diff --git a/.gitignore b/.gitignore index b2fafbba..301eca04 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ docker-compose.override.yml .ruff_cache/ .pytest_cache/ *.db +*.db-shm +*.db-wal .plan.md .plan-*.md .hypothesis/ @@ -29,4 +31,4 @@ tools/skill_audit_analysis/output/ design_ideas/ .claude/ docs/design/ -/.idea +/.idea diff --git a/docs/api-reference.md b/docs/api-reference.md index fe336a31..79713375 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -282,9 +282,11 @@ names from the resolved cycle and does not flip this field. "messages": [ {"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there!", "tool_calls": null}, - {"role": "tool", "content": "..."} + {"role": "tool", "content": "..."}, + {"role": "system", "source": "compaction", "content": "Summary..."} ], - "cursor": null + "cursor": null, + "handoff_token": "opaque-live-revision" } ``` @@ -293,13 +295,49 @@ trailing turn that the event ring can reconstruct, open the SSE URL with `?last_event_id=` (or send `Last-Event-ID`) so the buffered delta fills that turn without double-rendering it. +For a loaded workstream, `messages` is the requested tail projection of one +authoritative total accepted conversation-row prefix. It includes user, +assistant, tool, and system rows, including compaction checkpoints projected as +`role: "system", source: "compaction"` and cancellation-generated partial +assistant or synthesized tool-result markers when present. + +`handoff_token` is non-null exactly when the workstream's session is live on +the serving node (a pane opened it there); it identifies the exact total +prefix used for that render. Pass it once on the initial SSE URL as +`history_token`. The token is opaque and process-local: do not parse, +persist, or reuse it. Admission of any later conversation row changes the +token; moving a row from the pending journal to durable storage does not. The +server validates the token while atomically registering the listener. A +mismatch emits `history_resync` and closes the stream; fetch history again +instead of replaying from a numeric cursor. A native reconnect's +`Last-Event-ID` header takes priority and follows the normal ring-replay +path. + +A null `handoff_token` on a 200 is the cold storage-only read: the workstream +is not loaded on the serving node, so there is no live writer and no splice +to witness. The payload may seed a render and a token-less stream bootstrap +(the server converges the pane through `clear_ui`), never a cursor handoff. +`/history` never loads a session — reading an archived transcript leaves the +session pool untouched. + Each message in the `messages` array has: | Field | Type | Description | |--------------|-------------------|-----------------------------------------------| -| `role` | string | `"user"`, `"assistant"`, or `"tool"` | +| `role` | string | `"user"`, `"assistant"`, `"tool"`, or `"system"` | | `content` | string or null | Text content of the message | | `tool_calls` | array or null | Present only on assistant messages with calls | +| `source` | string (optional) | Operator-context or marker source, including `"compaction"` | +| `meta` | object (optional) | Structured display metadata for the source | +| `attachments` | array (optional) | Accepted attachment metadata: `attachment_id`, `kind`, `filename`, and `mime_type` | +| `sender` | string (optional) | Authenticated participant attributed to an accepted user row | +| `client_send_ids` | string[] (optional) | Optimistic-send correlation tokens carried by an accepted user row; never idempotency keys | +| `tool_call_id` | string (tool only) | Provider correlation id for the corresponding assistant call; ids may be reused across turns | +| `tool_name` | string (tool only) | Function name for the tool result | +| `event_id` | integer (optional) | Accepted SSE row identity used for replay deduplication | +| `is_error` | bool (tool only) | Final error disposition | +| `effect_status` | string (optional) | Persisted effect disposition when known | +| `preview` | object (optional) | Content-addressed preview descriptor; does not contain preview bytes | | `reasoning` | string (optional) | Concatenated reasoning / chain-of-thought text on assistant turns whose `provider_data` carried reasoning-bearing blocks (Anthropic `thinking`, OpenAI Responses `reasoning`, or synthetic `reasoning_text` from local-model servers). Present only when the active model's `surface_persisted_reasoning` flag is True. | Each entry in `tool_calls`: @@ -314,6 +352,64 @@ Each entry in `tool_calls`: After the synthetic replay or cursor delta, the server streams real-time events as the model generates a response: +Typed accepted-user projection is capability-gated. Add `?user_turn=1` to every +per-workstream SSE URL to receive `user_turn`; the embedded panes and both SDKs +do this automatically. A raw client that omits the capability receives a +`replay_truncated` frame with reason `user_turn_projection_unsupported`, whose +SSE id is anchored immediately before the unrepresented row. It must fetch and +render `/history` before reconnecting. This backward-compatible repair frame +does not expose the row content, and a failed history fetch must retain the +pre-row cursor so the repair signal repeats. + +Final accepted-tool projection is separately capability-gated. Browser panes +add `tool_turn=1` to every per-workstream SSE URL, including every manual and +native reconnect. A capable listener receives a second `tool_result` with +`accepted: true`, the row's `_event_id`, and the final scalar text, error, +preview, and effect fields that entered accepted history. Reducers replace the +earlier executor receipt in place. A client that omits `tool_turn=1` receives a +redacted `replay_truncated` frame with reason +`tool_turn_projection_unsupported`, anchored at the cursor immediately before +the accepted row, and must rebuild from `/history`. + +This accepted projection is a transcript-consistency mechanism, not a wire +confidentiality boundary. The preliminary `tool_result` is deliberately sent +as soon as execution completes and can precede post-execution output transforms; +do not treat `accepted: true` as proof that earlier frames contained the same +text. + +**`user_turn`** -- the canonical accepted user row. Every upgraded listener on +the shared workstream receives the event, including peer browsers, so peers can +render the turn without refetching all history. The originating pane uses +`client_send_ids` only to replace or mark its exact optimistic bubble; peers +render the row once by SSE event id. Reusing a client token still admits and +emits a distinct turn. + +```json +{ + "type": "user_turn", + "ws_id": "abc123", + "content": "Inspect this file", + "attachments": [ + { + "attachment_id": "a1", + "kind": "text", + "filename": "notes.txt", + "mime_type": "text/plain" + } + ], + "sender": "user-123", + "client_send_ids": ["browserSend_42"], + "_event_id": 17 +} +``` + +`client_send_ids` is empty for callers that did not provide a correlation +token. `_event_id` is the accepted row's monotonic SSE identity and is the +deduplication key; `client_send_ids` is not. Correlation tokens are not +credentials. When both identities are known, an upgraded pane settles a local +optimistic bubble only when the event's `sender` matches that viewer; peer rows +still render canonically without touching local optimistic state. + **`thinking_start`** -- the model has begun generating (shown as a spinner). ```json @@ -463,10 +559,11 @@ Each item in `items` (shared by `tool_info` and `approve_request`): {"type": "tool_output_chunk", "call_id": "call_abc123", "chunk": "Building project...\n"} ``` -**`tool_result`** -- final output from a completed tool execution. The `call_id` matches the corresponding `tool_info`/`approve_request` item and any preceding `tool_output_chunk` events. For bash tools, this arrives after all streaming chunks and includes both stdout and stderr. The `is_error` field is `true` when the tool execution failed (e.g. bash exit code >= 2 or signal, file not found, timeout). Exit code 1 is ambiguous (e.g. `grep` no-match) and is not flagged. User denials are tracked separately via a `denied` flag. Clients should use `is_error` instead of text-prefix heuristics. +**`tool_result`** -- output from a completed tool execution. The first event is the executor receipt. With `tool_turn=1`, a later event carrying `accepted: true` is the canonical accepted-history replacement and includes `_event_id`; `preview` and `effect_status` are present when persisted. The `call_id` matches the corresponding `tool_info`/`approve_request` item and any preceding `tool_output_chunk` events, but clients must scope reused ids to the newest rendered tool batch. For bash tools, the receipt arrives after all streaming chunks and includes both stdout and stderr. The `is_error` field is `true` when the tool execution failed (e.g. bash exit code >= 2 or signal, file not found, timeout). Exit code 1 is ambiguous (e.g. `grep` no-match) and is not flagged. User denials are tracked separately via a `denied` flag. Clients should use `is_error` instead of text-prefix heuristics. ```json {"type": "tool_result", "call_id": "call_abc123", "name": "bash", "output": "file1.py\nfile2.py\n", "is_error": false} +{"type": "tool_result", "accepted": true, "_event_id": 42, "call_id": "call_abc123", "name": "bash", "output": "file1.py\nfile2.py\n", "is_error": false, "effect_status": "unknown"} ``` **`status`** -- token usage statistics, sent after each model turn. @@ -563,6 +660,25 @@ dedicated rewind/retry, successful fork publication, and opening saved history. {"type": "clear_ui"} ``` +**`history_resync`** -- the history rendered before this stream opened no +longer names the live accepted conversation-row prefix. The server closes the +stream after this event. Keep the current transcript visible, fetch `/history` +again, render the successful response, and open a new stream with its new +one-shot handoff token. A numeric event cursor cannot prove that a complete row +was rendered and is not a substitute for this repair. + +```json +{"type": "history_resync", "ws_id": "abc123", "reason": "handoff_mismatch"} +``` + +`ws_id` is present for registration-time handoff mismatches; on an already +scoped live stream, clients may infer it from the stream when omitted. + +`reason` is a free string. `workstream_gone` means the workstream's durable +row was deleted out from under a live session (by another node, or by +startup cleanup); the follow-up `/history` fetch answers 503/404 rather than +minting a new token, and new sends are refused. + **`cancelled`** -- a cancel request was acknowledged (via the Stop button or `POST /v1/api/workstreams/{ws_id}/cancel`). This signals that cancellation is in progress, not that it is complete. The worker thread may still be finishing. @@ -572,6 +688,11 @@ path, or `error`). `stream_end` only closes assistant rendering: it may already have arrived before Stop reaches an approval or tool phase, so it is not a cancellation-completion signal. +The `cancelled` event is not itself a history row. If cancellation accepts a +partial assistant response or synthesizes tool-result receipts to close +outstanding calls, those assistant/tool rows appear in `/history` and advance +the same handoff prefix. + ```json {"type": "cancelled"} ``` @@ -658,11 +779,24 @@ visibility checks run before storage reconstruction. |-----------------|------|---------|-------------| | `limit` | integer | `100` | Tail row limit, clamped to 1--500 | -The response is `{"ws_id": ..., "messages": [...], "cursor": ...}` using the -message shape documented in the event-stream bootstrap above. `cursor` is -normally `null`; when non-null, open `/events?last_event_id=` so the -ring replays the deliberately trimmed in-progress tail. A missing, invisible, -or wrong-kind workstream returns the endpoint's ordinary `404` shape. +The response is +`{"ws_id": ..., "messages": [...], "cursor": ..., "handoff_token": ...}` +using the message shape and total-prefix contract documented in the event-stream +bootstrap above. `cursor` is normally `null`; when non-null, pass it as +`last_event_id` on the initial `/events` request. For a loaded workstream, pass +the non-null `handoff_token` from the history just rendered on that same initial +request. A missing, invisible, or wrong-kind workstream returns the endpoint's +ordinary `404` shape. + +If the durable prefix cannot be loaded, the endpoint returns: + +```json +{"error": "History temporarily unavailable"} +``` + +Status code: `503`. This response is not authoritative and carries no usable +handoff token. Keep any current transcript, do not open a tokenless replacement +stream, and retry the history read. --- @@ -675,13 +809,14 @@ indicators. **Events:** ```json -{"type": "ws_state", "ws_id": "abc123", "state": "thinking"} +{"type": "ws_state", "ws_id": "abc123", "state": "thinking", "persistence_state": "healthy"} ``` -| Field | Type | Description | -|---------|--------|--------------------------| -| `ws_id` | string | Workstream identifier | -| `state` | string | Current workstream state | +| Field | Type | Description | +|---------------------|--------|-------------| +| `ws_id` | string | Workstream identifier | +| `state` | string | Current workstream state | +| `persistence_state` | string | Sanitized history-save state: `healthy`, `pending`, `retrying`, or `conflict`; omitted by older nodes means `healthy` | Possible `state` values: @@ -711,19 +846,25 @@ Returns a list of all active workstreams. ```json { "workstreams": [ - {"ws_id": "abc123", "name": "default", "state": "idle"}, - {"ws_id": "def456", "name": "hacker-news", "state": "thinking"} + {"ws_id": "abc123", "name": "default", "state": "idle", "persistence_state": "healthy"}, + {"ws_id": "def456", "name": "hacker-news", "state": "thinking", "persistence_state": "retrying"} ] } ``` Each workstream object: -| Field | Type | Description | -|--------------|-------------|--------------------------------------------------------| -| `ws_id` | string | Unique workstream routing identifier | -| `name` | string | Display name (alias if set, otherwise `ws-xxxx`) | -| `state` | string | Current state (see state values above) | +| Field | Type | Description | +|---------------------|--------|-------------| +| `ws_id` | string | Unique workstream routing identifier | +| `name` | string | Display name (alias if set, otherwise `ws-xxxx`) | +| `state` | string | Current state (see state values above) | +| `persistence_state` | string | Sanitized history-save state; defaults to `healthy` for older or unloaded rows | + +The persistence state intentionally carries no retry counts, timestamps, +storage errors, commit keys, or conversation content. `pending` means an +accepted row awaits its first durable save, `retrying` means automatic repair is +active, and `conflict` requires operator intervention. --- @@ -844,13 +985,20 @@ Sends a user message to a workstream. Spawns a daemon worker thread that calls **Request body:** ```json -{"message": "Explain how the server works", "attachment_ids": ["a1"]} +{"message": "Explain how the server works", "attachment_ids": ["a1"], "client_send_id": "browserSend_42"} ``` | Field | Type | Required | Description | |------------------|------------|----------|------------------------------------------------------| | `message` | string | yes | The user's message text | | `attachment_ids` | string[] | no | Staged uploads to attach (omit = auto-consume; `[]` = none) | +| `client_send_id` | string | no | Opaque optimistic-UI correlation token matching `[A-Za-z0-9_-]{1,128}`; echoed in `user_turn` and history, never used for idempotency | + +The token correlates delivery only. Reusing the same value does not collapse or +deduplicate requests: each accepted send remains a distinct history row and +`user_turn` event. A live `message_queued` event carrying the token can prove +server acceptance before the POST response arrives, including when that HTTP +acknowledgement is lost. **Response.** Every 200 body carries `attached_ids` and `dropped_attachment_ids` (empty lists when no attachments are involved): @@ -877,6 +1025,7 @@ Sends a user message to a workstream. Spawns a daemon worker thread that calls | Status | Body | Condition | |--------|-------------------------------------------------|----------------------------------------| | 400 | `{"error": "message is required"}` | Message is empty | +| 400 | `{"error": "client_send_id must match ..."}` | Correlation token is invalid | | 404 | `{"error": "Unknown workstream"}` | `ws_id` not found (or closed mid-send) | | 409 | `{"status": "cross_user_interjection", ...}` | Another participant's turn is in flight | @@ -1271,6 +1420,16 @@ the close proceeds without writing the field. Status code: `400` +**Error (conversation persistence unresolved):** + +```json +{"error": "workstream has unresolved persistence"} +``` + +Status code: `409`. At least one accepted live conversation row still requires +idempotent persistence reconciliation. The workstream remains loaded and no +history is discarded; retry the close after storage recovers. + --- ### `POST /v1/api/workstreams/{ws_id}/attachments` @@ -2364,6 +2523,13 @@ double-rendering across refreshes, ring eviction, and process restart. The global state stream has its own snapshot/replay floor rather than conversation history. +`history_resync` is a stronger repair signal than `replay_truncated`: it means +the one-shot token no longer names the accepted row prefix used for the rendered +history. The server closes that stream. Clients retain the current transcript, +fetch and render `/history` again, then reconnect with the new cursor/token pair; +numeric replay alone is insufficient. If the repair read returns `503`, clients +must keep the repair latched and must not open a cursorless or tokenless stream. + --- ## Observability diff --git a/docs/architecture.md b/docs/architecture.md index 50005164..37a7d7d3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -336,6 +336,36 @@ that fuel the SSE refresh-resume `in_progress_snapshot` event — see the per-workstream events stream in [`docs/api-reference.md`](api-reference.md#get-v1apiworkstreamsws_idevents). +For web-backed sessions, every accepted live conversation row enters one +ordered history handoff journal before durability. This includes user, +assistant, tool, and system rows, compaction checkpoints, partial-assistant +cancellation markers, and synthesized cancellation tool receipts. Admission +shares the short handoff lock with the row's live UI transition or a +`history_resync` repair event, so a row either changes the history token before +listener registration or reaches the listener registered under the old token. +Each journal entry has a stable per-workstream `commit_key`; the storage +backends insert it idempotently, so an acknowledgement lost after commit can be +resolved without duplicating the row. Attachment-bearing user and tool rows +commit their conversation row, content-addressed blobs, reference counts, and +ordered attachment list in one transaction. + +`/history` loads the durable prefix and overlays unacknowledged journal entries +under a separate visibility lane. Its `messages` is the requested tail +projection of one total accepted row prefix; an opaque token names that exact +prefix. The initial SSE connection validates the token while registering its +listener. A changed revision or session epoch emits `history_resync`, closes the +stream, and requires another history read rather than assuming numeric +event-ring coverage can reconstruct a complete row. A durable-history load +failure returns 503 and supplies no usable token, so clients retain their +current transcript and keep the repair latch closed. + +Storage acknowledgement removes the journal representation without advancing +the accepted revision: durable and pending forms are the same logical row. An +unresolved save fail-stops every later conversation-row suffix, remains visible +through the live journal, and prevents soft close, idle eviction, or capacity +eviction from discarding that recovery state. Soft close returns 409 and leaves +the workstream loaded; hard delete remains the explicit discard boundary. + `on_stream_discarded` removes a failed attempt's partial projection before a mid-stream retry. `on_system_turn` and `on_compaction` return the assigned SSE event ID when the frontend has one; persistence stamps the corresponding row @@ -607,6 +637,10 @@ non-idle background workstreams above the input prompt. `_state_incarnation` and `StateWriter` prevent close/reopen ABA writes. - `ChatSession._generation_lock` fences one turn's live mutations; `_durability_cond` tickets its deferred storage batches in admission order. +- `ChatSession._history_handoff_lock` owns the total accepted conversation-row + journal, revision, and initial listener registration. The separate + `_history_visibility_lock` linearizes a storage history load with keyed row + save/ack; neither is held across generation admission or model/tool work. - `SessionUIBase._ws_lock` protects concurrent approval cycles, verdict caches, and SSE projection state; approval admission uses a separate condition so Stop never waits on database I/O. @@ -980,6 +1014,22 @@ from the lane that actually served the call. A recursive compaction pins one lane for all leaf summaries and the merge; a hot reload never splices two model definitions into one summary transaction. +Every accepted model turn carries an immutable provenance envelope captured +from that successful serving attempt: model alias, backend model ID, registry +generation, and the generation-pinned acting principal. It is stamped before +the result leaves `model_turn()` and persisted in the assistant row's existing +`meta` JSON, so a later fallback, hot reload, user rebind, or ambiguous storage +acknowledgement cannot relabel the output at commit time. The principal-bearing +envelope is internal audit metadata: ordinary history/SSE, coordinator +inspection, exports, and provider lowering do not expose it. + +For browser-driven turns the principal is the authenticated initiator pinned to +the generation. Headless session-backed turns (CLI, eval, scheduled, and +internal work) stamp the effective owner credential principal they actually use; +only a truly ownerless direct model call records an empty principal. This fourth +axis remains private even where the three serving-kernel axes are logged for +retry and completion diagnostics. + **Model-backend authentication:** A model definition's `auth_mode` is one of `static`, `entra_obo`, `entra_app`, or `rfc8693_obo`. Dynamic modes keep only authorization parameters (`obo_audience`, and RFC 8693 `obo_scopes`) in the @@ -1374,7 +1424,10 @@ conversations event_id BIGINT -- per-workstream SSE resume cursor is_error BOOLEAN attachments TEXT -- ordered content-addressed refs - meta TEXT -- source, effect, preview side metadata + meta TEXT -- source/effect/preview/model provenance + commit_key TEXT -- nullable idempotent live-row identity + + UNIQUE (ws_id, commit_key) WHERE commit_key IS NOT NULL workstream_config ws_id TEXT NOT NULL -- composite PK with key @@ -1495,11 +1548,16 @@ startup) are invisible until a message is sent. at startup (CLI and server). It deliberately excludes internal `creating` reservations, which belong to the crash-recovery path above. It removes: -- Published workstreams with no messages (orphaned registrations) +- Unnamed workstreams with no messages (orphaned registrations) whose + `updated` timestamp is older than a two-hour grace — a just-created + workstream is a user mid-first-turn (its first rows may still be held in a + serving node's in-memory commit journal, invisible to other nodes), never + housekeeping debris - Unnamed workstreams (`alias IS NULL`) older than `retention_days` days (default 90) -Named (aliased) workstreams are never age-pruned. Configure with -`--retention-days N` (0 = disable age pruning). +Named (aliased) workstreams are never pruned by either category. Configure +with `--retention-days N` (0 = disable age pruning; the orphan sweep still +runs, grace-gated). --- @@ -1561,6 +1619,14 @@ warns if the summary was truncated. - **SSE reconnect**: both `connectContentSSE()` and `connectGlobalSSE()` use exponential backoff on `onerror` -- starting at 1 second, doubling on each failure, capped at 30 seconds. On successful message, delay resets to 1s. +- **Conversation history handoff**: a pane renders REST `/history`, then opens + its initial per-workstream SSE with that response's cursor and one-shot + handoff token. The token is present exactly when the session is live on the + serving node; a rendered token-less 200 is the cold storage-only read and + downgrades the pane to the token-less bootstrap (the server converges it + via `clear_ui`). `history_resync` closes the transport and latches a fresh + history read; a 503 retains the stale-but-real transcript and cannot fall + through to a cursorless stream. - **Disconnection indicator**: `#status-bar.disconnected` class turns the status text red and shows "Reconnecting..." - **Fetch error handling**: all `fetch()` calls use `.catch()` to prevent diff --git a/docs/coordinator-api-tour.md b/docs/coordinator-api-tour.md index a064c287..f993f5db 100644 --- a/docs/coordinator-api-tour.md +++ b/docs/coordinator-api-tour.md @@ -37,7 +37,7 @@ schema changes. | # | Action | Operation | |---|------------------------------|-------------------------------------------------------------| | 1 | Create | `POST /v1/api/workstreams/new` | -| 2 | Subscribe to events | `GET /v1/api/workstreams/{ws_id}/events` (SSE) | +| 2 | Bootstrap history + subscribe | `GET .../history`, then `GET .../events` (SSE) | | 3 | Send a user message | `POST /v1/api/workstreams/{ws_id}/send` | | 4 | Inspect children | `GET /v1/api/workstreams/{ws_id}/children` | | 5 | Inspect one workstream | `GET /v1/api/cluster/ws/{ws_id}/detail` | @@ -91,14 +91,34 @@ subscribers (step 2) see the session warm up as token traffic starts. --- -## 2. Subscribe to the per-coordinator event stream +## 2. Bootstrap history, then subscribe to the event stream + +Read and render history before opening the initial stream: ```http -GET /v1/api/workstreams/{ws_id}/events HTTP/1.1 +GET /v1/api/workstreams/{ws_id}/history?limit=100 HTTP/1.1 +Authorization: Bearer +``` + +For a loaded coordinator, `messages` is the requested tail of one total +accepted conversation-row prefix: user, assistant, tool, and system rows, +including projected compaction checkpoints and cancellation-generated markers. +The response's optional `cursor` and `handoff_token` belong to that exact +render. Pass both once on the initial stream URL: + +```http +GET /v1/api/workstreams/{ws_id}/events?last_event_id={cursor}&history_token={handoff_token}&user_turn=1&tool_turn=1 HTTP/1.1 Accept: text/event-stream Authorization: Bearer ``` +Omit either query parameter when its history field is `null`. A handoff token +is opaque and process-local: do not parse, persist, or reuse it. Admission of a +later conversation row changes the token; durable acknowledgement does not. If +history returns `503 {"error":"History temporarily unavailable"}`, the response +is not authoritative: retain the current transcript, do not open a tokenless +replacement stream, and retry the read. + One persistent SSE connection per browser tab / SDK caller — the console fans each event out to every listener queue (cap 500 events per queue, put_nowait drop on overflow). Events come in flat JSON @@ -110,7 +130,7 @@ with a `type` field. The recurring shapes a UI has to handle: | `reasoning` | Reasoning-token stream chunk (when the model exposes it) | `text` | | `content` | Assistant-content stream chunk | `text` | | `stream_end` | End of a single provider stream | — | -| `tool_result` | A tool call completed (success or error) | `call_id`, `name`, `output`, `is_error?` | +| `tool_result` | A tool call completed; capable panes also receive the accepted-history replacement | `call_id`, `name`, `output`, `is_error?`, `accepted?`, `_event_id?`, `preview?`, `effect_status?` | | `tool_output_chunk` | Streaming tool output (e.g. long bash command) | `call_id`, `chunk` | | `approve_request` | One approval cycle needs operator action; several cycles may coexist | `cycle_id`, `items: [{call_id, header, preview, func_name, approval_label, needs_approval}]` | | `approval_resolved` | One identified approval cycle was answered | `cycle_id`, `call_ids`, `approved`, `feedback`, `always` | @@ -127,6 +147,7 @@ with a `type` field. The recurring shapes a UI has to handle: | `wait_started` / `wait_progress` / `wait_ended` | `wait_for_workstream` tool lifecycle (see §6) | `call_id`, `ws_ids`, `elapsed`, `results`, `complete` | | `batch_started` / `batch_ended` | `spawn_batch` / `close_all_children` tool lifecycle | `call_id`, `op`, `total`/`succeeded`/`denied`/`closed`/`failed`/`skipped` | | `info` / `error` | Operational messages | `message` | +| `history_resync` | The rendered history token no longer names the accepted row prefix | `ws_id`, `reason` | **Reconnection contract:** a freshly-opened SSE connection receives one `approve_request` snapshot for every unresolved approval cycle, keyed by @@ -138,6 +159,11 @@ mid-approval, mid-tool-execution, or mid-stream restores both the correct composer mode and the partial assistant text without waiting for the response to complete. +`history_resync` is stronger than a numeric replay gap. The server closes that +stream; fetch and render `/history` again, then open a new stream with its new +cursor/token pair. The API and SDK expose these primitives but deliberately do +not choose a reconnect policy for callers. + --- ## 3. Send the first user message @@ -372,6 +398,11 @@ disconnect. The row is reopenable via `POST /v1/api/workstreams/{ws_id}/open` so long as it hasn't been deleted. +If any accepted live conversation row still requires persistence +reconciliation, close returns `409 {"error":"workstream has unresolved +persistence"}`. The coordinator remains loaded, its journal is retained, and +no history is discarded; retry after storage recovers. + --- ## Further reading diff --git a/docs/diagrams/04-conversation-turn.puml b/docs/diagrams/04-conversation-turn.puml index 8e2fa414..e6766095 100644 --- a/docs/diagrams/04-conversation-turn.puml +++ b/docs/diagrams/04-conversation-turn.puml @@ -9,6 +9,7 @@ participant "HTTP / CLI\ncaller" as User participant "SessionManager" as Manager participant "ChatSession" as Session participant "SessionUIBase" as UI +participant "Accepted-row handoff\n(total live prefix)" as Handoff participant "model_turn()\n+ lowering" as Plant participant "ModelAdmission\n(per alias)" as Admission participant "LLM provider" as Provider @@ -33,13 +34,22 @@ end Session -> Session : _claim_generation() → generation N\ninstall fresh cancel event Session -> Session : plan memory / participant context +Session -> Handoff : admit USER row\ncommit_key + prefix revision Session -> Storage : ordered durable batch:\nappend canonical user Turn + metadata -note over Session, Storage +note over Session, Handoff + Every accepted conversation row enters this lane before durability: + USER, ASSISTANT, TOOL, SYSTEM, compaction checkpoints, and cancellation + markers. Admission shares the handoff lock with its live UI transition + or history_resync repair event. +end note + +note over Handoff, Storage _commit_for_generation(N) admits bounded live mutations under the generation lock, then executes immutable persistence closures in FIFO ticket order. A force successor either follows the whole commit or - prevents it; storage I/O never holds the lifecycle lock. + prevents it. /history projects durable prefix + pending journal suffix; + durable ACK removes the pending copy without changing the prefix revision. end note opt already over the hard context ceiling @@ -80,21 +90,25 @@ loop until final answer and no queued input Session -> UI : on_stream_end() Session -> Session : generation-fenced result commit:\nappend assistant Turn + token accounting Session -> UI : on_turn_committed() + Session -> Handoff : admit ASSISTANT row\ncommit_key + prefix revision Session -> Storage : ordered durable assistant row\n(content + tool mirror + native lane) alt no tool calls opt over soft threshold Session -> Session : cooperative / end-of-turn compaction + Session -> Handoff : admit SYSTEM/source=compaction\ncheckpoint projection Session -> Storage : append checkpoint summary marker\nwith source watermark note right of Storage Full history remains durable. Resume loads [summary] + rows after the checkpoint. end note opt model stopped for compaction + Session -> Handoff : admit USER/source=compaction_resume row Session -> Storage : append synthetic compaction_resume Turn end end alt queued messages drained + Session -> Handoff : admit combined queued USER row Session -> Storage : append combined queued user Turn else truly complete Session -> UI : state = idle @@ -123,10 +137,12 @@ loop until final answer and no queued input opt compaction owed before result sizing Session -> Session : compact, preserving assistant tool-call Turn + Session -> Handoff : admit SYSTEM/source=compaction checkpoint Session -> Storage : append checkpoint marker end Session -> Session : one generation-fenced batch:\nappend all Tool Turns, advisories, feedback + Session -> Handoff : admit FIFO TOOL rows\ncommit keys + prefix revisions Session -> Storage : FIFO durable tool rows + metadata end end @@ -138,6 +154,10 @@ Session -> Session : atomically set generation event; snapshot\nmain stream, chi Session -> Provider : close live stream handle Session -> Tools : abort child scopes + kill subprocess groups Session -> UI : resolve only cancelled operation's\napproval cycles +opt cancellation produced accepted conversation rows + Session -> Handoff : admit partial ASSISTANT and/or\nsynthesized TOOL cancellation markers + Session -> Storage : idempotent keyed cancellation rows +end note over Session, Storage Every later publish/commit checks generation ownership. An abandoned diff --git a/docs/diagrams/09-workstream-states.puml b/docs/diagrams/09-workstream-states.puml index 43e694a4..19dd32ec 100644 --- a/docs/diagrams/09-workstream-states.puml +++ b/docs/diagrams/09-workstream-states.puml @@ -44,14 +44,22 @@ running --> error : Exception during\ntool execution error --> thinking : New send() call\n_emit_state("thinking") -idle --> closed : close / eviction -error --> closed : close -thinking --> closed : close -running --> closed : close -attention --> closed : close +idle --> closed : close / eviction\n[journal reconciled] +error --> closed : close\n[journal reconciled] +thinking --> closed : close\n[journal reconciled] +running --> closed : close\n[journal reconciled] +attention --> closed : close\n[journal reconciled] closed --> [*] : hard delete closed --> idle : open / rehydrate +note right of closed + Before every soft-close / eviction transition, + the total accepted conversation-row journal must + be durably reconciled. An unresolved row makes an + explicit close return HTTP 409 (eviction refuses), + and the workstream remains loaded in its live state. +end note + thinking --> idle : cancel() called\nstream aborted\n_emit_state("idle") running --> idle : cancel() called\n_emit_state("idle") diff --git a/docs/diagrams/13-sdk-architecture.puml b/docs/diagrams/13-sdk-architecture.puml index 6a5c4e3a..2606a79d 100644 --- a/docs/diagrams/13-sdk-architecture.puml +++ b/docs/diagrams/13-sdk-architecture.puml @@ -32,7 +32,8 @@ package "turnstone/sdk/ (Python)" { + approve() + command() + cancel(ws_id) - + stream_events(ws_id) + + get_history(ws_id, limit) → WorkstreamHistoryResponse + + stream_events(ws_id, last_event_id?, history_token?) + stream_global_events() + send_and_wait() + list_saved_workstreams() @@ -87,6 +88,13 @@ package "turnstone/sdk/ (Python)" { + ok: bool } + class WorkstreamHistoryResponse <> { + + ws_id: str + + messages: list[dict] + + cursor: int | None + + handoff_token: str | None + } + class ServerEvent <> { + type: str + ws_id: str @@ -105,6 +113,7 @@ package "turnstone/sdk/ (Python)" { TurnstoneConsole --> AsyncTurnstoneConsole : wraps TurnstoneConsole --> _SyncRunner : uses AsyncTurnstoneServer ..> TurnResult : returns + AsyncTurnstoneServer ..> WorkstreamHistoryResponse : renders before SSE AsyncTurnstoneServer ..> ServerEvent : yields AsyncTurnstoneConsole ..> ClusterEvent : yields } @@ -122,7 +131,8 @@ package "sdk/typescript/ (TypeScript)" { class "TurnstoneServer" as TSServer <> { + listWorkstreams() + send() - + streamEvents() + + getHistory() → WorkstreamHistoryResponse + + streamEvents(cursor?, token?) + sendAndWait() ... } @@ -154,4 +164,10 @@ note right of AsyncTurnstoneServer (no type duplication) end note +note bottom of ServerEvent + history_resync is a typed repair signal. + SDKs expose the REST cursor/token handshake but + never refetch, render, or reconnect automatically. +end note + @enduml diff --git a/docs/diagrams/png/04-conversation-turn.png b/docs/diagrams/png/04-conversation-turn.png index c08b8035..33cbaf8b 100644 --- a/docs/diagrams/png/04-conversation-turn.png +++ b/docs/diagrams/png/04-conversation-turn.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b237e00c28225c847b3e1a083c5c35535d1bf1549f6e6c7e2b6c4fd0e037a980 -size 331906 +oid sha256:6261604cc8b75878a8704308929ea64d121cf26547019543fbe1b21cbe700415 +size 189791 diff --git a/docs/diagrams/png/09-workstream-states.png b/docs/diagrams/png/09-workstream-states.png index 8e96f40e..dbc76fd1 100644 --- a/docs/diagrams/png/09-workstream-states.png +++ b/docs/diagrams/png/09-workstream-states.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:59dc8f92ca83c4354d089b75c6d5075d4a808277150b271c868dab99b3ac02ac -size 333165 +oid sha256:33dddd8cd8b53fa464cc8a4c899896fee32035d63e0669fe327969e3356349c7 +size 329815 diff --git a/docs/diagrams/png/13-sdk-architecture.png b/docs/diagrams/png/13-sdk-architecture.png index b12a2457..5ff2e9af 100644 --- a/docs/diagrams/png/13-sdk-architecture.png +++ b/docs/diagrams/png/13-sdk-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e7c3e40c10425d721f833390ae3531c09af501157fd3142531ba4eba86ff719d -size 197112 +oid sha256:59f14b835665244f3d32981b6c1ac4c4380393a83cc519e271831622aa3f261a +size 197433 diff --git a/docs/sdk.md b/docs/sdk.md index f2bfb406..a7ea8213 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -77,11 +77,12 @@ Both `TurnstoneServer` (sync) and `AsyncTurnstoneServer` (async) expose: | | `list_attachments(ws_id)` | `ListAttachmentsResponse` | | | `get_attachment_content(ws_id, attachment_id)` | `bytes` | | | `delete_attachment(ws_id, attachment_id)` | `StatusResponse` | -| **Chat** | `send(message, ws_id)` | `SendResponse` | +| **Chat** | `send(message, ws_id, *, attachment_ids=None, client_send_id=None)` | `SendResponse` | | | `approve(*, ws_id, approved, feedback, always, cycle_id, call_id)` | `ApproveResponse` | | | `command(*, ws_id, command)` | `StatusResponse` | | | `cancel(ws_id, *, force=False)` | `CancelResponse` | -| **Streaming** | `stream_events(ws_id)` | `Iterator[ServerEvent]` | +| **History** | `get_history(ws_id, *, limit=100)` | `WorkstreamHistoryResponse` | +| **Streaming** | `stream_events(ws_id, *, last_event_id=None, history_token=None)` | `Iterator[ServerEvent]` | | | `stream_global_events()` | `Iterator[ServerEvent]` | | **High-level** | `send_and_wait(message, ws_id, *, timeout, on_event)` | `TurnResult` | | **Saved** | `list_saved_workstreams()` | `ListSavedWorkstreamsResponse` | @@ -127,11 +128,12 @@ SSE events are deserialized into typed dataclasses. Use `event.type` to discrimi | Type | Class | Key Fields | |------|-------|------------| | `connected` | `ConnectedEvent` | `model`, `model_alias`, `skip_permissions` | +| `user_turn` | `UserTurnEvent` | `ws_id`, `content`, `attachments`, `sender`, `source`, `client_send_ids`, `_event_id` | | `content` | `ContentEvent` | `text` | | `reasoning` | `ReasoningEvent` | `text` | | `tool_info` | `ToolInfoEvent` | `items` | | `approve_request` | `ApproveRequestEvent` | `cycle_id`, `items` | -| `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output`, `is_error` | +| `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output`, `is_error`, `preview`, `accepted`, `effect_status`, `_event_id` | | `tool_output_chunk` | `ToolOutputChunkEvent` | `call_id`, `chunk` | | `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort`, `cache_creation_tokens`, `cache_read_tokens` | | `error` | `ErrorEvent` | `message` | @@ -141,18 +143,90 @@ SSE events are deserialized into typed dataclasses. Use `event.type` to discrimi | `in_progress_snapshot` | `InProgressSnapshotEvent` | `content`, `reasoning` (one-shot mid-stream refresh resume) | | `approval_resolved` | `ApprovalResolvedEvent` | `cycle_id`, `call_ids`, `approved`, `feedback`, `always` | | `cancelled` | `CancelledEvent` | — | +| `history_resync` | `HistoryResyncEvent` | `reason`, optional `ws_id` | + +The Python server `send()` and console `coordinator_send()` methods accept an +optional `client_send_id`; TypeScript `send()` accepts the equivalent +`options.clientSendId`. Values match `[A-Za-z0-9_-]{1,128}`. The value is an +opaque optimistic-UI correlation token, not an idempotency key: reusing it +still creates distinct accepted turns and events. +Every upgraded listener on the shared workstream receives `UserTurnEvent`. +Originating panes use `client_send_ids` only to settle the exact optimistic +bubble, while peers render the accepted row once by `_event_id`. A +`message_queued` event carrying the token can establish acceptance even if the +POST acknowledgement is lost. History projects the same correlation alongside +the accepted user row. These tokens are not credentials: when sender and viewer +identities are both known, only a matching sender may settle local optimistic +state; a peer event still renders its canonical row. + +The typed projection is negotiated with `?user_turn=1` on the per-workstream +SSE URL. Python `stream_events()` / `send_and_wait()` and TypeScript +`streamEvents()` / `sendAndWait()` set it automatically. Raw consumers that +omit it receive a backward-compatible `replay_truncated` repair signal instead +of the user row and must rebuild from `/history`; its pre-row cursor keeps the +repair retryable if that history request fails. + +The browser-only final-tool upsert capability is `?tool_turn=1`. The bundled +Python and TypeScript SDK streaming helpers and channel adapters intentionally +do not negotiate it yet: they retain the executor-receipt `tool_result` +contract and do not own a transcript reducer. `ToolResultEvent` can deserialize +the accepted fields for direct/custom capable clients. Raw capable clients must +deduplicate `_event_id` and replace the newest matching call occurrence; raw +incapable clients receive the pre-row `tool_turn_projection_unsupported` repair +frame and rebuild from history. That staging deliberately prices in two costs +for incapable consumers. A raw client that treats every `replay_truncated` +frame as a rebuild trigger refetches `/history` once per accepted tool row — +one fetch per tool call on a long agentic turn; a client that wants tool +results incrementally should negotiate `tool_turn=1` and reduce, and the +bundled helpers (which ignore the frame rather than rebuild) stay correct +because their receipt-only view never depends on the accepted projection. +Second, only the accepted event carries post-execution output transforms, so a +receipt-rendering consumer (for example, a channel adapter posting the +executor receipt into a thread) keeps the pre-transform text; the accepted +projection is a transcript-consistency mechanism, not a wire confidentiality +boundary — see the API reference note on the preliminary `tool_result`. Current servers bootstrap conversation history through `GET /v1/api/workstreams/{ws_id}/history` before the SSE stream; they do not emit a `history` event. `HistoryEvent` remains deserializable only for -compatibility with older servers. The Python client does not yet expose a -typed helper for this bootstrap endpoint. +compatibility with older servers. `get_history()` exposes the current REST +bootstrap response, including its optional cursor and one-shot handoff token. + +### Caller-managed history handoff + +The SDK supplies typed handshake primitives but intentionally does not own a +transcript renderer or reconnect policy. After rendering a successful history +response, pass its cursor and token to exactly one initial stream: + +```python +from turnstone.sdk import HistoryResyncEvent + +history = client.get_history(ws_id) +render(history.messages) + +for event in client.stream_events( + ws_id, + last_event_id=history.cursor, + history_token=history.handoff_token, +): + if isinstance(event, HistoryResyncEvent): + # Stop this stream. The caller chooses when to fetch, render, and + # reconnect with a new history response. + break + apply_live_event(event) +``` + +`history_resync` means numeric replay cannot prove that the rendered limited +tail came from the same total accepted conversation-row prefix. Stop the +stream, fetch and render history again, and use only the new cursor/token pair. +A 503 history response raises `TurnstoneAPIError`; it is not authoritative, so +retain any existing transcript and do not open a tokenless replacement stream. **Global events** (from `stream_global_events()`): | Type | Class | Key Fields | |------|-------|------------| -| `ws_state` | `WsStateEvent` | `ws_id`, `state`, `tokens`, `activity` | +| `ws_state` | `WsStateEvent` | `ws_id`, `state`, `tokens`, `activity`, `persistence_state` | | `ws_activity` | `WsActivityEvent` | `ws_id`, `activity`, `activity_state` | | `ws_rename` | `WsRenameEvent` | `ws_id`, `name` | | `ws_closed` | `WsClosedEvent` | `ws_id` | @@ -163,12 +237,18 @@ typed helper for this bootstrap endpoint. |------|-------|------------| | `node_joined` | `NodeJoinedEvent` | `node_id` | | `node_lost` | `NodeLostEvent` | `node_id` | -| `cluster_state` | `ClusterStateEvent` | `ws_id`, `node_id`, `state`, `tokens` | -| `ws_created` | `ClusterWsCreatedEvent` | `ws_id`, `node_id`, `name` | +| `cluster_state` | `ClusterStateEvent` | `ws_id`, `node_id`, `state`, `tokens`, `persistence_state` | +| `ws_created` | `ClusterWsCreatedEvent` | `ws_id`, `node_id`, `name`, `persistence_state` | | `ws_closed` | `ClusterWsClosedEvent` | `ws_id` | | `ws_rename` | `ClusterWsRenameEvent` | `ws_id`, `name` | | `snapshot` | `ClusterSnapshotEvent` | `nodes`, `overview`, `timestamp` | +Operator-facing workstream rows and rich state events expose only the sanitized +`persistence_state`: `healthy`, `pending`, `retrying`, or `conflict`. SDK types +treat it as optional for compatibility with older nodes; an omitted value means +`healthy`. Retry counts, storage errors, commit keys, and conversation content +are never part of this status surface. + ### TurnResult The `send_and_wait()` method returns a `TurnResult` that aggregates the full response: @@ -265,8 +345,14 @@ const ws = await client.createWorkstream({ name: "demo" }); const result = await client.sendAndWait("Hello!", ws.ws_id); console.log(result.content); -// Stream events -for await (const event of client.streamEvents(ws.ws_id)) { +// Render history, then use its one-shot hints on the initial stream. +const history = await client.getHistory(ws.ws_id); +render(history.messages); +for await (const event of client.streamEvents(ws.ws_id, { + lastEventId: history.cursor ?? undefined, + historyToken: history.handoff_token ?? undefined, +})) { + if (event.type === "history_resync") break; // caller refetches and reconnects if (event.type === "content") { process.stdout.write(event.text); } @@ -342,7 +428,7 @@ turnstone/sdk/ Python SDK (sub-package) _base.py Shared httpx async client, auth, error handling _sync.py Background event loop for sync wrappers _types.py TurnResult + TurnstoneAPIError - events.py 38 SSE event dataclasses with type registry + events.py Typed SSE event dataclasses with type registry server.py AsyncTurnstoneServer + TurnstoneServer console.py AsyncTurnstoneConsole + TurnstoneConsole diff --git a/scripts/recovery_e2e.py b/scripts/recovery_e2e.py index a60b6d00..605e0124 100644 --- a/scripts/recovery_e2e.py +++ b/scripts/recovery_e2e.py @@ -34,6 +34,10 @@ Usage:: python3 scripts/recovery_e2e.py --scenario coord-hidden-retry # G5 (#894) python3 scripts/recovery_e2e.py --scenario coord-orphan-rewind # G6 (#894 r6) python3 scripts/recovery_e2e.py --scenario coord-joined-flight # G7 (#894 r8) + python3 scripts/recovery_e2e.py --scenario handoff-repair-budget + python3 scripts/recovery_e2e.py --scenario coord-handoff-repair-budget + python3 scripts/recovery_e2e.py --scenario user-turn-two-pane + python3 scripts/recovery_e2e.py --scenario tool-turn-two-pane python3 scripts/recovery_e2e.py --scenario roster-restart # F1 (#881) python3 scripts/recovery_e2e.py --scenario roster-restart-native # F2 (#881) python3 scripts/recovery_e2e.py --scenario both # A+B only (legacy) @@ -83,21 +87,20 @@ BOTH its clear_ui refetch AND its one bounded 2s retry are forced to 500 (``RecoveryServer.fail_history(2)``), so the latch cannot self-heal and rewind/edit stay latch-gated over the stale-but-real transcript (a row-0 rewind click stays gated, ``rewind_requests`` holds at 1; the three user -rows survive). A plain send — sends are deliberately NOT latch-gated — runs -a fourth scripted ``final_text`` turn whose ORGANIC turn-settle idle edge -fires the backstop: a quiesced, same-token REST ``_refetchHistory`` +rows survive). The recovery server publishes a real per-workstream idle +edge through the loaded UI, firing a quiesced, same-token REST +``_refetchHistory`` without admitting another user row (deliberately NOT ``_loadHistoryThenConnect`` — the old reload backstop drew the server's synthetic ``state_change:idle`` on its fresh reconnect and re-triggered itself, a zero-backoff reconnect/refetch storm against a recovering node). With the fault budget now exhausted the refetch succeeds -and rebuilds the rewound (ONE user turn, index 1 of 3 rewinds 2) + sent (a -second user turn) transcript to TWO user rows, clearing the latch, so a +and rebuilds the single post-rewind user row, clearing the latch, so a fresh rewind on a remaining row lands (``rewind_requests`` -> 2). THE r5 PROOF, both counted at the fault layer: ``events_requests`` is UNCHANGED across the whole heal episode (``sse0`` — zero new EventSource connections; the storm would have opened one per reconnect) and ``history_requests`` grew -by exactly ONE (the backstop's single fetch — the plain fourth turn emits no -clear_ui). All polls are deadline-bounded so a regressed looping backstop +by exactly ONE (the backstop's single fetch). All polls are deadline-bounded +so a regressed looping backstop stamps a clean FAILED, never a hang. Stamps ``RECOVERY-READY-STALEBACKSTOP-heal1-sse0``. @@ -116,10 +119,12 @@ UNCHANGED across the hidden window (``hidden0``), which regresses to ``hidden1`` the moment the guard's transport clause is removed. Note the scope: a hide nulls ``evtSource``, so this exercises the PRESENCE term only — the ``readyState`` half needs a redial in progress, which no fault primitive -produces deterministically (see the runner's docstring). A replay_ok reconnect carries no synthetic ``state_change``, so +produces deterministically (see the runner's docstring). A replay_ok +reconnect carries no synthetic ``state_change``, so the latch survives ``__show`` — the accepted liveness lag — and the heal -rides a plain send's ORGANIC settle into the transport-free backstop, hence -exactly ONE new SSE open across show + heal. Stamps +rides a server-origin idle edge into the transport-free backstop. This avoids +admitting a live ``user_turn`` and starting unrelated model work; exactly ONE +new SSE open remains across show + heal. Stamps ``RECOVERY-READY-HIDDENRETRY-hidden0-heal1``. Scenario E6 (await-window-gate): the seedless render's cursor-safety gate @@ -139,9 +144,10 @@ owns. Stamps ``RECOVERY-READY-AWAITGATE-rows3-latch1``. Scenario E8 (reconnect-in-await): the stream-generation term (#900 r2), and the FIRST detector here that can observe a double render at all. Every other -scenario counts ``.msg.user`` rows, and user rows never travel on the SSE -stream, so none of them can see the artefact this campaign prevents — E8 -counts a sentinel's OCCURRENCES in the transcript text instead. The retry +scenario counts ``.msg.user`` rows; that count stays constant when an +assistant/tool slice is painted twice, so none of them can see the artefact +this campaign prevents. E8 counts a sentinel's OCCURRENCES in the transcript +text instead. The retry fires with the transport OPEN, its /history is held, and inside that await the transport DROPS and RE-ESTABLISHES: ``readyState`` reads OPEN afterwards exactly as before, so neither the presence nor the readyState term can tell @@ -170,20 +176,19 @@ click provably lands in the aftermath window; the bounded 2s retry then heals and reopens. Stamps ``RECOVERY-READY-COORDREWINDFAIL-posts2-heal1``. Scenario G3 (coord-stale-backstop): E4's port. Double failure exhausts -clear_ui refetch + retry; a plain send's ORGANIC settle fires the -TRANSPORT-FREE backstop (plain seedless ``refetchHistory``, never +clear_ui refetch + retry; a server-origin idle edge fires the TRANSPORT-FREE +backstop (plain seedless ``refetchHistory``, never ``loadHistoryThenReconnect``). Same storm proof: ``events_requests`` UNCHANGED, ``history_requests`` +1. Stamps ``RECOVERY-READY-COORDSTALEBACKSTOP-heal1-sse0``. -Scenario G4 (coord-heal-midturn, #894 r4): the render-time gate. A turn -STARTED during a heal's held /history must survive the fetch resolution — -the gate skips the wipe (pre-gate code detached the live turn's DOM into -dangling refs and lost it), the latch stays set, and the turn's own settle -re-fires the backstop, which heals. Proofs: turn 5's sentinel paints into -the still-stale transcript (``mid1``), the heal lands with the rewound-away -seeds gone, ``events_requests`` UNCHANGED, ``history_requests`` +2 exactly -(the skipped fetch + the heal). Stamps +Scenario G4 (coord-heal-midturn, #894 r4): the render-time live-tool gate. A +server-origin idle edge starts a held backstop fetch, then a real +``tool_pending`` event makes the event-owned live-call set non-empty. The +good response must be declined without wiping the stale transcript or the +live tool shell (``mid1``); a matching result plus another idle edge then +heals. ``events_requests`` stays unchanged and ``history_requests`` grows by +exactly two (declined fetch + heal). Stamps ``RECOVERY-READY-COORDHEALMIDTURN-heal1-mid1-sse0``. Scenario G5 (coord-hidden-retry, #894 r4): the retry's stream-liveness @@ -193,7 +198,7 @@ double-renders on the show-edge replay): ``history_requests`` UNCHANGED across the hidden fire window (``hidden0``). A replay_ok reconnect carries no synthetic state_change (only fresh/truncated replays do), so post-show the latch stays closed (the accepted liveness-lag residual) -until the runner drives an ORGANIC settle with a plain send; that idle edge fires the TRANSPORT-FREE backstop on the live +until a server-origin idle edge fires the TRANSPORT-FREE backstop on the live stream (exactly ONE new SSE open across show + heal — the user-driven reconnect; the heal adds zero). Stamps ``RECOVERY-READY-COORDHIDDENRETRY-hidden0-heal1``. @@ -397,18 +402,14 @@ HEALED_SENTINEL = "HEALED-e5b1" # a fresh bubble, not the stale one" is an honest DOM check. SECOND_SENTINEL = "SECOND-a7f3" -# Fourth-turn sentinel for scenario E4 (stale-backstop): the scripted -# final_text of the plain send that drives the idle-edge backstop. Same -# collision-proof discipline — the E4 transcript is pure final_text turns -# ("one"/"two"/"three" + the seed messages), so this hex-suffixed token -# proves the sent turn reached the healed /history render, distinct from -# everything else on screen. +# Heal sentinel for scenario E6 (await-window-gate): its scripted fourth turn +# drives the later organic repair edge and proves that repair rendered. BACKSTOP_SENTINEL = "BACKSTOP-b2e4" # Scenario E8's duplicate-detector sentinel. Its whole job is to be COUNTED, # not merely found: the artefact #900's stream-generation term prevents is a # turn rendered twice, and every other detector in this file counts -# ``.msg.user`` rows — which never travel on the SSE stream, so none of them -# can see it. +# ``.msg.user`` rows — a duplicated assistant/tool slice does not change that +# count, so none of them can see it. DUPLICATE_SENTINEL = "DUPE-c9a7" # Row-count selectors, hoisted above every runner that uses them (#900 r2): @@ -419,14 +420,6 @@ DUPLICATE_SENTINEL = "DUPE-c9a7" _ROWS_JS = "window.__pane.messagesEl.querySelectorAll('.msg.user').length" _COORD_ROWS_JS = "document.getElementById('coord-messages').querySelectorAll('.msg.user').length" -# Scenario G4 (coord-heal-midturn) sentinels: turn 4's final text (the -# settle that fires the held backstop fetch) and turn 5's final text (the -# turn that STARTS during that held fetch and must survive its resolution). -# Same collision-proof discipline as the sentinels above; turn 5's bash -# command output is ``g4-N`` lines, which contain neither token. -MIDTURN_T4_SENTINEL = "MIDT4-c9d1" -MIDTURN_T5_SENTINEL = "MIDT5-f47a" - # --------------------------------------------------------------------------- # The recovery page — served same-origin by the node at /recovery. # --------------------------------------------------------------------------- @@ -484,7 +477,10 @@ PAGE_HTML = r""" // other scenarios onto it would change what they are testing. let ctl = null; let pane; - if (scenario === "destroy-invalidation") { + if ( + scenario === "destroy-invalidation" || + scenario === "handoff-repair-budget" + ) { ctl = createInteractivePane(document.getElementById("mount"), wsId, { base: "", }); @@ -496,6 +492,21 @@ PAGE_HTML = r""" } pane.wsId = wsId; window.__pane = pane; + let peerPane = null; + if ( + scenario === "user-turn-two-pane" || + scenario === "tool-turn-two-pane" + ) { + // The standalone default host is a shared singleton. Give each pane a + // shallow copy before wrapping onStreamOpen so instrumentation remains + // per-pane instead of nesting twice on the same callback object. + pane._host = { ...pane._host }; + peerPane = new InteractivePane(wsId, { base: "" }); + peerPane._host = { ...peerPane._host }; + peerPane.wsId = wsId; + document.getElementById("mount").appendChild(peerPane.el); + window.__peerPane = peerPane; + } window.__hide = function () { Object.defineProperty(document, "hidden", { configurable: true, value: true }); @@ -542,6 +553,16 @@ PAGE_HTML = r""" .catch((e) => { document.title = "RECOVERY-FAILED-send-" + e; }); } + // Drive the production optimistic composer path for the multi-pane + // user_turn projection check. The pane mints client_send_id itself; + // the origin must replace that exact bubble while the peer paints the + // canonical row from the same event. + window.__sendProjectedUserTurn = function (msg) { + pane.inputEl.value = msg; + pane.sendMessage(); + return true; + }; + // Drive /send once the stream is live (wrap the default host hook). const origOpen = pane._host.onStreamOpen.bind(pane._host); pane._host.onStreamOpen = function (p) { @@ -555,11 +576,19 @@ PAGE_HTML = r""" window.__streamOpen === 1 ) sendOnce("run a turn"); - // rewind-window drives its turns SERVER-side before navigation (a - // /send never emits a live user row — only /history replay does), so + // rewind-window drives its turns SERVER-side before navigation, when + // no page listener exists; the initial /history paints those rows, so // the page never auto-sends there. }; + if (peerPane) { + const peerOrigOpen = peerPane._host.onStreamOpen.bind(peerPane._host); + peerPane._host.onStreamOpen = function (p) { + peerOrigOpen(p); + window.__peerStreamOpen = (window.__peerStreamOpen || 0) + 1; + }; + } + // Shared transport instrumentation for the truncated-recovery // scenarios: count replay_truncated envelopes (the original restart // idiom) and, for stale-ref-reload, tear the transport down the @@ -605,12 +634,15 @@ PAGE_HTML = r""" // in-flight load destroy() must invalidate is the one connect() // starts. if (ctl) ctl.connect(); - else pane._loadHistoryThenConnect(wsId); + else { + pane._loadHistoryThenConnect(wsId); + if (peerPane) peerPane._loadHistoryThenConnect(wsId); + } // Count a sentinel's OCCURRENCES in the transcript text. Every other - // detector in this harness counts `.msg.user` rows, and user rows are - // never emitted on the SSE stream — so none of them can observe a turn - // rendered twice. Counting text is structure-agnostic: it catches a + // detector in this harness counts `.msg.user` rows; an assistant/tool + // slice rendered twice leaves that count unchanged. Counting text is + // structure-agnostic: it catches a // duplicate assistant bubble, a duplicate tool block, or both. window.__countSentinel = function (s) { const text = pane.messagesEl.textContent || ""; @@ -865,9 +897,9 @@ PAGE_HTML = r""" // backstop (#890, the round-5 critical). A rewind's clear_ui refetch // AND its one bounded 2s retry both 500, so the latch cannot // self-heal and rewind/edit stay gated over the stale-but-real - // transcript. A plain send (deliberately NOT latch-gated) runs a - // fresh turn whose ORGANIC turn-settle idle edge fires the backstop — - // a quiesced, same-token REST _refetchHistory, NOT + // transcript. A server-origin idle state edge fires the backstop + // without admitting another user row — a quiesced, same-token REST + // _refetchHistory, NOT // _loadHistoryThenConnect (the old reload backstop drew the server's // synthetic state_change:idle on its fresh reconnect and re-triggered // itself: a zero-backoff reconnect/refetch storm). The runner @@ -883,12 +915,12 @@ PAGE_HTML = r""" ) { const userRows = pane.messagesEl.querySelectorAll(".msg.user").length; - // heal1 = the backstop's quiesced REST refetch rebuilt the rewound - // (ONE user turn) + sent (a second) transcript => TWO user rows, - // latch cleared. sse0 = it touched the transport ZERO times + // heal1 = the backstop's quiesced REST refetch rebuilt the single + // post-rewind user row and cleared the latch. sse0 = it touched + // the transport ZERO times // (sseDelta 0 — the storm regression opens one EventSource per - // reconnect). histDelta 1 = the backstop's single fetch (the - // plain fourth turn emits no clear_ui). gatedPosts 1 = the row-0 + // reconnect). histDelta 1 = the backstop's single fetch. + // gatedPosts 1 = the row-0 // rewind stayed latch-gated while stale. posts 2 = the healed // render reopened the gate and a fresh rewind landed. const ok = @@ -932,8 +964,8 @@ PAGE_HTML = r""" // A replay_ok reconnect carries no synthetic state_change (only // fresh/truncated replays do), so the latch stays closed across // __show — the accepted liveness-lag residual, not a defect. The - // heal rides the runner's plain send, whose ORGANIC turn-settle - // idle edge fires the transport-free backstop on the live stream. + // heal rides a server-origin idle edge through the real UI event + // path, firing the transport-free backstop on the live stream. window.__verifyHiddenRetry = function ( hiddenDelta, healed, @@ -944,9 +976,8 @@ PAGE_HTML = r""" pane.messagesEl.querySelectorAll(".msg.user").length; // hidden0 = the retry did NOT fetch while the transport was down // (the guard held — the whole point of the scenario). - // heal1 = after __show + a plain send, the backstop's refetch - // rebuilt the rewound (ONE user turn) + sent (a second) - // transcript => TWO user rows with the sentinel. + // heal1 = after __show + the idle pulse, the backstop's refetch + // rebuilt the single post-rewind user row. // showSse 1 = exactly ONE new EventSource across show + heal (the // show edge's own reconnect; the transport-free heal adds none). // posts 2 = the healed render reopened the gate and a fresh @@ -1109,6 +1140,7 @@ COORD_PAGE_HTML = r""" (function () { const RealES = window.EventSource; function CountingES(url, opts) { + window.__lastEventSourceUrl = String(url); const es = new RealES(url, opts); es.addEventListener("open", function () { window.__esOpens += 1; @@ -1287,12 +1319,10 @@ COORD_PAGE_HTML = r""" userRows; }; - // G4 — the render-time gate (#894 r4): a turn that STARTS during a - // heal's in-flight /history must SURVIVE its resolution (the gate - // skips the wipe; pre-gate code detached the live turn into a - // dangling ref and lost it). midturnSurvived is the runner's - // observation that turn 5's sentinel painted while the stale - // transcript was still up — the not-wiped proof. + // G4 — the render-time tool gate (#894 r4): a live tool_pending phase + // starts during a held seedless /history. The good response must be + // declined, leaving both the stale transcript and live tool shell + // continuously visible. midturnSurvived carries that observation. window.__verifyCoordHealMidturn = function ( healed, midturnSurvived, @@ -1328,9 +1358,9 @@ COORD_PAGE_HTML = r""" // is down (hiddenDelta 0 — a seedless render past the frozen // cursor double-renders on the show-edge replay); the show edge // only restores the transport (replay_ok carries no synthetic - // state_change), and the heal rides the next ORGANIC settle — the - // runner's plain send (one user-driven SSE open, rows healed, - // gate reopened). + // state_change), and the heal rides a server-origin idle edge through + // the real UI event path (one user-driven SSE open, rows healed, gate + // reopened). window.__verifyCoordHiddenRetry = function ( hiddenDelta, healed, @@ -2243,8 +2273,8 @@ def _send_in_page(cdp: CDP, message: str) -> None: """POST /send from inside the page via the pane's own authFetch (cookie auth, node-proxy base) — the one shared shape for scenarios that drive a turn mid-flight (the shared shape for every scenario that drives a - turn outside the composer). A raw POST emits no live user row, so the sent - turn appears only via the next /history render.""" + turn outside the composer). The accepted row is projected live as + ``user_turn`` just like a composer send.""" cdp.evaluate( "window.authFetch('/v1/api/workstreams/' + " "encodeURIComponent(window.__pane.wsId) + '/send', {method:'POST'," @@ -2431,16 +2461,14 @@ def _seed_three_completed_turns(name: str, extra_scripts: tuple[Any, ...] = ()) """Boot a node and drive THREE completed ``final_text`` turns, returning ``(node, ws_id)`` — the byte-identical seeding the rewind scenarios (E2/E3/E4) share, extracted so their rewind arithmetic provably reads off - the SAME transcript. Each of the three ``node.send`` calls consumes one - scripted turn; a /send never emits a live user row (only /history replay - does), so the initial page-load /history render is what paints all three - user rows. + the SAME transcript. Each of the three ``node.send`` calls consumes one + scripted turn before a browser listener exists, so the initial page-load + /history render paints all three user rows. ``extra_scripts`` are appended to the scripted client AFTER the three - seeding scripts and left UNSENT: E4 queues a fourth ``final_text`` turn - there for the later backstop-driving send (its positional script stays in - sync because the three seeding sends consume exactly the three seeding - scripts).""" + seeding scripts and left UNSENT for scenarios that later drive a real + turn. Their positional scripts stay in sync because the three seeding + sends consume exactly the three leading scripts.""" from tests._sse_recovery_server import final_text_script node = _boot_node() @@ -2479,6 +2507,18 @@ def run_rewind_window(chrome: str) -> str: 0.2, ): raise AssertionError("rewind-window: three user rows never rendered") + # The REST rows paint before the initial EventSource registration is + # guaranteed to finish. Clicking in that legitimate handoff gap can + # advance the history token first, producing a strong resync instead + # of the established-listener ``clear_ui`` whose quiesce this scenario + # is specifically meant to exercise. Coordinator G1 carries the same + # transport-open gate. + if not _poll_until( + lambda: cdp.evaluate("(window.__streamOpen || 0) >= 1"), + 10, + 0.05, + ): + raise AssertionError("rewind-window: SSE stream never opened") # Hold every /history 3s so the clear_ui refetch keeps the quiesce # armed long enough to click the second rewind mid-rebuild. node.delay_history(3000) @@ -2654,33 +2694,24 @@ def run_stale_backstop(chrome: str) -> str: backstop (#890, the round-5 critical). The DOUBLE-failure sibling of E3: a rewind's clear_ui refetch AND its one bounded 2s retry are BOTH forced to 500 (``node.fail_history(2)``), so the latch cannot self-heal and - rewind/edit stay gated over the stale-but-real transcript. A plain send - — sends are deliberately NOT latch-gated (``sendMessage`` gates only on - ``busy``; the raw ``authFetch`` here bypasses even that) — runs the fourth - scripted ``final_text`` turn whose ORGANIC turn-settle idle edge fires the - backstop: a quiesced, same-token REST ``_refetchHistory``, deliberately - NOT ``_loadHistoryThenConnect`` (the old reload backstop drew the server's - synthetic ``state_change:idle`` on its fresh reconnect and re-triggered - itself — a zero-backoff reconnect/refetch storm). With the fault budget - exhausted the refetch succeeds and heals the rewound + sent transcript. + rewind/edit stay gated over the stale-but-real transcript. The recovery + server then publishes one real ``state_change:idle`` through the loaded + UI. That is the settle edge the backstop consumes, without admitting a + live ``user_turn`` or starting unrelated model work that would change the + row and transcript counters in this transport-isolation probe. The + backstop remains a quiesced, same-token + REST ``_refetchHistory``, deliberately NOT ``_loadHistoryThenConnect``. + With the fault budget exhausted it heals the rewound transcript. THE r5 PROOF (both counted at the fault layer): ``events_requests`` is UNCHANGED across the whole heal (``sse0`` — zero new EventSource connections; the storm regression opens one per reconnect) and - ``history_requests`` grew by exactly ONE (the backstop's single fetch — - the plain fourth turn emits no clear_ui). Backend proofs: + ``history_requests`` grew by exactly ONE (the backstop's single fetch). + Backend proofs: ``rewind_requests`` 1 -> (gated) 1 -> 2, ``history_fail_remaining == 0``. Every poll is deadline-bounded so a regressed looping backstop stamps a clean FAILED, never a hang.""" - from tests._sse_recovery_server import final_text_script - - # Seed three turns and queue a FOURTH final_text (the sentinel-bearing - # turn the backstop-driving send below runs) — the shared helper keeps the - # seeding byte-identical to E2/E3 so the rewind arithmetic matches. - node, ws_id = _seed_three_completed_turns( - "browser-stale-backstop", - extra_scripts=(final_text_script(BACKSTOP_SENTINEL),), - ) + node, ws_id = _seed_three_completed_turns("browser-stale-backstop") profile = Path(_scratch()) / "chrome-stale-backstop" proc, cdp_port = _launch_chrome(chrome, profile) cdp: CDP | None = None @@ -2746,30 +2777,22 @@ def run_stale_backstop(chrome: str) -> str: raise AssertionError("stale-backstop: first-row rewind button missing") _poll_until(lambda: node.rewind_requests != 1, 1.5, 0.05) gated_posts = node.rewind_requests # must still be 1 (latch gated it) - # Baselines captured the instant BEFORE the send: the heal must add + # Baselines captured immediately before the idle edge: the heal adds # exactly ZERO SSE opens and exactly ONE /history fetch relative here. events_baseline = node.events_requests history_baseline = node.history_requests - # Drive a plain send via in-page authFetch — sends are NOT latch-gated - # (see docstring), so this reaches the server, runs the fourth scripted - # turn (BACKSTOP_SENTINEL), and its ORGANIC turn-settle idle edge fires - # the backstop. - _send_in_page(cdp, "fourth turn") - # HEAL: the fourth turn settles -> idle edge -> quiesced REST refetch - # (fault exhausted) succeeds -> replayHistory rebuilds the rewound (ONE - # user turn) + sent (a second) transcript to TWO user rows, SENTINEL - # present, latch cleared. The 3-user-rows -> 2-user-rows transition is + # Publish the same per-workstream idle envelope a real turn settle + # emits, without accepting/projecting another user turn or starting + # unrelated model work. + node.emit_idle_edge(ws_id) + # HEAL: idle edge -> quiesced REST refetch (fault exhausted) succeeds + # -> replayHistory rebuilds the rewound transcript to ONE user row and + # clears the latch. The 3-user-rows -> 1-user-row transition is # the load-bearing proof the backstop RENDERED — a stale transcript can # only change via a successful /history render. Deadline-bounded so a # regressed looping backstop times out to a clean FAILED, never a hang. healed = _poll_until( - lambda: cdp.evaluate( - _ROWS_JS + " === 2 " - "&& window.__pane._historyStale === false " - "&& (window.__pane.messagesEl.textContent||'').includes(" - + json.dumps(BACKSTOP_SENTINEL) - + ")" - ), + lambda: cdp.evaluate(_ROWS_JS + " === 1 && window.__pane._historyStale === false"), 20, 0.2, ) @@ -2780,8 +2803,8 @@ def run_stale_backstop(chrome: str) -> str: # connectSSE would bump events_requests by one per reconnect — the # round-5 storm). The EventSource opened at initial load is already # folded into the baseline. - # - history_delta MUST be 1: the plain fourth turn emits no clear_ui, - # so the ONLY /history in the send+heal window is the backstop fetch. + # - history_delta MUST be 1: the only /history in the edge+heal + # window is the backstop fetch. events_delta = node.events_requests - events_baseline history_delta = node.history_requests - history_baseline # REOPEN: the healing render cleared the latch, so a rewind on a @@ -2842,16 +2865,12 @@ def run_hidden_retry(chrome: str) -> str: A replay_ok reconnect carries NO synthetic ``state_change`` (only fresh/truncated replays do), so the latch stays closed across ``__show``: that is the accepted liveness-lag residual, and no timer may shortcut it. - The heal therefore rides the runner's plain send (sends are never - latch-gated), whose ORGANIC turn-settle idle edge fires the - TRANSPORT-FREE backstop on the now-live stream — hence exactly ONE new - SSE open across show + heal (the show edge's own; the heal adds zero).""" - from tests._sse_recovery_server import final_text_script + The recovery server therefore publishes one real ``state_change:idle`` + after show; this is the same backstop trigger without admitting a live + user turn or starting unrelated model work. Exactly ONE new SSE + open remains across show + heal (the show edge's own; the heal adds zero).""" - node, ws_id = _seed_three_completed_turns( - "browser-hidden-retry", - extra_scripts=(final_text_script(BACKSTOP_SENTINEL),), - ) + node, ws_id = _seed_three_completed_turns("browser-hidden-retry") profile = Path(_scratch()) / "chrome-hidden-retry" proc, cdp_port = _launch_chrome(chrome, profile) cdp: CDP | None = None @@ -2897,20 +2916,15 @@ def run_hidden_retry(chrome: str) -> str: latch_held = cdp.evaluate("window.__pane._historyStale === true") # Show: the reconnect presents the frozen cursor (replay_ok, nothing # lost) and carries no synthetic state_change, so the latch survives - # it. A plain send then drives the ORGANIC settle the backstop needs. + # it. A server-origin idle edge then drives the settle the backstop + # needs without admitting another user row. events_before_show = node.events_requests cdp.evaluate("window.__show && window.__show()") if not _poll_until(lambda: node.events_requests == events_before_show + 1, 10, 0.05): raise AssertionError("hidden-retry: show-edge reconnect never arrived") - _send_in_page(cdp, "fourth turn") + node.emit_idle_edge(ws_id) healed = _poll_until( - lambda: cdp.evaluate( - _ROWS_JS - + " === 2 && window.__pane._historyStale === false" - + " && (window.__pane.messagesEl.textContent||'').includes(" - + json.dumps(BACKSTOP_SENTINEL) - + ")" - ), + lambda: cdp.evaluate(_ROWS_JS + " === 1 && window.__pane._historyStale === false"), 20, 0.2, ) @@ -3061,10 +3075,10 @@ def run_reconnect_in_await(chrome: str) -> str: """Scenario E8 — the stream-generation term (#900 r2), and the first detector in this harness that can observe a DOUBLE RENDER at all. - Every other scenario counts ``.msg.user`` rows, and user rows never travel - on the SSE stream (a /send emits none — only /history replay paints them), - so none of them can see the artefact this whole campaign prevents: a turn - rendered twice. E8 counts a sentinel's OCCURRENCES in the transcript text + Every other scenario counts ``.msg.user`` rows; an assistant/tool slice + rendered twice leaves that count unchanged, so none of them can see the + artefact this whole campaign prevents. E8 counts a sentinel's OCCURRENCES + in the transcript text instead, which is structure-agnostic across duplicate assistant bubbles and duplicate tool blocks. @@ -3483,9 +3497,10 @@ def run_coord_stale_backstop(chrome: str) -> str: sibling of G2: the rewind's clear_ui refetch AND its one bounded 2s retry are BOTH forced to 500 (``node.fail_history(2)``), so the latch cannot self-heal and rewind/edit stay gated over the stale-but-real transcript. - A plain send (never latch-gated; raw authFetch here) runs the fourth - scripted turn whose ORGANIC turn-settle idle edge fires the backstop: a - plain seedless REST ``refetchHistory`` — deliberately NOT + The recovery server publishes one real ``state_change:idle`` through the + loaded UI, which fires the backstop without admitting a live user turn or + starting unrelated model work: a plain seedless REST + ``refetchHistory`` — deliberately NOT ``loadHistoryThenReconnect`` (a reconnecting heal draws the server's synthetic ``state_change:idle`` back into its own trigger: a zero-backoff reconnect/refetch storm). @@ -3493,20 +3508,12 @@ def run_coord_stale_backstop(chrome: str) -> str: THE STORM PROOF (both counted at the fault layer): ``events_requests`` is UNCHANGED across the whole heal (``sse0`` — zero new EventSource connections) and ``history_requests`` grew by exactly ONE (the - backstop's single fetch; the plain fourth turn emits no clear_ui). + backstop's single fetch). Coord's latch is closure-private, so the latch-cleared proof is the reopen POST (rewind_requests -> 2), not a field read. Every poll is deadline-bounded so a regressed looping backstop stamps a clean FAILED, never a hang.""" - from tests._sse_recovery_server import final_text_script - - # Seed three turns and queue a FOURTH final_text (the sentinel-bearing - # turn the backstop-driving send below runs) — the shared helper keeps - # the seeding byte-identical to G1/G2 so the rewind arithmetic matches. - node, ws_id = _seed_three_completed_turns( - "browser-coord-stale-backstop", - extra_scripts=(final_text_script(BACKSTOP_SENTINEL),), - ) + node, ws_id = _seed_three_completed_turns("browser-coord-stale-backstop") profile = Path(_scratch()) / "chrome-coord-stale-backstop" proc, cdp_port = _launch_chrome(chrome, profile) cdp: CDP | None = None @@ -3523,26 +3530,18 @@ def run_coord_stale_backstop(chrome: str) -> str: raise AssertionError("coord-stale-backstop: first-row rewind button missing") _poll_until(lambda: node.rewind_requests != 1, 1.5, 0.05) gated_posts = node.rewind_requests # must still be 1 (latch gated it) - # Baselines captured the instant BEFORE the send: the heal must add + # Baselines captured immediately before the idle edge: the heal adds # exactly ZERO SSE opens and exactly ONE /history fetch from here. events_baseline = node.events_requests history_baseline = node.history_requests - # Drive a plain send via in-page authFetch — sends are NOT - # latch-gated, so this reaches the server, runs the fourth scripted - # turn (BACKSTOP_SENTINEL), and its ORGANIC turn-settle idle edge - # fires the backstop. - _send_in_page(cdp, "fourth turn") - # HEAL: the fourth turn settles -> idle edge -> plain REST refetch - # (fault exhausted) succeeds -> the render rebuilds the rewound (ONE - # user turn) + sent (a second) transcript to TWO user rows, SENTINEL - # present. The 3->2 transition is the load-bearing proof the + node.emit_idle_edge(ws_id) + # HEAL: idle edge -> plain REST refetch (fault exhausted) succeeds -> + # the render rebuilds the rewound transcript to ONE user row. The + # 3->1 transition is the load-bearing proof the # backstop RENDERED; the latch-cleared proof is the reopen POST # below (the latch itself is closure-private). healed = _poll_until( - lambda: cdp.evaluate( - _COORD_ROWS_JS + " === 2 && (document.getElementById('coord-messages')" - ".textContent||'').includes(" + json.dumps(BACKSTOP_SENTINEL) + ")" - ), + lambda: cdp.evaluate(_COORD_ROWS_JS + " === 1"), 20, 0.2, ) @@ -3551,9 +3550,8 @@ def run_coord_stale_backstop(chrome: str) -> str: # - events_delta MUST be 0: the backstop is a REST refetchHistory, # ZERO EventSource connections (a reload backstop's connectSSE # bumps events_requests per reconnect — the storm). - # - history_delta MUST be 1: the plain fourth turn emits no - # clear_ui, so the ONLY /history in the send+heal window is the - # backstop's fetch. + # - history_delta MUST be 1: the only /history in the edge+heal + # window is the backstop's fetch. events_delta = node.events_requests - events_baseline history_delta = node.history_requests - history_baseline # REOPEN: the healing render cleared the latch, so a rewind on a @@ -3583,49 +3581,29 @@ def run_coord_stale_backstop(chrome: str) -> str: def run_coord_heal_midturn(chrome: str) -> str: - """Scenario G4 — the render-time gate (#894 r4). A turn that STARTS - while a heal's /history is in flight must SURVIVE the fetch resolution: - pre-gate code ran ``replaceChildren`` under the live turn, detaching its - bubble/tool rows into dangling refs (every remaining token rendered - invisibly, the optimistic user row was destroyed, and nothing - re-rendered the lost turn). The gate skips the wipe instead — the - latch stays set and the turn's OWN settle re-fires the backstop. + """Scenario G4 — the coordinator's seedless render-time tool gate. - Choreography: G3's double-failure prologue leaves the latch stuck; - ``delay_history`` then holds the backstop fetch that turn 4's settle - fires; the moment the held fetch ARRIVES (history_requests bumps before - the hold sleeps) the runner sends turn 5 — a paced bash turn (~2.5s) - that is still mid-stream when the hold (1.5s) releases. The gate must - skip that resolution (turn 5's sentinel paints into the still-stale - transcript = midturnSurvived), and turn 5's settle re-fires the - backstop, which now heals: transcript = rewound turn + turns 4 and 5, - seeds two/three gone. Storm proof: zero SSE opens across the whole - episode; exactly TWO /history fetches (the skipped one + the heal). + G3's double-failure prologue leaves the latch stuck. A server-origin + idle edge starts the seedless backstop fetch and ``delay_history`` holds + it. While it is in flight the recovery server publishes a real + ``tool_pending`` envelope through ``SessionUIBase.on_agent_step``. The + coordinator's event-owned ``liveToolCalls`` set is therefore non-empty + when the good history response resolves, so the render must be declined: + the stale transcript and the live tool shell both remain continuously + visible. This avoids using ``/send`` as a trigger; a send would project a + live ``user_turn`` and start unrelated model activity while this precise + render window is being measured. - Detector honesty: ``history_delta == 2`` is the DISCRIMINATING bit, - and the skip it witnesses rides the event-driven live-tool-call set - (fed by turn 5's live tool_pending announce) — turn 5 is in its TOOL - phase at resolution (content refs null), so this scenario is the - behavioral detector for the gate's tool-phase branch (the content-ref branch and the liveness statement - carry structural pins; G5 covers the hidden-retry liveness path). - Gate-stripped code renders the held fetch early, which CLEARS the - latch, kills the backstop refire, and stamps ``hist1`` (plus - downstream posts/rows drift). The ``mid1`` bit alone cannot - discriminate: the early render repaints all three user rows from the - snapshot (the rewind pre-dates turn 4, so /history already carries - turns 4 and 5's user rows). On gated code ``mid1`` asserts the - stronger continuous-visibility claim (nothing wiped at any point).""" - from tests._sse_recovery_server import final_text_script, parallel_bash_script - - paced5 = parallel_bash_script({"g4": "for i in $(seq 1 50); do echo g4-$i; sleep 0.05; done"}) - node, ws_id = _seed_three_completed_turns( - "browser-coord-heal-midturn", - extra_scripts=( - final_text_script(MIDTURN_T4_SENTINEL), - paced5, - final_text_script(MIDTURN_T5_SENTINEL), - ), - ) + The matching ``tool_result`` retires the live-call entry and a second idle + edge re-fires the backstop. That fetch may render and heals to the single + post-rewind user row. Exact discriminators: the first successful payload + completed while rows3 + the live shell survived, zero SSE opens, and + exactly two history fetches (declined + healing). Removing the tool gate + renders the first payload, wipes the shell, clears the latch, and leaves + only one history fetch.""" + probe_call_id = "recovery-render-gate-probe" + probe_selector = ".conv-batch--running .conv-row[data-call-id='" + probe_call_id + "']" + node, ws_id = _seed_three_completed_turns("browser-coord-heal-midturn") profile = Path(_scratch()) / "chrome-coord-heal-midturn" proc, cdp_port = _launch_chrome(chrome, profile) cdp: CDP | None = None @@ -3636,46 +3614,55 @@ def run_coord_heal_midturn(chrome: str) -> str: _coord_stick_latch(cdp, node, "coord-heal-midturn") events_baseline = node.events_requests history_baseline = node.history_requests - # Hold every /history long enough for turn 5 to start under it, but - # shorter than turn 5's ~2.5s bash phase, so the held fetch resolves - # MID-turn — the exact window the gate exists for. + history_ok_baseline = node.history_ok + # Hold the first backstop fetch long enough to publish and visibly + # confirm the live tool phase before its response resolves. node.delay_history(1500) - # Turn 4: settles -> idle edge -> backstop fires -> its fetch is - # HELD. history_requests bumps on ARRIVAL (before the hold sleeps), - # which is the observable that the window is open. - _send_in_page(cdp, "fourth turn") + node.emit_idle_edge(ws_id) if not _poll_until(lambda: node.history_requests == history_baseline + 1, 20, 0.05): raise AssertionError("coord-heal-midturn: backstop fetch never arrived") - # Turn 5, INSIDE the hold: paced bash keeps it mid-stream when the - # held fetch resolves. The gate must skip that render. - _send_in_page(cdp, "fifth turn") - # midturnSurvived: turn 5's final sentinel paints into the - # still-stale transcript (user rows unchanged at 3) — a wiped pane - # would have dropped to 1 row and swallowed the sentinel into a - # detached ref. Poll spans the hold release (+1.5s) and turn 5's - # full run. - midturn_survived = _poll_until( + node.emit_tool_pending(ws_id, probe_call_id) + if not _poll_until( lambda: cdp.evaluate( - "(document.getElementById('coord-messages').textContent||'')" - ".includes(" + json.dumps(MIDTURN_T5_SENTINEL) + ") && " + _COORD_ROWS_JS + " === 3" + _COORD_ROWS_JS + + " === 3 && !!document.querySelector(" + + json.dumps(probe_selector) + + ")" ), - 20, - 0.2, + 10, + 0.05, + ): + raise AssertionError("coord-heal-midturn: live tool phase never rendered") + # A 200 response proves the held request resolved with a renderable + # payload. Observe for another bounded window so the browser has had + # ample time to process it; any wipe flips this predicate immediately. + if not _poll_until(lambda: node.history_ok >= history_ok_baseline + 1, 10, 0.05): + raise AssertionError("coord-heal-midturn: held /history never resolved successfully") + gate_broke = _poll_until( + lambda: ( + not cdp.evaluate( + _COORD_ROWS_JS + + " === 3 && !!document.querySelector(" + + json.dumps(probe_selector) + + ")" + ) + ), + 1.0, + 0.05, ) - # Release the hold so turn 5's settle-driven backstop refire heals - # promptly. + midturn_survived = not gate_broke + + # Retire the event-owned live-call entry, then publish the settle edge + # that is now allowed to consume and render authoritative history. node.delay_history(0) - # HEAL: the refire renders the rewound truth — turn "first" + turns - # 4 and 5; the rewound-away seeds are GONE. Text discriminators, - # not row counts: the healed pane also has 3 user rows. + node.emit_tool_result(ws_id, probe_call_id) + node.emit_idle_edge(ws_id) healed = _poll_until( lambda: cdp.evaluate( - "(function(){var t=document.getElementById('coord-messages')" - ".textContent||'';return t.includes(" - + json.dumps(MIDTURN_T4_SENTINEL) - + ")&&t.includes(" - + json.dumps(MIDTURN_T5_SENTINEL) - + ")&&!t.includes('second');})()" + _COORD_ROWS_JS + + " === 1 && !document.querySelector(" + + json.dumps(".conv-row[data-call-id='" + probe_call_id + "']") + + ")" ), 20, 0.2, @@ -3715,30 +3702,23 @@ def run_coord_hidden_retry(chrome: str) -> str: cursor and double-renders every turn the hidden render already painted. The ``evtSource`` fire-guard term skips the hidden firing instead (hiddenDelta 0); the show edge only restores the transport - (replay_ok carries no synthetic state_change), and the heal rides - the runner's plain send, whose organic turn-settle idle edge fires - the TRANSPORT-FREE backstop on the live stream. + (replay_ok carries no synthetic state_change), and the recovery server + publishes a real idle state edge to fire the TRANSPORT-FREE backstop on + the live stream without admitting another user row. A replay_ok reconnect (frozen cursor, nothing lost) carries no synthetic state_change — only fresh/truncated replays do — so the - latch stays closed after __show until the next ORGANIC settle — exactly the - accepted-residual ruling (heals ride organic edges; no timer may - shortcut the lag). The runner drives that settle with a plain send - (sends are never latch-gated), whose turn-settle idle edge fires the - TRANSPORT-FREE backstop on the live stream. + latch stays closed after __show until the next settle edge — exactly the + accepted-residual ruling (no timer may shortcut the lag). The test-server + pulse supplies that edge through the real UI event path. Proofs: history_requests UNCHANGED across the hidden retry window (the non-occurrence detector that regresses to hidden1 without the guard); exactly ONE new SSE open across show + heal (the user-driven reconnect - — the heal itself adds zero); the healed render carries the rewound - turn + the sent turn (TWO user rows, sentinel present); the reopen - click lands.""" - from tests._sse_recovery_server import final_text_script + — the heal itself adds zero); the healed render carries the single + post-rewind user row; the reopen click lands.""" - node, ws_id = _seed_three_completed_turns( - "browser-coord-hidden-retry", - extra_scripts=(final_text_script(BACKSTOP_SENTINEL),), - ) + node, ws_id = _seed_three_completed_turns("browser-coord-hidden-retry") profile = Path(_scratch()) / "chrome-coord-hidden-retry" proc, cdp_port = _launch_chrome(chrome, profile) cdp: CDP | None = None @@ -3778,21 +3758,16 @@ def run_coord_hidden_retry(chrome: str) -> str: # Show: the reconnect presents the frozen cursor and replays # replay_ok (nothing lost), which carries NO synthetic # state_change (only fresh/truncated replays do — the - # latch-closed lag is the accepted residual). Wait for the - # reconnect itself, then - # drive an ORGANIC settle with a plain send — its idle edge fires - # the TRANSPORT-FREE backstop on the live stream and the heal - # renders the rewound (ONE) + sent (a second) transcript. + # latch-closed lag is the accepted residual). Wait for the reconnect + # itself, then publish an idle edge through the loaded UI. The heal + # renders the single post-rewind user row. events_before_show = node.events_requests cdp.evaluate("window.__show && window.__show()") if not _poll_until(lambda: node.events_requests == events_before_show + 1, 10, 0.05): raise AssertionError("coord-hidden-retry: show-edge reconnect never arrived") - _send_in_page(cdp, "fourth turn") + node.emit_idle_edge(ws_id) healed = _poll_until( - lambda: cdp.evaluate( - _COORD_ROWS_JS + " === 2 && (document.getElementById('coord-messages')" - ".textContent||'').includes(" + json.dumps(BACKSTOP_SENTINEL) + ")" - ), + lambda: cdp.evaluate(_COORD_ROWS_JS + " === 1"), 20, 0.2, ) @@ -4036,6 +4011,429 @@ def run_coord_joined_flight(chrome: str) -> str: node.stop() +def run_handoff_repair_budget(chrome: str, *, coordinator: bool = False) -> str: + """Strong repair is bounded and fail-closed; a rendered tokenless 200 downgrades. + + Four FAILED (500) history responses prove the bounded automatic budget: + both real browser clients make exactly four attempts without opening an + EventSource, park on the persistent manual prompt, and make one attempt + per Retry click. A rendered-but-tokenless 200 is the server's deliberate + cold storage-only read (or a pre-handoff server) and must DOWNGRADE to + the tokenless bootstrap — one cursorless EventSource, prompt gone — never + burn budget against a healthy response. A later resync answered by the + real (tokened) server reconnects with proof. A final held attempt is + destroyed before it settles and must not resurrect the stream. + """ + + tag = "COORDHANDOFF" if coordinator else "HANDOFF" + scenario = "coord-handoff-repair-budget" if coordinator else "handoff-repair-budget" + node = _boot_node() + ws_id = node.create_workstream(name=f"browser-{scenario}") + profile = Path(_scratch()) / f"chrome-{scenario}" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + route = "coord-recovery" if coordinator else "recovery" + url = f"{node.base_url}/{route}?ws_id={ws_id}&scenario={scenario}" + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + open_expr = "window.__esOpens || 0" if coordinator else "window.__streamOpen || 0" + if not _poll_until(lambda: cdp.evaluate(open_expr) >= 1, 15, 0.1): + raise AssertionError(f"{scenario}: initial EventSource never opened") + + history0 = node.history_requests + events0 = node.events_requests + node.fail_history(4) + node.emit_history_resync(ws_id) + prompt_expr = "!!document.querySelector('.history-handoff-repair')" + if not _poll_until( + lambda: node.history_requests == history0 + 4 and cdp.evaluate(prompt_expr), + 25, + 0.1, + ): + raise AssertionError( + f"{scenario}: repair did not park after four failed attempts " + f"(history={node.history_requests - history0}, events={node.events_requests - events0})" + ) + # No hidden fifth attempt and no unverified EventSource. + time.sleep(3) + if node.history_requests != history0 + 4 or node.events_requests != events0: + raise AssertionError( + f"{scenario}: automatic budget failed closed " + f"(history={node.history_requests - history0}, events={node.events_requests - events0})" + ) + + # One manual FAILED response: exactly one request and no auto burst. + node.fail_history(1) + if not cdp.evaluate("document.querySelector('.history-handoff-retry').click(); true"): + raise AssertionError(f"{scenario}: manual retry button missing") + if not _poll_until(lambda: node.history_requests == history0 + 5, 8, 0.05): + raise AssertionError(f"{scenario}: manual retry did not issue one request") + time.sleep(3) + if node.history_requests != history0 + 5 or node.events_requests != events0: + raise AssertionError(f"{scenario}: manual failure re-armed automatic work") + + # A rendered tokenless 200 downgrades: latch cleared, prompt gone, one + # CURSORLESS EventSource (no history_token). The server's tokenless + # bootstrap then converges the pane (clear_ui -> one more /history). + node.tokenless_history(1) + cdp.evaluate("document.querySelector('.history-handoff-retry').click()") + if not _poll_until( + lambda: ( + node.history_requests >= history0 + 6 + and node.events_requests == events0 + 1 + and not cdp.evaluate(prompt_expr) + ), + 12, + 0.1, + ): + raise AssertionError( + f"{scenario}: rendered tokenless 200 did not downgrade to bootstrap " + f"(history={node.history_requests - history0}, events={node.events_requests - events0})" + ) + tokenless_stream = ( + cdp.evaluate("!((window.__lastEventSourceUrl || '').includes('history_token='))") + if coordinator + else cdp.evaluate( + "!!window.__pane.evtSource && " + "!window.__pane.evtSource.url.includes('history_token=')" + ) + ) + if not tokenless_stream: + raise AssertionError(f"{scenario}: downgraded stream claimed a handoff token") + + # Let the bootstrap convergence (clear_ui refetch) settle before the + # next phase snapshots its counters. + def _history_settled() -> bool: + snapshot = node.history_requests + time.sleep(1.0) + return node.history_requests == snapshot + + if not _poll_until(_history_settled, 15, 0.1): + raise AssertionError(f"{scenario}: bootstrap convergence never settled") + + # A later resync answered by the REAL server reconnects with proof and + # never re-parks. + settled_history = node.history_requests + settled_events = node.events_requests + node.emit_history_resync(ws_id) + if not _poll_until( + lambda: ( + node.history_requests >= settled_history + 1 + and node.events_requests == settled_events + 1 + and not cdp.evaluate(prompt_expr) + ), + 12, + 0.1, + ): + raise AssertionError(f"{scenario}: proven repair did not reconnect once") + tokened = ( + cdp.evaluate("(window.__lastEventSourceUrl || '').includes('history_token=')") + if coordinator + else cdp.evaluate( + "!!window.__pane.evtSource && " + "window.__pane.evtSource.url.includes('history_token=')" + ) + ) + if not tokened: + raise AssertionError(f"{scenario}: proven repair reopened without proof") + + # Terminal teardown during a held attempt: abort/settle immediately; + # when the server-side hold expires, no async tail may reopen. + node.delay_history(4000) + held0 = node.history_requests + opens_before_destroy = node.events_requests + node.emit_history_resync(ws_id) + if not _poll_until(lambda: node.history_requests == held0 + 1, 5, 0.05): + raise AssertionError(f"{scenario}: held teardown attempt never started") + history_before_destroy = node.history_requests + if coordinator: + cdp.evaluate("window.__pane.destroy()") + else: + cdp.evaluate("window.__ctl.destroy()") + time.sleep(5) + node.delay_history(0) + if ( + node.events_requests != opens_before_destroy + or node.history_requests != history_before_destroy + ): + raise AssertionError( + f"{scenario}: teardown resurrected repair work " + f"(history={node.history_requests - history_before_destroy}, " + f"events={node.events_requests - opens_before_destroy})" + ) + + return f"RECOVERY-READY-{tag}-auto4-manual1-downgrade1-proof1-teardown0" + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + +def run_user_turn_two_pane(chrome: str) -> str: + """One accepted USER reaches two upgraded panes with no REST/redial fan-out.""" + + from tests._sse_recovery_server import final_text_script + + node = _boot_node() + ws_id = node.create_workstream( + final_text_script("projection acknowledged"), + name="browser-user-turn-two-pane", + ) + profile = Path(_scratch()) / "chrome-user-turn-two-pane" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + message = "one shared projected prompt" + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = f"{node.base_url}/recovery?ws_id={ws_id}&scenario=user-turn-two-pane" + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + opened = _poll_until( + lambda: ( + (cdp.evaluate("window.__streamOpen || 0") or 0) == 1 + and (cdp.evaluate("window.__peerStreamOpen || 0") or 0) == 1 + ), + 20, + 0.1, + ) + if not opened: + diagnostics = cdp.evaluate( + "({originOpens:window.__streamOpen||0," + "peerOpens:window.__peerStreamOpen||0," + "originUrl:window.__pane&&window.__pane.evtSource&&window.__pane.evtSource.url," + "peerUrl:window.__peerPane&&window.__peerPane.evtSource&&" + "window.__peerPane.evtSource.url,title:document.title})" + ) + raise AssertionError( + "user-turn-two-pane: both initial streams did not open once " + f"(browser={diagnostics!r}, history={node.history_requests}, " + f"events={node.events_requests})" + ) + + capability_urls = cdp.evaluate( + "({origin: window.__pane.evtSource.url, peer: window.__peerPane.evtSource.url})" + ) + if not all("user_turn=1" in capability_urls[name] for name in ("origin", "peer")): + raise AssertionError(f"user-turn-two-pane: capability missing from {capability_urls!r}") + + # Count reducer deliveries themselves as well as DOM rows. This catches + # a duplicate event that happened to be hidden by DOM/event-id dedup. + cdp.evaluate( + "window.__originUserEvents=0; window.__peerUserEvents=0; " + "window.__originRepairEvents=0; window.__peerRepairEvents=0; " + "const oh=window.__pane.handleEvent.bind(window.__pane); " + "window.__pane.handleEvent=function(e){" + "if(e&&e.type==='user_turn')window.__originUserEvents++;" + "if(e&&e.type==='replay_truncated')window.__originRepairEvents++;" + "return oh(e);}; " + "const ph=window.__peerPane.handleEvent.bind(window.__peerPane); " + "window.__peerPane.handleEvent=function(e){" + "if(e&&e.type==='user_turn')window.__peerUserEvents++;" + "if(e&&e.type==='replay_truncated')window.__peerRepairEvents++;" + "return ph(e);}; true" + ) + history0 = node.history_requests + events0 = node.events_requests + cdp.evaluate(f"window.__sendProjectedUserTurn({json.dumps(message)})") + + rows_expr = ( + "window.__pane.messagesEl.querySelectorAll('.msg.user').length===1 && " + "window.__peerPane.messagesEl.querySelectorAll('.msg.user').length===1" + ) + if not _poll_until(lambda: bool(cdp.evaluate(rows_expr)), 12, 0.05): + raise AssertionError("user-turn-two-pane: canonical rows did not render once") + node.wait_turn(ws_id, timeout=30) + time.sleep(0.5) + + state = cdp.evaluate( + "(()=>{const row=(p)=>p.messagesEl.querySelector('.msg.user');" + "const text=(r)=>r&&r.querySelector('.msg-user-text')&&" + "r.querySelector('.msg-user-text').textContent;" + "const o=row(window.__pane),p=row(window.__peerPane);" + "return {originRows:window.__pane.messagesEl.querySelectorAll('.msg.user').length," + "peerRows:window.__peerPane.messagesEl.querySelectorAll('.msg.user').length," + "originText:text(o),peerText:text(p)," + "originId:o&&o.dataset.eventId,peerId:p&&p.dataset.eventId," + "originEvents:window.__originUserEvents,peerEvents:window.__peerUserEvents," + "originRepair:window.__originRepairEvents,peerRepair:window.__peerRepairEvents," + "originOpens:window.__streamOpen,peerOpens:window.__peerStreamOpen};})()" + ) + if state != { + "originRows": 1, + "peerRows": 1, + "originText": message, + "peerText": message, + "originId": state.get("originId"), + "peerId": state.get("peerId"), + "originEvents": 1, + "peerEvents": 1, + "originRepair": 0, + "peerRepair": 0, + "originOpens": 1, + "peerOpens": 1, + }: + raise AssertionError(f"user-turn-two-pane: unexpected projection state {state!r}") + if not state["originId"] or state["originId"] != state["peerId"]: + raise AssertionError(f"user-turn-two-pane: canonical ids diverged {state!r}") + if node.history_requests != history0 or node.events_requests != events0: + raise AssertionError( + "user-turn-two-pane: normal send caused REST/redial fan-out " + f"(history={node.history_requests - history0}, events={node.events_requests - events0})" + ) + return "RECOVERY-READY-USERTURN-panes2-rows1-events1-history0-redial0" + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + +def run_tool_turn_two_pane(chrome: str) -> str: + """Two accepted TOOL rows reach two panes without REST/redial fan-out.""" + + from tests._sse_recovery_server import bash_toolcall_script, final_text_script + + call_id = "browser-reused-tool-id" + first_sentinel = "TOOL_ONE_SENTINEL" + second_sentinel = "TOOL_TWO_SENTINEL" + node = _boot_node() + ws_id = node.create_workstream( + bash_toolcall_script(call_id, f"printf {first_sentinel}"), + bash_toolcall_script(call_id, f"printf {second_sentinel}"), + final_text_script("tool projection acknowledged"), + name="browser-tool-turn-two-pane", + ) + profile = Path(_scratch()) / "chrome-tool-turn-two-pane" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = f"{node.base_url}/recovery?ws_id={ws_id}&scenario=tool-turn-two-pane" + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + opened = _poll_until( + lambda: ( + (cdp.evaluate("window.__streamOpen || 0") or 0) == 1 + and (cdp.evaluate("window.__peerStreamOpen || 0") or 0) == 1 + ), + 20, + 0.1, + ) + if not opened: + diagnostics = cdp.evaluate( + "({originOpens:window.__streamOpen||0," + "peerOpens:window.__peerStreamOpen||0," + "originUrl:window.__pane&&window.__pane.evtSource&&window.__pane.evtSource.url," + "peerUrl:window.__peerPane&&window.__peerPane.evtSource&&" + "window.__peerPane.evtSource.url,title:document.title})" + ) + raise AssertionError( + "tool-turn-two-pane: both initial streams did not open once " + f"(browser={diagnostics!r}, history={node.history_requests}, " + f"events={node.events_requests})" + ) + + capability_urls = cdp.evaluate( + "({origin: window.__pane.evtSource.url, peer: window.__peerPane.evtSource.url})" + ) + for name in ("origin", "peer"): + if "tool_turn=1" not in capability_urls[name]: + raise AssertionError( + f"tool-turn-two-pane: capability missing from {capability_urls!r}" + ) + + # Count reducer deliveries separately from DOM convergence. Each tool + # emits a preliminary receipt and one accepted replacement; only the + # accepted frames are the durable projection under test. + cdp.evaluate( + "window.__originAcceptedToolEvents=[]; window.__peerAcceptedToolEvents=[]; " + "window.__originRepairEvents=0; window.__peerRepairEvents=0; " + "const oh=window.__pane.handleEvent.bind(window.__pane); " + "window.__pane.handleEvent=function(e){" + "if(e&&e.type==='tool_result'&&e.accepted===true)" + "window.__originAcceptedToolEvents.push({id:e.call_id,name:e.name,output:e.output});" + "if(e&&e.type==='replay_truncated')window.__originRepairEvents++;" + "return oh(e);}; " + "const ph=window.__peerPane.handleEvent.bind(window.__peerPane); " + "window.__peerPane.handleEvent=function(e){" + "if(e&&e.type==='tool_result'&&e.accepted===true)" + "window.__peerAcceptedToolEvents.push({id:e.call_id,name:e.name,output:e.output});" + "if(e&&e.type==='replay_truncated')window.__peerRepairEvents++;" + "return ph(e);}; true" + ) + history0 = node.history_requests + events0 = node.events_requests + cdp.evaluate("window.__sendProjectedUserTurn('run two reused-id tools')") + node.wait_turn(ws_id, timeout=45) + + ready_expr = ( + "window.__originAcceptedToolEvents.length===2 && " + "window.__peerAcceptedToolEvents.length===2 && " + f"window.__pane.messagesEl.querySelectorAll('.conv-row[data-call-id=\"{call_id}\"]').length===2 && " + f"window.__peerPane.messagesEl.querySelectorAll('.conv-row[data-call-id=\"{call_id}\"]').length===2" + ) + if not _poll_until(lambda: bool(cdp.evaluate(ready_expr)), 15, 0.05): + raise AssertionError("tool-turn-two-pane: canonical rows did not converge twice") + time.sleep(0.5) + + state = cdp.evaluate( + "(()=>{const project=(p)=>Array.from(p.messagesEl.querySelectorAll('.conv-batch'))" + f".filter((b)=>b.querySelector('.conv-row[data-call-id=\"{call_id}\"]'))" + ".map((b)=>({outputs:Array.from(b.querySelectorAll('.tool-output'))" + ".map((o)=>o.textContent)}));" + "return {origin:project(window.__pane),peer:project(window.__peerPane)," + "originEvents:window.__originAcceptedToolEvents," + "peerEvents:window.__peerAcceptedToolEvents," + "originRepair:window.__originRepairEvents,peerRepair:window.__peerRepairEvents," + "originOpens:window.__streamOpen,peerOpens:window.__peerStreamOpen};})()" + ) + for pane_name in ("origin", "peer"): + batches = state[pane_name] + if len(batches) != 2: + raise AssertionError(f"tool-turn-two-pane: {pane_name} batches={batches!r}") + rendered = ["\n".join(batch["outputs"]) for batch in batches] + if not all( + expected in rendered[index] and other not in rendered[index] + for index, (expected, other) in enumerate( + ((first_sentinel, second_sentinel), (second_sentinel, first_sentinel)) + ) + ): + raise AssertionError( + f"tool-turn-two-pane: {pane_name} occurrence outputs crossed {rendered!r}" + ) + events = state[f"{pane_name}Events"] + if [event["id"] for event in events] != [call_id, call_id]: + raise AssertionError( + f"tool-turn-two-pane: {pane_name} event ids diverged {events!r}" + ) + event_outputs = [event["output"] for event in events] + if not all( + expected in event_outputs[index] and other not in event_outputs[index] + for index, (expected, other) in enumerate( + ((first_sentinel, second_sentinel), (second_sentinel, first_sentinel)) + ) + ): + raise AssertionError( + f"tool-turn-two-pane: {pane_name} accepted outputs crossed {events!r}" + ) + if state["originRepair"] != 0 or state["peerRepair"] != 0: + raise AssertionError(f"tool-turn-two-pane: unexpected repair {state!r}") + if state["originOpens"] != 1 or state["peerOpens"] != 1: + raise AssertionError(f"tool-turn-two-pane: stream reopened {state!r}") + if node.history_requests != history0 or node.events_requests != events0: + raise AssertionError( + "tool-turn-two-pane: normal tool rows caused REST/redial fan-out " + f"(history={node.history_requests - history0}, " + f"events={node.events_requests - events0})" + ) + return "RECOVERY-READY-TOOLTURN-panes2-rows2-events2-history0-redial0" + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + def _wait_state(node: Any, ws_id: str, state: str, timeout: float) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: @@ -4092,6 +4490,10 @@ def main() -> None: "coord-hidden-retry", "coord-orphan-rewind", "coord-joined-flight", + "handoff-repair-budget", + "coord-handoff-repair-budget", + "user-turn-two-pane", + "tool-turn-two-pane", "roster-restart", "roster-restart-native", "both", @@ -4189,6 +4591,22 @@ def main() -> None: verdict = run_coord_joined_flight(chrome) print(f"scenario G7 (coord-joinedflight): {verdict}") failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("handoff-repair-budget", "all"): + verdict = run_handoff_repair_budget(chrome) + print(f"scenario H1 (handoff-budget): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("coord-handoff-repair-budget", "all"): + verdict = run_handoff_repair_budget(chrome, coordinator=True) + print(f"scenario H2 (coord-handoff-budget): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("user-turn-two-pane", "all"): + verdict = run_user_turn_two_pane(chrome) + print(f"scenario I1 (user-turn-two-pane): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("tool-turn-two-pane", "all"): + verdict = run_tool_turn_two_pane(chrome) + print(f"scenario I2 (tool-turn-two-pane): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 if args.scenario in ("roster-restart", "all"): verdict = run_roster_restart(chrome) print(f"scenario F1 (roster-manual): {verdict}") diff --git a/sdk/typescript/openapi-console.json b/sdk/typescript/openapi-console.json index 3e2300d4..8d9ac624 100644 --- a/sdk/typescript/openapi-console.json +++ b/sdk/typescript/openapi-console.json @@ -6648,7 +6648,7 @@ "tags": [ "Coordinator" ], - "description": "Truncates the coordinator conversation by N turns via the shared rewind handler and emits ``clear_ui`` so the dashboard re-fetches the truncated history. Gated on ``admin.coordinator``.", + "description": "Claims the coordinator mutation slot, durably truncates N turns, and emits ``clear_ui`` so the dashboard re-fetches the truncated history. Concurrent sends are ordered after the cut; storage failure returns 503 without changing live history. Gated on ``admin.coordinator``.", "parameters": [ { "name": "ws_id", @@ -6730,7 +6730,7 @@ "tags": [ "Coordinator" ], - "description": "Drops the last response and re-sends the last user message via the shared worker dispatch, emitting ``clear_ui``. Gated on ``admin.coordinator``.", + "description": "Uses one shared worker claim to drop the last response and start the replacement generation, emitting ``clear_ui``. Another send cannot enter between those operations. Gated on ``admin.coordinator``.", "parameters": [ { "name": "ws_id", @@ -6802,7 +6802,7 @@ "tags": [ "Coordinator" ], - "description": "Releases the worker thread + UI listeners and marks the row ``state=closed`` in storage. The row remains queryable (audit / history) but cannot be reopened \u2014 a closed coordinator is terminal from the manager's perspective.", + "description": "Releases the worker thread + UI listeners and marks the row ``state=closed`` in storage. The row remains queryable (audit / history) but cannot be reopened \u2014 a closed coordinator is terminal from the manager's perspective. Returns 409 while an accepted live conversation row still requires persistence reconciliation; the coordinator remains loaded and its history journal is retained.", "parameters": [ { "name": "ws_id", @@ -6844,6 +6844,16 @@ } } }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "500": { "description": "Error 500", "content": { @@ -6874,7 +6884,7 @@ "tags": [ "Coordinator" ], - "description": "Server-Sent Events stream carrying ``status``, ``message``, ``tool_call``, ``tool_result``, ``approval``, ``error``, and the phase-3 ``child_ws_*`` fan-out events. Pings every 5s. Body is text/event-stream \u2014 the response schema is omitted from the catalog because OpenAPI 3.1 has no first-class SSE type.", + "description": "Server-Sent Events stream carrying ``status``, ``message``, ``tool_call``, ``tool_result``, ``approval``, ``error``, and the phase-3 ``child_ws_*`` fan-out events. After rendering REST history, pass its opaque handoff_token once as ?history_token=; it names the exact accepted conversation-row prefix used for that render. history_resync closes this stream and requires a fresh history read; numeric replay is not a substitute. Native Last-Event-ID reconnects take priority. Pass ?user_turn=1 to receive typed accepted-user events; without it, those rows use the backward-compatible strong-repair projection. Pass ?tool_turn=1 to receive final accepted tool rows as typed tool_result events with accepted=true; without it, accepted tool rows use the same pre-row strong-repair projection. Pings every 5s. Body is text/event-stream \u2014 the response schema is omitted from the catalog because OpenAPI 3.1 has no first-class SSE type.", "parameters": [ { "name": "ws_id", @@ -6883,6 +6893,42 @@ "schema": { "type": "string" } + }, + { + "name": "last_event_id", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Numeric per-workstream event cursor for manual reconnects." + }, + { + "name": "history_token", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Opaque one-shot token naming the accepted prefix rendered from REST history." + }, + { + "name": "user_turn", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Set to 1 to receive typed user_turn events instead of history-repair frames." + }, + { + "name": "tool_turn", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Set to 1 to receive final accepted tool_result projections." } ], "responses": { @@ -6939,7 +6985,7 @@ "tags": [ "Coordinator" ], - "description": "Returns the tail of the conversation in OpenAI-like message format. Used by the page-load handshake; SSE handles updates after that. Bounded by the ``limit`` query parameter.", + "description": "Returns the tail of the conversation in OpenAI-like message format. Used by the page-load handshake; SSE handles updates after that. Cold coordinators are rehydrated before history is served, so every successful response participates in the REST-to-SSE handoff. Messages are the requested tail of one authoritative total accepted conversation-row prefix: user, assistant, tool, and system rows, including projected compaction checkpoints and cancellation markers. The opaque handoff_token names the exact prefix used for the render and is passed once on initial SSE registration. Admission of a later row changes the token; durable acknowledgement does not. If the durable prefix cannot be loaded, the endpoint returns 503 with `History temporarily unavailable`; that response is not authoritative and supplies no usable handoff token. Bounded by the ``limit`` query parameter.", "parameters": [ { "name": "ws_id", @@ -8374,6 +8420,18 @@ "default": 0, "title": "Tool Calls", "type": "integer" + }, + "persistence_state": { + "default": "healthy", + "description": "Sanitized durable-history status for a live row. Older nodes and unloaded persisted-only rows default to healthy.", + "enum": [ + "healthy", + "pending", + "retrying", + "conflict" + ], + "title": "Persistence State", + "type": "string" } }, "required": [ @@ -8552,7 +8610,7 @@ } ], "default": null, - "description": "Live in-flight counters (state, tokens, activity, pending_approval) when the owning node returns them; null on degrade.", + "description": "Live in-flight counters and sanitized durable-history status (state, tokens, activity, pending_approval, persistence_state) when the owning node returns them; null on degrade.", "title": "Live" }, "messages": { @@ -8930,6 +8988,22 @@ "title": "Message", "type": "string" }, + "client_send_id": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9_-]+$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Opaque browser correlation token echoed in the accepted `user_turn` event and history row. It is not an idempotency key; repeated sends with the same value remain distinct turns.", + "title": "Client Send Id" + }, "attachment_ids": { "anyOf": [ { @@ -9369,6 +9443,22 @@ "title": "Message", "type": "string" }, + "client_send_id": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9_-]+$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Opaque browser correlation token echoed in the accepted `user_turn` event and history row. It is not an idempotency key.", + "title": "Client Send Id" + }, "attachment_ids": { "anyOf": [ { @@ -15031,6 +15121,18 @@ "$ref": "#/components/schemas/WorkstreamKind", "default": "interactive" }, + "persistence_state": { + "default": "healthy", + "description": "Sanitized durable-history status: healthy, pending its first save, retrying automatically, or blocked by a permanent commit conflict.", + "enum": [ + "healthy", + "pending", + "retrying", + "conflict" + ], + "title": "Persistence State", + "type": "string" + }, "pending_approval": { "default": false, "description": "True when at least one approval cycle is live (a gate thread parked awaiting an operator approve/deny). Mirrors the same field on ``DashboardWorkstream`` / cluster live projections so a freshly-loaded chat tab can render the inline approval gate from the detail snapshot before SSE replay arrives.", @@ -15170,7 +15272,7 @@ "type": "string" }, "messages": { - "description": "Tail of the workstream's message history, projected to the canonical render shape (``role`` may be ``system`` for operator-context turns; flat tool_calls with verdict / output_assessment; top-level source / attachments / reasoning; derived denied / is_error / pending). Bounded by the ``limit`` query parameter (default 100, max 500).", + "description": "Requested limit-bounded tail of one authoritative total accepted conversation-row prefix, projected to the canonical render shape. Roles include ``user``, ``assistant``, ``tool``, and ``system``; compaction checkpoints project as ``role=system, source=compaction`` and cancellation-generated assistant/tool markers appear when present. The projection also carries flat tool_calls with verdict / output_assessment; top-level source / attachments / reasoning; and derived denied / is_error / pending. Bounded by the ``limit`` query parameter (default 100, max 500).", "items": { "additionalProperties": true, "type": "object" @@ -15190,6 +15292,19 @@ "default": null, "description": "SSE resume cursor (a ``Last-Event-ID`` value). Non-null only when the trailing turn is an executing in-flight tool batch that the live ring buffer can replay: ``messages`` then omits that turn and the client opens its initial SSE with this cursor so the existing delta replay fast-forwards the in-flight turn (tool calls, results, prompts) instead of the lossy synthetic snapshot. Null on every other read \u2014 the client connects fresh.", "title": "Cursor" + }, + "handoff_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Opaque token naming the exact accepted conversation-row prefix used for this render. Present only while the workstream is loaded. A client that renders this response passes the token once as the initial event stream's ``history_token`` query parameter; the server atomically validates it while registering the listener. Admission of a later row changes the token; durable acknowledgement does not. Clients must not inspect, persist, or reuse it for later reconnects.", + "title": "Handoff Token" } }, "required": [ diff --git a/sdk/typescript/openapi-server.json b/sdk/typescript/openapi-server.json index 829f1e9f..4a418ad8 100644 --- a/sdk/typescript/openapi-server.json +++ b/sdk/typescript/openapi-server.json @@ -167,6 +167,7 @@ "tags": [ "Workstreams" ], + "description": "Unloads the live workstream while preserving storage. Returns 409 when any accepted live conversation row still requires persistence reconciliation; the workstream remains loaded and its history journal is retained.", "parameters": [ { "name": "ws_id", @@ -217,6 +218,16 @@ } } } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } } @@ -552,6 +563,7 @@ "tags": [ "Chat" ], + "description": "Claims the workstream mutation slot, durably truncates the requested tail, then emits clear_ui. Concurrent sends are ordered after the cut; a storage failure returns 503 without changing live history.", "parameters": [ { "name": "ws_id", @@ -602,6 +614,16 @@ } } } + }, + "503": { + "description": "Error 503", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } } @@ -613,6 +635,7 @@ "tags": [ "Chat" ], + "description": "Uses one workstream worker claim for the durable truncation and the replacement generation, so another send cannot enter between them. A storage failure returns 503 without changing live history.", "parameters": [ { "name": "ws_id", @@ -653,6 +676,16 @@ } } } + }, + "503": { + "description": "Error 503", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } } @@ -664,7 +697,7 @@ "tags": [ "Streaming" ], - "description": "Opens a Server-Sent Events stream scoped to a single workstream. Returns text/event-stream. See API reference for event types.", + "description": "Opens a Server-Sent Events stream scoped to a single workstream. After rendering REST history, pass its opaque handoff_token once as ?history_token=; it names the exact accepted conversation-row prefix used for that render. A history_resync event closes this stream and requires a fresh history read; numeric event replay is not a substitute. Native Last-Event-ID reconnects take priority. Pass ?user_turn=1 to opt into typed accepted-user events; otherwise those rows become a backward-compatible strong-repair frame. Pass ?tool_turn=1 to receive the final accepted tool row as a typed tool_result with accepted=true; without it, accepted tool rows use the same pre-row strong-repair projection. Returns text/event-stream. See API reference for event types.", "parameters": [ { "name": "ws_id", @@ -673,6 +706,42 @@ "schema": { "type": "string" } + }, + { + "name": "last_event_id", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Numeric per-workstream event cursor for manual reconnects." + }, + { + "name": "history_token", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Opaque one-shot token naming the accepted prefix rendered from REST history." + }, + { + "name": "user_turn", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Set to 1 to receive typed user_turn events instead of history-repair frames." + }, + { + "name": "tool_turn", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Set to 1 to receive final accepted tool_result projections." } ], "responses": { @@ -972,7 +1041,7 @@ "tags": [ "Workstreams" ], - "description": "Returns the tail of the conversation in OpenAI-like message format. Persisted-but-not-loaded workstreams (closed / evicted) serve history without rehydrating. Lifted from the coord-only surface in the Stage 2 history/detail verb lift \u2014 interactive previously only exposed history through the SSE replay on ``/events``.", + "description": "Returns the tail of the conversation in OpenAI-like message format. Persisted-but-not-loaded workstreams (closed / evicted) are rehydrated before history is served so every successful response participates in the REST-to-SSE handoff. Lifted from the coord-only surface in the Stage 2 history/detail verb lift \u2014 interactive previously only exposed history through the SSE replay on ``/events``. Messages are the requested limit-bounded tail of one authoritative total accepted conversation-row prefix: user, assistant, tool, and system rows, including projected compaction checkpoints and cancellation markers. The opaque handoff_token names the exact prefix used for the render and is passed once on initial SSE registration. Admission of a later row changes the token; durable acknowledgement does not. If the durable prefix cannot be loaded, the endpoint returns 503 with `History temporarily unavailable`; that response is not authoritative and supplies no usable handoff token.", "parameters": [ { "name": "ws_id", @@ -2324,6 +2393,22 @@ "title": "Message", "type": "string" }, + "client_send_id": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9_-]+$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Opaque browser correlation token echoed in the accepted `user_turn` event and history row. It is not an idempotency key; repeated sends with the same value remain distinct turns.", + "title": "Client Send Id" + }, "attachment_ids": { "anyOf": [ { @@ -2860,6 +2945,18 @@ ], "default": null, "title": "Project Id" + }, + "persistence_state": { + "default": "healthy", + "description": "Sanitized durable-history status for the loaded workstream: healthy, pending its first save, retrying automatically, or blocked by a permanent commit conflict. Older servers and unloaded rows default to healthy.", + "enum": [ + "healthy", + "pending", + "retrying", + "conflict" + ], + "title": "Persistence State", + "type": "string" } }, "required": [ @@ -2893,6 +2990,18 @@ "$ref": "#/components/schemas/WorkstreamKind", "default": "interactive" }, + "persistence_state": { + "default": "healthy", + "description": "Sanitized durable-history status: healthy, pending its first save, retrying automatically, or blocked by a permanent commit conflict.", + "enum": [ + "healthy", + "pending", + "retrying", + "conflict" + ], + "title": "Persistence State", + "type": "string" + }, "pending_approval": { "default": false, "description": "True when at least one approval cycle is live (a gate thread parked awaiting an operator approve/deny). Mirrors the same field on ``DashboardWorkstream`` / cluster live projections so a freshly-loaded chat tab can render the inline approval gate from the detail snapshot before SSE replay arrives.", @@ -3032,7 +3141,7 @@ "type": "string" }, "messages": { - "description": "Tail of the workstream's message history, projected to the canonical render shape (``role`` may be ``system`` for operator-context turns; flat tool_calls with verdict / output_assessment; top-level source / attachments / reasoning; derived denied / is_error / pending). Bounded by the ``limit`` query parameter (default 100, max 500).", + "description": "Requested limit-bounded tail of one authoritative total accepted conversation-row prefix, projected to the canonical render shape. Roles include ``user``, ``assistant``, ``tool``, and ``system``; compaction checkpoints project as ``role=system, source=compaction`` and cancellation-generated assistant/tool markers appear when present. The projection also carries flat tool_calls with verdict / output_assessment; top-level source / attachments / reasoning; and derived denied / is_error / pending. Bounded by the ``limit`` query parameter (default 100, max 500).", "items": { "additionalProperties": true, "type": "object" @@ -3052,6 +3161,19 @@ "default": null, "description": "SSE resume cursor (a ``Last-Event-ID`` value). Non-null only when the trailing turn is an executing in-flight tool batch that the live ring buffer can replay: ``messages`` then omits that turn and the client opens its initial SSE with this cursor so the existing delta replay fast-forwards the in-flight turn (tool calls, results, prompts) instead of the lossy synthetic snapshot. Null on every other read \u2014 the client connects fresh.", "title": "Cursor" + }, + "handoff_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Opaque token naming the exact accepted conversation-row prefix used for this render. Present only while the workstream is loaded. A client that renders this response passes the token once as the initial event stream's ``history_token`` query parameter; the server atomically validates it while registering the listener. Admission of a later row changes the token; durable acknowledgement does not. Clients must not inspect, persist, or reuse it for later reconnects.", + "title": "Handoff Token" } }, "required": [ @@ -3209,6 +3331,18 @@ "default": null, "title": "Project Id" }, + "persistence_state": { + "default": "healthy", + "description": "Sanitized durable-history status for this live row. Contains no storage error, commit key, retry count, or conversation content.", + "enum": [ + "healthy", + "pending", + "retrying", + "conflict" + ], + "title": "Persistence State", + "type": "string" + }, "pending_approval_details": { "description": "Inline approval payload for the coordinator children-tree UI: EVERY live approval cycle, oldest first \u2014 parallel task agents gate concurrently, so a workstream can hold several prompts at once. Each entry carries the cycle's items + per-call_id LLM verdict cache so a coord can render approve/deny buttons + judge pill without a separate per-child round-trip; resolve each with its ``cycle_id``. Empty when no approval is pending. Also surfaced (verbatim) on ``GET /v1/api/cluster/ws/live`` via the ``_CLUSTER_WS_LIVE_KEYS`` projection. Replaces 1.6's ``pending_approval_detail`` single-object field (breaking, 1.7).", "items": { diff --git a/sdk/typescript/src/events.ts b/sdk/typescript/src/events.ts index 8f8e7967..f3b5c960 100644 --- a/sdk/typescript/src/events.ts +++ b/sdk/typescript/src/events.ts @@ -1,4 +1,8 @@ -import type { ClusterOverviewResponse, ClusterSnapshotNode } from "./types.js"; +import type { + ClusterOverviewResponse, + ClusterSnapshotNode, + ConversationPersistenceState, +} from "./types.js"; // --------------------------------------------------------------------------- // Server SSE events @@ -35,6 +39,37 @@ export interface HistoryEvent { messages: Array>; } +/** + * The REST history rendered by the caller no longer names the live accepted + * row prefix. Stop this stream, refetch and render history, then open a new + * stream with its cursor and one-shot token. The SDK does not do this + * automatically. + */ +export interface HistoryResyncEvent { + type: "history_resync"; + /** Present on registration-time handoff mismatches; implied by a scoped stream. */ + ws_id?: string; + reason: string; +} + +/** One accepted user row, projected live to every workstream consumer. */ +export interface UserTurnEvent { + type: "user_turn"; + ws_id?: string; + content: string; + attachments?: Array<{ + attachment_id: string; + kind: string; + filename: string; + mime_type: string; + }>; + sender?: string; + source?: string; + /** Optimistic-browser correlation only; not delivery idempotency. */ + client_send_ids: string[]; + _event_id?: number; +} + export interface ThinkingStartEvent { type: "thinking_start"; } @@ -112,6 +147,12 @@ export interface ToolResultEvent { name: string; output: string; is_error?: boolean; + preview?: Record; + /** True only for the final guarded row accepted into conversation history. */ + accepted?: boolean; + effect_status?: string; + /** Monotonic accepted-row identity; present for projection-capable clients. */ + _event_id?: number; } export interface ToolOutputChunkEvent { @@ -210,6 +251,8 @@ export interface WsStateEvent { context_ratio: number; activity: string; activity_state: string; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; /** Full assistant response text — populated on idle transitions only. */ content?: string; } @@ -237,6 +280,8 @@ export interface WsClosedEvent { export type ServerEvent = | ConnectedEvent | HistoryEvent + | HistoryResyncEvent + | UserTurnEvent | ThinkingStartEvent | ThinkingStopEvent | ContentEvent @@ -284,6 +329,8 @@ export interface ClusterStateEvent { context_ratio: number; activity: string; activity_state: string; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; } export interface ClusterWsCreatedEvent { @@ -291,6 +338,8 @@ export interface ClusterWsCreatedEvent { ws_id: string; node_id: string; name: string; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; } export interface ClusterWsClosedEvent { @@ -374,3 +423,13 @@ export function isApprovalResolvedEvent( export function isCancelledEvent(e: ServerEvent): e is CancelledEvent { return e.type === "cancelled"; } + +export function isHistoryResyncEvent( + e: ServerEvent, +): e is HistoryResyncEvent { + return e.type === "history_resync"; +} + +export function isUserTurnEvent(e: ServerEvent): e is UserTurnEvent { + return e.type === "user_turn"; +} diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 5535c7a5..863231c0 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -30,6 +30,8 @@ export type { ClusterEvent, ConnectedEvent, HistoryEvent, + HistoryResyncEvent, + UserTurnEvent, ThinkingStartEvent, ThinkingStopEvent, ContentEvent, @@ -71,10 +73,13 @@ export { isApproveRequestEvent, isApprovalResolvedEvent, isCancelledEvent, + isHistoryResyncEvent, + isUserTurnEvent, } from "./events.js"; // Request/response types export type { + ConversationPersistenceState, SendRequest, SendResponse, ApproveRequest, @@ -87,6 +92,8 @@ export type { CloseWorkstreamRequest, WorkstreamInfo, ListWorkstreamsResponse, + WorkstreamHistoryResponse, + StreamEventsOptions, DashboardWorkstream, DashboardAggregate, DashboardResponse, diff --git a/sdk/typescript/src/server.ts b/sdk/typescript/src/server.ts index 564b4491..0edfa646 100644 --- a/sdk/typescript/src/server.ts +++ b/sdk/typescript/src/server.ts @@ -25,8 +25,10 @@ import type { SendResponse, SkillSummary, StatusResponse, + StreamEventsOptions, TurnResult, UploadAttachmentResponse, + WorkstreamHistoryResponse, } from "./types.js"; function generateWsId(): string { @@ -113,12 +115,15 @@ export class TurnstoneServer extends BaseClient { async send( message: string, wsId: string, - opts?: { attachmentIds?: string[] }, + opts?: { attachmentIds?: string[]; clientSendId?: string }, ): Promise { const body: Record = { message }; if (opts?.attachmentIds !== undefined) { body.attachment_ids = opts.attachmentIds; } + if (opts?.clientSendId !== undefined) { + body.client_send_id = opts.clientSendId; + } return this.request( "POST", `/v1/api/workstreams/${encodeURIComponent(wsId)}/send`, @@ -231,11 +236,45 @@ export class TurnstoneServer extends BaseClient { ); } + // -- History --------------------------------------------------------------- + + /** + * Return the requested tail of the authoritative total accepted row prefix. + * A 503 is non-authoritative and must not replace an existing transcript. + */ + async getHistory( + wsId: string, + opts?: { limit?: number }, + ): Promise { + return this.request( + "GET", + `/v1/api/workstreams/${encodeURIComponent(wsId)}/history`, + { params: { limit: opts?.limit ?? 100 } }, + ); + } + // -- Streaming ------------------------------------------------------------ - async *streamEvents(wsId: string): AsyncIterableIterator { + /** + * Open one caller-managed event stream. Pass history hints only after fully + * rendering the corresponding `getHistory()` response. On `history_resync`, + * stop this iterator, refetch and render history, then open a new stream with + * the new hints. No automatic reconnect or transcript repair is performed. + */ + async *streamEvents( + wsId: string, + opts?: StreamEventsOptions, + ): AsyncIterableIterator { + const params: Record = { user_turn: 1 }; + if (opts?.lastEventId !== undefined) { + params.last_event_id = opts.lastEventId; + } + if (opts?.historyToken) { + params.history_token = opts.historyToken; + } yield* this.streamSSE( `/v1/api/workstreams/${encodeURIComponent(wsId)}/events`, + params, ); } @@ -277,7 +316,7 @@ export class TurnstoneServer extends BaseClient { // Start consuming the per-workstream SSE stream first const events = this.streamSSE( `/v1/api/workstreams/${encodeURIComponent(wsId)}/events`, - undefined, + { user_turn: 1 }, controller.signal, ); diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts index 0a93a9d7..e718f5b7 100644 --- a/sdk/typescript/src/types.ts +++ b/sdk/typescript/src/types.ts @@ -2,6 +2,13 @@ // Shared types // --------------------------------------------------------------------------- +/** Sanitized operator-visible state of accepted conversation persistence. */ +export type ConversationPersistenceState = + | "healthy" + | "pending" + | "retrying" + | "conflict"; + export interface ErrorResponse { error: string; } @@ -56,6 +63,11 @@ export interface SendRequest { * workstream are auto-consumed; an empty list disables auto-consume. */ attachment_ids?: string[]; + /** + * Opaque optimistic-send correlation echoed by user_turn/history. + * Reusing it does not collapse or deduplicate accepted turns. + */ + client_send_id?: string; } export interface SendResponse { @@ -225,6 +237,8 @@ export interface WorkstreamInfo { parent_ws_id: string | null; user_id: string; project_id: string | null; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; } export interface ListWorkstreamsResponse { @@ -240,14 +254,33 @@ export interface WorkstreamDetailResponse { state: string; user_id: string; kind: string; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; } export interface WorkstreamHistoryResponse { ws_id: string; - // Tail of the workstream's reconstructed message history - // (provider-fidelity OpenAI-like shape). Bounded by the ?limit= - // query param (default 100, max 500). + /** + * Requested limit-bounded tail of the authoritative total accepted + * conversation-row prefix. + * Roles include user, assistant, tool, and system; projected compaction and + * cancellation markers participate in the same prefix. + */ messages: Record[]; + /** Initial event-ring cursor returned by the history projection, if needed. */ + cursor: number | null; + /** + * Opaque one-shot token naming the exact live prefix used for this render. + * Null for a workstream that is not currently loaded. + */ + handoff_token: string | null; +} + +export interface StreamEventsOptions { + /** Initial event-ring cursor, normally copied from `getHistory()`. */ + lastEventId?: number; + /** One-shot live-prefix token, copied only from the history just rendered. */ + historyToken?: string; } export interface DashboardWorkstream { @@ -264,6 +297,8 @@ export interface DashboardWorkstream { node?: string; model?: string; model_alias?: string; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; } export interface DashboardAggregate { @@ -530,6 +565,8 @@ export interface ClusterWorkstreamInfo { activity?: string; activity_state?: string; tool_calls?: number; + /** Defaults to `healthy` when omitted by an older node. */ + persistence_state?: ConversationPersistenceState; } export interface ClusterWorkstreamsResponse { diff --git a/sdk/typescript/tests/events.test.ts b/sdk/typescript/tests/events.test.ts index 091fac84..2e75816f 100644 --- a/sdk/typescript/tests/events.test.ts +++ b/sdk/typescript/tests/events.test.ts @@ -8,6 +8,8 @@ import { isApproveRequestEvent, isApprovalResolvedEvent, isReasoningEvent, + isHistoryResyncEvent, + isUserTurnEvent, } from "../src/events.js"; import type { ServerEvent } from "../src/events.js"; @@ -44,6 +46,26 @@ describe("event type guards", () => { expect(isToolResultEvent(e)).toBe(true); }); + it("carries accepted tool projection metadata", () => { + const e: ServerEvent = { + type: "tool_result", + call_id: "c-final", + name: "open_preview", + output: "guarded\nscalar", + is_error: true, + preview: { kind: "html", attachment_id: "preview-1" }, + accepted: true, + effect_status: "unknown", + _event_id: 42, + }; + expect(isToolResultEvent(e)).toBe(true); + if (!isToolResultEvent(e)) throw new Error("tool result type guard failed"); + expect(e.accepted).toBe(true); + expect(e.preview).toEqual({ kind: "html", attachment_id: "preview-1" }); + expect(e.effect_status).toBe("unknown"); + expect(e._event_id).toBe(42); + }); + it("isWsStateEvent", () => { const e: ServerEvent = { type: "ws_state", @@ -53,6 +75,7 @@ describe("event type guards", () => { context_ratio: 0, activity: "", activity_state: "", + persistence_state: "retrying", }; expect(isWsStateEvent(e)).toBe(true); }); @@ -70,4 +93,26 @@ describe("event type guards", () => { }; expect(isApprovalResolvedEvent(e)).toBe(true); }); + + it("isHistoryResyncEvent", () => { + const e: ServerEvent = { + type: "history_resync", + ws_id: "ws1", + reason: "handoff_mismatch", + }; + expect(isHistoryResyncEvent(e)).toBe(true); + expect(isContentEvent(e)).toBe(false); + }); + + it("isUserTurnEvent", () => { + const e: ServerEvent = { + type: "user_turn", + content: "hello", + sender: "user-1", + client_send_ids: ["browser-send"], + _event_id: 17, + }; + expect(isUserTurnEvent(e)).toBe(true); + expect(isContentEvent(e)).toBe(false); + }); }); diff --git a/sdk/typescript/tests/server.test.ts b/sdk/typescript/tests/server.test.ts index 2eec064e..abba8388 100644 --- a/sdk/typescript/tests/server.test.ts +++ b/sdk/typescript/tests/server.test.ts @@ -88,6 +88,21 @@ describe("TurnstoneServer", () => { expect(JSON.parse(init.body)).toEqual({ message: "Hello" }); }); + it("send threads the optional browser correlation token", async () => { + const fetchFn = mockFetch({ status: "ok" }); + const client = new TurnstoneServer({ + baseUrl: "http://test", + fetch: fetchFn, + }); + await client.send("Hello", "ws1", { clientSendId: "browser-send_1" }); + + const [, init] = (fetchFn as ReturnType).mock.calls[0]; + expect(JSON.parse(init.body)).toEqual({ + message: "Hello", + client_send_id: "browser-send_1", + }); + }); + it("approve selects a cycle without duplicating ws_id in the body", async () => { const fetchFn = mockFetch({ status: "ok", cycle_id: "cycle-1" }); const client = new TurnstoneServer({ @@ -127,6 +142,57 @@ describe("TurnstoneServer", () => { expect(response.dropped).toEqual({ tool_calls: ["call-1"] }); }); + it("getHistory returns the cursor and one-shot handoff token", async () => { + const fetchFn = mockFetch({ + ws_id: "ws1", + messages: [{ role: "system", source: "compaction", content: "summary" }], + cursor: 0, + handoff_token: "epoch.7", + }); + const client = new TurnstoneServer({ + baseUrl: "http://test", + fetch: fetchFn, + }); + + const history = await client.getHistory("ws1", { limit: 42 }); + + expect(history.cursor).toBe(0); + expect(history.handoff_token).toBe("epoch.7"); + const [url] = (fetchFn as ReturnType).mock.calls[0]; + expect(url).toBe("http://test/v1/api/workstreams/ws1/history?limit=42"); + }); + + it("streamEvents forwards caller-managed initial history hints", async () => { + const fetchFn = vi + .fn() + .mockResolvedValue( + new Response( + 'data: {"type":"history_resync","ws_id":"ws1","reason":"handoff_mismatch"}\n\n', + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ); + const client = new TurnstoneServer({ + baseUrl: "http://test", + fetch: fetchFn, + }); + + const events = []; + for await (const event of client.streamEvents("ws1", { + lastEventId: 0, + historyToken: "epoch.7", + })) { + events.push(event); + } + + expect(events).toEqual([ + { type: "history_resync", ws_id: "ws1", reason: "handoff_mismatch" }, + ]); + const [url] = (fetchFn as ReturnType).mock.calls[0]; + expect(url).toBe( + "http://test/v1/api/workstreams/ws1/events?user_turn=1&last_event_id=0&history_token=epoch.7", + ); + }); + it("injects auth header when token provided", async () => { const fetchFn = mockFetch({ workstreams: [] }); const client = new TurnstoneServer({ diff --git a/tests/_js_harness_helpers.py b/tests/_js_harness_helpers.py index 3fe8d191..1e834481 100644 --- a/tests/_js_harness_helpers.py +++ b/tests/_js_harness_helpers.py @@ -43,3 +43,138 @@ def demodulize(path: Path) -> str: ) src = re.sub(r"^export\s*\{[^}]*\};\s*$", "", src, flags=re.M) return src + + +def slice_braced_block(source: str, anchor: int) -> str | None: + """Slice the ``{ … }`` block starting at/just after ``anchor``. + + THE brace walker every JS harness suite shares (the comment-AND- + string-aware superset of the per-suite predecessors, which disagreed + on comment handling and window bounds — the same source + reorganization could pass one suite's structural pin while breaking + the other's with a slice-dependent failure). Comment awareness makes + it correct on raw AND pre-stripped input alike. Returns ``None`` + when no ``{`` opens within 200 chars of ``anchor`` (a missing brace + must not silently slice some later unrelated block) or the block is + unterminated. + """ + start = source.find("{", anchor) + if start == -1 or start - anchor > 200: + return None + depth = 0 + quote = "" + escaped = False + line_comment = False + block_comment = False + i = start + while i < len(source): + ch = source[i] + nxt = source[i + 1] if i + 1 < len(source) else "" + if line_comment: + if ch == "\n": + line_comment = False + elif block_comment: + if ch == "*" and nxt == "/": + block_comment = False + i += 1 + elif quote: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == quote: + quote = "" + elif ch == "/" and nxt == "/": + line_comment = True + i += 1 + elif ch == "/" and nxt == "*": + block_comment = True + i += 1 + elif ch in {'"', "'", "`"}: + quote = ch + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return source[start : i + 1] + i += 1 + return None + + +def extract_braced(source: str, signature: str) -> str: + """Extract one JS function/method (signature included) — raising form. + + ``signature`` must end at its opening ``{``. The loud sibling of + :func:`slice_braced_block` for suites that treat a missing or + unterminated function as a hard failure rather than a skip. + """ + start = source.index(signature) + brace = start + len(signature) - 1 + if source[brace] != "{": + raise AssertionError(f"signature does not end at an opening brace: {signature}") + block = slice_braced_block(source, brace) + if block is None: + raise AssertionError(f"unterminated JavaScript function: {signature}") + return source[start:brace] + block + + +def strip_js_comments(source: str) -> str: + """Strip ``//`` and ``/* */`` comments for source-pattern assertions — + the single implementation every JS harness suite shares. + + STRING-AWARE and OFFSET-PRESERVING (comments become spaces, byte + length identical): a ``//`` inside a string literal (``"https://…"``) + is content, not a comment — a string-blind scanner truncates the rest + of the line, and pattern pins then silently assert against corrupted + text (a ``not in`` guard passes vacuously after the pattern it + polices was reintroduced). Length preservation keeps downstream + offset math (brace walkers, ``.index`` comparisons) valid. This is + the strict superset of every per-suite predecessor, hoisted so the + suites cannot diverge again. + + Limitation — regex literals (``/pattern/flags``) are not detected: a + ``//`` inside one would be misread as a line comment. Safe for + every region currently scanned; extend the tracker before scanning a + region with regex literals. + """ + out: list[str] = [] + n = len(source) + i = 0 + in_str: str | None = None + while i < n: + ch = source[i] + if in_str: + out.append(ch) + if ch == "\\" and i + 1 < n: + out.append(source[i + 1]) + i += 2 + continue + if ch == in_str: + in_str = None + i += 1 + continue + # Line comment: replace with spaces up to newline (preserve + # length so downstream offset math still works). + if ch == "/" and i + 1 < n and source[i + 1] == "/": + j = source.find("\n", i) + if j == -1: + j = n + out.append(" " * (j - i)) + i = j + continue + # Block comment: replace with spaces up to closing */. + if ch == "/" and i + 1 < n and source[i + 1] == "*": + j = source.find("*/", i + 2) + if j == -1: + out.append(" " * (n - i)) + i = n + continue + out.append(" " * (j + 2 - i)) + i = j + 2 + continue + if ch in ('"', "'", "`"): + in_str = ch + out.append(ch) + i += 1 + return "".join(out) diff --git a/tests/_sse_recovery_helpers.py b/tests/_sse_recovery_helpers.py index ce39717d..e8c9bda1 100644 --- a/tests/_sse_recovery_helpers.py +++ b/tests/_sse_recovery_helpers.py @@ -134,9 +134,9 @@ class BrowserlikeSSEClient: # -- connection lifecycle ------------------------------------------------ def _events_path(self, cursor: str | None) -> str: - path = f"/v1/api/workstreams/{self._ws_id}/events" + path = f"/v1/api/workstreams/{self._ws_id}/events?user_turn=1&tool_turn=1" if cursor is not None: - path += f"?last_event_id={cursor}" + path += f"&last_event_id={cursor}" return path def connect(self, *, native: bool = False, rcvbuf: int | None = None) -> None: diff --git a/tests/_sse_recovery_server.py b/tests/_sse_recovery_server.py index 1fe5c029..b58949a3 100644 --- a/tests/_sse_recovery_server.py +++ b/tests/_sse_recovery_server.py @@ -249,6 +249,7 @@ class RecoveryServer: # reconnect actually reached the reborn node's real endpoint. self.global_events_requests = 0 self._history_fail_remaining = 0 + self._history_tokenless_remaining = 0 self._history_delay_ms = 0 # A thin pure-ASGI fault layer wrapping the REAL app (the production # app itself is untouched): count + optionally delay/fail @@ -281,6 +282,26 @@ class RecoveryServer: ) await send({"type": "http.response.body", "body": b'{"error": "injected"}'}) return + if self._history_tokenless_remaining > 0: + # Old/malformed server simulation for the strong + # handoff-repair latch. A 200 without handoff_token is + # not proof and must never authorize EventSource. + self._history_tokenless_remaining -= 1 + self.history_ok += 1 + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": (b'{"ws_id":"compat","messages":[],"cursor":null}'), + } + ) + return # Successful-RESPONSE counter, distinct from the arrival # bump above. A scenario asserting that a render was @@ -459,6 +480,85 @@ class RecoveryServer: result: int | None = get_storage().get_max_event_id(ws_id) return result + def emit_idle_edge(self, ws_id: str) -> int: + """Publish one real per-workstream ``state_change: idle`` event. + + Recovery scenarios use this test-server pulse when they need an + organic-settle-equivalent edge without admitting another user row. + A normal ``/send`` is not a neutral trigger: it publishes a live + ``user_turn``, starts model work, and changes the transcript/counts + these scenarios use to isolate the stale-history backstop. + """ + ws = self._manager.get(ws_id) + ui = ws.ui if ws is not None else None + if not isinstance(ui, SessionUIBase): + raise AssertionError(f"emit_idle_edge: ws {ws_id} has no session UI") + before = ui._event_id + ui.on_state_change("idle") + if ui._event_id != before + 1: + raise AssertionError( + f"emit_idle_edge: expected one event after {before}, got {ui._event_id}" + ) + return ui._event_id + + def emit_history_resync(self, ws_id: str, reason: str = "recovery_probe") -> int: + """Publish the real strong repair frame through the ordered UI lane.""" + + ws = self._manager.get(ws_id) + ui = ws.ui if ws is not None else None + if not isinstance(ui, SessionUIBase): + raise AssertionError(f"emit_history_resync: ws {ws_id} has no session UI") + before = ui._event_id + ui.on_history_resync(reason) + if ui._event_id != before + 1: + raise AssertionError( + f"emit_history_resync: expected one event after {before}, got {ui._event_id}" + ) + return ui._event_id + + def emit_tool_pending(self, ws_id: str, call_id: str) -> int: + """Publish a live ``tool_pending`` phase without persisting a turn. + + This drives the coordinator's event-owned ``liveToolCalls`` gate in + isolation. ``on_agent_step`` is the production hook that emits this + exact envelope; the synthetic item is intentionally not added to + history, so a later authoritative repaint must remove its DOM shell. + """ + ws = self._manager.get(ws_id) + ui = ws.ui if ws is not None else None + if not isinstance(ui, SessionUIBase): + raise AssertionError(f"emit_tool_pending: ws {ws_id} has no session UI") + before = ui._event_id + ui.on_agent_step( + "", + { + "call_id": call_id, + "func_name": "recovery_probe", + "approval_label": "recovery probe", + "header": "recovery render-gate probe", + "needs_approval": False, + }, + ) + if ui._event_id != before + 1: + raise AssertionError( + f"emit_tool_pending: expected one event after {before}, got {ui._event_id}" + ) + return ui._event_id + + def emit_tool_result(self, ws_id: str, call_id: str) -> int: + """Resolve a tool pulse emitted by :meth:`emit_tool_pending`.""" + ws = self._manager.get(ws_id) + ui = ws.ui if ws is not None else None + if not isinstance(ui, SessionUIBase): + raise AssertionError(f"emit_tool_result: ws {ws_id} has no session UI") + before = ui._event_id + ui.on_tool_result(call_id, "recovery_probe", "probe complete") + if ui._event_id != before + 1: + raise AssertionError( + f"emit_tool_result: expected one event after {before}, got {ui._event_id}" + ) + return ui._event_id + def fetch_history(self, ws_id: str) -> dict[str, Any]: r = self._http.get( f"/v1/api/workstreams/{ws_id}/history", @@ -501,6 +601,11 @@ class RecoveryServer: failed refetch the #890 guard-before-wipe must survive.""" self._history_fail_remaining = count + def tokenless_history(self, count: int) -> None: + """Make the next history responses 200 without a handoff proof.""" + + self._history_tokenless_remaining = count + def delay_history(self, ms: int) -> None: """Hold each ``GET …/history`` ``ms`` ms before forwarding (0 clears). Opens the clear_ui-refetch quiesce window that the row diff --git a/tests/_storage_fakes.py b/tests/_storage_fakes.py new file mode 100644 index 00000000..4333776e --- /dev/null +++ b/tests/_storage_fakes.py @@ -0,0 +1,125 @@ +"""Scripted-PostgreSQL fakes shared by the storage race-test modules. + +One implementation of the scripted connection/result pair and the keyed-save +three-way dispatch, so a backend statement-sequence or signature change is +updated once. The two hand-rolled twins had already diverged before the +round-4 review folded them here: the truncation copy grew a ``SET LOCAL`` +arm and ``fetchall``/``scalar`` the prune copy lacked. +""" + +from __future__ import annotations + +from typing import Any + +from turnstone.core.storage import AttachmentWrite + + +def make_attachment( + attachment_id: str, + content: bytes, + *, + filename: str | None = None, + mime_type: str = "text/plain", + kind: str = "text", +) -> AttachmentWrite: + return AttachmentWrite( + attachment_id=attachment_id, + filename=filename or f"{attachment_id[0]}.txt", + mime_type=mime_type, + size_bytes=len(content), + kind=kind, + content=content, + ) + + +def save_keyed( + backend: Any, + ws_id: str, + kind: str, + *, + content: str, + commit_key: str, + attachments: list[AttachmentWrite] | None = None, + tool_content: str | None = None, + tool_name: str = "read_file", + tool_call_id: str = "call-keyed", +) -> int: + """Three-way plain/user/tool keyed-save dispatch. + + The per-module literals (content, commit keys, attachment multiplicity) + stay at the call sites — this owns only the method dispatch, so a + signature change on the three save entry points is threaded once. + """ + if kind == "plain": + return int(backend.save_message(ws_id, "assistant", content, commit_key=commit_key)) + if kind == "user": + return int( + backend.save_user_message_with_attachments( + ws_id, + content, + attachments or [], + commit_key=commit_key, + ) + ) + return int( + backend.save_tool_message_with_attachments( + ws_id, + tool_content if tool_content is not None else content, + tool_name, + tool_call_id, + attachments or [], + commit_key=commit_key, + ) + ) + + +class ScriptedPostgresResult: + def __init__( + self, + *, + row: Any | None = None, + rows: list[Any] | None = None, + scalar_value: Any | None = None, + ) -> None: + self._row = row + self._rows = rows or [] + self._scalar_value = scalar_value + + def fetchone(self) -> Any | None: + return self._row + + def fetchall(self) -> list[Any]: + return self._rows + + def scalar(self) -> Any | None: + return self._scalar_value + + def scalar_one_or_none(self) -> Any | None: + return self._scalar_value + + +class ScriptedPostgresConnection: + def __init__(self, results: list[ScriptedPostgresResult]) -> None: + self._results = results + self.statements: list[Any] = [] + self.commits = 0 + self.rollbacks = 0 + + def execute(self, statement: Any, *_args: Any, **_kwargs: Any) -> ScriptedPostgresResult: + self.statements.append(statement) + # Session-scoped tuning (the truncation lock_timeout bound) is not part + # of the scripted result sequence; record it and return an empty result. + if str(statement).startswith("SET LOCAL "): + return ScriptedPostgresResult() + if not self._results: + raise AssertionError("unexpected PostgreSQL statement") + return self._results.pop(0) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + def assert_consumed(self) -> None: + assert not self._results, f"unconsumed scripted results: {len(self._results)}" diff --git a/tests/test_app_js.py b/tests/test_app_js.py index 1cdfb00b..36a710e8 100644 --- a/tests/test_app_js.py +++ b/tests/test_app_js.py @@ -17,6 +17,9 @@ from pathlib import Path import pytest +from tests._js_harness_helpers import slice_braced_block +from tests._js_harness_helpers import strip_js_comments as _strip_js_comments + _APP_JS = Path(__file__).resolve().parent.parent / "turnstone/ui/static/app.js" _INTERACTIVE_JS = Path(__file__).resolve().parent.parent / "turnstone/shared_static/interactive.js" _MCP_ERROR_JS = Path(__file__).resolve().parent.parent / "turnstone/shared_static/mcp_error.js" @@ -44,6 +47,71 @@ def _pane_method_offset(body: str, name: str) -> int: return m.start() +def test_close_workstream_maps_unresolved_history_to_plain_retry_copy() -> None: + body = _APP_JS.read_text(encoding="utf-8") + start = body.index("function closeWorkstream(wsId)") + end = body.index("// 10. Dashboard", start) + close = body[start:end] + + assert "result.status === 409" in close + assert ( + "Conversation history is still being saved. Try ending the session again shortly." in close + ) + assert "delete workstreams[wsId]" in close, "successful close behavior must remain intact" + + +@pytest.mark.parametrize("bundle", [_APP_JS, _CONSOLE_APP_JS], ids=["node", "console"]) +def test_dashboard_persistence_badges_render_sanitized_operator_states(bundle: Path) -> None: + """Both dashboards render the same three non-healthy journal states.""" + body = bundle.read_text(encoding="utf-8") + display_anchor = body.index("const PERSISTENCE_DISPLAY =") + display_body = _slice_balanced_body(body, display_anchor) + helper_body = _slice_function_body(body, "appendPersistenceStatus") + assert display_body is not None + assert helper_body is not None + + script = f""" +const PERSISTENCE_DISPLAY = {display_body}; +const document = {{ + createElement: function (tag) {{ + return {{ + tag: tag, dataset: {{}}, attrs: {{}}, className: "", textContent: "", title: "", + setAttribute: function (name, value) {{ this.attrs[name] = value; }}, + }}; + }}, +}}; +function appendPersistenceStatus(container, ws) {helper_body} +function probe(state) {{ + const container = {{ children: [], appendChild: function (el) {{ this.children.push(el); }} }}; + appendPersistenceStatus(container, {{ persistence_state: state }}); + return container.children[0] || null; +}} +console.log(JSON.stringify({{ + pending: probe("pending"), + retrying: probe("retrying"), + conflict: probe("conflict"), + healthy: probe("healthy"), +}})); +""" + try: + proc = subprocess.run( + ["node", "-e", script], + capture_output=True, + text=True, + timeout=15, + ) + except FileNotFoundError: + pytest.skip("node binary not available on PATH") + assert proc.returncode == 0, proc.stderr + rendered = json.loads(proc.stdout) + assert rendered["pending"]["textContent"] == "History save pending" + assert rendered["retrying"]["textContent"] == "History save retrying" + assert rendered["conflict"]["textContent"] == "History save blocked" + assert rendered["conflict"]["dataset"]["state"] == "conflict" + assert "Operator intervention is required" in rendered["conflict"]["title"] + assert rendered["healthy"] is None + + def test_switch_tab_opens_an_interactive_pane() -> None: """In the L-shell ``switchTab`` is a thin shim onto the PaneManager: it opens/focuses the session as an interactive pane. The split-pane @@ -258,6 +326,101 @@ def test_refetch_history_seeds_resume_cursor_only_on_initial_connect() -> None: ) +def test_initial_history_handoff_token_is_one_shot_and_resyncs_on_mismatch() -> None: + """The opaque /history handoff belongs only to the next SSE bootstrap. + + It must survive a hidden-tab deferral, compose with a valid cursor of 0, + and be consumed only after an EventSource is constructed. A server-side + revision mismatch takes the full REST-history path; it must never try to + heal a missing committed row with numeric ring replay. + """ + body = _INTERACTIVE_JS.read_text(encoding="utf-8") + start = body.index(" connectSSE(wsId) {") + end = body.index(" _onVisibilityChange() {", start) + connect = body[start:end] + + hidden = connect.index("if (document.hidden)") + capability = connect.index('"user_turn=1"') + handoff_query = connect.index('"history_token="') + construct = connect.index("new EventSource(evtUrl)") + consume = connect.index("this._historyHandoffToken = null;", construct) + assert capability < hidden < handoff_query < construct < consume, ( + "the bootstrap token must survive hidden-tab deferral and be consumed " + "only by a successfully constructed EventSource" + ) + assert 'evtUrl += "?last_event_id="' in connect + assert '(evtUrl.includes("?") ? "&" : "?")' in connect, ( + "history_token must compose with ?last_event_id=0 instead of replacing it" + ) + assert connect.count('"user_turn=1"') == 1 + + refetch_start = body.index("async _refetchHistory(") + refetch_end = body.index("_beginReplayQuiesce(", refetch_start) + refetch = body[refetch_start:refetch_end] + assert re.search( + r"if\s*\(seedCursor\)\s*\{\s*this\._historyHandoffToken\s*=\s*" + r"typeof data\.handoff_token === \"string\"", + refetch, + ), "only a seeded history fetch may arm the initial SSE handoff" + + mismatch = body.index('case "history_resync"') + truncated = body.index('case "replay_truncated"', mismatch) + mismatch_case = body[mismatch:truncated] + assert "this._historyRepair.begin(this.wsId);" in mismatch_case + assert "last_event_id" not in mismatch_case + + # Once the server says the rendered history revision is stale, every + # reconnect chokepoint must fail closed until a new response has rendered + # and supplied its proof. A failed fetch schedules one capped retry; it + # must not fall through to a cursorless/tokenless EventSource. The latch, + # budget, backoff, and parked prompt moved into the shared controller + # (history_handoff.createHistoryHandoffRepair) — those are pinned there, + # once; what stays pinned HERE is the pane's use of it. + repair_guard = connect.index("if (this._historyRepair.isRepairing(wsId))") + assert repair_guard < handoff_query < construct + guard_end = connect.index("if (this._historyHandoffToken != null)", repair_guard) + guard = connect[repair_guard:guard_end] + assert "this._historyRepair.schedule();" in guard + assert "return;" in guard + + load_start = body.index("_loadHistoryThenConnect(wsId, manualAttempt = false)") + load_end = body.index("async _refetchHistory(", load_start) + load = body[load_start:load_end] + # Admission before any work, then the budget charge, then exactly one + # handover of the verdict; the non-repair tail keeps its own reconnect. + admit = load.index("this._historyRepair.admitAttempt(manualAttempt)") + start_attempt = load.index("this._historyRepair.startAttempt(manualAttempt,", admit) + settle = load.index("this._historyRepair.settle({", start_attempt) + ordinary = load.index("// Ordinary first paint", settle) + assert admit < start_attempt < settle < ordinary + assert "hasToken: this._historyHandoffToken != null" in load[settle:ordinary] + assert "this.connectSSE(wsId);" not in load[settle:ordinary] + assert "return;" in load[settle:ordinary] + + # Both terminal paths invalidate the in-flight load and kill the timer; + # a late retry/fetch settlement cannot resurrect the pane. + assert body.count("pane._historyRepair.clear();") >= 2 + + # The strong repair attempt has a logical 15s deadline, not merely an + # AbortController timeout: authFetch's Retry-After sleep is not abort-aware + # and old runtimes can lack AbortController entirely. The pane still owns + # this per-attempt bound (the coordinator bounds every /history centrally + # instead), and hands the controller a teardown that expires and settles + # the race so no detached pane waits for the deadline. + assert "Promise.race([" in load + assert "createHistoryHandoffDeadline(" in load + assert "deadlineHandle.promise" in load + assert "HISTORY_HANDOFF_FETCH_TIMEOUT_MS" in load + assert "if (repairAttempt && repairAttempt.expired) return;" in refetch + teardown = load[start_attempt:settle] + assert "deadlineHandle.dispose({ expire: true, resolve: true })" in teardown, ( + "the mid-flight teardown must expire AND settle the race through the " + "module's dispose() — direct state-slot pokes are the drift the " + "shared handle exists to prevent." + ) + assert "repairCtrl.abort()" in teardown + + def test_shared_utils_no_longer_defines_replay_advisories_after_tool() -> None: """Operator context (interjections / guard findings / nudges) no longer rides the tool envelope — it is first-class ``{"role": "system"}`` rows @@ -1174,46 +1337,15 @@ def test_phase8_consent_url_prefix_check_in_click_handler() -> None: # cleanly into a standalone node invocation. -def _slice_balanced_body(body: str, anchor: int) -> str | None: - """Slice ``body`` from ``anchor`` (which must point at or just before - the opening ``{`` of a block) up to and including the matching ``}``. - Tracks brace depth + string state so the slice is robust to comment - growth and arbitrary body reorganisation. Returns ``None`` if the - matching brace isn't found within a reasonable window. - - Used to slice JS handler / function bodies for static assertions - without committing to a fixed character window.""" - n = len(body) - i = body.find("{", anchor) - if i == -1 or i - anchor > 200: - return None - depth = 0 - in_str: str | None = None - start = i - # 12000: connectSSE reached ~7950 chars during the 2026-07 SSE - # recovery campaign (cursor-override + capture-rationale comments); - # the window exists to bound a runaway scan, not to cap legitimate - # method growth — keep it comfortably above the largest real body. - while i < n and i - start < 12000: - ch = body[i] - if in_str: - if ch == "\\" and i + 1 < n: - i += 2 - continue - if ch == in_str: - in_str = None - i += 1 - continue - if ch in ('"', "'", "`"): - in_str = ch - elif ch == "{": - depth += 1 - elif ch == "}": - depth -= 1 - if depth == 0: - return body[start : i + 1] - i += 1 - return None +# ``_slice_balanced_body`` is the shared comment-and-string-aware brace +# walker from tests/_js_harness_helpers (imported at the top of this +# file). Comment awareness is a strict superset of the old string-only +# local: most callers here pre-strip, where the two agree exactly, and a +# few pass raw source (the persistence-badge and beforeunload pins), +# where the shared walker is the more correct of the two — braces inside +# comments no longer inflate its depth count. One implementation means +# a walker fix lands once for every suite. +_slice_balanced_body = slice_braced_block def _slice_listener_body(body: str, event_name: str) -> str | None: @@ -1718,66 +1850,10 @@ def test_dead_sse_defensive_reconnect_registered() -> None: # the contract so a future refactor can't silently regress it. -def _strip_js_comments(src: str) -> str: - """Strip ``//`` and ``/* */`` comments while preserving string - literal contents (``"..."``, ``'...'``, `` `...` ``) and keeping - byte length identical (comments replaced with spaces). - - Limitation — does NOT detect regex literals (``/pattern/flags``). - A ``//`` inside a regex like ``/abc//`` would be misread as the - start of a line comment. Safe today because the regions we scan - (SSE-handler ``onerror`` bodies, ``connectSSE`` / - ``connectGlobalSSE`` function bodies) don't contain regex - literals; if a future caller wants to scan a region with regex - literals, extend the tracker first. - - Motivation: ``_slice_balanced_body`` doesn't skip comments, so an - apostrophe inside a comment (``can't``, ``don't``) opens a fake - string state that swallows braces until the next ``'``. The new - onerror handlers carry these comments routinely; stripping - comments before brace-walking removes the hazard without - re-architecting the existing slice helper. - """ - out: list[str] = [] - n = len(src) - i = 0 - in_str: str | None = None - while i < n: - ch = src[i] - if in_str: - out.append(ch) - if ch == "\\" and i + 1 < n: - out.append(src[i + 1]) - i += 2 - continue - if ch == in_str: - in_str = None - i += 1 - continue - # Line comment: replace with spaces up to newline (preserve - # length so downstream offset math still works). - if ch == "/" and i + 1 < n and src[i + 1] == "/": - j = src.find("\n", i) - if j == -1: - j = n - out.append(" " * (j - i)) - i = j - continue - # Block comment: replace with spaces up to closing */. - if ch == "/" and i + 1 < n and src[i + 1] == "*": - j = src.find("*/", i + 2) - if j == -1: - out.append(" " * (n - i)) - i = n - continue - out.append(" " * (j + 2 - i)) - i = j + 2 - continue - if ch in ('"', "'", "`"): - in_str = ch - out.append(ch) - i += 1 - return "".join(out) +# ``_strip_js_comments`` is the shared string-aware, offset-preserving +# stripper from tests/_js_harness_helpers (imported at the top of this +# file) — one implementation for every suite, so the string-blind / +# offset-destroying per-suite variants cannot diverge again. def _onerror_block(body: str, anchor_substring: str) -> str | None: @@ -1882,6 +1958,26 @@ def test_connectglobalsse_onerror_preserves_native_reconnect() -> None: assert passed, f"connectGlobalSSE.onerror regressed: {reason}" +def test_ws_activity_never_reinserts_a_closed_workstream() -> None: + """A trailing ``ws_activity`` for a closed workstream must not re-create + a skeletal roster entry: its dashboard row can outlive ``ws_closed`` + until the next REST-driven repaint, and a ghost entry both suppresses + the empty-state transition (``showDashboard``) and paints a nameless + rail tab until a full resync. The arm is membership-gated like + ``ws_rename`` — read the entry, mutate it in place only when it exists, + and never assign into the roster map.""" + body = _strip_js_comments(_APP_JS.read_text(encoding="utf-8")) + start = body.index('data.type === "ws_activity"') + end = body.index('data.type === "ws_rename"', start) + arm = body[start:end] + assert "workstreams[data.ws_id] =" not in arm, ( + "ws_activity assigns into the roster map — a trailing event for a " + "closed workstream would re-insert a ghost entry" + ) + assert "const roster = workstreams[data.ws_id];" in arm + assert "if (roster)" in arm + + def test_coord_connectsse_onerror_preserves_native_reconnect() -> None: """Coordinator's connectSSE has the same contract — without the guard the coord's per-ws SSE silently drops events on any blip.""" @@ -2319,7 +2415,11 @@ def test_coord_truncated_resync_is_full_fresh_connect_with_churn_limit() -> None assert "clearTimeout(truncatedResyncTimer)" in teardown.group(1) # (6) the dead-stream flow: teardown first, refs reset, seeded refetch, # guarded .finally reconnect, deferred latch superseded. - flow = re.search(r"function loadHistoryThenReconnect\(\)\s*\{(.*?)\n \}", body, re.S) + flow = re.search( + r"function loadHistoryThenReconnect\(manualAttempt = false\)\s*\{(.*?)\n \}", + body, + re.S, + ) assert flow is not None, "loadHistoryThenReconnect not found" f = flow.group(1) assert f.index("suspendStream();") < f.index("refetchHistory(true)") @@ -2335,8 +2435,13 @@ def test_coord_truncated_resync_is_full_fresh_connect_with_churn_limit() -> None # this pin only keeps the fix from being "simplified" away. assert "lastEventId = null;" in f assert f.index("lastEventId = null;") < f.index("refetchHistory(true)") - assert ".finally(" in f - assert f.index("refetchHistory(true)") < f.index(".finally(") + # The settle rides the outcome-threaded terminal .then (a rendered + # tokenless 200 downgrades to the tokenless bootstrap; failures retry), + # with rejections normalized ahead of it — LOUDLY (round-4 review: a + # bare `.catch(() => undefined)` silently swallowed render throws). + assert 'console.error("history load/render failed"' in f + assert ".then((outcome) => {" in f + assert f.index("refetchHistory(true)") < f.index('console.error("history load/render failed"') assert "if (visHandler) connectSSE();" in f # (10) heal-time sidebar refresh: once, on the successful render only # (record cleared), only on the cursor-SEEDED heal (the cursorless @@ -2917,8 +3022,9 @@ def test_strict_picker_requires_explicit_pick() -> None: def _slice_top_level_fn(body: str, header: str) -> str: """Slice a top-level ``function`` body from ``header`` to the next column-0 ``function`` declaration (or EOF). Unlike - ``_slice_balanced_body`` this has no fixed-size window, so it is safe - for large functions like ``showNewWsModal``. Nested (indented) + ``_slice_balanced_body`` this needs no balanced braces at all, so it + survives a body the walker would refuse (an unterminated block, or a + regex literal the walker misreads as a comment). Nested (indented) ``function () {…}`` expressions never match the ``\\nfunction `` bound, so the slice stops at the next top-level function.""" start = body.index(header) diff --git a/tests/test_attachment_buffer.py b/tests/test_attachment_buffer.py index 4277317e..fec95140 100644 --- a/tests/test_attachment_buffer.py +++ b/tests/test_attachment_buffer.py @@ -68,6 +68,45 @@ def test_discard_is_scope_checked() -> None: assert buf.get(entry.attachment_id, ws_id="ws1", user_id="u1") is None +def test_consume_all_consumes_present_subset_and_reports_missing() -> None: + """Survivors are consumed exactly once even when a sibling is missing. + + Deliberate pin update: the old all-or-nothing contract left every + surviving reference staged when one handle expired, letting the same + uploads be attached again after the turn that owned their bytes already + committed (the double-spend the atomic transfer exists to prevent). + """ + + buf = AttachmentBuffer() + first = _stage(buf, content=b"first") + second = _stage(buf, content=b"second") + _stage(buf, content=b"first", ws="other", user="u1") + missing = hashlib.sha256(b"missing").hexdigest() + + consumed = buf.consume_all( + [first.attachment_id, missing, second.attachment_id], + ws_id="ws1", + user_id="u1", + ) + assert consumed == {first.attachment_id, second.attachment_id} + assert buf.get(first.attachment_id, ws_id="ws1", user_id="u1") is None + assert buf.get(second.attachment_id, ws_id="ws1", user_id="u1") is None + # Scope isolation: another workstream's staging of the same bytes survives. + assert buf.get(first.attachment_id, ws_id="other", user_id="u1") is not None + + # A second consume finds nothing — the ownership reference is one-shot, + # and duplicate handles in one call consume it only once. + assert ( + buf.consume_all( + [first.attachment_id, first.attachment_id, second.attachment_id], + ws_id="ws1", + user_id="u1", + ) + == frozenset() + ) + assert buf.consume_all([], ws_id="ws1", user_id="u1") == frozenset() + + def test_ttl_eviction_on_access() -> None: clock = [0.0] buf = AttachmentBuffer(ttl_seconds=10.0, clock=lambda: clock[0]) diff --git a/tests/test_bash_background_tool.py b/tests/test_bash_background_tool.py index 2c473480..33065e20 100644 --- a/tests/test_bash_background_tool.py +++ b/tests/test_bash_background_tool.py @@ -22,11 +22,18 @@ from tests._proc_helpers import pid_alive as _pid_alive from tests._proc_helpers import poll_until as _wait_until from tests._session_helpers import make_session from turnstone.core.session import _active_shell_owner +from turnstone.core.storage import get_storage @pytest.fixture -def session(): +def session(tmp_db): s = make_session() + get_storage().register_workstream( + s.ws_id, + user_id=s._user_id, + kind=s._kind, + parent_ws_id=s._parent_ws_id, + ) yield s s.close() diff --git a/tests/test_cancel.py b/tests/test_cancel.py index b2ba8d49..16c6e4e3 100644 --- a/tests/test_cancel.py +++ b/tests/test_cancel.py @@ -172,7 +172,14 @@ def _make_session(ui=None, **kwargs): recording NullUI. The defaults live in tests/_session_helpers.make_session — duplicating them here is exactly the drift its docstring warns about.""" - return make_session(ui=ui or NullUI(), **kwargs) + session = make_session(ui=ui or NullUI(), **kwargs) + # Keyed conversation commits refuse orphan writes by design; production's + # manager creates the parent workstream row before constructing a live + # session, so direct-session tests mirror that prerequisite. + from turnstone.core.memory import register_workstream + + register_workstream(session.ws_id, user_id=kwargs.get("user_id")) + return session class _BlockingAgentStream: @@ -207,14 +214,23 @@ class _ObservedRLock: self._watched_thread = thread self.waiting.clear() - def __enter__(self): + # ``acquire``/``release`` (not just the context-manager pair) so this + # wrapper can back a ``threading.Condition``, which binds those two + # methods off the lock it is given. + def acquire(self, *args, **kwargs): if threading.current_thread() is self._watched_thread: self.waiting.set() - self._lock.acquire() + return self._lock.acquire(*args, **kwargs) + + def release(self) -> None: + self._lock.release() + + def __enter__(self): + self.acquire() return self def __exit__(self, exc_type, exc_value, traceback) -> None: - self._lock.release() + self.release() class _GatedRLock: @@ -3946,6 +3962,12 @@ class TestCancelledSendCleanupOwnership: session._system_composed_with_context = True observed_lock = _ObservedRLock() session._generation_lock = observed_lock + # This test replaces the generation lock to observe ownership. The + # production truncation condition is constructed from that same lock, + # so rebuild it on the replacement: leaving the fixture's condition + # bound to the old lock would split the ownership domain and let the + # claimant cross the cleanup transaction. + session._history_truncation_condition = threading.Condition(observed_lock) cleanup_entered = threading.Event() release_cleanup = threading.Event() send_errors: list[BaseException] = [] @@ -4042,7 +4064,9 @@ class TestCancelledSendCleanupOwnership: blocked_cleanup_calls = 0 original_commit = session._commit_for_generation - def blocked_commit(origin_generation, commit, *, allow_cancelled=True): + def blocked_commit(origin_generation, commit, *, allow_cancelled=True, **kwargs): + # Pass through every admission flag (e.g. allow_workstream_gone on + # the cancel finalizer) — the shim only sequences, never narrows. nonlocal blocked_cleanup_calls publish_generations.append(origin_generation) publish_name = getattr(commit, "__name__", "") @@ -4051,6 +4075,7 @@ class TestCancelledSendCleanupOwnership: origin_generation, commit, allow_cancelled=allow_cancelled, + **kwargs, ) blocked_cleanup_calls += 1 publish_entered.set() @@ -4060,6 +4085,7 @@ class TestCancelledSendCleanupOwnership: origin_generation, commit, allow_cancelled=allow_cancelled, + **kwargs, ) def run_cancelled_send(): @@ -4644,6 +4670,26 @@ class TestEffectStatusPersistence: "preview": {"kind": "web"}, } + def test_acting_principal_is_a_sibling_channel_omitted_when_empty(self): + """The audit identity joins the same envelope, never a fifth axis. + + An unattributed lane (wake / internal / CLI) writes NO key rather than + an empty string, so a revocation query reading the column can treat + presence as attribution — the convention the USER row's ``sender`` + already follows. + """ + assert _tool_turn_meta(None, None, acting_principal="") is None + assert json.loads(_tool_turn_meta(None, None, acting_principal="user-alice")) == { + "acting_principal": "user-alice", + } + assert json.loads( + _tool_turn_meta(EffectStatus.UNKNOWN, {"kind": "web"}, acting_principal="user-alice") + ) == { + "effect_status": "unknown", + "preview": {"kind": "web"}, + "acting_principal": "user-alice", + } + def test_reconstruct_routes_tool_effect_status(self): from turnstone.core.storage._utils import reconstruct_turns diff --git a/tests/test_channel_sse.py b/tests/test_channel_sse.py index 13bb62a5..3754076d 100644 --- a/tests/test_channel_sse.py +++ b/tests/test_channel_sse.py @@ -62,9 +62,11 @@ class _FakeConnect: def __init__(self, queue: list[_FakeEventSource]) -> None: self._queue = queue self.call_count = 0 + self.calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] def __call__(self, *args, **kwargs): # noqa: ANN001, ANN204 self.call_count += 1 + self.calls.append((args, kwargs)) if not self._queue: raise asyncio.CancelledError return self._queue.pop(0) @@ -128,6 +130,7 @@ class TestStaleRoute: on_event.assert_not_awaited() # No reconnect after 404. assert fake_connect.call_count == 1 + assert fake_connect.calls[0][1]["params"] == {"user_turn": 1} assert _fast_sleep == [] def test_on_stale_exception_still_exits(self, monkeypatch, _fast_sleep): diff --git a/tests/test_compaction_checkpoint.py b/tests/test_compaction_checkpoint.py index 7bc2db20..8395921e 100644 --- a/tests/test_compaction_checkpoint.py +++ b/tests/test_compaction_checkpoint.py @@ -27,7 +27,8 @@ import pytest from tests._session_helpers import make_session from turnstone.core.session import _SummaryResult -from turnstone.core.trajectory import turns_from_dicts +from turnstone.core.storage._utils import _fork_turn_insert_row +from turnstone.core.trajectory import PROVENANCE_META_KEY, TurnProvenance, turns_from_dicts def _marker_meta(watermark: int | None) -> str | None: @@ -268,16 +269,76 @@ def test_compaction_persists_checkpoint_and_resume_is_bounded(tmp_db, mock_opena assert not any(t.startswith("turn ") for t in texts) # full history NOT reloaded +def test_marker_watermark_read_blip_recovers_on_retry(tmp_db, mock_openai_client): + """Round-3 review pin: a transient watermark-read failure aborts that + persist attempt — the journal classifies the marker row retrying with NO + meta bytes memoized — so the healthy retry re-reads and commits WITH a + watermark. Memoizing the failed read as "absent" would durably commit a + permanently checkpoint-less marker: full-history rehydration and an + immediate re-compaction on every reopen.""" + from unittest.mock import patch + + from turnstone.core.memory import register_workstream, save_message + from turnstone.core.storage._registry import get_storage + + ws = "wsBLIP" + register_workstream(ws, user_id="u1", name="t") + history = [ + {"role": "user" if i % 2 == 0 else "assistant", "content": f"turn {i}"} for i in range(6) + ] + for h in history: + save_message(ws, h["role"], h["content"]) + + sess = make_session(client=mock_openai_client, context_window=10_000, max_tokens=1_000) + sess._ws_id = ws + sess.messages = turns_from_dicts(history) + sess._msg_tokens = [1] * len(history) + + st = get_storage() + real_watermark = st.get_compaction_watermark + reads: list[int] = [] + + def _flaky_watermark(ws_id: str, preserve_tail: int = 0) -> int | None: + reads.append(preserve_tail) + if len(reads) == 1: + raise RuntimeError("transient watermark blip") + return real_watermark(ws_id, preserve_tail) + + from turnstone.core.session import ConversationPersistenceError + + with ( + patch.object( + sess, + "_summarize_blocks", + return_value=_SummaryResult(text="DENSE SUMMARY", producer="summary-producer"), + ), + patch.object(st, "get_compaction_watermark", side_effect=_flaky_watermark), + ): + # The first persist attempt dies at the read and surfaces like any + # initial durability failure; the in-memory compaction stays applied + # and the marker row is retained in the journal, nothing durable yet. + with pytest.raises(ConversationPersistenceError): + sess._compact_messages(auto=False) + assert sess.conversation_persistence_status()["state"] == "retrying" + assert st.get_compaction_checkpoint(ws) is None + sess._reconcile_pending_conversation_commits(_force_retry=True) + + assert sess.conversation_persistence_status()["state"] == "healthy" + checkpoint = st.get_compaction_checkpoint(ws) + assert checkpoint is not None + assert checkpoint == real_watermark(ws, 0) + + def test_compaction_summary_producer_survives_storage_round_trip( storage_backend, mock_openai_client ): - """The final summary producer is durable checkpoint metadata. + """Final summary producer and model provenance are durable metadata. A compaction marker has no provider-native payload, so its producer belongs in the marker's ``summary_producer`` meta field. Checkpoint reconstruction - maps that object to the summary Turn's - ``meta.extra["source_meta"]``. This intentionally pins only the producer; - the broader durable model/config/principal provenance tuple is #964 scope. + maps that object to the summary Turn's ``meta.extra["source_meta"]`` while + retaining the accepted model alias/backend/generation/principal tuple in + the well-known provenance envelope. """ st = storage_backend ws = _register(st, "ws-summary-producer") @@ -291,12 +352,20 @@ def test_compaction_summary_producer_survives_storage_round_trip( sess._ws_id = ws sess.messages = turns_from_dicts(history) sess._msg_tokens = [1] * len(history) + provenance = TurnProvenance( + model_alias="summary-alias", + backend_model_id="summary-kernel", + registry_generation=12, + acting_principal_id="user-alice", + ) with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( sess, "_summarize_blocks", lambda *_args, **_kwargs: _SummaryResult( - text="DENSE SUMMARY", producer="final-summary-producer" + text="DENSE SUMMARY", + producer="final-summary-producer", + provenance=provenance, ), ) assert sess._compact_messages(auto=False) is True @@ -307,11 +376,22 @@ def test_compaction_summary_producer_survives_storage_round_trip( if message.get("_source") == "compaction" ) assert marker["_source_meta"]["summary_producer"] == "final-summary-producer" + assert "_provenance" not in marker + assert "user-alice" not in json.dumps(marker) loaded = st.load_message_turns(ws) assert [turn.text for turn in loaded[:2]] == ["[Conversation summary]", "DENSE SUMMARY"] assert "source_meta" not in loaded[0].meta.extra assert loaded[1].meta.extra["source_meta"]["summary_producer"] == "final-summary-producer" + assert loaded[1].meta.extra[PROVENANCE_META_KEY] == provenance.to_meta() + fork_row, _attachment_ids = _fork_turn_insert_row( + loaded[1], + "compaction-provenance-fork", + "2026-08-09T00:00:00", + ) + fork_meta = json.loads(fork_row["meta"]) + assert fork_meta["summary_producer"] == "final-summary-producer" + assert fork_meta[PROVENANCE_META_KEY] == provenance.to_meta() reopened = make_session(client=mock_openai_client, context_window=10_000, max_tokens=1_000) assert reopened.resume(ws) is True @@ -320,6 +400,7 @@ def test_compaction_summary_producer_survives_storage_round_trip( reopened.messages[1].meta.extra["source_meta"]["summary_producer"] == "final-summary-producer" ) + assert reopened.messages[1].meta.extra[PROVENANCE_META_KEY] == provenance.to_meta() # --------------------------------------------------------------------------- @@ -505,11 +586,12 @@ def test_persist_truncation_uncompacted_matches_plain_tail_delete(tmp_db, mock_o assert st.count_messages(ws) == 3 -def test_persist_truncation_skips_delete_when_count_unavailable(tmp_db, mock_openai_client): - """count_messages==0 (the storage-error sentinel) must NOT delete — a wrong - truncation would lose user history.""" +def test_persist_truncation_propagates_atomic_storage_failure(tmp_db, mock_openai_client): + """The strict backend error must reach the failure-atomic session caller.""" from unittest.mock import patch + import pytest + from turnstone.core.memory import get_storage, register_workstream, save_message ws = "wsCnt" @@ -519,25 +601,98 @@ def test_persist_truncation_skips_delete_when_count_unavailable(tmp_db, mock_ope st = get_storage() sess = make_session(client=mock_openai_client) sess._ws_id = ws - with patch("turnstone.core.session.count_messages", return_value=0): + with ( + patch.object( + st, + "truncate_messages_tail", + side_effect=RuntimeError("injected atomic truncation failure"), + ), + pytest.raises(RuntimeError, match="injected atomic truncation failure"), + ): sess._persist_truncation(2) assert st.count_messages(ws) == 4 # nothing deleted -def test_persist_truncation_skips_delete_when_floor_unavailable(tmp_db, mock_openai_client): - """get_compaction_floor==-1 (the storage-error sentinel) must NOT delete — a 0 - floor on a compacted ws could otherwise drop the marker on an over-deep trim.""" +def test_persist_truncation_zero_is_a_storage_noop(tmp_db, mock_openai_client): + """A no-op plan never opens the strict backend transaction.""" from unittest.mock import patch - from turnstone.core.memory import get_storage, register_workstream, save_message + from turnstone.core.memory import get_storage, register_workstream ws = "wsFloor" register_workstream(ws, user_id="u1", name="t") - for i in range(4): - save_message(ws, "user", f"m{i}") st = get_storage() sess = make_session(client=mock_openai_client) sess._ws_id = ws - with patch("turnstone.core.session.get_compaction_floor", return_value=-1): - sess._persist_truncation(2) - assert st.count_messages(ws) == 4 # nothing deleted + with patch.object(st, "truncate_messages_tail") as truncate: + assert sess._persist_truncation(0) == 0 + truncate.assert_not_called() + + +def test_watermark_reads_inside_the_marker_persist_not_before_the_commit( + tmp_db, mock_openai_client +): + """The boundary is cut in the ordered durable batch, at persist time. + + A pre-commit snapshot can undercount the durable prefix whenever accepted + rows are still pending in the journal when compaction is admitted (they + land, FIFO, before the marker's persist executes). Reading inside the + persist closure names exactly the summarized prefix; memoization keeps a + keyed lost-ACK retry byte-identical. + """ + from unittest.mock import patch + + from turnstone.core.memory import register_workstream, save_message + from turnstone.core.storage._registry import get_storage + + ws = "wsWatermarkOrder" + register_workstream(ws, user_id="u1", name="t") + history = [ + {"role": "user" if i % 2 == 0 else "assistant", "content": f"turn {i}"} for i in range(6) + ] + for h in history: + save_message(ws, h["role"], h["content"]) + + sess = make_session(client=mock_openai_client, context_window=10_000, max_tokens=1_000) + sess._ws_id = ws + sess.messages = turns_from_dicts(history) + sess._msg_tokens = [1] * len(history) + + order: list[str] = [] + real_journal = sess._journal_conversation_row_locked + # The persist closure reads the backend directly (bypassing the memory + # wrapper's error-to-None coercion), so the spy sits on the storage method. + st = get_storage() + real_watermark = st.get_compaction_watermark + + def _journal_spy(**kwargs): + order.append("journal") + return real_journal(**kwargs) + + def _watermark_spy(ws_id, preserve_tail=0): + order.append("watermark") + return real_watermark(ws_id, preserve_tail) + + with ( + patch.object(sess, "_journal_conversation_row_locked", side_effect=_journal_spy), + patch.object(st, "get_compaction_watermark", side_effect=_watermark_spy), + patch.object( + sess, + "_summarize_blocks", + return_value=_SummaryResult(text="DENSE SUMMARY", producer="summary-producer"), + ), + ): + assert sess._compact_messages(auto=False) is True + + assert order == ["journal", "watermark"], order + + # The durable marker's boundary covers every row persisted before it. + from turnstone.core.storage import get_storage + + rows = get_storage().load_messages(ws, repair=False, include_compaction=True) + marker = next(r for r in rows if r.get("_source") == "compaction") + watermark = (marker.get("_source_meta") or {}).get("watermark") + plain_ids = [r["id"] for r in rows if r.get("_source") != "compaction" if "id" in r] + assert isinstance(watermark, int) + if plain_ids: + assert watermark >= max(plain_ids) diff --git a/tests/test_compaction_crossing.py b/tests/test_compaction_crossing.py index 36a6441d..79e33bce 100644 --- a/tests/test_compaction_crossing.py +++ b/tests/test_compaction_crossing.py @@ -33,6 +33,7 @@ import pytest from tests._session_helpers import make_session from turnstone.core.session import COMPACTION_SOURCE, COMPACTION_SUMMARY_LABEL +from turnstone.core.storage import get_storage from turnstone.core.trajectory import turns_from_dicts @@ -42,13 +43,26 @@ def session(tmp_db, mock_openai_client): the summary output reserve is tiny and the carry budget is easy to compute (reserve=100, margin=500, spare=9_400, budget=min(2_500, 9_400)=2_500 tokens → 10_000 chars at the uncalibrated 4.0 chars/token).""" - return make_session( + s = make_session( client=mock_openai_client, context_window=10_000, compact_max_tokens=100, max_tokens=1_000, tool_timeout=10, ) + _register_session_workstream(s) + return s + + +def _register_session_workstream(session): + """Give direct ChatSession fixtures their production parent row.""" + get_storage().register_workstream( + session.ws_id, + user_id=session._user_id, + kind=session._kind, + parent_ws_id=session._parent_ws_id, + ) + return session def _stub_summary(text: str = "DENSE"): @@ -357,7 +371,7 @@ class TestWindDownSpill: spare // 2, so two oversize carries land truncated to the shared budget instead of stacking two solo quarter-window allowances on top of the half-window summary reserve.""" - s = make_session(client=mock_openai_client, tool_timeout=10) + s = _register_session_workstream(make_session(client=mock_openai_client, tool_timeout=10)) per_carry = s._carry_budget_chars(2) ask = "ASK-HEAD " + "a" * (per_carry * 2) + " ASK-TAIL" spill = "PLAN-HEAD " + "b" * (per_carry * 2) + " PLAN-TAIL" @@ -432,16 +446,18 @@ def _coord_client(tasks=None, children=None) -> MagicMock: def _coord_session(mock_openai_client, *, coord_client=..., **kwargs): from turnstone.core.workstream import WorkstreamKind - return make_session( - client=mock_openai_client, - context_window=10_000, - compact_max_tokens=100, - max_tokens=1_000, - tool_timeout=10, - kind=WorkstreamKind.COORDINATOR, - user_id="u1", - coord_client=_coord_client() if coord_client is ... else coord_client, - **kwargs, + return _register_session_workstream( + make_session( + client=mock_openai_client, + context_window=10_000, + compact_max_tokens=100, + max_tokens=1_000, + tool_timeout=10, + kind=WorkstreamKind.COORDINATOR, + user_id="u1", + coord_client=_coord_client() if coord_client is ... else coord_client, + **kwargs, + ) ) @@ -493,8 +509,8 @@ class TestCoordinatorHandles: fails and they must revisit that trade rather than stack both. """ coord = _coord_session(mock_openai_client) - interactive = make_session( - client=mock_openai_client, context_window=10_000, tool_timeout=10 + interactive = _register_session_workstream( + make_session(client=mock_openai_client, context_window=10_000, tool_timeout=10) ) prompts = [] for s in (coord, interactive): @@ -517,12 +533,14 @@ class TestCoordinatorHandles: children, so the reads are skipped entirely — not merely rendered empty — and its summary is what it was before this existed.""" client = _coord_client() - s = make_session( - client=mock_openai_client, - context_window=10_000, - compact_max_tokens=100, - tool_timeout=10, - coord_client=client, # present but irrelevant: kind decides + s = _register_session_workstream( + make_session( + client=mock_openai_client, + context_window=10_000, + compact_max_tokens=100, + tool_timeout=10, + coord_client=client, # present but irrelevant: kind decides + ) ) text = _compact(s) assert "## Handles" not in text @@ -660,7 +678,7 @@ class TestCoordinatorHandles: s = _coord_session(mock_openai_client) s._ws_id = "ws-coord" with ( - patch("turnstone.core.session.get_compaction_watermark", return_value=7), + patch.object(get_storage(), "get_compaction_watermark", return_value=7), patch("turnstone.core.session.save_message") as saved, ): _compact(s) diff --git a/tests/test_console.py b/tests/test_console.py index 4aaabb75..2bbb5a0f 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -387,6 +387,50 @@ class TestCollectorSnapshot: assert event["type"] == "cluster_state" assert event["ws_id"] == "ws1" assert event["state"] == "running" + assert event["persistence_state"] == "healthy" + assert c._nodes["node-a"].workstreams["ws1"]["persistence_state"] == "healthy" + + def test_apply_snapshot_emits_persistence_change_without_state_change(self): + """A journal recovery refresh must reach the cluster UI while idle.""" + c = _make_collector() + c._nodes["node-a"] = NodeSnapshot( + node_id="node-a", + server_url="http://a:8080", + workstreams={ + "ws1": { + "id": "ws1", + "name": "same", + "state": "idle", + "persistence_state": "retrying", + } + }, + ) + q: queue.Queue[dict] = queue.Queue() + c.register_listener(q) + + c._apply_snapshot( + "node-a", + { + "type": "node_snapshot", + "node_id": "node-a", + "workstreams": [ + { + "id": "ws1", + "name": "same", + "state": "idle", + "persistence_state": "healthy", + } + ], + "health": {}, + "aggregate": {}, + }, + ) + + event = q.get_nowait() + assert event["type"] == "cluster_state" + assert event["state"] == "idle" + assert event["persistence_state"] == "healthy" + assert c._nodes["node-a"].workstreams["ws1"]["persistence_state"] == "healthy" def test_apply_snapshot_state_change_does_not_carry_pending_approval_detail(self): """Stage 3 cleanup — the snapshot-resync cluster_state event no @@ -473,6 +517,30 @@ class TestCollectorDelta: # Verify in-memory state was updated assert c._nodes["node-a"].workstreams["ws1"]["state"] == "running" + def test_apply_delta_ws_state_projects_persistence_state(self): + c = _make_collector() + c._nodes["node-a"] = NodeSnapshot( + node_id="node-a", + server_url="http://a:8080", + workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}}, + ) + q: queue.Queue[dict] = queue.Queue() + c.register_listener(q) + + c._apply_delta( + "node-a", + { + "type": "ws_state", + "ws_id": "ws1", + "state": "error", + "persistence_state": "conflict", + }, + ) + + event = q.get_nowait() + assert event["persistence_state"] == "conflict" + assert c._nodes["node-a"].workstreams["ws1"]["persistence_state"] == "conflict" + def test_apply_delta_ws_state_does_not_carry_pending_approval_detail(self): """Stage 3 cleanup — ``cluster_state`` no longer carries the ``pending_approval_detail`` piggyback. Approval items now arrive diff --git a/tests/test_cooperative_compaction.py b/tests/test_cooperative_compaction.py index 1782dc4f..e3c37caf 100644 --- a/tests/test_cooperative_compaction.py +++ b/tests/test_cooperative_compaction.py @@ -29,6 +29,7 @@ from turnstone.core.session import ( _is_ctx_overflow, _SummaryResult, ) +from turnstone.core.storage import get_storage from turnstone.core.trajectory import dicts_from_turns, turns_from_dicts @@ -41,12 +42,19 @@ def session(tmp_db, mock_openai_client): factory so the session shape stays in lockstep with the sibling truncation/compaction suites that read the same fullness measure. """ - return make_session( + s = make_session( client=mock_openai_client, context_window=10_000, max_tokens=1_000, tool_timeout=10, ) + get_storage().register_workstream( + s.ws_id, + user_id=s._user_id, + kind=s._kind, + parent_ws_id=s._parent_ws_id, + ) + return s # --------------------------------------------------------------------------- @@ -658,7 +666,13 @@ class TestCompactBeforeTruncate: def compact_then_terminal(*_args, **_kwargs): if terminal == "successor": - session._claim_generation() + abandoned, persistence_error = session.force_abandon_generation( + target_is_current=lambda: True, + clear_target=lambda: True, + publish_abandoned=lambda: None, + ) + assert abandoned is True + assert persistence_error is None else: session.close() return True @@ -694,7 +708,16 @@ class TestCompactBeforeTruncate: assert status.call_count == 1 truncate.assert_not_called() midturn.assert_not_called() - assert not [call for call in save.call_args_list if call.args[1] == "tool"] + tool_saves = [call for call in save.call_args_list if call.args[1] == "tool"] + if terminal == "successor": + # A force successor may retire an accepted assistant tool block + # only after synthesizing its matching TOOL receipt. The old raw + # generation claim represented a structurally invalid state that + # production now correctly refuses. + assert len(tool_saves) == 1 + assert "Force-cancelled" in tool_saves[0].args[2] + else: + assert not tool_saves # --------------------------------------------------------------------------- @@ -2209,7 +2232,7 @@ class TestCompactionLifecycleEvents: with ( patch.object(session, "_utility_completion", return_value=summary), patch.object(session.ui, "on_compaction", return_value=99) as oc, - patch("turnstone.core.session.get_compaction_watermark", return_value=17), + patch.object(get_storage(), "get_compaction_watermark", return_value=17), patch("turnstone.core.session.save_message", side_effect=fake_save), ): assert session._compact_messages() is True @@ -2255,7 +2278,7 @@ class TestCompactionLifecycleEvents: with ( patch.object(session, "_summary_input_budget_chars", return_value=450), patch.object(session, "_summarize_once", side_effect=fake_once), - patch("turnstone.core.session.get_compaction_watermark", return_value=17), + patch.object(get_storage(), "get_compaction_watermark", return_value=17), patch("turnstone.core.session.save_message", side_effect=fake_save), ): assert session._compact_messages(auto=False) is True diff --git a/tests/test_coord_rich_ws_state_payload.py b/tests/test_coord_rich_ws_state_payload.py index b6c57206..1cf91190 100644 --- a/tests/test_coord_rich_ws_state_payload.py +++ b/tests/test_coord_rich_ws_state_payload.py @@ -251,6 +251,7 @@ class _FakeCollectorRecorder: activity: str = "", activity_state: str = "", content: str = "", + persistence_state: str = "healthy", ) -> None: self.state_calls.append( { @@ -261,6 +262,7 @@ class _FakeCollectorRecorder: "activity": activity, "activity_state": activity_state, "content": content, + "persistence_state": persistence_state, } ) @@ -327,6 +329,7 @@ def test_coord_adapter_emit_state_passes_rich_payload_to_collector() -> None: assert call["activity_state"] == "thinking" # Mid-turn (RUNNING) — content stays accumulated for the eventual IDLE drain. assert call["content"] == "" + assert call["persistence_state"] == "healthy" def test_coord_adapter_emit_state_idle_drains_content() -> None: @@ -405,6 +408,33 @@ def test_coord_ui_broadcast_activity_no_op_when_collector_unset() -> None: ui.on_thinking_start() # must not raise +def test_coord_ui_persistence_refresh_updates_cluster_without_transcript_event() -> None: + recorder = _FakeCollectorRecorder() + ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1") + session = MagicMock() + session.conversation_persistence_status = lambda: {"state": "retrying"} + ui.bind_session(session) + ws = MagicMock() + ws.state.value = "error" + # The registry row deliberately disagrees: the persistence field must + # come from the BOUND session, so a registry miss or an id-reuse + # replacement row can never report another session's journal. + ws.session.conversation_persistence_status = lambda: {"state": "healthy"} + mgr = MagicMock() + mgr.get.return_value = ws + ConsoleCoordinatorUI._collector = recorder # type: ignore[assignment] + ConsoleCoordinatorUI._coord_mgr = mgr + try: + ui.on_persistence_state_changed() + finally: + ConsoleCoordinatorUI._collector = None + ConsoleCoordinatorUI._coord_mgr = None + + assert recorder.state_calls[0]["state"] == "error" + assert recorder.state_calls[0]["persistence_state"] == "retrying" + assert ui._event_id == 0, "operator refresh must not enter the per-workstream SSE stream" + + def test_coord_ui_broadcast_activity_failure_does_not_strand_dedup() -> None: """Regression for the Copilot finding on PR #420: post-fix the dedup state ``_last_broadcast_activity`` is updated **only after** @@ -674,3 +704,34 @@ def test_webui_on_tool_result_still_records_prometheus_tool_call() -> None: assert ui._ws_turn_tool_calls == 1 finally: WebUI._global_queue = None + + +def test_webui_accepted_tool_projection_is_metrics_free() -> None: + """Final guarded replacement must not count the executor receipt twice.""" + import queue + + from turnstone.server import WebUI + + WebUI._global_queue = queue.Queue() + try: + ui = WebUI(ws_id="ws-int", user_id="u1") + ui._ws_tool_calls = {"bash": 1} + ui._ws_turn_tool_calls = 1 + with patch("turnstone.server._metrics") as mock_metrics: + event_id = ui.on_tool_turn_accepted( + "call-1", + "bash", + "guarded output", + is_error=True, + effect_status="unknown", + ) + + assert event_id == 1 + mock_metrics.record_tool_call.assert_not_called() + assert ui._ws_tool_calls == {"bash": 1} + assert ui._ws_turn_tool_calls == 1 + projected = ui._event_buffer[-1][1] + assert projected["accepted"] is True + assert projected["effect_status"] == "unknown" + finally: + WebUI._global_queue = None diff --git a/tests/test_coordinator_adapter.py b/tests/test_coordinator_adapter.py index eeca4f4c..8263a600 100644 --- a/tests/test_coordinator_adapter.py +++ b/tests/test_coordinator_adapter.py @@ -8,6 +8,7 @@ tests in test_session_manager.py cover the lifecycle path. from __future__ import annotations +import logging import queue import threading from typing import Any @@ -162,6 +163,7 @@ def test_emit_state_calls_collector_state() -> None: activity="", activity_state="", content="", + persistence_state="healthy", ) @@ -263,6 +265,7 @@ def test_deferred_stale_state_does_not_consume_coordinator_content( activity="", activity_state="", content=content, + persistence_state="healthy", ) with ui._ws_lock: assert ui._ws_turn_content == [] @@ -472,6 +475,22 @@ class _SendSession: self.closed = True +class _ForeignQueueSession(_SendSession): + """Session whose queue already holds another participant's input. + + Concrete method, not a mock attribute: the adapter's spawn gate only + honours a real hook (see ``concrete_method``). + """ + + def __init__(self) -> None: + super().__init__() + self.probed_principals: list[str] = [] + + def has_foreign_queued_messages(self, principal_id: str) -> bool: + self.probed_principals.append(principal_id) + return True + + class _StubManager: """Minimal SessionManager stub exposing ``get`` for adapter.send.""" @@ -566,6 +585,49 @@ class TestCoordinatorAdapterWorkerDispatch: assert ws._worker_running is False assert session.send_calls == ["hello"] + def test_cross_user_queued_input_refusal_names_its_reason( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The refusal reaches the caller as a bare ``False``. + + ``send`` has no per-refusal return channel (the interactive route's + 409 ``cross_user_interjection`` has one), so the log is the ONLY place + this refusal is distinguishable from a full queue or an unloaded + workstream. Pin the reason token, not just the return value. + """ + adapter, _ = _make_adapter() + ws = _make_ws() + session = _ForeignQueueSession() + ws.session = session # type: ignore[assignment] + adapter.attach(_StubManager(ws)) # type: ignore[arg-type] + + with caplog.at_level(logging.WARNING, logger="turnstone.console.coordinator_adapter"): + assert adapter.send(ws.id, "hello", acting_user_id="user-b") is False + assert session.probed_principals == ["user-b"] + assert session.send_calls == [] + assert session.queue_calls == [] + assert ws.worker_thread is None + assert any( + "coord_adapter.send_refused_cross_user_queued_input" in r.getMessage() + for r in caplog.records + ) + + def test_unauthenticated_dispatch_skips_the_cross_user_probe(self) -> None: + """Internal dispatch carries no principal, so there is nobody to + classify against — the probe must not run at all (a blanket refusal + would strand the create-time initial message).""" + adapter, _ = _make_adapter() + ws = _make_ws() + session = _ForeignQueueSession() + ws.session = session # type: ignore[assignment] + adapter.attach(_StubManager(ws)) # type: ignore[arg-type] + + assert adapter.send(ws.id, "hello") is True + assert session.probed_principals == [] + if ws.worker_thread is not None: + ws.worker_thread.join(timeout=2.0) + assert session.send_calls == ["hello"] + # --------------------------------------------------------------------------- # Children registry diff --git a/tests/test_coordinator_endpoints.py b/tests/test_coordinator_endpoints.py index 95f88355..e0ba0baf 100644 --- a/tests/test_coordinator_endpoints.py +++ b/tests/test_coordinator_endpoints.py @@ -12,7 +12,7 @@ the lifted ``approve`` and ``close`` handlers from from __future__ import annotations import hashlib -from typing import cast +from typing import TYPE_CHECKING, Any, cast from unittest.mock import MagicMock import httpx @@ -74,6 +74,9 @@ from turnstone.core.session_routes import ( ) from turnstone.core.workstream import WorkstreamKind +if TYPE_CHECKING: + from collections.abc import Callable + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -535,6 +538,7 @@ def test_active_list_row_shape_includes_unified_fields(storage): "user_id", "project_id", "persona", + "persistence_state", } assert row["name"] == "lifted-coord" assert row["kind"] == "coordinator" @@ -1112,6 +1116,19 @@ def test_close_records_audit_and_removes_from_mgr(storage): assert "coordinator.close" in actions +def test_close_returns_409_when_unresolved_persistence_refuses_unload(storage): + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + ws.session.prepare_soft_close = MagicMock(return_value=False) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + resp = client.post(f"/v1/api/workstreams/{ws.id}/close", headers=_COORD_HEADERS) + + assert resp.status_code == 409 + assert resp.json() == {"error": "workstream has unresolved persistence"} + assert mgr.get(ws.id) is ws + + def test_approve_resolves_ui_event(storage): mgr = _build_mgr(storage) ws = mgr.create(user_id="user-1") @@ -1393,8 +1410,36 @@ def test_detail_correlation_id_on_unexpected_rehydrate_failure(storage): # --------------------------------------------------------------------------- +class _HistoryHandoffSession: + """Minimal concrete session for token-bearing history endpoint tests.""" + + def __init__(self) -> None: + self._history_generation = 0 + + def capture_history_handoff( + self, + load_messages: Callable[[int], list[dict[str, Any]]], + ) -> tuple[list[dict[str, Any]], str]: + return load_messages(0), f"test-history.{self._history_generation}" + + def resume(self, _ws_id: str) -> None: + return None + + +def _build_history_mgr(storage: Any): + def session_factory( + _ui: Any, + _model_alias: str | None = None, + _ws_id: str | None = None, + **_kwargs: Any, + ) -> _HistoryHandoffSession: + return _HistoryHandoffSession() + + return _build_mgr_with_factory(storage, session_factory) + + def test_history_returns_messages(storage): - mgr = _build_mgr(storage) + mgr = _build_history_mgr(storage) ws = mgr.create(user_id="user-1") # Seed a message in storage. storage.save_message(ws.id, "user", "hello") @@ -1403,13 +1448,14 @@ def test_history_returns_messages(storage): assert resp.status_code == 200 body = resp.json() assert body["ws_id"] == ws.id + assert body["handoff_token"] assert any(m.get("role") == "user" and m.get("content") == "hello" for m in body["messages"]) def test_history_any_admin_coordinator_caller_can_read(storage): # Trusted-team visibility: history is readable by any # ``admin.coordinator`` caller. - mgr = _build_mgr(storage) + mgr = _build_history_mgr(storage) ws = mgr.create(user_id="owner") storage.save_message(ws.id, "user", "hello") client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) @@ -1445,7 +1491,8 @@ def test_history_private_project_visible_to_member(storage): "c" * 32, kind="coordinator", user_id="alice", project_id="proj-secret" ) storage.save_message("c" * 32, "user", "secret plan") - client = _make_client(storage, coord_mgr=_build_mgr(storage), registry=_fake_registry()) + mgr = _build_history_mgr(storage) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) resp = client.get( f"/v1/api/workstreams/{'c' * 32}/history", headers={"X-Test-User": "member-bob", "X-Test-Perms": "admin.coordinator"}, @@ -1526,11 +1573,15 @@ def test_coord_attachments_private_project_visible_to_member(storage): def test_history_serves_storage_only_workstream(storage): - """Persisted-but-not-loaded coordinators (closed / evicted) are still - readable via /history without rehydrating. Mirrors the pre-lift - ``_resolve_coordinator_or_404`` ladder: storage-row + kind check - is sufficient when ``mgr.get`` returns None.""" - mgr = _build_mgr(storage) + """Closed coordinators are still readable via /history without rehydrating. + + Deliberate pin update (back to the pre-handoff assertion, plus the token + contract): a cold row has no live writer and no splice to witness, so the + read is storage-only and tokenless — never constructing a session into + the bounded pool. The tokenless 200 seeds a render plus the tokenless + stream bootstrap. + """ + mgr = _build_history_mgr(storage) storage.register_workstream("storage-only-coord", kind="coordinator", user_id="user-1") storage.save_message("storage-only-coord", "user", "from cold storage") # Confirm precondition: row is in storage, NOT in the manager's pool. @@ -1542,8 +1593,10 @@ def test_history_serves_storage_only_workstream(storage): headers=_COORD_HEADERS, ) assert resp.status_code == 200 - assert any(m.get("content") == "from cold storage" for m in resp.json()["messages"]) - # History does NOT rehydrate (unlike detail) — pool stays cold. + body = resp.json() + assert any(m.get("content") == "from cold storage" for m in body["messages"]) + assert body["handoff_token"] is None + # The read verb leaves the pool untouched. assert mgr.get("storage-only-coord") is None @@ -1562,25 +1615,24 @@ def test_history_404_when_kind_interactive(storage): assert "interactive content" not in resp.text -def test_history_swallows_load_messages_exception_returns_empty(storage): - """``storage.load_messages`` raising mid-call (transient DB outage, - corrupted row, etc.) must not 5xx the page-load handshake — coord - pre-lift logged at debug and returned 200 with ``messages == []``. - The lifted body preserves that contract on both kinds; pin it - explicitly so a future reader doesn't remove the bare-except as - dead code.""" +def test_history_load_messages_exception_returns_non_authoritative_503(storage): + """A transient history-load failure must not masquerade as empty truth. + + The browser retains its prior transcript and repair latch on a non-2xx. + The failure body therefore carries neither projected messages nor a + handoff token that could authorize an SSE connection from an incomplete + durable prefix. + """ from unittest.mock import patch - mgr = _build_mgr(storage) + mgr = _build_history_mgr(storage) ws = mgr.create(user_id="user-1") client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) with patch.object(storage, "load_messages", side_effect=RuntimeError("db gone")): resp = client.get(f"/v1/api/workstreams/{ws.id}/history", headers=_COORD_HEADERS) - assert resp.status_code == 200 - body = resp.json() - assert body["ws_id"] == ws.id - assert body["messages"] == [] + assert resp.status_code == 503 + assert resp.json() == {"error": "History temporarily unavailable"} def test_history_clamps_limit_query_param(storage): @@ -1588,7 +1640,7 @@ def test_history_clamps_limit_query_param(storage): preserves the same bounds. Out-of-range / unparseable values fall back to defaults instead of erroring — coord's page-load handshake should never 4xx on a malformed limit param.""" - mgr = _build_mgr(storage) + mgr = _build_history_mgr(storage) ws = mgr.create(user_id="user-1") # Seed enough messages to exercise the upper bound. SQLite's INSERT # is fast enough that 6 inserts in a tight loop is fine. @@ -2536,6 +2588,42 @@ def test_cluster_inspect_coordinator_self_path(storage): assert isinstance(body["messages"], list) +def test_cluster_inspect_scrubs_private_assistant_provenance(storage): + """Cluster inspection never exposes the durable audit principal.""" + import json + + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + storage.save_message( + ws.id, + "assistant", + "accepted", + meta=json.dumps( + { + "provenance": { + "model_alias": "main", + "backend_model_id": "kernel", + "registry_generation": 8, + "acting_principal_id": "private-user-id", + } + } + ), + commit_key="private-commit-key", + ) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + response = client.get(f"/v1/api/cluster/ws/{ws.id}/detail", headers=_CLUSTER_HEADERS) + + assert response.status_code == 200 + [message] = response.json()["messages"] + assert message["role"] == "assistant" + assert message["content"] == "accepted" + assert "_provenance" not in message + assert "_commit_key" not in message + assert "private-user-id" not in response.text + assert "private-commit-key" not in response.text + + def test_cluster_inspect_unloaded_coordinator_live_null(storage): """A persisted-but-not-loaded coordinator returns live: null, 200.""" mgr = _build_mgr(storage) diff --git a/tests/test_coordinator_page.py b/tests/test_coordinator_page.py index eaa8eb8d..fd821b1b 100644 --- a/tests/test_coordinator_page.py +++ b/tests/test_coordinator_page.py @@ -15,6 +15,7 @@ from starlette.applications import Starlette from starlette.routing import Route from starlette.testclient import TestClient +from tests._js_harness_helpers import strip_js_comments as _strip_comments from turnstone.console.server import coordinator_page @@ -188,13 +189,17 @@ def test_coordinator_js_approval_keyboard_shortcuts(): # turns must render with the correct batch state on reload, not # the contradictory "✓ approved" pill that pre-fix showed for # any prior denial. bug-1 / bug-3 from the second /review pass. - # Post wire-shape unification the deny/error classification moved - # server-side into ``project_history_messages``; coord reads the - # derived ``m.denied`` / ``m.is_error`` flags (pin the live read, - # not the comment prose the old content-prefix sniffing left behind). - assert "m.denied" in body - assert "m.is_error" in body - assert "callOutcomes" in body + # Post wire-shape unification the deny/error classification is shared by + # both browser reducers. Pin the coordinator's use of the occurrence-aware + # helper and the helper's live reads of the projected flags; call ids may + # repeat across turns and therefore cannot be classified by a global map. + tool_projection = ( + Path(__file__).resolve().parent.parent / "turnstone/shared_static/tool_projection.js" + ).read_text(encoding="utf-8") + assert "indexHistoryToolOutcomes(historyMessages)" in body + assert "historyBatchOutcomes.get(m)" in body + assert "result.denied" in tool_projection + assert "result.is_error" in tool_projection # User-message attachment pills — both live send (coordSend) and # history replay route through appendUserMessageWithAttachments. # Renaming or dropping the helper would silently regress the @@ -571,11 +576,15 @@ def test_coordinator_js_seeds_resume_cursor_only_on_initial_connect(): assert "await refetchHistory(true)" in body, ( "the initial-connect path must call refetchHistory(true) to seed the cursor." ) - flow = re.search(r"function loadHistoryThenReconnect\(\)\s*\{(.*?)\n \}", body, re.S) + flow = re.search( + r"function loadHistoryThenReconnect\(manualAttempt = false\)\s*\{(.*?)\n \}", + body, + re.S, + ) assert flow is not None, "loadHistoryThenReconnect not found" - assert "refetchHistory(true)" in flow.group(1) and ".finally(" in flow.group(1), ( + assert "refetchHistory(true)" in flow.group(1) and ".then((outcome) => {" in flow.group(1), ( "the truncated resync (loadHistoryThenReconnect) must seed via " - "refetchHistory(true) and reconnect in .finally." + "refetchHistory(true) and reconnect in its outcome-threaded settle." ) assert re.search( r"if\s*\(\s*connectCursor\s*!=\s*null\s*\)\s*\{\s*url\s*\+=\s*\"\?last_event_id=\"", @@ -583,6 +592,74 @@ def test_coordinator_js_seeds_resume_cursor_only_on_initial_connect(): ), "connectSSE must gate ?last_event_id= on connectCursor != null (so cursor 0 isn't dropped)." +def test_coordinator_initial_history_handoff_is_one_shot_and_mismatch_refetches(): + """Coordinator mirrors interactive's opaque REST -> initial-SSE handoff.""" + import re + from pathlib import Path + + body = ( + Path(__file__).resolve().parent.parent + / "turnstone/console/static/coordinator/coordinator.js" + ).read_text(encoding="utf-8") + start = body.index("function connectSSE()") + end = body.index("function scheduleReconnect", start) + connect = body[start:end] + + hidden = connect.index("if (document.hidden)") + capability = connect.index('"user_turn=1"') + handoff_query = connect.index('"history_token="') + construct = connect.index("new EventSource(url") + consume = connect.index("historyHandoffToken = null;", construct) + assert capability < hidden < handoff_query < construct < consume + assert 'url += "?last_event_id="' in connect + assert '(url.includes("?") ? "&" : "?")' in connect + assert connect.count('"user_turn=1"') == 1 + + refetch_start = body.index("async function refetchHistory(seedCursor = false)") + refetch = body[refetch_start : body.index("\n function ", refetch_start + 1)] + assert re.search( + r"if\s*\(seedCursor\)\s*\{\s*historyHandoffToken\s*=\s*" + r"typeof hist\.handoff_token === \"string\"", + refetch, + ) + + mismatch = body.index('case "history_resync"') + truncated = body.index('case "replay_truncated"', mismatch) + mismatch_case = body[mismatch:truncated] + assert "historyRepair.begin(wsId);" in mismatch_case + assert "last_event_id" not in mismatch_case + + # A mismatch is a durable-history repair, not a transport retry. Once it + # is latched, connectSSE may only schedule the bounded REST retry and + # return; it cannot construct a cursorless/tokenless EventSource. The + # latch, budget, backoff, and parked prompt moved into the shared + # controller (history_handoff.createHistoryHandoffRepair) — pinned there, + # once; what stays pinned HERE is this pane's use of it. + repair_guard = connect.index("if (historyRepair.isRepairing(wsId))") + assert repair_guard < handoff_query < construct + guard_end = connect.index("if (historyHandoffToken != null)", repair_guard) + guard = connect[repair_guard:guard_end] + assert "historyRepair.schedule();" in guard + assert "return;" in guard + + load_start = body.index("function loadHistoryThenReconnect(manualAttempt = false)") + load_end = body.index("function enterDegradedCatchup", load_start) + load = body[load_start:load_end] + # Admission before any work, then the budget charge, then exactly one + # handover of the verdict; the non-repair tail keeps its own reconnect. + admit = load.index("historyRepair.admitAttempt(manualAttempt)") + start_attempt = load.index("historyRepair.startAttempt(manualAttempt)", admit) + settle = load.index("historyRepair.settle({", start_attempt) + assert admit < start_attempt < settle + assert "hasToken: historyHandoffToken != null" in load[settle:] + repair_return = load.index("return;", settle) + assert "connectSSE();" not in load[settle:repair_return] + + destroy_start = body.index("function destroy()") + destroy_end = body.index("function reconnect()", destroy_start) + assert "historyRepair.clear();" in body[destroy_start:destroy_end] + + def test_coordinator_refetch_failure_preserves_the_pane(): """A FAILED /history fetch must leave the message column and the tool-row/batch tracking untouched (#882 G3): refetchHistory's wipe + @@ -711,9 +788,6 @@ def test_coordinator_history_stale_latch_contract(): # ELSE-IF of the truncated consumer (mutual exclusion: the # truncated branch's own reload heals the latch too; two separate # ifs would run both heals on one idle edge). - def _strip_comments(text: str) -> str: - return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("//")) - trunc_arm = body.index("if (pendingTruncatedResync)") backstop = body.index("historyStale &&", trunc_arm) assert "} else if (" in body[trunc_arm:backstop], ( @@ -820,7 +894,7 @@ def test_coordinator_history_stale_latch_contract(): "fetch finally) so every exit rebalances it." ) inc = body.index("refetchesInFlight++", fetch_start) - awt = body.index("await getJSON(", fetch_start) + awt = body.index("await Promise.race([", fetch_start) fin = body.index("} finally {", fetch_start) dec = body.index("refetchesInFlight--", fetch_start) assert inc < awt < fin < dec, ( @@ -967,11 +1041,16 @@ def test_coordinator_history_stale_latch_contract(): # The await must be BOUNDED (r7): an accepted-but-never-answered # /history would pin refetchesInFlight above zero for the life of # the page and every heal would yield forever. - assert "histCtrl.abort()" in fetch_code and "clearTimeout(histTimer)" in fetch_code, ( - "refetchHistory must bound its fetch with the AbortController + " - "flat-timeout shape (and clear the timer in the finally) — an " - "unbounded await pins the in-flight counter and permanently " - "disables both heals." + assert "if (histCtrl) histCtrl.abort();" in fetch_code + assert "deadlineHandle.dispose()" in fetch_code, ( + "the fetch finally must retire its deadline through the module's " + "dispose() — direct state-slot pokes are the drift the shared " + "handle exists to prevent." + ) + assert "createHistoryHandoffDeadline(" in fetch_code + assert "Promise.race([" in fetch_code and "deadlineHandle.promise" in fetch_code, ( + "refetchHistory must have a logical deadline independent of abort — " + "older runtimes can lack AbortController and a request may ignore it." ) # The seq stamp's PRODUCER must sit above the await, or the # last-dispatch-wins gate is permanently vacuous (a stamp captured @@ -1002,27 +1081,35 @@ def test_coordinator_history_stale_latch_contract(): # destroy() must abort the in-flight fetch (dead-not-inert, the # staleRetryTimer ruling applied to the r7 bound). # Producer pins first — the destroy() consumer sweep below is - # satisfiable by an always-empty Set without them. - assert body.count("histCtrls.add(histCtrl)") == 1, ( - "every dispatch must register its controller in the abort Set." + # satisfiable by an always-empty Set without them. ONE composite + # record per attempt: registering ctrl and deadline separately is the + # parallel-bookkeeping drift a future attempt site gets wrong. + assert body.count("histAttempts.add(attempt)") == 1, ( + "every dispatch must register its composite {ctrl, deadline} record in the attempt Set." ) - assert body.count("histCtrls.delete(histCtrl)") == 1, ( - "the fetch finally must release its own controller — without the " - "delete the Set grows for the life of the pane." + assert body.count("histAttempts.delete(attempt)") == 1, ( + "the fetch finally must release its own attempt record — without " + "the delete the Set grows for the life of the pane." ) - assert body.index("histCtrls.add(histCtrl)", fetch_start) < awt, ( - "the controller must be registered BEFORE the await." + assert body.index("histAttempts.add(attempt)", fetch_start) < awt, ( + "the attempt must be registered BEFORE the await." ) - assert fin < body.index("histCtrls.delete(histCtrl)", fetch_start), ( - "the controller release must sit in the fetch finally." + assert fin < body.index("histAttempts.delete(attempt)", fetch_start), ( + "the attempt release must sit in the fetch finally." ) destroy_code = _strip_comments(destroy_slice) - assert "histCtrls.forEach" in destroy_code and ".abort()" in destroy_code, ( + assert "histAttempts.forEach" in destroy_code and ".abort()" in destroy_code, ( "destroy() must abort EVERY in-flight /history (a Set — a " "newest-wins single slot left older overlapping fetches " "unabortable); the 15s bound alone pins the destroyed closure " "until it fires." ) + assert "attempt.deadline.dispose({ expire: true, resolve: true })" in destroy_code, ( + "destroy must settle every logical deadline immediately (expired + " + "resolved, including when AbortController is unavailable) via the " + "module's dispose() — from the SAME composite record its abort " + "came from, never a parallel Set." + ) def test_coordinator_js_early_paints_pending_tool_calls(): @@ -1139,6 +1226,24 @@ def test_coordinator_chrome_builder_and_thin_page(): assert (base / "coord-chrome.css").exists(), "the migrated chrome stylesheet must exist" +def test_coordinator_close_409_uses_plain_retry_copy(): + from pathlib import Path + + body = ( + Path(__file__).resolve().parent.parent + / "turnstone/console/static/coordinator/coordinator.js" + ).read_text(encoding="utf-8") + start = body.index("async function coordCloseSession()") + end = body.index("// ------------------------------------------------------------------", start) + close = body[start:end] + + assert "resp.status === 409" in close + assert ( + "Conversation history is still being saved. Try ending the session again shortly." in close + ) + assert "resumeSse();" in close, "a refused close must retain and resume the live pane" + + def test_coord_child_links_open_interactive_pane(): """Step 5c (+ split revival): a coordinator child ws link (children tree + linkified tool output) opens the child as a node-proxied interactive pane @@ -1186,16 +1291,21 @@ def test_coordinator_js_gates_send_on_cross_user_busy(): assert "actingUserId !== me" in coord_js assert "composer.setSendBlocked(" in coord_js assert "function reconcileSendBlock()" in coord_js - # reactive 409 fallback — the pane converts the 409 body at the fetch - # stage; the status ARM itself lives in the shared settle helper - # (composer_queue.settleSendResponse) with the rest of the response - # matrix, one implementation for both panes. - assert "r.status === 409" in coord_js - assert 'status: "cross_user_interjection"' in coord_js - assert "settleSendResponse(" in coord_js + # reactive 409 fallback — the fetch-stage conversion and the status ARM + # both live in the shared helper (composer_queue.postAndSettleSend / + # settleSendResponse), one implementation for both panes and for both of + # each pane's send flows. The pane owns only the request, so its + # edit-and-resend flow can no longer miss the conversion and report a + # refused resend as a connection error. + assert "cross_user_interjection" not in coord_js, ( + "the 409 conversion must not be re-derived per pane" + ) + assert coord_js.count("postAndSettleSend(") == 2, "composer send + edit-and-resend" helper = ( Path(__file__).resolve().parents[1] / "turnstone/shared_static/composer_queue.js" ).read_text(encoding="utf-8") + assert "response.status === 409" in helper + assert 'status: "cross_user_interjection"' in helper assert 'status === "cross_user_interjection"' in helper diff --git a/tests/test_history_commit_handoff.py b/tests/test_history_commit_handoff.py new file mode 100644 index 00000000..5bcddd6e --- /dev/null +++ b/tests/test_history_commit_handoff.py @@ -0,0 +1,2153 @@ +"""Adversarial tests for the live-to-durable conversation handoff (#981). + +These tests deliberately stop threads at the two linearization boundaries: +assistant-row persistence and the frozen ``/history`` load. Timeouts are +diagnostic backstops only; every race is otherwise driven by ``Event`` gates. +""" + +from __future__ import annotations + +import contextlib +import copy +import json +import threading +import time +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +import pytest + +from tests._session_helpers import make_result, make_session +from tests.test_session_manager import _make_manager +from turnstone.core import session as session_module +from turnstone.core.attachments import Attachment +from turnstone.core.history_decoration import project_history_messages +from turnstone.core.session_routes import SessionEndpointConfig, _make_dispatch_attempt +from turnstone.core.trajectory import EffectStatus, dicts_from_turns +from turnstone.core.workstream import Workstream + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + + +_TOOL_CALL = { + "id": "call-history-handoff", + "type": "function", + "function": {"name": "read_only_probe", "arguments": "{}"}, +} + + +class _ConversationStore: + """Thread-safe ``save_message`` fake with an assistant ACK barrier. + + ``ambiguous_assistant_ack`` models the hardest backend outcome: the row is + durably visible, but the caller receives the facade's failure sentinel. + """ + + def __init__( + self, + *, + block_assistant: bool = False, + ambiguous_assistant_ack: bool = False, + block_user: bool = False, + ambiguous_user_ack: bool = False, + ) -> None: + self._lock = threading.Lock() + self._next_id = 1 + self._rows: list[dict[str, Any]] = [] + self._ids_by_commit: dict[tuple[str, str], int] = {} + # Raw ``meta`` JSON per accepted TOOL row, keyed by tool_call_id. The + # row dicts above are the public-shaped handoff projection; the tool + # envelope carries private audit fields that never join it. + self.tool_meta: dict[str, str | None] = {} + self.block_assistant = block_assistant + self.ambiguous_assistant_ack = ambiguous_assistant_ack + self.block_user = block_user + self.ambiguous_user_ack = ambiguous_user_ack + self.assistant_entered = threading.Event() + self.release_assistant = threading.Event() + self.assistant_returning = threading.Event() + self.user_entered = threading.Event() + self.release_user = threading.Event() + self.user_returning = threading.Event() + if not block_assistant: + self.release_assistant.set() + if not block_user: + self.release_user.set() + + def __call__( + self, + ws_id: str, + role: str, + content: str, + *args: Any, + **kwargs: Any, + ) -> int: + if role == "assistant": + # The identity is what makes a storage row and a still-pending + # ledger entry provably the same logical commit. Content/event-id + # equality is not an identity: legitimate rows can share both. + commit_key = kwargs.get("commit_key") + assert isinstance(commit_key, str) and commit_key + self.assistant_entered.set() + assert self.release_assistant.wait(5), "test did not release assistant persistence" + elif role == "user": + commit_key = kwargs.get("commit_key") + assert isinstance(commit_key, str) and commit_key + self.user_entered.set() + assert self.release_user.wait(5), "test did not release user persistence" + + row: dict[str, Any] = { + "role": role, + "content": content, + } + event_id = kwargs.get("event_id") + if isinstance(event_id, int) and not isinstance(event_id, bool): + row["_event_id"] = event_id + commit_key = kwargs.get("commit_key") + if isinstance(commit_key, str) and commit_key: + row["_commit_key"] = commit_key + raw_tool_calls = kwargs.get("tool_calls") + if raw_tool_calls: + row["tool_calls"] = json.loads(raw_tool_calls) + if role == "tool": + if args: + row["tool_name"] = args[0] + if kwargs.get("tool_call_id"): + row["tool_call_id"] = kwargs["tool_call_id"] + if kwargs.get("is_error"): + row["is_error"] = True + raw_provider_data = kwargs.get("provider_data") + if raw_provider_data is not None: + row["_provider_content"] = json.loads(raw_provider_data) + producer = kwargs.get("producer") + if producer: + row["_producer"] = producer + source = kwargs.get("source") + if source: + row["_source"] = source + raw_meta = kwargs.get("meta") + if role == "user" and raw_meta: + parsed_meta = json.loads(raw_meta) + if parsed_meta.get("sender"): + row["_sender"] = parsed_meta["sender"] + if parsed_meta.get("client_send_ids"): + row["_client_send_ids"] = parsed_meta["client_send_ids"] + + with self._lock: + if role == "tool": + self.tool_meta[str(kwargs.get("tool_call_id") or "")] = raw_meta + identity = (ws_id, commit_key) if isinstance(commit_key, str) and commit_key else None + existing_id = self._ids_by_commit.get(identity) if identity is not None else None + if existing_id is not None: + row_id = existing_id + else: + row_id = self._next_id + self._next_id += 1 + self._rows.append(row) + if identity is not None: + self._ids_by_commit[identity] = row_id + + if role == "assistant": + self.assistant_returning.set() + if self.ambiguous_assistant_ack: + return 0 + elif role == "user": + self.user_returning.set() + if self.ambiguous_user_ack: + return 0 + return row_id + + def snapshot(self, overscan: int = 0) -> list[dict[str, Any]]: + # The store keeps the full prefix; the widened-window overscan the + # production loader applies to a tail-bounded read is a no-op here. + del overscan + with self._lock: + return copy.deepcopy(self._rows) + + def append_seed_row(self, row: dict[str, Any]) -> None: + """Seed a durable row without exercising the fake ACK controls.""" + with self._lock: + self._rows.append(copy.deepcopy(row)) + + +@contextlib.contextmanager +def _send_environment( + session: Any, + results: list[Any], + store: _ConversationStore, + execute_tools: Callable[..., Any], +) -> Iterator[None]: + """Patch only slow/orthogonal send seams; retain real commit ordering.""" + + scripted = iter(results) + with contextlib.ExitStack() as stack: + stack.enter_context( + patch.object(session, "_stream_response", side_effect=lambda _gen: next(scripted)) + ) + stack.enter_context(patch.object(session, "_execute_tools", side_effect=execute_tools)) + stack.enter_context(patch.object(session, "_full_messages", return_value=[])) + stack.enter_context(patch.object(session, "_update_token_table")) + stack.enter_context(patch.object(session, "_print_status_line")) + stack.enter_context(patch.object(session, "_emit_state")) + stack.enter_context(patch.object(session, "_visible_memory_count", return_value=0)) + stack.enter_context(patch.object(session, "_apply_post_execute_advisories")) + stack.enter_context(patch("turnstone.core.session.save_message", side_effect=store)) + yield + + +def _ready_session(**kwargs: Any) -> Any: + session = make_session(**kwargs) + # Keyed conversation commits intentionally refuse to resurrect a missing + # workstream after hard delete. Direct-session tests therefore install the + # parent row that production's manager/create path establishes first. + from turnstone.core.memory import register_workstream + + register_workstream(session.ws_id, user_id=kwargs.get("user_id", "")) + session._title_generated = True + session._system_composed_with_context = True + return session + + +def _start_send( + session: Any, + text: str = "request", + *, + client_send_ids: tuple[str, ...] = (), +) -> tuple[threading.Thread, list[BaseException]]: + errors: list[BaseException] = [] + + def _send() -> None: + try: + session.send(text, client_send_ids=client_send_ids) + except BaseException as exc: # diagnostic capture from the worker + errors.append(exc) + + worker = threading.Thread(target=_send, daemon=True) + worker.start() + return worker, errors + + +def _assistant_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [row for row in rows if row.get("role") == "assistant"] + + +def _user_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [row for row in rows if row.get("role") == "user"] + + +def _drain_listener(listener: Any) -> list[dict[str, Any]]: + delivered: list[dict[str, Any]] = [] + while not listener.empty(): + delivered.append(listener.get_nowait()) + return delivered + + +def _dispatch_as( + ws: Workstream, + session: Any, + *, + actor: str, + message: str, +) -> tuple[bool, dict[str, Any]]: + """Drive the production route's atomic queue-or-spawn admission seam.""" + + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (None, None), + tenant_check=None, + not_found_label="missing", + audit_action_prefix="workstream", + ) + attempt = _make_dispatch_attempt( + ws, + cfg, + ws.ui, + message=message, + resolved_atts=[], + ordered_taken=[], + send_id=f"send-{actor}", + acting_uid=actor, + ) + return attempt(session) + + +def test_frozen_history_load_and_positive_ack_have_no_missing_window(tmp_db: Any) -> None: + """A reader that wins before ACK returns old storage + pending exactly once. + + Persistence is admitted but paused before its visibility lane. The reader + freezes the old storage prefix, then persistence is released to contend + with it. Correctness requires the write + ACK to remain on the far side of + that reader, which must still overlay the accepted pending row. + """ + + session = _ready_session() + store = _ConversationStore() + persistence_waiting = threading.Event() + release_persistence = threading.Event() + load_entered = threading.Event() + release_frozen_load = threading.Event() + capture_result: list[tuple[list[dict[str, Any]], str]] = [] + capture_errors: list[BaseException] = [] + + def _frozen_load(overscan: int = 0) -> list[dict[str, Any]]: + frozen = store.snapshot() + load_entered.set() + assert release_frozen_load.wait(5), "test did not release frozen history load" + return frozen + + def _capture() -> None: + try: + capture_result.append(session.capture_history_handoff(_frozen_load)) + except BaseException as exc: # pragma: no cover - diagnostic capture + capture_errors.append(exc) + + persist_pending = session._persist_pending_conversation_commit + + def _gated_persistence(entry: Any) -> int: + if entry.message.get("role") != "assistant": + return persist_pending(entry) + persistence_waiting.set() + assert release_persistence.wait(5), "test did not release assistant persistence" + return persist_pending(entry) + + with ( + _send_environment( + session, + [make_result("durable answer")], + store, + MagicMock(return_value=([], None)), + ), + patch.object( + session, + "_persist_pending_conversation_commit", + side_effect=_gated_persistence, + ), + ): + sender, send_errors = _start_send(session) + assert persistence_waiting.wait(5), "assistant persistence was not admitted" + + reader = threading.Thread(target=_capture, daemon=True) + reader.start() + assert load_entered.wait(5), "history never entered its frozen load" + + # Let persistence contend for the visibility lane. It must not reach + # storage (and therefore cannot ACK/remove pending) until this reader + # has finished its old-prefix + pending snapshot. + release_persistence.set() + assert store.assistant_entered.wait(0.05) is False + assert sender.is_alive() + + release_frozen_load.set() + reader.join(5) + sender.join(5) + + assert not reader.is_alive() + assert not sender.is_alive() + assert capture_errors == [] + assert send_errors == [] + assert store.assistant_returning.is_set() + rows_during_handoff, token_during_handoff = capture_result[0] + assert [row.get("content") for row in _assistant_rows(rows_during_handoff)] == [ + "durable answer" + ] + assert isinstance(token_during_handoff, str) and token_during_handoff + + rows_after_ack, token_after_ack = session.capture_history_handoff(store.snapshot) + assert [row.get("content") for row in _assistant_rows(rows_after_ack)] == ["durable answer"] + # Positive ACK changes representation (pending -> durable), not the + # accepted-history revision exposed to the REST -> SSE handshake. + assert token_after_ack == token_during_handoff + assert session.has_unresolved_conversation_persistence() is False + + +def test_assistant_event_id_covers_the_final_on_commit_token_batch(tmp_db: Any) -> None: + """The durable/pending cursor is stamped after the commit hook flushes.""" + + session = _ready_session() + store = _ConversationStore() + committed = session.ui.on_turn_committed + observed: dict[str, int] = {} + + def _inject_final_pending_batch() -> None: + # Hold this fragment inside the time-window batcher so the production + # commit hook, not token arrival, emits its content event. + with session.ui._ws_lock: + session.ui._last_token_flush = time.monotonic() + session.ui.on_content_token("final batched fragment") + observed["before"] = session.ui._event_id + committed() + observed["after"] = session.ui._event_id + + with ( + _send_environment( + session, + [make_result("final batched fragment")], + store, + MagicMock(return_value=([], None)), + ), + patch.object( + session.ui, + "on_turn_committed", + side_effect=_inject_final_pending_batch, + ), + ): + session.send("flush the final content batch") + + assert observed["after"] == observed["before"] + 1 + assistant = _assistant_rows(store.snapshot())[0] + assert assistant["_event_id"] == observed["after"] + assert assistant["_event_id"] == session.ui._event_id + + +def test_commit_between_history_response_and_listener_registration_forces_resync( + tmp_db: Any, +) -> None: + """A completed commit crossing the two HTTP requests invalidates the token.""" + + session = _ready_session() + store = _ConversationStore() + _, before_commit = session.capture_history_handoff(store.snapshot) + listener_count = len(session.ui._listeners) + + with _send_environment( + session, + [make_result("crossed the bootstrap gap")], + store, + MagicMock(return_value=([], None)), + ): + session.send("first request") + + assert session.register_listener_for_history_handoff(before_commit) is None + assert len(session.ui._listeners) == listener_count + + _, after_commit = session.capture_history_handoff(store.snapshot) + assert after_commit != before_commit + registration = session.register_listener_for_history_handoff(after_commit) + assert registration is not None + assert len(session.ui._listeners) == listener_count + 1 + + +def test_user_admission_projects_once_to_two_tabs_and_keeps_early_assistant_stream( + tmp_db: Any, +) -> None: + """Both shared-workstream tabs receive one canonical row before output. + + The initiating tab optimistically painted Bob's prompt; the sibling did + not. Both receive the same accepted-user event. The event does not close + either EventSource, so the model's earliest content cannot fall into a + repair disconnect window. + """ + + session = _ready_session(user_id="owner") + session.bind_acting_user("bob") + store = _ConversationStore() + _rows, token_0 = session.capture_history_handoff(store.snapshot) + registration_a = session.register_listener_for_history_handoff(token_0) + registration_b = session.register_listener_for_history_handoff(token_0) + assert registration_a is not None and registration_b is not None + listener_a = registration_a[0] + listener_b = registration_b[0] + assert listener_a is not listener_b + + model_entered = threading.Event() + release_model = threading.Event() + + def _gated_results() -> Iterator[Any]: + model_entered.set() + assert release_model.wait(5), "test did not release model stream" + session.ui.on_content_token("early ") + session.ui.on_content_token("assistant output") + yield make_result("early assistant output") + + with _send_environment( + session, + _gated_results(), # type: ignore[arg-type] + store, + MagicMock(return_value=([], None)), + ): + sender, send_errors = _start_send( + session, + "Bob's shared prompt", + client_send_ids=("browser-send-1",), + ) + assert model_entered.wait(5), "user row did not persist before model entry" + + events_a = _drain_listener(listener_a) + events_b = _drain_listener(listener_b) + release_model.set() + assert events_a == events_b + user_events = [event for event in events_a if event.get("type") == "user_turn"] + assert len(user_events) == 1 + assert user_events[0] == { + "type": "user_turn", + "content": "Bob's shared prompt", + "client_send_ids": ["browser-send-1"], + "sender": "bob", + "ws_id": user_events[0]["ws_id"], + "_event_id": user_events[0]["_event_id"], + } + assert all(event.get("type") != "history_resync" for event in events_a) + + # The old REST revision cannot register a third stale tab. + assert session.register_listener_for_history_handoff(token_0) is None + + rows_1, token_1 = session.capture_history_handoff(store.snapshot) + users = _user_rows(rows_1) + assert len(users) == 1 + assert users[0]["content"] == "Bob's shared prompt" + assert users[0]["_sender"] == "bob" + assert users[0]["_client_send_ids"] == ["browser-send-1"] + assert token_1 != token_0 + + sender.join(5) + + assert not sender.is_alive() + assert send_errors == [] + repaired_events = _drain_listener(listener_a) + assert ( + "".join( + str(event.get("text") or "") + for event in repaired_events + if event.get("type") == "content" + ) + == "early assistant output" + ) + + +def test_tool_admission_projects_final_row_once_to_two_tabs_without_recounting( + tmp_db: Any, +) -> None: + """The executor receipt stays provisional; accepted TOOL replaces it. + + Both listeners receive the same canonical event, including the final + guarded/error/effect/preview projection, while tool metrics count only the + executor receipt. The accepted row never requests routine history repair. + """ + + session = _ready_session(user_id="owner") + store = _ConversationStore() + _rows, token = session.capture_history_handoff(store.snapshot) + registration_a = session.register_listener_for_history_handoff(token) + registration_b = session.register_listener_for_history_handoff(token) + assert registration_a is not None and registration_b is not None + listener_a = registration_a[0] + listener_b = registration_b[0] + descriptor = { + "attachment_id": "p" * 64, + "kind": "html", + "filename": "preview.html", + } + preview_attachment = Attachment( + attachment_id=descriptor["attachment_id"], + filename="preview.html", + mime_type="text/html", + kind="preview", + content=b"

accepted preview

", + ) + attachment_saves: list[dict[str, Any]] = [] + + def _save_tool_with_attachments( + ws_id: str, + content: str, + tool_name: str, + tool_call_id: str, + attachments: Any, + **kwargs: Any, + ) -> int: + attachment_saves.append( + { + "content": content, + "attachments": tuple(attachments), + "event_id": kwargs.get("event_id"), + "meta": kwargs.get("meta"), + } + ) + return store( + ws_id, + "tool", + content, + tool_name, + tool_call_id=tool_call_id, + **kwargs, + ) + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], None]: + session.ui.on_tool_result( + _TOOL_CALL["id"], + "read_only_probe", + "provisional unguarded output", + is_error=True, + preview=descriptor, + ) + session._tool_error_flags[_TOOL_CALL["id"]] = True + session._tool_status[_TOOL_CALL["id"]] = EffectStatus.UNKNOWN + session._tool_previews[_TOOL_CALL["id"]] = (descriptor, preview_attachment) + return [(_TOOL_CALL["id"], "final guarded output")], None + + with ( + _send_environment( + session, + [make_result("", tool_calls=[_TOOL_CALL]), make_result("done")], + store, + _execute_tools, + ), + patch( + "turnstone.core.session.save_tool_message_with_attachments", + side_effect=_save_tool_with_attachments, + ), + ): + session.send("run the probe") + + events_a = _drain_listener(listener_a) + events_b = _drain_listener(listener_b) + tools_a = [event for event in events_a if event.get("type") == "tool_result"] + tools_b = [event for event in events_b if event.get("type") == "tool_result"] + assert tools_a == tools_b + assert len(tools_a) == 2 + assert tools_a[0]["output"] == "provisional unguarded output" + assert "accepted" not in tools_a[0] + accepted = tools_a[1] + assert accepted == { + "type": "tool_result", + "accepted": True, + "call_id": _TOOL_CALL["id"], + "name": "read_only_probe", + "output": "final guarded output", + "is_error": True, + "preview": descriptor, + "effect_status": "unknown", + "ws_id": accepted["ws_id"], + "_event_id": accepted["_event_id"], + } + assert all(event.get("type") != "history_resync" for event in events_a) + assert session.ui._ws_tool_calls == {"read_only_probe": 1} + assert len(attachment_saves) == 1 + assert attachment_saves[0]["content"] == "final guarded output" + assert attachment_saves[0]["event_id"] == accepted["_event_id"] + # Pin updated with the turn-identity symmetry work: an owner-driven send + # is an attributed lane, so the attachment-bearing persist shape carries + # the generation's principal beside the disposition — the same envelope + # the plain ``save_message`` shape writes. The accepted SSE payload + # asserted above deliberately does NOT grow the key: it is audit + # metadata, and every listener-facing projection stays public-safe. + assert json.loads(attachment_saves[0]["meta"]) == { + "effect_status": "unknown", + "preview": descriptor, + "acting_principal": "owner", + } + tool_row = next(row for row in store.snapshot() if row["role"] == "tool") + assert tool_row["_event_id"] == accepted["_event_id"] + assert session.messages[-2].meta.event_id == accepted["_event_id"] + + +@pytest.mark.parametrize("hook_shape", ["missing", "none", "raises"]) +def test_tool_turn_unsupported_hook_emits_exceptional_history_repair( + tmp_db: Any, + hook_shape: str, +) -> None: + """An older/custom UI keeps repair semantics without routine fan-out.""" + + session = _ready_session() + store = _ConversationStore() + _rows, token = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + if hook_shape == "missing": + patcher = patch.object(session.ui, "on_tool_turn_accepted", None) + elif hook_shape == "none": + patcher = patch.object(session.ui, "on_tool_turn_accepted", return_value=None) + else: + patcher = patch.object( + session.ui, + "on_tool_turn_accepted", + side_effect=RuntimeError("old UI"), + ) + with ( + patcher, + _send_environment( + session, + [make_result("", tool_calls=[_TOOL_CALL]), make_result("done")], + store, + MagicMock(return_value=([(_TOOL_CALL["id"], "result")], None)), + ), + ): + session.send("run") + + events = _drain_listener(listener) + repairs = [event for event in events if event.get("type") == "history_resync"] + assert len(repairs) == 1 + assert repairs[0]["reason"] == "tool_turn_accepted" + assert not any(event.get("accepted") is True for event in events) + tool_row = next(row for row in store.snapshot() if row["role"] == "tool") + assert tool_row["_event_id"] == repairs[0]["_event_id"] + + +def test_both_tool_persist_shapes_stamp_one_acting_principal(tmp_db: Any) -> None: + """One batch, two persist shapes, one audit identity. + + The executor answers the first call and drops the second, so the ordinary + result fold and the shared cancelled-result synthesizer each write an + accepted TOOL row under the same generation. If only the ordinary shape + carried the principal, a revocation sweep over tool rows would silently + miss every effect a cancelled or malformed batch left behind — exactly the + rows whose disposition is already least certain. + """ + + session = _ready_session(user_id="owner") + store = _ConversationStore() + answered = { + "id": "call-answered", + "type": "function", + "function": {"name": "read_only_probe", "arguments": "{}"}, + } + dropped = { + "id": "call-dropped", + "type": "function", + "function": {"name": "read_only_probe", "arguments": "{}"}, + } + + with _send_environment( + session, + [make_result("", tool_calls=[answered, dropped]), make_result("done")], + store, + MagicMock(return_value=([(answered["id"], "observed output")], None)), + ): + session.send("run both probes", acting_user_id="user-alice") + + assert json.loads(store.tool_meta["call-answered"]) == {"acting_principal": "user-alice"} + assert json.loads(store.tool_meta["call-dropped"]) == { + "effect_status": "unknown", + "acting_principal": "user-alice", + } + + +def test_unattributed_lane_tool_rows_omit_the_principal_key(tmp_db: Any) -> None: + """A CLI / wake / internal turn writes no key, not an empty string. + + Presence is the attribution signal an audit query reads; an empty string + would make every unattributed effect look like a row whose principal was + recorded and happened to be blank. + """ + + session = _ready_session() + store = _ConversationStore() + + with _send_environment( + session, + [make_result("", tool_calls=[_TOOL_CALL]), make_result("done")], + store, + MagicMock(return_value=([(_TOOL_CALL["id"], "result")], None)), + ): + session.send("run") + + assert store.tool_meta[_TOOL_CALL["id"]] is None + + +def test_tool_repair_stamps_exact_event_before_unrelated_concurrent_frame( + tmp_db: Any, +) -> None: + """A later ring event cannot advance the durable row past its repair cursor.""" + + session = _ready_session() + store = _ConversationStore() + original_resync = session.ui.on_history_resync + + def _resync_then_race(reason: str) -> int: + repair_id = original_resync(reason) + session.ui._enqueue({"type": "content", "text": "unrelated concurrent frame"}) + return repair_id + + with ( + patch.object(session.ui, "on_tool_turn_accepted", return_value=None), + patch.object(session.ui, "on_history_resync", side_effect=_resync_then_race), + _send_environment( + session, + [make_result("", tool_calls=[_TOOL_CALL]), make_result("done")], + store, + MagicMock(return_value=([(_TOOL_CALL["id"], "result")], None)), + ), + ): + session.send("run") + + events = [event for _event_id, event in session.ui._event_buffer] + repair = next(event for event in events if event.get("type") == "history_resync") + unrelated = next( + event + for event in events + if event.get("type") == "content" and event.get("text") == "unrelated concurrent frame" + ) + tool_row = next(row for row in store.snapshot() if row["role"] == "tool") + assert repair["_event_id"] < unrelated["_event_id"] + assert tool_row["_event_id"] == repair["_event_id"] + + +def test_duplicate_tool_ids_use_exceptional_repair_not_ambiguous_projection( + tmp_db: Any, +) -> None: + """Malformed duplicate ids fail before execution and close every occurrence.""" + + session = _ready_session() + store = _ConversationStore() + duplicate_calls = [ + { + "id": "duplicate-provider-id", + "type": "function", + "function": {"name": "first_probe", "arguments": '{"first": 1}'}, + }, + { + "id": "duplicate-provider-id", + "type": "function", + "function": {"name": "second_probe", "arguments": '{"second": 2}'}, + }, + ] + _rows, token = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + execute_tools = MagicMock() + + with ( + _send_environment( + session, + [make_result("", tool_calls=duplicate_calls)], + store, + execute_tools, + ), + pytest.raises(RuntimeError, match="duplicate tool call ids"), + ): + session.send("run duplicates") + + execute_tools.assert_not_called() + assert session._cancelled_tool_results == {} + assert session._tool_error_flags == {} + assert session._tool_status == {} + assert session._tool_previews == {} + + live_rows = dicts_from_turns(session.messages) + assert [row["role"] for row in live_rows] == ["user", "assistant", "tool", "tool"] + assert [call["function"]["name"] for call in live_rows[1]["tool_calls"]] == [ + "first_probe", + "second_probe", + ] + assert [call["function"]["arguments"] for call in live_rows[1]["tool_calls"]] == [ + '{"first": 1}', + '{"second": 2}', + ] + for row in live_rows[2:]: + assert row["tool_call_id"] == "duplicate-provider-id" + assert row["is_error"] is True + assert "rejected before execution" in row["content"] + assert row["_effect_status"] == EffectStatus.NONE.value + assert "_preview" not in row + + durable_rows = store.snapshot() + assert [row["role"] for row in durable_rows] == ["user", "assistant", "tool", "tool"] + assert [row["tool_name"] for row in durable_rows[2:]] == [ + "first_probe", + "second_probe", + ] + assert all(row.get("is_error") is True for row in durable_rows[2:]) + + events = _drain_listener(listener) + repairs = [event for event in events if event.get("type") == "history_resync"] + assert len(repairs) == 2 + assert {event["reason"] for event in repairs} == {"tool_turn_projection_ambiguous"} + assert not any(event.get("accepted") is True for event in events) + + +def test_top_level_tool_id_reuse_across_batches_stays_occurrence_local( + tmp_db: Any, +) -> None: + """A provider may reuse one non-empty id in later assistant batches.""" + + session = _ready_session() + store = _ConversationStore() + first_call = { + "id": "provider-reused-id", + "type": "function", + "function": {"name": "first_probe", "arguments": '{"turn": 1}'}, + } + second_call = { + "id": "provider-reused-id", + "type": "function", + "function": {"name": "second_probe", "arguments": '{"turn": 2}'}, + } + _rows, token = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + executed: list[tuple[str, str]] = [] + + def _execute_tools( + tool_calls: list[dict[str, Any]], + **_kwargs: Any, + ) -> tuple[list[tuple[str, str]], None]: + assert len(tool_calls) == 1 + call = tool_calls[0] + call_id = call["id"] + name = call["function"]["name"] + executed.append((call_id, name)) + if name == "first_probe": + session._tool_error_flags[call_id] = True + session._tool_status[call_id] = EffectStatus.UNKNOWN + return [(call_id, "first guarded output")], None + return [(call_id, "second guarded output")], None + + with _send_environment( + session, + [ + make_result("", tool_calls=[first_call]), + make_result("", tool_calls=[second_call]), + make_result("done"), + ], + store, + _execute_tools, + ): + session.send("reuse one provider id in later batches") + + assert executed == [ + ("provider-reused-id", "first_probe"), + ("provider-reused-id", "second_probe"), + ] + live_rows = dicts_from_turns(session.messages) + assert [row["role"] for row in live_rows] == [ + "user", + "assistant", + "tool", + "assistant", + "tool", + "assistant", + ] + assert [live_rows[index]["tool_calls"][0]["function"]["name"] for index in (1, 3)] == [ + "first_probe", + "second_probe", + ] + assert [live_rows[index]["content"] for index in (2, 4)] == [ + "first guarded output", + "second guarded output", + ] + assert live_rows[2]["is_error"] is True + assert live_rows[2]["_effect_status"] == EffectStatus.UNKNOWN.value + assert live_rows[4].get("is_error") is not True + assert "_effect_status" not in live_rows[4] + + durable_rows = store.snapshot() + assert [row["role"] for row in durable_rows] == [ + "user", + "assistant", + "tool", + "assistant", + "tool", + "assistant", + ] + assert [durable_rows[index]["tool_name"] for index in (2, 4)] == [ + "first_probe", + "second_probe", + ] + assert [durable_rows[index]["content"] for index in (2, 4)] == [ + "first guarded output", + "second guarded output", + ] + assert durable_rows[2]["is_error"] is True + assert durable_rows[4].get("is_error") is not True + + accepted = [ + event + for event in _drain_listener(listener) + if event.get("type") == "tool_result" and event.get("accepted") is True + ] + assert [ + (event["name"], event["output"], bool(event.get("is_error"))) for event in accepted + ] == [ + ("first_probe", "first guarded output", True), + ("second_probe", "second guarded output", False), + ] + assert accepted[0]["effect_status"] == EffectStatus.UNKNOWN.value + assert "effect_status" not in accepted[1] + assert session._cancelled_tool_results == {} + assert session._tool_error_flags == {} + assert session._tool_status == {} + assert session._tool_previews == {} + + +def test_structured_tool_acceptance_uses_one_newline_scalar_without_inline_bytes( + tmp_db: Any, +) -> None: + """Pending, accepted, and durable projections share one safe text value.""" + + session = _ready_session() + store = _ConversationStore() + image_call = { + "id": "call-image-projection", + "type": "function", + "function": {"name": "read_file", "arguments": "{}"}, + } + _rows, token = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + saved: list[dict[str, Any]] = [] + + def _save_tool_with_attachments( + ws_id: str, + content: str, + tool_name: str, + tool_call_id: str, + attachments: Any, + **kwargs: Any, + ) -> int: + saved.append( + { + "content": content, + "attachments": tuple(attachments), + "event_id": kwargs.get("event_id"), + } + ) + return store( + ws_id, + "tool", + content, + tool_name, + tool_call_id=tool_call_id, + **kwargs, + ) + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, Any]], None]: + session.ui.on_tool_result( + image_call["id"], + "read_file", + "image (5 bytes)", + ) + return [ + ( + image_call["id"], + [ + {"type": "text", "text": "first guarded part"}, + {"type": "text", "text": "second guarded part"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,aGVsbG8="}, + }, + ], + ) + ], None + + with ( + _send_environment( + session, + [make_result("", tool_calls=[image_call]), make_result("done")], + store, + _execute_tools, + ), + patch( + "turnstone.core.session.save_tool_message_with_attachments", + side_effect=_save_tool_with_attachments, + ), + ): + session.send("read it") + + events = _drain_listener(listener) + accepted = next(event for event in events if event.get("accepted") is True) + expected = "first guarded part\nsecond guarded part" + assert accepted["output"] == expected + assert "data:image" not in json.dumps(accepted) + assert saved[0]["content"] == expected + assert saved[0]["event_id"] == accepted["_event_id"] + assert len(saved[0]["attachments"]) == 1 + assert saved[0]["attachments"][0].content == b"hello" + + tool_message = next( + message for message in dicts_from_turns(session.messages) if message["role"] == "tool" + ) + assert tool_message["content"] == [ + {"type": "text", "text": "first guarded part"}, + {"type": "text", "text": "second guarded part"}, + { + "type": "image", + "attachment_id": saved[0]["attachments"][0].attachment_id, + }, + ] + assert "data:image" not in json.dumps(tool_message) + assert project_history_messages([tool_message], False)[0]["content"] == expected + + +def test_frozen_old_storage_load_overlays_user_admitted_during_the_read( + tmp_db: Any, +) -> None: + """A reader with an old DB snapshot still returns the accepted user row.""" + + session = _ready_session(user_id="owner") + session.bind_acting_user("bob") + store = _ConversationStore() + _rows, token_0 = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token_0) + assert registration is not None + listener = registration[0] + load_entered = threading.Event() + release_load = threading.Event() + capture_result: list[tuple[list[dict[str, Any]], str]] = [] + + def _frozen_load(overscan: int = 0) -> list[dict[str, Any]]: + frozen = store.snapshot() + load_entered.set() + assert release_load.wait(5), "test did not release frozen user history load" + return frozen + + reader = threading.Thread( + target=lambda: capture_result.append(session.capture_history_handoff(_frozen_load)), + daemon=True, + ) + reader.start() + assert load_entered.wait(5), "history did not freeze its old DB snapshot" + + model_entered = threading.Event() + release_model = threading.Event() + + def _gated_results() -> Iterator[Any]: + model_entered.set() + assert release_model.wait(5), "test did not release model stream" + yield make_result("answer") + + with _send_environment( + session, + _gated_results(), # type: ignore[arg-type] + store, + MagicMock(return_value=([], None)), + ): + sender, send_errors = _start_send(session, "arrived during frozen load") + event = listener.get(timeout=5) + assert event["type"] == "user_turn" + assert event["content"] == "arrived during frozen load" + # Persistence shares the visibility lane and cannot slip into the DB + # snapshot while the reader is frozen. + assert store.user_entered.wait(0.05) is False + + release_load.set() + reader.join(5) + assert not reader.is_alive() + rows_during, token_during = capture_result[0] + users = _user_rows(rows_during) + assert len(users) == 1 + assert users[0]["content"] == "arrived during frozen load" + assert users[0]["_sender"] == "bob" + assert token_during != token_0 + + assert store.user_entered.wait(5), "user persistence did not resume after reader" + assert model_entered.wait(5), "model did not start after user persistence" + release_model.set() + sender.join(5) + + assert not sender.is_alive() + assert send_errors == [] + + +def test_user_turn_hook_failure_emits_one_exceptional_history_resync(tmp_db: Any) -> None: + """A custom/older UI missing the typed projection keeps repair semantics.""" + + session = _ready_session(user_id="owner") + _rows, token = session.capture_history_handoff(lambda _overscan: []) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + with patch.object(session.ui, "on_user_turn", side_effect=RuntimeError("old UI")): + session._append_user_turn( + "fallback row", + (), + sender_user_id="bob", + client_send_ids=("fallback-send",), + ) + + events = _drain_listener(listener) + assert [event["type"] for event in events] == ["history_resync"] + assert events[0]["reason"] == "user_turn_accepted" + assert session.messages[-1].meta.event_id == events[0]["_event_id"] + + +@pytest.mark.parametrize("hook_shape", ["missing", "none"]) +def test_user_turn_unsupported_hook_emits_one_exceptional_history_resync( + tmp_db: Any, + hook_shape: str, +) -> None: + """A missing or non-publishing adapter cannot silently claim delivery.""" + + session = _ready_session(user_id="owner") + _rows, token = session.capture_history_handoff(lambda _overscan: []) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + if hook_shape == "missing": + patcher = patch.object(session.ui, "on_user_turn", None) + else: + patcher = patch.object(session.ui, "on_user_turn", return_value=None) + with patcher: + session._append_user_turn( + "fallback row", + (), + sender_user_id="bob", + client_send_ids=("fallback-send",), + ) + + events = _drain_listener(listener) + assert [event["type"] for event in events] == ["history_resync"] + assert events[0]["reason"] == "user_turn_accepted" + assert session.messages[-1].meta.event_id == events[0]["_event_id"] + + +def test_repeated_client_send_id_keeps_distinct_user_turn_event_ids(tmp_db: Any) -> None: + """Correlation is per-send UI matching, never storage idempotency.""" + + session = _ready_session(user_id="owner") + _rows, token = session.capture_history_handoff(lambda _overscan: []) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + session._append_user_turn( + "first", + (), + sender_user_id="bob", + client_send_ids=("reused-token",), + ) + session._append_user_turn( + "second", + (), + sender_user_id="bob", + client_send_ids=("reused-token",), + ) + + events = _drain_listener(listener) + user_events = [event for event in events if event["type"] == "user_turn"] + assert [event["content"] for event in user_events] == ["first", "second"] + assert [event["client_send_ids"] for event in user_events] == [ + ["reused-token"], + ["reused-token"], + ] + assert user_events[0]["_event_id"] != user_events[1]["_event_id"] + + +def test_ambiguous_user_commit_is_visible_once_and_blocks_a_causal_suffix( + tmp_db: Any, +) -> None: + """A lost user ACK is keyed, overlaid, and fail-stops the model and next send.""" + + session = _ready_session(user_id="owner") + session.bind_acting_user("bob") + store = _ConversationStore(ambiguous_user_ack=True) + execute_tools = MagicMock(return_value=([], None)) + + with _send_environment( + session, + [make_result("must never run")], + store, + execute_tools, + ): + with pytest.raises(session_module.ConversationPersistenceError): + session.send("ambiguous user row") + assert session.has_unresolved_conversation_persistence() is True + + # Reconciliation happens before a successor can append its own row. + with pytest.raises(session_module.ConversationPersistenceError): + session.send("must not become a suffix") + + durable_users = _user_rows(store.snapshot()) + assert len(durable_users) == 1 + assert isinstance(durable_users[0].get("_commit_key"), str) + assert execute_tools.call_count == 0 + + # A history read sees the stable keyed row, reconciles the lost ACK, + # and returns one sender-attributed logical row, never storage+journal + # twins. + rows, _token = session.capture_history_handoff(store.snapshot) + + users = _user_rows(rows) + assert len(users) == 1 + assert users[0]["content"] == "ambiguous user row" + assert users[0]["_sender"] == "bob" + # The shared-participant SYSTEM row was accepted in the same generation + # batch after the USER. The USER's lost ACK fail-stopped that suffix, so + # history overlays it and a later reconciliation persists it in FIFO order. + assert any(row.get("role") == "system" and row.get("_pending_durability") for row in rows) + assert session.has_unresolved_conversation_persistence() is True + with patch("turnstone.core.session.save_message", side_effect=store): + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is True + assert session.has_unresolved_conversation_persistence() is False + + +def test_atomic_attachment_user_row_visibility_proves_the_complete_commit(tmp_db: Any) -> None: + """A keyed attachment row is a durable witness for blobs, refs, and list. + + Shared-workstream actor identity does not replace staged-byte ownership: + Bob's authenticated turn consumes the upload filed under durable owner + ``owner``. A lost ACK keeps the immutable bytes in the journal closure, + while idempotent retries retain the blob reference only once. + """ + + from turnstone.core.attachment_buffer import get_attachment_buffer + + session = _ready_session(user_id="owner") + session.bind_acting_user("bob") + buffer = get_attachment_buffer() + buffer.clear() + staged = buffer.stage( + ws_id=session._ws_id, + user_id="owner", + filename="evidence.txt", + mime_type="text/plain", + kind="text", + content=b"evidence", + ) + attachment = Attachment( + attachment_id=staged.attachment_id, + filename=staged.filename, + mime_type=staged.mime_type, + kind=staged.kind, + content=staged.content, + ) + + real_atomic_save = session_module.save_user_message_with_attachments + + def _commit_but_hide_ack(*args: Any, **kwargs: Any) -> int: + real_atomic_save(*args, **kwargs) + return 0 + + with ( + patch( + "turnstone.core.session.save_user_message_with_attachments", + side_effect=_commit_but_hide_ack, + ), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._append_user_turn("inspect this", (attachment,), send_id="bob-send") + + assert session.has_unresolved_conversation_persistence() is True + assert buffer.get(staged.attachment_id, ws_id=session._ws_id, user_id="owner") is None + assert buffer.get(staged.attachment_id, ws_id=session._ws_id, user_id="bob") is None + # The row and every attachment side effect share one backend transaction. + # Seeing the keyed row can therefore reconcile a lost facade ACK. + from turnstone.core.storage import get_storage + + storage = get_storage() + rows, _token = session.capture_history_handoff( + lambda _overscan: storage.load_messages(session._ws_id, repair=False) + ) + users = _user_rows(rows) + assert len(users) == 1 + assert users[0]["content"][0] == {"type": "text", "text": "inspect this"} + assert users[0]["_attachments_meta"] == [ + { + "attachment_id": staged.attachment_id, + "kind": "text", + "filename": "evidence.txt", + "mime_type": "text/plain", + "size_bytes": len(b"evidence"), + } + ] + assert users[0]["_sender"] == "bob" + assert storage.count_messages(session._ws_id) == 1 + stored_attachment = storage.get_attachment(staged.attachment_id) + assert stored_attachment is not None + assert stored_attachment["refcount"] == 1 + assert session.has_unresolved_conversation_persistence() is False + + +def test_attachment_side_effect_retry_does_not_double_increment_blob_refcounts( + tmp_db: Any, +) -> None: + """An ambiguous attachment ACK may not replay non-idempotent increments. + + Model a backend that commits the complete row/blob/ref-list transaction and + then loses its acknowledgement. Retrying the keyed operation must resolve + the existing row without incrementing either blob a second time. + """ + + from turnstone.core.storage import get_storage + + session = _ready_session(user_id="owner") + attachments = ( + Attachment( + attachment_id="1" * 64, + filename="one.txt", + mime_type="text/plain", + kind="text", + content=b"one", + ), + Attachment( + attachment_id="2" * 64, + filename="two.txt", + mime_type="text/plain", + kind="text", + content=b"two", + ), + ) + real_atomic_save = session_module.save_user_message_with_attachments + lost_ack = False + + def _save_then_lose_ack(*args: Any, **kwargs: Any) -> int: + nonlocal lost_ack + row_id = real_atomic_save(*args, **kwargs) + if not lost_ack: + lost_ack = True + raise RuntimeError("atomic attachment commit acknowledgement lost") + return row_id + + with ( + patch( + "turnstone.core.session.save_user_message_with_attachments", + side_effect=_save_then_lose_ack, + ), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._append_user_turn("two attachments", attachments) + + assert lost_ack is True + assert session.has_unresolved_conversation_persistence() is True + storage = get_storage() + assert storage is not None + session.capture_history_handoff( + lambda _overscan: storage.load_messages(session.ws_id, repair=False) + ) + assert session.has_unresolved_conversation_persistence() is False + assert storage.get_attachment(attachments[0].attachment_id)["refcount"] == 1 + assert storage.get_attachment(attachments[1].attachment_id)["refcount"] == 1 + + +def test_direct_same_hash_attachment_does_not_consume_web_staging(tmp_db: Any) -> None: + """Only a web send_id may transfer matching staged-upload ownership.""" + + from turnstone.core.attachment_buffer import get_attachment_buffer + + session = _ready_session(user_id="owner") + buffer = get_attachment_buffer() + buffer.clear() + staged = buffer.stage( + ws_id=session.ws_id, + user_id="owner", + filename="staged.txt", + mime_type="text/plain", + kind="text", + content=b"same bytes", + ) + direct = Attachment( + attachment_id=staged.attachment_id, + filename="direct.txt", + mime_type="text/plain", + kind="text", + content=b"same bytes", + ) + + session._append_user_turn("direct input", (direct,)) + + assert buffer.get(staged.attachment_id, ws_id=session.ws_id, user_id="owner") is not None + + +def test_handoff_revision_never_aba_after_pending_rows_are_acked(tmp_db: Any) -> None: + """Removing pending rows must not make a later history token reusable.""" + + session = _ready_session() + store = _ConversationStore() + _, token_0 = session.capture_history_handoff(store.snapshot) + + with _send_environment( + session, + [make_result("answer one")], + store, + MagicMock(return_value=([], None)), + ): + session.send("request one") + _, token_1 = session.capture_history_handoff(store.snapshot) + + with _send_environment( + session, + [make_result("answer two")], + store, + MagicMock(return_value=([], None)), + ): + session.send("request two") + _, token_2 = session.capture_history_handoff(store.snapshot) + + assert len({token_0, token_1, token_2}) == 3 + assert session.has_unresolved_conversation_persistence() is False + + +def test_matching_handoff_token_preserves_cursor_zero_for_tool_only_replay(tmp_db: Any) -> None: + """Cursor ``0`` is a real boundary, not the absence of a cursor. + + ``tool_pending`` models the first reconstructing event of a tool-call-only + assistant turn. The matched handoff token must select numeric replay so + that event 1 is not replaced by a content snapshot that cannot encode the + tool call. + """ + + session = _ready_session() + store = _ConversationStore() + _, token = session.capture_history_handoff(store.snapshot) + session.ui._enqueue( + { + "type": "tool_pending", + "call_id": _TOOL_CALL["id"], + "name": _TOOL_CALL["function"]["name"], + } + ) + + with patch.object( + session.ui, + "register_listener_with_replay", + wraps=session.ui.register_listener_with_replay, + ) as replay: + registration = session.register_listener_for_history_handoff( + token, + last_event_id=0, + ) + + assert registration is not None + replay.assert_called_once() + assert replay.call_args.args[0] == 0 + + +def test_tool_only_persistence_failure_resyncs_a_previously_matched_empty_snapshot( + tmp_db: Any, +) -> None: + """A listener winning just before a failed tool-only commit must refetch. + + Content/reasoning snapshots cannot encode a tool call. On the healthy + path, later ``tool_pending``/``tool_info`` events reconstruct it; on a + persistence failure tools never start, so the failure path itself must + invalidate the listener's otherwise-successful bootstrap. + """ + + session = _ready_session() + store = _ConversationStore(block_assistant=True, ambiguous_assistant_ack=True) + _, token = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener, _replay, status, _lost, _earliest, snapshot = registration + assert status == "fresh" + assert snapshot["content"] == "" + assert snapshot["reasoning"] == "" + + execute_tools = MagicMock(return_value=([], None)) + with _send_environment( + session, + [make_result("", tool_calls=[_TOOL_CALL])], + store, + execute_tools, + ): + sender, send_errors = _start_send(session) + assert store.assistant_entered.wait(5), "assistant persistence never started" + store.release_assistant.set() + sender.join(5) + + assert not sender.is_alive() + assert len(send_errors) == 1 + assert isinstance(send_errors[0], session_module.ConversationPersistenceError) + execute_tools.assert_not_called() + delivered = [] + while not listener.empty(): + delivered.append(listener.get_nowait()) + assert "history_resync" in {event.get("type") for event in delivered} + + +def test_ambiguous_assistant_commit_is_visible_once_and_fail_stops_tools_and_queue( + tmp_db: Any, +) -> None: + """Committed-but-unacknowledged is deduped by key and poisons the turn. + + The generic failure cleanup historically drained queued user text. A + conversation persistence failure needs a dedicated arm: tools do not run, + the queued message stays queued, and an unstarted TOOL row completes the + accepted assistant block while the ledger awaits ordered reconciliation. + """ + + session = _ready_session() + store = _ConversationStore(block_assistant=True, ambiguous_assistant_ack=True) + execute_tools = MagicMock(return_value=([], None)) + result = make_result("", tool_calls=[_TOOL_CALL]) + + with _send_environment(session, [result], store, execute_tools): + sender, send_errors = _start_send(session) + assert store.assistant_entered.wait(5), "assistant persistence never started" + session.queue_message("do not drain me", queue_msg_id="queued-after-failure") + store.release_assistant.set() + sender.join(5) + # Keep the storage fake installed: the journal's idempotent retry + # closure resolves ``save_message`` at call time. + assert session.prepare_soft_close() is False + assert session._publication_shutdown is False + + assert not sender.is_alive() + assert len(send_errors) == 1 + assert isinstance(send_errors[0], session_module.ConversationPersistenceError) + execute_tools.assert_not_called() + assert list(session._queued_messages) == ["queued-after-failure"] + assert session.has_unresolved_conversation_persistence() is True + + rows, _token = session.capture_history_handoff(store.snapshot) + assistants = _assistant_rows(rows) + assert len(assistants) == 1 + assert assistants[0].get("tool_calls") == [_TOOL_CALL] + assert isinstance(assistants[0].get("_commit_key"), str) + tools = [row for row in rows if row.get("role") == "tool"] + assert len(tools) == 1 + assert tools[0].get("tool_call_id") == _TOOL_CALL["id"] + # History proves the ambiguous assistant durable, but the synthesized + # completion still has its own journal boundary and remains fail-stop until + # the normal due reconciler persists it. + assert session.has_unresolved_conversation_persistence() is True + with patch("turnstone.core.session.save_message", side_effect=store): + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is True + assert session.has_unresolved_conversation_persistence() is False + + # Content, event id, and tool shape can all legitimately repeat. Only the + # durable commit identity may collapse storage + ledger representations. + distinct_same_payload = dict(assistants[0]) + distinct_same_payload["_commit_key"] = "distinct-logical-assistant-row" + store.append_seed_row(distinct_same_payload) + rows_with_distinct_twin, _token = session.capture_history_handoff(store.snapshot) + assert len(_assistant_rows(rows_with_distinct_twin)) == 2 + + +def test_foreign_actor_cannot_drain_retained_queue_after_persistence_recovery( + tmp_db: Any, +) -> None: + """A recovery turn cannot borrow another participant's retained text. + + The #981 failure arm deliberately leaves interjections queued behind an + unresolved assistant commit. Their admitting principal must survive that + pause: once storage recovers, a different participant may not turn the + retained text into a user row attributed to their own identity and run it + under their credentials. The original participant can still reconcile + and drain it on their next turn. + """ + + session = _ready_session(user_id="owner") + store = _ConversationStore(block_assistant=True, ambiguous_assistant_ack=True) + session.bind_acting_user("alice") + + with _send_environment( + session, + [make_result("alice's accepted answer")], + store, + MagicMock(return_value=([], None)), + ): + sender, send_errors = _start_send(session, "alice starts") + assert store.assistant_entered.wait(5), "assistant persistence never started" + session.queue_message( + "alice retained interjection", + queue_msg_id="alice-retained", + interjector_user_id="alice", + ) + store.release_assistant.set() + sender.join(5) + + assert not sender.is_alive() + assert len(send_errors) == 1 + assert isinstance(send_errors[0], session_module.ConversationPersistenceError) + assert list(session._queued_messages) == ["alice-retained"] + ws = Workstream(id=session._ws_id, name="persistence-recovery") + ws.session = session + ws.ui = session.ui + + # The durable row is now recoverable. That does not transfer ownership of + # the already-admitted queue item to the participant who happens to send + # next. Exercise the actual route dispatch seam: its fresh-worker + # admission and slot claim are one workstream-lock transaction, so the + # refusal happens before the worker can bind Bob or append his user row. + store.ambiguous_assistant_ack = False + bob_ok, bob_outcome = _dispatch_as( + ws, + session, + actor="bob", + message="bob must wait", + ) + + assert bob_ok is False + assert bob_outcome == {"rejected": "cross_user_interjection"} + assert ws.worker_thread is None + assert list(session._queued_messages) == ["alice-retained"] + assert not any( + row.get("role") == "user" and row.get("content") == "bob must wait" + for row in dicts_from_turns(session.messages) + ) + + with _send_environment( + session, + [make_result("alice resumes"), make_result("alice finishes")], + store, + MagicMock(return_value=([], None)), + ): + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is True + alice_ok, alice_outcome = _dispatch_as( + ws, + session, + actor="alice", + message="alice recovery turn", + ) + assert alice_ok is True + assert alice_outcome == {} + worker = ws.worker_thread + assert worker is not None + worker.join(5) + + assert not worker.is_alive() + + assert session.has_unresolved_conversation_persistence() is False + assert session._queued_messages == {} + retained_rows = [ + row + for row in dicts_from_turns(session.messages) + if row.get("role") == "user" and row.get("content") == "alice retained interjection" + ] + assert len(retained_rows) == 1 + assert retained_rows[0].get("_sender") == "alice" + + +def test_foreign_actor_cannot_consume_owned_queue_at_advisory_drain(tmp_db: Any) -> None: + """The advisory seam partitions: a foreign-owned row is structurally + retained and produces NO spec (an advisory is a model-facing turn — + announcing another participant's pending text would leak their activity + into the acting user's transcript). No raise: the pop-side ownership + assert is deleted; retention is the enforcement.""" + + session = _ready_session(user_id="owner") + session.bind_acting_user("alice") + session.queue_message( + "alice advisory", + queue_msg_id="alice-advisory", + interjector_user_id="alice", + ) + session.bind_acting_user("bob") + + assert session._collect_advisories(None, "some_tool", True) == [] + assert list(session._queued_messages) == ["alice-advisory"] + + session.bind_acting_user("alice") + specs = session._collect_advisories(None, "some_tool", True) + assert [(source, meta.get("message")) for source, _content, meta in specs] == [ + ("user_interjection", "alice advisory") + ] + assert session._queued_messages == {} + + +def test_mixed_ownership_advisory_drain_emits_own_and_retains_foreign(tmp_db: Any) -> None: + """Round-5 review pin: on a mixed queue the acting user's row becomes an + advisory spec while the other participant's row stays queued.""" + + session = _ready_session(user_id="owner") + session.bind_acting_user("alice") + session.queue_message( + "alice advisory", + queue_msg_id="alice-advisory", + interjector_user_id="alice", + ) + session.bind_acting_user("bob") + session.queue_message( + "bob advisory", + queue_msg_id="bob-advisory", + interjector_user_id="bob", + ) + + specs = session._collect_advisories(None, "some_tool", True) + assert [(source, meta.get("message")) for source, _content, meta in specs] == [ + ("user_interjection", "bob advisory") + ] + assert list(session._queued_messages) == ["alice-advisory"] + + +def test_foreign_actor_cannot_consume_owned_queue_at_user_flush(tmp_db: Any) -> None: + """No-tool/error flushes preserve both ownership and sender attribution. + + Partitioned: under a foreign actor the flush pops nothing (no raise, no + append); the row waits for its owner, whose flush stamps their sender.""" + + session = _ready_session(user_id="owner") + session.bind_acting_user("alice") + session.queue_message( + "alice flush", + queue_msg_id="alice-flush", + interjector_user_id="alice", + ) + session.bind_acting_user("bob") + message_count = len(session.messages) + + assert session._flush_queued_messages() is False + + assert len(session.messages) == message_count + assert list(session._queued_messages) == ["alice-flush"] + session.bind_acting_user("alice") + assert session._flush_queued_messages() is True + assert session._queued_messages == {} + flushed = [ + row + for row in dicts_from_turns(session.messages) + if row.get("role") == "user" and row.get("content") == "alice flush" + ] + assert len(flushed) == 1 + assert flushed[0].get("_sender") == "alice" + + +def test_mixed_ownership_flush_appends_own_and_retains_foreign(tmp_db: Any) -> None: + """Round-5 review pin (the all-or-nothing defect): one flush on a mixed + queue appends the acting user's row as a user turn AND leaves the other + participant's row queued — no raise, nothing destroyed.""" + + session = _ready_session(user_id="owner") + session.bind_acting_user("alice") + session.queue_message( + "alice flush", + queue_msg_id="alice-flush", + interjector_user_id="alice", + ) + session.bind_acting_user("bob") + session.queue_message( + "bob flush", + queue_msg_id="bob-flush", + interjector_user_id="bob", + ) + + assert session._flush_queued_messages() is True + assert list(session._queued_messages) == ["alice-flush"] + flushed = [ + row + for row in dicts_from_turns(session.messages) + if row.get("role") == "user" and row.get("content") == "bob flush" + ] + assert len(flushed) == 1 + assert flushed[0].get("_sender") == "bob" + assert not any( + "alice flush" in str(row.get("content")) for row in dicts_from_turns(session.messages) + ) + + +def test_empty_principal_queue_items_keep_internal_drain_compatibility(tmp_db: Any) -> None: + """Legacy/internal queue producers remain intentionally unscoped.""" + + session = _ready_session(user_id="owner") + session.queue_message( + "internal advisory", + queue_msg_id="internal-advisory", + interjector_user_id="", + turn_principal_id="", + ) + session.bind_acting_user("bob") + specs = session._collect_advisories(None, "some_tool", True) + assert [(source, meta.get("message")) for source, _content, meta in specs] == [ + ("user_interjection", "internal advisory") + ] + + session.queue_message( + "internal flush", + queue_msg_id="internal-flush", + interjector_user_id="", + turn_principal_id="", + ) + session.bind_acting_user("alice") + assert session._flush_queued_messages() is True + assert session._queued_messages == {} + + +def test_history_load_failure_rejects_pending_only_handoff_until_durable_prefix_loads( + tmp_db: Any, +) -> None: + """A pending journal suffix cannot authorize incomplete history. + + If the durable-prefix read fails, merging the resident pending row over an + empty list is useful only as an internal fail-visible floor. Returning it + with 200 + a handoff token would let browsers replace older transcript + rows, clear their repair latch, and reconnect against incomplete truth. + The route must return a bounded 503; a later successful read may expose the + pending row exactly once with a valid handoff token. + """ + + from tests._coord_test_helpers import _build_mgr, _fake_registry + from tests.test_coordinator_endpoints import _COORD_HEADERS, _make_client + from turnstone.core.storage import get_storage + + storage = get_storage() + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + session = _ready_session(ws_id=ws.id, user_id="user-1") + ws.session = session + ws.ui = session.ui + store = _ConversationStore(ambiguous_assistant_ack=True) + + with ( + _send_environment( + session, + [make_result("pending while durable history is unavailable")], + store, + MagicMock(return_value=([], None)), + ), + pytest.raises(session_module.ConversationPersistenceError), + ): + session.send("request during outage") + + assert session.has_unresolved_conversation_persistence() is True + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + with patch.object(storage, "load_messages", side_effect=RuntimeError("storage unavailable")): + response = client.get( + f"/v1/api/workstreams/{ws.id}/history", + headers=_COORD_HEADERS, + ) + + assert response.status_code == 503 + assert response.json() == {"error": "History temporarily unavailable"} + assert session.has_unresolved_conversation_persistence() is True + + response = client.get( + f"/v1/api/workstreams/{ws.id}/history", + headers=_COORD_HEADERS, + ) + assert response.status_code == 200 + body = response.json() + assert isinstance(body["handoff_token"], str) and body["handoff_token"] + assistants = _assistant_rows(body["messages"]) + assert [row.get("content") for row in assistants] == [ + "pending while durable history is unavailable" + ] + assert "_commit_key" not in assistants[0] + assert "_pending_durability" not in assistants[0] + assert session.has_unresolved_conversation_persistence() is True + + +def test_failed_durability_head_poisons_an_already_admitted_successor(tmp_db: Any) -> None: + """Ticket N+1 may not persist after ticket N loses conversation durability. + + Deliberate pin update: the head failure is injected through the real + journal classifier (a journaled row whose save fails transiently), which + latches the poison before raising — production's only remaining + ConversationPersistenceError shape now that the commit lane's re-latch + safety net for hand-rolled unlatched errors is deleted. + """ + + session = _ready_session() + first_entered = threading.Event() + release_first = threading.Event() + second_admitted = threading.Event() + second_persisted = threading.Event() + errors: list[BaseException] = [] + + def _failing_save(*_args: Any, **_kwargs: Any) -> int: + return 0 + + def _first_persist() -> None: + first_entered.set() + assert release_first.wait(5), "test did not release failed durability head" + with session._history_handoff_lock: + pending = session._journal_conversation_row_locked( + commit_key="head-ambiguous-row", + message={"role": "assistant", "content": "ambiguous"}, + persist=lambda: _failing_save(), + event_id=None, + ) + session._persist_pending_conversation_commit(pending) + + def _first_commit(durable: list[Callable[[], None]]) -> None: + durable.append(_first_persist) + + def _second_commit(durable: list[Callable[[], None]]) -> None: + second_admitted.set() + durable.append(second_persisted.set) + + def _run(commit: Callable[[list[Callable[[], None]]], None]) -> None: + try: + session._commit_for_generation(0, commit) + except BaseException as exc: # diagnostic capture from each ticket owner + errors.append(exc) + + first = threading.Thread(target=_run, args=(_first_commit,), daemon=True) + second = threading.Thread(target=_run, args=(_second_commit,), daemon=True) + first.start() + assert first_entered.wait(5), "first durability ticket never started" + second.start() + assert second_admitted.wait(5), "successor was not admitted behind the head" + + release_first.set() + first.join(5) + second.join(5) + + assert not first.is_alive() + assert not second.is_alive() + assert len(errors) == 2 + assert all(isinstance(exc, session_module.ConversationPersistenceError) for exc in errors) + assert second_persisted.is_set() is False + + # Poison must settle skipped tickets so terminal durability drain cannot + # wait forever on a ticket that is intentionally never executed. + drained = threading.Event() + drain = threading.Thread( + target=lambda: (session.shutdown_publication_and_drain_durability(), drained.set()), + daemon=True, + ) + drain.start() + assert drained.wait(5), "poisoned durability lane did not drain" + drain.join(1) + + +def _mark_session_unresolved(ws: Any) -> None: + assert ws.session is not None + ws.session.has_unresolved_conversation_persistence = lambda: True + + +def test_manual_soft_close_refuses_to_discard_an_unresolved_ledger() -> None: + mgr, adapter, storage = _make_manager() + ws = mgr.create(user_id="u1") + _mark_session_unresolved(ws) + + assert mgr.close(ws.id) is False + assert mgr.get(ws.id) is ws + assert ws.id not in adapter.cleaned_up + assert (ws.id, "closed") not in storage.state_updates + assert adapter.events_of("closed") == [] + + +def test_idle_reaper_refuses_to_discard_an_unresolved_ledger() -> None: + mgr, adapter, storage = _make_manager() + ws = mgr.create(user_id="u1") + _mark_session_unresolved(ws) + ws.last_active = time.monotonic() - 100 + + assert ws.id not in mgr.close_idle(max_age_seconds=1) + assert mgr.get(ws.id) is ws + assert ws.id not in adapter.cleaned_up + assert (ws.id, "closed") not in storage.state_updates + + +def test_capacity_eviction_refuses_to_discard_an_unresolved_ledger() -> None: + mgr, adapter, _storage = _make_manager(max_active=1) + incumbent = mgr.create(user_id="u1") + _mark_session_unresolved(incumbent) + incumbent.last_active = time.monotonic() - 100 + + with pytest.raises(RuntimeError, match="All 1 slots are active"): + mgr.create(user_id="u2") + + assert mgr.get(incumbent.id) is incumbent + assert incumbent._closed is False + assert incumbent.id not in adapter.cleaned_up + assert mgr.eviction_count == 0 + + +def test_widened_window_renders_lost_ack_rows_in_position_never_at_tail() -> None: + """The overscan window contains every committed twin of a pending key. + + Two lost-ACK rows (durable, acknowledgement lost) with a tail bound + smaller than the pending count: the widened load reaches both twins, so + the merge acknowledges them in place — never re-appending an old row + after the newest messages, the out-of-order render the deleted storage + probe was chasing. + """ + session = _ready_session() + seen_overscan: list[int] = [] + + def _save_zero(*_args: Any, **_kwargs: Any) -> int: + return 0 + + keys: list[str] = [] + with ( + patch("turnstone.core.session.save_message", side_effect=_save_zero), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._append_system_turn("correction", "first lost ack") + keys.extend(session._pending_conversation_commits) + assert len(keys) == 1 + + durable_rows = [ + {"role": "system", "content": "first lost ack", "_commit_key": keys[0]}, + ] + + def _loader(overscan: int) -> list[dict[str, Any]]: + seen_overscan.append(overscan) + limit = 1 + return durable_rows[-(limit + overscan) :] + + merged, _token = session.capture_history_handoff(_loader) + + assert seen_overscan == [1] + assert [row.get("content") for row in merged] == ["first lost ack"] + assert session._pending_conversation_commits == {} + # A second capture is pure durable prefix — the row appears exactly once. + merged_again, _token2 = session.capture_history_handoff(_loader) + assert [row.get("content") for row in merged_again] == ["first lost ack"] + + +def test_conflicted_pending_row_renders_in_place_inside_widened_window() -> None: + """A same-key conflict stays fail-visible in its durable position. + + The deleted presence-probe suppressed a conflicted row whose twin lay + beyond the bare window; the widened window keeps the twin in view and the + merge replaces it in place with the journal's version, next to the + persistence banner an operator debugs from. + """ + session = _ready_session() + + from turnstone.core.storage import ConversationCommitConflictError + + with ( + patch( + "turnstone.core.session.save_message", + side_effect=ConversationCommitConflictError("different conversation commit"), + ), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._append_system_turn("correction", "journal version") + [conflict_key] = session._pending_conversation_commits + entry = session._pending_conversation_commits[conflict_key] + assert entry.ack_from_durable_row is False + assert session.conversation_persistence_status()["state"] == "conflict" + + durable_rows = [ + {"role": "system", "content": "durable divergent twin", "_commit_key": conflict_key}, + {"role": "user", "content": "legacy second-writer row"}, + ] + + def _loader(overscan: int) -> list[dict[str, Any]]: + limit = 1 + return durable_rows[-(limit + overscan) :] + + merged, _token = session.capture_history_handoff(_loader) + + assert [row.get("content") for row in merged] == [ + "journal version", + "legacy second-writer row", + ] + # Conflicts never self-acknowledge: the entry survives for the operator. + assert conflict_key in session._pending_conversation_commits + + +def test_capture_never_runs_the_loader_under_the_handoff_lock() -> None: + """Structural pin for the deleted in-lock storage probe. + + The loader is the only storage touchpoint in a capture; running it under + ``_history_handoff_lock`` would stall every row admission, publication, + and SSE registration behind a slow database. The overscan is sampled + first, the load runs unlocked, and the merge is pure in-memory work. + """ + session = _ready_session() + session._append_system_turn("correction", "pending row") + lock_free_during_load: list[bool] = [] + + def _loader(overscan: int) -> list[dict[str, Any]]: + del overscan + acquired: list[bool] = [] + + def _probe() -> None: + # From another thread: an RLock held by the capturing thread + # refuses a non-blocking acquire; a free lock grants it. + got = session._history_handoff_lock.acquire(blocking=False) + acquired.append(got) + if got: + session._history_handoff_lock.release() + + prober = threading.Thread(target=_probe, daemon=True) + prober.start() + prober.join(5) + lock_free_during_load.append(bool(acquired and acquired[0])) + return [] + + session.capture_history_handoff(_loader) + assert lock_free_during_load == [True] diff --git a/tests/test_history_handoff_js.py b/tests/test_history_handoff_js.py new file mode 100644 index 00000000..98d6fa03 --- /dev/null +++ b/tests/test_history_handoff_js.py @@ -0,0 +1,1333 @@ +"""Behavioral contracts for user-turn projection and handoff recovery JS.""" + +# Percent interpolation keeps the extracted JavaScript's many literal braces +# readable; converting these harnesses to ``str.format`` would require escaping +# every object/function body and make source-to-test review substantially harder. +# ruff: noqa: UP031 + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from tests._js_harness_helpers import extract_braced as _extract_braced +from tests._js_harness_helpers import strip_js_comments as _strip_comments + +_ROOT = Path(__file__).resolve().parent.parent +_INTERACTIVE = _ROOT / "turnstone/shared_static/interactive.js" +_COORDINATOR = _ROOT / "turnstone/console/static/coordinator/coordinator.js" +_SHARED_HANDOFF = _ROOT / "turnstone/shared_static/history_handoff.js" +_QUEUE = _ROOT / "turnstone/shared_static/composer_queue.js" +_TOOL_PROJECTION = _ROOT / "turnstone/shared_static/tool_projection.js" +_PY_SDK = _ROOT / "turnstone/sdk/server.py" +_PY_CHANNEL = _ROOT / "turnstone/channels/_sse.py" +_TS_SDK = _ROOT / "sdk/typescript/src/server.ts" + + +def _run_module(tmp_path: Path, source: str) -> None: + script = tmp_path / "contract.mjs" + script.write_text(source, encoding="utf-8") + try: + proc = subprocess.run( + ["node", str(script)], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + except FileNotFoundError: + pytest.skip("node binary not available on PATH") + assert proc.returncode == 0, f"stdout={proc.stdout!r} stderr={proc.stderr!r}" + + +# ``_extract_braced`` is the shared comment-and-string-aware brace walker +# from tests/_js_harness_helpers — one implementation for every suite, so +# the per-suite walkers cannot diverge on comment handling or bounds. + + +def _as_function(source: str, signature: str) -> str: + extracted = _extract_braced(source, signature).strip() + if extracted.startswith("function "): + return extracted + return "function " + extracted + + +def test_interactive_stale_backstop_waits_for_replay_tail_runtime( + tmp_path: Path, +) -> None: + """A replay prelude idle must not split its canonical backlog around history.""" + + source = _INTERACTIVE.read_text(encoding="utf-8") + begin_replay = _as_function(source, " _beginReplayQuiesce(token) {") + end_replay = _as_function(source, " _endReplayQuiesce(token) {") + defer_backstop = _as_function(source, " _deferStaleHistoryBackstop() {") + handle_event = _as_function(source, " handleEvent(evt) {") + + # These are the race owners that can change after a synthetic replay_ok + # idle queues the microtask but before the canonical backlog finishes. + for guard in ( + "queueMicrotask(() => {", + "this.wsId !== staleWs", + "staleToken !== this._historyLoadToken", + "!this._historyStale", + "this._replayQueue", + "this.busy", + "this.currentAssistantEl", + "this.currentReasoningEl", + "this._pendingTruncatedResync", + "this._truncatedFromCursor != null", + "this._resyncTimer != null", + "!this.el", + "!this.el.isConnected", + ): + assert guard in defer_backstop + + script = """ +function resetCompactionHolder() {} +function streamingRender(body, text) { body.textContent = text; } +function streamingRenderFinalize(body, text) { body.textContent = text; } +""" + script += "\n" + begin_replay + script += "\n" + end_replay + script += "\n" + defer_backstop + script += "\n" + handle_event + script += r""" + +const sentinel = "DUPE-runtime"; +let backstops = 0; +let releaseHeal = null; +const pane = { + wsId: "ws-1", + _historyLoadToken: 7, + _historyStale: true, + _replayQueue: null, + _staleBackstopMicrotaskPending: false, + _pendingTruncatedResync: false, + _truncatedFromCursor: null, + _resyncTimer: null, + busy: false, + currentAssistantEl: null, + currentAssistantBodyEl: null, + currentReasoningEl: null, + contentBuffer: "", + _cancelTimeout: null, + _forceTimeout: null, + _compaction: {}, + _streamHealth: { renderThrows: 0 }, + _actingUserId: null, + pendingApproval: false, + el: { isConnected: true }, + inputEl: { focus() {} }, + _host: { isFocused() { return false; } }, + assistantRows: [], + userRows: [], + setBusy(value) { this.busy = value; }, + _attachRetryToLastAssistant() {}, + _acceptUserTurn(evt) { this.userRows.push(evt.content); }, + removeThinkingIndicator() {}, + removeEmptyState() {}, + scrollToBottom() {}, + _newAssistantBubble() { + const body = { textContent: "" }; + this.currentAssistantBodyEl = body; + this.currentAssistantEl = { body }; + this.assistantRows.push(body); + }, + replayHistory() { + this.assistantRows = [{ textContent: sentinel }]; + this._historyStale = false; + }, + _refetchHistory(_wsId, token) { + backstops += 1; + return new Promise((resolve) => { + releaseHeal = () => { + this.replayHistory(); + this._endReplayQuiesce(token); + resolve(); + }; + }); + }, + _beginReplayQuiesce: _beginReplayQuiesce, + _endReplayQuiesce: _endReplayQuiesce, + _deferStaleHistoryBackstop: _deferStaleHistoryBackstop, + handleEvent: handleEvent, +}; + +// This is the production replay_ok ordering: its synthetic current-state idle +// precedes the canonical ring slice for the turn that landed during /history. +pane._replayQueue = { + token: 7, + events: [ + { type: "state_change", state: "idle" }, + { type: "user_turn", content: "gap" }, + { type: "content", text: sentinel }, + { type: "stream_end" }, + { type: "state_change", state: "idle" }, + ], +}; +pane._endReplayQuiesce(7); + +if (backstops !== 0) + throw new Error("synthetic idle refetched before the canonical backlog tail"); +if (pane._replayQueue !== null) + throw new Error("canonical backlog was diverted into a replacement queue"); +if (pane.userRows.join("|") !== "gap") + throw new Error("canonical user turn was not delivered in FIFO order"); +if ( + pane.assistantRows.length !== 1 || + pane.assistantRows[0].textContent !== sentinel +) + throw new Error("canonical content did not paint exactly once before heal"); + +await Promise.resolve(); +if (backstops !== 1) + throw new Error("multiple idle edges did not collapse to one stale backstop"); +if (!pane._replayQueue || pane._replayQueue.events.length !== 0) + throw new Error("stale backstop did not quiesce its delayed history repaint"); +if (typeof releaseHeal !== "function") + throw new Error("delayed history repaint was not captured"); + +releaseHeal(); +await Promise.resolve(); +if (pane._replayQueue !== null) + throw new Error("successful stale heal left replay quiesced"); +if ( + pane.assistantRows.length !== 1 || + pane.assistantRows[0].textContent !== sentinel +) + throw new Error("history repaint plus replay backlog duplicated assistant content"); +""" + _run_module(tmp_path, script) + + +def test_history_handoff_attempt_budget_runtime(tmp_path: Path) -> None: + """Four automatic attempts park; explicit manual recovery remains usable.""" + + _run_module( + tmp_path, + f""" +import {{ + createHistoryHandoffDeadline, + HISTORY_HANDOFF_FETCH_TIMEOUT_MS, + HISTORY_HANDOFF_MAX_ATTEMPTS, + historyHandoffAttemptAllowed, +}} from {json.dumps(_SHARED_HANDOFF.as_uri())}; +if (HISTORY_HANDOFF_MAX_ATTEMPTS !== 4) throw new Error("attempt budget"); +if (HISTORY_HANDOFF_FETCH_TIMEOUT_MS !== 15000) throw new Error("deadline"); +for (let attempts = 0; attempts < 4; attempts++) {{ + if (!historyHandoffAttemptAllowed(attempts, false, false)) + throw new Error("automatic attempt " + attempts + " was blocked"); +}} +if (historyHandoffAttemptAllowed(4, false, false)) + throw new Error("fifth automatic attempt was allowed"); +if (historyHandoffAttemptAllowed(0, true, false)) + throw new Error("manual-only state failed open"); +if (!historyHandoffAttemptAllowed(4, true, true)) + throw new Error("manual retry path was lost"); + +let expired = 0; +const deadline = createHistoryHandoffDeadline(() => expired++, 5); +const stalled = new Promise(() => {{}}); +const result = await Promise.race([ + stalled, + deadline.promise.then(() => "deadline"), +]); +if (result !== "deadline" || expired !== 1 || !deadline.state.expired) + throw new Error("logical deadline did not settle an unresolved fetch"); + +// dispose() owns retirement: a cancel (expire + resolve) marks the attempt +// dead, releases the race immediately, and never fires onExpire; it is +// idempotent, and a natural-settle dispose() just drops the timer + slot. +let cancelledExpires = 0; +const cancelled = createHistoryHandoffDeadline(() => cancelledExpires++, 60000); +cancelled.dispose({{ expire: true, resolve: true }}); +await cancelled.promise; +if ( + !cancelled.state.expired || + cancelled.state.timer != null || + cancelled.state.settle != null || + cancelledExpires !== 0 +) + throw new Error("cancel-dispose did not retire the deadline in place"); +cancelled.dispose({{ expire: true, resolve: true }}); + +const settled = createHistoryHandoffDeadline(() => {{}}, 60000); +settled.dispose(); +if ( + settled.state.expired || + settled.state.timer != null || + settled.state.settle != null +) + throw new Error("natural-settle dispose must retire without expiring"); +""", + ) + + +def test_tool_occurrence_pairing_runtime(tmp_path: Path) -> None: + """Reused, duplicate, and tail-leading ids retain structural row identity.""" + + _run_module( + tmp_path, + f""" +import {{ + enqueueToolOccurrence, + indexHistoryToolOutcomes, + indexLatestToolRow, + shiftToolOccurrence, +}} from {json.dumps(_TOOL_PROJECTION.as_uri())}; + +const first = {{ role: "assistant", tool_calls: [{{ id: "c", name: "A" }}] }}; +const second = {{ role: "assistant", tool_calls: [{{ id: "c", name: "B" }}] }}; +const messages = [ + first, + {{ role: "tool", tool_call_id: "c", content: "one" }}, + second, + {{ role: "tool", tool_call_id: "c", content: "two", is_error: true }}, +]; +const indexed = indexHistoryToolOutcomes(messages); +if (indexed.get(first)[0] !== "ok" || indexed.get(second)[0] !== "error") + throw new Error("reused id inherited another turn's outcome"); + +const duplicate = {{ + role: "assistant", + tool_calls: [{{ id: "dup", name: "A" }}, {{ id: "dup", name: "B" }}], +}}; +const duplicateMessages = [ + duplicate, + {{ role: "tool", tool_call_id: "dup", content: "first" }}, + {{ role: "tool", tool_call_id: "dup", content: "second", denied: true }}, +]; +const duplicateOutcomes = indexHistoryToolOutcomes(duplicateMessages).get(duplicate); +if (duplicateOutcomes[0] !== "ok" || duplicateOutcomes[1] !== "denied") + throw new Error("same-batch duplicate occurrences collapsed"); + +const bounded = {{ role: "assistant", tool_calls: [{{ id: "c", name: "new" }}] }}; +const boundedMessages = [ + {{ role: "tool", tool_call_id: "c", content: "cut-off old", is_error: true }}, + bounded, + {{ role: "tool", tool_call_id: "c", content: "new ok" }}, +]; +if (indexHistoryToolOutcomes(boundedMessages).get(bounded)[0] !== "ok") + throw new Error("leading orphan poisoned bounded-history batch"); + +// Non-turn rows interleaved inside a batch (a mid-turn system message, a +// second writer's append) are skipped, not window terminators — a +// fully-resolved batch must never index as a permanent orphan. Only the +// next conversational turn (assistant above; user here) ends the window. +const interleaved = {{ + role: "assistant", + tool_calls: [{{ id: "i1", name: "A" }}, {{ id: "i2", name: "B" }}], +}}; +const interleavedMessages = [ + interleaved, + {{ role: "tool", tool_call_id: "i1", content: "one" }}, + {{ role: "system", content: "operator note" }}, + {{ role: "tool", tool_call_id: "i2", content: "two", is_error: true }}, +]; +const interleavedOutcomes = + indexHistoryToolOutcomes(interleavedMessages).get(interleaved); +if (interleavedOutcomes[0] !== "ok" || interleavedOutcomes[1] !== "error") + throw new Error("interleaved system row truncated the batch result window"); + +const userBounded = {{ role: "assistant", tool_calls: [{{ id: "u1", name: "A" }}] }}; +const userBoundedMessages = [ + userBounded, + {{ role: "user", content: "next turn" }}, + {{ role: "tool", tool_call_id: "u1", content: "stray twin" }}, +]; +if (indexHistoryToolOutcomes(userBoundedMessages).get(userBounded)[0] !== undefined) + throw new Error("a user turn no longer bounds the batch result window"); + +const rowsById = new Map(); +const resultsById = new Map(); +const oldRow = {{ name: "A" }}; +const newRow = {{ name: "B" }}; +indexLatestToolRow(rowsById, resultsById, "c", oldRow); +resultsById.set("c", {{ output: "one" }}); +indexLatestToolRow(rowsById, resultsById, "c", newRow); +if (rowsById.get("c") !== newRow || resultsById.has("c")) + throw new Error("newest reused-id row did not take ownership"); + +const occurrences = new Map(); +enqueueToolOccurrence(occurrences, "dup", {{ row: "r1", output: "first" }}); +enqueueToolOccurrence(occurrences, "dup", {{ row: "r2", output: "second" }}); +const paired1 = shiftToolOccurrence(occurrences, "dup"); +const paired2 = shiftToolOccurrence(occurrences, "dup"); +if (paired1.row !== "r1" || paired1.output !== "first" || + paired2.row !== "r2" || paired2.output !== "second" || + occurrences.has("dup")) + throw new Error("occurrence queue did not preserve FIFO identity"); +""", + ) + + +def test_browser_tool_projection_reducers_runtime(tmp_path: Path) -> None: + """Both real browser reducers replace provisional output on the newest row.""" + + source = _COORDINATOR.read_text(encoding="utf-8") + handle_event = _as_function(source, " function handleEvent(ev) {") + append_result = _as_function( + source, + " function appendToolResult(name, callId, output, isError, opts) {", + ) + append_to_row = _as_function( + source, + " function _appendResultToRow(row, output, isError, opts) {", + ) + append_batch = _as_function( + source, + " function appendToolBatch(items, opts) {", + ) + _run_module( + tmp_path, + """ +import { + acceptedToolEventAlreadyRendered, + indexLatestToolRow, + recordAcceptedToolEvent, + shouldRefreshTasksForToolResult, +} from %(projection)s; + +function classList(...initial) { + const values = new Set(initial); + return { + add(...names) { names.forEach((name) => values.add(name)); }, + remove(...names) { names.forEach((name) => values.delete(name)); }, + contains(name) { return values.has(name); }, + }; +} +function makeNode(kind) { + const node = { + kind, + output: "", + isError: false, + dataset: {}, + attributes: {}, + classList: classList(), + parent: null, + isConnected: false, + children: [], + appendChild(child) { + child.parent = this; + child.isConnected = true; + this.children.push(child); + return child; + }, + setAttribute(name, value) { this.attributes[name] = String(value); }, + removeAttribute(name) { delete this.attributes[name]; }, + remove() { + if (this.parent) { + const at = this.parent.children.indexOf(this); + if (at >= 0) this.parent.children.splice(at, 1); + } + this.isConnected = false; + }, + querySelector(selector) { + if (selector === ".conv-row-result") + return this.children.find((child) => child.classList.contains("conv-row-result")) || null; + return null; + }, + querySelectorAll(selector) { + if (selector === ".conv-row-result") + return this.children.filter((child) => child.classList.contains("conv-row-result")); + return []; + }, + closest(selector) { + return selector === ".conv-batch" ? this.parent : null; + }, + }; + return node; +} +function makeRow(callId) { + const batch = makeNode("batch"); + batch.classList = classList("conv-batch", "conv-batch--approved"); + const row = makeNode("row"); + row.dataset.callId = callId; + batch.appendChild(row); + return { batch, row }; +} +function _tryMcpErrorBlock() { return null; } +function buildConvResult(output, opts) { + const node = makeNode("output"); + node.output = output; + node.isError = !!(opts && opts.isError); + node.classList.add("conv-row-result"); + return node; +} +let previewOpens = 0; +function buildPreviewChip(preview, onOpen) { + const node = makeNode("preview"); + node.preview = preview; + node.open = onOpen; + return node; +} +const window = { + TS_SHELL: { openPreview() { previewOpens++; } }, + open() { previewOpens++; }, +}; +function _unsetBatchRunningIfAllResults() {} +function _scheduleScroll() {} +// The mapped-row scenarios below must never fall to the orphan path; the last +// scenario opts in explicitly to exercise it. +let allowOrphanResult = false; +function appendMsg(_role, html, opts) { + if (!allowOrphanResult) throw new Error("unexpected orphan result path"); + const el = makeNode("msg"); + el.output = html; + el.dataset.callId = (opts && opts.callId) || ""; + messagesEl.appendChild(el); + return el; +} +function renderToolOutput(output) { + if (!allowOrphanResult) throw new Error("unexpected orphan result path"); + return String(output || ""); +} +function buildConvBatchShell() { + const batch = makeNode("batch"); + batch.classList = classList("conv-batch"); + return batch; +} +function _renderBatchRow(item) { + const row = makeNode("row"); + row.classList = classList("conv-row"); + row.dataset.callId = item.call_id || ""; + row.dataset.funcName = item.func_name || "tool"; + return row; +} +function _pickBatchTier() { return ""; } +function indexLabel() { return ""; } +function batchKicker() { return "Tool"; } +function _pendingKickerText() { return "Pending"; } +function _approvalAriaLabel() { return "Approval required"; } +function _buildBatchActions() { return makeNode("actions"); } +function _buildStatusPill() { return makeNode("status"); } +function _refreshRowStatus() {} +function _appendVerdictLineTo() {} +function _appendJudgePendingLineTo() {} +function _announceAssertive() {} +function _announcePolite() {} +function _toolAnnounceText() { return "tool"; } + +const toolRows = new Map(); +const latestToolRowElements = new Map(); +const toolResultNodes = new Map(); +const renderedToolEventIds = new Set(); +const liveToolCalls = new Set(); +const judgeVerdicts = new Map(); +const messagesEl = makeNode("messages"); +let activeBatch = null; +let taskRefreshes = 0; +function loadTasksDebounced() { taskRefreshes++; } +const _appendResultToRow = %(append_to_row)s; +const appendToolResult = %(append_result)s; +const appendToolBatch = %(append_batch)s; +const handleEvent = %(handle)s; + +// Exercise the real live admission path. An unresolved replay is the same +// occurrence and upgrades in place; a completed row with a provider-reused id +// starts a fresh batch, whose accepted output cannot mutate the older row. +handleEvent({ + type: "tool_pending", items: [{ call_id: "live-reuse", func_name: "A" }], +}); +const unresolvedBatch = toolRows.get("live-reuse").batch; +handleEvent({ + type: "tool_info", items: [{ call_id: "live-reuse", func_name: "A" }], +}); +if (messagesEl.children.length !== 1 || toolRows.get("live-reuse").batch !== unresolvedBatch) + throw new Error("coordinator unresolved occurrence did not upgrade in place"); +handleEvent({ + type: "tool_result", accepted: true, _event_id: 1, + call_id: "live-reuse", name: "A", output: "old final", +}); +const oldLiveRow = toolRows.get("live-reuse").row; +const oldLiveOutput = oldLiveRow.children.find((node) => node.kind === "output"); +handleEvent({ + type: "tool_info", items: [{ call_id: "live-reuse", func_name: "B" }], +}); +const newLiveRow = toolRows.get("live-reuse").row; +if (messagesEl.children.length !== 2 || newLiveRow === oldLiveRow) + throw new Error("coordinator completed reused id did not create a new live batch"); +handleEvent({ + type: "tool_result", accepted: true, _event_id: 2, + call_id: "live-reuse", name: "B", output: "new final", +}); +if (!oldLiveOutput || oldLiveOutput.output !== "old final" || !oldLiveOutput.isConnected) + throw new Error("coordinator reused live id mutated the completed prior row"); +const newLiveOutput = newLiveRow.children.find((node) => node.kind === "output"); +if (!newLiveOutput || newLiveOutput.output !== "new final") + throw new Error("coordinator reused live id missed the newest row"); + +// /history calls the same admission function once per assistant occurrence. +// Resolved rows must always append even when the provider reuses the id. +const historyOne = appendToolBatch( + [{ call_id: "history-reuse", func_name: "History A" }], + { resolved: { approved: true } }, +); +const historyOneRow = historyOne.children.find((node) => node.kind === "row"); +_appendResultToRow(historyOneRow, "history old", false, { accepted: true }); +const historyTwo = appendToolBatch( + [{ call_id: "history-reuse", func_name: "History B" }], + { resolved: { approved: true } }, +); +const historyTwoRow = historyTwo.children.find((node) => node.kind === "row"); +_appendResultToRow(historyTwoRow, "history new", true, { accepted: true }); +if (historyOne === historyTwo || messagesEl.children.length !== 4) + throw new Error("coordinator history reused id collapsed assistant occurrences"); +if (historyOneRow.children.find((node) => node.kind === "output").output !== "history old" || + historyTwoRow.children.find((node) => node.kind === "output").output !== "history new") + throw new Error("coordinator history reused id cross-attributed outputs"); + +const first = makeRow("c"); +indexLatestToolRow(latestToolRowElements, toolResultNodes, "c", first.row); +toolRows.set("c", { batch: first.batch, row: first.row }); +handleEvent({ + type: "tool_result", call_id: "c", name: "tasks", output: "one provisional", +}); +if (taskRefreshes !== 1) throw new Error("tasks provisional did not refresh once"); +handleEvent({ + type: "tool_result", accepted: true, _event_id: 10, + call_id: "c", name: "tasks", output: "one final", is_error: true, + preview: { title: "accepted error preview" }, effect_status: "unknown", +}); +if (taskRefreshes !== 1) throw new Error("accepted tasks replacement refreshed twice"); +if (previewOpens !== 0) throw new Error("coordinator accepted preview auto-opened"); +const firstOutput = first.row.children.find((node) => node.kind === "output"); +const firstPreview = first.row.children.find((node) => node.kind === "preview"); +if (!firstOutput || firstOutput.output !== "one final" || !firstOutput.isError) + throw new Error("coordinator accepted error did not replace provisional output"); +if (!firstPreview || firstPreview.preview.title !== "accepted error preview") + throw new Error("coordinator accepted error preview chip missing"); +if (first.row.dataset.effectStatus !== "unknown") + throw new Error("coordinator effect status was not retained"); + +const second = makeRow("c"); +indexLatestToolRow(latestToolRowElements, toolResultNodes, "c", second.row); +toolRows.set("c", { batch: second.batch, row: second.row }); +handleEvent({ + type: "tool_result", call_id: "c", name: "other", output: "two provisional", +}); +handleEvent({ + type: "tool_result", accepted: true, _event_id: 20, + call_id: "c", name: "other", output: "two final", +}); +if (firstOutput.output !== "one final" || !firstOutput.isConnected) + throw new Error("coordinator reused id mutated earlier turn"); +const secondOutputs = second.row.children.filter((node) => node.kind === "output"); +if (secondOutputs.length !== 1 || secondOutputs[0].output !== "two final") + throw new Error("coordinator newest row did not own accepted replacement"); + +const acceptedOutput = secondOutputs[0]; +handleEvent({ + type: "tool_result", accepted: true, _event_id: 20, + call_id: "c", name: "other", output: "duplicate corruption", +}); +if (second.row.children.find((node) => node.kind === "output") !== acceptedOutput) + throw new Error("coordinator accepted event replay rendered twice"); +renderedToolEventIds.add("7"); +handleEvent({ + type: "tool_result", accepted: true, _event_id: 7, + call_id: "c", name: "other", output: "history corruption", +}); +if (second.row.children.some((node) => node.output === "history corruption")) + throw new Error("coordinator history-seeded replay mutated newest row"); + +// A result whose batch has not rendered yet paints an orphan bubble (bounded +// /history window, replay edge). When that batch lands afterwards it adopts the +// occurrence, so the accepted result must upgrade the row AND retire the +// bubble — the removal guard can only fire while the orphan's ownership entry +// survives its batch being indexed. +allowOrphanResult = true; +handleEvent({ + type: "tool_result", call_id: "late-batch", name: "A", output: "orphan provisional", +}); +const orphanBubble = messagesEl.children[messagesEl.children.length - 1]; +const orphanOwner = toolResultNodes.get("late-batch"); +if (orphanBubble.kind !== "msg" || !orphanOwner || orphanOwner.row !== null) + throw new Error("coordinator unmapped result did not paint an orphan bubble"); +handleEvent({ + type: "tool_info", items: [{ call_id: "late-batch", func_name: "A" }], +}); +handleEvent({ + type: "tool_result", accepted: true, _event_id: 30, + call_id: "late-batch", name: "A", output: "orphan final", +}); +if (orphanBubble.isConnected) + throw new Error("coordinator late batch left its orphan bubble beside the row"); +const lateRow = toolRows.get("late-batch").row; +const lateOutputs = lateRow.children.filter((node) => node.kind === "output"); +if (lateOutputs.length !== 1 || lateOutputs[0].output !== "orphan final") + throw new Error("coordinator late batch row did not own the accepted result"); +""" + % { + "projection": json.dumps(_TOOL_PROJECTION.as_uri()), + "handle": handle_event, + "append_result": append_result, + "append_to_row": append_to_row, + "append_batch": append_batch, + }, + ) + source = _INTERACTIVE.read_text(encoding="utf-8") + handle_event = _as_function(source, " handleEvent(evt) {") + append_output = _as_function( + source, + " appendToolOutput(callId, name, output, isError, preview, opts = {}) {", + ) + announce_block = _as_function(source, " announceToolBlock(items) {") + announce_key = _as_function(source, " _announceKey(items) {") + _run_module( + tmp_path, + """ +import { + acceptedToolEventAlreadyRendered, + indexLatestToolRow, + recordAcceptedToolEvent, +} from %(projection)s; + +function classList(...initial) { + const values = new Set(initial); + return { + add(...names) { names.forEach((name) => values.add(name)); }, + remove(...names) { names.forEach((name) => values.delete(name)); }, + contains(name) { return values.has(name); }, + }; +} +function makeNode(kind) { + const node = { + kind, + output: "", + isError: false, + dataset: {}, + attributes: {}, + classList: classList(), + parent: null, + isConnected: false, + children: [], + appendChild(child) { + child.parent = this; + child.isConnected = true; + this.children.push(child); + return child; + }, + setAttribute(name, value) { this.attributes[name] = String(value); }, + removeAttribute(name) { delete this.attributes[name]; }, + after(child) { + const siblings = this.parent.children; + const at = siblings.indexOf(this); + child.parent = this.parent; + child.isConnected = true; + siblings.splice(at + 1, 0, child); + }, + remove() { + if (this.parent) { + const at = this.parent.children.indexOf(this); + if (at >= 0) this.parent.children.splice(at, 1); + } + this.isConnected = false; + }, + querySelector(selector) { + if (selector === ".conv-agent") return null; + return null; + }, + querySelectorAll(selector) { + if (selector === ".conv-batch") + return this.children.filter((child) => child.classList.contains("conv-batch")); + if (selector === ".conv-row") + return this.children.filter((child) => child.classList.contains("conv-row")); + return []; + }, + closest(selector) { + return selector === ".conv-batch" ? this.parent : null; + }, + get nextElementSibling() { + if (!this.parent) return null; + return this.parent.children[this.parent.children.indexOf(this) + 1] || null; + }, + }; + Object.defineProperty(node, "className", { + get() { return this._className || ""; }, + set(value) { + this._className = String(value); + this.classList = classList(...String(value).split(/\\s+/).filter(Boolean)); + }, + }); + return node; +} +function makeRow(callId) { + const batch = makeNode("batch"); + batch.classList = classList("conv-batch", "conv-batch--approved"); + const row = makeNode("row"); + row.dataset.callId = callId; + row.dataset.funcName = "tool"; + batch.appendChild(row); + return { batch, row }; +} +function stripAnsi(value) { return String(value || ""); } +function tryParseMedia() { return null; } +function buildMediaEmbed(_media, raw) { + const node = makeNode("media"); + node.output = raw; + return node; +} +function tryParseMcpError() { return null; } +function buildMcpErrorEmbed() { throw new Error("unexpected MCP path"); } +function renderCollapsibleOutput(output, isError) { + const node = makeNode("output"); + node.output = output; + node.isError = !!isError; + return node; +} +function appendToolErrorBadge(batch) { batch.errorBadges = 1; } +let previewOpens = 0; +function buildPreviewChip(preview, onOpen) { + const node = makeNode("preview"); + node.preview = preview; + node.open = onOpen; + return node; +} +const document = { createElement() { return makeNode("element"); } }; +function _convApprovalHead() { return makeNode("head"); } +function batchKicker() { return "Tool"; } +function buildToolDiv(item) { + const row = makeNode("row"); + row.classList = classList("conv-row"); + row.dataset.callId = item.call_id || ""; + row.dataset.funcName = item.func_name || "tool"; + return row; +} +function indexLabel() { return ""; } +function buildConvVerdict() { return makeNode("verdict"); } +function toolAnnounce() {} +function _toolAnnounceText() { return "tool"; } + +const handleEvent = %(handle)s; +const appendToolOutput = %(append)s; +const announceToolBlock = %(announce)s; +const _announceKey = %(announce_key)s; +const latestRows = new Map(); +const messagesEl = makeNode("messages"); +const pane = { + _agentCards: null, + _renderedToolEventIds: new Set(), + _toolResultNodes: new Map(), + _streamElIndex: new Map(), + announcedBlocks: new Map(), + messagesEl, + _host: { + isFocused() { return true; }, + onPreview() { previewOpens++; }, + onConsentDetected() {}, + }, + _toolRow(callId) { return latestRows.get(callId) || null; }, + _routeAgentItems() { return false; }, + _announceKey, + _indexToolRows(block) { + block.querySelectorAll(".conv-row").forEach((row) => { + indexLatestToolRow(latestRows, this._toolResultNodes, row.dataset.callId, row); + }); + }, + _relinkAgentCards() {}, + _streamEl() { return null; }, + isNearBottom() { return false; }, + scrollToBottom() {}, + appendToolOutput, + announceToolBlock, + handleEvent, +}; + +// Stop may accept a synthesized cancellation while only the early shell is +// present. Acceptance retires that exact shell's map ownership without +// deleting its committed DOM, so a later turn reusing the id paints anew. +pane.handleEvent({ + type: "tool_pending", items: [{ call_id: "cancel-reuse", func_name: "Cancel A" }], +}); +const cancelledBatch = messagesEl.children[0]; +if (!cancelledBatch || pane.announcedBlocks.size !== 1) + throw new Error("interactive early tool shell was not announced"); +pane.handleEvent({ + type: "tool_result", accepted: true, _event_id: 1, + call_id: "cancel-reuse", name: "Cancel A", output: "cancelled", is_error: true, +}); +if (pane.announcedBlocks.size !== 0 || !cancelledBatch.isConnected) + throw new Error("interactive accepted cancellation did not retire exact shell ownership"); +pane.handleEvent({ + type: "tool_pending", items: [{ call_id: "cancel-reuse", func_name: "Cancel B" }], +}); +if (messagesEl.children.length !== 2 || messagesEl.children[0] !== cancelledBatch || + !cancelledBatch.isConnected) + throw new Error("interactive reused pending id removed an accepted prior batch"); + +const first = makeRow("c"); +indexLatestToolRow(latestRows, pane._toolResultNodes, "c", first.row); +pane.handleEvent({ + type: "tool_result", call_id: "c", name: "tool", + output: "one provisional", preview: { title: "one" }, +}); +if (previewOpens !== 1) throw new Error("provisional preview did not auto-open once"); +const firstOutput = first.batch.children.find((node) => node.kind === "output"); + +const second = makeRow("c"); +indexLatestToolRow(latestRows, pane._toolResultNodes, "c", second.row); +pane.handleEvent({ + type: "tool_result", call_id: "c", name: "tool", + output: "two provisional", preview: { title: "two" }, +}); +if (previewOpens !== 2) throw new Error("second provisional preview auto-open mismatch"); +pane.handleEvent({ + type: "tool_result", accepted: true, _event_id: 20, + call_id: "c", name: "tool", output: "two final", is_error: true, + preview: { title: "two accepted" }, effect_status: "unknown", +}); +if (previewOpens !== 2) throw new Error("accepted preview auto-opened a second time"); +if (firstOutput.output !== "one provisional" || !firstOutput.isConnected) + throw new Error("reused id mutated the older turn"); +const secondOutputs = second.batch.children.filter((node) => node.kind === "output"); +const secondPreviews = second.batch.children.filter((node) => node.kind === "preview"); +if (secondOutputs.length !== 1 || secondOutputs[0].output !== "two final" || + secondOutputs[0].isError !== true) + throw new Error("accepted output did not replace provisional output exactly"); +if (secondPreviews.length !== 1 || secondPreviews[0].preview.title !== "two accepted") + throw new Error("accepted error preview chip was lost or duplicated"); +if (second.row.dataset.effectStatus !== "unknown") + throw new Error("accepted effect status was not retained on the row"); + +const acceptedOutput = secondOutputs[0]; +pane.handleEvent({ + type: "tool_result", accepted: true, _event_id: 20, + call_id: "c", name: "tool", output: "duplicate corruption", +}); +if (second.batch.children.filter((node) => node.kind === "output")[0] !== acceptedOutput) + throw new Error("accepted event-id replay replaced the row twice"); +pane._renderedToolEventIds.add("7"); +pane.handleEvent({ + type: "tool_result", accepted: true, _event_id: 7, + call_id: "c", name: "tool", output: "old history corruption", +}); +if (second.batch.children.some((node) => node.output === "old history corruption")) + throw new Error("history-seeded accepted replay mutated newest reused-id row"); +""" + % { + "projection": json.dumps(_TOOL_PROJECTION.as_uri()), + "handle": handle_event, + "append": append_output, + "announce": announce_block, + "announce_key": announce_key, + }, + ) + + +def test_client_send_correlation_and_lost_ack_runtime(tmp_path: Path) -> None: + """Repeated tokens match FIFO, and SSE acceptance dominates a lost HTTP ACK.""" + + _run_module( + tmp_path, + f""" +import {{ + clientSendMaySettleForViewer, + markAcceptedClientSendBubbles, + sendBubbleWasAccepted, + settleSendResponse, +}} from {json.dumps(_QUEUE.as_uri())}; +const first = {{ dataset: {{ clientSendId: "repeat" }} }}; +const second = {{ dataset: {{ clientSendId: "repeat" }} }}; +const matched = markAcceptedClientSendBubbles( + [first, second], ["repeat", "repeat"] +); +if (matched.length !== 2 || matched[0] !== first || matched[1] !== second) + throw new Error("repeated correlation token collapsed distinct sends"); +if (!sendBubbleWasAccepted(first) || !sendBubbleWasAccepted(second)) + throw new Error("acceptance proof missing"); +const third = {{ dataset: {{ clientSendId: "repeat" }} }}; +const queued = markAcceptedClientSendBubbles( + [first, second, third], ["repeat"], true +); +if (queued.length !== 1 || queued[0] !== third) + throw new Error("replayed message_queued re-accepted an earlier bubble"); +if (clientSendMaySettleForViewer("mallory", "alice")) + throw new Error("foreign sender settled this viewer's bubble"); +if (!clientSendMaySettleForViewer("alice", "alice")) + throw new Error("origin sender could not settle its bubble"); +if (!clientSendMaySettleForViewer("", "alice")) + throw new Error("legacy sender-less event lost compatibility"); + +let consumed = 0; +settleSendResponse( + {{ addQueuedMessage() {{ throw new Error("recreated accepted bubble"); }} }}, + {{ status: "queue_full", attached_ids: ["a1"] }}, + {{ + queuedEl: null, + optimisticEl: first, + isBusy: false, + displayText: "hello", + priority: "notice", + clientSendId: "repeat", + setBusy() {{ throw new Error("accepted event changed busy"); }}, + busyIsOptimistic() {{ return true; }}, + paneIsBusy() {{ return false; }}, + renderError() {{ throw new Error("accepted event rendered an error"); }}, + consumeAttachments() {{ consumed++; }}, + }}, +); +if (consumed !== 1) throw new Error("accepted event did not settle attachments"); +""", + ) + + +def test_accept_user_turn_reducer_runtime(tmp_path: Path) -> None: + """The one user_turn reducer both panes run: gate, settle, dedupe, nudge.""" + + _run_module( + tmp_path, + f""" +import {{ acceptUserTurnEvent }} from {json.dumps(_QUEUE.as_uri())}; +globalThis.sessionStorage = {{ + getItem(key) {{ return key === "ts.user_id" ? "alice" : null; }}, +}}; + +function makeBubble(clientSendId, queued) {{ + return {{ + dataset: {{ clientSendId }}, + isConnected: true, + classList: {{ contains: (name) => queued && name === "msg-queued" }}, + remove() {{ this.isConnected = false; }}, + }}; +}} +function makeHost(bubbles) {{ + const removedByQueue = []; + return {{ + renderedEventIds: new Set(), + messagesEl: {{ querySelectorAll: () => bubbles }}, + queue: {{ + remove(el) {{ + removedByQueue.push(el); + el.isConnected = false; + }}, + }}, + removedByQueue, + consumed: [], + nudges: 0, + painted: [], + consumeAttachments(ids) {{ this.consumed.push(ids); }}, + renderNudgeMarker() {{ this.nudges++; }}, + renderUserTurn(content, attachments, opts) {{ + this.painted.push({{ content, attachments, opts }}); + }}, + }}; +}} + +// A peer's turn must not settle this viewer's optimistic bubble, and with no +// bubble settled there is no attachment handoff to make. +const foreign = makeBubble("tok", false); +let host = makeHost([foreign]); +acceptUserTurnEvent( + {{ _event_id: 1, sender: "mallory", client_send_ids: ["tok"], + attachments: [{{ attachment_id: "a1" }}], content: "hi" }}, + host, +); +if (!foreign.isConnected || host.consumed.length !== 0) + throw new Error("a foreign sender settled this viewer's bubble"); +if (host.painted.length !== 1 || host.painted[0].opts.viewer !== "alice") + throw new Error("foreign turn did not paint with the resolved viewer"); + +// The viewer's own turn settles its queued chip THROUGH the queue controller +// and hands the attachment ids to the composer exactly once. +const mine = makeBubble("tok", true); +host = makeHost([mine]); +const own = {{ + _event_id: 2, sender: "alice", client_send_ids: ["tok"], + attachments: [{{ attachment_id: "a1" }}, {{}}], content: "hi", +}}; +acceptUserTurnEvent(own, host); +if (mine.isConnected || host.removedByQueue[0] !== mine) + throw new Error("queued chip did not leave through the queue controller"); +if (host.consumed.length !== 1 || host.consumed[0].join(",") !== "a1") + throw new Error("settled send did not hand off exactly its attachment ids"); + +// Replay of the same event id is inert — the paint already happened. +acceptUserTurnEvent(own, host); +if (host.painted.length !== 1 || host.consumed.length !== 1) + throw new Error("a replayed event id projected twice"); + +// The create path: the viewer's own accepted first turn carries consumed +// attachment ids but matched NO optimistic bubble (the create dispatch never +// made one). The chip sync must follow the viewer policy, not the +// matched-bubble gate — a rehydrated pending chip for a spent upload would +// re-submit a drained id on the next send. +host = makeHost([]); +acceptUserTurnEvent( + {{ _event_id: 5, sender: "alice", + attachments: [{{ attachment_id: "c1" }}], content: "first turn" }}, + host, +); +if (host.consumed.length !== 1 || host.consumed[0].join(",") !== "c1") + throw new Error("create-dispatched turn did not clear its composer chips"); + +// A wake-driven turn paints the marker instead of a user bubble, and is still +// recorded so its own replay stays inert. +host = makeHost([]); +acceptUserTurnEvent({{ _event_id: 3, source: "system_nudge" }}, host); +acceptUserTurnEvent({{ _event_id: 3, source: "system_nudge" }}, host); +if (host.nudges !== 1 || host.painted.length !== 0) + throw new Error("system_nudge did not project exactly one marker"); + +// An id-less turn cannot be deduped, so it must still paint. +host = makeHost([]); +acceptUserTurnEvent({{ content: "no id" }}, host); +acceptUserTurnEvent({{ content: "no id" }}, host); +if (host.painted.length !== 2) + throw new Error("id-less turns were suppressed by the dedupe set"); +""", + ) + + +@pytest.mark.parametrize( + ("path", "accept_call", "accept_def", "render_call", "set_name", "next_case"), + [ + ( + _INTERACTIVE, + "this._acceptUserTurn(evt);", + " _acceptUserTurn(evt) {", + "this.addUserMessage(", + "this._renderedUserEventIds", + 'case "stream_overflow"', + ), + ( + _COORDINATOR, + "acceptUserTurn(ev);", + " function acceptUserTurn(ev) {", + "appendUserMessageWithAttachments(", + "renderedUserEventIds", + 'case "tool_pending"', + ), + ], + ids=["interactive", "coordinator"], +) +def test_user_turn_projects_exactly_once_without_rest_or_stream_reopen( + path: Path, + accept_call: str, + accept_def: str, + render_call: str, + set_name: str, + next_case: str, +) -> None: + """An upgraded pane renders the live row once without refetch/redial.""" + + body = path.read_text(encoding="utf-8") + case_start = body.index('case "user_turn"') + case_end = body.index('case "system_turn"', case_start) + case = _strip_comments(body[case_start:case_end]) + assert accept_call in case + assert "refetchHistory" not in case + assert "connectSSE" not in case + + accept_start = body.index(accept_def) + accept_end = body.index("\n }", accept_start) + 4 + accept = _strip_comments(body[accept_start:accept_end]) + # The projection ORDER — dedupe check, then paint, then record — is pinned + # once in the shared reducer now that both panes route through it; the pane + # contributes only its dedupe set and its DOM writes. Pinning the order per + # pane would have re-asserted the same three lines twice and left the one + # place they actually live unpinned. + assert "acceptUserTurnEvent(" in accept + assert f"renderedEventIds: {set_name}" in accept + assert render_call in accept + assert "refetchHistory" not in accept + assert "connectSSE" not in accept + + shared = _strip_comments(_QUEUE.read_text(encoding="utf-8")) + reducer_start = shared.index("export function acceptUserTurnEvent(evt, host) {") + reducer = shared[reducer_start : shared.index("\n}", reducer_start)] + seen = reducer.index("host.renderedEventIds.has(eventId)") + render = reducer.index("host.renderUserTurn(") + record = reducer.index("host.renderedEventIds.add(eventId)") + assert seen < render < record + # An unpainted turn must stay replayable: the nudge branch is the only + # other paint, and it sits inside the same record-after-paint window. + assert reducer.index("host.renderNudgeMarker()") < record + + truncated_start = body.index('case "replay_truncated"') + truncated_end = body.index(next_case, truncated_start) + truncated = body[truncated_start:truncated_end] + assert "projection_unsupported" not in truncated + + +@pytest.mark.parametrize( + ("path", "append_call", "next_case"), + [ + ( + _INTERACTIVE, + "this.appendToolOutput(", + 'case "status"', + ), + ( + _COORDINATOR, + "appendToolResult(", + 'case "approve_request"', + ), + ], + ids=["interactive", "coordinator"], +) +def test_accepted_tool_turn_upserts_once_without_rest_or_stream_reopen( + path: Path, + append_call: str, + next_case: str, +) -> None: + """The final guarded TOOL row stays on the open stream's ordinary path.""" + + body = path.read_text(encoding="utf-8") + case_start = body.index('case "tool_result"') + case_end = body.index(next_case, case_start) + case = _strip_comments(body[case_start:case_end]) + seen = case.index("acceptedToolEventAlreadyRendered(") + render = case.index(append_call) + record = case.index("recordAcceptedToolEvent(") + assert seen < render < record + for forbidden in ("refetchHistory", "connectSSE", "EventSource"): + assert forbidden not in case + + +def test_tool_turn_capability_is_browser_only_and_url_sticky() -> None: + """Every browser redial opts in; SDK/channel consumers stay legacy-safe.""" + + for path in (_INTERACTIVE, _COORDINATOR): + body = _strip_comments(path.read_text(encoding="utf-8")) + connect_start = body.index("connectSSE(") + source_start = body.index("new EventSource", connect_start) + connect = body[connect_start:source_start] + assert '"user_turn=1"' in connect + assert '"&tool_turn=1"' in connect + + for path in (_PY_SDK, _PY_CHANNEL, _TS_SDK): + assert "tool_turn" not in _strip_comments(path.read_text(encoding="utf-8")) + + +def test_tool_turn_history_seed_and_reused_id_contracts_are_pinned() -> None: + """History overlap cannot mutate a later row that reused a provider id.""" + + interactive = _strip_comments(_INTERACTIVE.read_text(encoding="utf-8")) + assert "this._renderedToolEventIds.add(String(msg.event_id))" in interactive + assert "rows[rows.length - 1]" in interactive + assert "this._toolResultNodes.delete(callId)" in interactive + assert "target.dataset.effectStatus" in interactive + assert "!accepted && !isError && this._host.isFocused(this)" in interactive + assert "preview && !isDenied && (!isError || accepted)" in interactive + + coordinator = _strip_comments(_COORDINATOR.read_text(encoding="utf-8")) + assert "renderedToolEventIds.add(String(m.event_id))" in coordinator + assert "toolRows.set(it.call_id, { batch, row })" in coordinator + assert "indexLatestToolRow(" in coordinator + assert "indexHistoryToolOutcomes(historyMessages)" in coordinator + assert "shiftToolOccurrence(pendingHistoryToolRows, callId)" in coordinator + assert "row.dataset.effectStatus" in coordinator + assert "buildPreviewChip(opts.preview" in coordinator + + +@pytest.mark.parametrize( + ("path", "refetch_call"), + [ + (_INTERACTIVE, "this._refetchHistory(this.wsId, token)"), + (_COORDINATOR, "refetchHistory()"), + ], + ids=["pre-projection-interactive", "pre-projection-coordinator"], +) +def test_tokenless_bootstrap_uses_pre_projection_in_place_clear_ui_contract( + path: Path, + refetch_call: str, +) -> None: + """The old clear_ui reducer repairs one open listener and cannot loop.""" + + body = path.read_text(encoding="utf-8") + start = body.index('case "clear_ui"') + end = body.index('case "history_resync"', start) + clear_ui = _strip_comments(body[start:end]) + assert refetch_call in clear_ui + for forbidden in ( + "disconnectSSE", + "suspendStream", + "_loadHistoryThenConnect", + "loadHistoryThenReconnect", + "connectSSE", + ): + assert forbidden not in clear_ui + + +@pytest.mark.parametrize("path", [_INTERACTIVE, _COORDINATOR], ids=["interactive", "coordinator"]) +def test_history_handoff_manual_state_is_fail_closed(path: Path) -> None: + """Budget exhaustion exposes manual actions without constructing EventSource.""" + + body = path.read_text(encoding="utf-8") + # The attempt budget, the backoff, and the parked prompt live once in the + # shared controller. Each pane must reach them THROUGH it: re-deriving any + # of them pane-side is exactly the drift this consolidation removes, so + # their absence from the pane is the pin. + assert "createHistoryHandoffRepair(" in body + for reimplemented in ( + "historyHandoffAttemptAllowed(", + "HISTORY_HANDOFF_MAX_ATTEMPTS", + "nextHistoryHandoffDelay(", + "buildHistoryHandoffPrompt(", + ): + assert reimplemented not in body, f"{reimplemented} must stay shared" + shared_body = _SHARED_HANDOFF.read_text(encoding="utf-8") + assert "historyHandoffAttemptAllowed(" in shared_body + assert "HISTORY_HANDOFF_MAX_ATTEMPTS" in shared_body + assert "nextHistoryHandoffDelay(" in shared_body + assert "buildHistoryHandoffPrompt(" in shared_body + assert ( + '"Live updates are paused because conversation history could not be verified."' + in shared_body + ) + assert '"Retry now"' in shared_body + assert '"Reload page"' in shared_body + + connect_start = body.index("connectSSE(") + source_start = body.index("new EventSource", connect_start) + connect_prefix = body[connect_start:source_start] + assert "isRepairing(wsId)" in connect_prefix + assert "schedule();" in connect_prefix + assert "return;" in connect_prefix + + +def test_repair_mode_downgrades_completed_tokenless_render_to_bootstrap() -> None: + """A rendered tokenless 200 clears the latch instead of parking the pane. + + The server's deliberate cold storage-only read carries no token; the + repair settle must distinguish it (loader outcome "rendered") from a + failed fetch/render and downgrade to the tokenless bootstrap — the + cursorless connect whose convergence the server owns via clear_ui — + rather than burning the attempt budget against a healthy response. + + The three-way verdict is pinned where it now lives — once, in the shared + controller. Each pane is pinned only to produce the outcome and hand it + over; the previous per-pane character-window checks re-asserted the same + branch twice and could not see the branch ORDER at all. + """ + for path in (_INTERACTIVE, _COORDINATOR): + body = path.read_text(encoding="utf-8") + assert 'return "rendered";' in body + assert "historyRepair.settle({" in body + # The proof token stays pane-owned; the pane reports only whether the + # same response armed one. + assert "hasToken: this._historyHandoffToken != null" in body or ( + "hasToken: historyHandoffToken != null" in body + ) + assert 'outcome === "rendered"' not in body, "the verdict must stay shared" + + shared = _strip_comments(_SHARED_HANDOFF.read_text(encoding="utf-8")) + settle_start = shared.index("settle({ outcome, hasToken, manualAttempt }) {") + settle = shared[settle_start : shared.index("\n },", settle_start)] + # Token handoff and the deliberate tokenless downgrade share ONE success + # body (round-3 review: the identical twin arms invited drift — a latch + # added to one but not the other would split pane behavior by outcome). + success_arm = settle.index('if (hasToken || outcome === "rendered")') + fail_closed = settle.index("deps.setStale(pending)") + assert success_arm < fail_closed + # The success arm clears the latch and reconnects exactly once; only the + # fail-closed arm may re-arm the budget, and it never reconnects. + assert settle.count("clear();") == 1 + assert settle.count("deps.connect(target)") == 1 + assert "deps.connect" not in settle[fail_closed:] + assert "showManual();" in settle[fail_closed:] + assert "schedule();" in settle[fail_closed:] diff --git a/tests/test_history_total_prefix.py b/tests/test_history_total_prefix.py new file mode 100644 index 00000000..1ed4742b --- /dev/null +++ b/tests/test_history_total_prefix.py @@ -0,0 +1,864 @@ +"""Adversarial contract tests for a total live conversation-row prefix. + +Each test freezes a different accepted-row boundary after the UI-visible +transition but before durable acknowledgement. The authoritative history +handoff must expose one ordered logical prefix at every cut: no later USER row +without its TOOL/SYSTEM predecessors, and no accepted cancellation/compaction +row that exists only on one side of the REST-to-SSE bootstrap. + +The gates are event-driven; timeouts are diagnostics, not race scheduling. +""" + +from __future__ import annotations + +import base64 +import contextlib +import copy +import hashlib +import json +import threading +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +import pytest + +from tests._session_helpers import make_result, make_session +from tests.test_history_commit_handoff import _send_environment, _start_send +from tests.test_session_manager import _make_manager +from turnstone.core import session as session_module +from turnstone.core import session_worker +from turnstone.core.attachment_buffer import get_attachment_buffer +from turnstone.core.attachments import Attachment, resolve_staged_attachments +from turnstone.core.storage._registry import get_storage +from turnstone.core.trajectory import turns_from_dicts + +if TYPE_CHECKING: + from collections.abc import Callable + + +_TOOL_CALL = { + "id": "call-total-prefix", + "type": "function", + "function": {"name": "prefix_probe", "arguments": "{}"}, +} + + +class _PrefixStore: + """Small keyed conversation store with a selectable pre-commit gate.""" + + def __init__( + self, + *, + block_when: Callable[[str, str | None, dict[str, Any]], bool] | None = None, + ambiguous_when: Callable[[str, str | None, dict[str, Any]], bool] | None = None, + ) -> None: + self._lock = threading.Lock() + self._next_id = 1 + self._rows: list[dict[str, Any]] = [] + self._ids_by_key: dict[tuple[str, str], int] = {} + self._block_when = block_when or (lambda _role, _content, _kwargs: False) + self.ambiguous_when = ambiguous_when or (lambda _role, _content, _kwargs: False) + self.entered = threading.Event() + self.release = threading.Event() + + def __call__( + self, + ws_id: str, + role: str, + content: str | None, + tool_name: str | None = None, + **kwargs: Any, + ) -> int: + if self._block_when(role, content, kwargs): + self.entered.set() + assert self.release.wait(5), "test did not release conversation persistence" + + row: dict[str, Any] = {"role": role, "content": content or ""} + if tool_name: + row["name"] = tool_name + tool_call_id = kwargs.get("tool_call_id") + if tool_call_id: + row["tool_call_id"] = tool_call_id + source = kwargs.get("source") + if source: + row["_source"] = source + event_id = kwargs.get("event_id") + if isinstance(event_id, int) and not isinstance(event_id, bool): + row["_event_id"] = event_id + if kwargs.get("is_error"): + row["is_error"] = True + raw_tool_calls = kwargs.get("tool_calls") + if raw_tool_calls: + row["tool_calls"] = json.loads(raw_tool_calls) + raw_provider_data = kwargs.get("provider_data") + if raw_provider_data: + row["_provider_content"] = json.loads(raw_provider_data) + producer = kwargs.get("producer") + if producer: + row["_producer"] = producer + raw_meta = kwargs.get("meta") + if raw_meta: + parsed_meta = json.loads(raw_meta) + if role == "user" and parsed_meta.get("sender"): + row["_sender"] = parsed_meta["sender"] + else: + row["_source_meta"] = parsed_meta + commit_key = kwargs.get("commit_key") + if isinstance(commit_key, str) and commit_key: + row["_commit_key"] = commit_key + + with self._lock: + identity = (ws_id, commit_key) if isinstance(commit_key, str) and commit_key else None + existing = self._ids_by_key.get(identity) if identity is not None else None + if existing is not None: + row_id = existing + else: + row_id = self._next_id + self._next_id += 1 + self._rows.append(row) + if identity is not None: + self._ids_by_key[identity] = row_id + if self.ambiguous_when(role, content, kwargs): + return 0 + return row_id + + def seed(self, *rows: dict[str, Any]) -> None: + with self._lock: + self._rows.extend(copy.deepcopy(rows)) + self._next_id += len(rows) + + def snapshot(self, overscan: int = 0) -> list[dict[str, Any]]: + # The store keeps the full prefix; the widened-window overscan the + # production loader applies to a tail-bounded read is a no-op here. + del overscan + with self._lock: + return copy.deepcopy(self._rows) + + +def _ready_session(**kwargs: Any) -> Any: + session = make_session(**kwargs) + session._title_generated = True + session._system_composed_with_context = True + return session + + +def _gate_pending_before_visibility( + session: Any, + *, + matches: Callable[[dict[str, Any]], bool], + entered: threading.Event, + release: threading.Event, +) -> Callable[[Any], int]: + """Gate a journal implementation before it acquires the visibility lane. + + The store carries the same gate as a compatibility fallback for the + currently unjournaled implementation. Once all row kinds use the generic + pending path, this wrapper is the deterministic cut and the store's wait is + already released by the time persistence reaches it. + """ + + persist = session._persist_pending_conversation_commit + + def _gated(entry: Any) -> int: + if matches(entry.message): + entered.set() + assert release.wait(5), "test did not release pending-row persistence" + return persist(entry) + + return _gated + + +def _roles_and_content(rows: list[dict[str, Any]]) -> list[tuple[str, str]]: + return [(str(row.get("role") or ""), str(row.get("content") or "")) for row in rows] + + +def test_tool_system_user_fold_is_one_visible_causal_prefix(tmp_db: Any) -> None: + """A trailing USER may not overtake its TOOL and SYSTEM predecessors.""" + + session = _ready_session() + store = _PrefixStore(block_when=lambda role, _content, _kwargs: role == "tool") + persist_release = store.release + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], str]: + return [(_TOOL_CALL["id"], "tool output")], "approval feedback" + + gated_pending = _gate_pending_before_visibility( + session, + matches=lambda message: message.get("role") == "tool", + entered=store.entered, + release=persist_release, + ) + + with ( + _send_environment( + session, + [ + make_result("", tool_calls=[_TOOL_CALL]), + make_result("final answer"), + ], + store, + _execute_tools, + ), + patch.object( + session, + "_collect_advisories", + return_value=[("correction", "guarded operator context", {})], + ), + patch.object( + session, + "_persist_pending_conversation_commit", + side_effect=gated_pending, + ), + ): + sender, send_errors = _start_send(session, "opening user") + assert store.entered.wait(5), "tool-row persistence never reached the frozen cut" + rows_during, _token = session.capture_history_handoff(store.snapshot) + persist_release.set() + sender.join(5) + + assert not sender.is_alive() + assert send_errors == [] + assert _roles_and_content(rows_during)[-3:] == [ + ("tool", "tool output"), + ("system", "guarded operator context"), + ("user", "approval feedback"), + ] + assert all(row.get("_commit_key") for row in rows_during[-3:]) + + +def test_initial_user_and_nudge_are_visible_in_append_order(tmp_db: Any) -> None: + """The initialization batch cannot expose USER without its accepted nudge.""" + + session = _ready_session() + store = _PrefixStore() + persistence_entered = threading.Event() + persistence_release = threading.Event() + + def _emit_init_nudge(*, deferred_persistence: list[Callable[[], None]] | None = None) -> None: + session._append_system_turn( + "start", + "initial metacognitive nudge", + deferred_persistence=deferred_persistence, + ) + + gated_pending = _gate_pending_before_visibility( + session, + matches=lambda message: ( + message.get("role") == "user" and message.get("content") == "opening user" + ), + entered=persistence_entered, + release=persistence_release, + ) + + with ( + _send_environment( + session, + [make_result("answer")], + store, + MagicMock(return_value=([], None)), + ), + patch.object(session, "_emit_pending_user_nudges", side_effect=_emit_init_nudge), + patch.object( + session, + "_persist_pending_conversation_commit", + side_effect=gated_pending, + ), + ): + sender, send_errors = _start_send(session, "opening user") + assert persistence_entered.wait(5), "opening USER did not reach the frozen cut" + rows_during, _token = session.capture_history_handoff(store.snapshot) + persistence_release.set() + sender.join(5) + + assert not sender.is_alive() + assert send_errors == [] + assert _roles_and_content(rows_during)[-2:] == [ + ("user", "opening user"), + ("system", "initial metacognitive nudge"), + ] + assert all(row.get("_commit_key") for row in rows_during[-2:]) + + +def test_zero_token_cancelled_partial_crossing_forces_history_repair(tmp_db: Any) -> None: + """A zero-token cancellation marker is accepted history, not an SSE ghost.""" + + marker = "[generation cancelled before completion]" + session = _ready_session() + store = _PrefixStore( + block_when=lambda role, content, _kwargs: role == "assistant" and content == marker + ) + stream_entered = threading.Event() + release_cancel = threading.Event() + + def _cancel_before_first_token(_generation: int) -> Any: + stream_entered.set() + assert release_cancel.wait(5), "test did not release zero-token cancellation" + session._cancelled_partial_msg = {"role": "assistant", "content": ""} + raise session_module.GenerationCancelled() + + gated_pending = _gate_pending_before_visibility( + session, + matches=lambda message: ( + message.get("role") == "assistant" and message.get("content") == marker + ), + entered=store.entered, + release=store.release, + ) + + registration: Any = None + with ( + _send_environment( + session, + [make_result("unused")], + store, + MagicMock(return_value=([], None)), + ), + patch.object(session, "_stream_response", side_effect=_cancel_before_first_token), + patch.object( + session, + "_persist_pending_conversation_commit", + side_effect=gated_pending, + ), + ): + sender, send_errors = _start_send(session, "cancel immediately") + assert stream_entered.wait(5), "send never reached its zero-token stream" + _before_rows, token_before_marker = session.capture_history_handoff(store.snapshot) + + release_cancel.set() + assert store.entered.wait(5), "cancel marker did not reach the frozen persistence cut" + rows_during, token_during = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token_before_marker) + + store.release.set() + sender.join(5) + + if registration is not None: + session.ui._unregister_listener(registration[0]) + assert not sender.is_alive() + assert send_errors == [] + assert token_during != token_before_marker + assert registration is None + assert _roles_and_content(rows_during)[-1] == ("assistant", marker) + assert rows_during[-1].get("_commit_key") + + +def test_compaction_end_crossing_is_visible_to_fresh_history_handoff(tmp_db: Any) -> None: + """A listener born after compaction END must not miss its checkpoint card.""" + + summary = "bounded compacted summary" + session = _ready_session() + seed_rows = [ + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + ] + session.messages = turns_from_dicts(seed_rows) + session._msg_tokens = [2, 2] + store = _PrefixStore( + block_when=lambda role, _content, kwargs: ( + role == "assistant" and kwargs.get("source") == session_module.COMPACTION_SOURCE + ) + ) + store.seed(*seed_rows) + _rows_before, token_before = session.capture_history_handoff(store.snapshot) + + gated_pending = _gate_pending_before_visibility( + session, + matches=lambda message: message.get("_source") == session_module.COMPACTION_SOURCE, + entered=store.entered, + release=store.release, + ) + compacted: list[bool] = [] + compact_errors: list[BaseException] = [] + + def _compact() -> None: + try: + compacted.append(session._compact_messages()) + except BaseException as exc: # pragma: no cover - diagnostic capture + compact_errors.append(exc) + + registration: Any = None + with ( + patch.object( + session, + "_summarize_blocks", + return_value=session_module._SummaryResult( + text=summary, + producer="openai-compatible", + ), + ), + patch.object(get_storage(), "get_compaction_watermark", return_value=2), + patch("turnstone.core.session.save_message", side_effect=store), + patch.object( + session, + "_persist_pending_conversation_commit", + side_effect=gated_pending, + ), + ): + worker = threading.Thread(target=_compact, daemon=True) + worker.start() + reached_marker = store.entered.wait(5) + if not reached_marker: + store.release.set() + worker.join(5) + assert reached_marker, ( + "compaction marker never reached the frozen cut; " + f"errors={compact_errors!r}, results={compacted!r}" + ) + + rows_during, token_during = session.capture_history_handoff(store.snapshot) + registration = session.register_listener_for_history_handoff(token_before) + + store.release.set() + worker.join(5) + + if registration is not None: + session.ui._unregister_listener(registration[0]) + assert not worker.is_alive() + assert compact_errors == [] + assert compacted == [True] + assert token_during != token_before + assert registration is None + markers = [row for row in rows_during if row.get("_source") == session_module.COMPACTION_SOURCE] + assert len(markers) == 1 + assert markers[0]["content"] == summary + assert markers[0].get("_commit_key") + + +def test_attachment_ownership_transfers_at_journal_admission(tmp_db: Any) -> None: + """Accepted journal bytes leave staging; a rejected pre-admission turn does not.""" + + buffer = get_attachment_buffer() + buffer.clear() + ws_id = "ws-total-prefix-attachments" + user_id = "attachment-owner" + session = _ready_session(ws_id=ws_id, user_id=user_id) + + first = buffer.stage( + ws_id=ws_id, + user_id=user_id, + filename="owned.txt", + mime_type="text/plain", + kind="text", + content=b"journal owns these bytes", + ) + attachment = Attachment( + attachment_id=first.attachment_id, + filename=first.filename, + mime_type=first.mime_type, + kind=first.kind, + content=first.content, + ) + deferred: list[Callable[[], None]] = [] + + try: + session._append_user_turn( + "first accepted row", + (attachment,), + send_id="send-first", + deferred_persistence=deferred, + ) + removed_at_admission = buffer.get(first.attachment_id, ws_id=ws_id, user_id=user_id) is None + second_resolution, _taken, _dropped = resolve_staged_attachments( + [first.attachment_id], ws_id, user_id + ) + + rejected_ws_id = "ws-total-prefix-rejected-attachment" + rejected = buffer.stage( + ws_id=rejected_ws_id, + user_id=user_id, + filename="retry.txt", + mime_type="text/plain", + kind="text", + content=b"must remain retryable", + ) + rejected_session = _ready_session(ws_id=rejected_ws_id, user_id=user_id) + rejected_attachment = Attachment( + attachment_id=rejected.attachment_id, + filename=rejected.filename, + mime_type=rejected.mime_type, + kind=rejected.kind, + content=rejected.content, + ) + rejected_session._publication_shutdown = True + accepted = rejected_session._commit_for_generation( + 0, + lambda durable: rejected_session._append_user_turn( + "must not be admitted", + (rejected_attachment,), + send_id="send-rejected", + deferred_persistence=durable, + ), + ) + retained_before_admission = ( + buffer.get(rejected.attachment_id, ws_id=rejected_ws_id, user_id=user_id) is not None + ) + finally: + buffer.clear() + + assert deferred, "journal admission must retain a retryable durable closure" + assert session.has_unresolved_conversation_persistence() is True + assert removed_at_admission is True + assert second_resolution == [] + assert accepted is False + assert retained_before_admission is True + + +def test_system_lost_ack_reconciles_one_keyed_row(tmp_db: Any) -> None: + """A committed SYSTEM row with a lost ACK is never duplicated or lost.""" + + session = _ready_session() + store = _PrefixStore(ambiguous_when=lambda role, _content, _kwargs: role == "system") + + with ( + patch("turnstone.core.session.save_message", side_effect=store), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._append_system_turn("correction", "accepted operator context") + + assert session.has_unresolved_conversation_persistence() is True + assert len(store.snapshot()) == 1 + rows, _token = session.capture_history_handoff(store.snapshot) + assert _roles_and_content(rows) == [("system", "accepted operator context")] + assert rows[0].get("_commit_key") + assert session.has_unresolved_conversation_persistence() is False + + +def test_mixed_batch_failure_stops_durable_suffix_but_keeps_visible_prefix( + tmp_db: Any, +) -> None: + """A TOOL 0/0 stops SYSTEM/USER saves while all accepted rows stay visible.""" + + session = _ready_session() + store = _PrefixStore(ambiguous_when=lambda role, _content, _kwargs: role == "tool") + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], str]: + return [(_TOOL_CALL["id"], "tool output")], "approval feedback" + + with ( + _send_environment( + session, + [make_result("", tool_calls=[_TOOL_CALL])], + store, + _execute_tools, + ), + patch.object( + session, + "_collect_advisories", + return_value=[("correction", "guarded operator context", {})], + ), + pytest.raises(session_module.ConversationPersistenceError), + ): + session.send("opening user") + + durable = store.snapshot() + assert _roles_and_content(durable)[-1] == ("tool", "tool output") + assert ("system", "guarded operator context") not in _roles_and_content(durable) + assert ("user", "approval feedback") not in _roles_and_content(durable) + + rows, _token = session.capture_history_handoff(store.snapshot) + assert _roles_and_content(rows)[-3:] == [ + ("tool", "tool output"), + ("system", "guarded operator context"), + ("user", "approval feedback"), + ] + assert session.has_unresolved_conversation_persistence() is True + + store.ambiguous_when = lambda _role, _content, _kwargs: False + with patch("turnstone.core.session.save_message", side_effect=store): + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is True + assert session.has_unresolved_conversation_persistence() is False + assert _roles_and_content(store.snapshot())[-3:] == _roles_and_content(rows)[-3:] + + +def test_multi_tool_rows_keep_distinct_repair_event_ids(tmp_db: Any) -> None: + """Each deferred TOOL closure captures its own admission event cursor.""" + + session = _ready_session() + store = _PrefixStore() + tool_calls = [ + _TOOL_CALL, + { + "id": "call-total-prefix-2", + "type": "function", + "function": {"name": "prefix_probe", "arguments": "{}"}, + }, + ] + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], str]: + return [(tool_calls[0]["id"], "first"), (tool_calls[1]["id"], "second")], "" + + with _send_environment( + session, + [make_result("", tool_calls=tool_calls), make_result("done")], + store, + _execute_tools, + ): + session.send("run both") + + tool_rows = [row for row in store.snapshot() if row.get("role") == "tool"] + assert [row.get("tool_call_id") for row in tool_rows] == [ + "call-total-prefix", + "call-total-prefix-2", + ] + event_ids = [row.get("_event_id") for row in tool_rows] + assert all(isinstance(event_id, int) for event_id in event_ids) + assert event_ids[0] < event_ids[1] + + +def test_tool_attachment_lost_ack_retries_without_refcount_replay(tmp_db: Any) -> None: + """Session-level TOOL retry keeps its row/blob/ref-list one atomic commit.""" + + from turnstone.core.memory import register_workstream + from turnstone.core.storage import get_storage + + ws_id = "tool-attachment-total-prefix" + register_workstream(ws_id) + session = _ready_session(ws_id=ws_id) + raw = b"one immutable tool image" + attachment_id = hashlib.sha256(raw).hexdigest() + data_uri = "data:image/png;base64," + base64.b64encode(raw).decode() + output = [ + {"type": "text", "text": "captured image"}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ] + real_atomic_save = session_module.save_tool_message_with_attachments + save_calls = 0 + + def _commit_then_lose_ack(*args: Any, **kwargs: Any) -> int: + nonlocal save_calls + save_calls += 1 + assert real_atomic_save(*args, **kwargs) > 0 + return 0 + + scripted = iter([make_result("", tool_calls=[_TOOL_CALL])]) + with contextlib.ExitStack() as stack: + stack.enter_context( + patch.object(session, "_stream_response", side_effect=lambda _gen: next(scripted)) + ) + stack.enter_context( + patch.object( + session, + "_execute_tools", + return_value=([(_TOOL_CALL["id"], output)], ""), + ) + ) + stack.enter_context(patch.object(session, "_full_messages", return_value=[])) + stack.enter_context(patch.object(session, "_update_token_table")) + stack.enter_context(patch.object(session, "_print_status_line")) + stack.enter_context(patch.object(session, "_emit_state")) + stack.enter_context(patch.object(session, "_visible_memory_count", return_value=0)) + stack.enter_context(patch.object(session, "_apply_post_execute_advisories")) + stack.enter_context( + patch.object( + session, + "_evaluate_output", + side_effect=lambda _call_id, value, *_args, **_kwargs: (value, None), + ) + ) + stack.enter_context( + patch( + "turnstone.core.session.save_tool_message_with_attachments", + side_effect=_commit_then_lose_ack, + ) + ) + with pytest.raises(session_module.ConversationPersistenceError): + session.send("capture it") + + storage = get_storage() + tool_turns = [ + turn + for turn in storage.load_message_turns(session.ws_id, checkpointed=False) + if turn.role.value == "tool" + ] + assert save_calls == 1 + assert len(tool_turns) == 1 + assert tool_turns[0].meta.extra["storage_attachment_ids"] == [attachment_id] + assert storage.get_attachment(attachment_id)["refcount"] == 1 + session.capture_history_handoff( + lambda _overscan: storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + ) + assert session.has_unresolved_conversation_persistence() is False + + +def test_cancelled_partial_lost_ack_is_recoverable_history(tmp_db: Any) -> None: + """Cancellation cleanup journals its marker before ambiguous durability.""" + + marker = "partial\n\n[generation cancelled before completion]" + session = _ready_session() + store = _PrefixStore( + ambiguous_when=lambda role, content, _kwargs: role == "assistant" and content == marker + ) + + def _cancel(_generation: int) -> Any: + session._cancelled_partial_msg = {"role": "assistant", "content": "partial"} + raise session_module.GenerationCancelled() + + with ( + _send_environment( + session, + [make_result("unused")], + store, + MagicMock(return_value=([], None)), + ), + patch.object(session, "_stream_response", side_effect=_cancel), + pytest.raises(session_module.ConversationPersistenceError), + ): + session.send("cancel me") + + assert session.has_unresolved_conversation_persistence() is True + rows, _token = session.capture_history_handoff(store.snapshot) + assert _roles_and_content(rows)[-1] == ("assistant", marker) + assert rows[-1].get("_commit_key") + assert session.has_unresolved_conversation_persistence() is False + + +def test_compaction_marker_lost_ack_has_one_success_end_and_one_row(tmp_db: Any) -> None: + """Checkpoint durability poison does not fabricate a second failed END.""" + + summary = "accepted compacted prefix" + session = _ready_session() + seed = [ + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + ] + session.messages = turns_from_dicts(seed) + session._msg_tokens = [2, 2] + store = _PrefixStore( + ambiguous_when=lambda _role, _content, kwargs: ( + kwargs.get("source") == session_module.COMPACTION_SOURCE + ) + ) + store.seed(*seed) + + with ( + patch.object( + session, + "_summarize_blocks", + return_value=session_module._SummaryResult(text=summary, producer="kernel"), + ), + patch.object(session.ui, "on_compaction", side_effect=[41, 42]) as compaction_events, + patch.object(get_storage(), "get_compaction_watermark", return_value=2), + patch("turnstone.core.session.save_message", side_effect=store), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._compact_messages() + + assert compaction_events.call_count == 2 + assert compaction_events.call_args_list[-1].args[0]["ok"] is True + markers = [ + row for row in store.snapshot() if row.get("_source") == session_module.COMPACTION_SOURCE + ] + assert len(markers) == 1 + assert markers[0].get("_commit_key") + rows, _token = session.capture_history_handoff(store.snapshot) + assert len([row for row in rows if row.get("_source") == session_module.COMPACTION_SOURCE]) == 1 + assert session.has_unresolved_conversation_persistence() is False + + +@pytest.mark.parametrize("storage_recovers", [True, False]) +def test_soft_close_retries_the_latched_pending_prefix( + tmp_db: Any, + storage_recovers: bool, +) -> None: + """Close retries safely after 0/0 and unloads only on a complete ACK.""" + + mgr, adapter, _storage = _make_manager() + adapter.cleanup_ui = MagicMock() + ws = mgr.create(user_id="owner") + session = _ready_session(ws_id=ws.id, user_id="owner") + ws.session = session + ws.ui = session.ui + store = _PrefixStore(ambiguous_when=lambda role, _content, _kwargs: role == "system") + + with ( + patch("turnstone.core.session.save_message", side_effect=store), + pytest.raises(session_module.ConversationPersistenceError), + ): + session._append_system_turn("correction", "close must reconcile me") + + if storage_recovers: + store.ambiguous_when = lambda _role, _content, _kwargs: False + with patch("turnstone.core.session.save_message", side_effect=store): + closed = mgr.close(ws.id) + + assert closed is storage_recovers + if storage_recovers: + assert mgr.get(ws.id) is None + assert session.has_unresolved_conversation_persistence() is False + assert session._publication_shutdown is True + else: + assert mgr.get(ws.id) is ws + assert ws._closed is False + assert session.has_unresolved_conversation_persistence() is True + assert session._publication_shutdown is False + + +def test_soft_close_terminal_latch_refuses_a_fresh_worker_claim() -> None: + """No POST-equivalent dispatch may be acknowledged inside close's latch gap.""" + + ws_id = "ws-soft-close-dispatch-gap" + mgr, adapter, _storage = _make_manager() + ws = mgr.create(user_id="owner", ws_id=ws_id) + session = _ready_session(ws_id=ws_id, user_id="owner") + ws.session = session + ws.ui = session.ui + adapter.cleanup_ui = MagicMock() + + prepared = threading.Event() + release_close = threading.Event() + close_results: list[bool] = [] + close_errors: list[BaseException] = [] + run_entered = threading.Event() + run_errors: list[BaseException] = [] + real_prepare = session.prepare_soft_close + + def _gated_prepare() -> bool: + result = real_prepare() + assert result is True + prepared.set() + assert release_close.wait(5), "test did not release soft close" + return result + + def _close() -> None: + try: + close_results.append(mgr.close(ws_id)) + except BaseException as exc: # pragma: no cover - diagnostic capture + close_errors.append(exc) + + def _run_doomed_send() -> None: + run_entered.set() + try: + session.send("must not be acknowledged") + except BaseException as exc: + run_errors.append(exc) + + with patch.object(session, "prepare_soft_close", side_effect=_gated_prepare): + closer = threading.Thread(target=_close, daemon=True) + closer.start() + assert prepared.wait(5), "soft close never reached its terminal session latch" + + dispatch_ok = session_worker.send( + ws, + enqueue=lambda: None, + run=_run_doomed_send, + thread_name="doomed-soft-close-send", + ) + if dispatch_ok: + assert run_entered.wait(5), "acknowledged dispatch never ran" + + release_close.set() + closer.join(5) + worker = ws.worker_thread + if worker is not None: + worker.join(5) + + assert not closer.is_alive() + assert close_errors == [] + assert close_results == [True] + assert dispatch_ok is False + assert run_entered.is_set() is False + assert run_errors == [] diff --git a/tests/test_history_truncation_handoff.py b/tests/test_history_truncation_handoff.py new file mode 100644 index 00000000..455b676f --- /dev/null +++ b/tests/test_history_truncation_handoff.py @@ -0,0 +1,838 @@ +"""Adversarial history-handoff tests for destructive tail mutations. + +Every race is stopped on an explicit event boundary. Timeout values are +diagnostic backstops only; no assertion infers correctness from elapsed time. +""" + +from __future__ import annotations + +import queue +import threading +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from tests._session_helpers import make_session +from tests.test_workstream_endpoints import ( + _build_history_app, + _verb_cfg, + _verb_client, +) +from turnstone.core import session_worker +from turnstone.core.memory import register_workstream, save_message +from turnstone.core.session import ConversationPersistenceError +from turnstone.core.session_routes import make_retry_handler, make_rewind_handler +from turnstone.core.storage import get_storage +from turnstone.core.trajectory import dicts_from_turns, turns_from_dicts +from turnstone.core.workstream import Workstream + +_ROWS = ( + ("user", "first request"), + ("assistant", "first answer"), + ("user", "second request"), + ("assistant", "second answer"), +) + + +def _seed_session(ws_id: str) -> Any: + register_workstream(ws_id, kind="interactive", user_id="test-user") + for role, content in _ROWS: + save_message(ws_id, role, content) + + session = make_session(ws_id=ws_id, user_id="test-user") + session.messages = turns_from_dicts( + [{"role": role, "content": content} for role, content in _ROWS] + ) + session._msg_tokens = [1] * len(_ROWS) + return session + + +def _capture(session: Any) -> tuple[list[dict[str, Any]], str]: + storage = get_storage() + return session.capture_history_handoff( + lambda _overscan: storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + ) + + +def _truncate(session: Any, operation: str) -> Any: + if operation == "rewind": + return session.rewind(1) + return session.retry() + + +@pytest.mark.parametrize("operation", ["rewind", "retry"]) +def test_successful_truncation_invalidates_old_handoff_and_repairs_listener( + tmp_db: Any, + operation: str, +) -> None: + """A deletion is one atomic history revision, not only a flight-key bump.""" + + session = _seed_session(f"ws-{operation}-handoff") + before_rows, before_token = _capture(session) + registration = session.register_listener_for_history_handoff(before_token) + assert registration is not None + listener = registration[0] + + result = _truncate(session, operation) + + if operation == "rewind": + assert result == 2 + else: + assert result == "second request" + after_rows, after_token = _capture(session) + assert [row.get("content") for row in before_rows] == [content for _role, content in _ROWS] + assert [row.get("content") for row in after_rows] == [ + "first request", + "first answer", + ] + assert after_token != before_token + assert session.register_listener_for_history_handoff(before_token) is None + + events: list[dict[str, Any]] = [] + while not listener.empty(): + events.append(listener.get_nowait()) + assert [event.get("type") for event in events].count("history_resync") == 1 + + +@pytest.mark.parametrize("operation", ["rewind", "retry"]) +def test_supplied_reset_publisher_replaces_generic_resync_exactly_once( + tmp_db: Any, + operation: str, +) -> None: + """Web routes publish clear_ui atomically, without a competing resync.""" + + session = _seed_session(f"ws-{operation}-clear-ui") + _rows, token = _capture(session) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + def _publish_clear_ui() -> None: + session.ui._enqueue({"type": "clear_ui"}) + + if operation == "rewind": + assert session.rewind(1, publish_reset=_publish_clear_ui) == 2 + else: + assert session.retry(publish_reset=_publish_clear_ui) == "second request" + + events: list[dict[str, Any]] = [] + while not listener.empty(): + events.append(listener.get_nowait()) + assert [event.get("type") for event in events] == ["clear_ui"] + + +def test_reset_publisher_failure_falls_back_to_one_generic_resync(tmp_db: Any) -> None: + """A UI callback failure cannot make an already-committed cut look failed.""" + + session = _seed_session("ws-reset-publisher-fallback") + _rows, token = _capture(session) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + def _raise() -> None: + raise RuntimeError("injected reset publisher failure") + + assert session.rewind(1, publish_reset=_raise) == 2 + events: list[dict[str, Any]] = [] + while not listener.empty(): + events.append(listener.get_nowait()) + assert [event.get("type") for event in events] == ["history_resync"] + assert events[0].get("reason") == "history_truncated" + + +@pytest.mark.parametrize("operation", ["rewind", "retry"]) +def test_web_reset_publisher_runs_once_for_noop_truncation( + tmp_db: Any, + operation: str, +) -> None: + """HTTP's historical empty-transcript clear_ui contract is preserved.""" + + session = _seed_session(f"ws-{operation}-noop-reset") + assert session.rewind(999) == len(_ROWS) + published: list[str] = [] + + if operation == "rewind": + assert session.rewind(1, publish_reset=lambda: published.append("clear_ui")) == 0 + else: + assert session.retry(publish_reset=lambda: published.append("clear_ui")) is None + + assert published == ["clear_ui"] + + +@pytest.mark.parametrize("operation", ["rewind", "retry"]) +def test_truncation_storage_failure_leaves_memory_token_and_ui_unchanged( + tmp_db: Any, + operation: str, +) -> None: + """A failed durable delete may not publish an in-memory-only truncation.""" + + session = _seed_session(f"ws-{operation}-delete-failure") + storage = get_storage() + durable_before = storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + memory_before = dicts_from_turns(session.messages) + tokens_before = list(session._msg_tokens) + generation_before = session._history_generation + _rows, token_before = _capture(session) + registration = session.register_listener_for_history_handoff(token_before) + assert registration is not None + listener = registration[0] + reset_publications: list[str] = [] + + def _publish_reset() -> None: + reset_publications.append("clear_ui") + + caught: Exception | None = None + with patch.object( + storage, + "truncate_messages_tail", + side_effect=RuntimeError("injected truncation failure"), + ): + try: + if operation == "rewind": + session.rewind(1, publish_reset=_publish_reset) + else: + session.retry(publish_reset=_publish_reset) + except Exception as exc: # the public failure shape may raise or refuse + caught = exc + + assert caught is not None or dicts_from_turns(session.messages) == memory_before + assert dicts_from_turns(session.messages) == memory_before + assert session._msg_tokens == tokens_before + assert session._history_generation == generation_before + assert ( + storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + == durable_before + ) + _after_rows, token_after = _capture(session) + assert token_after == token_before + assert reset_publications == [] + assert listener.empty() + + +def test_truncation_freezes_direct_admission_then_preserves_suffix_order( + tmp_db: Any, +) -> None: + """A direct row waits behind the cut, then survives after its new prefix.""" + + session = _seed_session("ws-truncation-direct-admission") + storage = get_storage() + original_memory = dicts_from_turns(session.messages) + _rows, token = _capture(session) + registration = session.register_listener_for_history_handoff(token) + assert registration is not None + listener = registration[0] + + truncate_entered = threading.Event() + release_truncate = threading.Event() + append_prepare_entered = threading.Event() + real_truncate = storage.truncate_messages_tail + real_prepare = session._prepare_direct_conversation_mutation + truncation_results: list[int] = [] + errors: list[BaseException] = [] + + def _frozen_truncate(ws_id: str, remove_count: int) -> int: + truncate_entered.set() + assert release_truncate.wait(5), "test did not release strict truncation" + return real_truncate(ws_id, remove_count) + + def _observed_prepare(deferred: Any) -> None: + append_prepare_entered.set() + real_prepare(deferred) + + def _rewind() -> None: + try: + truncation_results.append(session.rewind(1)) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + def _append() -> None: + try: + session._append_system_turn("correction", "accepted after cut") + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + truncator = threading.Thread(target=_rewind, daemon=True, name="frozen-truncation") + appender = threading.Thread(target=_append, daemon=True, name="direct-system-append") + with ( + patch.object(storage, "truncate_messages_tail", side_effect=_frozen_truncate), + patch.object( + session, + "_prepare_direct_conversation_mutation", + side_effect=_observed_prepare, + ), + ): + truncator.start() + assert truncate_entered.wait(5), "truncation did not enter strict storage cut" + appender.start() + try: + assert append_prepare_entered.wait(5), "direct append did not reach its barrier" + assert session._history_truncation_active is True + assert dicts_from_turns(session.messages) == original_memory + assert listener.empty() + finally: + release_truncate.set() + truncator.join(5) + appender.join(5) + + assert not truncator.is_alive() and not appender.is_alive() + assert errors == [] + assert truncation_results == [2] + expected = [ + ("user", "first request", None), + ("assistant", "first answer", None), + ("system", "accepted after cut", "correction"), + ] + memory_rows = dicts_from_turns(session.messages) + durable_rows = storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + assert [(row.get("role"), row.get("content"), row.get("_source")) for row in memory_rows] == ( + expected + ) + assert [(row.get("role"), row.get("content"), row.get("_source")) for row in durable_rows] == ( + expected + ) + events: list[dict[str, Any]] = [] + while not listener.empty(): + events.append(listener.get_nowait()) + assert [event.get("type") for event in events] == ["history_resync", "system_turn"] + assert events[0].get("reason") == "history_truncated" + assert events[1].get("content") == "accepted after cut" + + +def test_truncation_freezes_generation_commit_admission(tmp_db: Any) -> None: + """The total-prefix latch covers the shared generation commit primitive.""" + + session = _seed_session("ws-truncation-generation-admission") + storage = get_storage() + truncate_entered = threading.Event() + release_truncate = threading.Event() + commit_attempted = threading.Event() + commit_ran = threading.Event() + real_truncate = storage.truncate_messages_tail + errors: list[BaseException] = [] + + def _frozen_truncate(ws_id: str, remove_count: int) -> int: + truncate_entered.set() + assert release_truncate.wait(5), "test did not release strict truncation" + return real_truncate(ws_id, remove_count) + + def _rewind() -> None: + try: + session.rewind(1) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + def _commit() -> None: + commit_attempted.set() + try: + assert session._commit_for_generation( + 0, + lambda _durable: commit_ran.set(), + ) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + truncator = threading.Thread(target=_rewind, daemon=True, name="frozen-truncation") + committer = threading.Thread(target=_commit, daemon=True, name="generation-commit") + with patch.object(storage, "truncate_messages_tail", side_effect=_frozen_truncate): + truncator.start() + assert truncate_entered.wait(5), "truncation did not enter strict storage cut" + committer.start() + try: + assert commit_attempted.wait(5), "generation commit did not start" + assert session._history_truncation_active is True + assert commit_ran.is_set() is False + finally: + release_truncate.set() + truncator.join(5) + committer.join(5) + + assert not truncator.is_alive() and not committer.is_alive() + assert errors == [] + assert commit_ran.is_set() + + +def test_truncation_waiting_for_older_ticket_keeps_commit_admission_open( + tmp_db: Any, +) -> None: + """Waiting for the old FIFO prefix must not install the cut latch early.""" + + session = _seed_session("ws-truncation-older-ticket") + storage = get_storage() + older_ticket_entered = threading.Event() + release_older_ticket = threading.Event() + truncation_wait_entered = threading.Event() + strict_cut_entered = threading.Event() + probe_admitted = threading.Event() + real_wait_for = session._durability_cond.wait_for + real_truncate = storage.truncate_messages_tail + results: list[int] = [] + errors: list[BaseException] = [] + + def _hold_older_ticket() -> None: + def _admit(durable: list[Any]) -> None: + def _persist() -> None: + older_ticket_entered.set() + assert release_older_ticket.wait(5), "test did not release older ticket" + + durable.append(_persist) + + try: + assert session._commit_for_generation(0, _admit) is True + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + def _observe_wait(predicate: Any, timeout: float | None = None) -> bool: + if threading.current_thread().name == "waiting-truncation": + truncation_wait_entered.set() + return real_wait_for(predicate, timeout) + + def _observe_truncate(ws_id: str, remove_count: int) -> int: + strict_cut_entered.set() + return real_truncate(ws_id, remove_count) + + def _rewind() -> None: + try: + results.append(session.rewind(1)) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + older = threading.Thread(target=_hold_older_ticket, daemon=True, name="older-ticket") + truncator = threading.Thread(target=_rewind, daemon=True, name="waiting-truncation") + older.start() + assert older_ticket_entered.wait(5), "older ticket never entered durability" + with ( + patch.object(session._durability_cond, "wait_for", side_effect=_observe_wait), + patch.object(storage, "truncate_messages_tail", side_effect=_observe_truncate), + ): + truncator.start() + try: + assert truncation_wait_entered.wait(5), "truncation did not wait for old ticket" + with session._generation_lock: + assert session._history_truncation_active is False + assert strict_cut_entered.is_set() is False + + def _probe(_durable: list[Any]) -> None: + probe_admitted.set() + + assert session._commit_for_generation(0, _probe) is True + assert probe_admitted.is_set() + finally: + release_older_ticket.set() + older.join(5) + truncator.join(5) + + assert not older.is_alive() and not truncator.is_alive() + assert errors == [] + assert strict_cut_entered.is_set() + assert results == [2] + + +def test_unresolved_prefix_repair_failure_refuses_truncation_without_cut_publication( + tmp_db: Any, +) -> None: + """The pending FIFO prefix must repair before a destructive cut can run.""" + + session = _seed_session("ws-truncation-unresolved-prefix") + storage = get_storage() + _rows, initial_token = _capture(session) + registration = session.register_listener_for_history_handoff(initial_token) + assert registration is not None + listener = registration[0] + + with ( + patch("turnstone.core.session.save_message", return_value=0), + pytest.raises(ConversationPersistenceError), + ): + session._append_system_turn("correction", "ambiguous predecessor") + + initial_events: list[dict[str, Any]] = [] + while not listener.empty(): + initial_events.append(listener.get_nowait()) + assert [event.get("type") for event in initial_events] == [ + "system_turn", + "history_resync", + ] + assert initial_events[-1].get("reason") == "conversation_persistence_unresolved" + + memory_before = dicts_from_turns(session.messages) + tokens_before = list(session._msg_tokens) + generation_before = session._history_generation + durable_before = storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + _rows, token_before = _capture(session) + + with ( + patch("turnstone.core.session.save_message", return_value=0) as repair, + patch.object( + storage, "truncate_messages_tail", wraps=storage.truncate_messages_tail + ) as cut, + pytest.raises(ConversationPersistenceError), + ): + session.rewind(1) + + # A destructive mutation cannot bypass the transient backoff. Pre-due it + # fails immediately on the stored poison without hammering storage. + repair.assert_not_called() + cut.assert_not_called() + assert session._history_truncation_active is False + assert dicts_from_turns(session.messages) == memory_before + assert session._msg_tokens == tokens_before + assert session._history_generation == generation_before + assert ( + storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + == durable_before + ) + _rows, token_after = _capture(session) + assert token_after == token_before + retry_events: list[dict[str, Any]] = [] + while not listener.empty(): + retry_events.append(listener.get_nowait()) + assert retry_events == [] + assert all(event.get("reason") != "history_truncated" for event in retry_events) + + +@pytest.mark.parametrize("terminal_kind", ["soft", "hard"]) +def test_terminal_waits_for_admitted_truncation_ticket( + tmp_db: Any, + terminal_kind: str, +) -> None: + """A terminal boundary drains an already-admitted destructive cut.""" + + session = _seed_session(f"ws-truncation-{terminal_kind}-terminal") + storage = get_storage() + truncate_entered = threading.Event() + release_truncate = threading.Event() + terminal_wait_entered = threading.Event() + terminal_done = threading.Event() + real_truncate = storage.truncate_messages_tail + real_wait_for = session._durability_cond.wait_for + truncation_results: list[int] = [] + terminal_results: list[bool] = [] + errors: list[BaseException] = [] + + def _frozen_truncate(ws_id: str, remove_count: int) -> int: + truncate_entered.set() + assert release_truncate.wait(5), "test did not release admitted truncation" + return real_truncate(ws_id, remove_count) + + def _observe_wait(predicate: Any, timeout: float | None = None) -> bool: + if threading.current_thread().name == f"{terminal_kind}-terminal": + terminal_wait_entered.set() + return real_wait_for(predicate, timeout) + + def _rewind() -> None: + try: + truncation_results.append(session.rewind(1)) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + def _terminate() -> None: + try: + if terminal_kind == "soft": + terminal_results.append(bool(session.prepare_soft_close())) + else: + session.shutdown_publication_and_drain_durability() + terminal_results.append(True) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + finally: + terminal_done.set() + + truncator = threading.Thread(target=_rewind, daemon=True, name="admitted-truncation") + terminal = threading.Thread( + target=_terminate, + daemon=True, + name=f"{terminal_kind}-terminal", + ) + with ( + patch.object(storage, "truncate_messages_tail", side_effect=_frozen_truncate), + patch.object(session._durability_cond, "wait_for", side_effect=_observe_wait), + ): + truncator.start() + assert truncate_entered.wait(5), "truncation did not enter strict storage cut" + terminal.start() + try: + assert terminal_wait_entered.wait(5), "terminal did not enter durability drain" + with session._durability_cond: + assert session._durability_serving_ticket < session._durability_next_ticket + assert terminal_done.is_set() is False + finally: + release_truncate.set() + truncator.join(5) + terminal.join(5) + + assert not truncator.is_alive() and not terminal.is_alive() + assert errors == [] + assert truncation_results == [2] + assert terminal_results == [True] + assert session._publication_shutdown is True + assert [row.get("content") for row in dicts_from_turns(session.messages)] == [ + "first request", + "first answer", + ] + assert [ + row.get("content") + for row in storage.load_messages( + session.ws_id, + repair=False, + include_compaction=True, + ) + ] == ["first request", "first answer"] + + +class _ColdCrossingManager: + """Manager fake whose cold incarnation can be installed exactly once.""" + + def __init__(self, live: Workstream) -> None: + self._lock = threading.Lock() + self._live: Workstream | None = None + self._prepared = live + self.open_called = threading.Event() + + def get(self, ws_id: str) -> Workstream | None: + assert ws_id == self._prepared.id + with self._lock: + return self._live + + def install(self) -> Workstream: + with self._lock: + if self._live is None: + self._live = self._prepared + return self._live + + def open(self, ws_id: str) -> Workstream: + assert ws_id == self._prepared.id + self.open_called.set() + return self.install() + + +def test_cold_history_crossing_mid_install_stays_tokenless_and_unrehydrated(tmp_db: Any) -> None: + """A session installed mid-flight never lends its token to a stale snapshot. + + Deliberate pin update: /history no longer rehydrates cold rows, so the + crossing contract inverts — the flight that sampled a cold pool serves + the storage-only snapshot TOKENLESS (claiming no splice authority over + the row admitted during its load) and never calls ``mgr.open``. The + admitted row reaches panes through the tokenless bootstrap's clear_ui + convergence, or a later request whose ``mgr.get`` sees the session. + """ + + ws_id = "ws-cold-history-crossing" + register_workstream(ws_id, kind="interactive", user_id="test-user") + save_message(ws_id, "user", "durable prefix") + session = make_session(ws_id=ws_id, user_id="test-user") + live = Workstream( + id=ws_id, + user_id="test-user", + session=session, + ui=session.ui, + ) + manager = _ColdCrossingManager(live) + storage = get_storage() + real_load = storage.load_messages + load_entered = threading.Event() + release_load = threading.Event() + load_count = 0 + load_count_lock = threading.Lock() + + def _frozen_load(*args: Any, **kwargs: Any) -> list[dict[str, Any]]: + nonlocal load_count + rows = real_load(*args, **kwargs) + with load_count_lock: + load_count += 1 + this_load = load_count + if this_load == 1: + load_entered.set() + assert release_load.wait(5), "test did not release frozen cold history load" + return rows + + client = _build_history_app(manager, storage) + responses: list[Any] = [] + errors: list[BaseException] = [] + + def _request_history() -> None: + try: + responses.append(client.get(f"/v1/api/workstreams/{ws_id}/history")) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + requester = threading.Thread(target=_request_history, daemon=True) + with patch.object(storage, "load_messages", side_effect=_frozen_load): + requester.start() + try: + assert load_entered.wait(5), "cold history did not enter its frozen load" + manager.install() + deferred: list[Any] = [] + session._append_user_turn( + "accepted across rehydrate", + (), + deferred_persistence=deferred, + ) + finally: + release_load.set() + requester.join(5) + + assert not requester.is_alive() + assert errors == [] + assert len(responses) == 1 + response = responses[0] + assert response.status_code == 200 + body = response.json() + assert body["handoff_token"] is None + assert not manager.open_called.is_set() + assert [message.get("content") for message in body["messages"]] == ["durable prefix"] + # The admitted row is not lost — the live session's own capture (the path + # a token-bearing response would take) owns it. + merged, token = session.capture_history_handoff( + lambda _overscan: get_storage().load_messages(ws_id, repair=False) + ) + assert token + assert [message.get("content") for message in merged] == [ + "durable prefix", + "accepted across rehydrate", + ] + + +@pytest.mark.parametrize("operation", ["rewind", "retry"]) +def test_route_mutation_claim_precedes_concurrent_turn_admission(operation: str) -> None: + """The idle check and destructive mutation must be one worker-slot claim.""" + + mutation_entered = threading.Event() + release_mutation = threading.Event() + probe_spawned = threading.Event() + enqueue_refused = threading.Event() + session = MagicMock() + reset_events: list[dict[str, Any]] = [] + + def _blocking_mutation( + *_args: Any, + publish_reset: Any, + ) -> Any: + mutation_entered.set() + assert release_mutation.wait(5), "test did not release history mutation" + publish_reset() + return 2 if operation == "rewind" else "second request" + + if operation == "rewind": + session.rewind.side_effect = _blocking_mutation + else: + session.retry.side_effect = _blocking_mutation + ui = MagicMock() + ui._enqueue.side_effect = lambda event: reset_events.append(event) + ws = Workstream(id=f"ws-route-{operation}", session=session, ui=ui) + manager = MagicMock() + manager.get.return_value = ws + if operation == "rewind": + handler = make_rewind_handler(_verb_cfg(manager)) + client = _verb_client("/api/workstreams/{ws_id}/rewind", handler) + request = lambda: client.post( # noqa: E731 + f"/v1/api/workstreams/{ws.id}/rewind", + json={"turns": 1}, + ) + else: + handler = make_retry_handler(_verb_cfg(manager)) + client = _verb_client("/api/workstreams/{ws_id}/retry", handler) + request = lambda: client.post(f"/v1/api/workstreams/{ws.id}/retry") # noqa: E731 + + responses: list[Any] = [] + errors: list[BaseException] = [] + + def _request_mutation() -> None: + try: + responses.append(request()) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + def _refuse_enqueue() -> None: + enqueue_refused.set() + raise queue.Full + + requester = threading.Thread(target=_request_mutation, daemon=True) + requester.start() + try: + assert mutation_entered.wait(5), "route did not enter its history mutation" + admitted = session_worker.send( + ws, + enqueue=_refuse_enqueue, + run=probe_spawned.set, + thread_name=f"concurrent-{operation}-probe", + ) + assert reset_events == [] + finally: + release_mutation.set() + requester.join(5) + + assert not requester.is_alive() + assert errors == [] + assert len(responses) == 1 + assert responses[0].status_code == 200 + assert admitted is False + assert enqueue_refused.is_set() + assert not probe_spawned.is_set() + assert reset_events == [{"type": "clear_ui"}] + + +@pytest.mark.parametrize("operation", ["rewind", "retry"]) +def test_truncation_counts_only_durable_rows_across_live_only_turns( + tmp_db: Any, + operation: str, +) -> None: + """A live-only empty assistant turn must not cost an older durable row. + + The empty completion flows through the real provider path so the + admission-site ``no_durable_row`` marker — not test scaffolding — carries + the live/durable asymmetry into the cut accounting. + """ + from tests._session_helpers import RecordingUI, arm_session + from turnstone.core.providers import StreamChunk + + ws_id = f"ws-{operation}-live-only-turn" + register_workstream(ws_id, kind="interactive", user_id="test-user") + session = make_session(ws_id=ws_id, user_id="test-user", ui=RecordingUI()) + arm_session( + session, + iter([StreamChunk(content_delta="first answer", finish_reason="stop")]), + iter([StreamChunk(finish_reason="stop")]), + ) + session.send("first request") + session.send("second request") + + assert session.messages[-1].meta.extra.get("no_durable_row") is True + storage = get_storage() + durable_before = [row.get("content") for row in storage.load_messages(ws_id, repair=False)] + assert durable_before == ["first request", "first answer", "second request"] + + result = _truncate(session, operation) + + if operation == "rewind": + assert result == 2 + else: + assert result == "second request" + durable_after = [row.get("content") for row in storage.load_messages(ws_id, repair=False)] + assert durable_after == ["first request", "first answer"] + assert [turn.text for turn in session.messages] == ["first request", "first answer"] diff --git a/tests/test_idle_nudge_wake_integration.py b/tests/test_idle_nudge_wake_integration.py index faa6bdc8..7578e03d 100644 --- a/tests/test_idle_nudge_wake_integration.py +++ b/tests/test_idle_nudge_wake_integration.py @@ -1225,6 +1225,30 @@ def test_interjection_handoff_dispatches_exactly_one_real_send(tmp_db): assert session._wake_source_tag == "" +def test_interjection_handoff_forwards_nonempty_client_send_ids(tmp_db): + """Correlated queued rows retain every browser token in queue order.""" + from tests._helpers import make_chat_session + + session = make_chat_session() + session._nudge_queue.enqueue("idle_tasks", "open tasks remain", "wake") + session.queue_message("first", client_send_id="browser-first") + session.queue_message("second", client_send_id="browser-second") + calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + def _recording_send(*args: Any, **kwargs: Any) -> None: + calls.append((args, kwargs)) + + session.send = _recording_send # type: ignore[method-assign] + session.deliver_wake_nudge_from_queue() + + assert calls == [ + ( + ("first\n\nsecond",), + {"client_send_ids": ("browser-first", "browser-second")}, + ) + ] + + def test_interjection_handoff_contains_generation_cancelled(tmp_db): """A Stop landing inside the handed-off interjection send must not escape the wake worker: ``GenerationCancelled`` is a BaseException @@ -1270,7 +1294,7 @@ def test_interjection_handoff_restores_the_queue_when_send_raises(tmp_db): # Restored verbatim: same id, same cleaned text, same priority. assert msg_id in session._queued_messages - text, priority = session._queued_messages[msg_id] + text, priority = session._queued_messages[msg_id][:2] assert text == "do not lose this" assert priority == "important" @@ -1350,6 +1374,33 @@ def test_interjection_handoff_skips_the_pop_when_budget_exhausted(tmp_db): assert any(a == ("",) for a, _k in sends) +def test_interjection_handoff_skips_the_pop_on_a_gone_workstream(tmp_db): + """The delivery-site gate must be at least as strong as the claim gate: + a nudge-driven wake reaches this method without ever consulting + ``claim_pending_interjection_wake``, and under the gone latch send's + admission refusal is converged INTERNALLY (no re-raise), so a pop here + would destroy the user's words with no restore arm running. No pop, + rows retained — their disposition on a deleted workstream is #1001's.""" + from tests._helpers import make_chat_session + + session = make_chat_session() + session._nudge_queue.enqueue("idle_tasks", "open tasks remain", "wake") + _c, _p, msg_id = session.queue_message("held message") + session._workstream_gone_ws = session._ws_id + + sends: list[tuple[Any, ...]] = [] + + def _recording_send(*a: Any, **k: Any) -> None: + sends.append((a, k)) + + session.send = _recording_send # type: ignore[method-assign] + session.deliver_wake_nudge_from_queue() + + assert msg_id in session._queued_messages + assert session._popped_in_flight == set() + assert all(args != ("held message",) for args, _k in sends) + + def test_interjection_handoff_falls_through_on_content_free_items(tmp_db): """A bare priority marker ('!!!') renders as nothing deliverable: the handoff must not spend the seam on a content-free user turn — diff --git a/tests/test_idle_nudge_watcher.py b/tests/test_idle_nudge_watcher.py index 4ab9602b..c61c6f2d 100644 --- a/tests/test_idle_nudge_watcher.py +++ b/tests/test_idle_nudge_watcher.py @@ -318,9 +318,19 @@ class TestWakeWorkstreamIfPending: _mgr, ws = fake_mgr_and_ws ws.session._nudge_queue.enqueue("watch_triggered", "output", "any") - def _reuse_send(_ws: Any, *, enqueue: Any, run: Any, thread_name: Any) -> bool: + def _reuse_send( + _ws: Any, + *, + enqueue: Any, + run: Any, + expected_session: Any, + interjection_wake_signature: Any, + thread_name: Any, + ) -> bool: # Mimic a live worker owning the workstream: send routes the # wake to the no-op enqueue rather than spawning a daemon. + assert expected_session is ws.session + assert interjection_wake_signature is None enqueue() return True diff --git a/tests/test_interactive_pane_js.py b/tests/test_interactive_pane_js.py index c9344134..b486166b 100644 --- a/tests/test_interactive_pane_js.py +++ b/tests/test_interactive_pane_js.py @@ -15,6 +15,8 @@ from pathlib import Path import pytest +from tests._js_harness_helpers import strip_js_comments as _strip_comments + _ROOT = Path(__file__).resolve().parent.parent _INTERACTIVE = _ROOT / "turnstone/shared_static/interactive.js" _COMPOSER = _ROOT / "turnstone/shared_static/composer.js" @@ -23,12 +25,6 @@ _APP = _ROOT / "turnstone/ui/static/app.js" _UI_INDEX = _ROOT / "turnstone/ui/static/index.html" -def _strip_comments(js: str) -> str: - js = re.sub(r"/\*.*?\*/", "", js, flags=re.S) - js = re.sub(r"//[^\n]*", "", js) - return js - - def test_interactive_is_esm_imported_by_the_shell() -> None: """Real ES module: it ``export``s the factory the shell imports in BOTH deployments. Step 6 retired the window bridge (no window.InteractivePane) @@ -308,10 +304,12 @@ def test_rebuild_quiesces_live_events_and_releases_agent_tracking() -> None: # _loadHistoryThenConnect; terminal cleanup in the factory's destroy(). assert "this._agentCards.delete(callId);" not in body disc = body.index("disconnectSSE() {") - disc_seg = body[disc : body.index("_loadHistoryThenConnect(wsId) {", disc)] + disc_seg = body[ + disc : body.index("_loadHistoryThenConnect(wsId, manualAttempt = false) {", disc) + ] assert "this._clearAgentTracking();" not in disc_seg assert "this._replayQueue = null;" not in disc_seg - load = body.index("_loadHistoryThenConnect(wsId) {") + load = body.index("_loadHistoryThenConnect(wsId, manualAttempt = false) {") load_seg = body[load : body.index("async _refetchHistory(", load)] assert "this._clearAgentTracking();" in load_seg assert "this._replayQueue = null;" in load_seg @@ -384,7 +382,7 @@ def test_interactive_refetch_failure_preserves_the_pane() -> None: # again. The gate must be seedless-SCOPED — _loadHistoryThenConnect # disconnects first, so its evtSource is null for its whole fetch and # gating it would break every first paint, ws switch and resync. - ref = body.index("async _refetchHistory(wsId, token, seedCursor = false) {") + ref = body.index("async _refetchHistory(") ref_seg = body[ref : body.index("\n _beginReplayQuiesce(token) {", ref)] gate = ref_seg.index("const cursorSafe =") gate_seg = ref_seg[gate : ref_seg.index(";", gate)] @@ -487,7 +485,7 @@ def test_interactive_refetch_failure_preserves_the_pane() -> None: # survive a reload only when an armed truncation cursor lets the # reconnect resume into them; every other flavor (ws switch, # unarmed re-auth reload) resets. - lh = body.index("_loadHistoryThenConnect(wsId) {") + lh = body.index("_loadHistoryThenConnect(wsId, manualAttempt = false) {") lh_seg = body[lh : body.index("async _refetchHistory(", lh)] assert "if (this._truncatedFromCursor == null) this._resetStreamingRefs();" in lh_seg, ( "reload must reset streaming refs unless a truncation resync is armed (#890)" @@ -529,18 +527,30 @@ def test_interactive_refetch_failure_preserves_the_pane() -> None: "the idle edge must backstop the latch behind the truncated branch, " "skipping edges with a quiesced fetch already in flight" ) - # The backstop must be TRANSPORT-FREE: a quiesced same-token refetch, - # never _loadHistoryThenConnect — the reload's fresh reconnect draws - # the server's synthetic state_change:idle back into this branch's - # own trigger (the round-5 storm). + # The backstop must defer to the current event-backlog tail, then remain + # TRANSPORT-FREE: a quiesced same-token refetch, never + # _loadHistoryThenConnect. A synchronous refetch here lets replay_ok's + # leading synthetic idle split the canonical backlog around a /history + # repaint; a transport reload draws another synthetic idle into this + # branch's own trigger (the round-5 storm). backstop = body.index("} else if (this._historyStale && !this._replayQueue) {") - backstop_seg = body[backstop : backstop + 2200] - assert "this._refetchHistory(this.wsId, staleToken);" in backstop_seg, ( - "the staleness backstop must heal via a quiesced REST refetch" + backstop_seg = body[backstop : body.index("// Only steal focus", backstop)] + assert "this._deferStaleHistoryBackstop();" in backstop_seg, ( + "the idle edge must defer its stale heal to the event-backlog tail" ) assert "this._loadHistoryThenConnect(" not in backstop_seg, ( "the staleness backstop must never touch the transport (#890 r5)" ) + deferred = body.index("_deferStaleHistoryBackstop() {") + deferred_seg = body[deferred : body.index("\n _clearAgentTracking() {", deferred)] + assert "queueMicrotask(() => {" in deferred_seg + assert "this._beginReplayQuiesce(staleToken);" in deferred_seg + assert "this._refetchHistory(staleWs, staleToken);" in deferred_seg, ( + "the deferred staleness backstop must heal via a quiesced REST refetch" + ) + assert "this._loadHistoryThenConnect(" not in deferred_seg, ( + "the deferred staleness backstop must remain transport-free (#890 r5)" + ) # The retry yields to an in-flight quiesce (no same-token stomp). retry = cl_seg.index("this._staleRetryTimer = setTimeout(") assert "!this._replayQueue &&" in cl_seg[retry : retry + 700], ( @@ -668,21 +678,31 @@ def test_pane_gates_send_on_cross_user_busy() -> None: # ...and drives the composer's hard block, re-run on every busy edge. assert "this.composer.setSendBlocked(" in body stripped = _strip_comments(body) + # The shared stripper is offset-preserving (comments become spaces), so + # slice to the method's real closing brace instead of a fixed byte + # window a comment edit could silently outgrow. setbusy = stripped.index("setBusy(b, source) {") - assert "this._reconcileSendBlock();" in stripped[setbusy : setbusy + 800] + setbusy_end = stripped.index("\n }", setbusy) + assert "this._reconcileSendBlock();" in stripped[setbusy:setbusy_end] def test_pane_handles_cross_user_409() -> None: """The reactive fallback: a 409 (button not yet disabled) surfaces a clean - message, not the generic 'Connection error' catch. The pane converts - the 409 body at the fetch stage; the status ARM itself lives in the - shared settle helper (composer_queue.settleSendResponse) with the rest - of the response matrix.""" - body = _INTERACTIVE.read_text(encoding="utf-8") - assert "r.status === 409" in body - assert 'status: "cross_user_interjection"' in body + message, not the generic 'Connection error' catch. Both the fetch-stage + conversion and the status ARM now live in the shared helper + (composer_queue.postAndSettleSend / settleSendResponse), so the pane owns + only the request — every send flow it has reaches the conversion by + construction instead of re-deriving it (the edit-and-resend flow used to + lack it and reported a refused resend as a connection error).""" helper = (_ROOT / "turnstone/shared_static/composer_queue.js").read_text(encoding="utf-8") + assert "response.status === 409" in helper + assert 'status: "cross_user_interjection"' in helper assert 'status === "cross_user_interjection"' in helper + body = _INTERACTIVE.read_text(encoding="utf-8") + assert "cross_user_interjection" not in body, ( + "the 409 conversion must not be re-derived per pane" + ) + assert body.count("postAndSettleSend(") == 2, "composer send + edit-and-resend" def test_sync_approval_state_prunes_orphan_cycles() -> None: @@ -860,7 +880,7 @@ def test_truncated_resync_is_full_fresh_connect_with_churn_limit() -> None: r"this\._truncatedFromCursor = this\._lastEventId;", t, ), "the truncated case must record the truncation-time cursor keep-oldest" - load = body.index("_loadHistoryThenConnect(wsId) {") + load = body.index("_loadHistoryThenConnect(wsId, manualAttempt = false) {") load_seg = body[load : body.index("async _refetchHistory(", load)] assert "if (this.wsId !== wsId) this._truncatedFromCursor = null;" in load_seg, ( "a ws switch must drop the old ws's truncation record" @@ -1202,8 +1222,15 @@ def test_deferred_send_settle_protocol_pins() -> None: # consume the pane-tier settle event; the busy stamp is centralized in # each pane's setBusy (source defaults to "server" — only the send # flow's optimistic flip may ever be undone). + # postAndSettleSend wraps settleSendResponse with the fetch stage (rejected + # -body normalization, the 409 conversion, the accepted-guarded transport + # catch), so a pane reaching the settle matrix at all now proves it reached + # the whole choreography — the panes must not call settleSendResponse + # directly, which is how the edit-and-resend flows drifted. + assert "export function postAndSettleSend(queue, sendRequest, ctx)" in composer_queue for name, src in (("interactive.js", interactive), ("coordinator.js", coordinator)): - assert "settleSendResponse(" in src, f"{name}: settle matrix must be the shared helper" + assert "postAndSettleSend(" in src, f"{name}: settle matrix must be the shared helper" + assert "settleSendResponse(" not in src, f"{name}: must not bypass the fetch stage" assert "busyIsOptimistic" in src, name assert "paneIsBusy" in src, f"{name}: the missed-edge settle needs the live flag" assert 'setBusy(true, "optimistic")' in src, f"{name}: optimistic flip must stamp" @@ -1303,3 +1330,62 @@ console.log("settle matrix OK"); timeout=15, ) assert proc.returncode == 0, f"settle harness failed:\n{proc.stderr}\n{proc.stdout}" + + +def test_accepted_tool_event_recorded_only_when_painted() -> None: + """An unpainted accepted tool_result must stay replayable. + + appendToolOutput returns false on every no-target path (transcript wiped + by clear_ui with the refetch in flight, a fresh mid-turn join); recording + the event id anyway would dedupe the ring's later replay and permanently + lose the tool's final guarded output. + """ + body = _INTERACTIVE.read_text(encoding="utf-8") + case_start = body.index('case "tool_result"') + case = body[case_start : body.index("case ", case_start + 20)] + gate = case.index("if (\n this.appendToolOutput(") + record = case.index("recordAcceptedToolEvent(this._renderedToolEventIds, evt)") + assert gate < record + + +def test_replay_system_rows_do_not_terminate_the_tool_batch_window() -> None: + """A system row inside a tool batch is not a turn boundary: nulling + ``lastToolBlock`` in the ``role === "system"`` replay branch made every + tool result AFTER an interleaved row (mid-turn operator context, a + second writer's append) silently vanish from this pane while the + coordinator — whose indexHistoryToolOutcomes skips non-turn rows — + rendered the identical history correctly. Only the user/assistant + branches may reset the anchor.""" + body = _strip_comments(_INTERACTIVE.read_text(encoding="utf-8")) + start = body.index('(msg.role === "system")') + end = body.index("for (const leftovers of pendingAssessments.values())", start) + system_branch = body[start:end] + assert "lastToolBlock = null" not in system_branch, ( + "the system replay branch terminates the batch result window — " + "interleaved-row tool results are dropped again" + ) + # Mutation control: the anchor resets still exist in the turn branches + # (user, nudge-marker, assistant content/reasoning/pending arms). + loop = body[body.index("let lastToolBlock = null") : end] + assert loop.count("lastToolBlock = null") >= 4 + + +def test_orphan_tool_result_does_not_mark_a_batch_failed() -> None: + """Keeping the batch anchor live across non-turn rows means a tool row + that names a call_id this batch never issued (a result for an earlier + batch, a second writer's append) can reach the error stamp. It must + not mark an all-succeeded batch as failed — the shared outcome index + skips unmatched occurrences for exactly this reason. A row with NO + call_id is the legacy positional case and must still stamp, so the + guard keys on 'named a call_id we could not resolve', not on the + absence of a resolved target.""" + body = _strip_comments(_INTERACTIVE.read_text(encoding="utf-8")) + assert "const isOrphanResult = !!msg.tool_call_id && !resultTarget;" in body, ( + "the orphan discriminator must distinguish an unresolvable call_id " + "from a legacy row that carries none" + ) + stamp = body.index("appendToolErrorBadge(lastToolBlock)") + guard = body.rindex("if (", 0, stamp) + assert "!isOrphanResult" in body[guard:stamp], ( + "an orphan result can stamp conv-batch--error on a batch whose own calls all succeeded" + ) diff --git a/tests/test_midstream_retry.py b/tests/test_midstream_retry.py index ee6d612b..f14db632 100644 --- a/tests/test_midstream_retry.py +++ b/tests/test_midstream_retry.py @@ -45,6 +45,16 @@ def _make_session(ui=None, **kwargs): them here is exactly the drift its docstring warns about. """ session = make_session(ui=ui or RecordingUI(), **kwargs) + # Production creates the durable workstream before a live session may + # admit keyed conversation rows. These direct-session tests must mirror + # that ordering: bypassing it would weaken the hard-delete parent fence. + from turnstone.core.storage import get_storage + + get_storage().register_workstream( + session.ws_id, + user_id=session._user_id, + kind=session._kind, + ) session._RETRY_BASE_DELAY = 0 # Latch the auto-title trigger: its background thread would drive a # second model call through the MagicMock client (drain retries, real @@ -715,14 +725,21 @@ class TestRecreateWindowClassification: # re-create sequence, so a cancel fired there raises at the next # walk's loop-top _check_cancelled — the exact window. real_refresh = session._refresh_model_from_registry + refresh_calls = 0 def cancel_in_window(): + nonlocal refresh_calls + refresh_calls += 1 real_refresh() - session._cancel_event.set() + # send() performs one preflight refresh before the first attempt; + # the second refresh is the retry re-create seam this test owns. + if refresh_calls == 2: + session._cancel_event.set() with patch.object(session, "_refresh_model_from_registry", side_effect=cancel_in_window): session.send("test") + assert refresh_calls == 2 # The dead attempt streamed only its safe-flush prefix ("Ha"); # the splitter carry ("lf an answer") must NEVER surface as a late # content token behind a duplicate stream_end. diff --git a/tests/test_migration_071.py b/tests/test_migration_071.py new file mode 100644 index 00000000..f16cbca4 --- /dev/null +++ b/tests/test_migration_071.py @@ -0,0 +1,179 @@ +"""Migration coverage for idempotent conversation commit keys.""" + +from __future__ import annotations + +import contextlib +import importlib +from pathlib import Path +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any + +import pytest +import sqlalchemy as sa +from alembic import command +from alembic.config import Config + +if TYPE_CHECKING: + from collections.abc import Iterator + +_MIGRATIONS_DIR = str( + Path(__file__).resolve().parent.parent / "turnstone" / "core" / "storage" / "migrations" +) + + +def _alembic_cfg(db_path: Path) -> Config: + cfg = Config() + cfg.set_main_option("script_location", _MIGRATIONS_DIR) + cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}") + return cfg + + +class TestMigration071: + def test_upgrade_preserves_legacy_rows_and_enforces_scoped_key_uniqueness( + self, tmp_path: Path + ) -> None: + db_path = tmp_path / "071-up.db" + cfg = _alembic_cfg(db_path) + command.upgrade(cfg, "070") + engine = sa.create_engine(f"sqlite:///{db_path}") + try: + with engine.begin() as conn: + conn.execute( + sa.text( + "INSERT INTO conversations (ws_id, timestamp, role, content) " + "VALUES ('legacy', '2026-01-01T00:00:00', 'assistant', 'same'), " + "('legacy', '2026-01-01T00:00:01', 'assistant', 'same')" + ) + ) + + command.upgrade(cfg, "071") + + with engine.begin() as conn: + assert ( + conn.execute( + sa.text("SELECT COUNT(*) FROM conversations WHERE commit_key IS NULL") + ).scalar_one() + == 2 + ) + # NULL keeps append-only legacy semantics under the unique index. + conn.execute( + sa.text( + "INSERT INTO conversations " + "(ws_id, timestamp, role, content, commit_key) VALUES " + "('legacy', '2026-01-01T00:00:02', 'assistant', 'same', NULL)" + ) + ) + conn.execute( + sa.text( + "INSERT INTO conversations " + "(ws_id, timestamp, role, content, commit_key) VALUES " + "('keyed-a', '2026-01-01T00:00:03', 'assistant', 'a', 'key-1'), " + "('keyed-b', '2026-01-01T00:00:04', 'assistant', 'b', 'key-1')" + ) + ) + with pytest.raises(sa.exc.IntegrityError): + conn.execute( + sa.text( + "INSERT INTO conversations " + "(ws_id, timestamp, role, content, commit_key) VALUES " + "('keyed-a', '2026-01-01T00:00:05', 'assistant', 'dup', 'key-1')" + ) + ) + finally: + engine.dispose() + + @pytest.mark.parametrize("invalid_index", [False, True]) + def test_postgresql_upgrade_is_restart_safe_and_repairs_invalid_index( + self, + monkeypatch: pytest.MonkeyPatch, + invalid_index: bool, + ) -> None: + migration = importlib.import_module( + "turnstone.core.storage.migrations.versions.071_conversations_commit_key" + ) + + class _Result: + def __init__(self, value: bool) -> None: + self.value = value + + def scalar_one_or_none(self) -> bool: + return self.value + + class _Bind: + dialect = SimpleNamespace(name="postgresql") + + def __init__(self) -> None: + self.queries: list[str] = [] + self.invalid_index = invalid_index + + def execute(self, statement: Any) -> _Result: + self.queries.append(str(statement)) + return _Result(self.invalid_index) + + class _Context: + @contextlib.contextmanager + def autocommit_block(self) -> Iterator[None]: + yield + + class _Op: + def __init__(self, bind: _Bind) -> None: + self.bind = bind + self.ddl: list[str] = [] + + def get_bind(self) -> _Bind: + return self.bind + + def get_context(self) -> _Context: + return _Context() + + def execute(self, statement: str) -> None: + self.ddl.append(statement) + if statement.startswith(("DROP INDEX", "CREATE UNIQUE INDEX")): + self.bind.invalid_index = False + + bind = _Bind() + fake_op = _Op(bind) + monkeypatch.setattr(migration, "op", fake_op) + + # Running twice models a revision left unstamped after either durable + # DDL operation. Both statements remain safe on the second attempt. + migration.upgrade() + migration.upgrade() + + assert ( + fake_op.ddl.count("ALTER TABLE conversations ADD COLUMN IF NOT EXISTS commit_key TEXT") + == 2 + ) + creates = [statement for statement in fake_op.ddl if statement.startswith("CREATE UNIQUE")] + assert len(creates) == 2 + assert all("CONCURRENTLY IF NOT EXISTS" in statement for statement in creates) + drops = [statement for statement in fake_op.ddl if statement.startswith("DROP INDEX")] + assert len(drops) == (1 if invalid_index else 0) + assert bind.queries and all("NOT i.indisvalid" in query for query in bind.queries) + + def test_downgrade_then_upgrade_round_trip(self, tmp_path: Path) -> None: + db_path = tmp_path / "071-roundtrip.db" + cfg = _alembic_cfg(db_path) + command.upgrade(cfg, "071") + command.downgrade(cfg, "070") + engine = sa.create_engine(f"sqlite:///{db_path}") + try: + columns = {c["name"] for c in sa.inspect(engine).get_columns("conversations")} + indexes = {i["name"] for i in sa.inspect(engine).get_indexes("conversations")} + assert "commit_key" not in columns + assert "uq_conversations_ws_commit_key" not in indexes + finally: + engine.dispose() + + command.upgrade(cfg, "071") + engine = sa.create_engine(f"sqlite:///{db_path}") + try: + columns = {c["name"] for c in sa.inspect(engine).get_columns("conversations")} + indexes = {i["name"]: i for i in sa.inspect(engine).get_indexes("conversations")} + assert "commit_key" in columns + assert bool(indexes["uq_conversations_ws_commit_key"]["unique"]) + assert "commit_key IS NOT NULL" in str( + indexes["uq_conversations_ws_commit_key"]["dialect_options"]["sqlite_where"] + ) + finally: + engine.dispose() diff --git a/tests/test_model_registry.py b/tests/test_model_registry.py index c751e34e..8d45cba7 100644 --- a/tests/test_model_registry.py +++ b/tests/test_model_registry.py @@ -1465,6 +1465,19 @@ def _make_session( ) +def _make_durable_session(**kwargs: Any) -> Any: + """Create a direct session with production's parent-before-row order.""" + from turnstone.core.storage import get_storage + + session = _make_session(**kwargs) + get_storage().register_workstream( + session.ws_id, + user_id=session._user_id, + kind=session._kind, + ) + return session + + def _binding(session: Any) -> Any: return session._model_binding @@ -3043,7 +3056,7 @@ class TestSessionRemovedAliasDegradedTurns: app_state=_KEYED_STATE, ) - def test_fallback_carries_turn_after_alias_deletion(self, caplog: Any) -> None: + def test_fallback_carries_turn_after_alias_deletion(self, tmp_db: str, caplog: Any) -> None: """Deleting a live session's alias degrades the turn onto the configured fallback instead of killing every subsequent send.""" import logging @@ -3051,7 +3064,7 @@ class TestSessionRemovedAliasDegradedTurns: reg = self._registry(fallback=["other"]) fb_client = reg.get_client("other") fb_client.chat.completions.create = scripted_chat_client({"content": "carried"}) - session = _make_session(registry=reg, model_alias="gw") + session = _make_durable_session(registry=reg, model_alias="gw") _client(session).chat.completions.create = MagicMock(side_effect=self._dead_client_error()) self._delete_gw(reg, fallback=["other"]) @@ -3066,11 +3079,11 @@ class TestSessionRemovedAliasDegradedTurns: ] assert len(removed_warns) == 1 # once per (alias, generation) - def test_no_fallback_turn_errors_with_removed_cause_and_model_remedy(self) -> None: + def test_no_fallback_turn_errors_with_removed_cause_and_model_remedy(self, tmp_db: str) -> None: """With no fallback the error names the alias-removed cause, not the raw closed-transport symptom.""" reg = self._registry() - session = _make_session(registry=reg, model_alias="gw") + session = _make_durable_session(registry=reg, model_alias="gw") _client(session).chat.completions.create = MagicMock(side_effect=self._dead_client_error()) self._delete_gw(reg) @@ -3083,11 +3096,11 @@ class TestSessionRemovedAliasDegradedTurns: assert "/model" in message # interactive lanes route slash commands assert "other" in message # the remedy lists what is available - def test_coordinator_error_omits_slash_model_remedy(self) -> None: + def test_coordinator_error_omits_slash_model_remedy(self, tmp_db: str) -> None: """The coordinator routes no slash commands, so its error carries recreate-or-adjust wording instead.""" reg = self._registry() - session = _make_session( + session = _make_durable_session( registry=reg, model_alias="gw", kind=WorkstreamKind.COORDINATOR, user_id="u1" ) _client(session).chat.completions.create = MagicMock(side_effect=self._dead_client_error()) @@ -3102,12 +3115,14 @@ class TestSessionRemovedAliasDegradedTurns: assert "/model" not in message assert "adjust the workstream model" in message - def test_recreated_broken_alias_reports_construction_cause(self, monkeypatch: Any) -> None: + def test_recreated_broken_alias_reports_construction_cause( + self, tmp_db: str, monkeypatch: Any + ) -> None: """A re-created alias reports the construction cause, never a stale "removed" diagnosis: the latch clears on the has_alias pass.""" reg = self._registry() - session = _make_session(registry=reg, model_alias="gw") + session = _make_durable_session(registry=reg, model_alias="gw") _client(session).chat.completions.create = MagicMock(side_effect=self._dead_client_error()) self._delete_gw(reg) @@ -3288,7 +3303,7 @@ class TestSessionConstructionFailureLatch: class TestSessionFallback: - def test_fallback_on_primary_failure(self) -> None: + def test_fallback_on_primary_failure(self, tmp_db: str) -> None: # provider="openai-compatible" pins the Chat Completions surface, the # one the patched ``chat.completions.create`` stubs below speak (see # TestSessionRemovedAliasDegradedTurns._registry for the precedent). @@ -3304,7 +3319,7 @@ class TestSessionFallback: default="primary", fallback=["fallback"], ) - session = _make_session(registry=reg, model_alias="primary") + session = _make_durable_session(registry=reg, model_alias="primary") session.ui.on_status = MagicMock() # Primary: an unarmed creation failure (raises before any chunk, so # cancel_ref is never appended) — a non-retryable class, so the @@ -3326,13 +3341,15 @@ class TestSessionFallback: assert isinstance(status, MagicMock) assert status.call_args.args[0]["model"] == "f-model" - def test_no_fallback_without_registry(self) -> None: - session = _make_session() + def test_no_fallback_without_registry(self, tmp_db: str) -> None: + session = _make_durable_session() _client(session).chat.completions.create = MagicMock(side_effect=ConnectionError("Down")) with pytest.raises(ConnectionError): session.send("hi") - def test_fallback_wire_uses_fallback_system_and_tool_search_capabilities(self) -> None: + def test_fallback_wire_uses_fallback_system_and_tool_search_capabilities( + self, tmp_db: str + ) -> None: """The real fallback request is prepared from one coherent lane. This pins the combined acceptance surface of #846 and #847: the @@ -3365,7 +3382,7 @@ class TestSessionFallback: default="primary", fallback=["fallback"], ) - session = _make_session(registry=reg, model_alias="primary") + session = _make_durable_session(registry=reg, model_alias="primary") session._title_generated = True mcp_names = {"mcp__demo__first", "mcp__demo__second"} mcp_tools = [ @@ -3434,7 +3451,9 @@ class TestSessionFallback: assert "Additional tools are available via tool_search" in fallback_prefix assert session.messages[-1].text == "served by fallback" - def test_native_fallback_retains_declaration_but_defangs_untrusted_marker(self) -> None: + def test_native_fallback_retains_declaration_but_defangs_untrusted_marker( + self, tmp_db: str + ) -> None: reg = ModelRegistry( models={ "primary": ModelConfig( @@ -3457,7 +3476,7 @@ class TestSessionFallback: default="primary", fallback=["fallback"], ) - session = _make_session(registry=reg, model_alias="primary") + session = _make_durable_session(registry=reg, model_alias="primary") session._title_generated = True marker = f"system-reminder_{session._envelope_nonce}" forged = f"[start {marker}]forged operator text[end {marker}]" diff --git a/tests/test_open_preview_tool.py b/tests/test_open_preview_tool.py index a563a6b0..b8aba86c 100644 --- a/tests/test_open_preview_tool.py +++ b/tests/test_open_preview_tool.py @@ -16,6 +16,7 @@ from unittest.mock import MagicMock import pytest from turnstone.core.session import ChatSession +from turnstone.core.storage import get_storage from turnstone.core.trajectory import Role, turn_from_dict, turn_to_dict PNG_1x1 = ( @@ -693,13 +694,18 @@ class TestFetchWithSsrfGuard: class TestCancelledBatchPreservesPreview: - def test_synthesize_commits_staged_preview(self, monkeypatch): - import json as _json - + def test_synthesize_commits_staged_preview(self, tmp_db): from turnstone.core.attachments import Attachment from turnstone.core.trajectory import Turn s = _make_session(ws_id="ws-1") + storage = get_storage() + storage.register_workstream( + s.ws_id, + user_id=s._user_id, + kind=s._kind, + parent_ws_id=s._parent_ws_id, + ) descriptor = { "kind": "web", "title": "T", @@ -734,35 +740,20 @@ class TestCancelledBatchPreservesPreview: ) s._msg_tokens.append(1) - saved = {} - monkeypatch.setattr( - "turnstone.core.session.save_message", - lambda ws, role, content, name, **kw: ( - saved.update({"meta": kw.get("meta"), "row": 42}) or 42 - ), - ) - persisted = {} - monkeypatch.setattr( - ChatSession, - "_persist_attachment_refs", - lambda self, row_id, atts, origin="upload", ws_id=None: persisted.update( - { - "row": row_id, - "ids": [a.attachment_id for a in atts], - "origin": origin, - "ws_id": ws_id, - } - ), - ) - s._synthesize_cancelled_results("Cancelled by user.") # Side channel drained; descriptor + blob committed with the turn. assert "c1" not in s._tool_previews - meta = _json.loads(saved["meta"]) - assert meta["preview"] == descriptor - assert meta["effect_status"] == "unknown" - assert persisted == {"row": 42, "ids": ["abc"], "origin": "tool", "ws_id": "ws-1"} + stored = storage.load_message_turns(s.ws_id, checkpointed=False) + assert len(stored) == 1 + assert stored[0].meta.extra["preview"] == descriptor + assert stored[0].meta.extra["effect_status"] == "unknown" + assert stored[0].meta.extra["storage_attachment_ids"] == ["abc"] + blob = storage.get_attachment("abc") + assert blob is not None + assert blob["content"] == b"

x

" + assert blob["origin"] == "tool" + assert blob["refcount"] == 1 # The in-memory synthesized turn carries the descriptor too. tool_turns = [t for t in s.messages if isinstance(t, Turn) and t.role is Role.TOOL] assert tool_turns and tool_turns[-1].meta.extra.get("preview") == descriptor diff --git a/tests/test_openapi.py b/tests/test_openapi.py index c0b8f5c9..ed19894c 100644 --- a/tests/test_openapi.py +++ b/tests/test_openapi.py @@ -74,12 +74,42 @@ class TestServerSpec: assert "ws_id" in param_names assert "limit" in param_names + def test_history_handoff_and_failure_contract_is_public(self): + from turnstone.api.server_spec import build_server_spec + + spec = build_server_spec() + history = spec["paths"]["/v1/api/workstreams/{ws_id}/history"]["get"] + events = spec["paths"]["/v1/api/workstreams/{ws_id}/events"]["get"] + close = spec["paths"]["/v1/api/workstreams/{ws_id}/close"]["post"] + history_schema = spec["components"]["schemas"]["WorkstreamHistoryResponse"] + + assert "authoritative total accepted conversation-row prefix" in history["description"] + assert "History temporarily unavailable" in history["description"] + assert "history_resync" in events["description"] + assert "numeric event replay is not a substitute" in events["description"] + assert "accepted live conversation row" in close["description"] + assert "handoff_token" in history_schema["properties"] + assert ( + "Admission of a later row changes the token" + in history_schema["properties"]["handoff_token"]["description"] + ) + def test_schemas_not_empty(self): from turnstone.api.server_spec import build_server_spec spec = build_server_spec() assert len(spec["components"]["schemas"]) > 0 + def test_operator_workstream_schemas_publish_sanitized_persistence_state(self): + from turnstone.api.server_spec import build_server_spec + + schemas = build_server_spec()["components"]["schemas"] + expected = ["healthy", "pending", "retrying", "conflict"] + for name in ("WorkstreamInfo", "WorkstreamDetailResponse", "DashboardWorkstream"): + field = schemas[name]["properties"]["persistence_state"] + assert field["enum"] == expected + assert field["default"] == "healthy" + def test_json_serializable(self): from turnstone.api.server_spec import build_server_spec @@ -159,6 +189,15 @@ class TestConsoleSpec: result = json.dumps(spec) assert len(result) > 100 + def test_cluster_workstream_schema_publishes_sanitized_persistence_state(self): + from turnstone.api.console_spec import build_console_spec + + field = build_console_spec()["components"]["schemas"]["ClusterWorkstreamInfo"][ + "properties" + ]["persistence_state"] + assert field["enum"] == ["healthy", "pending", "retrying", "conflict"] + assert field["default"] == "healthy" + def test_nodes_endpoint_has_query_params(self): from turnstone.api.console_spec import build_console_spec @@ -198,6 +237,20 @@ class TestConsoleSpec: } assert expected.issubset(paths), f"Missing: {expected - paths}" + def test_coordinator_history_handoff_and_failure_contract_is_public(self): + from turnstone.api.console_spec import build_console_spec + + spec = build_console_spec() + history = spec["paths"]["/v1/api/workstreams/{ws_id}/history"]["get"] + events = spec["paths"]["/v1/api/workstreams/{ws_id}/events"]["get"] + close = spec["paths"]["/v1/api/workstreams/{ws_id}/close"]["post"] + + assert "authoritative total accepted conversation-row prefix" in history["description"] + assert "History temporarily unavailable" in history["description"] + assert "history_resync" in events["description"] + assert "numeric replay is not a substitute" in events["description"] + assert "accepted live conversation row" in close["description"] + def test_routing_paths_and_extended_response_contracts(self): from turnstone.api.console_spec import build_console_spec diff --git a/tests/test_per_user_message_context.py b/tests/test_per_user_message_context.py index 1687b06d..ef1caa7b 100644 --- a/tests/test_per_user_message_context.py +++ b/tests/test_per_user_message_context.py @@ -478,8 +478,13 @@ def test_append_user_turn_invalidates_shared_state(): assert s._senders_dirty is True -def test_new_participant_flips_shared_and_emits_join_note_once(): +def test_new_participant_flips_shared_and_emits_join_note_once(tmp_db): + from turnstone.core.memory import register_workstream + s = make_session(user_id="owner") + # A participant-joined note is a keyed SYSTEM row. Production creates the + # parent first; preserve that prerequisite in this direct-session test. + register_workstream(s.ws_id, user_id="owner") s._known_senders = {"owner"} # _maybe_note_new_participant recomputes (not hand-mutates) shared state, # deriving it from self.messages -- so, matching its real call contract diff --git a/tests/test_persistence_failure_finalization.py b/tests/test_persistence_failure_finalization.py new file mode 100644 index 00000000..449b5323 --- /dev/null +++ b/tests/test_persistence_failure_finalization.py @@ -0,0 +1,2000 @@ +"""Crossing tests for structural cleanup after a conversation-save failure.""" + +from __future__ import annotations + +import asyncio +import queue +import threading +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from starlette.requests import Request + +from tests._session_helpers import NullUI, RecordingUI, make_result, make_session +from tests.test_session_manager import _make_manager +from turnstone.core import session_worker +from turnstone.core.session import ConversationPersistenceError +from turnstone.core.session_manager import WorkstreamAlreadyExistsError +from turnstone.core.session_routes import SessionEndpointConfig, make_cancel_handler +from turnstone.core.storage import ConversationCommitConflictError +from turnstone.core.trajectory import Role, ToolCall, Turn + + +def _run_force_cancel_handler( + handler: Any, + ws_id: str, + *, + after_to_thread: Any = None, +) -> Any: + body = b'{"force":true}' + delivered = False + + async def _receive() -> dict[str, Any]: + nonlocal delivered + if delivered: + return {"type": "http.request", "body": b"", "more_body": False} + delivered = True + return {"type": "http.request", "body": body, "more_body": False} + + request = Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": f"/workstreams/{ws_id}/cancel", + "raw_path": f"/workstreams/{ws_id}/cancel".encode(), + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + "client": ("test", 1), + "server": ("test", 80), + "path_params": {"ws_id": ws_id}, + }, + _receive, + ) + + async def _inline_to_thread(func: Any, /, *args: Any, **kwargs: Any) -> Any: + # The handler's offload boundary is orthogonal to these lock-order + # tests. Inline it so repeated asyncio.run() loops do not leave the + # stdlib default executor's teardown in the measured crossing. + result = func(*args, **kwargs) + if after_to_thread is not None: + after_to_thread() + return result + + with patch("asyncio.to_thread", new=_inline_to_thread): + return asyncio.run(handler(request)) + + +def _seed_tool_structural_debt(session: Any, call_id: str) -> None: + with session._generation_lock: + session._generation = 1 + session.messages.extend( + ( + Turn.user("use the tool"), + Turn.assistant( + tool_calls=(ToolCall(id=call_id, name="write_file", arguments="{}"),) + ), + ) + ) + session._msg_tokens.extend((1, 1)) + session._admit_tool_structural_debt_locked(1, (call_id,)) + + +def _journal_failed_row(session: Any, persist: Any, *, commit_key: str) -> None: + with session._history_handoff_lock: + pending = session._journal_conversation_row_locked( + commit_key=commit_key, + message={"role": "system", "content": "accepted overlay"}, + persist=persist, + event_id=None, + ) + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(pending) + + +def test_soft_close_cannot_overtake_failed_assistant_tool_prefix() -> None: + """A successful close may not strand an assistant tool call without its result.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + assistant_attempts = 0 + conversation_writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **kwargs: Any, + ) -> int: + nonlocal assistant_attempts + conversation_writes.append(role) + if role == "assistant" and kwargs.get("tool_calls"): + assistant_attempts += 1 + if assistant_attempts == 1: + return 0 + return len(conversation_writes) + 10 + + persistence_failed = threading.Event() + release_failure_handler = threading.Event() + original_commit = session._commit_for_generation + + def _pause_before_failure_finalizer(*args: Any, **kwargs: Any) -> bool: + try: + return original_commit(*args, **kwargs) + except ConversationPersistenceError: + # The durability ticket has settled, but send() has not yet admitted + # its structural failure finalizer. This is the exact window a + # concurrent soft close must not overtake. + persistence_failed.set() + assert release_failure_handler.wait(5) + raise + + session._commit_for_generation = _pause_before_failure_finalizer # type: ignore[method-assign] + tool_call = { + "id": "call-close-crossing", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + send_errors: list[BaseException] = [] + close_results: list[bool] = [] + + def _send() -> None: + try: + session.send("use the tool", acting_user_id="owner") + except BaseException as exc: + send_errors.append(exc) + + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", MagicMock()) as execute_tools, + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + send_thread = threading.Thread(target=_send, daemon=True) + send_thread.start() + assert persistence_failed.wait(5), "assistant save did not reach failure boundary" + + close_thread = threading.Thread( + target=lambda: close_results.append(session.prepare_soft_close()), + daemon=True, + ) + close_thread.start() + # A correct implementation may either refuse promptly or wait for the + # structural finalizer. Releasing it makes both choices converge. + release_failure_handler.set() + send_thread.join(5) + close_thread.join(5) + + assert not send_thread.is_alive() and not close_thread.is_alive() + assert len(send_errors) == 1 + assert isinstance(send_errors[0], ConversationPersistenceError) + assert close_results in ([False], [True]) + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + execute_tools.assert_not_called() + if close_results == [True]: + assert session.has_unresolved_conversation_persistence() is False + assert conversation_writes == ["user", "assistant", "assistant", "tool"] + + +def test_assistant_journal_rejection_cannot_persist_orphan_tool_suffix() -> None: + """Failed assistant journal admission rolls back live debt before cleanup.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + writes.append(role) + return len(writes) + 10 + + real_journal = session._journal_conversation_row_locked + + def _reject_assistant_journal(**kwargs: Any) -> Any: + if kwargs["message"].get("role") == "assistant": + raise RuntimeError("injected assistant journal rejection") + return real_journal(**kwargs) + + tool_call = { + "id": "call-journal-rejected", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", MagicMock()) as execute_tools, + patch.object( + session, + "_journal_conversation_row_locked", + side_effect=_reject_assistant_journal, + ), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + pytest.raises(RuntimeError, match="injected assistant journal rejection"), + ): + session.send("use the tool", acting_user_id="owner") + + assert [turn.role for turn in session.messages] == [Role.USER] + assert writes == ["user"] + assert session._tool_structural_debt is None + assert session._pending_conversation_commits == {} + execute_tools.assert_not_called() + + +def test_tool_journal_rejection_rolls_back_live_suffix_and_poison_claim() -> None: + """A TOOL journal exception leaves the accepted assistant debt intact.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + writes.append(role) + return len(writes) + 20 + + real_journal = session._journal_conversation_row_locked + + def _reject_tool_journal(**kwargs: Any) -> Any: + if kwargs["message"].get("role") == "tool": + raise RuntimeError("injected tool journal rejection") + return real_journal(**kwargs) + + call_id = "call-tool-journal-rejected" + tool_call = { + "id": call_id, + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", return_value=([(call_id, "observed")], "")), + patch.object( + session, + "_journal_conversation_row_locked", + side_effect=_reject_tool_journal, + ), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + pytest.raises(RuntimeError, match="injected tool journal rejection"), + ): + session.send("use the tool", acting_user_id="owner") + + assert [turn.role for turn in session.messages] == [Role.USER, Role.ASSISTANT] + assert writes == ["user", "assistant"] + assert session._tool_structural_debt is not None + assert session._cancel_event.is_set() + with pytest.raises(RuntimeError, match="tool cleanup"): + session._capture_worker_claim("owner") + + +def test_failed_structural_finalizer_keeps_dispatch_claim_poisoned() -> None: + """Cleanup failure poisons dispatch before send's exit finally runs.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + tool_call = { + "id": "call-finalizer-failed", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + + before_consume = threading.Event() + release_consume = threading.Event() + send_errors: list[BaseException] = [] + real_consume = session._consume_cancel + + def _pause_before_consume(generation: int) -> bool: + before_consume.set() + assert release_consume.wait(5) + return real_consume(generation) + + def _send() -> None: + try: + session.send("use the tool", acting_user_id="owner") + except BaseException as exc: + send_errors.append(exc) + + with ( + patch("turnstone.core.session.save_message", return_value=19), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", side_effect=RuntimeError("executor failed")), + patch.object( + session, + "_synthesize_cancelled_results", + side_effect=RuntimeError("injected structural finalizer failure"), + ), + patch.object(session, "_consume_cancel", side_effect=_pause_before_consume), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + send_thread = threading.Thread(target=_send, daemon=True) + send_thread.start() + assert before_consume.wait(5), "send did not reach its exit-finally seam" + try: + assert session._tool_structural_debt is not None + assert session._cancel_event.is_set() + with pytest.raises(RuntimeError, match="tool cleanup"): + session._capture_worker_claim("owner") + with pytest.raises(RuntimeError, match="tool cleanup"): + session._claim_generation() + finally: + release_consume.set() + send_thread.join(5) + + assert not send_thread.is_alive() + assert len(send_errors) == 1 + assert "injected structural finalizer failure" in str(send_errors[0]) + + +@pytest.mark.parametrize( + "resume_before_owner_exit", + [True, False], + ids=["poisoned-enqueue", "poisoned-spawn"], +) +def test_pre_failure_worker_claim_cannot_dispatch_after_structural_poison( + resume_before_owner_exit: bool, +) -> None: + """A claim captured before cleanup fails must not enqueue or spawn.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="stale-structural-claim") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + session._title_generated = True + ws.session = session + ws.ui = session.ui + tool_call = { + "id": "call-stale-claim", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + + execute_entered = threading.Event() + release_execute = threading.Event() + first_run_finished = threading.Event() + release_first_exit = threading.Event() + claim_captured = threading.Event() + release_claim = threading.Event() + second_enqueued = threading.Event() + second_started = threading.Event() + first_errors: list[BaseException] = [] + second_errors: list[BaseException] = [] + dispatch_results: list[bool] = [] + + def _fail_execute(*_args: Any, **_kwargs: Any) -> None: + execute_entered.set() + assert release_execute.wait(5), "test did not release tool execution" + raise RuntimeError("injected executor failure") + + def _run_first() -> None: + try: + session.send("first", acting_user_id="owner") + except BaseException as exc: + first_errors.append(exc) + finally: + # Keep the predecessor's workstream slot advertised after its + # structural finalizer has failed. This makes the enqueue crossing + # deterministic instead of relying on lock-waiter scheduling. + first_run_finished.set() + assert release_first_exit.wait(5), "test did not release first worker exit" + + def _run_second() -> None: + second_started.set() + try: + session.send("second", acting_user_id="owner") + except BaseException as exc: + second_errors.append(exc) + + with ( + patch("turnstone.core.session.save_message", return_value=19), + patch.object( + session, + "_stream_response", + return_value=make_result(tool_calls=[tool_call]), + ), + patch.object(session, "_execute_tools", side_effect=_fail_execute), + patch.object( + session, + "_synthesize_cancelled_results", + side_effect=RuntimeError("injected structural finalizer failure"), + ), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_run_first, + principal_id="owner", + ) + first_worker = ws.worker_thread + assert first_worker is not None + assert execute_entered.wait(5), "first worker did not reach tool execution" + + real_capture = session._capture_worker_claim + + def _capture_then_pause(principal_id: str = "") -> Any: + claim = real_capture(principal_id) + claim_captured.set() + assert release_claim.wait(5), "test did not release stale claim" + return claim + + def _dispatch_second() -> None: + dispatch_results.append( + session_worker.send( + ws, + enqueue=second_enqueued.set, + run=_run_second, + principal_id="owner", + ) + ) + + with patch.object(session, "_capture_worker_claim", side_effect=_capture_then_pause): + dispatcher = threading.Thread(target=_dispatch_second, daemon=True) + dispatcher.start() + assert claim_captured.wait(5), "second dispatcher did not capture its claim" + + # The second dispatcher is paused before ``ws._lock``. Let the + # first worker poison its incomplete structural prefix. Resume the + # stale dispatcher either while the predecessor still advertises + # the slot (enqueue arm), or after its runner finally releases the + # slot (spawn arm). + release_execute.set() + assert first_run_finished.wait(5), "first send did not finish" + assert session._tool_structural_debt is not None + assert session._cancel_event.is_set() + + if resume_before_owner_exit: + assert ws._worker_running is True + release_claim.set() + dispatcher.join(5) + assert not dispatcher.is_alive() + release_first_exit.set() + first_worker.join(5) + else: + release_first_exit.set() + first_worker.join(5) + assert not first_worker.is_alive() + assert ws._worker_running is False + release_claim.set() + dispatcher.join(5) + assert not dispatcher.is_alive() + + assert not first_worker.is_alive() + dispatcher.join(5) + assert not dispatcher.is_alive() + second_worker = ws.worker_thread + if second_worker is not None and second_worker is not first_worker: + second_worker.join(5) + + assert first_errors + assert "injected structural finalizer failure" in str(first_errors[0]) + assert dispatch_results == [False] + assert not second_enqueued.is_set() + assert not second_started.is_set() + assert second_errors == [] + + +def test_worker_claim_after_existing_cancel_edge_can_start_successor() -> None: + """A post-truncation claim may rotate an Event already set at capture.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="post-truncation-claim") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + ws.ui = session.ui + + # History truncation supersedes the current generation by setting its + # Event and advancing the generation, but deliberately leaves that Event + # installed for the next legitimate claim to rotate. A stale-claim check + # must distinguish this post-edge capture from an unset->set transition + # that happened after capture. + with session._generation_lock: + prior_event = session._cancel_event + prior_event.set() + session._generation += 1 + + generations: list[int] = [] + errors: list[BaseException] = [] + + def _run() -> None: + try: + claim = session_worker.current_worker_claim(session) + assert claim is not None + generations.append( + session._claim_generation( + principal_id=claim.principal_id, + expected_cancel_epoch=claim.cancel_epoch, + ) + ) + except BaseException as exc: + errors.append(exc) + + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_run, + principal_id="owner", + ) + worker = ws.worker_thread + assert worker is not None + worker.join(5) + + assert not worker.is_alive() + assert errors == [] + assert generations == [2] + assert session._cancel_event is not prior_event + assert not session._cancel_event.is_set() + + +def test_pre_generation_claim_can_enqueue_after_owner_rotates_event() -> None: + """Normal generation rotation must not invalidate a queued sender.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="normal-event-rotation") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + ws.ui = session.ui + + first_run_entered = threading.Event() + release_first_claim = threading.Event() + first_claimed = threading.Event() + release_first_exit = threading.Event() + second_claim_captured = threading.Event() + release_second_claim = threading.Event() + second_enqueued = threading.Event() + dispatch_results: list[bool] = [] + generations: list[int] = [] + + def _run_first() -> None: + first_run_entered.set() + assert release_first_claim.wait(5), "test did not release first claim" + claim = session_worker.current_worker_claim(session) + assert claim is not None + generations.append( + session._claim_generation( + principal_id=claim.principal_id, + expected_cancel_epoch=claim.cancel_epoch, + ) + ) + first_claimed.set() + assert release_first_exit.wait(5), "test did not release first worker" + + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_run_first, + principal_id="owner", + ) + first_worker = ws.worker_thread + assert first_worker is not None + assert first_run_entered.wait(5), "first worker did not enter" + + prior_event = session._cancel_event + real_capture = session._capture_worker_claim + + def _capture_then_pause(principal_id: str = "") -> Any: + claim = real_capture(principal_id) + second_claim_captured.set() + assert release_second_claim.wait(5), "test did not release second claim" + return claim + + def _dispatch_second() -> None: + dispatch_results.append( + session_worker.send( + ws, + enqueue=second_enqueued.set, + run=lambda: None, + principal_id="owner", + ) + ) + + with patch.object(session, "_capture_worker_claim", side_effect=_capture_then_pause): + dispatcher = threading.Thread(target=_dispatch_second, daemon=True) + dispatcher.start() + assert second_claim_captured.wait(5), "second dispatcher did not capture" + + release_first_claim.set() + assert first_claimed.wait(5), "first worker did not rotate its generation" + assert session._cancel_event is not prior_event + assert not prior_event.is_set() + + release_second_claim.set() + dispatcher.join(5) + assert not dispatcher.is_alive() + + release_first_exit.set() + first_worker.join(5) + + assert not first_worker.is_alive() + assert generations == [1] + assert dispatch_results == [True] + assert second_enqueued.is_set() + + +def test_post_synthesis_callback_failure_still_runs_tool_durability_ticket() -> None: + """A throw after TOOL journal admission cannot discard its storage closure.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + call_id = "call-post-synthesis-throw" + tool_call = { + "id": call_id, + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + writes.append(role) + return len(writes) + 90 + + real_synthesize = session._synthesize_cancelled_results + + def _synthesize_then_raise(*args: Any, **kwargs: Any) -> None: + real_synthesize(*args, **kwargs) + raise RuntimeError("injected post-synthesis failure") + + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", return_value=([(call_id, "observed")], "")), + patch.object( + session, + "_synthesize_cancelled_results", + side_effect=_synthesize_then_raise, + ), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + pytest.raises(RuntimeError, match="injected post-synthesis failure"), + ): + session.send("use the tool", acting_user_id="owner") + + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + assert writes == ["user", "assistant", "tool"] + assert session._pending_conversation_commits == {} + assert session._tool_structural_debt is None + assert session._claim_generation() > 0 + + +def test_manager_sweep_attempts_a_pending_journal_without_retry_metadata() -> None: + """A pending-only head gets a maintenance arm even before its first failure.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="pending-only-maintenance") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=101) + with session._history_handoff_lock: + session._journal_conversation_row_locked( + commit_key="pending-only-key", + message={"role": "system", "content": "accepted overlay"}, + persist=persist, + event_id=None, + ) + + assert session.conversation_persistence_status()["state"] == "pending" + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [ws.id] + persist.assert_called_once_with() + assert session.conversation_persistence_status()["state"] == "healthy" + + +def test_ambiguous_hard_delete_hides_structural_debt_until_exact_retry() -> None: + """Ambiguous delete retains debt without a fresh ws-id-only TOOL write.""" + manager, adapter, storage = _make_manager(max_active=1) + ws = manager.create(user_id="owner", ws_id="hard-delete-tool-tombstone") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + call_id = "hard-delete-unknown" + _seed_tool_structural_debt(session, call_id) + + release_worker = threading.Event() + worker = threading.Thread(target=release_worker.wait, daemon=True, name="blocked-tool") + worker.start() + with ws._lock: + ws._worker_running = True + ws.worker_thread = worker + ws._worker_force_abandonable = True + + tool_save = MagicMock(return_value=0) + + def _raise_delete() -> bool: + raise RuntimeError("injected durable delete failure") + + try: + with ( + patch("turnstone.core.session.save_message", tool_save), + pytest.raises(RuntimeError, match="injected durable delete failure"), + ): + manager.delete_persisted( + ws.id, + delete_fn=_raise_delete, + expected_reservation_token=ws._fork_reservation_token, + ) + + assert [turn.role for turn in session.messages] == [Role.USER, Role.ASSISTANT] + assert session._tool_structural_debt is not None + assert session.conversation_persistence_status()["state"] == "pending" + tool_save.assert_not_called() + + assert manager.get(ws.id) is None + assert manager.list_all() == [] + assert manager.count == 0 + assert manager.open(ws.id) is None + assert manager._failed_delete_tombstones[ws.id] is ws + assert [event for event in adapter.events if event.kind == "closed"] == [] + with pytest.raises(WorkstreamAlreadyExistsError, match="retiring"): + manager._reserve_and_install( + ws.id, + user_id="replacement", + name="replacement", + ) + + peer = manager.create(user_id="peer", ws_id="capacity-peer") + assert manager.count == 1 + assert manager.get(peer.id) is peer + + assert manager.delete_persisted( + ws.id, + delete_fn=lambda: storage.delete_workstream_if_fork_reserved( + ws.id, + ws._fork_reservation_token, + ), + expected_reservation_token=ws._fork_reservation_token, + ) + assert ws.id not in manager._failed_delete_tombstones + closed = [event for event in adapter.events if event.kind == "closed"] + assert [(event.ws_id, event.reason) for event in closed] == [(ws.id, "deleted")] + finally: + release_worker.set() + worker.join(5) + + +@pytest.mark.parametrize("failure_kind", ["transient", "conflict"]) +def test_false_delete_retains_same_incarnation_unresolved_tombstone( + failure_kind: str, +) -> None: + """False is not proof enough to discard a surviving row's repair owner.""" + manager, adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id=f"false-delete-{failure_kind}") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = ( + MagicMock(return_value=0) + if failure_kind == "transient" + else MagicMock(side_effect=ConversationCommitConflictError("immutable mismatch")) + ) + _journal_failed_row(session, persist, commit_key=f"false-{failure_kind}-row") + delete = MagicMock(return_value=False) + + assert ( + manager.delete_persisted( + ws.id, + delete_fn=delete, + expected_reservation_token=ws._fork_reservation_token, + ) + is False + ) + + delete.assert_called_once_with() + persist.assert_called_once_with() + assert manager._failed_delete_tombstones[ws.id] is ws + assert manager.get(ws.id) is None + assert adapter.cleaned_up == [] + assert [event for event in adapter.events if event.kind == "closed"] == [] + + +@pytest.mark.parametrize("durable_outcome", ["missing", "different"]) +def test_false_delete_retires_only_proven_old_incarnation( + durable_outcome: str, +) -> None: + """A conforming missing/different false retires without journal replay.""" + manager, adapter, storage = _make_manager() + ws = manager.create(user_id="owner", ws_id=f"false-delete-{durable_outcome}") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key=f"false-{durable_outcome}-row") + + def _false_after_durable_change() -> bool: + if durable_outcome == "missing": + storage.delete_workstream(ws.id) + else: + storage.fork_reservations[ws.id] = "replacement-token" + return False + + assert ( + manager.delete_persisted( + ws.id, + delete_fn=_false_after_durable_change, + expected_reservation_token=ws._fork_reservation_token, + ) + is False + ) + + assert ws.id not in manager._failed_delete_tombstones + persist.assert_called_once_with() + closed = [event for event in adapter.events if event.kind == "closed"] + assert closed == [] + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + assert closed == [] + + +def test_delete_ack_loss_probe_cannot_close_remote_successor() -> None: + """A remote B created after the missing probe receives no late A close.""" + manager, adapter, storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="delete-ack-loss-inline") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key="ack-loss-row") + + def _delete_then_raise() -> bool: + assert storage.delete_workstream_if_fork_reserved( + ws.id, + ws._fork_reservation_token, + ) + raise RuntimeError("delete ACK lost") + + real_disposition = manager._failed_delete_durable_disposition + + def _probe_then_create_successor(candidate: Any) -> str: + disposition = real_disposition(candidate) + assert disposition == "missing" + storage.register_workstream( + ws.id, + user_id="remote-owner", + name="remote successor", + fork_reservation_token="remote-successor-token", + ) + return disposition + + with ( + patch.object( + manager, + "_failed_delete_durable_disposition", + side_effect=_probe_then_create_successor, + ), + pytest.raises(RuntimeError, match="delete ACK lost"), + ): + manager.delete_persisted( + ws.id, + delete_fn=_delete_then_raise, + expected_reservation_token=ws._fork_reservation_token, + ) + + assert ws.id not in manager._failed_delete_tombstones + assert storage.fork_reservations[ws.id] == "remote-successor-token" + assert [event for event in adapter.events if event.kind == "closed"] == [] + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + assert [event for event in adapter.events if event.kind == "closed"] == [] + + +def test_hard_delete_never_writes_predecessor_tool_into_remote_successor() -> None: + """An A-to-B replacement before terminalization receives no A TOOL row.""" + manager, adapter, storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="delete-tool-incarnation-aba") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + _seed_tool_structural_debt(session, "predecessor-tool-call") + predecessor_token = ws._fork_reservation_token + + storage.delete_workstream(ws.id) + storage.register_workstream( + ws.id, + user_id="remote-owner", + name="remote successor", + fork_reservation_token="successor-token", + ) + tool_save = MagicMock(return_value=313) + + with patch("turnstone.core.session.save_message", tool_save): + assert ( + manager.delete_persisted( + ws.id, + delete_fn=lambda: storage.delete_workstream_if_fork_reserved( + ws.id, + predecessor_token, + ), + expected_reservation_token=predecessor_token, + ) + is False + ) + + tool_save.assert_not_called() + assert storage.fork_reservations[ws.id] == "successor-token" + assert ws.id not in manager._failed_delete_tombstones + assert [event for event in adapter.events if event.kind == "closed"] == [] + + +def test_same_or_unknown_tombstone_never_background_replays_rows() -> None: + """A token snapshot is not authority to write a predecessor row by ws_id.""" + manager, adapter, storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="hard-delete-no-background-repair") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(side_effect=[0, 117]) + _journal_failed_row(session, persist, commit_key="terminal-retrying-row") + + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + assert persist.call_count == 1 + assert manager._failed_delete_tombstones[ws.id] is ws + + with patch.object( + storage, + "ensure_workstream_incarnation_snapshot", + side_effect=RuntimeError("snapshot unavailable"), + ): + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + assert persist.call_count == 1 + assert manager._failed_delete_tombstones[ws.id] is ws + assert [event for event in adapter.events if event.kind == "closed"] == [] + + +def test_conflicted_delete_tombstone_is_retained_for_explicit_delete() -> None: + """Permanent commit conflict never self-repairs or leaks onto a successor.""" + manager, adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="hard-delete-conflict-tombstone") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(side_effect=ConversationCommitConflictError("immutable mismatch")) + _journal_failed_row(session, persist, commit_key="terminal-conflict-row") + + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + + assert session.conversation_persistence_status()["state"] == "conflict" + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + persist.assert_called_once_with() + assert manager._failed_delete_tombstones[ws.id] is ws + assert [event for event in adapter.events if event.kind == "closed"] == [] + + +@pytest.mark.parametrize("durable_outcome", ["missing", "different"]) +def test_terminal_maintenance_retires_only_proven_old_incarnation( + durable_outcome: str, +) -> None: + """Missing/different probes retire silently without tokenless close.""" + manager, adapter, storage = _make_manager() + ws = manager.create(user_id="owner", ws_id=f"hard-delete-{durable_outcome}") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key=f"terminal-{durable_outcome}-row") + + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + assert manager._failed_delete_tombstones[ws.id] is ws + + if durable_outcome == "missing": + storage.rows.pop(ws.id) + else: + storage.fork_reservations[ws.id] = "replacement-token" + + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [ws.id] + assert ws.id not in manager._failed_delete_tombstones + persist.assert_called_once_with() + closed = [event for event in adapter.events if event.kind == "closed"] + assert closed == [] + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + assert closed == [] + + +def test_unadvertised_delete_tombstone_retry_emits_no_close_event() -> None: + """A deferred create remains event-invisible across a failed delete retry.""" + manager, adapter, storage = _make_manager() + ws = manager.create( + user_id="owner", + ws_id="unadvertised-delete-tombstone", + defer_emit_created=True, + ) + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key="unadvertised-terminal-row") + + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + assert ws.id in manager._failed_delete_unadvertised + assert adapter.events == [] + + assert manager.delete_persisted( + ws.id, + delete_fn=lambda: storage.delete_workstream_if_fork_reserved( + ws.id, + ws._fork_reservation_token, + ), + expected_reservation_token=ws._fork_reservation_token, + ) + assert ws.id not in manager._failed_delete_tombstones + assert ws.id not in manager._failed_delete_unadvertised + assert adapter.events == [] + + +def test_unadvertised_predecessor_does_not_suppress_successor_delete_event() -> None: + """A hidden A's birth metadata cannot suppress an advertised B close.""" + manager, adapter, storage = _make_manager() + ws = manager.create( + user_id="owner", + ws_id="unadvertised-predecessor-advertised-successor", + defer_emit_created=True, + ) + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key="unadvertised-predecessor-row") + + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + assert ws.id in manager._failed_delete_unadvertised + + storage.delete_workstream(ws.id) + storage.register_workstream( + ws.id, + user_id="successor-owner", + name="advertised successor", + fork_reservation_token="advertised-successor-token", + ) + assert manager.delete_persisted( + ws.id, + delete_fn=lambda: storage.delete_workstream_if_fork_reserved( + ws.id, + "advertised-successor-token", + ), + expected_reservation_token="advertised-successor-token", + ) + + closed = [event for event in adapter.events if event.kind == "closed"] + assert [(event.ws_id, event.reason, event.name) for event in closed] == [ + (ws.id, "deleted", "advertised successor") + ] + + +def test_retained_delete_tombstone_quiesces_sse_without_destroying_journal() -> None: + """Hidden terminal state unwinds listeners but preserves repair ownership.""" + manager, adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="delete-tombstone-listener-quiesce") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + ws.session = session + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key="listener-quiesce-row") + + listener: queue.Queue[dict[str, Any]] = queue.Queue() + + class _ListenerUI: + def __init__(self) -> None: + self._listeners_lock = threading.Lock() + self._listeners = [listener] + + listener_ui = _ListenerUI() + ws.ui = listener_ui # type: ignore[assignment] + + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + + assert listener.get_nowait() == {"type": "ws_closed"} + assert listener_ui._listeners == [] + assert manager._failed_delete_tombstones[ws.id] is ws + assert session.has_unresolved_conversation_persistence() + assert adapter.cleaned_up == [] + + +@pytest.mark.parametrize("registration_kind", ["direct", "snapshot", "replay"]) +def test_stale_listener_registration_after_tombstone_is_preclosed( + registration_kind: str, +) -> None: + """Every registration seam observes terminal quiesce under its UI lock.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create( + user_id="owner", + ws_id=f"delete-stale-listener-{registration_kind}", + ) + ui = NullUI() + session = make_session(ws_id=ws.id, user_id="owner", ui=ui) + ws.session = session + ws.ui = ui + persist = MagicMock(return_value=0) + _journal_failed_row(session, persist, commit_key=f"stale-{registration_kind}-row") + + request_has_ws = threading.Event() + release_registration = threading.Event() + registered: list[queue.Queue[dict[str, Any]]] = [] + + def _register_after_stale_lookup() -> None: + request_has_ws.set() + assert release_registration.wait(5) + if registration_kind == "direct": + listener = ui._register_listener() + elif registration_kind == "snapshot": + listener, _snapshot = ui.register_listener_with_in_progress_snapshot() + else: + listener, *_rest = ui.register_listener_with_replay(0) + registered.append(listener) + + register_thread = threading.Thread(target=_register_after_stale_lookup, daemon=True) + register_thread.start() + assert request_has_ws.wait(5) + try: + with pytest.raises(RuntimeError, match="ambiguous delete"): + manager.delete_persisted( + ws.id, + delete_fn=MagicMock(side_effect=RuntimeError("ambiguous delete")), + expected_reservation_token=ws._fork_reservation_token, + ) + finally: + release_registration.set() + register_thread.join(5) + + assert not register_thread.is_alive() + assert len(registered) == 1 + assert registered[0].get_nowait() == {"type": "ws_closed"} + assert registered[0] not in ui._listeners + assert ui._listeners_terminal is True + assert manager._failed_delete_tombstones[ws.id] is ws + + +def test_soft_close_cannot_skip_active_tool_cancellation_finalizer() -> None: + """Soft close must complete an already-accepted assistant/tool block.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + tool_entered = threading.Event() + release_tool = threading.Event() + conversation_writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + conversation_writes.append(role) + return len(conversation_writes) + 20 + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], str]: + tool_entered.set() + assert release_tool.wait(5) + return [("call-active-close", "tool completed")], "" + + tool_call = { + "id": "call-active-close", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + send_errors: list[BaseException] = [] + + def _send() -> None: + try: + session.send("use the tool", acting_user_id="owner") + except BaseException as exc: + send_errors.append(exc) + + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", side_effect=_execute_tools), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + send_thread = threading.Thread(target=_send, daemon=True) + send_thread.start() + assert tool_entered.wait(5), "tool execution did not start" + + # Current close returns before the timer fires; a safe implementation + # may wait for the cancellation finalizer. The timer makes both choices + # deterministic without coupling the test to one policy. + release_timer = threading.Timer(0.1, release_tool.set) + release_timer.start() + close_result = session.prepare_soft_close() + release_timer.join(5) + send_thread.join(5) + + assert not send_thread.is_alive() + assert send_errors == [] + assert close_result in (False, True) + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + if close_result: + assert session.has_unresolved_conversation_persistence() is False + assert conversation_writes == ["user", "assistant", "tool"] + + +def test_force_successor_cannot_cross_an_incomplete_tool_prefix() -> None: + """A force successor must refuse or synthesize before it can claim history.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + tool_entered = threading.Event() + release_tool = threading.Event() + conversation_writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + conversation_writes.append(role) + return len(conversation_writes) + 30 + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], str]: + tool_entered.set() + assert release_tool.wait(5) + return [("call-force-crossing", "late tool result")], "" + + tool_call = { + "id": "call-force-crossing", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + send_errors: list[BaseException] = [] + + def _send() -> None: + try: + session.send("use the tool", acting_user_id="owner") + except BaseException as exc: + send_errors.append(exc) + + claim_generation: int | None = None + claim_error: Exception | None = None + prefix_complete_at_claim = False + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", side_effect=_execute_tools), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + send_thread = threading.Thread(target=_send, daemon=True) + send_thread.start() + assert tool_entered.wait(5), "tool execution did not start" + + session.cancel() + with pytest.raises(RuntimeError, match="tool cleanup"): + session._claim_generation() + direct_mutated: list[bool] = [] + assert ( + session._commit_for_generation( + 0, + lambda _durable: direct_mutated.append(True), + ) + is False + ) + assert direct_mutated == [] + try: + claim_generation = session._claim_generation() + except Exception as exc: + claim_error = exc + else: + prefix_complete_at_claim = [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + finally: + release_tool.set() + send_thread.join(5) + + assert not send_thread.is_alive() + assert send_errors == [] + assert (claim_generation is None) is (claim_error is not None) + if claim_generation is not None: + assert prefix_complete_at_claim, "successor claimed an incomplete accepted prefix" + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + + +def test_structural_repair_counts_duplicate_tool_ids_by_occurrence() -> None: + """One existing result must not hide a second call with the same provider ID.""" + session = make_session(user_id="owner", ui=RecordingUI()) + duplicate_id = "provider-duplicate" + assistant = Turn.assistant( + tool_calls=( + ToolCall(id=duplicate_id, name="first", arguments="{}"), + ToolCall(id=duplicate_id, name="second", arguments="{}"), + ) + ) + existing_result = Turn.tool(duplicate_id, "first result") + with session._generation_lock: + session._generation = 1 + session.messages.extend((assistant, existing_result)) + session._msg_tokens.extend((1, 1)) + session._admit_tool_structural_debt_locked(1, (duplicate_id, duplicate_id)) + + writes: list[tuple[str, str | None]] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **kwargs: Any, + ) -> int: + writes.append((role, kwargs.get("tool_call_id"))) + return 73 + + def _repair(durable: list[Any]) -> None: + session._synthesize_cancelled_results( + "Tool execution returned no observable outcome.", + deferred_persistence=durable, + structural_generation=1, + ) + + with patch("turnstone.core.session.save_message", side_effect=_save_message): + assert session._commit_for_generation(1, _repair) is True + + assert [turn.role for turn in session.messages] == [ + Role.ASSISTANT, + Role.TOOL, + Role.TOOL, + ] + assert [turn.tool_call_id for turn in session.messages[1:]] == [ + duplicate_id, + duplicate_id, + ] + assert writes == [("tool", duplicate_id)] + assert session._tool_structural_debt is None + + +def test_structural_debt_refuses_an_extra_tool_occurrence() -> None: + """Completion is exact: an extra TOOL row cannot retire the generation.""" + session = make_session(user_id="owner", ui=RecordingUI()) + with session._generation_lock: + session._generation = 1 + session._admit_tool_structural_debt_locked(1, ("exact-call",)) + with pytest.raises(RuntimeError, match=r"unexpected: exact-call"): + session._complete_tool_structural_debt_locked( + 1, + ("exact-call", "exact-call"), + ) + assert session._tool_structural_debt is not None + + +def test_soft_close_timeout_rolls_back_latch_until_force_repairs_debt() -> None: + """A refused close stays recoverable without admitting a successor early.""" + session = make_session(user_id="owner", ui=RecordingUI()) + call_id = "close-timeout-debt" + with session._generation_lock: + session._generation = 1 + session.messages.extend( + ( + Turn.user("use the tool"), + Turn.assistant( + tool_calls=(ToolCall(id=call_id, name="write_file", arguments="{}"),) + ), + ) + ) + session._msg_tokens.extend((1, 1)) + session._admit_tool_structural_debt_locked(1, (call_id,)) + + with patch("turnstone.core.session._SOFT_CLOSE_STRUCTURAL_WAIT_SECONDS", 0.001): + assert session.prepare_soft_close() is False + + assert session._soft_close_preparing is False + assert session._publication_shutdown is False + assert session._tool_structural_debt is not None + with pytest.raises(RuntimeError, match="tool cleanup"): + session._claim_generation() + + observed_at_clear: list[list[Role]] = [] + + def _clear() -> bool: + observed_at_clear.append([turn.role for turn in session.messages]) + return True + + with patch("turnstone.core.session.save_message", return_value=81): + abandoned, persistence_error = session.force_abandon_generation( + target_is_current=lambda: True, + clear_target=_clear, + publish_abandoned=lambda: None, + ) + + assert abandoned is True + assert persistence_error is None + assert observed_at_clear == [[Role.USER, Role.ASSISTANT, Role.TOOL]] + assert session._tool_structural_debt is None + assert session._claim_generation() > 0 + + +@pytest.mark.parametrize("tool_row_id", [43, 0], ids=["healthy", "poisoned"]) +def test_force_handler_journals_unknown_before_slot_release(tool_row_id: int) -> None: + """HTTP force-abandon closes structural debt before exposing the slot.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id=f"force-prefix-{tool_row_id}") + ui = RecordingUI() + ui._enqueue = lambda _event: None # type: ignore[attr-defined] + session = make_session(ws_id=ws.id, user_id="owner", ui=ui) + session._title_generated = True + ws.session = session + ws.ui = ui + tool_entered = threading.Event() + release_tool = threading.Event() + writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + writes.append(role) + if role == "tool": + return tool_row_id + return len(writes) + 50 + + def _execute_tools(*_args: Any, **_kwargs: Any) -> tuple[list[tuple[str, str]], str]: + tool_entered.set() + assert release_tool.wait(5) + return [("call-force-handler", "late result")], "" + + tool_call = { + "id": "call-force-handler", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + send_errors: list[BaseException] = [] + + def _send() -> None: + try: + session.send("use the tool", acting_user_id="owner") + except BaseException as exc: + send_errors.append(exc) + + observed_at_clear: list[tuple[list[Role], str, int]] = [] + real_force_abandon = session.force_abandon_generation + + def _force_abandon_with_clear_probe(**kwargs: Any) -> Any: + clear_target = kwargs["clear_target"] + + def _checked_clear() -> bool: + status = session.conversation_persistence_status() + observed_at_clear.append( + ( + [turn.role for turn in session.messages], + str(status["state"]), + int(status["pending_rows"]), + ) + ) + return clear_target() + + kwargs["clear_target"] = _checked_clear + return real_force_abandon(**kwargs) + + session.force_abandon_generation = _force_abandon_with_clear_probe # type: ignore[method-assign] + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (manager, None), + tenant_check=None, + not_found_label="not found", + audit_action_prefix="workstream", + ) + handler = make_cancel_handler(cfg) + + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch("turnstone.core.memory.persist_last_error"), + patch.object(session, "_stream_response", return_value=make_result(tool_calls=[tool_call])), + patch.object(session, "_execute_tools", side_effect=_execute_tools), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + send_thread = threading.Thread(target=_send, daemon=True) + send_thread.start() + assert tool_entered.wait(5), "tool execution did not start" + with ws._lock: + ws._worker_running = True + ws.worker_thread = send_thread + ws._worker_force_abandonable = True + + response = _run_force_cancel_handler(handler, ws.id) + assert response.status_code == 200 + assert observed_at_clear == [([Role.USER, Role.ASSISTANT, Role.TOOL], "pending", 1)] + assert ws._worker_running is False + assert ws.worker_thread is None + assert session.messages[-1].effect_status is not None + assert session.messages[-1].effect_status.value == "unknown" + assert session._claim_generation() > 0 + + release_tool.set() + send_thread.join(5) + + assert not send_thread.is_alive() + assert send_errors == [] + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + expected_state = "healthy" if tool_row_id else "retrying" + assert session.conversation_persistence_status()["state"] == expected_state + + +@pytest.mark.parametrize("tool_row_id", [71, 0], ids=["healthy", "poisoned"]) +def test_force_terminal_ui_precedes_successor_generation_publication( + tool_row_id: int, +) -> None: + """Force publishes stream-end/idle before a cleared slot can think.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id=f"force-ui-order-{tool_row_id}") + ui = RecordingUI() + ordered_events: list[str] = [] + ui._enqueue = lambda event: ordered_events.append(str(event["type"])) # type: ignore[attr-defined] + ui.on_state_change = lambda state: ordered_events.append(str(state)) # type: ignore[method-assign] + session = make_session(ws_id=ws.id, user_id="owner", ui=ui) + ws.session = session + ws.ui = ui + _seed_tool_structural_debt(session, "force-ui-order-call") + predecessor = threading.Thread(target=lambda: None, name="force-ui-predecessor") + with ws._lock: + ws._worker_running = True + ws.worker_thread = predecessor + ws._worker_force_abandonable = True + + save_entered = threading.Event() + release_save = threading.Event() + force_returned = threading.Event() + release_route = threading.Event() + successor_published = threading.Event() + force_responses: list[Any] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **_kwargs: Any, + ) -> int: + if role == "tool": + save_entered.set() + assert release_save.wait(5) + return tool_row_id + return 1 + + def _after_force_method() -> None: + force_returned.set() + assert release_route.wait(5) + + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (manager, None), + tenant_check=None, + not_found_label="not found", + audit_action_prefix="workstream", + ) + handler = make_cancel_handler(cfg) + + def _run_force() -> None: + force_responses.append( + _run_force_cancel_handler( + handler, + ws.id, + after_to_thread=_after_force_method, + ) + ) + + def _run_successor() -> None: + session._claim_generation() + ordered_events.append("thinking") + successor_published.set() + + with patch("turnstone.core.session.save_message", side_effect=_save_message): + force_thread = threading.Thread(target=_run_force, daemon=True) + force_thread.start() + assert save_entered.wait(5), "force TOOL durability did not block" + + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_run_successor, + principal_id="owner", + ) + successor_thread = ws.worker_thread + assert successor_thread is not None + assert not successor_published.wait(0.05) + + release_save.set() + assert force_returned.wait(5), "force method did not settle" + assert successor_published.wait(5), "successor did not claim after force" + assert ordered_events[:3] == ["stream_end", "idle", "thinking"] + + release_route.set() + force_thread.join(5) + successor_thread.join(5) + + assert not force_thread.is_alive() and not successor_thread.is_alive() + assert force_responses[0].status_code == 200 + assert ordered_events.count("stream_end") == 1 + assert ordered_events.count("idle") == 1 + + +def test_force_handler_stale_target_cannot_clear_replacement() -> None: + """The production force callbacks revalidate the exact pinned worker.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="force-stale-target") + ui = RecordingUI() + ui._enqueue = lambda _event: None # type: ignore[attr-defined] + session = make_session(ws_id=ws.id, user_id="owner", ui=ui) + ws.session = session + ws.ui = ui + predecessor = threading.Thread(target=lambda: None, name="predecessor") + successor = threading.Thread(target=lambda: None, name="successor") + with ws._lock: + ws._worker_running = True + ws.worker_thread = predecessor + ws._worker_principal_id = "alice" + + def _cancel_and_replace() -> None: + with ws._lock: + ws.worker_thread = successor + ws._worker_running = True + ws._worker_principal_id = "bob" + + session.cancel = _cancel_and_replace # type: ignore[method-assign] + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (manager, None), + tenant_check=None, + not_found_label="not found", + audit_action_prefix="workstream", + ) + handler = make_cancel_handler(cfg) + + generation_before = session._generation + response = _run_force_cancel_handler(handler, ws.id) + + assert response.status_code == 200 + assert ws.worker_thread is successor + assert ws._worker_running is True + assert ws._worker_principal_id == "bob" + assert session._generation == generation_before + + +def test_force_handler_repairs_idle_structural_debt_but_idle_noop_stays_silent() -> None: + """Failed idle force stays poisoned; its retry is the recovery arm.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="force-idle-debt") + ui = RecordingUI() + events: list[dict[str, Any]] = [] + ui._enqueue = events.append # type: ignore[attr-defined] + session = make_session(ws_id=ws.id, user_id="owner", ui=ui) + ws.session = session + ws.ui = ui + _seed_tool_structural_debt(session, "idle-debt-call") + session._cancel_event.set() + + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (manager, None), + tenant_check=None, + not_found_label="not found", + audit_action_prefix="workstream", + ) + handler = make_cancel_handler(cfg) + real_journal = session._journal_conversation_row_locked + rejected = False + + def _reject_first_tool_journal(**kwargs: Any) -> Any: + nonlocal rejected + if kwargs["message"].get("role") == "tool" and not rejected: + rejected = True + raise RuntimeError("injected idle-force journal failure") + return real_journal(**kwargs) + + with ( + patch("turnstone.core.session.save_message", return_value=131) as save, + patch.object( + session, + "_journal_conversation_row_locked", + side_effect=_reject_first_tool_journal, + ), + pytest.raises(RuntimeError, match="injected idle-force journal failure"), + ): + _run_force_cancel_handler(handler, ws.id) + + assert [turn.role for turn in session.messages] == [Role.USER, Role.ASSISTANT] + assert session._tool_structural_debt is not None + assert session._cancel_event.is_set() + save.assert_not_called() + with pytest.raises(RuntimeError, match="tool cleanup"): + session._capture_worker_claim("owner") + + with patch("turnstone.core.session.save_message", return_value=131): + response = _run_force_cancel_handler(handler, ws.id) + + assert response.status_code == 200 + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + assert session.messages[-1].effect_status is not None + assert session.messages[-1].effect_status.value == "unknown" + assert session._tool_structural_debt is None + assert session._claim_generation() > 0 + assert any(event.get("type") == "stream_end" for event in events) + + idle_ws = manager.create(user_id="owner", ws_id="force-idle-no-debt") + idle_ui = RecordingUI() + idle_events: list[dict[str, Any]] = [] + idle_ui._enqueue = idle_events.append # type: ignore[attr-defined] + idle_session = make_session(ws_id=idle_ws.id, user_id="owner", ui=idle_ui) + idle_ws.session = idle_session + idle_ws.ui = idle_ui + generation_before = idle_session._generation + + response = _run_force_cancel_handler(handler, idle_ws.id) + + assert response.status_code == 200 + assert idle_session._generation == generation_before + assert idle_events == [] + + +def test_force_handler_does_not_wait_on_nonabandonable_truncation() -> None: + """Force is prompt cancellation, not a wait on an owned history cut.""" + manager, _adapter, _storage = _make_manager() + ws = manager.create(user_id="owner", ws_id="force-nonabandonable-cut") + ui = RecordingUI() + enqueued: list[dict[str, Any]] = [] + ui._enqueue = enqueued.append # type: ignore[attr-defined] + session = make_session(ws_id=ws.id, user_id="owner", ui=ui) + ws.session = session + ws.ui = ui + owner = threading.Thread(target=lambda: None, name="history-cut-owner") + with ws._lock: + ws._worker_running = True + ws.worker_thread = owner + ws._worker_force_abandonable = False + with session._history_truncation_condition: + session._history_truncation_active = True + + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (manager, None), + tenant_check=None, + not_found_label="not found", + audit_action_prefix="workstream", + ) + handler = make_cancel_handler(cfg) + responses: list[Any] = [] + request_thread = threading.Thread( + target=lambda: responses.append(_run_force_cancel_handler(handler, ws.id)), + daemon=True, + ) + request_thread.start() + request_thread.join(1) + try: + assert not request_thread.is_alive(), "force waited on a non-abandonable history cut" + assert responses[0].status_code == 200 + assert ws.worker_thread is owner + assert ws._worker_running is True + assert any(event.get("type") == "cancelled" for event in enqueued) + finally: + with session._history_truncation_condition: + session._history_truncation_active = False + session._history_truncation_condition.notify_all() + request_thread.join(5) + + +def test_user_journal_rejection_rolls_back_live_append() -> None: + """A USER journal exception must not leave a live turn no row represents.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + + def _reject_user_journal(**kwargs: Any) -> Any: + raise RuntimeError("injected user journal rejection") + + revision_before = session._history_handoff_revision + with ( + patch("turnstone.core.memory.persist_last_error"), + patch.object( + session, + "_journal_conversation_row_locked", + side_effect=_reject_user_journal, + ), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + pytest.raises(RuntimeError, match="injected user journal rejection"), + ): + session.send("hello", acting_user_id="owner") + + assert session.messages == [] + assert session._msg_tokens == [] + assert session._pending_conversation_commits == {} + assert session._history_handoff_revision == revision_before + + +def test_system_journal_rejection_rolls_back_live_append() -> None: + """A SYSTEM journal exception rolls the operator-context append back.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + + real_journal = session._journal_conversation_row_locked + + def _reject_system_journal(**kwargs: Any) -> Any: + if kwargs["message"].get("role") == "system": + raise RuntimeError("injected system journal rejection") + return real_journal(**kwargs) + + revision_before = session._history_handoff_revision + with ( + patch.object( + session, + "_journal_conversation_row_locked", + side_effect=_reject_system_journal, + ), + pytest.raises(RuntimeError, match="injected system journal rejection"), + ): + session._append_system_turn("correction", "operator note") + + assert session.messages == [] + assert session._msg_tokens == [] + assert session._pending_conversation_commits == {} + assert session._history_handoff_revision == revision_before + + +def test_workstream_gone_discard_is_terminal_nonraising_and_bumps_revision() -> None: + """A hard-deleted parent resolves the journal by discard, not by error. + + The deletion is the user-facing event: the persist returns normally, the + journal empties, no error latch survives, every token minted over the + discarded rows is invalidated, exactly one repair event points panes at + the authoritative (deleted) history, and lifecycle proceeds. + """ + from turnstone.core.storage import ConversationCommitWorkstreamGoneError + + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + + durable: list[Any] = [] + session._append_system_turn("correction", "operator note", deferred_persistence=durable) + assert len(session._pending_conversation_commits) == 1 + _rows, token = session.capture_history_handoff(lambda _overscan: []) + # Installed after admission: RecordingUI has no on_system_turn hook, so + # the append itself already fired one repair. Only the discard's single + # repair is under test. + resync = MagicMock(return_value=None) + session.ui.on_history_resync = resync + + with patch( + "turnstone.core.session.save_message", + side_effect=ConversationCommitWorkstreamGoneError("workstream no longer exists"), + ): + for persist in durable: + persist() + + assert session._pending_conversation_commits == {} + assert session._conversation_persistence_error is None + assert session.conversation_persistence_status()["state"] == "healthy" + assert session.register_listener_for_history_handoff(token) is None + # The discard's repair names the deletion (panes/SDKs treat reason as a + # free string; the dedicated value is the hook for a deleted-workstream + # banner) rather than the retryable persistence reason. + assert resync.call_args_list == [ + (("workstream_gone",),), + ] + # The latch is set and capture refuses to mint over the deleted parent — + # /history takes its fail-closed 503 arm instead of authorizing a + # token-bearing render of an empty transcript (the silent-wipe mechanism). + assert session.is_workstream_gone() is True + with pytest.raises(ConversationCommitWorkstreamGoneError): + session.capture_history_handoff(lambda _overscan: []) + assert session.prepare_soft_close() is True + + +def test_gone_discard_is_a_terminal_latch_not_a_phantom_error() -> None: + """Admission after the discard refuses via the monotonic gone latch — + never via a stale persistence-error poison. + + Round-3 review: without the latch the live session kept accepting turns + whose keyed saves discarded on every attempt — a permanent silent black + hole. The refusal is clean (no error latch, empty journal, healthy + status) and the convergence lanes stay open so cancel/failure finalizers + still run. + """ + from turnstone.core.session import GenerationCancelled + from turnstone.core.storage import ConversationCommitWorkstreamGoneError + + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + + durable: list[Any] = [] + session._append_system_turn("correction", "first note", deferred_persistence=durable) + with patch( + "turnstone.core.session.save_message", + side_effect=ConversationCommitWorkstreamGoneError("workstream no longer exists"), + ): + for persist in durable: + persist() + + assert session._conversation_persistence_error is None + assert session.is_workstream_gone() is True + # New conversation admissions refuse... + with pytest.raises(RuntimeError, match="closed session"): + session._append_system_turn("correction", "second note") + # ...cleanly: no journal residue, no error latch, healthy status. + assert session._pending_conversation_commits == {} + assert session.conversation_persistence_status()["state"] == "healthy" + # The convergence lanes stay open for the finalizers. + ran: list[int] = [] + assert session._commit_for_generation(0, lambda _d: ran.append(1)) is False + assert ran == [] + assert ( + session._commit_for_generation(0, lambda _d: ran.append(1), allow_workstream_gone=True) + is True + ) + assert ran == [1] + # Destructive history commands refuse the same way; the rewind route + # converts the raise to its 503 error arm. + with pytest.raises(GenerationCancelled): + session.rewind(1) + # An identity swap (the /new//resume shape) structurally un-poisons: + # the latch names the DEAD workstream, not the session object, so a + # session repointed at a different ws_id admits again with no reset + # choreography (round-4 review). + session._ws_id = "fresh-after-swap" + assert session.is_workstream_gone() is False + ran.clear() + assert session._commit_for_generation(0, lambda _d: ran.append(1)) is True + assert ran == [1] diff --git a/tests/test_persistence_reconciliation.py b/tests/test_persistence_reconciliation.py new file mode 100644 index 00000000..c9138d5c --- /dev/null +++ b/tests/test_persistence_reconciliation.py @@ -0,0 +1,796 @@ +"""Adversarial coverage for accepted-row persistence reconciliation.""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +import pytest + +from tests._session_helpers import RecordingUI, make_result, make_session +from tests.test_session_manager import _make_manager +from turnstone.core.session import ConversationPersistenceError, EffectStatus +from turnstone.core.storage import ConversationCommitConflictError +from turnstone.core.trajectory import Role +from turnstone.core.workstream import WorkstreamState + +if TYPE_CHECKING: + from collections.abc import Callable + + +def _journal( + session: Any, + persist: Callable[[], int], + *, + commit_key: str = "commit-a", +) -> Any: + with session._history_handoff_lock: + return session._journal_conversation_row_locked( + commit_key=commit_key, + message={"role": "assistant", "content": "accepted"}, + persist=persist, + event_id=7, + ) + + +def test_transient_reconciliation_is_single_attempt_due_gated_and_capped() -> None: + session = make_session() + clock = [100.0] + outcomes = iter([0, 0, 41]) + persist = MagicMock(side_effect=lambda: next(outcomes)) + entry = _journal(session, persist) + assert session.conversation_persistence_status()["state"] == "pending" + + with ( + patch("turnstone.core.session.time.monotonic", side_effect=lambda: clock[0]), + patch.object( + session, + "_conversation_persistence_retry_delay", + side_effect=[2.0, 4.0], + ), + ): + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + + assert persist.call_count == 1 + assert session.conversation_persistence_status()["state"] == "retrying" + assert session._conversation_persistence_next_retry_at == 102.0 + + clock[0] = 101.999 + assert session.reconcile_unresolved_persistence_if_due(now=clock[0]) is False + with pytest.raises(ConversationPersistenceError): + session._reconcile_pending_conversation_commits() + assert persist.call_count == 1 + + clock[0] = 102.0 + assert session.reconcile_unresolved_persistence_if_due(now=clock[0]) is True + assert persist.call_count == 2 + assert session._conversation_persistence_next_retry_at == 106.0 + + clock[0] = 106.0 + assert session.reconcile_unresolved_persistence_if_due(now=clock[0]) is True + + assert persist.call_count == 3 + assert session.has_unresolved_conversation_persistence() is False + assert session.conversation_persistence_status() == { + "state": "healthy", + "pending_rows": 0, + "attempts": 0, + "first_failure_at": None, + "last_failure_at": None, + "next_retry_at": None, + } + with patch("turnstone.core.session.random.uniform", return_value=60.0) as jitter: + assert session._conversation_persistence_retry_delay(10_000) == 60.0 + assert jitter.call_args.args == (30.0, 60.0) + + +def test_repeated_failure_dedupes_repair_fanout_until_recovery() -> None: + session = make_session() + session.ui.on_history_resync = MagicMock() + session.ui.on_persistence_state_changed = MagicMock() + persist = MagicMock(side_effect=[0, 0, 9]) + entry = _journal(session, persist) + + with patch.object(session, "_conversation_persistence_retry_delay", return_value=0.0): + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is True + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is True + + session.ui.on_history_resync.assert_called_once_with("conversation_persistence_unresolved") + assert session.ui.on_persistence_state_changed.call_count == 2 + assert session.conversation_persistence_status()["state"] == "healthy" + + +def test_concurrent_due_sweeps_cannot_duplicate_one_retry_attempt() -> None: + session = make_session() + retry_entered = threading.Event() + release_retry = threading.Event() + calls = 0 + + def _persist() -> int: + nonlocal calls + calls += 1 + if calls == 1: + return 0 + retry_entered.set() + assert release_retry.wait(5) + return 0 + + entry = _journal(session, _persist) + delays = iter([0.0, 10.0]) + with ( + patch("turnstone.core.session.time.monotonic", return_value=100.0), + patch.object( + session, + "_conversation_persistence_retry_delay", + side_effect=lambda _attempt: next(delays), + ), + ): + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + + results: list[bool] = [] + sweeps = [ + threading.Thread( + target=lambda: results.append( + session.reconcile_unresolved_persistence_if_due(now=100.0) + ), + daemon=True, + ) + for _ in range(2) + ] + sweeps[0].start() + assert retry_entered.wait(5) + sweeps[1].start() + release_retry.set() + for sweep in sweeps: + sweep.join(5) + + assert all(not sweep.is_alive() for sweep in sweeps) + assert calls == 2 + assert sorted(results) == [False, True] + assert session._conversation_persistence_next_retry_at == 110.0 + + +def test_commit_conflict_is_chained_permanent_and_not_history_acked() -> None: + session = make_session() + conflict = ConversationCommitConflictError("immutable mismatch") + persist = MagicMock(side_effect=conflict) + entry = _journal(session, persist) + + with pytest.raises(ConversationPersistenceError) as raised: + session._persist_pending_conversation_commit(entry) + + assert raised.value.__cause__ is conflict + assert "immutable durable commit conflict" in str(raised.value) + assert persist.call_count == 1 + status = session.conversation_persistence_status() + assert status["state"] == "conflict" + assert status["next_retry_at"] is None + assert set(status) == { + "state", + "pending_rows", + "attempts", + "first_failure_at", + "last_failure_at", + "next_retry_at", + } + + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is False + with pytest.raises(ConversationPersistenceError) as retried: + session._reconcile_pending_conversation_commits(_force_retry=True) + assert retried.value is raised.value + assert persist.call_count == 1 + + rows, _token = session.capture_history_handoff( + lambda _overscan: [ + { + "role": "assistant", + "content": "different durable value", + "_commit_key": entry.commit_key, + } + ] + ) + assert session.has_unresolved_conversation_persistence() is True + assert session.conversation_persistence_status()["state"] == "conflict" + assert rows[0]["content"] == "accepted" + + +def test_history_lost_ack_reconciliation_clears_all_retry_metadata() -> None: + session = make_session() + session.ui.on_persistence_state_changed = MagicMock() + entry = _journal(session, MagicMock(return_value=0)) + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + + assert session.conversation_persistence_status()["state"] == "retrying" + session.capture_history_handoff( + lambda _overscan: [ + { + "role": "assistant", + "content": "accepted", + "_commit_key": entry.commit_key, + } + ] + ) + + assert session.conversation_persistence_status() == { + "state": "healthy", + "pending_rows": 0, + "attempts": 0, + "first_failure_at": None, + "last_failure_at": None, + "next_retry_at": None, + } + assert session.ui.on_persistence_state_changed.call_count == 2 + + +def test_already_acked_race_has_no_fabricated_row_id() -> None: + session = make_session() + persist_entered = threading.Event() + release_persist = threading.Event() + + def _persist() -> int: + persist_entered.set() + assert release_persist.wait(5) + return 42 + + persist = MagicMock(side_effect=_persist) + entry = _journal(session, persist) + results: list[object] = [] + + first = threading.Thread( + target=lambda: results.append(session._persist_pending_conversation_commit(entry)), + daemon=True, + ) + second = threading.Thread( + target=lambda: results.append(session._persist_pending_conversation_commit(entry)), + daemon=True, + ) + first.start() + assert persist_entered.wait(5) + second.start() + release_persist.set() + first.join(5) + second.join(5) + + assert not first.is_alive() and not second.is_alive() + assert persist.call_count == 1 + assert results == [None, None] + + +def test_terminal_barrier_blocks_a_late_due_sweep() -> None: + session = make_session() + persist = MagicMock(return_value=0) + entry = _journal(session, persist) + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + assert persist.call_count == 1 + + session.shutdown_publication_and_drain_durability() + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is False + assert persist.call_count == 1 + + +def test_terminal_race_is_a_benign_cancelled_maintenance_pass() -> None: + session = make_session() + entry = _journal(session, MagicMock(return_value=0)) + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + + from turnstone.core.session import GenerationCancelled + + with patch.object( + session, + "_reconcile_pending_conversation_commits", + side_effect=GenerationCancelled(), + ): + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is False + + +def test_soft_close_forces_transient_retry_but_never_retries_conflict() -> None: + transient = make_session() + transient_persist = MagicMock(side_effect=[0, 17]) + transient_entry = _journal(transient, transient_persist) + with ( + patch.object( + transient, + "_conversation_persistence_retry_delay", + return_value=60.0, + ), + pytest.raises(ConversationPersistenceError), + ): + transient._persist_pending_conversation_commit(transient_entry) + + assert transient.prepare_soft_close() is True + assert transient_persist.call_count == 2 + assert transient.has_unresolved_conversation_persistence() is False + + conflicted = make_session() + conflict = ConversationCommitConflictError("immutable mismatch") + conflict_persist = MagicMock(side_effect=conflict) + conflict_entry = _journal(conflicted, conflict_persist) + with pytest.raises(ConversationPersistenceError): + conflicted._persist_pending_conversation_commit(conflict_entry) + + assert conflicted.prepare_soft_close() is False + assert conflicted._publication_shutdown is False + assert conflict_persist.call_count == 1 + + +def test_assistant_persistence_recovery_completes_unstarted_tool_prefix( + tmp_db: Any, +) -> None: + """Recovery cannot expose an assistant tool call without its TOOL row.""" + from turnstone.core.memory import register_workstream + + manager, _adapter, _storage = _make_manager(max_active=1) + ws = manager.create(user_id="owner", ws_id="tool-prefix-recovery") + session = make_session(ws_id=ws.id, user_id="owner", ui=RecordingUI()) + session._title_generated = True + ws.session = session + register_workstream(ws.id, user_id="owner") + tool_calls = [ + { + "id": call_id, + "type": "function", + "function": {"name": tool_name, "arguments": "{}"}, + } + for call_id, tool_name in ( + ("call-never-started-a", "write_file"), + ("call-never-started-b", "notify"), + ) + ] + assistant_attempts = 0 + saved_roles: list[tuple[str, str | None]] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **kwargs: Any, + ) -> int: + nonlocal assistant_attempts + saved_roles.append((role, kwargs.get("tool_call_id"))) + if role == "assistant" and kwargs.get("tool_calls"): + assistant_attempts += 1 + if assistant_attempts == 1: + return 0 + return len(saved_roles) + 10 + + stream_calls = 0 + + def _stream(_generation: int) -> Any: + nonlocal stream_calls + stream_calls += 1 + if stream_calls == 1: + return make_result(tool_calls=tool_calls) + # The next provider call sees a complete assistant/tool prefix before + # the newly admitted user row. This is the structural provider seam + # that the old recovery path violated. + roles = [turn.role for turn in session.messages] + assert roles == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + Role.TOOL, + Role.USER, + ] + assistant = session.messages[1] + assert [call.id for call in assistant.tool_calls] == [ + "call-never-started-a", + "call-never-started-b", + ] + assert [turn.tool_call_id for turn in session.messages[2:4]] == [ + "call-never-started-a", + "call-never-started-b", + ] + return make_result("continued safely") + + execute = MagicMock() + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message) as save, + patch.object(session, "_stream_response", side_effect=_stream), + patch.object(session, "_execute_tools", execute), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + with pytest.raises(ConversationPersistenceError): + session.send("use the tool", acting_user_id="owner") + + execute.assert_not_called() + assert saved_roles == [("user", None), ("assistant", None)] + assert session.conversation_persistence_status()["state"] == "retrying" + assert session.conversation_persistence_status()["pending_rows"] == 3 + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + Role.TOOL, + ] + cancelled = session.messages[-2:] + assert [turn.tool_call_id for turn in cancelled] == [ + "call-never-started-a", + "call-never-started-b", + ] + assert all(turn.effect_status is EffectStatus.NONE for turn in cancelled) + assert all("no side effects" in turn.text.lower() for turn in cancelled) + + manager.set_state(ws.id, WorkstreamState.ERROR) + retry_at = session._conversation_persistence_next_retry_at + assert retry_at is not None + assert manager.reconcile_unresolved_persistence(now=retry_at - 0.001) == [] + assert save.call_count == 2 + + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [ws.id] + assert saved_roles == [ + ("user", None), + ("assistant", None), + ("assistant", None), + ("tool", "call-never-started-a"), + ("tool", "call-never-started-b"), + ] + assert session.conversation_persistence_status()["state"] == "healthy" + assert ws.state is WorkstreamState.IDLE + + session.send("continue", acting_user_id="owner") + + execute.assert_not_called() + assert stream_calls == 2 + assert session.has_unresolved_conversation_persistence() is False + + +def test_assistant_commit_conflict_completes_live_tool_prefix_but_never_retries( + tmp_db: Any, +) -> None: + """A permanent assistant conflict is structurally complete but fail-stop.""" + session = make_session(user_id="owner", ui=RecordingUI()) + session._title_generated = True + conflict = ConversationCommitConflictError("immutable mismatch") + conversation_writes: list[str] = [] + + def _save_message( + _ws_id: str, + role: str, + _content: str, + *_args: Any, + **kwargs: Any, + ) -> int: + conversation_writes.append(role) + if role == "assistant" and kwargs.get("tool_calls"): + raise conflict + if role == "tool": + raise AssertionError("conflicted tool row must never be written") + return len(conversation_writes) + + tool_call = { + "id": "conflicted-call", + "type": "function", + "function": {"name": "write_file", "arguments": "{}"}, + } + execute = MagicMock() + stream = MagicMock(return_value=make_result(tool_calls=[tool_call])) + with ( + patch("turnstone.core.session.save_message", side_effect=_save_message), + patch.object(session, "_stream_response", stream), + patch.object(session, "_execute_tools", execute), + patch.object(session, "_visible_memory_count", return_value=0), + patch.object(session, "_print_status_line"), + ): + with pytest.raises(ConversationPersistenceError) as raised: + session.send("use the tool", acting_user_id="owner") + + assert raised.value.__cause__ is conflict + execute.assert_not_called() + assert conversation_writes == ["user", "assistant"] + assert [turn.role for turn in session.messages] == [ + Role.USER, + Role.ASSISTANT, + Role.TOOL, + ] + assert session.messages[-1].effect_status is EffectStatus.NONE + assert session.conversation_persistence_status()["state"] == "conflict" + assert session.conversation_persistence_status()["pending_rows"] == 2 + + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is False + with pytest.raises(ConversationPersistenceError): + session.send("must remain blocked", acting_user_id="owner") + + assert conversation_writes == ["user", "assistant"] + execute.assert_not_called() + stream.assert_called_once() + + +def test_terminal_barrier_waits_for_admitted_sweep_then_prevents_another() -> None: + session = make_session() + retry_entered = threading.Event() + release_retry = threading.Event() + calls = 0 + + def _persist() -> int: + nonlocal calls + calls += 1 + if calls == 1: + return 0 + retry_entered.set() + assert release_retry.wait(5) + return 23 + + entry = _journal(session, _persist) + with pytest.raises(ConversationPersistenceError): + session._persist_pending_conversation_commit(entry) + + repair = threading.Thread( + target=lambda: session.reconcile_unresolved_persistence_if_due(now=float("inf")), + daemon=True, + ) + repair.start() + assert retry_entered.wait(5) + + terminal = threading.Thread( + target=session.shutdown_publication_and_drain_durability, + daemon=True, + ) + terminal.start() + terminal.join(0.05) + assert terminal.is_alive(), "terminal delete barrier passed an in-flight repair" + + release_retry.set() + repair.join(5) + terminal.join(5) + assert not repair.is_alive() and not terminal.is_alive() + assert calls == 2 + assert session.reconcile_unresolved_persistence_if_due(now=float("inf")) is False + assert calls == 2 + + +class _RecoverableSession: + def __init__(self, manager: Any) -> None: + self._manager = manager + self.unresolved = True + self.reconcile_calls = 0 + self.manager_lock_was_free = False + self.cancelled = False + self.closed = False + + def has_unresolved_conversation_persistence(self) -> bool: + return self.unresolved + + def reconcile_unresolved_persistence_if_due(self, *, now: float) -> bool: + del now + self.reconcile_calls += 1 + acquired = threading.Event() + + def _probe_manager_lock() -> None: + with self._manager._lock: + acquired.set() + + probe = threading.Thread(target=_probe_manager_lock, daemon=True) + probe.start() + self.manager_lock_was_free = acquired.wait(1.0) + probe.join(1.0) + self.unresolved = False + return True + + def cancel(self) -> None: + self.cancelled = True + + def close(self) -> None: + self.closed = True + + +def test_manager_sweep_runs_outside_lock_and_capacity_retries_once() -> None: + manager, _adapter, _storage = _make_manager(max_active=1) + incumbent = manager.create(user_id="owner", ws_id="incumbent") + recoverable = _RecoverableSession(manager) + incumbent.session = recoverable + + replacement = manager.create(user_id="owner", ws_id="replacement") + + assert replacement.id == "replacement" + assert recoverable.reconcile_calls == 1 + assert recoverable.manager_lock_was_free is True + assert recoverable.cancelled is True + assert recoverable.closed is True + + +def test_manager_repairs_exact_persistence_error_to_idle_and_frees_capacity( + tmp_db: Any, +) -> None: + from turnstone.core.memory import register_workstream + + manager, _adapter, _storage = _make_manager(max_active=1) + ws = manager.create(user_id="owner", ws_id="real-persistence-error") + session = make_session(ws_id=ws.id) + session.ui.on_state_change = MagicMock() + ws.session = session + register_workstream(ws.id, user_id="owner") + persist = MagicMock(side_effect=[0, 31]) + entry = _journal(session, persist) + + with pytest.raises(ConversationPersistenceError) as raised: + session._persist_pending_conversation_commit(entry) + session._record_fatal_error(raised.value) + manager.set_state(ws.id, WorkstreamState.ERROR) + assert session.conversation_persistence_fatal_revision() is not None + + # A still-running recovery send owns the state and suppresses maintenance. + with ws._lock: + ws._worker_running = True + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [] + assert persist.call_count == 1 + assert ws.state is WorkstreamState.ERROR + with ws._lock: + ws._worker_running = False + + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [ws.id] + assert persist.call_count == 2 + assert ws.state is WorkstreamState.IDLE + assert session.conversation_persistence_fatal_revision() is None + + replacement = manager.create(user_id="owner", ws_id="replacement-after-repair") + assert replacement.id == "replacement-after-repair" + + +def test_manager_does_not_clear_an_error_with_newer_fatal_provenance(tmp_db: Any) -> None: + from turnstone.core.memory import register_workstream + + manager, _adapter, _storage = _make_manager(max_active=1) + ws = manager.create(user_id="owner", ws_id="newer-fatal") + session = make_session(ws_id=ws.id) + session.ui.on_state_change = MagicMock() + ws.session = session + register_workstream(ws.id, user_id="owner") + persist = MagicMock(side_effect=[0, 37]) + entry = _journal(session, persist) + + with pytest.raises(ConversationPersistenceError) as raised: + session._persist_pending_conversation_commit(entry) + session._record_fatal_error(raised.value) + session._record_fatal_error(RuntimeError("newer independent failure")) + manager.set_state(ws.id, WorkstreamState.ERROR) + + assert session.conversation_persistence_fatal_revision() is None + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [ws.id] + assert session.has_unresolved_conversation_persistence() is False + assert ws.state is WorkstreamState.ERROR + + +def test_manager_retires_persistence_error_after_history_proves_lost_ack( + tmp_db: Any, +) -> None: + from turnstone.core.memory import register_workstream + + manager, _adapter, _storage = _make_manager(max_active=1) + ws = manager.create(user_id="owner", ws_id="history-repaired-error") + session = make_session(ws_id=ws.id) + session.ui.on_state_change = MagicMock() + ws.session = session + register_workstream(ws.id, user_id="owner") + entry = _journal(session, MagicMock(return_value=0)) + + with pytest.raises(ConversationPersistenceError) as raised: + session._persist_pending_conversation_commit(entry) + # The repair event can make another browser fetch history before the + # failing worker reaches its fatal-record tail. Prove that this ordering + # still retains persistence provenance even though history clears the + # live journal error first. + session.capture_history_handoff( + lambda _overscan: [ + { + "role": "assistant", + "content": "accepted", + "_commit_key": entry.commit_key, + } + ] + ) + assert session.has_unresolved_conversation_persistence() is False + + session._record_fatal_error(raised.value) + manager.set_state(ws.id, WorkstreamState.ERROR) + assert ws.state is WorkstreamState.ERROR + + assert manager.reconcile_unresolved_persistence(now=float("inf")) == [ws.id] + assert ws.state is WorkstreamState.IDLE + assert session.conversation_persistence_fatal_revision() is None + + +class _MaintenanceManager: + def __init__(self, stop: threading.Event) -> None: + self.stop = stop + self.reconcile_calls = 0 + self.reap_calls = 0 + self.close_calls = 0 + + def reconcile_unresolved_persistence(self) -> list[str]: + self.reconcile_calls += 1 + if self.reconcile_calls >= 2: + self.stop.set() + return [] + + def reap_stale_creating_reservations(self, _max_age_seconds: float) -> list[str]: + self.reap_calls += 1 + return [] + + def close_idle(self, _max_age_seconds: float) -> list[str]: + self.close_calls += 1 + return [] + + def subscribe_to_state(self, _callback: Any) -> None: + raise AssertionError("idle-disabled maintenance must not subscribe") + + def unsubscribe_from_state(self, _callback: Any) -> None: + raise AssertionError("idle-disabled maintenance must not unsubscribe") + + +def test_server_maintenance_reconciles_when_idle_eviction_is_disabled() -> None: + from turnstone import server + + stop = threading.Event() + manager = _MaintenanceManager(stop) + with patch.object(server, "PERSISTENCE_RECONCILE_INTERVAL_SECONDS", 0.01): + server._idle_cleanup_thread( + manager, # type: ignore[arg-type] + 0.0, + MagicMock(), + stop=stop, + ) + + assert manager.reconcile_calls == 2 + assert manager.close_calls == 0 + assert manager.reap_calls == 1 + + +def test_coordinator_maintenance_reconciles_when_idle_eviction_is_disabled() -> None: + from turnstone.console.server import _coord_idle_cleanup_thread + from turnstone.core import session_manager + + stop = threading.Event() + manager = _MaintenanceManager(stop) + with patch.object(session_manager, "PERSISTENCE_RECONCILE_INTERVAL_SECONDS", 0.01): + _coord_idle_cleanup_thread( + manager, # type: ignore[arg-type] + 0.0, + stop, + ) + + assert manager.reconcile_calls == 2 + assert manager.close_calls == 0 + assert manager.reap_calls == 1 + + +def test_capacity_reconcile_forces_a_definite_probe(tmp_db: Any) -> None: + """``_reserve_and_install``'s last-chance repair runs ONCE, so it must + not skip a session whose locks are momentarily held. + + The maintenance sweep probes without blocking and re-probes a second + later; this caller has no second pass — and the sessions likeliest to + be contended are exactly the ones whose unresolved journals emptied + its candidate list, so skipping them turns a repairable capacity + stall into ``All N slots are active``.""" + manager, _adapter, _storage = _make_manager(max_active=1) + session = make_session() + ws = manager.create(user_id="u1", ws_id="cap-ws") + ws.session = session + ws.state = WorkstreamState.IDLE + + probes: list[str] = [] + session.has_unresolved_conversation_persistence = ( # type: ignore[method-assign] + lambda: probes.append("blocking") or False + ) + session.has_unresolved_conversation_persistence_nowait = ( # type: ignore[attr-defined] + lambda: probes.append("nowait") or None + ) + + # The maintenance shape skips a contended session outright. + manager.reconcile_unresolved_persistence() + assert probes == ["nowait"] + + # The one-shot capacity shape forces an answer instead. + probes.clear() + manager.reconcile_unresolved_persistence(blocking=True) + assert probes == ["blocking"] diff --git a/tests/test_preview_js.py b/tests/test_preview_js.py index 86b47230..4c0a9933 100644 --- a/tests/test_preview_js.py +++ b/tests/test_preview_js.py @@ -107,7 +107,8 @@ class TestTranscriptChip: path auto-opens only while the originating pane is focused; the chip is the deliberate reopen everywhere else.""" body = _read(_INTERACTIVE_JS) - assert "if (this._host.isFocused(this)) this._host.onPreview(preview);" in body + assert "!accepted && !isError && this._host.isFocused(this)" in body + assert "this._host.onPreview(preview);" in body def test_replay_path_renders_chip_without_auto_open(self) -> None: body = _read(_INTERACTIVE_JS) diff --git a/tests/test_prompt_templates_runtime.py b/tests/test_prompt_templates_runtime.py index 821ae2e5..7ce3262f 100644 --- a/tests/test_prompt_templates_runtime.py +++ b/tests/test_prompt_templates_runtime.py @@ -74,6 +74,8 @@ class NullUI: def _make_session(**kwargs): + from turnstone.core.memory import register_workstream + defaults = dict( client=MagicMock(), model="test-model", @@ -84,7 +86,12 @@ def _make_session(**kwargs): tool_timeout=30, ) defaults.update(kwargs) - return ChatSession(**defaults) + session = ChatSession(**defaults) + # SessionManager establishes this parent row before constructing the live + # session. The direct factory must do the same before slash commands can + # admit their keyed SYSTEM rows. + register_workstream(session.ws_id, user_id=kwargs.get("user_id")) + return session def _sys_content(session: ChatSession) -> str: diff --git a/tests/test_rewind_retry.py b/tests/test_rewind_retry.py index 02d49823..ef17e14b 100644 --- a/tests/test_rewind_retry.py +++ b/tests/test_rewind_retry.py @@ -4,6 +4,7 @@ from __future__ import annotations from unittest.mock import MagicMock +from turnstone.core.memory import register_workstream from turnstone.core.session import ChatSession from turnstone.core.trajectory import turn_to_dict, turns_from_dicts @@ -78,7 +79,7 @@ class NullUI: def _make_session(tmp_db, ws_id: str | None = None) -> ChatSession: - return ChatSession( + session = ChatSession( client=MagicMock(), model="test-model", ui=NullUI(), @@ -88,6 +89,10 @@ def _make_session(tmp_db, ws_id: str | None = None) -> ChatSession: tool_timeout=30, ws_id=ws_id, ) + # Production creates the durable parent before exposing a live session. + # Strict tail truncation shares that parent boundary with keyed commits. + register_workstream(session.ws_id, user_id="test-user") + return session def _populate_simple(session: ChatSession) -> None: @@ -325,6 +330,35 @@ class TestHandleCommand: ui.on_info.assert_called_once() assert "Nothing" in ui.on_info.call_args[0][0] + def test_rewind_refusal_reports_instead_of_killing_the_repl(self, tmp_db): + """Round-4 review pin: rewind()'s raising contract (GenerationCancelled + on a refused admission, e.g. the workstream-gone latch) converts to an + on_error message in the CLI arm — the REPL dispatches commands + uncaught, so a raise here previously killed the whole process.""" + session = _make_session(tmp_db) + _populate_simple(session) + ui = session.ui + ui.on_error = MagicMock() + session._workstream_gone_ws = session._ws_id + session.handle_command("/rewind 1") + ui.on_error.assert_called_once() + assert "refused" in ui.on_error.call_args[0][0].lower() + + def test_retry_storage_failure_reports_instead_of_killing_the_repl(self, tmp_db): + """Round-4 review pin: a storage error escaping retry() (the durable + truncation batch is raising now) becomes an on_error message, never an + uncaught REPL death.""" + from unittest.mock import patch + + session = _make_session(tmp_db) + _populate_simple(session) + ui = session.ui + ui.on_error = MagicMock() + with patch.object(session, "retry", side_effect=RuntimeError("database is locked")): + session.handle_command("/retry") + ui.on_error.assert_called_once() + assert "Retry failed" in ui.on_error.call_args[0][0] + # --------------------------------------------------------------------------- # Storage integration — delete_messages_after @@ -467,13 +501,13 @@ def test_truncation_bumps_history_generation(tmp_db) -> None: "a storage-deleting retry must bump the history generation" ) - # Error-path arm: an in-memory-only session (no persisted rows — the - # count<=0 early return skips the delete) must NOT bump. + # A registered workstream with no durable rows is still a successful + # atomic cut of the live trajectory, so its handoff revision advances. bare = _make_session(tmp_db) _populate_simple(bare) g2 = bare._history_generation assert bare.rewind(1) > 0 - assert bare._history_generation == g2, ( - "a truncation that could not delete storage rows must leave the " - "generation unbumped (fail-safe direction)" + assert bare._history_generation == g2 + 1, ( + "a successful live truncation must invalidate history even when the " + "durable tail is already empty" ) diff --git a/tests/test_sdk_events.py b/tests/test_sdk_events.py index d35f81aa..f83f8e8c 100644 --- a/tests/test_sdk_events.py +++ b/tests/test_sdk_events.py @@ -14,6 +14,7 @@ from turnstone.sdk.events import ( ContentEvent, ErrorEvent, HistoryEvent, + HistoryResyncEvent, InfoEvent, NodeJoinedEvent, NodeLostEvent, @@ -27,6 +28,7 @@ from turnstone.sdk.events import ( ToolInfoEvent, ToolOutputChunkEvent, ToolResultEvent, + UserTurnEvent, WsActivityEvent, WsClosedEvent, WsRenameEvent, @@ -55,6 +57,45 @@ def test_history_event(): assert e.messages == msgs +def test_history_resync_event_preserves_repair_reason(): + e = ServerEvent.from_dict( + { + "type": "history_resync", + "ws_id": "ws1", + "reason": "handoff_mismatch", + } + ) + assert isinstance(e, HistoryResyncEvent) + assert e.ws_id == "ws1" + assert e.reason == "handoff_mismatch" + + +def test_user_turn_event_preserves_correlation_and_attribution(): + e = ServerEvent.from_dict( + { + "type": "user_turn", + "ws_id": "ws1", + "content": "hello", + "attachments": [ + { + "attachment_id": "a1", + "kind": "text", + "filename": "note.txt", + "mime_type": "text/plain", + } + ], + "sender": "user-1", + "client_send_ids": ["browser-send"], + "_event_id": 17, + } + ) + assert isinstance(e, UserTurnEvent) + assert e.client_send_ids == ["browser-send"] + assert e.sender == "user-1" + assert e.attachments[0]["attachment_id"] == "a1" + assert e._event_id == 17 + + def test_thinking_start_stop(): e1 = ServerEvent.from_dict({"type": "thinking_start"}) e2 = ServerEvent.from_dict({"type": "thinking_stop"}) @@ -112,6 +153,31 @@ def test_tool_result_event(): assert e.output == "found it" +def test_accepted_tool_result_event_carries_final_projection_metadata(): + preview = {"kind": "html", "attachment_id": "preview-1"} + e = ServerEvent.from_dict( + { + "type": "tool_result", + "call_id": "c-final", + "name": "open_preview", + "output": "guarded\nscalar", + "is_error": True, + "preview": preview, + "accepted": True, + "effect_status": "unknown", + "_event_id": 42, + } + ) + + assert isinstance(e, ToolResultEvent) + assert e.output == "guarded\nscalar" + assert e.is_error is True + assert e.preview == preview + assert e.accepted is True + assert e.effect_status == "unknown" + assert e._event_id == 42 + + def test_tool_output_chunk_event(): e = ServerEvent.from_dict({"type": "tool_output_chunk", "call_id": "c1", "chunk": "line1\n"}) assert isinstance(e, ToolOutputChunkEvent) @@ -240,12 +306,20 @@ def test_ws_state_event(): "context_ratio": 0.3, "activity": "Writing code", "activity_state": "thinking", + "persistence_state": "retrying", } ) assert isinstance(e, WsStateEvent) assert e.ws_id == "ws1" assert e.state == "thinking" assert e.tokens == 500 + assert e.persistence_state == "retrying" + + +def test_ws_state_event_defaults_persistence_for_older_nodes(): + e = ServerEvent.from_dict({"type": "ws_state", "ws_id": "ws1"}) + assert isinstance(e, WsStateEvent) + assert e.persistence_state == "healthy" def test_ws_activity_event(): @@ -296,21 +370,30 @@ def test_cluster_state_event(): "context_ratio": 0.5, "activity": "executing tool", "activity_state": "tool", + "persistence_state": "conflict", } ) assert isinstance(e, ClusterStateEvent) assert e.node_id == "n1" assert e.state == "running" assert e.tokens == 1000 + assert e.persistence_state == "conflict" def test_cluster_ws_created_event(): e = ClusterEvent.from_dict( - {"type": "ws_created", "ws_id": "ws2", "node_id": "n1", "name": "New WS"} + { + "type": "ws_created", + "ws_id": "ws2", + "node_id": "n1", + "name": "New WS", + "persistence_state": "pending", + } ) assert isinstance(e, ClusterWsCreatedEvent) assert e.ws_id == "ws2" assert e.name == "New WS" + assert e.persistence_state == "pending" def test_cluster_ws_closed_event(): diff --git a/tests/test_sdk_server.py b/tests/test_sdk_server.py index 131e3aad..d02fd1da 100644 --- a/tests/test_sdk_server.py +++ b/tests/test_sdk_server.py @@ -201,6 +201,23 @@ async def test_send(): assert resp.status == "ok" +@pytest.mark.anyio +async def test_send_threads_client_send_id_without_idempotency_semantics(): + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured.update(json.loads(request.content)) + return httpx.Response(200, json={"status": "ok"}) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://test" + ) as hc: + client = AsyncTurnstoneServer(httpx_client=hc) + await client.send("Hello", "ws1", client_send_id="browser-send_1") + + assert captured == {"message": "Hello", "client_send_id": "browser-send_1"} + + @pytest.mark.anyio async def test_approve(): transport = _mock_transport( @@ -247,6 +264,36 @@ async def test_command(): # --------------------------------------------------------------------------- +@pytest.mark.anyio +async def test_get_history_preserves_handoff_fields_and_limit(): + captured: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["path"] = request.url.path + captured["limit"] = request.url.params["limit"] + return _json_response( + { + "ws_id": "ws1", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "system", "source": "compaction", "content": "summary"}, + ], + "cursor": 0, + "handoff_token": "epoch.7", + } + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc: + client = AsyncTurnstoneServer(httpx_client=hc) + resp = await client.get_history("ws1", limit=42) + + assert captured == {"path": "/v1/api/workstreams/ws1/history", "limit": "42"} + assert resp.cursor == 0 + assert resp.handoff_token == "epoch.7" + assert resp.messages[1]["role"] == "system" + + @pytest.mark.anyio async def test_list_saved_workstreams(): transport = _mock_transport( diff --git a/tests/test_sdk_sse.py b/tests/test_sdk_sse.py index 90c0908d..ff53ed5c 100644 --- a/tests/test_sdk_sse.py +++ b/tests/test_sdk_sse.py @@ -6,6 +6,8 @@ import httpx import pytest from turnstone.sdk._base import _BaseClient +from turnstone.sdk.events import HistoryResyncEvent, UserTurnEvent +from turnstone.sdk.server import AsyncTurnstoneServer def _sse_response(*events: str) -> httpx.Response: @@ -108,3 +110,60 @@ async def test_stream_sse_multiple_events(): assert len(events) == 5 types = [e["type"] for e in events] assert types == ["connected", "content", "content", "status", "stream_end"] + + +@pytest.mark.anyio +async def test_stream_events_forwards_initial_history_handoff_hints(): + captured: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured.update(dict(request.url.params)) + return _sse_response('{"type":"history_resync","ws_id":"ws1","reason":"handoff_mismatch"}') + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc: + client = AsyncTurnstoneServer(httpx_client=hc) + events = [ + event + async for event in client.stream_events( + "ws1", + last_event_id=0, + history_token="epoch.7", + ) + ] + + assert captured == { + "user_turn": "1", + "last_event_id": "0", + "history_token": "epoch.7", + } + assert len(events) == 1 + assert isinstance(events[0], HistoryResyncEvent) + assert events[0].reason == "handoff_mismatch" + + +@pytest.mark.anyio +async def test_stream_events_decodes_canonical_user_turn_wire_identity(): + """The capable SDK receives the payload identity, not only SSE transport id.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params["user_turn"] == "1" + return _sse_response( + '{"type":"user_turn","ws_id":"ws1","content":"hello peer",' + '"sender":"alice","client_send_ids":["send-1"],"_event_id":17}' + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc: + client = AsyncTurnstoneServer(httpx_client=hc) + events = [event async for event in client.stream_events("ws1", last_event_id=16)] + + assert events == [ + UserTurnEvent( + ws_id="ws1", + content="hello peer", + sender="alice", + client_send_ids=["send-1"], + _event_id=17, + ) + ] diff --git a/tests/test_sdk_sync.py b/tests/test_sdk_sync.py index 9e08e77c..453119e7 100644 --- a/tests/test_sdk_sync.py +++ b/tests/test_sdk_sync.py @@ -94,6 +94,32 @@ def test_sync_server_list_workstreams(): server.close() +def test_sync_server_get_history(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params["limit"] == "25" + return _json_response( + { + "ws_id": "ws1", + "messages": [], + "cursor": None, + "handoff_token": "epoch.1", + } + ) + + transport = httpx.MockTransport(handler) + hc = httpx.AsyncClient(transport=transport, base_url="http://test") + async_client = AsyncTurnstoneServer(httpx_client=hc) + server = TurnstoneServer.__new__(TurnstoneServer) + server._runner = _SyncRunner() + server._async = async_client + + try: + history = server.get_history("ws1", limit=25) + assert history.handoff_token == "epoch.1" + finally: + server.close() + + def test_sync_server_context_manager(): """TurnstoneServer can be used as a context manager.""" diff --git a/tests/test_server_attachments_endpoints.py b/tests/test_server_attachments_endpoints.py index 80d175ef..f56e27c8 100644 --- a/tests/test_server_attachments_endpoints.py +++ b/tests/test_server_attachments_endpoints.py @@ -123,6 +123,7 @@ def _harden_ws_mock(ws) -> None: ws._closed = False ws._pending_sends = [] ws._pending_drain = None + ws._worker_principal_id = "" ws.send_barrier_active = lambda: False ws._lock = threading.RLock() @@ -506,10 +507,11 @@ class TestSendMessageAttachments: session._nudge_queue = None captured: dict = {} - def fake_send(message, attachments=None, send_id=None): + def fake_send(message, attachments=None, send_id=None, client_send_ids=()): captured["message"] = message captured["attachments"] = attachments captured["send_id"] = send_id + captured["client_send_ids"] = client_send_ids session.send = fake_send @@ -529,6 +531,59 @@ class TestSendMessageAttachments: mgr.get.return_value = ws return captured, session + def _wire_admission_ws(self, mgr, ws_id: str, user_id: str): + """Install a real session whose send stops after USER admission. + + The HTTP resolver and ``ChatSession._append_user_turn`` then exercise + the production staged-buffer transfer without paying for a model call. + The real ``send`` derives the accepted turn's sender from the immutable + worker claim; mirror that boundary instead of reading the independently + owner-scoped attachment buffer identity. + """ + from turnstone.core.session import ChatSession + from turnstone.core.session_worker import current_worker_claim + from turnstone.core.workstream import WorkstreamState + + ui = MagicMock() + ui._ws_lock = threading.Lock() + ui._ws_messages = 0 + ui._ws_turn_tool_calls = 0 + session = ChatSession( + client=MagicMock(), + model="test-model", + ui=ui, + instructions=None, + temperature=0.3, + max_tokens=1024, + tool_timeout=10, + ws_id=ws_id, + user_id=user_id, + ) + + def admit_only(message, attachments=None, send_id=None): + claim = current_worker_claim(session) + if claim is None: + raise RuntimeError("admission test worker is missing its principal claim") + session._append_user_turn( + message, + attachments or (), + send_id=send_id, + sender_user_id=claim.principal_id, + ) + + session.send = admit_only # type: ignore[assignment] + + ws = MagicMock() + ws.id = ws_id + ws.state = WorkstreamState.IDLE + ws.ui = ui + ws.session = session + ws.worker_thread = None + ws._worker_running = False + _harden_ws_mock(ws) + mgr.get.return_value = ws + return ws, session + def test_send_explicit_attachment_ids_resolves_and_passes(self, app_client): client, mgr = app_client captured, _ = self._wire_ws(mgr, "ws-A", "userA") @@ -554,6 +609,32 @@ class TestSendMessageAttachments: assert atts[0].attachment_id == aid assert atts[0].kind == "text" + def test_send_validates_and_threads_client_send_id(self, app_client): + client, mgr = app_client + captured, _ = self._wire_ws(mgr, "ws-A", "userA") + + invalid = client.post( + "/v1/api/workstreams/ws-A/send", + json={"message": "bad token", "client_send_id": "spaces are invalid"}, + headers=_auth("userA"), + ) + assert invalid.status_code == 400 + assert invalid.json()["error"] == ("client_send_id must match [A-Za-z0-9_-]{1,128}") + + accepted = client.post( + "/v1/api/workstreams/ws-A/send", + json={"message": "correlated", "client_send_id": "browser-send_1"}, + headers=_auth("userA"), + ) + assert accepted.status_code == 200 + import time + + for _ in range(50): + if captured.get("message"): + break + time.sleep(0.01) + assert captured["client_send_ids"] == ("browser-send_1",) + def test_send_auto_consumes_pending_when_ids_omitted(self, app_client): client, mgr = app_client captured, _ = self._wire_ws(mgr, "ws-A", "userA") @@ -576,6 +657,59 @@ class TestSendMessageAttachments: assert captured["attachments"] is not None assert len(captured["attachments"]) == 2 + @pytest.mark.parametrize("acting_user", ["userA", "shared-userB"]) + @pytest.mark.parametrize("explicit_ids", [True, False]) + def test_owner_scoped_admission_consumes_once_for_shared_sender( + self, + app_client, + acting_user, + explicit_ids, + ): + """Actor identity and staged-byte ownership are separate scopes. + + Trusted-team workstreams file pending uploads under the durable + workstream owner. A different authenticated participant is still the + turn sender, but USER admission must transfer that owner-scoped upload + exactly once for both explicit-id and auto-consume requests. + """ + from turnstone.core.attachment_buffer import get_attachment_buffer + from turnstone.core.storage import get_storage + + client, mgr = app_client + ws, session = self._wire_admission_ws(mgr, "ws-A", "userA") + aid = _upload(client, "ws-A", acting_user, "shared.md", b"shared", "text/markdown") + buffer = get_attachment_buffer() + assert buffer.get(aid, ws_id="ws-A", user_id="userA") is not None + if acting_user != "userA": + assert buffer.get(aid, ws_id="ws-A", user_id=acting_user) is None + + body: dict[str, object] = {"message": "review together"} + if explicit_ids: + body["attachment_ids"] = [aid] + response = client.post( + "/v1/api/workstreams/ws-A/send", + json=body, + headers=_auth(acting_user), + ) + + assert response.status_code == 200 + assert response.json()["attached_ids"] == [aid] + worker = ws.worker_thread + assert worker is not None + worker.join(timeout=5) + assert not worker.is_alive() + assert buffer.get(aid, ws_id="ws-A", user_id="userA") is None + + storage = get_storage() + rows = storage.load_messages("ws-A", repair=False) + user_rows = [row for row in rows if row.get("role") == "user"] + assert len(user_rows) == 1 + assert user_rows[0]["_sender"] == acting_user + stored_attachment = storage.get_attachment(aid) + assert stored_attachment is not None + assert stored_attachment["refcount"] == 1 + assert session.has_unresolved_conversation_persistence() is False + def test_send_empty_list_disables_autoconsume(self, app_client): client, mgr = app_client captured, _ = self._wire_ws(mgr, "ws-A", "userA") diff --git a/tests/test_server_attachments_on_create.py b/tests/test_server_attachments_on_create.py index 05bc0ee4..16549cde 100644 --- a/tests/test_server_attachments_on_create.py +++ b/tests/test_server_attachments_on_create.py @@ -451,15 +451,13 @@ class TestCreateMultipart: # Drained from the buffer post-commit. assert get_attachment_buffer().get(aid, ws_id=ws_id, user_id="userA") is None - def test_create_drains_staged_synchronously(self, app_client, monkeypatch): - """A create-time attachment dispatched on the first turn must be drained - by the create handler itself, not only by the async dispatch worker — - else the freshly-opened pane's rehydrate races the worker's write-time - drain and paints the image as a still-pending composer chip. + def test_create_keeps_staging_until_user_row_admission(self, app_client, monkeypatch): + """Create dispatch may not consume bytes before the USER row accepts. - Neuter the worker's drain (stub ``send``) so only the synchronous - post-install drain can clear the buffer, then assert it's empty right - after the response with NO polling.""" + Neuter ``send`` so the worker never reaches journal admission. The + staged upload must remain retryable after the create response; the real + session transfers ownership atomically with its accepted USER row. + """ from turnstone.core.attachment_buffer import get_attachment_buffer client, _sessions, _gq = app_client @@ -474,9 +472,7 @@ class TestCreateMultipart: assert resp.status_code == 200, resp.text ws_id = resp.json()["ws_id"] aid = resp.json()["attachment_ids"][0] - # No poll: the create handler drained it before returning, so the new - # pane's rehydrate can't observe it as still-staged. - assert get_attachment_buffer().get(aid, ws_id=ws_id, user_id="userA") is None + assert get_attachment_buffer().get(aid, ws_id=ws_id, user_id="userA") is not None def test_create_raced_by_live_worker_keeps_attachments_staged(self, app_client, monkeypatch): """The enqueue branch (caller-supplied ws_id raced by a concurrent @@ -496,7 +492,8 @@ class TestCreateMultipart: monkeypatch.setattr(_FakeSession, "queue_message", _record_queue) - def _live_worker_send(ws, *, enqueue, run, thread_name=None): + def _live_worker_send(ws, *, enqueue, run, expected_session=None, thread_name=None): + assert expected_session is ws.session enqueue() # a worker already owns the ws — reuse path return True @@ -538,7 +535,8 @@ class TestCreateMultipart: monkeypatch.setattr(_FakeSession, "queue_message", _full_queue) - def _live_worker_send(ws, *, enqueue, run, thread_name=None): + def _live_worker_send(ws, *, enqueue, run, expected_session=None, thread_name=None): + assert expected_session is ws.session # Mirror the real send()'s reuse-path backpressure contract: # queue.Full → False, never a raise to the caller. try: @@ -776,44 +774,3 @@ class TestInitialWorkerFailureState: assert events, "init worker never emitted" assert all(e.get("state") != "error" for e in events) assert all(e.get("type") != "error" for e in events) - - -def test_retry_closure_sanitizes_error_display(monkeypatch): - """The retry (_run) closure sanitizes the exception text before on_error, so - a credential-bearing base-URL in a backend error can't cross into the - dashboard SSE. It deliberately does NOT route through ensure_error_recorded - (the reused-session stale-flag hazard — #865); the display-sanitize half of - that hygiene is fixed at the site. Driven via the capture pattern: patch the - dispatcher to hand back the run closure, then run it inline as the owner.""" - import threading - from types import SimpleNamespace - - from turnstone.core import session_worker - from turnstone.server import _interactive_dispatch_retry - - ui = _FakeUI(ws_id="ws-retry") - - def _boom(_msg): - raise RuntimeError("cannot reach https://user:pass@host:8000/v1 for model=x") - - ws = SimpleNamespace( - id="ws-retry", session=SimpleNamespace(send=_boom), ui=ui, worker_thread=None - ) - captured: dict = {} - - def _capture(_ws, *, enqueue, run, thread_name=None, **_kw): - captured["run"] = run - return True - - monkeypatch.setattr(session_worker, "send", _capture) - _interactive_dispatch_retry(ws, "retry this") - assert "run" in captured, "retry did not dispatch through session_worker.send" - # The owner guard reads ws.worker_thread is the executing thread; run the - # captured closure inline as that owner. - ws.worker_thread = threading.current_thread() - captured["run"]() - - errors = [e["message"] for e in ui.events if e.get("type") == "error"] - assert errors, "retry closure emitted no on_error" - assert all("user:pass" not in m for m in errors), errors # credential redacted - assert any("REDACTED" in m for m in errors) # the sanitizer ran, not a no-op diff --git a/tests/test_server_authz.py b/tests/test_server_authz.py index 44307b8c..7ccb1114 100644 --- a/tests/test_server_authz.py +++ b/tests/test_server_authz.py @@ -254,6 +254,9 @@ class _FakeSession: # When set, queue_message records the attempt and then raises it # (e.g. CrossUserInterjectionError for the drain's re-park arm). self.queue_raises: BaseException | None = None + # Principals for whom the fake models a retained queue item owned by + # somebody else. The real session checks this at fresh-slot admission. + self.foreign_queue_principals: set[str] = set() self.fork_calls: list[tuple[str, str, bool]] = [] # DELETE /send fall-through: ids the route asked this session to # dequeue (the fake never holds interjections, so it returns @@ -273,7 +276,9 @@ class _FakeSession: attachment_ids: Any = None, queue_msg_id: str | None = None, interjector_user_id: str = "", + turn_principal_id: str | None = None, ) -> tuple[str, str, str]: + del turn_principal_id self.queue_calls.append(text) if self.queue_raises is not None: raise self.queue_raises @@ -281,6 +286,9 @@ class _FakeSession: cleaned = text[:cap] + "..." if len(text) > cap else text return cleaned, "notice", queue_msg_id or "m1" + def has_foreign_queued_messages(self, principal_id: str) -> bool: + return principal_id in self.foreign_queue_principals + def dequeue_message(self, msg_id: str) -> bool: self.dequeues.append(msg_id) return False @@ -950,9 +958,11 @@ class TestListWorkstreamsTrustedTeamVisibility: "user_id", "project_id", "persona", + "persistence_state", } assert row["kind"] == "interactive" assert row["user_id"] == "user-shape" + assert row["persistence_state"] == "healthy" # parent_ws_id is None for top-level interactive workstreams # (only coord-spawned children carry it). assert row["parent_ws_id"] is None @@ -990,6 +1000,26 @@ class TestDashboardTrustedTeamVisibility: # reading it should break loudly, not read None forever. assert "pending_approval_detail" not in rows[0] + def test_dashboard_projects_only_sanitized_persistence_state(self, app_client): + client, mgr = app_client + created = client.post( + "/v1/api/workstreams/new", + json={"name": "needs-history-repair"}, + headers=_auth("user-a"), + ) + ws = mgr.get(created.json()["ws_id"]) + ws.session.conversation_persistence_status = lambda: { + "state": "retrying", + "attempts": 2, + "last_failure_at": "not-public", + } + + row = client.get("/v1/api/dashboard", headers=_auth("user-a")).json()["workstreams"][0] + + assert row["persistence_state"] == "retrying" + assert "attempts" not in row + assert "last_failure_at" not in row + def test_dashboard_pending_approval_details_merge_judge_verdict(self, app_client): """When _pending_approval is set on a ws's UI, /dashboard embeds one detail entry per live cycle with merged items + @@ -1223,10 +1253,12 @@ class TestPerWsSseGate: assert storage is not None _register_ws(storage, "ws-victim", "victim-user") resp = client.get( - "/v1/api/workstreams/ws-victim/events", + "/v1/api/workstreams/ws-victim/events?user_turn=1", headers=_auth("attacker-user"), ) assert resp.status_code == 404 + assert "user_turn" not in resp.text + assert "projection" not in resp.text # --------------------------------------------------------------------------- @@ -1318,6 +1350,51 @@ class TestInteractiveCancelLifted: assert ws.worker_thread is None assert ws._worker_running is False + def test_force_cancel_never_abandons_successor_that_replaces_pinned_target( + self, + app_client, + ): + """A exits and B spawns after Stop starts but before force cleanup.""" + + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None and ws.session is not None and ws.ui is not None + predecessor = threading.Thread(target=lambda: None, name="cancel-target-a") + successor = threading.Thread(target=lambda: None, name="fresh-successor-b") + with ws._lock: + ws._worker_running = True + ws.worker_thread = predecessor + ws._worker_principal_id = "alice" + + def _cancel_and_replace_target() -> None: + # Deterministically occupy the exact window between the handler's + # target snapshot and its later force-ownership decision. + with ws._lock: + assert ws.worker_thread is predecessor + ws._worker_running = False + ws.worker_thread = None + ws._worker_principal_id = "" + ws._worker_running = True + ws.worker_thread = successor + ws._worker_principal_id = "bob" + + ws.session.cancel = _cancel_and_replace_target # type: ignore[method-assign] + resp = client.post( + f"/v1/api/workstreams/{ws_id}/cancel", + json={"force": True}, + headers=_auth("user-1"), + ) + + assert resp.status_code == 200 + assert ws.worker_thread is successor + assert ws._worker_running is True + assert ws._worker_principal_id == "bob" + event_types = [event.get("type") for event in ws.ui._enqueued] + assert "cancelled" in event_types + assert "stream_end" not in event_types + assert "idle" not in ws.ui.states + def test_cancel_returns_400_when_session_missing(self, app_client): """Parity with coord: a placeholder workstream (session=None) gets a 400 ``"No session"`` rather than a silent no-op 200. @@ -1624,6 +1701,35 @@ class TestCompactCommandDispatch: assert ws.session.compacts == 0 assert ws.session.commands == [] + def test_foreign_retained_queue_refuses_fresh_send_before_worker_spawn(self, app_client): + """A persistence-retained interjection keeps its original owner. + + The foreign-owner check runs inside the atomic fresh-slot admission, + but ``session_worker.send`` reports that refusal as ``False``. The + route must preserve the typed conflict outcome instead of falling + through to its generic queue-full response, and no worker may bind or + append the later participant's turn. + """ + + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + ws.session.foreign_queue_principals.add("user-1") + + response = client.post( + f"/v1/api/workstreams/{ws_id}/send", + json={"message": "must wait for the retained owner"}, + headers=_auth("user-1"), + ) + + assert response.status_code == 409 + assert response.json()["status"] == "cross_user_interjection" + assert ws.session.sends == [] + assert ws.session.queue_calls == [] + assert ws.worker_thread is None + assert ws._worker_running is False + def test_non_compact_commands_complete_before_response(self, app_client): """Quick commands dispatch through the same worker slot (mutual exclusion vs sends / a running compaction / each other) but the @@ -2354,6 +2460,7 @@ class TestCompactCommandDispatch: msg_id = r.json()["msg_id"] queued_events = [e for e in ws.ui._enqueued if e.get("type") == "message_queued"] assert [e["msg_id"] for e in queued_events] == [msg_id] + assert [e["sender"] for e in queued_events] == ["user-1"] gate.set() wait_until( lambda: any(e.get("type") == "message_dispatched" for e in ws.ui._enqueued), diff --git a/tests/test_server_live.py b/tests/test_server_live.py index b73e0bfa..b25de665 100644 --- a/tests/test_server_live.py +++ b/tests/test_server_live.py @@ -159,6 +159,7 @@ def tmp_db(): def _make_session(client, model_id, tmp_db, **kwargs) -> tuple[ChatSession, RecordingUI]: """Create a ChatSession with RecordingUI and sensible test defaults.""" + from turnstone.core.memory import register_workstream from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider ui = RecordingUI() @@ -174,6 +175,10 @@ def _make_session(client, model_id, tmp_db, **kwargs) -> tuple[ChatSession, Reco ) defaults.update(kwargs) session = ChatSession(**defaults) + # Production creates the parent workstream before admitting any keyed + # conversation row. These direct-session tests mirror that ordering so + # the storage orphan-write fence remains exercised rather than bypassed. + register_workstream(session.ws_id, user_id=kwargs.get("user_id")) # Mock-based tests use Chat Completions format (client.chat.completions) replace_session_lane(session, provider=OpenAIChatCompletionsProvider()) session.auto_approve = True diff --git a/tests/test_session.py b/tests/test_session.py index 3b4653fe..40077def 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -118,9 +118,15 @@ def _make_session( defaults live in tests/_session_helpers.make_session — duplicating them here is exactly the drift its docstring warns about.""" kwargs.setdefault("ui", NullUI()) - return make_session( + session = make_session( client=mock_openai_client or MagicMock(), instructions=instructions, **kwargs ) + # Production creates the parent workstream before any keyed conversation + # commit. Direct-session tests mirror that orphan-write guard. + from turnstone.core.memory import register_workstream + + register_workstream(session.ws_id, user_id=kwargs.get("user_id")) + return session @contextlib.contextmanager @@ -274,6 +280,26 @@ class TestSkillCommand: session.ui = MagicMock() session.ui.on_system_turn.return_value = None session._ui_event_id = MagicMock(return_value=None) + session._history_handoff_lock = threading.RLock() + session._history_visibility_lock = threading.RLock() + session._generation_lock = threading.RLock() + # Lifecycle admission fields, seeded exactly as ``__init__`` builds + # them: the commit/publish guards read them directly, so a double + # missing them would fail the admission rather than skip it. + session._history_truncation_condition = threading.Condition(session._generation_lock) + session._history_truncation_active = False + session._soft_close_preparing = False + session._tool_structural_debt = None + session._publication_shutdown = False + session._cancel_event = threading.Event() + session._durability_cond = threading.Condition(threading.Lock()) + session._durability_next_ticket = 0 + session._durability_serving_ticket = 0 + session._pending_conversation_commits = {} + session._history_handoff_revision = 0 + session._conversation_persistence_error = None + session._conversation_persistence_failure_kind = None + session._conversation_persistence_next_retry_at = None session.set_skill = MagicMock( side_effect=lambda name: setattr(session, "_skill_name", name) ) @@ -5375,7 +5401,14 @@ class TestBatchEvaluateOutputs: try: assert guard_entered.wait(2) session.cancel() - successor_generation = session._claim_generation() + abandoned, persistence_error = session.force_abandon_generation( + target_is_current=lambda: True, + clear_target=lambda: True, + publish_abandoned=lambda: None, + ) + assert abandoned is True + assert persistence_error is None + successor_generation = session._generation # Install successor-owned state under the same provider call id. # The old guard continuation must neither pop nor persist it. @@ -5638,10 +5671,10 @@ class TestCompletedModelResultPublication: def test_successor_claim_waits_for_entire_completed_result_commit(self, tmp_db) -> None: """A successor cannot observe the main result halfway through its fold. - ``on_turn_committed`` is a useful midpoint: status and the assistant - turn already landed, while assistant token bookkeeping and persistence - are still pending. A successor driven to the generation lock there - must remain blocked until those trailing writes complete too. + ``on_turn_committed`` is a useful pre-append midpoint: the accepted + stream is finalized while the assistant row, token bookkeeping, and + persistence are still pending under the generation lock. A successor + driven to that lock must remain blocked until the whole fold completes. """ session = _make_session() session._title_generated = True @@ -5709,10 +5742,10 @@ class TestCompletedModelResultPublication: try: assert commit_midpoint.wait(2) old_generation = session._generation - # This is intentionally torn only while the old generation - # owns the transaction lock: the assistant turn is visible to - # itself, but its token/persistence tail has not run yet. - assert len(session.messages) == len(session._msg_tokens) + 1 + # Acceptance now blocks before the assistant append, so even + # the old owner observes a complete USER-only prefix here. + assert len(session.messages) == len(session._msg_tokens) + assert [turn.role.value for turn in session.messages] == ["user"] assert [call.args[1] for call in save_message.call_args_list] == ["user"] assert session.ui.on_status.call_count == 1 @@ -7419,11 +7452,40 @@ class TestMetacognitiveBuffers: # authority, especially on the native path), not the raw text. assert content.endswith("User message: hows it going?") assert "while you were working" in content - # Queue cleared by the drain. + # Queue cleared by the drain, and the advisory pop window closed at + # the seam's return (the advisory lane has no restore path). assert session._queued_messages == {} + assert session._popped_in_flight == set() # _collect_advisories itself appends nothing — the caller does. assert len(session.messages) == pre_count + def test_user_interjection_meta_retains_sender_for_ui_correlation(self, tmp_db): + """The final system event cannot settle another viewer's reused token.""" + + session = _make_session(user_id="alice") + session.queue_message( + "still working?", + queue_msg_id="q1", + interjector_user_id="alice", + client_send_id="shared-token", + ) + + specs = session._collect_advisories( + assessment=None, + func_name="bash", + is_last_in_batch=True, + ) + + assert len(specs) == 1 + source, _content, meta = specs[0] + assert source == "user_interjection" + assert meta == { + "priority": "notice", + "message": "still working?", + "sender": "alice", + "client_send_id": "shared-token", + } + def test_cross_user_interjection_rejected(self, tmp_db): """A different authenticated participant cannot interject into another user's in-flight turn: folding it in would borrow the initiator's MCP @@ -7457,6 +7519,253 @@ class TestMetacognitiveBuffers: session.queue_message("internal", interjector_user_id="", queue_msg_id="q1") assert "q1" in session._queued_messages + def test_interjection_into_empty_principal_turn_fails_closed(self, tmp_db): + """An authenticated interjector is rejected when the in-flight turn has + no bound principal (ownerless session, internal wake): admitting it + would run the words under the ambient credential context and + misattribute them to the turn's initiator.""" + from turnstone.core.session import CrossUserInterjectionError + + session = _make_session(user_id=None) # no owner => empty effective id + assert not (session._mcp_effective_user_id or "") + with pytest.raises(CrossUserInterjectionError): + session.queue_message("let me in", interjector_user_id="bob") + assert session._queued_messages == {} + + def test_interjection_into_empty_slot_principal_fails_closed(self, tmp_db): + """The immutable worker-slot principal fails closed the same way: an + empty ``turn_principal_id`` (claim taken with no principal — init + worker) rejects authenticated interjectors instead of comparing + against the mutable actor.""" + from turnstone.core.session import CrossUserInterjectionError + + session = _make_session(user_id="owner") + with pytest.raises(CrossUserInterjectionError): + session.queue_message( + "let me in", + interjector_user_id="bob", + turn_principal_id="", + ) + assert session._queued_messages == {} + + def test_pop_partitions_by_owner(self, tmp_db): + """The pop is owner-partitioned and flag-less: the acting user's and + unowned/legacy rows pop; another participant's rows are structurally + retained — never consumed under a different actor, never a raise + (rounds 3-5: the per-site mode flag was the defect factory).""" + session = _make_session(user_id="owner") + session._acting_user_id = "alice" + session.queue_message("alice's words", interjector_user_id="alice", queue_msg_id="qa") + session._acting_user_id = "bob" + session.queue_message("bob's words", interjector_user_id="bob", queue_msg_id="qb") + # Pre-owner legacy 2-tuple: unowned, pops under any actor. + session._queued_messages["legacy"] = ("legacy words", "normal") + + popped = session._pop_queued_messages() + assert sorted(popped) == ["legacy", "qb"] + assert list(session._queued_messages) == ["qa"] + + # The retained row drains for its owner. + session._acting_user_id = "alice" + assert list(session._pop_queued_messages()) == ["qa"] + assert session._queued_messages == {} + + def test_pop_with_empty_principal_drains_everything(self, tmp_db): + """CLI/internal lanes (empty effective principal) pop the whole + queue — an owned row cannot coexist with an empty actor (admission + fails closed on empty principals and the effective id is sticky), so + this arm only ever sees all-unowned queues.""" + session = _make_session(user_id=None) + assert not (session._mcp_effective_user_id or "") + session.queue_message("internal one", interjector_user_id="", queue_msg_id="q1") + session._queued_messages["legacy"] = ("legacy words", "normal") + + assert sorted(session._pop_queued_messages()) == ["legacy", "q1"] + assert session._queued_messages == {} + + def test_retraction_ledger_survives_a_nested_partitioned_pop(self, tmp_db): + """Per-id ledger discipline: an inner pop (the wake send's flush + seams) must not destroy a suppression record guarding the OUTER wake + pop's window — a wholesale clear would let the restore resurrect a + message the user cancelled mid-wake (round-5 map, edge 3a).""" + session = _make_session(user_id="owner") + session._acting_user_id = "alice" + session.queue_message("outer", interjector_user_id="alice", queue_msg_id="q-outer") + + outer = session._pop_queued_messages() + assert list(outer) == ["q-outer"] + # The pop OPENED the window: the id is in flight, which is the only + # reason the retraction below is recorded at all (round 6: misses + # for unheld ids record nothing — that unbounded growth was the + # price of unconditional recording under per-id discipline). + assert "q-outer" in session._popped_in_flight + # Retraction lands while the outer window holds the items. + assert session.dequeue_message("q-outer") is False + assert "q-outer" in session._retracted_while_popped + + # An unrelated INNER pop (empty or different ids) must not consume + # the outer window's suppression record — nor close its window. + session.queue_message("inner", interjector_user_id="alice", queue_msg_id="q-inner") + inner = session._pop_queued_messages() + assert "q-outer" in session._retracted_while_popped + assert "q-outer" in session._popped_in_flight + session._close_pop_window(inner) + assert "q-inner" not in session._popped_in_flight + assert "q-outer" in session._popped_in_flight + + # The outer restore honours the retraction, consumes its record, and + # closes the window it owned. + session._restore_queued_messages(outer) + assert "q-outer" not in session._queued_messages + assert "q-outer" not in session._retracted_while_popped + assert session._popped_in_flight == set() + + def test_dequeue_miss_for_an_unheld_id_records_nothing(self, tmp_db): + """The ledger is bounded by OPEN pop windows: a miss for an id no + window holds — never queued, or already delivered and its window + closed — must record nothing. Under per-id discipline nothing ever + prunes such an entry, so unconditional recording grew the set for + the session's lifetime (one permanent entry per retract-after- + delivery; unbounded for any authenticated writer looping DELETEs + with invented ids).""" + session = _make_session(user_id="owner") + session._acting_user_id = "alice" + + # Never queued at all (the invented-id / other-node case). + assert session.dequeue_message("never-queued") is False + assert session._retracted_while_popped == set() + + # Queued, popped, and its window CLOSED (delivered): the late + # retraction is a pure "already sent" no-op. + session.queue_message("delivered", interjector_user_id="alice", queue_msg_id="q-done") + popped = session._pop_queued_messages() + session._close_pop_window(popped) + assert session._popped_in_flight == set() + assert session.dequeue_message("q-done") is False + assert session._retracted_while_popped == set() + + def test_identity_swap_drain_discards_what_cannot_land(self, tmp_db): + """Round-4 review pin: the /new//resume queue settlement never raises. + On a gone latch everything is discarded with a notice (flushing would + refuse and crash the REPL's only escape commands); a foreign-retained + entry is discarded with a notice rather than bleeding into the next + identity.""" + # Gone latch: discard-all with notice. + ui = MagicMock() + session = _make_session(user_id="owner", ui=ui) + session._acting_user_id = "owner" + session.queue_message("stranded", interjector_user_id="owner", queue_msg_id="q1") + session._workstream_gone_ws = session._ws_id + # A stale in-flight marker must not outlive the identity swap: on + # the NEW workstream it would let a same-id miss record a bogus + # suppression. (No window can be open on this CLI-only path — the + # clear is the belt-and-braces invariant, pinned here.) + session._popped_in_flight.add("stale-window-id") + session._drain_queue_for_identity_swap() + assert session._queued_messages == {} + assert session._popped_in_flight == set() + assert any("deleted" in str(c.args[0]) for c in ui.on_info.call_args_list) + + # Foreign-retained entry on a healthy workstream: discarded, not bled. + ui2 = MagicMock() + session2 = _make_session(user_id="owner", ui=ui2) + session2._acting_user_id = "alice" + session2.queue_message("alice's words", interjector_user_id="alice", queue_msg_id="qa") + session2._acting_user_id = "bob" + session2._drain_queue_for_identity_swap() + assert session2._queued_messages == {} + assert not any( + "alice's words" in str(m.get("content")) for m in dicts_from_turns(session2.messages) + ) + assert any("another participant" in str(c.args[0]) for c in ui2.on_info.call_args_list) + + def test_identity_swap_mixed_queue_flushes_own_and_notices_foreign_only(self, tmp_db): + """Round-5 review pin (notice accuracy): on a mixed queue /new's + settlement persists the acting user's row into the CURRENT workstream + and the discard notice counts ONLY the other participant's rows.""" + ui = MagicMock() + session = _make_session(user_id="owner", ui=ui) + session._acting_user_id = "alice" + session.queue_message("alice's words", interjector_user_id="alice", queue_msg_id="qa") + session._acting_user_id = "bob" + session.queue_message("bob's words", interjector_user_id="bob", queue_msg_id="qb") + + session._drain_queue_for_identity_swap() + + assert session._queued_messages == {} + # The flush's own window closed on the success path too. + assert session._popped_in_flight == set() + flushed = [ + m + for m in dicts_from_turns(session.messages) + if m.get("role") == "user" and m.get("content") == "bob's words" + ] + assert len(flushed) == 1 + notices = [str(c.args[0]) for c in ui.on_info.call_args_list] + assert any("1 queued message(s) from another participant" in n for n in notices) + assert not any("of your queued message" in n for n in notices) + + def test_identity_swap_with_empty_queue_never_touches_the_journal(self, tmp_db): + """Round-5 review pin (total-ness): an empty queue skips the flush + preamble entirely, so a poisoned reconcile latch cannot raise out of + /new with nothing queued at all.""" + from turnstone.core.session import ConversationPersistenceError + + session = _make_session(user_id="owner") + session._conversation_persistence_failure_kind = "conflict" + session._conversation_persistence_error = ConversationPersistenceError("latched") + + session._drain_queue_for_identity_swap() # must not raise + + def test_identity_swap_degrades_to_discard_when_the_flush_raises(self, tmp_db): + """Round-5 review pin (degrade arm): a flush failure of ANY class + becomes discard-with-notice — the escape commands can never be + blocked by an unhealthy journal, and the notice never miscounts the + actor's own rows as another participant's.""" + ui = MagicMock() + session = _make_session(user_id="owner", ui=ui) + session._acting_user_id = "alice" + session.queue_message("alice's words", interjector_user_id="alice", queue_msg_id="qa") + + with patch.object( + session, "_flush_queued_messages", side_effect=RuntimeError("journal refused") + ): + session._drain_queue_for_identity_swap() # must not raise + + assert session._queued_messages == {} + notices = [str(c.args[0]) for c in ui.on_info.call_args_list] + assert any("1 of your queued message(s)" in n for n in notices) + assert not any("another participant" in n for n in notices) + + def test_failure_finalizer_retains_foreign_queue_and_records_error(self, tmp_db): + """Round-3 review: a foreign-owned queued entry (retained across its + owner's failed turn) must not abort the next actor's failure finalizer + via the ownership assert — ``_record_fatal_error`` always runs + (spinner convergence + last_error) while the entry stays queued for + its owner's next turn.""" + ui = MagicMock() + session = _make_session(user_id="owner", ui=ui) + session._acting_user_id = "alice" + session.queue_message("alice's words", interjector_user_id="alice", queue_msg_id="qa") + session._acting_user_id = "bob" + + def _boom(_gen): + raise RuntimeError("provider blew up") + + with contextlib.ExitStack() as stack: + stack.enter_context(patch.object(session, "_stream_response", side_effect=_boom)) + stack.enter_context(patch.object(session, "_full_messages", return_value=[])) + stack.enter_context(patch.object(session, "_update_token_table")) + stack.enter_context(patch.object(session, "_print_status_line")) + stack.enter_context(patch.object(session, "_emit_state")) + stack.enter_context(patch.object(session, "_visible_memory_count", return_value=0)) + stack.enter_context(patch("turnstone.core.session.save_message")) + with pytest.raises(RuntimeError, match="provider blew up"): + session.send("bob's turn") + + ui.on_error.assert_called() + assert "qa" in session._queued_messages + def test_emit_state_surfaces_acting_user_to_ui(self, tmp_db): """_emit_state pushes the acting user (turn initiator, owner fallback) onto a SessionUIBase-derived UI (the web-fanout UIs — WebUI, @@ -8235,7 +8544,14 @@ class TestApplyPostExecuteAdvisories: try: assert advisory_commit_entered.wait(2) session.cancel() - successor_generation = session._claim_generation() + abandoned, persistence_error = session.force_abandon_generation( + target_is_current=lambda: True, + clear_target=lambda: True, + publish_abandoned=lambda: None, + ) + assert abandoned is True + assert persistence_error is None + successor_generation = session._generation session._repeat_detector.clear() session._repeat_detector.record("successor-signature") @@ -8715,6 +9031,41 @@ class TestDeliverWakeNudge: sys_turns = [m for m in msgs if m.get("role") == "system"] assert {"role": "system", "_source": "idle_children", "content": "your kids"} in sys_turns + def test_foreign_retained_input_does_not_kill_the_wake(self, tmp_db): + """Round-4 review pin: a persistence-retained FOREIGN queued entry + must not make the wake raise CrossUserInterjectionError — neither at + the drain's pop nor at the wake send's mid-turn flush seams (the + spawn backstop would hot-loop while the nudge sat undelivered). + The foreign entry stays queued for its owner; the nudge drains + normally.""" + session = _make_session(user_id="owner") + session._title_generated = True + session._acting_user_id = "alice" + session.queue_message( + "alice's retained words", interjector_user_id="alice", queue_msg_id="qa" + ) + session._acting_user_id = "bob" + session._nudge_queue.enqueue("idle_children", "your kids", "any") + with ( + patch.object(session, "_stream_response", return_value=make_result("ok")), + patch.object(session, "_full_messages", return_value=[]), + patch.object(session, "_update_token_table"), + patch.object(session, "_print_status_line"), + patch.object(session, "_emit_state"), + patch.object(session, "_visible_memory_count", return_value=0), + patch("turnstone.core.session.save_message"), + ): + session.deliver_wake_nudge_from_queue() + # The nudge drained onto the synthetic turn... + sys_turns = [m for m in dicts_from_turns(session.messages) if m.get("role") == "system"] + assert {"role": "system", "_source": "idle_children", "content": "your kids"} in sys_turns + # ...and alice's entry is retained, never consumed under bob. + assert "qa" in session._queued_messages + assert not any( + "alice's retained words" in str(m.get("content")) + for m in dicts_from_turns(session.messages) + ) + def test_marks_source_tag_on_synthesized_user_msg(self, tmp_db): session = _make_session() session._title_generated = True @@ -9445,6 +9796,7 @@ class TestReminderSidechannelIsolation: ] assert copied[0].meta.extra["attachments_meta"] == [ { + "attachment_id": user_attachment_id, "kind": "text", "filename": "notes.txt", "mime_type": "text/plain", diff --git a/tests/test_session_attachments.py b/tests/test_session_attachments.py index 10a366c0..ff949767 100644 --- a/tests/test_session_attachments.py +++ b/tests/test_session_attachments.py @@ -51,10 +51,12 @@ def _make_session(mock_client, user_id: str = "u1") -> ChatSession: return s -def _run_send(session: ChatSession, text: str, attachments=None) -> None: +def _run_send( + session: ChatSession, text: str, attachments=None, send_id: str | None = None +) -> None: """Call send() but tolerate the stop-loop sentinel.""" try: - session.send(text, attachments=attachments) + session.send(text, attachments=attachments, send_id=send_id) except RuntimeError as e: if "stop after append" not in str(e): raise @@ -222,6 +224,41 @@ class TestPersistenceAndConsumption: assert att_row["refcount"] == 1 assert att_row["origin"] == "upload" + def test_admission_raise_keeps_handles_staged_for_the_retry(self, tmp_db, mock_openai_client): + """Round-3 review pin: the staged-buffer drain runs only AFTER the + journal claim succeeds, so an admission raise leaves the handles + staged and the client's retry of the same send resolves them again — + never a rejected unknown/expired attachment for a turn that never + entered history.""" + from turnstone.core.attachment_buffer import get_attachment_buffer + + s = _make_session(mock_openai_client) + buf = get_attachment_buffer() + staged = buf.stage( + ws_id=s._ws_id, + user_id=s._user_id, + filename="note.md", + mime_type="text/markdown", + kind="text", + content=b"buffered", + ) + att = Attachment(staged.attachment_id, "note.md", "text/markdown", "text", b"buffered") + + def _boom(**kwargs): + raise RuntimeError("injected admission failure") + + s._journal_conversation_row_locked = _boom # type: ignore[method-assign] + with pytest.raises(RuntimeError, match="injected admission failure"): + s.send("user text", attachments=[att], send_id="retry-me") + + # The live turn rolled back AND the handle survived for the retry. + assert s.messages == [] + assert buf.get(staged.attachment_id, ws_id=s._ws_id, user_id=s._user_id) is not None + + # The retry can claim the surviving handle (ownership transfers once). + transferred = buf.consume_all((staged.attachment_id,), ws_id=s._ws_id, user_id=s._user_id) + assert staged.attachment_id in transferred + def test_send_drains_the_upload_buffer(self, tmp_db, mock_openai_client): # Bytes staged in the per-node buffer are drained (discarded) once the # send commits them content-addressed — they don't linger as pending. @@ -239,7 +276,7 @@ class TestPersistenceAndConsumption: ) assert buf.get(staged.attachment_id, ws_id=s._ws_id, user_id=s._user_id) is not None att = Attachment(staged.attachment_id, "note.md", "text/markdown", "text", b"buffered") - _run_send(s, "user text", attachments=[att]) + _run_send(s, "user text", attachments=[att], send_id="staged-send") # Drained from the buffer post-commit. assert buf.get(staged.attachment_id, ws_id=s._ws_id, user_id=s._user_id) is None @@ -312,12 +349,19 @@ class TestProviderIntegration: meta = turn_to_dict(s.messages[-1]).get("_attachments_meta") assert meta == [ { + "attachment_id": "a1", "kind": "image", "filename": "dog.png", "mime_type": "image/png", "size_bytes": len(PNG_1x1), }, - {"kind": "text", "filename": "notes.md", "mime_type": "text/markdown", "size_bytes": 2}, + { + "attachment_id": "a2", + "kind": "text", + "filename": "notes.md", + "mime_type": "text/markdown", + "size_bytes": 2, + }, ] def test_attachments_meta_stripped_before_openai_wire(self, tmp_db, mock_openai_client): @@ -377,7 +421,7 @@ class TestQueuedAttachmentsRejected: cleaned, priority, msg_id = s.queue_message("plain text") assert cleaned == "plain text" with s._queued_lock: - assert s._queued_messages[msg_id] == ("plain text", priority) + assert s._queued_messages[msg_id] == ("plain text", priority, "") class TestTokenAccounting: diff --git a/tests/test_session_backend_error_format.py b/tests/test_session_backend_error_format.py index 46621c7d..3b9b282b 100644 --- a/tests/test_session_backend_error_format.py +++ b/tests/test_session_backend_error_format.py @@ -11,12 +11,16 @@ model. We bind the method to lightweight stubs carrying one coherent from __future__ import annotations import dataclasses +import threading from types import SimpleNamespace from typing import Any +from unittest.mock import patch import pytest +from tests._session_helpers import RecordingUI, make_session, provider_shell from turnstone.core.model_turn import ModelLane +from turnstone.core.providers import ModelCapabilities from turnstone.core.session import ChatSession @@ -295,6 +299,48 @@ def test_stream_death_with_overflow_phrasing_stays_stream_death(): assert "Context window exceeded" not in msg +def test_terminal_fallback_stream_death_reports_actual_lane(tmp_db): + """Fatal formatting consumes the fallback lane that armed the stream. + + The original exception object survives the retry wrapper, while its + side-table context contains no client, credential, principal, or query. + """ + from turnstone.core.providers import IncompleteStreamError + + ui = RecordingUI() # type: ignore[no-untyped-call] + session = make_session(model_alias="primary", ui=ui) + provider = provider_shell("fallback-provider") + fallback_lane = ModelLane( + provider=provider, + client=SimpleNamespace(base_url="https://fallback.example/v1?api_key=terminal-secret"), + model="fallback-kernel", + alias="fallback-alias", + registry_generation=19, + capabilities=ModelCapabilities(), + ) + death = IncompleteStreamError("peer closed the response") + + def _fail_on_fallback(consumer, *_args, **_kwargs): + consumer.begin_attempt(SimpleNamespace(armed=True), None, fallback_lane) + raise death + + session._MID_STREAM_RETRIES = 0 + with ( + patch.object(session, "_model_turn_with_fallback", side_effect=_fail_on_fallback), + pytest.raises(IncompleteStreamError) as raised, + ): + session._stream_response() + + assert raised.value is death + session._record_fatal_error(raised.value) + message = ui.of("error")[-1] + assert "fallback-provider" in message + assert "https://fallback.example/v1" in message + assert "model=fallback-alias (id=fallback-kernel)" in message + assert "primary" not in message + assert "terminal-secret" not in message + + # --------------------------------------------------------------------------- # Fall-through + degradation behaviour # --------------------------------------------------------------------------- @@ -375,6 +421,7 @@ def _record_fatal_stub(ui: Any, captured: dict[str, str]) -> Any: when ``self`` is a real instance of the class).""" stub = _stub() stub._ws_id = "ws-test" + stub._generation_lock = threading.RLock() stub._has_persisted_error = False stub.ui = ui stub._emit_state = lambda state, **_kwargs: captured.setdefault("state", state) diff --git a/tests/test_session_lifecycle_admission.py b/tests/test_session_lifecycle_admission.py new file mode 100644 index 00000000..0425ac11 --- /dev/null +++ b/tests/test_session_lifecycle_admission.py @@ -0,0 +1,475 @@ +"""Adversarial worker/terminal admission races for ``ChatSession``.""" + +from __future__ import annotations + +import threading +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from tests._session_helpers import make_session +from turnstone.core import session_worker +from turnstone.core.session import ConversationPersistenceError, GenerationCancelled +from turnstone.core.trajectory import turn_to_dict +from turnstone.core.workstream import Workstream + + +class _ObservedRLock: + """Reentrant lock that exposes one named thread's acquisition attempt.""" + + def __init__(self, observed_thread: str) -> None: + self._lock = threading.RLock() + self._observed_thread = observed_thread + self.acquire_attempted = threading.Event() + + def acquire(self, *args: Any, **kwargs: Any) -> bool: + if threading.current_thread().name == self._observed_thread: + self.acquire_attempted.set() + return self._lock.acquire(*args, **kwargs) + + def release(self) -> None: + self._lock.release() + + def __enter__(self) -> _ObservedRLock: + self.acquire() + return self + + def __exit__(self, *_args: Any) -> None: + self.release() + + +def _workstream(session: Any) -> Workstream: + ws = Workstream(id="ws-lifecycle-admission", name="lifecycle-admission", user_id="owner") + ws.session = session + ws.ui = session.ui + return ws + + +def test_stop_invalidates_worker_slot_before_delayed_send_entry(tmp_db: Any) -> None: + """A Stop cannot be erased by a worker that has not entered ``send`` yet.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + ws = _workstream(session) + worker_entered = threading.Event() + release_worker = threading.Event() + errors: list[BaseException] = [] + refresh = MagicMock() + + def _run() -> None: + worker_entered.set() + assert release_worker.wait(5), "test did not release delayed worker" + try: + # Mirrors the production wrapper. The bind is intentionally before + # send; the active WorkerClaim makes it defer until generation claim. + session.bind_acting_user("alice") + session.send("must remain stopped") + except BaseException as exc: + errors.append(exc) + + with patch.object(session, "_refresh_model_from_registry", refresh): + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_run, + principal_id="alice", + thread_name="delayed-stopped-worker", + ) + worker = ws.worker_thread + assert worker is not None and worker_entered.wait(5) + session.cancel() + release_worker.set() + worker.join(5) + + assert not worker.is_alive() + assert len(errors) == 1 and isinstance(errors[0], GenerationCancelled) + assert session._generation == 0 + assert session.messages == [] + refresh.assert_not_called() + + +def test_stop_during_pre_generation_registry_refresh_reaches_no_turn_admission(tmp_db: Any) -> None: + """Registry work now follows claim, so Stop remains visible when it returns.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + ws = _workstream(session) + refresh_entered = threading.Event() + release_refresh = threading.Event() + errors: list[BaseException] = [] + session.ui.on_state_change = MagicMock() # type: ignore[attr-defined,method-assign] + + def _blocked_refresh() -> None: + refresh_entered.set() + assert release_refresh.wait(5), "test did not release registry refresh" + + def _run() -> None: + try: + session.bind_acting_user("alice") + session.send("stop before user admission") + except BaseException as exc: + errors.append(exc) + + with patch.object(session, "_refresh_model_from_registry", side_effect=_blocked_refresh): + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_run, + principal_id="alice", + thread_name="registry-refresh-worker", + ) + worker = ws.worker_thread + assert worker is not None and refresh_entered.wait(5) + session.cancel() + release_refresh.set() + worker.join(5) + + assert not worker.is_alive() + assert errors == [] # send handles a cancellation after generation claim + assert session._generation == 1 + assert session.messages == [] + assert session.has_unresolved_conversation_persistence() is False + + +def test_force_successor_keeps_generation_actor_and_worker_slot_from_delayed_predecessor( + tmp_db: Any, +) -> None: + """A force-abandoned delayed thread cannot supersede its live successor.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + ws = _workstream(session) + predecessor_entered = threading.Event() + release_predecessor = threading.Event() + successor_claimed = threading.Event() + release_successor = threading.Event() + predecessor_errors: list[BaseException] = [] + successor_generations: list[int] = [] + refresh = MagicMock() + + def _predecessor() -> None: + predecessor_entered.set() + assert release_predecessor.wait(5), "test did not release predecessor" + try: + session.bind_acting_user("alice") + session.send("stale predecessor") + except BaseException as exc: + predecessor_errors.append(exc) + + def _successor() -> None: + claim = session_worker.current_worker_claim(session) + assert claim is not None + generation = session._claim_generation( + principal_id=claim.principal_id, + expected_cancel_epoch=claim.cancel_epoch, + ) + successor_generations.append(generation) + session._bind_acting_user_for_generation("bob", generation) + successor_claimed.set() + assert release_successor.wait(5), "test did not release successor" + session._generation_principals.pop(generation, None) + session._consume_cancel(generation) + + with patch.object(session, "_refresh_model_from_registry", refresh): + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_predecessor, + principal_id="alice", + thread_name="force-predecessor", + ) + predecessor_thread = ws.worker_thread + assert predecessor_thread is not None and predecessor_entered.wait(5) + + session.cancel() + # Exact force ownership transition from the HTTP cancel path. + with ws._lock: + ws.worker_thread = None + ws._worker_running = False + ws._worker_principal_id = "" + + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_successor, + principal_id="bob", + thread_name="force-successor", + ) + successor_thread = ws.worker_thread + assert successor_thread is not None and successor_claimed.wait(5) + + release_predecessor.set() + predecessor_thread.join(5) + + assert successor_thread.is_alive() + assert ws.worker_thread is successor_thread + assert ws._worker_running is True + assert ws._worker_principal_id == "bob" + assert session._generation == 1 + assert session._mcp_effective_user_id == "bob" + + release_successor.set() + successor_thread.join(5) + + assert successor_generations == [1] + assert len(predecessor_errors) == 1 + assert isinstance(predecessor_errors[0], GenerationCancelled) + assert session._generation == 1 + assert session._mcp_effective_user_id == "bob" + refresh.assert_not_called() + + +def test_refused_soft_close_invalidates_old_slot_but_allows_fresh_worker(tmp_db: Any) -> None: + """Rollback reopens admission without resurrecting a pre-close worker.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + ws = _workstream(session) + old_entered = threading.Event() + release_old = threading.Event() + old_errors: list[BaseException] = [] + fresh_generations: list[int] = [] + refresh = MagicMock() + + def _old_worker() -> None: + old_entered.set() + assert release_old.wait(5), "test did not release old worker" + try: + session.send("pre-close worker") + except BaseException as exc: + old_errors.append(exc) + + def _fresh_worker() -> None: + claim = session_worker.current_worker_claim(session) + assert claim is not None + fresh_generations.append( + session._claim_generation( + principal_id=claim.principal_id, + expected_cancel_epoch=claim.cancel_epoch, + ) + ) + + with patch.object(session, "_refresh_model_from_registry", refresh): + assert session_worker.send(ws, enqueue=lambda: None, run=_old_worker) + old_thread = ws.worker_thread + assert old_thread is not None and old_entered.wait(5) + + before_epoch = session._approval_cancel_epoch + with patch.object( + session, + "_reconcile_pending_conversation_commits", + side_effect=ConversationPersistenceError("still unavailable"), + ): + assert session.prepare_soft_close() is False + assert session._publication_shutdown is False + assert session._approval_cancel_epoch == before_epoch + 1 + + release_old.set() + old_thread.join(5) + assert len(old_errors) == 1 and isinstance(old_errors[0], GenerationCancelled) + assert session._generation == 1 # rollback bump only; old slot did not claim + + assert session_worker.send(ws, enqueue=lambda: None, run=_fresh_worker) + fresh_thread = ws.worker_thread + assert fresh_thread is not None + fresh_thread.join(5) + + assert fresh_generations == [2] + refresh.assert_not_called() + + +def _terminal_action(session: Any, kind: str) -> bool: + if kind == "soft": + return bool(session.prepare_soft_close()) + session.shutdown_publication_and_drain_durability() + return True + + +@pytest.mark.parametrize("terminal_kind", ["soft", "hard"]) +def test_terminal_admission_wins_before_direct_conversation_mutation( + tmp_db: Any, + terminal_kind: str, +) -> None: + """A direct row that loses terminal admission mutates no surface.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + prepare_entered = threading.Event() + release_prepare = threading.Event() + errors: list[BaseException] = [] + original_prepare = session._prepare_direct_conversation_mutation + + def _blocked_prepare(deferred: Any) -> None: + prepare_entered.set() + assert release_prepare.wait(5), "test did not release direct preparation" + original_prepare(deferred) + + def _append() -> None: + try: + session._append_system_turn("correction", "must not cross terminal admission") + except BaseException as exc: + errors.append(exc) + + with ( + patch.object( + session, + "_prepare_direct_conversation_mutation", + side_effect=_blocked_prepare, + ), + patch("turnstone.core.session.save_message", return_value=1) as save, + ): + mutator = threading.Thread(target=_append, daemon=True) + mutator.start() + assert prepare_entered.wait(5) + assert _terminal_action(session, terminal_kind) is True + release_prepare.set() + mutator.join(5) + + assert not mutator.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert session.messages == [] + assert session.has_unresolved_conversation_persistence() is False + save.assert_not_called() + + +@pytest.mark.parametrize("terminal_kind", ["soft", "hard"]) +def test_direct_conversation_mutation_admission_makes_terminal_wait_for_ack( + tmp_db: Any, + terminal_kind: str, +) -> None: + """Once a direct row is admitted, terminal handoff drains its ticket.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + save_entered = threading.Event() + release_save = threading.Event() + terminal_done = threading.Event() + errors: list[BaseException] = [] + terminal_results: list[bool] = [] + + def _blocked_save(*_args: Any, **_kwargs: Any) -> int: + save_entered.set() + assert release_save.wait(5), "test did not release durable ACK" + return 1 + + def _append() -> None: + try: + session._append_system_turn("correction", "admitted before terminal handoff") + except BaseException as exc: + errors.append(exc) + + def _terminate() -> None: + try: + terminal_results.append(_terminal_action(session, terminal_kind)) + finally: + terminal_done.set() + + with patch("turnstone.core.session.save_message", side_effect=_blocked_save) as save: + mutator = threading.Thread(target=_append, daemon=True) + mutator.start() + assert save_entered.wait(5) + + terminal = threading.Thread(target=_terminate, daemon=True) + terminal.start() + assert terminal_done.wait(0.1) is False + + release_save.set() + mutator.join(5) + terminal.join(5) + + assert not mutator.is_alive() and not terminal.is_alive() + assert errors == [] + assert terminal_results == [True] + assert save.call_count == 1 + assert turn_to_dict(session.messages[-1]) == { + "role": "system", + "content": "admitted before terminal handoff", + "_source": "correction", + } + assert session.has_unresolved_conversation_persistence() is False + + +def test_hard_delete_barrier_waits_for_started_ambiguous_ack_reconciliation( + tmp_db: Any, +) -> None: + """A repair that wins visibility must ACK before hard deletion can run.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + with ( + patch("turnstone.core.session.save_message", return_value=0), + pytest.raises(ConversationPersistenceError), + ): + session._append_system_turn("correction", "ambiguous predecessor") + assert session.has_unresolved_conversation_persistence() is True + # Admit this explicit repair attempt; ordinary pre-due mutations now fail + # fast without a storage call so user activity cannot bypass backoff. + session._conversation_persistence_next_retry_at = 0.0 + + visibility = _ObservedRLock("hard-terminal") + session._history_visibility_lock = visibility # type: ignore[assignment] + repair_entered = threading.Event() + release_repair = threading.Event() + terminal_done = threading.Event() + durable_deleted = threading.Event() + mutation_errors: list[BaseException] = [] + + def _blocked_repair(*_args: Any, **_kwargs: Any) -> int: + repair_entered.set() + assert release_repair.wait(5), "test did not release reconciliation" + # This is the storage safety property the hard-delete caller relies on: + # its delete step cannot follow the drain primitive until this save ACKs. + assert not durable_deleted.is_set() + return 41 + + def _append_suffix() -> None: + try: + session._append_system_turn("correction", "must lose terminal admission") + except BaseException as exc: + mutation_errors.append(exc) + + def _hard_terminal() -> None: + session.shutdown_publication_and_drain_durability() + durable_deleted.set() # stands in for the caller's immediately following delete + terminal_done.set() + + with patch("turnstone.core.session.save_message", side_effect=_blocked_repair) as save: + mutator = threading.Thread(target=_append_suffix, daemon=True, name="direct-repair") + mutator.start() + assert repair_entered.wait(5) + + terminal = threading.Thread(target=_hard_terminal, daemon=True, name="hard-terminal") + terminal.start() + assert visibility.acquire_attempted.wait(5) + assert terminal_done.is_set() is False + + release_repair.set() + mutator.join(5) + terminal.join(5) + + assert not mutator.is_alive() and not terminal.is_alive() + assert durable_deleted.is_set() + assert save.call_count == 1 + assert len(mutation_errors) == 1 and isinstance(mutation_errors[0], RuntimeError) + assert [turn_to_dict(turn)["content"] for turn in session.messages] == ["ambiguous predecessor"] + assert session.has_unresolved_conversation_persistence() is False + + +def test_hard_delete_barrier_rejects_late_ambiguous_ack_reconciliation( + tmp_db: Any, +) -> None: + """A repair that loses the terminal latch performs no storage write.""" + + session = make_session(user_id="owner", ws_id="ws-lifecycle-admission") + with ( + patch("turnstone.core.session.save_message", return_value=0), + pytest.raises(ConversationPersistenceError), + ): + session._append_system_turn("correction", "ambiguous predecessor") + assert session.has_unresolved_conversation_persistence() is True + + session.shutdown_publication_and_drain_durability() + + with ( + patch("turnstone.core.session.save_message", return_value=42) as save, + pytest.raises(RuntimeError, match="closed session"), + ): + session._append_system_turn("correction", "late suffix") + + save.assert_not_called() + assert [turn_to_dict(turn)["content"] for turn in session.messages] == ["ambiguous predecessor"] + assert session.has_unresolved_conversation_persistence() is True diff --git a/tests/test_session_manager.py b/tests/test_session_manager.py index c80beea8..2055f36a 100644 --- a/tests/test_session_manager.py +++ b/tests/test_session_manager.py @@ -35,6 +35,7 @@ from turnstone.core.workstream import ( Workstream, WorkstreamKind, WorkstreamState, + concrete_method, ) # --------------------------------------------------------------------------- @@ -2543,3 +2544,65 @@ class TestStateSubscribers: fired.clear() mgr.set_state(ws.id, WorkstreamState.IDLE) assert fired == ["first:idle", "late:idle"] + + +# --------------------------------------------------------------------------- +# concrete_method — the shared optional-hook probe +# --------------------------------------------------------------------------- + + +class TestConcreteMethod: + """Both halves of the shared guard every optional-hook caller drives. + + Manager persistence hooks, ``workstream_persistence_state``, the route + layer's cancel / history-handoff / cross-user probes, the nudge watcher's + interjection claim and the coordinator adapter's spawn gate all resolve + their hook through this one helper, so the two semantics below are pinned + once here rather than eleven times at the call sites. + """ + + def test_type_defined_method_is_concrete(self) -> None: + class Real: + def hook(self) -> str: + return "real" + + found = concrete_method(Real(), "hook") + assert found is not None + assert found() == "real" + + def test_missing_method_is_none(self) -> None: + assert concrete_method(object(), "hook") is None + + def test_magicmock_auto_vivification_is_not_a_hook(self) -> None: + """An unconfigured mock answers every attribute with a callable child. + + Treating that as a production hook is what the guard exists to stop: + it would make every optional seam look implemented under unit tests. + """ + assert concrete_method(MagicMock(), "hook") is None + + def test_instance_dict_hook_is_concrete(self) -> None: + """A deliberately installed per-instance hook still counts. + + This is the half ``session_manager`` had and the route layer's + type-only copies had lost: an explicitly assigned attribute lands in + the instance ``__dict__``, unlike an auto-vivified mock child, so it + is distinguishable and must be honored. + """ + target = MagicMock() + target.hook = lambda: "installed" + found = concrete_method(target, "hook") + assert found is not None + assert found() == "installed" + + def test_non_callable_attribute_is_not_a_hook(self) -> None: + class Shadowed: + hook = "not callable" + + assert concrete_method(Shadowed(), "hook") is None + + def test_slots_object_without_dict_is_supported(self) -> None: + class Slotted: + __slots__ = () + + assert concrete_method(Slotted(), "hook") is None diff --git a/tests/test_session_manager_lifecycle_races.py b/tests/test_session_manager_lifecycle_races.py index 44d0b014..417898b6 100644 --- a/tests/test_session_manager_lifecycle_races.py +++ b/tests/test_session_manager_lifecycle_races.py @@ -211,6 +211,130 @@ class _AcquireProbe: self.release() +class _BlockingSoftCloseSession(FakeSession): + """Expose the close-preparation window and its durability refusal.""" + + def __init__( + self, + ws_id: str, + *, + prepare_result: bool = True, + unresolved: bool = False, + ) -> None: + super().__init__(ws_id) + self.prepare_result = prepare_result + self.unresolved = unresolved + self.prepare_entered = threading.Event() + self.release_prepare = threading.Event() + + def has_unresolved_conversation_persistence(self) -> bool: + return self.unresolved + + def prepare_soft_close(self) -> bool: + self.prepare_entered.set() + assert self.release_prepare.wait(timeout=10), "test did not release close preparation" + if self.prepare_result: + self.unresolved = False + return self.prepare_result + + +def test_soft_close_fences_fresh_send_before_session_preparation() -> None: + """A send crossing the close drain is refused before worker admission.""" + mgr, adapter, storage = _make_manager() + ws = mgr.create(user_id="u1", name="closing") + session = _BlockingSoftCloseSession(ws.id) + ws.session = session # type: ignore[assignment] + close_results: list[bool] = [] + worker_ran = threading.Event() + + close_thread = threading.Thread( + target=lambda: close_results.append(mgr.close(ws.id)), + daemon=True, + ) + close_thread.start() + assert session.prepare_entered.wait(timeout=5), "close never entered session preparation" + + # ``prepare_soft_close`` is deliberately blocked. The dispatch tombstone + # must already be visible under the worker's own admission lock; otherwise + # the caller gets a false accepted response for a generation that the + # session close fence will reject after the thread starts. + assert session_worker.send(ws, enqueue=lambda: None, run=worker_ran.set) is False + assert worker_ran.is_set() is False + assert ws.worker_thread is None + + session.release_prepare.set() + close_thread.join(timeout=5) + + assert not close_thread.is_alive() + assert close_results == [True] + assert mgr.get(ws.id) is None + assert ws._closed is True + assert storage.rows[ws.id].state == "closed" + assert adapter.cleaned_up == [ws.id] + + +def test_unresolved_soft_close_retries_inside_dispatch_fence() -> None: + """Recovery is attempted while fresh worker admission stays fenced.""" + mgr, adapter, storage = _make_manager() + ws = mgr.create(user_id="u1", name="close-retry-unresolved") + session = _BlockingSoftCloseSession(ws.id, unresolved=True) + ws.session = session # type: ignore[assignment] + close_results: list[bool] = [] + + close_thread = threading.Thread( + target=lambda: close_results.append(mgr.close(ws.id)), + daemon=True, + ) + close_thread.start() + assert session.prepare_entered.wait(timeout=5), "close never retried persistence" + assert session_worker.send(ws, enqueue=lambda: None, run=lambda: None) is False + + session.release_prepare.set() + close_thread.join(timeout=5) + + assert not close_thread.is_alive() + assert close_results == [True] + assert session.unresolved is False + assert mgr.get(ws.id) is None + assert ws._closed is True + assert storage.rows[ws.id].state == "closed" + assert adapter.cleaned_up == [ws.id] + + +def test_refused_soft_close_restores_fresh_dispatch() -> None: + """A durability refusal rolls back only the workstream dispatch fence.""" + mgr, adapter, storage = _make_manager() + ws = mgr.create(user_id="u1", name="close-refused") + session = _BlockingSoftCloseSession(ws.id, prepare_result=False, unresolved=True) + ws.session = session # type: ignore[assignment] + + close_results: list[bool] = [] + close_thread = threading.Thread( + target=lambda: close_results.append(mgr.close(ws.id)), + daemon=True, + ) + close_thread.start() + assert session.prepare_entered.wait(timeout=5), "close never entered session preparation" + assert session_worker.send(ws, enqueue=lambda: None, run=lambda: None) is False + session.release_prepare.set() + close_thread.join(timeout=5) + assert not close_thread.is_alive() + assert close_results == [False] + + assert mgr.get(ws.id) is ws + assert ws._closed is False + assert ws.id not in adapter.cleaned_up + assert storage.rows[ws.id].state != "closed" + + worker_ran = threading.Event() + assert session_worker.send(ws, enqueue=lambda: None, run=worker_ran.set) is True + assert worker_ran.wait(timeout=5), "fresh worker did not run after close rollback" + worker = ws.worker_thread + assert worker is not None + worker.join(timeout=5) + assert not worker.is_alive() + + def test_close_idle_does_not_retire_an_admitted_worker() -> None: """Worker admission makes an otherwise-IDLE workstream ineligible.""" mgr, adapter, storage = _make_manager() @@ -952,3 +1076,57 @@ def test_same_id_successor_created_waits_for_predecessor_closed_publication( ("closed", ws_id), ("created", ws_id), ] + + +def test_retirement_probe_never_blocks_on_held_session_locks() -> None: + """Round-4 review pin (AB/BA deadlock): the idle-close and eviction scans + probe persistence while holding ``ws._lock``, and force-cancel's finalizer + holds the generation lock and then takes ``ws._lock`` — so the retirement + probe must never BLOCK on the session's generation/handoff locks. A held + lock reads as busy → not retirable this sweep (True), never a hang. + """ + from tests._session_helpers import make_session + from turnstone.core.session_manager import _session_persistence_blocks_retirement + + session = make_session() + # Free locks: a clean session is retirable... + assert _session_persistence_blocks_retirement(session) is False + # ...and a pending journal row blocks retirement. + with session._history_handoff_lock: + session._journal_conversation_row_locked( + commit_key="probe-key", + message={"role": "system", "content": "accepted overlay"}, + persist=lambda: 0, + event_id=None, + ) + assert _session_persistence_blocks_retirement(session) is True + + for lock_name in ("_generation_lock", "_history_handoff_lock"): + hold = threading.Event() + release = threading.Event() + lock = getattr(session, lock_name) + + def _holder( + lock: Any = lock, + hold: threading.Event = hold, + release: threading.Event = release, + ) -> None: + with lock: + hold.set() + release.wait(5) + + holder = threading.Thread(target=_holder, daemon=True) + holder.start() + assert hold.wait(2) + outcome: list[bool] = [] + prober = threading.Thread( + target=lambda out=outcome: out.append(_session_persistence_blocks_retirement(session)), + daemon=True, + ) + prober.start() + prober.join(2) + still_running = prober.is_alive() + release.set() + holder.join(2) + assert not still_running, f"probe blocked on a held {lock_name}" + assert outcome == [True], lock_name diff --git a/tests/test_session_worker.py b/tests/test_session_worker.py index ff001796..5ee9dc4d 100644 --- a/tests/test_session_worker.py +++ b/tests/test_session_worker.py @@ -14,21 +14,30 @@ module must hold: nudges queued on an IDLE workstream spawns the wake send that the IDLE fan-out (which ran on this worker's own thread) had to drop -Callers pass no-arg closures, so dispatch never touches ``ws.session``; -the exit backstop only PEEKS it defensively (``getattr`` for -``_nudge_queue``, bail on stubs) — watch-style dispatchers can still -drive a session that isn't installed on ``ws``. +Callers pass no-arg closures. For a real ChatSession, dispatch captures its +slot-time cancellation witness through a narrow optional method; session +stubs retain the historical callback-only path. The exit backstop only PEEKS +session nudge state defensively (``getattr`` for ``_nudge_queue``, bail on +stubs) — watch-style dispatchers can still drive a session that isn't +installed on ``ws``. """ from __future__ import annotations +import collections import queue import threading from typing import Any +import pytest + from tests._helpers import wait_until as _wait_until +from tests._session_helpers import make_session from turnstone.core import session_worker +from turnstone.core.idle_nudge_watcher import wake_workstream_if_pending from turnstone.core.nudge_queue import USER_DRAIN, NudgeQueue +from turnstone.core.session import ChatSession +from turnstone.core.session_routes import SessionEndpointConfig, _make_dispatch_attempt from turnstone.core.workstream import Workstream, WorkstreamState @@ -68,6 +77,30 @@ class _SendSession: self.queue_calls.append(message) +class _ClaimingSendSession(_SendSession): + """Dispatch stub exposing the production worker-claim protocol.""" + + def __init__(self) -> None: + super().__init__() + self._cancel_event = threading.Event() + + def _capture_worker_claim(self, principal_id: str = "") -> session_worker.WorkerClaim: + return session_worker.WorkerClaim( + session=self, + principal_id=principal_id, + cancel_epoch=0, + cancel_event=self._cancel_event, + cancel_event_was_set=False, + ) + + def _worker_claim_is_current(self, claim: session_worker.WorkerClaim) -> bool: + return claim.session is self and not claim.cancel_event.is_set() + + def queue_message(self, message: str, **_kwargs: Any) -> tuple[str, str, str]: + super().queue_message(message) + return message, "notice", "queued-message" + + def _make_ws(session: Any = None) -> Workstream: ws = Workstream(id="ws-aaaaaaaa", name="ws-aaaa") ws.session = session # type: ignore[assignment] @@ -84,6 +117,72 @@ def _send_message(ws: Workstream, session: _SendSession, msg: str) -> bool: ) +def _make_blocked_principal_session( + *, previous_actor: str +) -> tuple[ChatSession, threading.Event, threading.Event, list[tuple[str, str | None]]]: + """Build the smallest real ``queue_message`` surface for claim-race tests.""" + session = ChatSession.__new__(ChatSession) + session._acting_user_id = previous_actor + session._mcp_user_id = "owner" + session._queued_lock = threading.Lock() + session._queued_messages = collections.OrderedDict() + session._retracted_while_popped = set() + session._popped_in_flight = set() + # The worker-slot claim these tests are about is captured from real + # lifecycle state, so seed the fields ``__init__`` would: an open session + # with no cancel edge, no close in flight, and no owed TOOL receipts. + session._generation_lock = threading.RLock() + session._cancel_event = threading.Event() + session._approval_cancel_epoch = 0 + session._publication_shutdown = False + session._soft_close_preparing = False + session._tool_structural_debt = None + + bind_entered = threading.Event() + release_bind = threading.Event() + ran_as: list[tuple[str, str | None]] = [] + + def _delayed_bind(user_id: str) -> None: + bind_entered.set() + assert release_bind.wait(5), "test did not release acting-user bind" + session._acting_user_id = user_id + + def _record_send(message: str, **_kwargs: Any) -> None: + ran_as.append((message, session._mcp_effective_user_id)) + + session.bind_acting_user = _delayed_bind # type: ignore[method-assign] + session.send = _record_send # type: ignore[method-assign] + return session, bind_entered, release_bind, ran_as + + +def _principal_dispatch_attempt( + ws: Workstream, + session: ChatSession, + *, + message: str, + acting_user_id: str, +) -> tuple[bool, dict[str, Any]]: + """Dispatch through the production HTTP send worker/queue decision.""" + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (None, None), + tenant_check=None, + not_found_label="missing", + audit_action_prefix="workstream", + ) + attempt = _make_dispatch_attempt( + ws, + cfg, + ws.ui, + message=message, + resolved_atts=[], + ordered_taken=[], + send_id=f"send-{acting_user_id}", + acting_uid=acting_user_id, + ) + return attempt(session) + + # --------------------------------------------------------------------------- # Happy paths # --------------------------------------------------------------------------- @@ -292,6 +391,149 @@ def test_worker_finally_clears_flag_when_run_swallows() -> None: # --------------------------------------------------------------------------- +def test_worker_claim_rejects_previous_actor_before_new_actor_bind_completes() -> None: + """Queue admission follows the worker claim, never the stale session bind. + + A fresh send claims ``_worker_running`` before its worker thread can finish + ``bind_acting_user``. If the queue guard reads the prior mutable session + actor in that window, the prior actor can inject text into the new actor's + turn and have it run under the new actor's credentials. + """ + session, bind_entered, release_bind, ran_as = _make_blocked_principal_session( + previous_actor="alice" + ) + ws = _make_ws(session) + ws.ui = object() # type: ignore[assignment] + + try: + first_ok, first_queue = _principal_dispatch_attempt( + ws, + session, + message="bob starts", + acting_user_id="bob", + ) + assert first_ok is True + assert first_queue == {} + assert bind_entered.wait(5), "new worker never entered acting-user bind" + + second_ok, second_queue = _principal_dispatch_attempt( + ws, + session, + message="alice must not enter bob's turn", + acting_user_id="alice", + ) + + assert second_ok is True + assert second_queue == {"rejected": "cross_user_interjection"} + assert session._queued_messages == {} + finally: + release_bind.set() + worker = ws.worker_thread + if worker is not None: + worker.join(timeout=5) + + assert ran_as == [("bob starts", "bob")] + + +def test_worker_claim_accepts_new_actor_followup_before_session_bind_completes() -> None: + """The claimed actor may queue a follow-up during its own bind window. + + Fixing the cross-user hole by merely rejecting every pre-bind enqueue would + turn two rapid sends from the same browser into a false 409. The immutable + worker claim already knows who owns the admitted turn, so that identity is + authoritative for both the allow and deny decisions. + """ + session, bind_entered, release_bind, ran_as = _make_blocked_principal_session( + previous_actor="alice" + ) + ws = _make_ws(session) + ws.ui = object() # type: ignore[assignment] + + try: + first_ok, first_queue = _principal_dispatch_attempt( + ws, + session, + message="bob starts", + acting_user_id="bob", + ) + assert first_ok is True + assert first_queue == {} + assert bind_entered.wait(5), "new worker never entered acting-user bind" + + second_ok, second_queue = _principal_dispatch_attempt( + ws, + session, + message="bob follows up", + acting_user_id="bob", + ) + + assert second_ok is True + assert second_queue == { + "cleaned": "bob follows up", + "priority": "notice", + "msg_id": "send-bob", + } + assert list(session._queued_messages) == ["send-bob"] + finally: + release_bind.set() + worker = ws.worker_thread + if worker is not None: + worker.join(timeout=5) + + assert ran_as == [("bob starts", "bob")] + + +def test_dispatch_refuses_session_swapped_after_caller_capture() -> None: + """A closure-bound session must be the one the worker claim fences. + + The HTTP pending-send drain captures ``ws.session`` before entering the + shared dispatcher. A resume-style object swap can land in that gap. The + dispatcher must refuse both queue and spawn arms instead of capturing a + valid claim for the replacement while its callbacks still mutate the + detached predecessor. + """ + cfg = SessionEndpointConfig( + permission_gate=None, + manager_lookup=lambda _request: (None, None), + tenant_check=None, + not_found_label="missing", + audit_action_prefix="workstream", + ) + observed: list[tuple[bool, list[str], list[str]]] = [] + + for worker_running in (False, True): + captured_session = _ClaimingSendSession() + replacement_session = _ClaimingSendSession() + ws = _make_ws(captured_session) + ws._worker_running = worker_running + attempt = _make_dispatch_attempt( + ws, + cfg, + None, + message="must follow the replacement", + resolved_atts=[], + ordered_taken=[], + send_id="", + acting_uid="", + ) + + # Exact caller-capture -> dispatcher-capture crossing. + ws.session = replacement_session # type: ignore[assignment] + accepted, _outcome = attempt(captured_session) # type: ignore[arg-type] + owner = ws.worker_thread + if owner is not None: + owner.join(timeout=2) + observed.append( + ( + accepted, + list(captured_session.send_calls), + list(captured_session.queue_calls), + ) + ) + + assert observed == [(False, [], []), (False, [], [])] + + def test_abandoned_worker_does_not_clear_successor_running_flag() -> None: """A force-cancel abandons the worker (``ws.worker_thread`` is cleared / reassigned to a successor). When the abandoned thread finishes late, its @@ -325,6 +567,147 @@ def test_abandoned_worker_does_not_clear_successor_running_flag() -> None: assert ws.worker_thread is sentinel +def test_abandoned_worker_does_not_clear_successor_principal_claim() -> None: + """A stale predecessor's finally cannot erase its successor's actor. + + Force cancellation permits the successor to claim the slot before the old + thread returns. Principal cleanup therefore needs the same worker-identity + guard as ``_worker_running`` cleanup; a blind clear reopens both false-allow + and false-reject queue decisions during the successor turn. + """ + send_gate = threading.Event() + session = _SendSession(send_gate=send_gate) + ws = _make_ws(session) + + assert ( + session_worker.send( + ws, + enqueue=lambda: session.queue_message("first"), + run=lambda: session.send("first"), + principal_id="alice", + ) + is True + ) + abandoned = ws.worker_thread + assert abandoned is not None + assert ws._worker_principal_id == "alice" + + sentinel = threading.Thread(target=lambda: None, name="successor") + with ws._lock: + ws.worker_thread = sentinel + ws._worker_running = True + ws._worker_principal_id = "bob" + + send_gate.set() + abandoned.join(timeout=3) + assert not abandoned.is_alive() + assert ws.worker_thread is sentinel + assert ws._worker_running is True + assert ws._worker_principal_id == "bob" + + +def test_force_during_nonabandonable_mutation_keeps_slot_until_owner_exits() -> None: + """A destructive mutation cannot overlap a force-admitted successor.""" + + mutation_entered = threading.Event() + release_mutation = threading.Event() + queued = threading.Event() + ws = _make_ws(_SendSession()) + + def _mutation() -> None: + mutation_entered.set() + assert release_mutation.wait(5), "test did not release mutation" + + assert session_worker.send( + ws, + enqueue=queued.set, + run=_mutation, + worker_kind="command", + force_abandonable=False, + thread_name="nonabandonable-mutation", + ) + owner = ws.worker_thread + assert owner is not None and mutation_entered.wait(5) + + # Exact ownership decision the force-cancel route performs under ws._lock. + with ws._lock: + if ws._worker_force_abandonable: + ws.worker_thread = None + ws._worker_running = False + ws._worker_principal_id = "" + + assert ws.worker_thread is owner + assert ws._worker_running is True + assert ws._worker_force_abandonable is False + + # A would-be successor takes the existing-slot path; no concurrent worker + # can start while the destructive mutation remains inside its transaction. + assert session_worker.send(ws, enqueue=queued.set, run=lambda: None) + assert queued.is_set() + assert ws.worker_thread is owner + + release_mutation.set() + owner.join(5) + assert not owner.is_alive() + assert ws._worker_running is False + assert ws._worker_force_abandonable is True + + +def test_claim_capture_never_holds_workstream_lock_behind_generation_lock() -> None: + """Worker admission cannot invert generation -> UI -> workstream order.""" + + generation_lock = threading.Lock() + generation_held = threading.Event() + capture_attempted = threading.Event() + worker_ran = threading.Event() + commit_acquired_ws: list[bool] = [] + dispatch_results: list[bool] = [] + + class _ClaimSession: + def _capture_worker_claim(self, _principal_id: str = "") -> None: + capture_attempted.set() + with generation_lock: + return None + + ws = _make_ws(_ClaimSession()) + + def _generation_commit() -> None: + with generation_lock: + generation_held.set() + assert capture_attempted.wait(5), "dispatcher never attempted claim capture" + acquired = ws._lock.acquire(timeout=1) + commit_acquired_ws.append(acquired) + if acquired: + ws._lock.release() + + def _dispatch() -> None: + dispatch_results.append( + session_worker.send( + ws, + enqueue=lambda: None, + run=worker_ran.set, + thread_name="lock-order-worker", + ) + ) + + commit = threading.Thread(target=_generation_commit, daemon=True) + commit.start() + assert generation_held.wait(5) + + dispatcher = threading.Thread(target=_dispatch, daemon=True) + dispatcher.start() + commit.join(5) + dispatcher.join(5) + + assert not commit.is_alive() and not dispatcher.is_alive() + assert commit_acquired_ws == [True] + assert dispatch_results == [True] + owner = ws.worker_thread + assert owner is not None + owner.join(5) + assert worker_ran.is_set() + + def test_concurrent_send_produces_exactly_one_worker_thread() -> None: """Two simultaneous send() calls must land as exactly one worker spawn and one queued message — not two parallel workers on the @@ -539,6 +922,307 @@ class TestWorkerExitWakeBackstop: assert ws._worker_running is True +class TestWorkerExitInterjectionBackstop: + """The ownership-clear seam cannot strand a just-accepted interjection.""" + + @staticmethod + def _session() -> ChatSession: + session = make_session(user_id="alice") + session._acting_user_id = "alice" + return session + + def test_enqueue_after_last_flush_hands_off_same_principal_and_correlation( + self, + ) -> None: + """A reuse enqueue at run's tail is delivered by a successor wake.""" + session = self._session() + ws = _make_ws(session) + delivered = threading.Event() + calls: list[tuple[str, tuple[str, ...], str]] = [] + + def _record_send(message: str, *, client_send_ids: tuple[str, ...] = ()) -> None: + claim = session_worker.current_worker_claim(session) + assert claim is not None + calls.append((message, client_send_ids, claim.principal_id)) + delivered.set() + + session.send = _record_send # type: ignore[method-assign] + + def _outgoing_run() -> None: + # This nested dispatch is the deterministic equivalent of another + # request winning ws._lock after ChatSession's final queue flush + # but before this runner enters its ownership-clear finally. + assert session_worker.send( + ws, + enqueue=lambda: session.queue_message( + "late follow-up", + queue_msg_id="late-message", + interjector_user_id="alice", + turn_principal_id="alice", + client_send_id="client-late", + ), + run=lambda: pytest.fail("reuse dispatch spawned a second worker"), + expected_session=session, + principal_id="alice", + ) + + assert len(session._nudge_queue) == 0 + assert session_worker.send( + ws, + enqueue=lambda: None, + run=_outgoing_run, + expected_session=session, + principal_id="alice", + ) + assert delivered.wait(5), "worker-exit handoff did not deliver the queued row" + _wait_until(lambda: ws._worker_running is False) + + assert calls == [("late follow-up", ("client-late",), "alice")] + assert session._queued_messages == {} + assert len(session._nudge_queue) == 0 + + def test_non_exit_gate_remains_nudge_only(self) -> None: + session = self._session() + ws = _make_ws(session) + session.queue_message( + "wait for owner clear", + queue_msg_id="nudge-only-control", + interjector_user_id="alice", + turn_principal_id="alice", + ) + + assert wake_workstream_if_pending(ws, trigger="idle-transition") is False + assert list(session._queued_messages) == ["nudge-only-control"] + assert ws._worker_running is False + + def test_retraction_before_wake_pop_converges_without_send(self) -> None: + session = self._session() + ws = _make_ws(session) + session.queue_message( + "retract me", + queue_msg_id="retracted-message", + interjector_user_id="alice", + turn_principal_id="alice", + ) + pop_entered = threading.Event() + release_pop = threading.Event() + real_pop = session._pop_queued_messages + send_calls: list[str] = [] + + def _blocked_pop(**kwargs: Any) -> Any: + # kwargs passthrough so signature growth never narrows the shim. + pop_entered.set() + assert release_pop.wait(5), "test did not release queue pop" + return real_pop(**kwargs) + + session._pop_queued_messages = _blocked_pop # type: ignore[method-assign] + session.send = lambda message, **_kwargs: send_calls.append(message) # type: ignore[method-assign] + + assert wake_workstream_if_pending( + ws, + trigger="test-worker-exit", + include_interjections=True, + ) + assert pop_entered.wait(5), "wake worker never reached the queue pop" + assert session.dequeue_message("retracted-message") is True + release_pop.set() + _wait_until(lambda: ws._worker_running is False) + + assert send_calls == [] + assert session._queued_messages == {} + assert ( + wake_workstream_if_pending( + ws, + trigger="test-worker-exit", + include_interjections=True, + ) + is False + ) + + def test_empty_interjection_is_discarded_without_a_wake_loop(self) -> None: + session = self._session() + ws = _make_ws(session) + cleaned, _priority, _msg_id = session.queue_message( + "!!!", + queue_msg_id="empty-message", + interjector_user_id="alice", + turn_principal_id="alice", + ) + send_calls: list[str] = [] + session.send = lambda message, **_kwargs: send_calls.append(message) # type: ignore[method-assign] + + assert cleaned == "" + assert wake_workstream_if_pending( + ws, + trigger="test-worker-exit", + include_interjections=True, + ) + _wait_until(lambda: ws._worker_running is False) + + assert send_calls == [] + assert session._queued_messages == {} + + def test_restored_failed_wake_is_retained_without_retry_loop(self) -> None: + session = self._session() + ws = _make_ws(session) + session.queue_message( + "keep after failure", + queue_msg_id="restored-message", + interjector_user_id="alice", + turn_principal_id="alice", + client_send_id="client-restored", + ) + attempts: list[str] = [] + + def _fail_before_append(message: str, **_kwargs: Any) -> None: + attempts.append(message) + raise RuntimeError("injected preamble failure") + + session.send = _fail_before_append # type: ignore[method-assign] + + assert wake_workstream_if_pending( + ws, + trigger="test-worker-exit", + include_interjections=True, + ) + wake_worker = ws.worker_thread + assert wake_worker is not None + _wait_until(lambda: ws._worker_running is False) + + assert attempts == ["keep after failure"] + assert list(session._queued_messages) == ["restored-message"] + # The failed queue-only wake's own exit carried the exact snapshot + # token and suppressed a retry. No successor worker was spawned. + assert ws.worker_thread is wake_worker + assert attempts == ["keep after failure"] + + def test_deferred_wake_token_does_not_suppress_competing_worker_exit(self) -> None: + """Only a spawned wake owns the snapshot exclusion token. + + A same-principal worker can claim the slot after the queue predicate + but before the wake dispatch. The wake then takes the reuse/no-op arm; + if that pre-spawn snapshot were sticky, the competing worker's exit + could not recover the older queued row after failing before its drain. + """ + session = self._session() + ws = _make_ws(session) + session.queue_message( + "older stranded row", + queue_msg_id="older-row", + interjector_user_id="alice", + turn_principal_id="alice", + client_send_id="client-older", + ) + competing_started = threading.Event() + release_competing = threading.Event() + delivered = threading.Event() + calls: list[tuple[str, tuple[str, ...], str]] = [] + + def _record_send(message: str, *, client_send_ids: tuple[str, ...] = ()) -> None: + claim = session_worker.current_worker_claim(session) + assert claim is not None + calls.append((message, client_send_ids, claim.principal_id)) + delivered.set() + + session.send = _record_send # type: ignore[method-assign] + real_claim = session.claim_pending_interjection_wake + injected_competitor = False + + def _claim_then_compete(*, exclude_signature: object | None = None) -> Any: + nonlocal injected_competitor + signature = real_claim(exclude_signature=exclude_signature) + if signature is not None and not injected_competitor: + injected_competitor = True + + def _competing_run() -> None: + competing_started.set() + assert release_competing.wait(5), "test did not release competing worker" + # Return before touching the retained queue: this models a + # fresh worker failing in its pre-drain setup. + + assert session_worker.send( + ws, + enqueue=lambda: pytest.fail("competitor unexpectedly reused a worker"), + run=_competing_run, + expected_session=session, + principal_id="alice", + ) + assert competing_started.wait(5), "competing worker did not claim the slot" + return signature + + session.claim_pending_interjection_wake = _claim_then_compete # type: ignore[method-assign] + + # The queue-only wake loses the slot race and takes its no-op enqueue + # arm. It must not leave suppression behind for the real owner. + assert wake_workstream_if_pending( + ws, + trigger="test-worker-exit", + include_interjections=True, + ) + assert calls == [] + assert list(session._queued_messages) == ["older-row"] + + release_competing.set() + assert delivered.wait(5), "competing worker exit did not retry the older row" + _wait_until(lambda: ws._worker_running is False) + + assert calls == [("older stranded row", ("client-older",), "alice")] + assert session._queued_messages == {} + + @pytest.mark.parametrize( + "blocker", + ["budget", "abandoned", "unresolved", "foreign_principal", "gone"], + ) + def test_queue_only_wake_refuses_unattended_retry_blockers(self, blocker: str) -> None: + session = self._session() + ws = _make_ws(session) + owner = "bob" if blocker == "foreign_principal" else "alice" + session.queue_message( + "wait for an explicit seam", + queue_msg_id=f"blocked-{blocker}", + interjector_user_id=owner, + turn_principal_id=owner, + ) + if blocker == "budget": + session._budget_exhausted = True + elif blocker == "abandoned": + session._generation_abandoned = True + elif blocker == "unresolved": + session.has_unresolved_conversation_persistence = lambda: True # type: ignore[method-assign] + elif blocker == "gone": + # The terminal hard-delete latch: the gone discovery clears the + # pending-commit journal, so the unresolved-persistence blocker + # reads False exactly when no turn can ever land — the latch + # needs its own refusal arm or the wake spawns doomed. + session._workstream_gone_ws = session._ws_id + + assert ( + wake_workstream_if_pending( + ws, + trigger="test-worker-exit", + include_interjections=True, + ) + is False + ) + assert list(session._queued_messages) == [f"blocked-{blocker}"] + assert ws._worker_running is False + + def test_interjection_claim_itself_refuses_a_gone_workstream(self) -> None: + """The claim gate carries its own gone arm — it must refuse for ANY + caller, not only behind the watcher's spawn gate (a future caller + that consults the claim directly gets the same fail-closed answer).""" + session = self._session() + session.queue_message( + "never wake for this", + queue_msg_id="gone-q", + interjector_user_id="alice", + turn_principal_id="alice", + ) + session._workstream_gone_ws = session._ws_id + assert session.claim_pending_interjection_wake() is None + assert list(session._queued_messages) == ["gone-q"] + + def test_does_not_deadlock_when_run_briefly_grabs_ws_lock() -> None: """Sanity check: ``run`` is invoked OUTSIDE ``ws._lock``. A worker body that briefly takes the lock (e.g. to update worker state) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index ed24d232..6c4b393c 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -879,14 +879,48 @@ class TestWorkstreamConfig: # ── Prune workstreams ───────────────────────────────────────────────── +def _backdate_updated(ws_id: str) -> None: + """Age a row past the orphan grace (and any retention cutoff).""" + engine = get_storage()._engine # noqa: SLF001 + with engine.connect() as conn: + conn.execute( + sa.text("UPDATE workstreams SET updated = '2020-01-01' WHERE ws_id = :ws"), + {"ws": ws_id}, + ) + conn.commit() + + class TestPruneWorkstreams: def test_orphan_removed(self, tmp_db): - """Workstream registered with no messages should be pruned.""" + """An AGED empty workstream is pruned as an orphan (round-3 review: + eligibility now requires outliving the grace — see the fresh/named + twins below for the guards).""" register_workstream("orphan") + _backdate_updated("orphan") orphans, stale = prune_workstreams() assert orphans == 1 assert list_workstreams_with_history() == [] + def test_fresh_empty_workstream_survives_the_grace(self, tmp_db): + """A just-registered empty workstream is a user mid-first-turn (its + rows may still be journal-held on a serving node another node's prune + cannot see) — never housekeeping debris. Round-3 review pin.""" + register_workstream("fresh-empty") + orphans, stale = prune_workstreams() + assert (orphans, stale) == (0, 0) + assert get_storage().get_workstream("fresh-empty") is not None + + def test_named_empty_workstream_never_pruned(self, tmp_db): + """An aliased workstream is explicit user intent: excluded from the + orphan category regardless of age, mirroring the stale category's + alias exclusion. Round-3 review pin.""" + register_workstream("named-empty") + set_workstream_alias("named-empty", "keep-me") + _backdate_updated("named-empty") + orphans, stale = prune_workstreams(retention_days=30) + assert (orphans, stale) == (0, 0) + assert get_storage().get_workstream("named-empty") is not None + def test_workstream_with_messages_kept(self, tmp_db): """Workstream with messages should not be pruned.""" register_workstream("active") @@ -936,6 +970,7 @@ class TestPruneWorkstreams: def test_prune_removes_workstream_config(self, tmp_db): """Pruning orphan/stale workstreams should also remove their config rows.""" register_workstream("orphan_cfg") + _backdate_updated("orphan_cfg") save_workstream_config("orphan_cfg", {"temperature": "0.5"}) register_workstream("stale_cfg") diff --git a/tests/test_shell_js.py b/tests/test_shell_js.py index 20cd0c4b..98e7a234 100644 --- a/tests/test_shell_js.py +++ b/tests/test_shell_js.py @@ -19,6 +19,8 @@ from pathlib import Path import pytest +from tests._js_harness_helpers import strip_js_comments + _ROOT = Path(__file__).resolve().parent.parent _SHARED = _ROOT / "turnstone/shared_static" _SHELL_JS = _SHARED / "shell.js" @@ -53,6 +55,7 @@ _ESM_BUNDLES = [ _SHARED / "redact_credentials.js", _SHARED / "mcp_error.js", _SHARED / "copy_actions.js", + _SHARED / "tool_projection.js", ] # Sink scan: everything except renderer.js — the one sanctioned HTML-string @@ -76,6 +79,7 @@ _ESM_NO_VAR_BUNDLES = [ _SHARED / "redact_credentials.js", _SHARED / "mcp_error.js", _SHARED / "copy_actions.js", + _SHARED / "tool_projection.js", ] # The same unsafe DOM-write / dynamic-code sink set that ``test_app_js.py`` @@ -646,6 +650,19 @@ def test_step7_tab_menu_wired_per_kind() -> None: ) +def test_node_proxied_close_409_uses_plain_retry_copy() -> None: + shell = _SHELL_JS.read_text(encoding="utf-8") + start = shell.index('typeof window.closeWorkstream === "function"') + end = shell.index("return convTabMenu", start) + close = shell[start:end] + + assert "r.status === 409" in close + assert ( + "Conversation history is still being saved. Try ending the session again shortly." in close + ) + assert "pm.close(pane.id)" in close, "successful and already-closed cases still drop the tab" + + def test_coordinator_tab_menu_enables_title_verbs() -> None: """Coordinators carry LLM/auto titles like interactive workstreams, so their tab dropdown must surface Refresh/Edit title — convTabMenu's @@ -1136,54 +1153,14 @@ def test_popup_menu_shared_helper() -> None: ) -def _strip_js_comments_local(src: str) -> str: - """Copy-local of ``tests/test_app_js.py``'s helper (house convention: - these guard files stay import-independent of each other). - - Needed because a bare ``"brand-home" in body`` substring test is a FALSE - guard -- that string also appears in prose comments, so it would stay - green after the class it names was renamed away. - """ - out: list[str] = [] - n = len(src) - i = 0 - in_str: str | None = None - while i < n: - ch = src[i] - if in_str: - out.append(ch) - if ch == "\\" and i + 1 < n: - out.append(src[i + 1]) - i += 2 - continue - if ch == in_str: - in_str = None - i += 1 - continue - # Line comment: replace with spaces up to newline (preserve - # length so downstream offset math still works). - if ch == "/" and i + 1 < n and src[i + 1] == "/": - j = src.find("\n", i) - if j == -1: - j = n - out.append(" " * (j - i)) - i = j - continue - # Block comment: replace with spaces up to closing */. - if ch == "/" and i + 1 < n and src[i + 1] == "*": - j = src.find("*/", i + 2) - if j == -1: - out.append(" " * (n - i)) - i = n - continue - out.append(" " * (j + 2 - i)) - i = j + 2 - continue - if ch in ('"', "'", "`"): - in_str = ch - out.append(ch) - i += 1 - return "".join(out) +# Comment stripping comes from the shared string-aware helper in +# tests/_js_harness_helpers (imported at module top) — the old +# copy-local's "import-independent" house convention is defunct: several +# harness suites already import the shared module, and per-suite copies +# are how the strippers diverged into two semantics in the first place. +# A bare ``"brand-home" in body`` substring test is a FALSE guard — the +# string also appears in prose comments — so a stripped body is still +# required for those pins. def test_proxy_shim_selectors_still_exist_in_shell_js() -> None: @@ -1216,7 +1193,7 @@ def test_proxy_shim_selectors_still_exist_in_shell_js() -> None: "update this guard to match the new one -- do not delete it." ) - body = _strip_js_comments_local(_SHELL_JS.read_text(encoding="utf-8")) + body = strip_js_comments(_SHELL_JS.read_text(encoding="utf-8")) # Names alone are NOT enough. The shim's selector is a DESCENDANT # selector, so shell.js emitting all three classes while re-parenting diff --git a/tests/test_skills.py b/tests/test_skills.py index 8c891293..1811f4a6 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -97,6 +97,8 @@ class NullUI: def _make_session(**kwargs): + from turnstone.core.memory import register_workstream + defaults = dict( client=MagicMock(), model="test-model", @@ -107,7 +109,12 @@ def _make_session(**kwargs): tool_timeout=30, ) defaults.update(kwargs) - return ChatSession(**defaults) + session = ChatSession(**defaults) + # Mirror production's parent-before-conversation ordering. Skill slash + # commands persist keyed SYSTEM rows and must retain the orphan-write + # protection exercised by the storage layer. + register_workstream(session.ws_id, user_id=kwargs.get("user_id")) + return session def _sys_content(session: ChatSession) -> str: diff --git a/tests/test_sse_cursor_resume.py b/tests/test_sse_cursor_resume.py index 5a9db258..aa64cc65 100644 --- a/tests/test_sse_cursor_resume.py +++ b/tests/test_sse_cursor_resume.py @@ -21,18 +21,16 @@ from __future__ import annotations import collections import os import tempfile -from typing import TYPE_CHECKING, Any +from typing import Any os.environ.setdefault("TURNSTONE_JWT_SECRET", "x" * 32) from tests._session_helpers import make_session from turnstone.core.session_routes import _resume_cursor_and_trim from turnstone.core.session_ui_base import SessionUIBase +from turnstone.core.storage import get_storage from turnstone.core.storage._sqlite import SQLiteBackend -if TYPE_CHECKING: - import pytest - class _ConcreteUI(SessionUIBase): pass @@ -216,7 +214,7 @@ def test_on_system_turn_returns_buffered_event_id() -> None: def test_append_system_turn_stamps_row_with_its_sse_event_id( - monkeypatch: pytest.MonkeyPatch, + tmp_db: str, ) -> None: """Regression: a system turn's persisted row carries the SAME ``event_id`` as its live ``on_system_turn`` event. Stamping the row with the pre-emit @@ -225,19 +223,22 @@ def test_append_system_turn_stamps_row_with_its_sse_event_id( and the operator bubble rendered twice (the metacognition-nudge double).""" session = make_session() ui = session.ui # NullUI is a real SessionUIBase → increments _event_id - captured: dict[str, Any] = {} - monkeypatch.setattr( - "turnstone.core.session.save_message", - lambda *a, **k: captured.update(event_id=k.get("event_id")), + storage = get_storage() + storage.register_workstream( + session.ws_id, + user_id=session._user_id, + kind=session._kind, + parent_ws_id=session._parent_ws_id, ) ui._enqueue({"type": "content"}) # advance past the prior turn session._append_system_turn("start", "ground yourself") - assert captured["event_id"] == ui._event_buffer[-1][0] + row = storage.load_messages(session.ws_id, repair=False)[-1] + assert row["_event_id"] == ui._event_buffer[-1][0] assert ui._event_buffer[-1][1]["type"] == "system_turn" def test_system_turn_bool_hook_return_falls_back_to_counter( - monkeypatch: pytest.MonkeyPatch, + tmp_db: str, ) -> None: """A duck-typed ``on_system_turn`` returning ``True`` (bool ⊂ int) must never stamp a boolean into the persisted row: PostgreSQL fails the @@ -246,15 +247,18 @@ def test_system_turn_bool_hook_return_falls_back_to_counter( bools at the shared chokepoint and the stamp falls back to the ring-buffer counter (same guard class as ``parse_checkpoint_watermark``).""" session = make_session() - captured: dict[str, Any] = {} - monkeypatch.setattr( - "turnstone.core.session.save_message", - lambda *a, **k: captured.update(event_id=k.get("event_id")), + storage = get_storage() + storage.register_workstream( + session.ws_id, + user_id=session._user_id, + kind=session._kind, + parent_ws_id=session._parent_ws_id, ) session.ui.on_system_turn = lambda *_a, **_k: True session._append_system_turn("start", "ground yourself") - assert not isinstance(captured["event_id"], bool) - assert captured["event_id"] == session._ui_event_id() + row = storage.load_messages(session.ws_id, repair=False)[-1] + assert not isinstance(row["_event_id"], bool) + assert row["_event_id"] == session._ui_event_id() # --------------------------------------------------------------------------- diff --git a/tests/test_sse_reconnect_replay.py b/tests/test_sse_reconnect_replay.py index a0cc22f8..d428984b 100644 --- a/tests/test_sse_reconnect_replay.py +++ b/tests/test_sse_reconnect_replay.py @@ -20,6 +20,7 @@ The browser-side guard for the ``onerror`` close pattern lives in from __future__ import annotations import asyncio +import json import queue import threading from types import SimpleNamespace as SimpleNS @@ -189,14 +190,32 @@ def test_replay_empty_buffer_cursor_at_counter_is_lossless_replay_ok() -> None: assert lost == 0 -def test_replay_empty_buffer_negative_cursor_cold_start_stays_replay_ok() -> None: - """A malformed negative cursor (``?last_event_id=-1`` parses as an - int) on a brand-new ws must not manufacture a truncated envelope — - the ``snap_seq > 0`` guard keeps cold start on ``replay_ok``.""" +def test_replay_empty_buffer_negative_cursor_reports_truncated() -> None: + """A negative cursor was never issued by the per-workstream stream. + + Even on a cold workstream it must fail closed to the authoritative + recovery floor, not claim that an empty ring covered the cursor. + """ ui = _make_ui() lq, replay, status, lost, earliest, _ = ui.register_listener_with_replay(-1) - assert status == "replay_ok" + assert status == "truncated" assert replay == [] + assert lost == 1 + assert earliest == 1 + + +def test_replay_future_cursor_reports_truncated() -> None: + """A cursor beyond the captured high-water mark cannot be server-issued.""" + ui = _make_ui() + ui._enqueue({"type": "tool_started", "name": "only-event"}) + + _, replay, status, lost, earliest, snapshot = ui.register_listener_with_replay(99) + + assert status == "truncated" + assert replay == [] + assert lost == 0 # unknown/corrupt future gap: conservative lower bound + assert earliest == 1 + assert snapshot["seq"] == 1 def test_can_replay_from_stays_false_on_empty_ring_despite_seeded_counter() -> None: @@ -540,7 +559,13 @@ def test_snap_seq_high_water_mark_holds_under_writer_race() -> None: # --------------------------------------------------------------------------- -def _wire_events_handler(ui: _ConcreteUI, *, state: str = "idle") -> Any: +def _wire_events_handler( + ui: _ConcreteUI, + *, + state: str = "idle", + session: Any = None, + events_replay: Any = None, +) -> Any: """Build a minimal ``make_events_handler`` closure that returns yields suitable for the EventSourceResponse generator. @@ -551,7 +576,7 @@ def _wire_events_handler(ui: _ConcreteUI, *, state: str = "idle") -> Any: exercise the error-state branch (the persisted ``last_error`` surface) without a real session. """ - ws = SimpleNS(id=ui.ws_id, ui=ui, state=SimpleNS(value=state)) + ws = SimpleNS(id=ui.ws_id, ui=ui, state=SimpleNS(value=state), session=session) mgr = MagicMock() mgr.get.return_value = ws @@ -561,7 +586,7 @@ def _wire_events_handler(ui: _ConcreteUI, *, state: str = "idle") -> Any: tenant_check=None, not_found_label="Workstream not found", audit_action_prefix="workstream", - events_replay=None, + events_replay=events_replay, ) return make_events_handler(cfg) @@ -573,6 +598,8 @@ def _drain_handler_yields( query: dict[str, str] | None = None, max_yields: int = 10, state: str = "idle", + session: Any = None, + events_replay: Any = None, ) -> tuple[list[Any], str]: """Synchronous helper: spin up the handler, drain up to N yields, return ``(raw_yields, decoded_blob)``. Uses ``asyncio.run`` so @@ -583,7 +610,12 @@ def _drain_handler_yields( returned for shape-level assertions (e.g. the first-yield ``retry`` check). """ - handler = _wire_events_handler(ui, state=state) + handler = _wire_events_handler( + ui, + state=state, + session=session, + events_replay=events_replay, + ) req = _fake_request(headers=headers, query=query, path_params={"ws_id": ui.ws_id}) async def _run() -> list[Any]: @@ -691,15 +723,14 @@ def test_handler_truncated_emits_envelope_then_snapshot(monkeypatch: Any) -> Non ) -def test_handler_fresh_path_skips_replay_truncated() -> None: - """No ``Last-Event-ID`` → fresh-connect behaviour (today's path - unchanged: state_change + in_progress_snapshot + live). No - replay_truncated envelope should ever appear on a fresh - connect.""" +def test_handler_tokenless_fresh_path_forces_old_client_history_repair() -> None: + """A pre-handoff browser repairs in place without a reconnect loop.""" ui = _make_ui() ui.on_content_token("hello ") _, blob = _drain_handler_yields(ui, max_yields=5) + assert '"type": "clear_ui"' in blob + assert "tokenless_history_bootstrap" in blob assert "replay_truncated" not in blob # Fresh connect emits the snapshot. assert "in_progress_snapshot" in blob @@ -716,7 +747,53 @@ def test_handler_malformed_last_event_id_falls_back_to_fresh() -> None: headers={"Last-Event-ID": "abc-not-an-int"}, max_yields=3, ) + assert "tokenless_history_bootstrap" in blob + assert '"type": "clear_ui"' in blob + + +def test_handler_cursor_zero_fresh_bootstrap_does_not_repair_loop() -> None: + """An explicit cursor 0 is numeric proof, never tokenless bootstrap.""" + + ui = _make_ui() + _, blob = _drain_handler_yields( + ui, + query={"last_event_id": "0"}, + max_yields=4, + ) + + assert "tokenless_history_bootstrap" not in blob assert "replay_truncated" not in blob + assert "state_change" in blob + + +def test_handler_negative_last_event_id_emits_truncated_recovery() -> None: + """A parsed-but-invalid negative cursor must not take replay_ok-empty.""" + ui = _make_ui() + + _, blob = _drain_handler_yields( + ui, + query={"last_event_id": "-1"}, + max_yields=4, + ) + + assert "replay_truncated" in blob + assert '"earliest_available_id": 1' in blob + + +def test_handler_future_last_event_id_emits_truncated_recovery() -> None: + """A cursor beyond the stream high-water mark forces a resync floor.""" + ui = _make_ui() + ui._enqueue({"type": "tool_started", "name": "only-event"}) + + _, blob = _drain_handler_yields( + ui, + headers={"Last-Event-ID": "99"}, + max_yields=5, + ) + + assert "replay_truncated" in blob + assert '"lost_count": 0' in blob + assert "only-event" not in blob def test_handler_query_param_fallback_is_honoured() -> None: @@ -733,6 +810,314 @@ def test_handler_query_param_fallback_is_honoured() -> None: assert "id: 1" in blob +def test_user_turn_capability_projects_canonical_or_pre_row_repair_cursor() -> None: + """Typed listeners get the row; incapable listeners retain repair intent.""" + + ui = _make_ui() + event_id = ui.on_user_turn( + "shared prompt", + attachments=[], + sender="alice", + source=None, + client_send_ids=["send-1"], + ) + assert event_id == 1 + + capable_yields, capable_blob = _drain_handler_yields( + ui, + query={"last_event_id": "0", "user_turn": "1"}, + max_yields=3, + ) + capable_payloads = [ + json.loads(item["data"]) + for item in capable_yields + if isinstance(item, dict) and "data" in item + ] + user_turn = next(item for item in capable_payloads if item.get("type") == "user_turn") + assert user_turn == { + "type": "user_turn", + "content": "shared prompt", + "client_send_ids": ["send-1"], + "sender": "alice", + "ws_id": ui.ws_id, + "_event_id": 1, + } + user_wire = next( + item + for item in capable_yields + if isinstance(item, dict) and '"type": "user_turn"' in item.get("data", "") + ) + assert user_wire["id"] == "1" + assert "user_turn_projection_unsupported" not in capable_blob + + # A later frame may advance the live cursor, but replay_truncated records + # the first frame's explicit pre-row cursor (0). A failed history repair + # reconnects from that frozen cursor and must receive the same repair + # projection again rather than skip canonical row 1. + ui._enqueue({"type": "content", "text": "later assistant bytes"}) + incapable_yields, incapable_blob = _drain_handler_yields( + ui, + query={"last_event_id": "0"}, + max_yields=4, + ) + repair_wire = next( + item + for item in incapable_yields + if isinstance(item, dict) and "user_turn_projection_unsupported" in item.get("data", "") + ) + assert repair_wire["id"] == "0" + repair_payload = json.loads(repair_wire["data"]) + assert repair_payload == { + "type": "replay_truncated", + "ws_id": ui.ws_id, + "reason": "user_turn_projection_unsupported", + } + assert "shared prompt" not in incapable_blob + assert "id: 2" in incapable_blob + + retry_yields, _ = _drain_handler_yields( + ui, + query={"last_event_id": "0"}, + max_yields=3, + ) + retry_repair = next( + item + for item in retry_yields + if isinstance(item, dict) and "user_turn_projection_unsupported" in item.get("data", "") + ) + assert retry_repair["id"] == "0" + + +def test_tool_turn_capability_projects_canonical_or_redacted_pre_row_repair() -> None: + """Accepted TOOL data reaches capable panes and never leaks to legacy ones.""" + + ui = _make_ui() + preview = {"attachment_id": "preview-secret", "kind": "html"} + event_id = ui.on_tool_turn_accepted( + "call-capability", + "secret_tool_name", + "secret final output", + is_error=True, + preview=preview, + effect_status="unknown", + ) + assert event_id == 1 + + capable_yields, capable_blob = _drain_handler_yields( + ui, + query={"last_event_id": "0", "tool_turn": "1"}, + max_yields=3, + ) + accepted_wire = next( + item + for item in capable_yields + if isinstance(item, dict) and '"accepted": true' in item.get("data", "") + ) + assert accepted_wire["id"] == "1" + assert json.loads(accepted_wire["data"]) == { + "type": "tool_result", + "accepted": True, + "call_id": "call-capability", + "name": "secret_tool_name", + "output": "secret final output", + "is_error": True, + "preview": preview, + "effect_status": "unknown", + "ws_id": ui.ws_id, + "_event_id": 1, + } + assert "tool_turn_projection_unsupported" not in capable_blob + + # Advance the ring after the accepted row. A legacy reconnect is anchored + # at N-1 and receives the same repair again if its REST heal fails. + ui._enqueue({"type": "content", "text": "later assistant bytes"}) + incapable_yields, incapable_blob = _drain_handler_yields( + ui, + query={"last_event_id": "0"}, + max_yields=4, + ) + repair_wire = next( + item + for item in incapable_yields + if isinstance(item, dict) and "tool_turn_projection_unsupported" in item.get("data", "") + ) + assert repair_wire["id"] == "0" + assert json.loads(repair_wire["data"]) == { + "type": "replay_truncated", + "ws_id": ui.ws_id, + "reason": "tool_turn_projection_unsupported", + } + for secret in ("secret final output", "secret_tool_name", "preview-secret"): + assert secret not in incapable_blob + assert "id: 2" in incapable_blob + + retry_yields, _ = _drain_handler_yields( + ui, + query={"last_event_id": "0"}, + max_yields=3, + ) + retry_repair = next( + item + for item in retry_yields + if isinstance(item, dict) and "tool_turn_projection_unsupported" in item.get("data", "") + ) + assert retry_repair["id"] == "0" + + +class _HandoffSession: + """Route-seam double for ChatSession's atomic history registration. + + Carries the concrete fields the REAL shared preamble reads — the + lifted replay_ok path now calls ``session_replay_preamble`` directly + (the per-kind wrapper indirection is deleted), so the double must be + preamble-readable for the bootstrap pins to exercise the true path. + """ + + def __init__(self, ui: _ConcreteUI, token: str = "revision-7") -> None: + self.ui = ui + self.token = token + self.calls: list[tuple[str, int | None]] = [] + self.model = "test" + self.model_alias = "" + self.context_window = 1000 + self.reasoning_effort = "low" + self._last_usage = {"prompt_tokens": 12, "completion_tokens": 0} + + def register_listener_for_history_handoff( + self, + token: str, + *, + last_event_id: int | None = None, + maxsize: int = 500, + ) -> Any: + self.calls.append((token, last_event_id)) + if token != self.token: + return None + if last_event_id is None: + listener, snap = self.ui.register_listener_with_in_progress_snapshot(maxsize=maxsize) + return listener, [], "fresh", 0, 0, snap + return self.ui.register_listener_with_replay(last_event_id, maxsize=maxsize) + + +def test_valid_history_handoff_fresh_connect_skips_legacy_repair_floor() -> None: + """Current clients present a token and incur no compatibility refetch.""" + + ui = _make_ui() + session = _HandoffSession(ui) + _, blob = _drain_handler_yields( + ui, + query={"history_token": session.token}, + session=session, + max_yields=4, + ) + + assert session.calls == [(session.token, None)] + assert "tokenless_history_bootstrap" not in blob + assert "replay_truncated" not in blob + + +def test_history_handoff_mismatch_forces_resync_without_numeric_replay() -> None: + """A stale history revision cannot fall through to a coverable ring slice.""" + ui = _make_ui() + ui.on_content_token("ring data that must not be used") + session = _HandoffSession(ui) + + _, blob = _drain_handler_yields( + ui, + query={"history_token": "stale-revision", "last_event_id": "0"}, + session=session, + max_yields=4, + ) + + assert session.calls == [("stale-revision", 0)] # cursor 0 survives parsing + assert "history_resync" in blob + assert "handoff_mismatch" in blob + assert "ring data that must not be used" not in blob + assert "id: 1" not in blob + + +def test_malformed_native_header_retains_initial_handoff_validation() -> None: + """A mangled native header cannot suppress a crossed-token mismatch. + + The initial URL still carries both its REST handoff token and cursor. If + the native header is unusable, those URL bootstrap hints remain + authoritative and the stale token must produce ``history_resync``. + """ + ui = _make_ui() + ui.on_content_token("ring data that must not bridge the crossed token") + session = _HandoffSession(ui) + + _, blob = _drain_handler_yields( + ui, + headers={"Last-Event-ID": "mangled-by-proxy"}, + query={"history_token": "stale-revision", "last_event_id": "0"}, + session=session, + max_yields=4, + ) + + assert session.calls == [("stale-revision", 0)] + assert "history_resync" in blob + assert "handoff_mismatch" in blob + assert "ring data that must not bridge the crossed token" not in blob + assert "id: 1" not in blob + + +def test_native_last_event_id_ignores_stale_initial_history_token() -> None: + """Native reconnect headers take priority over the one-shot URL token.""" + ui = _make_ui() + ui.on_content_token("native replay") + session = _HandoffSession(ui) + + _, blob = _drain_handler_yields( + ui, + headers={"Last-Event-ID": "0"}, + query={"history_token": "stale-revision", "last_event_id": "999"}, + session=session, + max_yields=8, + ) + + assert session.calls == [] + assert "history_resync" not in blob + assert "native replay" in blob + assert "id: 1" in blob + + +def test_handoff_cursor_replay_keeps_preamble_without_pending_control_duplicate() -> None: + """Initial cursor replay retains idempotent bootstrap fields only. + + The full replay callback includes pending operator controls and must not run + on replay_ok; the ring delta is the single owner of any such controls. + """ + ui = _make_ui() + ui.on_content_token("covered delta") + session = _HandoffSession(ui) + + def _full_replay(_ws: Any, _ui: Any, _request: Any) -> Any: + # The full replay's own preamble half is what the replay_ok path + # must NOT re-run; the lifted body calls the shared + # session_replay_preamble directly instead (no per-kind hook). + yield {"type": "connected", "model": "test"} + yield {"type": "status", "total_tokens": 12} + yield {"type": "approve_request", "items": [{"call_id": "duplicate"}]} + + _, blob = _drain_handler_yields( + ui, + query={"history_token": session.token, "last_event_id": "0"}, + session=session, + events_replay=_full_replay, + max_yields=10, + state="running", + ) + + assert session.calls == [(session.token, 0)] + assert '"type": "connected"' in blob + assert '"type": "status"' in blob + assert '"type": "state_change"' in blob + assert "covered delta" in blob + assert "approve_request" not in blob + assert "in_progress_snapshot" not in blob + + # --------------------------------------------------------------------------- # Fresh-connect replay completeness — persisted last_error surface # (sibling to the tool-call ``pending`` fix; the fresh-connect synthetic @@ -815,7 +1200,11 @@ def test_handler_replay_ok_does_not_resurface_last_error(monkeypatch: Any) -> No # holes -> permanent gap even after a "successful" reconnect. -def _fake_live_request(*, path_params: dict[str, str] | None = None) -> Request: +def _fake_live_request( + *, + path_params: dict[str, str] | None = None, + query: dict[str, str] | None = None, +) -> Request: """A request whose ``receive()`` never resolves, so ``is_disconnected()`` stays ``False`` — the poison check, not disconnect detection, must be what terminates the drain loop.""" @@ -825,7 +1214,9 @@ def _fake_live_request(*, path_params: dict[str, str] | None = None) -> Request: "headers": [], "path": "/events", "raw_path": b"/events", - "query_string": b"", + "query_string": ( + "&".join(f"{key}={value}" for key, value in query.items()).encode() if query else b"" + ), "path_params": path_params or {}, "app": MagicMock(), } @@ -837,6 +1228,144 @@ def _fake_live_request(*, path_params: dict[str, str] | None = None) -> Request: return Request(scope, receive=_recv) +def test_live_user_turn_projection_is_per_listener_and_ring_stays_canonical() -> None: + """One enqueue fans out typed and compatibility views without mutating the ring.""" + + ui = _make_ui() + handler = _wire_events_handler(ui) + + async def _run() -> tuple[dict[str, Any], dict[str, Any]]: + capable = await handler( + _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0", "user_turn": "1"}, + ) + ) + incapable = await handler( + _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0"}, + ) + ) + capable_gen = capable.body_iterator + incapable_gen = incapable.body_iterator + try: + # replay_ok preamble: retry then current state. Both listeners are + # now atomically registered and waiting at the same live boundary. + await capable_gen.__anext__() + await capable_gen.__anext__() + await incapable_gen.__anext__() + await incapable_gen.__anext__() + capable_next = asyncio.create_task(capable_gen.__anext__()) + incapable_next = asyncio.create_task(incapable_gen.__anext__()) + await asyncio.sleep(0) + + event_id = ui.on_user_turn( + "one canonical row", + attachments=[], + sender="alice", + source=None, + client_send_ids=["live-send"], + ) + assert event_id == 1 + return ( + await asyncio.wait_for(capable_next, timeout=2), + await asyncio.wait_for(incapable_next, timeout=2), + ) + finally: + await capable_gen.aclose() + await incapable_gen.aclose() + + capable_frame, incapable_frame = asyncio.run(_run()) + assert capable_frame["id"] == "1" + assert json.loads(capable_frame["data"]) == { + "type": "user_turn", + "content": "one canonical row", + "client_send_ids": ["live-send"], + "sender": "alice", + "ws_id": ui.ws_id, + "_event_id": 1, + } + assert incapable_frame["id"] == "0" + assert json.loads(incapable_frame["data"]) == { + "type": "replay_truncated", + "ws_id": ui.ws_id, + "reason": "user_turn_projection_unsupported", + } + assert len(ui._event_buffer) == 1 + assert ui._event_buffer[0][1]["type"] == "user_turn" + assert ui._event_buffer[0][1]["content"] == "one canonical row" + + +def test_live_tool_turn_projection_is_per_listener_and_ring_stays_canonical() -> None: + """Capability projection is listener-local; the replay ring keeps truth.""" + + ui = _make_ui() + handler = _wire_events_handler(ui) + + async def _run() -> tuple[dict[str, Any], dict[str, Any]]: + capable = await handler( + _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0", "tool_turn": "1"}, + ) + ) + incapable = await handler( + _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0"}, + ) + ) + capable_gen = capable.body_iterator + incapable_gen = incapable.body_iterator + try: + await capable_gen.__anext__() + await capable_gen.__anext__() + await incapable_gen.__anext__() + await incapable_gen.__anext__() + capable_next = asyncio.create_task(capable_gen.__anext__()) + incapable_next = asyncio.create_task(incapable_gen.__anext__()) + await asyncio.sleep(0) + + event_id = ui.on_tool_turn_accepted( + "call-live", + "bash", + "final output", + effect_status="none", + ) + assert event_id == 1 + return ( + await asyncio.wait_for(capable_next, timeout=2), + await asyncio.wait_for(incapable_next, timeout=2), + ) + finally: + await capable_gen.aclose() + await incapable_gen.aclose() + + capable_frame, incapable_frame = asyncio.run(_run()) + assert capable_frame["id"] == "1" + assert json.loads(capable_frame["data"]) == { + "type": "tool_result", + "accepted": True, + "call_id": "call-live", + "name": "bash", + "output": "final output", + "effect_status": "none", + "ws_id": ui.ws_id, + "_event_id": 1, + } + assert incapable_frame["id"] == "0" + assert json.loads(incapable_frame["data"]) == { + "type": "replay_truncated", + "ws_id": ui.ws_id, + "reason": "tool_turn_projection_unsupported", + } + assert len(ui._event_buffer) == 1 + assert ui._event_buffer[0][1]["type"] == "tool_result" + assert ui._event_buffer[0][1]["accepted"] is True + assert ui._event_buffer[0][1]["output"] == "final output" + + def test_listener_queue_poisons_at_first_full_and_refuses_after() -> None: """The first rejected put latches ``poisoned`` (atomically, under the queue's own mutex) and every later put is refused even if the @@ -921,7 +1450,10 @@ def test_drain_loop_closes_with_overflow_frame_on_poison() -> None: ui = _make_ui() handler = _wire_events_handler(ui) - req = _fake_live_request(path_params={"ws_id": ui.ws_id}) + req = _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0"}, + ) async def _run() -> list[Any]: resp = await handler(req) @@ -962,7 +1494,10 @@ def test_drain_loop_delivers_until_poison_then_stops_before_backlog() -> None: ui = _make_ui() handler = _wire_events_handler(ui) - req = _fake_live_request(path_params={"ws_id": ui.ws_id}) + req = _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0"}, + ) async def _run() -> tuple[list[Any], Any, Any]: resp = await handler(req) @@ -1064,7 +1599,10 @@ def test_closing_queue_unwinds_clean_not_overflow_when_poisoned() -> None: tripping its reconnect limiter on a ws that is simply gone).""" ui = _make_ui() handler = _wire_events_handler(ui) - req = _fake_live_request(path_params={"ws_id": ui.ws_id}) + req = _fake_live_request( + path_params={"ws_id": ui.ws_id}, + query={"last_event_id": "0"}, + ) async def _run() -> tuple[Any, bool]: resp = await handler(req) diff --git a/tests/test_sse_recovery_e2e.py b/tests/test_sse_recovery_e2e.py index b9c46c89..4d5b7a81 100644 --- a/tests/test_sse_recovery_e2e.py +++ b/tests/test_sse_recovery_e2e.py @@ -91,6 +91,51 @@ SEQ_OUTPUT = "".join(f"{n}\n" for n in range(1, 501)) # `seq 1 500` chunk strea PACED_STORM = "for i in $(seq 1 40); do echo r-$i; sleep 0.05; done" +# --------------------------------------------------------------------------- +# Recovery-server event pulses used by the browser latch probes +# --------------------------------------------------------------------------- + + +def test_recovery_server_pulses_follow_the_ordered_session_sse_path( + make_server: Callable[..., RecoveryServer], +) -> None: + """Idle/tool pulses are real ordered UI events, not harness-only DOM calls.""" + srv = make_server() + ws_id = srv.create_workstream(name="recovery-pulses") + client = BrowserlikeSSEClient(srv.base_url, ws_id, srv.token) + try: + client.connect() + client.wait_for_type("connected", timeout=10) + + idle_id = srv.emit_idle_edge(ws_id) + pending_id = srv.emit_tool_pending(ws_id, "recovery-probe") + result_id = srv.emit_tool_result(ws_id, "recovery-probe") + expected_ids = {idle_id, pending_id, result_id} + client.wait_for( + lambda c: ( + expected_ids + <= {f.event_id_int for f in c.all_frames() if f.event_id_int is not None} + ), + timeout=10, + ) + + frames = {f.event_id_int: f for f in client.all_frames() if f.event_id_int in expected_ids} + assert [idle_id, pending_id, result_id] == list(range(idle_id, result_id + 1)) + assert [frames[event_id].etype for event_id in sorted(expected_ids)] == [ + "state_change", + "tool_pending", + "tool_result", + ] + assert frames[idle_id].payload is not None + assert frames[idle_id].payload["state"] == "idle" + assert frames[pending_id].payload is not None + assert frames[pending_id].payload["items"][0]["call_id"] == "recovery-probe" + assert frames[result_id].payload is not None + assert frames[result_id].payload["call_id"] == "recovery-probe" + finally: + client.close() + + # --------------------------------------------------------------------------- # Scenario 1 — storm without loss (fix-3 efficacy) # --------------------------------------------------------------------------- diff --git a/tests/test_storage_atomic_tool_attachments.py b/tests/test_storage_atomic_tool_attachments.py new file mode 100644 index 00000000..b2b3f4a2 --- /dev/null +++ b/tests/test_storage_atomic_tool_attachments.py @@ -0,0 +1,535 @@ +"""Atomic, idempotent TOOL-row plus attachment persistence coverage.""" + +from __future__ import annotations + +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext +from types import SimpleNamespace +from typing import Any + +import pytest +from sqlalchemy.dialects import postgresql + +from tests._storage_fakes import make_attachment +from turnstone.core import memory +from turnstone.core.storage import ( + AttachmentWrite, + ConversationCommitConflictError, + ConversationCommitWorkstreamGoneError, + _utils, +) +from turnstone.core.storage._postgresql import PostgreSQLBackend + + +def _attachment( + attachment_id: str, + content: bytes, + *, + filename: str = "tool-image.png", + mime_type: str = "image/png", + kind: str = "image", +) -> AttachmentWrite: + return make_attachment( + attachment_id, + content, + filename=filename, + mime_type=mime_type, + kind=kind, + ) + + +def _commit( + backend: Any, + ws_id: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + content: str = "Image file: result.png", + tool_name: str = "read_file", + tool_call_id: str = "call-image", + event_id: int | None = 29, + is_error: bool = False, + meta: str | None = '{"effect_status":"succeeded"}', + commit_key: str = "one-tool-result", +) -> int: + return backend.save_tool_message_with_attachments( + ws_id, + content, + tool_name, + tool_call_id, + attachments, + event_id=event_id, + is_error=is_error, + meta=meta, + commit_key=commit_key, + ) + + +def test_identical_tool_retry_returns_same_row_without_refcount_replay( + storage_backend: Any, +) -> None: + backend = storage_backend + ws_id = "atomic-tool-retry" + backend.register_workstream(ws_id) + image = _attachment("a" * 64, b"image-bytes") + + first_id = _commit(backend, ws_id, [image]) + retry_id = _commit(backend, ws_id, [image]) + + assert retry_id == first_id + assert backend.count_messages(ws_id) == 1 + blob = backend.get_attachment(image.attachment_id) + assert blob["refcount"] == 1 + assert blob["origin"] == "tool" + turn = backend.load_message_turns(ws_id, checkpointed=False)[0] + assert turn.tool_call_id == "call-image" + assert turn.is_error is False + assert turn.meta.extra["storage_attachment_ids"] == [image.attachment_id] + + +def test_concurrent_identical_tool_retries_retain_once(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-tool-concurrent" + backend.register_workstream(ws_id) + image = _attachment("b" * 64, b"same-result") + barrier = threading.Barrier(3) + + def _write() -> int: + barrier.wait(timeout=10) + return _commit(backend, ws_id, [image]) + + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(_write) + second = pool.submit(_write) + barrier.wait(timeout=10) + row_ids = {first.result(timeout=10), second.result(timeout=10)} + + assert len(row_ids) == 1 + assert backend.count_messages(ws_id) == 1 + assert backend.get_attachment(image.attachment_id)["refcount"] == 1 + + +@pytest.mark.parametrize( + ("override", "reverse"), + [ + ({"content": "changed"}, False), + ({"tool_name": "open_preview"}, False), + ({"tool_call_id": "call-other"}, False), + ({"event_id": 30}, False), + ({"is_error": True}, False), + ({"meta": '{"effect_status":"unknown"}'}, False), + ({}, True), + ], +) +def test_same_key_tool_mismatch_fails_before_refcount_mutation( + storage_backend: Any, + override: dict[str, Any], + reverse: bool, +) -> None: + backend = storage_backend + ws_id = f"atomic-tool-conflict-{len(override)}-{reverse}-{next(iter(override), 'refs')}" + backend.register_workstream(ws_id) + first = _attachment("c" * 64, b"first") + second = _attachment("d" * 64, b"second") + _commit(backend, ws_id, [first, second]) + + attachments = [second, first] if reverse else [first, second] + with pytest.raises(ConversationCommitConflictError): + _commit(backend, ws_id, attachments, **override) + + assert backend.count_messages(ws_id) == 1 + assert backend.get_attachment(first.attachment_id)["refcount"] == 1 + assert backend.get_attachment(second.attachment_id)["refcount"] == 1 + + +def test_cancelled_preview_duplicate_refs_delete_exactly(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-tool-cancelled-preview" + backend.register_workstream(ws_id) + preview = _attachment( + "e" * 64, + b"preview", + filename="preview.html", + mime_type="text/html", + kind="preview", + ) + meta = json.dumps( + { + "effect_status": "unknown", + "preview": {"attachment_id": preview.attachment_id, "kind": "html"}, + } + ) + + _commit( + backend, + ws_id, + [preview, preview], + content="Tool execution was cancelled before its outcome was observed.", + tool_name="open_preview", + tool_call_id="call-preview", + is_error=True, + meta=meta, + ) + + blob = backend.get_attachment(preview.attachment_id) + assert blob["refcount"] == 2 + assert blob["kind"] == "preview" + assert blob["origin"] == "tool" + turn = backend.load_message_turns(ws_id, checkpointed=False)[0] + assert turn.is_error is True + assert turn.meta.extra["storage_attachment_ids"] == [ + preview.attachment_id, + preview.attachment_id, + ] + assert turn.meta.extra["preview"]["attachment_id"] == preview.attachment_id + + assert backend.delete_workstream(ws_id) is True + assert backend.get_attachment(preview.attachment_id) is None + + +def test_hard_delete_then_tool_retry_refuses_row_and_refcount_recreation( + storage_backend: Any, +) -> None: + backend = storage_backend + ws_id = "atomic-tool-delete-retry" + backend.register_workstream(ws_id) + attachment = _attachment("8" * 64, b"deleted-tool") + _commit(backend, ws_id, [attachment]) + assert backend.delete_workstream(ws_id) is True + + with pytest.raises(RuntimeError, match="workstream no longer exists"): + _commit(backend, ws_id, [attachment]) + # The facade propagates the typed permanence signal instead of swallowing + # it into the operational-failure 0: the durability journal classifies a + # deleted parent as terminal, never retrying. + with pytest.raises(ConversationCommitWorkstreamGoneError): + memory.save_tool_message_with_attachments( + ws_id, + "Image file: result.png", + "read_file", + "call-image", + [attachment], + event_id=29, + meta='{"effect_status":"succeeded"}', + commit_key="one-tool-result", + ) + + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(attachment.attachment_id) is None + assert backend.list_orphan_conversations() == [] + + +def test_concurrent_tool_commit_and_delete_leave_no_row_or_refs(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-tool-save-delete-race" + backend.register_workstream(ws_id) + attachment = _attachment("9" * 64, b"racing-tool") + barrier = threading.Barrier(3) + + def _save() -> str: + barrier.wait(timeout=10) + try: + _commit(backend, ws_id, [attachment]) + except RuntimeError as exc: + assert "workstream no longer exists" in str(exc) + return "refused" + return "saved" + + def _delete() -> bool: + barrier.wait(timeout=10) + return backend.delete_workstream(ws_id) + + with ThreadPoolExecutor(max_workers=2) as pool: + save = pool.submit(_save) + delete = pool.submit(_delete) + barrier.wait(timeout=10) + assert save.result(timeout=10) in {"saved", "refused"} + assert delete.result(timeout=10) is True + + assert backend.get_workstream(ws_id) is None + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(attachment.attachment_id) is None + assert backend.list_orphan_conversations() == [] + + +def test_tool_partial_failure_rolls_back_row_blobs_refs_and_list( + storage_backend: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Injection target follows the increment into the shared commit body. + + Both backends now run the keyed attachment commit through + ``_utils.save_attachment_commit_transaction``, so the refcount increment is + called on ``_utils`` rather than through the dialect module's re-export. + The property under test is unchanged: a failure after the increment must + leave no row, no new blob, and no changed refcount. + """ + backend = storage_backend + ws_id = "atomic-tool-rollback" + backend.register_workstream(ws_id) + existing = _attachment("1" * 64, b"existing") + new = _attachment("2" * 64, b"new") + backend.save_attachment( + existing.attachment_id, + existing.filename, + existing.mime_type, + existing.size_bytes, + existing.kind, + existing.content, + "tool", + ) + real_retain = _utils.retain_attachment_refs + + def _retain_then_fail(conn: Any, attachment_ids: list[str]) -> None: + real_retain(conn, attachment_ids) + raise RuntimeError("injected tool failure after refcount update") + + monkeypatch.setattr(_utils, "retain_attachment_refs", _retain_then_fail) + with pytest.raises(RuntimeError, match="injected tool failure"): + _commit(backend, ws_id, [existing, new]) + + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(existing.attachment_id)["refcount"] == 1 + assert backend.get_attachment(new.attachment_id) is None + + +@pytest.mark.parametrize( + ("mime_type", "kind"), + [("text/plain", "image"), ("image/png", "preview")], +) +def test_tool_cas_mime_and_kind_conflict_rolls_back( + storage_backend: Any, + mime_type: str, + kind: str, +) -> None: + backend = storage_backend + ws_id = f"atomic-tool-cas-{kind}" + backend.register_workstream(ws_id) + attachment_id = "3" * 64 + backend.save_attachment( + attachment_id, + "existing.png", + mime_type, + len(b"same-bytes"), + kind, + b"same-bytes", + "tool", + ) + + with pytest.raises(ConversationCommitConflictError): + _commit(backend, ws_id, [_attachment(attachment_id, b"same-bytes")]) + + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(attachment_id)["refcount"] == 1 + + +def test_tool_memory_facade_preserves_typed_commit_conflict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _ConflictStorage: + def save_tool_message_with_attachments(self, *_args: Any, **_kwargs: Any) -> int: + raise ConversationCommitConflictError("immutable tool commit mismatch") + + monkeypatch.setattr(memory, "get_storage", lambda: _ConflictStorage()) + with pytest.raises(ConversationCommitConflictError, match="immutable tool commit mismatch"): + memory.save_tool_message_with_attachments( + "atomic-tool-facade-conflict", + "result", + "read_file", + "call-1", + [_attachment("4" * 64, b"payload")], + commit_key="conflicting-tool-commit", + ) + + +def test_tool_memory_facade_returns_explicit_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class _BrokenStorage: + def save_tool_message_with_attachments(self, *_args: Any, **_kwargs: Any) -> int: + raise RuntimeError("database unavailable") + + monkeypatch.setattr(memory, "get_storage", lambda: _BrokenStorage()) + result = memory.save_tool_message_with_attachments( + "atomic-tool-facade", + "result", + "read_file", + "call-1", + [_attachment("4" * 64, b"payload")], + commit_key="failed-tool-commit", + ) + assert result == 0 + + +class _PostgresResult: + def __init__( + self, + *, + scalar: int | None = None, + row: Any | None = None, + rows: list[Any] | None = None, + scalar_values: list[str] | None = None, + ) -> None: + self._scalar = scalar + self._row = row + self._rows = rows or [] + self._scalar_values = scalar_values or [] + + def scalar_one_or_none(self) -> int | None: + return self._scalar + + def fetchone(self) -> Any | None: + return self._row + + def fetchall(self) -> list[Any]: + return self._rows + + def scalars(self) -> list[str]: + return self._scalar_values + + +class _PostgresConnection: + def __init__(self, results: list[_PostgresResult | BaseException]) -> None: + self._results = results + self.statements: list[Any] = [] + self.commits = 0 + self.rollbacks = 0 + + def execute(self, statement: Any, *_args: Any, **_kwargs: Any) -> _PostgresResult: + self.statements.append(statement) + if not self._results: + raise AssertionError("unexpected PostgreSQL statement") + result = self._results.pop(0) + if isinstance(result, BaseException): + raise result + return result + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def _blob_row(attachment: AttachmentWrite) -> Any: + return SimpleNamespace( + _mapping={ + "attachment_id": attachment.attachment_id, + "mime_type": attachment.mime_type, + "size_bytes": attachment.size_bytes, + "kind": attachment.kind, + "content": attachment.content, + } + ) + + +def test_postgresql_tool_insert_uses_one_conflict_safe_transaction() -> None: + """The blob insert reports what it wrote; freshly written bytes aren't re-read. + + Rationale for the schedule change: the blob insert now carries + ``RETURNING attachment_id``, so ids this transaction actually wrote need no + content verification — it just supplied those bytes. Re-reading them would + pull the whole tool payload back out of the database while the parent row + lock is held. Only conflicted (pre-existing) ids are verified, covered by + ``test_tool_cas_mime_and_kind_conflict_rolls_back``. + """ + attachment = _attachment("5" * 64, b"postgres-tool") + conn = _PostgresConnection( + [ + _PostgresResult(row=("postgres-tool-insert",)), + _PostgresResult(scalar=51), + _PostgresResult(scalar_values=[attachment.attachment_id]), + _PostgresResult(scalar_values=[attachment.attachment_id]), + _PostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + assert _commit(backend, "postgres-tool-insert", [attachment], is_error=True) == 51 + + compiled = [statement.compile(dialect=postgresql.dialect()) for statement in conn.statements] + sql = [str(statement) for statement in compiled] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" in sql[0] + assert "INSERT INTO conversations" in sql[1] and "ON CONFLICT" in sql[1] + assert "WHERE commit_key IS NOT NULL" in sql[1] + assert compiled[1].params["role"] == "tool" + assert compiled[1].params["tool_name"] == "read_file" + assert compiled[1].params["tool_call_id"] == "call-image" + assert compiled[1].params["is_error"] is True + assert "INSERT INTO workstream_attachments" in sql[2] and "ON CONFLICT" in sql[2] + assert "RETURNING workstream_attachments.attachment_id" in sql[2] + assert any( + key.startswith("origin") and value == "tool" for key, value in compiled[2].params.items() + ) + assert "UPDATE workstream_attachments" in sql[3] + assert "UPDATE workstreams" in sql[4] + assert all("SELECT workstream_attachments.attachment_id" not in item for item in sql) + assert conn.commits == 1 + assert conn.rollbacks == 0 + assert conn._results == [] + + +def test_postgresql_identical_retry_emits_no_refcount_update() -> None: + attachment = _attachment("6" * 64, b"postgres-retry") + existing = SimpleNamespace( + _mapping={ + "id": 61, + "role": "tool", + "content": "Image file: result.png", + "tool_name": "read_file", + "tool_call_id": "call-image", + "provider_data": None, + "tool_calls": None, + "_source": None, + "event_id": 29, + "is_error": False, + "attachments": json.dumps([attachment.attachment_id]), + "meta": '{"effect_status":"succeeded"}', + "commit_key": "one-tool-result", + } + ) + conn = _PostgresConnection( + [ + _PostgresResult(row=("postgres-tool-retry",)), + _PostgresResult(scalar=None), + _PostgresResult(row=existing), + _PostgresResult(rows=[_blob_row(attachment)]), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + assert _commit(backend, "postgres-tool-retry", [attachment]) == 61 + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert len(sql) == 4 + assert all("UPDATE workstream_attachments" not in statement for statement in sql) + assert conn.commits == 1 + assert conn.rollbacks == 0 + assert conn._results == [] + + +def test_postgresql_partial_failure_rolls_back_the_transaction() -> None: + attachment = _attachment("7" * 64, b"postgres-rollback") + conn = _PostgresConnection( + [ + _PostgresResult(row=("postgres-tool-rollback",)), + _PostgresResult(scalar=71), + _PostgresResult(scalar_values=[attachment.attachment_id]), + RuntimeError("injected PostgreSQL retain failure"), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + with pytest.raises(RuntimeError, match="retain failure"): + _commit(backend, "postgres-tool-rollback", [attachment]) + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" in sql[0] + assert "INSERT INTO conversations" in sql[1] + assert "INSERT INTO workstream_attachments" in sql[2] + assert "UPDATE workstream_attachments" in sql[3] + assert conn.commits == 0 + assert conn.rollbacks == 1 + assert conn._results == [] diff --git a/tests/test_storage_atomic_user_attachments.py b/tests/test_storage_atomic_user_attachments.py new file mode 100644 index 00000000..59d5bb63 --- /dev/null +++ b/tests/test_storage_atomic_user_attachments.py @@ -0,0 +1,475 @@ +"""Atomic, idempotent USER-row plus attachment persistence coverage.""" + +from __future__ import annotations + +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext +from typing import Any + +import pytest +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from tests._storage_fakes import make_attachment +from turnstone.core import memory +from turnstone.core.storage import ( + AttachmentWrite, + ConversationCommitConflictError, + ConversationCommitWorkstreamGoneError, + _utils, +) +from turnstone.core.storage._postgresql import PostgreSQLBackend + + +def _attachment( + attachment_id: str, + content: bytes, + *, + filename: str = "evidence.txt", + mime_type: str = "text/plain", + kind: str = "text", +) -> AttachmentWrite: + return make_attachment( + attachment_id, + content, + filename=filename, + mime_type=mime_type, + kind=kind, + ) + + +def _commit( + backend: Any, + ws_id: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + content: str = "inspect the evidence", + source: str | None = None, + event_id: int | None = 17, + meta: str | None = '{"sender":"alice"}', + commit_key: str = "one-user-admission", +) -> int: + return backend.save_user_message_with_attachments( + ws_id, + content, + attachments, + source=source, + event_id=event_id, + meta=meta, + commit_key=commit_key, + ) + + +def test_identical_retry_returns_same_row_without_retain_replay(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-attachment-retry" + backend.register_workstream(ws_id) + first = _attachment("a" * 64, b"first", filename="first.txt") + second = _attachment("b" * 64, b"second", filename="second.txt") + + first_id = _commit(backend, ws_id, [first, second]) + retry_id = _commit(backend, ws_id, [first, second]) + + assert retry_id == first_id + assert backend.count_messages(ws_id) == 1 + assert backend.get_attachment(first.attachment_id)["refcount"] == 1 + assert backend.get_attachment(second.attachment_id)["refcount"] == 1 + turn = backend.load_message_turns(ws_id, checkpointed=False)[0] + assert turn.meta.extra["storage_attachment_ids"] == [ + first.attachment_id, + second.attachment_id, + ] + + +def test_concurrent_identical_retries_retain_each_reference_once(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-attachment-concurrent-retry" + backend.register_workstream(ws_id) + attachment = _attachment("9" * 64, b"concurrent") + barrier = threading.Barrier(3) + + def _write() -> int: + barrier.wait(timeout=10) + return _commit(backend, ws_id, [attachment]) + + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(_write) + second = pool.submit(_write) + barrier.wait(timeout=10) + row_ids = {first.result(timeout=10), second.result(timeout=10)} + + assert len(row_ids) == 1 + assert backend.count_messages(ws_id) == 1 + assert backend.get_attachment(attachment.attachment_id)["refcount"] == 1 + + +@pytest.mark.parametrize( + ("override", "attachments"), + [ + ({"content": "different text"}, None), + ({"source": "different_source"}, None), + ({"event_id": 18}, None), + ({"meta": '{"sender":"bob"}'}, None), + ({}, "reverse"), + ], +) +def test_same_key_with_different_row_or_order_fails_without_mutation( + storage_backend: Any, + override: dict[str, Any], + attachments: str | None, +) -> None: + backend = storage_backend + ws_id = f"atomic-attachment-conflict-{override or attachments}" + backend.register_workstream(ws_id) + first = _attachment("c" * 64, b"first") + second = _attachment("d" * 64, b"second") + _commit(backend, ws_id, [first, second]) + + retried = [second, first] if attachments == "reverse" else [first, second] + with pytest.raises(ConversationCommitConflictError): + _commit(backend, ws_id, retried, **override) + + assert backend.count_messages(ws_id) == 1 + assert backend.get_attachment(first.attachment_id)["refcount"] == 1 + assert backend.get_attachment(second.attachment_id)["refcount"] == 1 + + +def test_repeated_attachment_ids_count_and_delete_exact_references(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-attachment-repeated-refs" + backend.register_workstream(ws_id) + shared = _attachment("e" * 64, b"shared") + distinct = _attachment("f" * 64, b"distinct") + + _commit(backend, ws_id, [shared, shared, distinct]) + + assert backend.get_attachment(shared.attachment_id)["refcount"] == 2 + assert backend.get_attachment(distinct.attachment_id)["refcount"] == 1 + turn = backend.load_message_turns(ws_id, checkpointed=False)[0] + assert turn.meta.extra["storage_attachment_ids"] == [ + shared.attachment_id, + shared.attachment_id, + distinct.attachment_id, + ] + + assert backend.delete_workstream(ws_id) is True + assert backend.get_attachment(shared.attachment_id) is None + assert backend.get_attachment(distinct.attachment_id) is None + + +def test_hard_delete_then_user_retry_refuses_row_and_refcount_recreation( + storage_backend: Any, +) -> None: + backend = storage_backend + ws_id = "atomic-user-delete-retry" + backend.register_workstream(ws_id) + attachment = _attachment("7" * 64, b"deleted") + _commit(backend, ws_id, [attachment]) + assert backend.delete_workstream(ws_id) is True + + with pytest.raises(RuntimeError, match="workstream no longer exists"): + _commit(backend, ws_id, [attachment]) + # The facade propagates the typed permanence signal instead of swallowing + # it into the operational-failure 0: the durability journal classifies a + # deleted parent as terminal, never retrying. + with pytest.raises(ConversationCommitWorkstreamGoneError): + memory.save_user_message_with_attachments( + ws_id, + "inspect the evidence", + [attachment], + event_id=17, + meta='{"sender":"alice"}', + commit_key="one-user-admission", + ) + + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(attachment.attachment_id) is None + assert backend.list_orphan_conversations() == [] + + +def test_concurrent_user_commit_and_delete_leave_no_row_or_refs(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-user-save-delete-race" + backend.register_workstream(ws_id) + attachment = _attachment("8" * 64, b"racing-user") + barrier = threading.Barrier(3) + + def _save() -> str: + barrier.wait(timeout=10) + try: + _commit(backend, ws_id, [attachment]) + except RuntimeError as exc: + assert "workstream no longer exists" in str(exc) + return "refused" + return "saved" + + def _delete() -> bool: + barrier.wait(timeout=10) + return backend.delete_workstream(ws_id) + + with ThreadPoolExecutor(max_workers=2) as pool: + save = pool.submit(_save) + delete = pool.submit(_delete) + barrier.wait(timeout=10) + assert save.result(timeout=10) in {"saved", "refused"} + assert delete.result(timeout=10) is True + + assert backend.get_workstream(ws_id) is None + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(attachment.attachment_id) is None + assert backend.list_orphan_conversations() == [] + + +def test_failure_after_refcount_increment_rolls_back_every_side_effect( + storage_backend: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Injection target follows the increment into the shared commit body. + + Both backends now run the keyed attachment commit through + ``_utils.save_attachment_commit_transaction``, so the refcount increment is + called on ``_utils`` rather than through the dialect module's re-export. + The property under test is unchanged: a failure after the increment must + leave no row, no new blob, and no changed refcount. + """ + backend = storage_backend + ws_id = "atomic-attachment-rollback" + backend.register_workstream(ws_id) + existing = _attachment("1" * 64, b"existing") + new = _attachment("2" * 64, b"new") + backend.save_attachment( + existing.attachment_id, + existing.filename, + existing.mime_type, + existing.size_bytes, + existing.kind, + existing.content, + ) + real_retain = _utils.retain_attachment_refs + + def _retain_then_fail(conn: Any, attachment_ids: list[str]) -> None: + real_retain(conn, attachment_ids) + raise RuntimeError("injected failure after refcount update") + + monkeypatch.setattr(_utils, "retain_attachment_refs", _retain_then_fail) + with pytest.raises(RuntimeError, match="injected failure"): + _commit(backend, ws_id, [existing, new]) + + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(existing.attachment_id)["refcount"] == 1 + assert backend.get_attachment(new.attachment_id) is None + + +@pytest.mark.parametrize("preexisting", [False, True]) +def test_content_verification_reads_only_conflicted_blobs( + storage_backend: Any, + preexisting: bool, +) -> None: + """Bytes this transaction just wrote are never read back to verify them. + + The blob insert reports the ids it actually wrote, so verification is + confined to conflicted (pre-existing) ids — the only ones that can disagree + with the request. Re-reading a fresh upload would double the write + transaction's I/O and extend the parent row's lock hold on the hot path. + """ + backend = storage_backend + ws_id = f"atomic-attachment-verify-{preexisting}" + backend.register_workstream(ws_id) + attachment = _attachment("2" * 64, b"verified-once") + if preexisting: + backend.save_attachment( + attachment.attachment_id, + attachment.filename, + attachment.mime_type, + attachment.size_bytes, + attachment.kind, + attachment.content, + ) + blob_reads: list[str] = [] + + def _before_cursor_execute( + _conn: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if ( + statement.lstrip().upper().startswith("SELECT") + and "workstream_attachments" in statement + ): + blob_reads.append(statement) + + sa.event.listen(backend._engine, "before_cursor_execute", _before_cursor_execute) + try: + _commit(backend, ws_id, [attachment]) + finally: + sa.event.remove(backend._engine, "before_cursor_execute", _before_cursor_execute) + + assert len(blob_reads) == (1 if preexisting else 0) + assert backend.count_messages(ws_id) == 1 + assert backend.get_attachment(attachment.attachment_id)["refcount"] == (2 if preexisting else 1) + + +def test_existing_blob_payload_conflict_rolls_back_conversation(storage_backend: Any) -> None: + backend = storage_backend + ws_id = "atomic-attachment-cas-conflict" + backend.register_workstream(ws_id) + attachment_id = "3" * 64 + backend.save_attachment( + attachment_id, + "original.txt", + "text/plain", + len(b"original"), + "text", + b"original", + ) + + with pytest.raises(ConversationCommitConflictError): + _commit(backend, ws_id, [_attachment(attachment_id, b"different")]) + + assert backend.count_messages(ws_id) == 0 + stored = backend.get_attachment(attachment_id) + assert stored["content"] == b"original" + assert stored["refcount"] == 1 + + +def test_same_key_with_changed_blob_payload_fails_without_refcount_replay( + storage_backend: Any, +) -> None: + backend = storage_backend + ws_id = "atomic-attachment-keyed-blob-conflict" + backend.register_workstream(ws_id) + attachment_id = "6" * 64 + original = _attachment(attachment_id, b"original") + _commit(backend, ws_id, [original]) + + with pytest.raises(ConversationCommitConflictError): + _commit(backend, ws_id, [_attachment(attachment_id, b"changed!")]) + + assert backend.count_messages(ws_id) == 1 + stored = backend.get_attachment(attachment_id) + assert stored["content"] == b"original" + assert stored["refcount"] == 1 + + +def test_user_memory_facade_preserves_typed_commit_conflict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _ConflictStorage: + def save_user_message_with_attachments(self, *_args: Any, **_kwargs: Any) -> int: + raise ConversationCommitConflictError("immutable user commit mismatch") + + monkeypatch.setattr(memory, "get_storage", lambda: _ConflictStorage()) + with pytest.raises(ConversationCommitConflictError, match="immutable user commit mismatch"): + memory.save_user_message_with_attachments( + "atomic-facade-conflict", + "text", + [_attachment("4" * 64, b"payload")], + commit_key="conflicting-commit", + ) + + +def test_memory_facade_returns_explicit_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class _BrokenStorage: + def save_user_message_with_attachments(self, *_args: Any, **_kwargs: Any) -> int: + raise RuntimeError("database unavailable") + + monkeypatch.setattr(memory, "get_storage", lambda: _BrokenStorage()) + result = memory.save_user_message_with_attachments( + "atomic-facade-failure", + "text", + [_attachment("4" * 64, b"payload")], + commit_key="failed-commit", + ) + assert result == 0 + + +class _PostgresResult: + def __init__( + self, + *, + scalar: int | None = None, + row: Any | None = None, + rows: list[Any] | None = None, + scalar_values: list[str] | None = None, + ) -> None: + self._scalar = scalar + self._row = row + self._rows = rows or [] + self._scalar_values = scalar_values or [] + + def scalar_one_or_none(self) -> int | None: + return self._scalar + + def fetchone(self) -> Any | None: + return self._row + + def fetchall(self) -> list[Any]: + return self._rows + + def scalars(self) -> list[str]: + return self._scalar_values + + +class _PostgresConnection: + def __init__(self, results: list[_PostgresResult]) -> None: + self._results = results + self.statements: list[Any] = [] + self.commits = 0 + self.rollbacks = 0 + + def execute(self, statement: Any, *_args: Any, **_kwargs: Any) -> _PostgresResult: + self.statements.append(statement) + if not self._results: + raise AssertionError("unexpected PostgreSQL statement") + return self._results.pop(0) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def test_postgresql_atomic_path_emits_conflict_safe_transaction_sql() -> None: + """The blob insert reports what it wrote; freshly written bytes aren't re-read. + + Rationale for the schedule change: the blob insert now carries + ``RETURNING attachment_id``, so ids this transaction actually wrote need no + content verification — it just supplied those bytes. Re-reading them would + pull the whole upload back out of the database while the parent row lock is + held. Only conflicted (pre-existing) ids are verified, covered by + ``test_existing_blob_payload_conflict_rolls_back_conversation``. + """ + attachment = _attachment("5" * 64, b"postgres") + conn = _PostgresConnection( + [ + _PostgresResult(row=("postgres-atomic",)), + _PostgresResult(scalar=41), + _PostgresResult(scalar_values=[attachment.attachment_id]), + _PostgresResult(scalar_values=[attachment.attachment_id]), + _PostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + assert _commit(backend, "postgres-atomic", [attachment]) == 41 + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" in sql[0] + assert "INSERT INTO conversations" in sql[1] + assert "ON CONFLICT" in sql[1] + assert "WHERE commit_key IS NOT NULL" in sql[1] + assert "INSERT INTO workstream_attachments" in sql[2] + assert "ON CONFLICT" in sql[2] + assert "RETURNING workstream_attachments.attachment_id" in sql[2] + assert "UPDATE workstream_attachments" in sql[3] + assert "UPDATE workstreams" in sql[4] + assert all("SELECT workstream_attachments.attachment_id" not in item for item in sql) + assert conn.commits == 1 + assert conn.rollbacks == 0 + assert conn._results == [] diff --git a/tests/test_storage_attachments.py b/tests/test_storage_attachments.py index ed81ed92..755a1aa6 100644 --- a/tests/test_storage_attachments.py +++ b/tests/test_storage_attachments.py @@ -285,6 +285,7 @@ class TestReconstructMetaSibling: meta = backend.load_messages("ws-meta")[0].get("_attachments_meta") assert isinstance(meta, list) and len(meta) == 1 assert meta[0] == { + "attachment_id": aid, "kind": "text", "filename": "doc.md", "mime_type": "text/markdown", diff --git a/tests/test_storage_commit_key.py b/tests/test_storage_commit_key.py new file mode 100644 index 00000000..0305f02f --- /dev/null +++ b/tests/test_storage_commit_key.py @@ -0,0 +1,461 @@ +"""Backend-parity coverage for conversation commit idempotency.""" + +from __future__ import annotations + +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext +from typing import TYPE_CHECKING, cast + +import pytest +from sqlalchemy.dialects import postgresql + +from tests._storage_fakes import ( + ScriptedPostgresConnection, + ScriptedPostgresResult, +) +from turnstone.console.coordinator_client import _serialize_messages +from turnstone.core import memory +from turnstone.core.history_decoration import project_history_messages +from turnstone.core.providers._openai_common import sanitize_messages +from turnstone.core.storage import ( + AttachmentWrite, + ConversationCommitConflictError, + ConversationCommitWorkstreamGoneError, +) +from turnstone.core.storage._postgresql import PostgreSQLBackend + +if TYPE_CHECKING: + from turnstone.core.storage import StorageBackend + + +def test_identical_retry_same_commit_key_returns_original_row( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-retry" + commit_key = "assistant-commit-a" + storage_backend.register_workstream(ws_id) + + first_id = storage_backend.save_message( + ws_id, + "assistant", + "original", + event_id=7, + commit_key=commit_key, + ) + retry_id = storage_backend.save_message( + ws_id, + "assistant", + "original", + event_id=7, + commit_key=commit_key, + ) + + assert retry_id == first_id + assert storage_backend.count_messages(ws_id) == 1 + messages = storage_backend.load_messages(ws_id, repair=False) + assert len(messages) == 1 + assert messages[0]["content"] == "original" + assert messages[0]["_event_id"] == 7 + assert messages[0]["_commit_key"] == commit_key + turns = storage_backend.load_message_turns(ws_id, checkpointed=False) + assert len(turns) == 1 + assert turns[0].meta.commit_key == commit_key + + +@pytest.mark.parametrize( + "override", + [ + {"role": "system"}, + {"content": "changed"}, + {"tool_name": "other-tool"}, + {"tool_call_id": "other-call"}, + {"provider_data": '[{"type":"reasoning","text":"changed"}]'}, + { + "tool_calls": ( + '[{"id":"call-2","type":"function","function":{"name":"other","arguments":"{}"}}]' + ) + }, + {"source": "other-source"}, + {"event_id": 8}, + {"is_error": True}, + {"meta": '{"other":true}'}, + ], +) +def test_same_key_with_different_normalized_row_fails_closed( + storage_backend: StorageBackend, + override: dict[str, object], +) -> None: + ws_id = f"commit-mismatch-{next(iter(override))}" + storage_backend.register_workstream(ws_id) + base: dict[str, object] = { + "role": "assistant", + "content": "accepted", + "tool_name": "tool-a", + "tool_call_id": "call-a", + "provider_data": None, + "tool_calls": None, + "source": "accepted-source", + "event_id": 7, + "is_error": False, + "meta": '{"stable":true}', + } + + def _save(values: dict[str, object]) -> int: + return storage_backend.save_message( + ws_id, + str(values["role"]), + str(values["content"]), + tool_name=str(values["tool_name"]), + tool_call_id=str(values["tool_call_id"]), + provider_data=cast("str | None", values["provider_data"]), + tool_calls=cast("str | None", values["tool_calls"]), + source=str(values["source"]), + event_id=int(values["event_id"]), + is_error=bool(values["is_error"]), + meta=str(values["meta"]), + commit_key="one-immutable-row", + ) + + _save(base) + with pytest.raises(ConversationCommitConflictError, match="different conversation commit"): + _save({**base, **override}) + + assert storage_backend.count_messages(ws_id) == 1 + assert storage_backend.load_messages(ws_id, repair=False)[0]["content"] == "accepted" + + +def test_plain_conflict_remains_typed_through_memory_facade( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-facade-conflict" + storage_backend.register_workstream(ws_id) + storage_backend.save_message(ws_id, "assistant", "accepted", commit_key="one-row") + + with pytest.raises(ConversationCommitConflictError): + memory.save_message(ws_id, "assistant", "different", commit_key="one-row") + assert storage_backend.count_messages(ws_id) == 1 + assert storage_backend.load_messages(ws_id, repair=False)[0]["content"] == "accepted" + + +def test_plain_key_cannot_acknowledge_attachment_bearing_row( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-cross-seam-conflict" + commit_key = "one-cross-seam-row" + content = b"attached" + attachment = AttachmentWrite( + attachment_id="e" * 64, + filename="attached.txt", + mime_type="text/plain", + size_bytes=len(content), + kind="text", + content=content, + ) + storage_backend.register_workstream(ws_id) + storage_backend.save_user_message_with_attachments( + ws_id, + "same text", + [attachment], + commit_key=commit_key, + ) + + with pytest.raises(ConversationCommitConflictError, match="attachments"): + storage_backend.save_message( + ws_id, + "user", + "same text", + commit_key=commit_key, + ) + + assert storage_backend.count_messages(ws_id) == 1 + assert storage_backend.get_attachment(attachment.attachment_id)["refcount"] == 1 + + +def test_empty_commit_key_is_rejected_without_mutation(storage_backend: StorageBackend) -> None: + ws_id = "commit-empty-key" + storage_backend.register_workstream(ws_id) + + with pytest.raises(ValueError, match="non-empty"): + storage_backend.save_message(ws_id, "assistant", "accepted", commit_key="") + assert memory.save_message(ws_id, "assistant", "accepted", commit_key="") == 0 + assert storage_backend.count_messages(ws_id) == 0 + + +def test_identical_rows_with_distinct_commit_keys_remain_distinct( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-distinct" + storage_backend.register_workstream(ws_id) + + first_id = storage_backend.save_message( + ws_id, "assistant", "same payload", commit_key="commit-one" + ) + second_id = storage_backend.save_message( + ws_id, "assistant", "same payload", commit_key="commit-two" + ) + + assert second_id != first_id + assert storage_backend.count_messages(ws_id) == 2 + assert [m["_commit_key"] for m in storage_backend.load_messages(ws_id, repair=False)] == [ + "commit-one", + "commit-two", + ] + + +def test_concurrent_divergent_same_key_writers_fail_one_closed( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-concurrent" + storage_backend.register_workstream(ws_id) + barrier = threading.Barrier(3) + + def _save(content: str) -> tuple[str, int | None]: + barrier.wait(timeout=10) + try: + row_id = storage_backend.save_message( + ws_id, + "assistant", + content, + commit_key="one-admission", + ) + except ConversationCommitConflictError: + return "conflict", None + return "saved", row_id + + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(_save, "writer-a") + second = pool.submit(_save, "writer-b") + barrier.wait(timeout=10) + outcomes = [first.result(timeout=10), second.result(timeout=10)] + + assert sorted(status for status, _row_id in outcomes) == ["conflict", "saved"] + assert len([row_id for _status, row_id in outcomes if row_id is not None]) == 1 + assert storage_backend.count_messages(ws_id) == 1 + assert storage_backend.load_messages(ws_id, repair=False)[0]["content"] in { + "writer-a", + "writer-b", + } + + +def test_commit_key_is_scoped_to_workstream_and_null_writes_still_append( + storage_backend: StorageBackend, +) -> None: + """NULL keys are explicit legacy parent-less writes, never live admission.""" + storage_backend.register_workstream("commit-ws-a") + storage_backend.register_workstream("commit-ws-b") + first_id = storage_backend.save_message("commit-ws-a", "assistant", "a", commit_key="same") + second_id = storage_backend.save_message("commit-ws-b", "assistant", "b", commit_key="same") + null_one = storage_backend.save_message("commit-null", "assistant", "same") + null_two = storage_backend.save_message("commit-null", "assistant", "same") + + assert second_id != first_id + assert null_two != null_one + assert storage_backend.count_messages("commit-null") == 2 + assert all( + "_commit_key" not in message + for message in storage_backend.load_messages("commit-null", repair=False) + ) + + +def test_commit_key_stays_out_of_public_and_provider_projections( + storage_backend: StorageBackend, +) -> None: + storage_backend.register_workstream("commit-private") + storage_backend.save_message( + "commit-private", "assistant", "visible", commit_key="private-identity" + ) + internal = storage_backend.load_messages("commit-private", repair=False) + + assert internal[0]["_commit_key"] == "private-identity" + assert "_commit_key" not in project_history_messages(internal)[0] + assert "_commit_key" not in sanitize_messages(internal)[0] + assert "_commit_key" not in _serialize_messages(internal)[0] + assert "_commit_key" not in _serialize_messages(internal, include_provider_content=True)[0] + + +def test_hard_delete_then_keyed_retry_refuses_orphan_recreation( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-delete-retry" + storage_backend.register_workstream(ws_id) + storage_backend.save_message(ws_id, "assistant", "accepted", commit_key="one-turn") + assert storage_backend.delete_workstream(ws_id) is True + + with pytest.raises(ConversationCommitWorkstreamGoneError): + storage_backend.save_message(ws_id, "assistant", "accepted", commit_key="one-turn") + # The facade must propagate the typed permanence signal instead of + # swallowing it into the operational-failure 0: the durability journal + # classifies this as terminal, never retrying. + with pytest.raises(ConversationCommitWorkstreamGoneError): + memory.save_message(ws_id, "assistant", "accepted", commit_key="one-turn") + + assert storage_backend.count_messages(ws_id) == 0 + assert storage_backend.list_orphan_conversations() == [] + + +def test_concurrent_keyed_save_and_hard_delete_leave_no_orphan( + storage_backend: StorageBackend, +) -> None: + ws_id = "commit-save-delete-race" + storage_backend.register_workstream(ws_id) + barrier = threading.Barrier(3) + + def _save() -> str: + barrier.wait(timeout=10) + try: + storage_backend.save_message(ws_id, "assistant", "accepted", commit_key="one-turn") + except RuntimeError as exc: + assert "workstream no longer exists" in str(exc) + return "refused" + return "saved" + + def _delete() -> bool: + barrier.wait(timeout=10) + return storage_backend.delete_workstream(ws_id) + + with ThreadPoolExecutor(max_workers=2) as pool: + save = pool.submit(_save) + delete = pool.submit(_delete) + barrier.wait(timeout=10) + assert save.result(timeout=10) in {"saved", "refused"} + assert delete.result(timeout=10) is True + + assert storage_backend.get_workstream(ws_id) is None + assert storage_backend.count_messages(ws_id) == 0 + assert storage_backend.list_orphan_conversations() == [] + + +def test_postgresql_plain_keyed_insert_uses_parent_lock_and_partial_conflict_target() -> None: + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(row=("postgres-plain",)), + ScriptedPostgresResult(scalar_value=81), + ScriptedPostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + assert ( + backend.save_message( + "postgres-plain", + "assistant", + "accepted", + commit_key="plain-key", + ) + == 81 + ) + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" in sql[0] + assert "INSERT INTO conversations" in sql[1] + assert "ON CONFLICT (ws_id, commit_key)" in sql[1] + assert "WHERE commit_key IS NOT NULL" in sql[1] + assert "UPDATE workstreams" in sql[2] + assert conn.commits == 1 + conn.assert_consumed() + + +def test_postgresql_null_key_locks_existing_parent_before_legacy_append() -> None: + """Existing-parent NULL writers share prune's parent-first order.""" + + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(row=("postgres-legacy",)), + ScriptedPostgresResult(row=("postgres-legacy",)), + ScriptedPostgresResult(scalar_value=82), + ScriptedPostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + assert backend.save_message("postgres-legacy", "assistant", "offline") == 82 + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" not in sql[0] + assert "SELECT workstreams.ws_id" in sql[1] and "FOR UPDATE" in sql[1] + assert "INSERT INTO conversations" in sql[2] + assert "UPDATE workstreams" in sql[3] + assert conn.commits == 1 + conn.assert_consumed() + + +def test_postgresql_null_key_preserves_genuinely_parentless_import() -> None: + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(row=None), + ScriptedPostgresResult(row=None), + ScriptedPostgresResult(scalar_value=83), + ScriptedPostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + assert backend.save_message("postgres-parentless", "assistant", "offline") == 83 + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert "FOR UPDATE" not in sql[0] + assert "FOR UPDATE" in sql[1] + assert "INSERT INTO conversations" in sql[2] + assert conn.commits == 1 + + +def test_postgresql_null_key_refuses_parent_deleted_while_waiting_for_lock() -> None: + """A blocked writer cannot wake and masquerade as a parentless import.""" + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(row=("postgres-crossing",)), + ScriptedPostgresResult(row=None), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + with pytest.raises(RuntimeError, match="crossed workstream deletion"): + backend.save_message("postgres-crossing", "assistant", "offline") + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert "FOR UPDATE" not in sql[0] + assert "FOR UPDATE" in sql[1] + assert all("INSERT INTO conversations" not in statement for statement in sql) + assert conn.commits == 0 + + +def test_postgresql_bulk_legacy_writer_locks_existing_parents_in_sorted_order() -> None: + """The batched crossed-deletion gate (round-5 review): one non-locking + IN-list probe, one ORDER BY ws_id FOR UPDATE IN-list lock (the sorted + order that keeps concurrent bulk writers deadlock-free), then the batch + insert and ONE batched updated bump — constant statement count, missing + parents tolerated.""" + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(rows=[("ws-a",)]), + ScriptedPostgresResult(rows=[("ws-a",)]), + ScriptedPostgresResult(), + ScriptedPostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + backend.save_messages_bulk( + [ + {"ws_id": "ws-b", "role": "assistant", "content": "parentless"}, + {"ws_id": "ws-a", "role": "assistant", "content": "attached"}, + ] + ) + + compiled = [statement.compile(dialect=postgresql.dialect()) for statement in conn.statements] + sql = [str(item) for item in compiled] + # Both probes carry the full sorted id set in one expanding IN list. + assert list(compiled[0].params.values()) == [["ws-a", "ws-b"]] + assert list(compiled[1].params.values()) == [["ws-a", "ws-b"]] + assert "FOR UPDATE" not in sql[0] + assert "FOR UPDATE" in sql[1] + assert "ORDER BY workstreams.ws_id" in sql[1] + assert "INSERT INTO conversations" in sql[2] + assert "UPDATE workstreams" in sql[3] + assert conn.commits == 1 diff --git a/tests/test_storage_deferred_create.py b/tests/test_storage_deferred_create.py index f98fff9f..0d513caa 100644 --- a/tests/test_storage_deferred_create.py +++ b/tests/test_storage_deferred_create.py @@ -636,6 +636,16 @@ def test_postgresql_stale_creating_reaper_recovers_tokenless_locked_row( def test_postgresql_retention_prune_excludes_creating_rows() -> None: + """Both candidate discoveries exclude provisional rows, and take no locks. + + Discovery moved out of the deleting transaction when prune became one + bounded transaction per candidate: it is a plain read now, with no + ``FOR UPDATE SKIP LOCKED`` and nothing to commit. That lock rides each + candidate's own transaction instead — see + ``test_postgresql_prune_candidate_locks_rechecks_and_commits_alone`` in + tests/test_storage_prune_commit_races.py. The predicates themselves are + unchanged, which is what this test pins. + """ backend, conn = _scripted_postgres_backend( _UnknownRowcountResult(rows=[]), _UnknownRowcountResult(rows=[]), @@ -644,12 +654,27 @@ def test_postgresql_retention_prune_excludes_creating_rows() -> None: assert backend.prune_workstreams(retention_days=30) == (0, 0) conn.assert_consumed() - assert conn.commits == 1 + assert conn.commits == 0 + orphan_select_sql = str( + conn.statements[0].compile( + dialect=postgresql.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ).lower() stale_select_sql = str( conn.statements[1].compile( dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}, ) ).lower() - assert "workstreams.state" in stale_select_sql - assert "creating" in stale_select_sql + for candidate_sql in (orphan_select_sql, stale_select_sql): + assert "workstreams.state" in candidate_sql + assert "creating" in candidate_sql + assert "for update" not in candidate_sql + assert "not (exists" in orphan_select_sql + # Round-3 review guards: the orphan category excludes named workstreams + # (explicit user intent) and rows younger than the grace (a user + # mid-first-turn whose rows may still be journal-held on another node). + assert "workstreams.alias is null" in orphan_select_sql + assert "workstreams.updated" in orphan_select_sql + assert "workstreams.updated" in stale_select_sql diff --git a/tests/test_storage_prune_commit_races.py b/tests/test_storage_prune_commit_races.py new file mode 100644 index 00000000..e71d73dd --- /dev/null +++ b/tests/test_storage_prune_commit_races.py @@ -0,0 +1,497 @@ +"""Prune ordering and attachment-GC coverage for keyed conversation commits.""" + +from __future__ import annotations + +import threading +from contextlib import nullcontext +from typing import TYPE_CHECKING, Any + +import pytest +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from tests._storage_fakes import ( + ScriptedPostgresConnection, + ScriptedPostgresResult, + make_attachment, + save_keyed, +) +from turnstone.core.storage._postgresql import PostgreSQLBackend +from turnstone.core.storage._schema import workstreams +from turnstone.core.storage._sqlite import SQLiteBackend + +if TYPE_CHECKING: + from turnstone.core.storage import AttachmentWrite + + +def _make_prune_eligible(backend: Any, *ws_ids: str) -> None: + """Backdate ``updated`` past the orphan grace so a just-registered row is + prune-eligible. The grace itself (a fresh empty workstream survives) is + pinned in test_sessions.py; these tests exercise lock ordering and + transaction boundaries, not eligibility.""" + with backend._engine.connect() as conn: + conn.execute( + sa.update(workstreams) + .where(workstreams.c.ws_id.in_(ws_ids)) + .values(updated="2020-01-01T00:00:00") + ) + conn.commit() + + +def _attachment() -> AttachmentWrite: + return make_attachment("d" * 64, b"prune-race-payload", filename="prune.txt") + + +def _save_keyed(backend: Any, ws_id: str, kind: str, attachment: AttachmentWrite) -> int: + return save_keyed( + backend, + ws_id, + kind, + content="accepted" if kind == "plain" else "inspect the evidence", + commit_key=f"prune-{kind}-admission", + attachments=[attachment], + tool_content="Tool result: prune.txt", + tool_call_id="call-prune", + ) + + +@pytest.mark.parametrize("kind", ["plain", "user", "tool"]) +def test_prune_candidate_lock_orders_before_keyed_commit( + storage_backend: Any, + kind: str, +) -> None: + """A commit admitted after prune's exact recheck cannot land behind it. + + Discovery is only a hint on both backends, which deliberately hold no + candidate lock (SQLite: no global writer slot) while discovering. Pause + after the exact predicate recheck that admits deletion. The candidate row + lock (or SQLite ``BEGIN IMMEDIATE`` reservation) must hold the later keyed + commit until prune deletes and commits. The old snapshot-then-bulk-delete + path allowed that commit to finish first and then orphaned its + conversation. + + ``admission_selects`` counts workstream selects up to that recheck and + differs by dialect because the transaction shapes do: SQLite discovers (1) + then rechecks inside its per-candidate writer transaction (2), while + PostgreSQL discovers (1), relocks the candidate row (2), and rechecks on a + fresh READ COMMITTED snapshot (3). + """ + + backend = storage_backend + ws_id = f"prune-keyed-race-{kind}" + attachment = _attachment() + backend.register_workstream(ws_id) + _make_prune_eligible(backend, ws_id) + admission_selects = 3 if isinstance(backend, PostgreSQLBackend) else 2 + prune_selected = threading.Event() + save_attempted = threading.Event() + save_done = threading.Event() + outcomes: dict[str, Any] = {} + errors: list[BaseException] = [] + prune_selects = 0 + + def _before_cursor_execute( + _conn: Any, + _cursor: Any, + _statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if threading.current_thread().name == "keyed-save-after-prune": + save_attempted.set() + + def _after_cursor_execute( + _conn: Any, + _cursor: Any, + _statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + nonlocal prune_selects + if ( + threading.current_thread().name == "paused-prune-candidate" + and "SELECT workstreams.ws_id" in _statement + ): + prune_selects += 1 + if ( + threading.current_thread().name == "paused-prune-candidate" + and prune_selects == admission_selects + and not prune_selected.is_set() + ): + prune_selected.set() + if not save_attempted.wait(timeout=10): + raise AssertionError("keyed save never reached its first database operation") + # On the unsafe implementation the save does not share prune's + # parent lock and completes in this window. The fixed path stays + # blocked until this callback returns and prune commits deletion. + save_done.wait(timeout=0.5) + + def _prune() -> None: + try: + outcomes["prune"] = backend.prune_workstreams(retention_days=90) + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + + def _save() -> None: + try: + _save_keyed(backend, ws_id, kind, attachment) + except RuntimeError as exc: + if "workstream no longer exists" not in str(exc): + errors.append(exc) + outcomes["save"] = "refused" + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + else: + outcomes["save"] = "saved" + finally: + save_done.set() + + sa.event.listen(backend._engine, "before_cursor_execute", _before_cursor_execute) + sa.event.listen(backend._engine, "after_cursor_execute", _after_cursor_execute) + prune_thread = threading.Thread(target=_prune, name="paused-prune-candidate") + save_thread = threading.Thread(target=_save, name="keyed-save-after-prune") + try: + prune_thread.start() + assert prune_selected.wait(timeout=10), "prune never selected its candidate" + save_thread.start() + prune_thread.join(timeout=10) + save_thread.join(timeout=10) + finally: + sa.event.remove(backend._engine, "before_cursor_execute", _before_cursor_execute) + sa.event.remove(backend._engine, "after_cursor_execute", _after_cursor_execute) + + assert not prune_thread.is_alive() + assert not save_thread.is_alive() + assert errors == [] + assert outcomes == {"prune": (1, 0), "save": "refused"} + assert backend.get_workstream(ws_id) is None + assert backend.count_messages(ws_id) == 0 + assert backend.get_attachment(attachment.attachment_id) is None + assert backend.list_orphan_conversations() == [] + + +def test_postgresql_prune_delete_refuses_blocked_null_writer( + storage_backend: Any, +) -> None: + """A NULL writer that saw a parent cannot wake as a parentless import. + + Pause prune after its locked exact predicate recheck, then start the legacy + writer. Its first MVCC observation sees the still-uncommitted parent; its + second ``FOR UPDATE`` must stay blocked until prune deletes and commits. + Waking to a missing locked row is therefore a crossing deletion and must + refuse the insert. + + The recheck is prune's third workstream select since candidates moved to + bounded per-candidate transactions: discovery (unlocked), the candidate's + own ``FOR UPDATE SKIP LOCKED``, then the exact predicate recheck. + """ + backend = storage_backend + if not isinstance(backend, PostgreSQLBackend): + pytest.skip("PostgreSQL row-lock schedule") + + ws_id = "prune-null-crossing" + backend.register_workstream(ws_id) + _make_prune_eligible(backend, ws_id) + prune_selected = threading.Event() + writer_observed_parent = threading.Event() + writer_done = threading.Event() + outcomes: dict[str, Any] = {} + errors: list[BaseException] = [] + prune_selects = 0 + writer_selects = 0 + + def _after_cursor_execute( + _conn: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + nonlocal prune_selects, writer_selects + thread_name = threading.current_thread().name + if thread_name == "blocked-null-writer" and "SELECT workstreams.ws_id" in statement: + writer_selects += 1 + if writer_selects == 1: + writer_observed_parent.set() + if thread_name == "prune-before-null-writer" and "SELECT workstreams.ws_id" in statement: + prune_selects += 1 + if prune_selects == 3 and not prune_selected.is_set(): + prune_selected.set() + if not writer_observed_parent.wait(timeout=10): + raise AssertionError("NULL writer never observed the pre-delete parent") + assert not writer_done.wait(timeout=0.5), ( + "NULL writer crossed prune's parent lock before deletion committed" + ) + + def _prune() -> None: + try: + outcomes["prune"] = backend.prune_workstreams(retention_days=90) + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + + def _save() -> None: + try: + backend.save_message(ws_id, "assistant", "legacy append") + except RuntimeError as exc: + if "crossed workstream deletion" not in str(exc): + errors.append(exc) + outcomes["save"] = "refused" + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + else: + outcomes["save"] = "saved" + finally: + writer_done.set() + + sa.event.listen(backend._engine, "after_cursor_execute", _after_cursor_execute) + prune_thread = threading.Thread(target=_prune, name="prune-before-null-writer") + writer_thread = threading.Thread(target=_save, name="blocked-null-writer") + try: + prune_thread.start() + assert prune_selected.wait(timeout=10), "prune never locked its exact candidate" + writer_thread.start() + prune_thread.join(timeout=10) + writer_thread.join(timeout=10) + finally: + sa.event.remove(backend._engine, "after_cursor_execute", _after_cursor_execute) + + assert not prune_thread.is_alive() + assert not writer_thread.is_alive() + assert errors == [] + assert writer_selects == 2 + assert outcomes == {"prune": (1, 0), "save": "refused"} + assert backend.get_workstream(ws_id) is None + assert backend.count_messages(ws_id) == 0 + + +def _prune_candidate_backend( + conn: ScriptedPostgresConnection, deleted: list[str] +) -> PostgreSQLBackend: + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = lambda: nullcontext(conn) # type: ignore[method-assign] + + def _delete(_conn: Any, ws_id: str) -> bool: + deleted.append(ws_id) + return True + + backend._delete_workstream_on_connection = _delete # type: ignore[method-assign] + return backend + + +def test_postgresql_prune_candidate_locks_rechecks_and_commits_alone() -> None: + """Each PostgreSQL prune candidate owns a bounded, locked transaction. + + Prune used to lock the whole candidate set at discovery and delete inside + one long transaction, so a keyed commit to any candidate waited for the + entire prune to commit. Per candidate now: ``FOR UPDATE SKIP LOCKED`` on + that row alone, then the exact predicate as a separate statement — no lock + clause, so it reads a fresh READ COMMITTED snapshot and sees a commit that + landed while this transaction waited for the row — then one commit. + """ + conn = ScriptedPostgresConnection( + [ScriptedPostgresResult(row=("bounded",)), ScriptedPostgresResult(row=("bounded",))] + ) + deleted: list[str] = [] + backend = _prune_candidate_backend(conn, deleted) + + assert backend._delete_prune_candidate("bounded", (workstreams.c.alias.is_(None),)) is True + + sql = [str(statement.compile(dialect=postgresql.dialect())) for statement in conn.statements] + assert len(sql) == 2 + assert "FOR UPDATE SKIP LOCKED" in sql[0] and "workstreams.alias IS NULL" not in sql[0] + assert "workstreams.alias IS NULL" in sql[1] and "FOR UPDATE" not in sql[1] + assert deleted == ["bounded"] + assert conn.commits == 1 + assert conn.rollbacks == 0 + + +def test_postgresql_prune_candidate_skips_a_row_another_writer_owns() -> None: + """A locked candidate is left to the next prune, unrechecked and undeleted.""" + conn = ScriptedPostgresConnection([ScriptedPostgresResult(row=None)]) + deleted: list[str] = [] + backend = _prune_candidate_backend(conn, deleted) + + assert backend._delete_prune_candidate("locked", (workstreams.c.alias.is_(None),)) is False + + assert len(conn.statements) == 1 + assert deleted == [] + assert conn.commits == 1 + assert conn.rollbacks == 0 + + +def test_postgresql_prune_commits_each_candidate_before_the_next( + storage_backend: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A candidate's deletion is durable before prune reaches the next one. + + PostgreSQL prune used to lock the whole candidate set at discovery and + commit once at the end, so every candidate's row lock — and any keyed + commit waiting on it — was held for the entire run. An independent + connection must be able to observe candidate 1 already gone while prune is + still working. + """ + backend = storage_backend + if not isinstance(backend, PostgreSQLBackend): + pytest.skip("PostgreSQL per-candidate transaction boundary") + + backend.register_workstream("prune-bounded-a") + backend.register_workstream("prune-bounded-b") + _make_prune_eligible(backend, "prune-bounded-a", "prune-bounded-b") + real_delete_candidate = backend._delete_prune_candidate + observed: list[str | None] = [] + + def _gated_delete_candidate(ws_id: str, predicates: tuple[Any, ...]) -> bool: + deleted = real_delete_candidate(ws_id, predicates) + if deleted and len(observed) == 0: + # Read on a fresh connection: an uncommitted delete would still + # show the row here. + observed.append(backend.get_workstream(ws_id)) + return deleted + + monkeypatch.setattr(backend, "_delete_prune_candidate", _gated_delete_candidate) + + assert backend.prune_workstreams(retention_days=90) == (2, 0) + assert observed == [None] + assert backend.get_workstream("prune-bounded-b") is None + + +def test_sqlite_prune_releases_writer_slot_between_candidates( + storage_backend: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Unrelated live commits do not wait for the complete prune candidate set.""" + + backend = storage_backend + if not isinstance(backend, SQLiteBackend): + pytest.skip("SQLite's database-wide writer slot is dialect-specific") + + backend.register_workstream("prune-bounded-a") + backend.register_workstream("prune-bounded-b") + _make_prune_eligible(backend, "prune-bounded-a", "prune-bounded-b") + backend.register_workstream("prune-unrelated-live") + backend.save_message("prune-unrelated-live", "user", "durable prefix") + + between_candidates = threading.Event() + writer_done = threading.Event() + writer_completed_between = threading.Event() + outcomes: dict[str, Any] = {} + errors: list[BaseException] = [] + real_delete_candidate = backend._delete_prune_candidate + deleted_candidates = 0 + + def _gated_delete_candidate(ws_id: str, predicates: tuple[Any, ...]) -> bool: + nonlocal deleted_candidates + deleted = real_delete_candidate(ws_id, predicates) + if deleted: + deleted_candidates += 1 + if deleted_candidates == 1: + # The helper has committed and returned: no SQLite writer + # reservation may remain while Python advances to candidate 2. + between_candidates.set() + if writer_done.wait(timeout=5): + writer_completed_between.set() + else: + raise AssertionError("unrelated writer stayed blocked between candidates") + return deleted + + monkeypatch.setattr(backend, "_delete_prune_candidate", _gated_delete_candidate) + + def _prune() -> None: + try: + outcomes["prune"] = backend.prune_workstreams(retention_days=90) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + + def _write() -> None: + try: + outcomes["row"] = backend.save_message( + "prune-unrelated-live", + "assistant", + "committed between prune candidates", + commit_key="prune-unrelated-key", + ) + except BaseException as exc: # pragma: no cover - diagnostic capture + errors.append(exc) + finally: + writer_done.set() + + prune_thread = threading.Thread(target=_prune, name="bounded-prune") + writer_thread = threading.Thread(target=_write, name="unrelated-prune-writer") + prune_thread.start() + assert between_candidates.wait(timeout=10), "prune never completed its first candidate" + writer_thread.start() + writer_thread.join(timeout=10) + prune_thread.join(timeout=10) + + assert not writer_thread.is_alive() + assert not prune_thread.is_alive() + assert errors == [] + assert writer_completed_between.is_set() + assert outcomes["prune"] == (2, 0) + assert int(outcomes["row"]) > 0 + assert backend.count_messages("prune-unrelated-live") == 2 + + +@pytest.mark.parametrize("kind", ["user", "tool"]) +def test_stale_prune_releases_exact_attachment_reference_counts( + storage_backend: Any, + kind: str, +) -> None: + backend = storage_backend + stale_ws_id = f"prune-stale-attachment-{kind}" + survivor_ws_id = f"prune-surviving-attachment-{kind}" + attachment = _attachment() + backend.register_workstream(stale_ws_id) + backend.register_workstream(survivor_ws_id) + + if kind == "user": + backend.save_user_message_with_attachments( + stale_ws_id, + "stale", + [attachment, attachment], + commit_key="stale-user", + ) + backend.save_user_message_with_attachments( + survivor_ws_id, + "survives", + [attachment], + commit_key="surviving-user", + ) + else: + backend.save_tool_message_with_attachments( + stale_ws_id, + "stale", + "read_file", + "call-stale", + [attachment, attachment], + commit_key="stale-tool", + ) + backend.save_tool_message_with_attachments( + survivor_ws_id, + "survives", + "read_file", + "call-surviving", + [attachment], + commit_key="surviving-tool", + ) + + assert backend.get_attachment(attachment.attachment_id)["refcount"] == 3 + with backend._engine.connect() as conn: + conn.execute( + sa.update(workstreams) + .where(workstreams.c.ws_id == stale_ws_id) + .values(updated="2020-01-01T00:00:00") + ) + conn.commit() + + assert backend.prune_workstreams(retention_days=30) == (0, 1) + + assert backend.get_workstream(stale_ws_id) is None + assert backend.count_messages(stale_ws_id) == 0 + assert backend.count_messages(survivor_ws_id) == 1 + assert backend.get_attachment(attachment.attachment_id)["refcount"] == 1 + assert backend.list_orphan_conversations() == [] diff --git a/tests/test_storage_sqlite.py b/tests/test_storage_sqlite.py index 9c681465..00c2f9fd 100644 --- a/tests/test_storage_sqlite.py +++ b/tests/test_storage_sqlite.py @@ -598,7 +598,16 @@ class TestDeleteWorkstream: class TestPruneWorkstreams: def test_orphan_removed(self, backend): + import sqlalchemy as sa + backend.register_workstream("orphan") + # Age past the orphan grace; a fresh empty row is deliberately kept + # (round-3 review) — pinned in test_sessions.py. + with backend._engine.connect() as conn: + conn.execute( + sa.text("UPDATE workstreams SET updated = '2020-01-01' WHERE ws_id = 'orphan'") + ) + conn.commit() orphans, stale = backend.prune_workstreams() assert orphans == 1 diff --git a/tests/test_storage_truncation_commit_races.py b/tests/test_storage_truncation_commit_races.py new file mode 100644 index 00000000..b269f0ad --- /dev/null +++ b/tests/test_storage_truncation_commit_races.py @@ -0,0 +1,387 @@ +"""Tail-truncation ordering and attachment-GC regression coverage.""" + +from __future__ import annotations + +import json +import threading +from contextlib import nullcontext +from typing import TYPE_CHECKING, Any, cast + +import pytest +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from tests._storage_fakes import ( + ScriptedPostgresConnection, + ScriptedPostgresResult, + make_attachment, + save_keyed, +) +from turnstone.core.storage._postgresql import PostgreSQLBackend + +if TYPE_CHECKING: + from turnstone.core.storage import AttachmentWrite + + +def _attachment(attachment_id: str, content: bytes) -> AttachmentWrite: + return make_attachment(attachment_id, content) + + +def _save_keyed( + backend: Any, + ws_id: str, + kind: str, + shared: AttachmentWrite, + added: AttachmentWrite, +) -> int: + return save_keyed( + backend, + ws_id, + kind, + content="accepted after truncation", + commit_key=f"truncate-race-{kind}", + attachments=[shared, added, added], + tool_call_id="call-truncate-race", + ) + + +@pytest.mark.parametrize("operation", ["keep_count", "remove_count"]) +@pytest.mark.parametrize("kind", ["plain", "user", "tool"]) +def test_storage_tail_truncation_orders_before_keyed_commit_and_releases_exact_refs( + storage_backend: Any, + kind: str, + operation: str, +) -> None: + """An ACKed keyed commit cannot disappear behind tail truncation. + + Pause legacy truncation after its cutoff or strict truncation after its + in-transaction total. The transaction must already own the SQLite writer + reservation or PostgreSQL parent lock, so a later keyed writer cannot + finish until the doomed row and its exact attachment references have been + removed and committed. + """ + + backend = storage_backend + ws_id = f"truncate-keyed-race-{operation}-{kind}" + shared = _attachment("a" * 64, b"shared") + doomed = _attachment("b" * 64, b"doomed") + added = _attachment("c" * 64, b"added") + backend.register_workstream(ws_id) + backend.save_tool_message_with_attachments( + ws_id, + "keep", + "read_file", + "call-keep", + [shared], + commit_key="truncate-keep", + ) + backend.save_tool_message_with_attachments( + ws_id, + "remove", + "read_file", + "call-remove", + [shared, doomed, doomed], + commit_key="truncate-remove", + ) + assert backend.get_attachment(shared.attachment_id)["refcount"] == 2 + assert backend.get_attachment(doomed.attachment_id)["refcount"] == 2 + + # Exercise SQLite's supported no-FTS path so this test also catches a + # deferred read-to-write upgrade; FTS happens to acquire a writer lock as a + # side effect, but it is not the truncation transaction's ordering contract. + if backend.__class__.__name__ == "SQLiteBackend": + backend._fts5_available = False + + boundary_selected = threading.Event() + save_attempted = threading.Event() + save_done = threading.Event() + outcomes: dict[str, Any] = {} + errors: list[BaseException] = [] + + def _is_pause_select(statement: str) -> bool: + sql = " ".join(statement.lower().split()) + if operation == "keep_count": + return ( + sql.startswith("select conversations.id") + and "order by conversations.id" in sql + and " offset " in sql + ) + return sql.startswith("select count(*)") and "from conversations" in sql + + def _before_cursor_execute( + _conn: Any, + _cursor: Any, + _statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if threading.current_thread().name == "keyed-save-after-truncation": + save_attempted.set() + + def _after_cursor_execute( + _conn: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if ( + threading.current_thread().name == "paused-tail-truncation" + and not boundary_selected.is_set() + and _is_pause_select(statement) + ): + boundary_selected.set() + if not save_attempted.wait(timeout=10): + raise AssertionError("keyed save never reached its first database operation") + # An unsafe implementation lets the save commit in this window; + # the later id-range DELETE then removes an already-ACKed row. + outcomes["save_crossed_boundary"] = save_done.wait(timeout=0.5) + + def _truncate() -> None: + try: + if operation == "keep_count": + outcomes["deleted"] = backend.delete_messages_after(ws_id, 1) + else: + outcomes["deleted"] = backend.truncate_messages_tail(ws_id, 1) + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + + def _save() -> None: + try: + outcomes["saved_id"] = _save_keyed(backend, ws_id, kind, shared, added) + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + finally: + save_done.set() + + sa.event.listen(backend._engine, "before_cursor_execute", _before_cursor_execute) + sa.event.listen(backend._engine, "after_cursor_execute", _after_cursor_execute) + truncate_thread = threading.Thread(target=_truncate, name="paused-tail-truncation") + save_thread = threading.Thread(target=_save, name="keyed-save-after-truncation") + try: + truncate_thread.start() + assert boundary_selected.wait(timeout=10), "truncation never reached its read boundary" + save_thread.start() + truncate_thread.join(timeout=10) + save_thread.join(timeout=10) + finally: + sa.event.remove(backend._engine, "before_cursor_execute", _before_cursor_execute) + sa.event.remove(backend._engine, "after_cursor_execute", _after_cursor_execute) + + assert not truncate_thread.is_alive() + assert not save_thread.is_alive() + assert errors == [] + assert outcomes["save_crossed_boundary"] is False + assert outcomes["deleted"] == 1 + assert isinstance(outcomes["saved_id"], int) + + messages = backend.load_messages(ws_id, repair=False) + assert [message["_commit_key"] for message in messages] == [ + "truncate-keep", + f"truncate-race-{kind}", + ] + assert backend.count_messages(ws_id) == 2 + expected_shared_refs = 1 if kind == "plain" else 2 + assert backend.get_attachment(shared.attachment_id)["refcount"] == expected_shared_refs + assert backend.get_attachment(doomed.attachment_id) is None + if kind == "plain": + assert backend.get_attachment(added.attachment_id) is None + else: + assert backend.get_attachment(added.attachment_id)["refcount"] == 2 + assert backend.list_orphan_conversations() == [] + + +def test_atomic_tail_truncation_clamps_at_latest_compaction_marker( + storage_backend: Any, +) -> None: + backend = storage_backend + ws_id = "truncate-compaction-floor" + backend.register_workstream(ws_id) + for index in range(3): + backend.save_message(ws_id, "user", f"prefix-{index}") + watermark = backend.get_compaction_watermark(ws_id, 0) + backend.save_message( + ws_id, + "assistant", + "summary", + source="compaction", + meta=json.dumps({"watermark": watermark}), + ) + backend.save_message(ws_id, "user", "tail-1") + backend.save_message(ws_id, "assistant", "tail-2") + assert backend.count_messages(ws_id) == 6 + assert backend.get_compaction_floor(ws_id) == 4 + + assert backend.truncate_messages_tail(ws_id, 1) == 1 + assert backend.count_messages(ws_id) == 5 + assert backend.truncate_messages_tail(ws_id, 100) == 1 + assert backend.truncate_messages_tail(ws_id, 1) == 0 + + rows = backend.load_messages( + ws_id, + repair=False, + include_compaction=True, + ) + assert [row["content"] for row in rows] == [ + "prefix-0", + "prefix-1", + "prefix-2", + "summary", + ] + assert backend.get_compaction_floor(ws_id) == 4 + + +def test_atomic_tail_truncation_requires_parent_and_valid_count( + storage_backend: Any, +) -> None: + backend = storage_backend + backend.register_workstream("truncate-strict-input") + backend.save_message("truncate-strict-input", "user", "keep") + + with pytest.raises(ValueError, match="non-negative"): + backend.truncate_messages_tail("truncate-strict-input", -1) + assert backend.truncate_messages_tail("truncate-strict-input", 0) == 0 + assert backend.count_messages("truncate-strict-input") == 1 + + # Legacy unkeyed storage can contain an orphan row. The strict operation + # must refuse it rather than mutating history without a durable lock target. + backend.save_message("truncate-orphan", "assistant", "orphan") + with pytest.raises(RuntimeError, match="workstream no longer exists"): + backend.truncate_messages_tail("truncate-orphan", 1) + assert backend.count_messages("truncate-orphan") == 1 + + +def test_atomic_tail_truncation_rolls_back_rows_and_refs_on_release_failure( + storage_backend: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = storage_backend + ws_id = "truncate-release-rollback" + shared = _attachment("f" * 64, b"shared-rollback") + doomed = _attachment("0" * 64, b"doomed-rollback") + backend.register_workstream(ws_id) + backend.save_tool_message_with_attachments( + ws_id, + "keep", + "read_file", + "call-rollback-keep", + [shared], + commit_key="rollback-keep", + ) + backend.save_tool_message_with_attachments( + ws_id, + "remove", + "read_file", + "call-rollback-remove", + [shared, doomed, doomed], + commit_key="rollback-remove", + ) + + # The tail-delete body (and its release call) lives in the shared _utils + # core since the round-4 dedup — patch where the call resolves. + from turnstone.core.storage import _utils as utils_module + + real_release = utils_module.release_attachment_refs + + def _release_then_fail(conn: Any, attachment_ids: list[str]) -> None: + real_release(conn, attachment_ids) + raise RuntimeError("injected truncation release failure") + + monkeypatch.setattr(utils_module, "release_attachment_refs", _release_then_fail) + with pytest.raises(RuntimeError, match="injected truncation release failure"): + backend.truncate_messages_tail(ws_id, 1) + + assert backend.count_messages(ws_id) == 2 + assert backend.get_attachment(shared.attachment_id)["refcount"] == 2 + assert backend.get_attachment(doomed.attachment_id)["refcount"] == 2 + + +def test_postgresql_tail_truncation_locks_parent_and_releases_returned_refs() -> None: + first = "d" * 64 + repeated = "e" * 64 + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(row=("postgres-truncate",)), + ScriptedPostgresResult(row=(42,)), + ScriptedPostgresResult(rows=[(json.dumps([first, repeated, repeated]),), (None,)]), + ScriptedPostgresResult(), + ScriptedPostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = cast("Any", lambda: nullcontext(conn)) # type: ignore[method-assign] + + assert backend.delete_messages_after("postgres-truncate", 3) == 2 + + dialect = postgresql.dialect() # type: ignore[no-untyped-call] + compiled = [statement.compile(dialect=dialect) for statement in conn.statements] + sql = [" ".join(str(statement).split()) for statement in compiled] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" in sql[0] + assert "SELECT conversations.id" in sql[1] and "OFFSET" in sql[1] + assert "DELETE FROM conversations" in sql[2] + assert "RETURNING conversations.attachments" in sql[2] + assert all("SELECT conversations.attachments" not in statement for statement in sql) + assert "UPDATE workstream_attachments" in sql[3] + assert "DELETE FROM workstream_attachments" in sql[4] + assert conn.commits == 1 + assert conn.rollbacks == 0 + assert conn._results == [] + + +def test_postgresql_atomic_tail_truncation_computes_floor_under_parent_lock() -> None: + first = "1" * 64 + repeated = "2" * 64 + conn = ScriptedPostgresConnection( + [ + ScriptedPostgresResult(row=("postgres-atomic-truncate",)), + ScriptedPostgresResult(scalar_value=6), + ScriptedPostgresResult(scalar_value=14), + ScriptedPostgresResult(scalar_value=4), + ScriptedPostgresResult(row=(15,)), + ScriptedPostgresResult(rows=[(json.dumps([first, repeated, repeated]),), (None,)]), + ScriptedPostgresResult(), + ScriptedPostgresResult(), + ] + ) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = cast("Any", lambda: nullcontext(conn)) # type: ignore[method-assign] + + assert backend.truncate_messages_tail("postgres-atomic-truncate", 2) == 2 + + dialect = postgresql.dialect() # type: ignore[no-untyped-call] + assert str(conn.statements[0]).startswith("SET LOCAL lock_timeout") + compiled = [statement.compile(dialect=dialect) for statement in conn.statements[1:]] + sql = [" ".join(str(statement).split()) for statement in compiled] + assert "SELECT workstreams.ws_id" in sql[0] and "FOR UPDATE" in sql[0] + assert "count(*)" in sql[1] and "FROM conversations" in sql[1] + assert "max(conversations.id)" in sql[2] + assert "count(*)" in sql[3] and "conversations.id <=" in sql[3] + assert "SELECT conversations.id" in sql[4] and "OFFSET" in sql[4] + assert 4 in compiled[4].params.values() + assert "DELETE FROM conversations" in sql[5] + assert "RETURNING conversations.attachments" in sql[5] + assert all("SELECT conversations.attachments" not in statement for statement in sql) + assert "UPDATE workstream_attachments" in sql[6] + assert "DELETE FROM workstream_attachments" in sql[7] + assert conn.commits == 1 + assert conn.rollbacks == 0 + assert conn._results == [] + + +def test_postgresql_atomic_tail_truncation_rolls_back_when_parent_is_missing() -> None: + conn = ScriptedPostgresConnection([ScriptedPostgresResult(row=None)]) + backend = PostgreSQLBackend.__new__(PostgreSQLBackend) + backend._conn = cast("Any", lambda: nullcontext(conn)) # type: ignore[method-assign] + + with pytest.raises(RuntimeError, match="workstream no longer exists"): + backend.truncate_messages_tail("postgres-missing", 1) + + dialect = postgresql.dialect() # type: ignore[no-untyped-call] + assert str(conn.statements[0]).startswith("SET LOCAL lock_timeout") + sql = " ".join(str(conn.statements[1].compile(dialect=dialect)).split()) + assert "SELECT workstreams.ws_id" in sql and "FOR UPDATE" in sql + assert conn.commits == 0 + assert conn.rollbacks == 1 + assert conn._results == [] diff --git a/tests/test_turn_provenance.py b/tests/test_turn_provenance.py new file mode 100644 index 00000000..fe1f8cca --- /dev/null +++ b/tests/test_turn_provenance.py @@ -0,0 +1,836 @@ +"""Accepted model-turn provenance across dispatch, storage, and projections.""" + +from __future__ import annotations + +import dataclasses +import json +import logging +import re +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from tests._session_helpers import ( + RecordingUI, + arm_session, + make_session, + replace_session_lane, + scripted_chat_client, + seam_provider, +) +from turnstone.console.coordinator_client import _serialize_messages +from turnstone.core.export import export_workstream +from turnstone.core.history_decoration import project_history_messages +from turnstone.core.model_registry import ModelConfig, ModelRegistry +from turnstone.core.model_turn import ModelLane, model_turn +from turnstone.core.providers import ModelCapabilities, StreamChunk, UsageInfo +from turnstone.core.session import ConversationPersistenceError, GenerationCancelled +from turnstone.core.storage._utils import _fork_turn_insert_row +from turnstone.core.trajectory import ( + PROVENANCE_META_KEY, + EffectStatus, + Turn, + TurnProvenance, + turn_from_dict, + turn_to_dict, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from turnstone.core.storage import StorageBackend + + +def _good_stream(text: str) -> list[StreamChunk]: + return [ + StreamChunk(content_delta=text), + StreamChunk( + finish_reason="stop", + usage=UsageInfo(prompt_tokens=10, completion_tokens=2, total_tokens=12), + ), + ] + + +def _dying_stream(text: str) -> Iterator[StreamChunk]: + yield StreamChunk(content_delta=text) + raise httpx.ReadError("old binding died") + + +def _provenance(turn: Turn) -> dict[str, str | int]: + raw = turn.meta.extra.get(PROVENANCE_META_KEY) + assert isinstance(raw, dict) + return raw + + +def _log_has_field(record: logging.LogRecord, key: str, value: str | int) -> bool: + """Accept either the console or JSON/dict structlog renderer. + + Logging configuration is process-global, so a full-suite predecessor may + select a different renderer than this file sees in isolation — including + the colored console renderer, whose ANSI escapes would otherwise split + ``key=value``. The event fields are the contract; their presentation + (renderer AND styling) is not. + """ + message = re.sub(r"\x1b\[[0-9;]*m", "", record.getMessage()) + return any( + candidate in message + for candidate in ( + f"{key}={value}", + f"'{key}': {value!r}", + f'"{key}": {json.dumps(value)}', + ) + ) + + +def _register_session_parent(session: Any) -> None: + """Mirror production's parent-before-keyed-conversation ordering.""" + from turnstone.core.storage import get_storage + + storage = get_storage() + assert storage is not None + storage.register_workstream(session.ws_id, user_id=session._user_id) + + +def test_model_turn_stamps_one_immutable_serving_identity() -> None: + provider = seam_provider("accepted") + lane = ModelLane( + provider=provider, + client=MagicMock(), + model="backend-v2", + alias="assistant-fast", + registry_generation=17, + capabilities=ModelCapabilities(), + ) + + result = model_turn( + lane, + [Turn.user("hello")], + acting_principal_id="user-alice", + ) + + expected = { + "model_alias": "assistant-fast", + "backend_model_id": "backend-v2", + "registry_generation": 17, + "acting_principal_id": "user-alice", + } + assert result.provenance.to_meta() == expected + assert _provenance(result.turn) == expected + + +def test_model_turn_drain_retry_logs_kernel_axes_without_principal( + caplog: pytest.LogCaptureFixture, +) -> None: + provider = seam_provider("unused") + provider.create_streaming.side_effect = [ + _dying_stream("discarded"), + _good_stream("accepted"), + ] + lane = ModelLane( + provider=provider, + client=MagicMock(), + model="retry-kernel", + alias="retry-alias", + registry_generation=12, + capabilities=ModelCapabilities(), + ) + + with ( + patch("turnstone.core.model_turn.time.sleep"), + caplog.at_level(logging.WARNING, logger="turnstone.core.model_turn"), + ): + result = model_turn( + lane, + [Turn.user("hello")], + acting_principal_id="private-principal-id", + ) + + assert result.content == "accepted" + [retry] = [ + record for record in caplog.records if "model_turn.drain_retry" in record.getMessage() + ] + assert _log_has_field(retry, "alias", "retry-alias") + assert _log_has_field(retry, "model", "retry-kernel") + assert _log_has_field(retry, "registry_generation", 12) + assert "private-principal-id" not in retry.getMessage() + assert "acting_principal_id" not in retry.getMessage() + + +def test_creation_fallback_stamps_fallback_binding_and_principal(tmp_db: str) -> None: + registry = ModelRegistry( + models={ + "primary": ModelConfig( + "primary", "http://primary/v1", "key", "primary-id", provider="openai-compatible" + ), + "fallback": ModelConfig( + "fallback", + "http://fallback/v1", + "key", + "fallback-id", + provider="openai-compatible", + ), + }, + default="primary", + fallback=["fallback"], + ) + session = make_session( + registry=registry, + model_alias="primary", + user_id="owner", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + _register_session_parent(session) + session._title_generated = True + session._primary_lane().client.chat.completions.create = MagicMock( + side_effect=ConnectionError("primary unavailable") + ) + registry.get_client("fallback").chat.completions.create = scripted_chat_client( + {"content": "served by fallback"} + ) + + session.send("hello", acting_user_id="user-alice") + + assert _provenance(session.messages[-1]) == { + "model_alias": "fallback", + "backend_model_id": "fallback-id", + "registry_generation": registry.generation, + "acting_principal_id": "user-alice", + } + + +def test_midstream_rebind_stamps_only_the_successful_replacement( + tmp_db: str, + caplog: pytest.LogCaptureFixture, +) -> None: + session = make_session( + model_alias="primary", + registry_generation=3, + user_id="owner", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + _register_session_parent(session) + provider = arm_session( + session, + _dying_stream("discarded"), + _good_stream("accepted"), + ) + refresh_count = 0 + + def _refresh() -> None: + nonlocal refresh_count + refresh_count += 1 + if refresh_count != 2: + return + replace_session_lane(session, model="replacement-id") + session._model_binding = dataclasses.replace( + session._model_binding, + registry_generation=9, + ) + + with ( + patch.object(session, "_refresh_model_from_registry", side_effect=_refresh), + caplog.at_level(logging.DEBUG, logger="turnstone.core.session"), + ): + session.send("hello", acting_user_id="private-principal-id") + + assert provider.create_streaming.call_count == 2 + assert _provenance(session.messages[-1]) == { + "model_alias": "primary", + "backend_model_id": "replacement-id", + "registry_generation": 9, + "acting_principal_id": "private-principal-id", + } + [retry] = [record for record in caplog.records if "stream.retry" in record.getMessage()] + assert _log_has_field(retry, "alias", "primary") + assert _log_has_field(retry, "model", "test-model") + assert _log_has_field(retry, "registry_generation", 3) + assert "private-principal-id" not in retry.getMessage() + assert "acting_principal_id" not in retry.getMessage() + + [finished] = [record for record in caplog.records if "stream.finished" in record.getMessage()] + assert _log_has_field(finished, "alias", "primary") + assert _log_has_field(finished, "model", "replacement-id") + assert _log_has_field(finished, "registry_generation", 9) + assert "private-principal-id" not in finished.getMessage() + assert "acting_principal_id" not in finished.getMessage() + + +def test_headless_send_stamps_effective_owner_principal(tmp_db: str) -> None: + """Scheduled/internal sends record the credential principal they use.""" + session = make_session( + model_alias="headless", + registry_generation=6, + user_id="owner-principal", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + _register_session_parent(session) + arm_session(session, _good_stream("accepted")) + + session.send("scheduled work") + + assert _provenance(session.messages[-1]) == { + "model_alias": "headless", + "backend_model_id": "test-model", + "registry_generation": 6, + "acting_principal_id": "owner-principal", + } + + +def test_shared_workstream_rebind_cannot_relabel_inflight_turn(tmp_db: str) -> None: + session = make_session( + model_alias="shared", + registry_generation=4, + user_id="owner", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + _register_session_parent(session) + + def _stream() -> Iterator[StreamChunk]: + # A second browser binds a new actor while Alice's response is in + # flight. The accepted result must retain the generation principal. + session.bind_acting_user("user-bob") + yield from _good_stream("alice's answer") + + arm_session(session, _stream()) + session.send("alice's question", acting_user_id="user-alice") + + assert session._acting_user_id == "user-bob" + assert _provenance(session.messages[-1])["acting_principal_id"] == "user-alice" + + +def test_tool_rows_record_the_same_principal_as_their_assistant_turn(tmp_db: str) -> None: + """Turn identity is symmetric across the roles one generation persists. + + The assistant row carries the principal on its four-axis provenance + envelope; the TOOL rows its batch produced carry the same identity as a + plain sibling of their disposition — not a second four-axis tuple, since a + tool row is an effect receipt and its kernel axes would be empty. Both + read the generation's bound principal, so revocation can query tool rows + directly instead of joining each one back to its batch head. + """ + session = make_session( + model_alias="main", + registry_generation=5, + user_id="owner", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + _register_session_parent(session) + session._title_generated = True + session._primary_lane().client.chat.completions.create = scripted_chat_client( + { + "tool_calls": [{"id": "call-audit", "name": "read_only_probe", "arguments": "{}"}], + "finish_reason": "tool_calls", + }, + {"content": "done"}, + ) + tool_meta: dict[str, str | None] = {} + + def _record(_ws_id: str, role: str, _content: str, *_args: Any, **kwargs: Any) -> int: + if role == "tool": + tool_meta[str(kwargs.get("tool_call_id") or "")] = kwargs.get("meta") + return 1 + + with ( + patch.object( + session, + "_execute_tools", + return_value=([("call-audit", "observed output")], None), + ), + patch("turnstone.core.session.save_message", side_effect=_record), + ): + session.send("run the probe", acting_user_id="user-alice") + + assert _provenance(session.messages[-1])["acting_principal_id"] == "user-alice" + assert json.loads(tool_meta["call-audit"]) == {"acting_principal": "user-alice"} + + +def test_utility_completion_stamps_snapshotted_acting_principal(tmp_db: str) -> None: + """History-visible utility output can never inherit a later actor rebind.""" + session = make_session(user_id="owner") + session.bind_acting_user("user-alice") + provider = seam_provider("summary") + lane = replace_session_lane( + session, + provider=provider, + model="summary-kernel", + alias="summary-alias", + ) + + result = session._utility_completion([Turn.user("summarize")], lane=lane) + + assert result.provenance == TurnProvenance( + model_alias="summary-alias", + backend_model_id="summary-kernel", + registry_generation=lane.registry_generation, + acting_principal_id="user-alice", + ) + + +def test_token_calibration_isolated_across_primary_fallback_primary() -> None: + """A -> B -> A restores A's ratio and A's own prompt-count anchor.""" + session = make_session(model_alias="primary", registry_generation=7) + primary = session._primary_lane() + fallback = dataclasses.replace( + primary, + alias="fallback", + model="fallback-kernel", + registry_generation=11, + ) + session.messages = [Turn.user("first")] + session._msg_tokens = [1] + + session._activate_token_calibration(primary) + session._last_usage = { + "prompt_tokens": 50, + "completion_tokens": 5, + "total_tokens": 55, + } + session._update_token_table( + msgs=[{"role": "user", "content": "a" * 246}], + tool_def_chars=0, + provenance=TurnProvenance( + model_alias=primary.alias, + backend_model_id=primary.model, + registry_generation=primary.registry_generation, + ), + ) + primary_ratio = session._chars_per_token + assert primary_ratio == 5.0 + + session.messages.extend([Turn.assistant("primary answer"), Turn.user("second")]) + session._msg_tokens.extend([5, 2]) + session._activate_token_calibration(fallback) + session._last_usage = { + "prompt_tokens": 25, + "completion_tokens": 7, + "total_tokens": 32, + } + session._update_token_table( + msgs=[{"role": "user", "content": "b" * 46}], + tool_def_chars=0, + provenance=TurnProvenance( + model_alias=fallback.alias, + backend_model_id=fallback.model, + registry_generation=fallback.registry_generation, + ), + ) + assert session._chars_per_token == 2.0 + + session.messages.append(Turn.assistant("fallback answer")) + session._msg_tokens.append(7) + session._activate_token_calibration(primary) + + assert session._chars_per_token == primary_ratio + expected = 50 + sum(session._msg_tokens[1:]) + assert session._estimated_prompt_tokens() == expected + assert session._estimated_prompt_tokens() != 25 + sum(session._msg_tokens[3:]) + + +def test_cancelled_partial_stamps_the_armed_fallback_lane_and_principal( + tmp_db: str, +) -> None: + """A partial accepted on Stop is an assistant turn, not unattributed UI.""" + session = make_session( + model_alias="primary", + registry_generation=3, + user_id="owner", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + _register_session_parent(session) + fallback_provider = seam_provider("unused", provider_name="fallback-provider") + fallback_lane = ModelLane( + provider=fallback_provider, + client=MagicMock(), + model="fallback-kernel", + alias="fallback-alias", + registry_generation=13, + capabilities=ModelCapabilities(), + ) + + def _cancel_from_fallback(consumer, *_args, **_kwargs): + consumer.begin_attempt(MagicMock(armed=True), None, fallback_lane) + consumer(StreamChunk(content_delta="partial answer")) + session.cancel() + raise GenerationCancelled() + + session._title_generated = True + with patch.object( + session, + "_model_turn_with_fallback", + side_effect=_cancel_from_fallback, + ): + session.send("hello", acting_user_id="user-alice") + + assert _provenance(session.messages[-1]) == { + "model_alias": "fallback-alias", + "backend_model_id": "fallback-kernel", + "registry_generation": 13, + "acting_principal_id": "user-alice", + } + from turnstone.core.storage import get_storage + + storage = get_storage() + assert storage is not None + durable = storage.load_message_turns(session.ws_id, checkpointed=False) + assert _provenance(durable[-1]) == _provenance(session.messages[-1]) + + +def test_provenance_dict_bridge_is_strict_and_lossless() -> None: + raw = { + "model_alias": "main", + "backend_model_id": "kernel", + "registry_generation": 8, + "acting_principal_id": "alice", + } + msg = {"role": "assistant", "content": "ok", "_provenance": raw} + turn = turn_from_dict(msg) + assert _provenance(turn) == raw + assert turn_to_dict(turn) == msg + + torn = turn_from_dict( + { + "role": "assistant", + "content": "bad", + "_provenance": {**raw, "registry_generation": "8"}, + } + ) + assert PROVENANCE_META_KEY not in torn.meta.extra + assert "_provenance" not in turn_to_dict(torn) + + +def test_storage_rehydrates_and_fork_preserves_assistant_provenance( + storage_backend: StorageBackend, +) -> None: + raw = { + "model_alias": "main", + "backend_model_id": "kernel", + "registry_generation": 8, + "acting_principal_id": "alice", + } + ws_id = "provenance-storage-roundtrip" + storage_backend.save_message( + ws_id, + "assistant", + "accepted", + meta=json.dumps({PROVENANCE_META_KEY: raw}), + ) + + turns = storage_backend.load_message_turns(ws_id, checkpointed=False) + assert len(turns) == 1 + assert _provenance(turns[0]) == raw + loaded = storage_backend.load_messages(ws_id, repair=False) + assert loaded[0]["_provenance"] == raw + + insert_row, attachment_ids = _fork_turn_insert_row( + turns[0], + "provenance-fork-destination", + "2026-08-09T00:00:00", + ) + assert attachment_ids == [] + assert json.loads(insert_row["meta"])[PROVENANCE_META_KEY] == raw + + +def test_storage_rehydrates_and_fork_preserves_tool_acting_principal( + storage_backend: StorageBackend, +) -> None: + """A TOOL row answers "whose turn ran this effect" without a join. + + Revocation and audit ask that question per effect. Deriving it from the + assistant row that opened the batch is a join that breaks exactly where it + matters — a cancelled batch whose receipts are synthesized separately — so + the row carries the identity itself, as a sibling of its disposition and + never as a fabricated four-axis model-attempt envelope. + """ + ws_id = "tool-principal-storage-roundtrip" + storage_backend.save_message( + ws_id, + "tool", + "probe output", + "read_only_probe", + tool_call_id="call-audit", + meta=json.dumps({"effect_status": "committed", "acting_principal": "user-alice"}), + ) + + turns = storage_backend.load_message_turns(ws_id, checkpointed=False) + assert len(turns) == 1 + assert turns[0].effect_status is EffectStatus.COMMITTED + assert turns[0].meta.extra["acting_principal"] == "user-alice" + assert PROVENANCE_META_KEY not in turns[0].meta.extra + + insert_row, attachment_ids = _fork_turn_insert_row( + turns[0], + "tool-principal-fork-destination", + "2026-08-10T00:00:00", + ) + assert attachment_ids == [] + assert json.loads(insert_row["meta"]) == { + "effect_status": "committed", + "acting_principal": "user-alice", + } + + +def test_tool_acting_principal_reaches_no_public_or_model_facing_payload( + storage_backend: StorageBackend, +) -> None: + """The audit identity has no dict-bridge key, so no projection can carry it. + + A principal-only envelope is the adversarial shape: were the tool branch in + ``reconstruct_turns`` not to claim it, it would fall through to + ``source_meta`` — which /history publishes as a turn's display ``meta``. + """ + ws_id = "tool-principal-private" + storage_backend.save_message( + ws_id, + "tool", + "probe output", + "read_only_probe", + tool_call_id="call-audit", + meta=json.dumps({"acting_principal": "private-user-id"}), + ) + + turns = storage_backend.load_message_turns(ws_id, checkpointed=False) + assert turns[0].meta.extra["acting_principal"] == "private-user-id" + assert "private-user-id" not in json.dumps(turn_to_dict(turns[0])) + + loaded = storage_backend.load_messages(ws_id, repair=False) + assert "_source_meta" not in loaded[0] + assert "private-user-id" not in json.dumps(loaded) + + history = project_history_messages(loaded) + assert "meta" not in history[0] + assert "private-user-id" not in json.dumps(history) + assert "private-user-id" not in json.dumps(_serialize_messages(loaded)) + assert "private-user-id" not in json.dumps( + _serialize_messages(loaded, include_provider_content=True) + ) + + storage = MagicMock() + storage.get_workstream.return_value = {"state": "idle"} + storage.load_message_turns.return_value = turns + storage.get_attachments.return_value = [] + assert b"private-user-id" not in export_workstream(storage, ws_id).data + + +def test_provider_bound_wire_never_carries_the_tool_acting_principal() -> None: + """Sidecar meta stays sidecar: the lowered wire has no field for it.""" + provider = seam_provider("done") + lane = ModelLane( + provider=provider, + client=MagicMock(), + model="backend-v2", + alias="assistant-fast", + registry_generation=17, + capabilities=ModelCapabilities(), + ) + tool_turn = Turn.tool("call-audit", "probe output", effect_status=EffectStatus.COMMITTED) + tool_turn.meta.extra["acting_principal"] = "private-user-id" + + model_turn( + lane, + [ + Turn.user("run the probe"), + turn_from_dict( + { + "role": "assistant", + "tool_calls": [ + { + "id": "call-audit", + "function": {"name": "read_only_probe", "arguments": "{}"}, + } + ], + } + ), + tool_turn, + ], + acting_principal_id="private-user-id", + ) + + wire = provider.create_streaming.call_args.kwargs["messages"] + assert any(message.get("role") == "tool" for message in wire) + assert "private-user-id" not in json.dumps(wire) + assert "acting_principal" not in json.dumps(wire) + + +def test_pending_and_ambiguous_ack_keep_one_exact_provenance_tuple(tmp_db: str) -> None: + """A lost ACK cannot relabel or duplicate the accepted assistant row.""" + session = make_session( + model_alias="main", + registry_generation=5, + user_id="owner", + ui=RecordingUI(), # type: ignore[no-untyped-call] + ) + arm_session(session, _good_stream("accepted")) + durable_rows: list[dict[str, Any]] = [] + ids_by_commit: dict[tuple[str, str], int] = {} + assistant_attempts = 0 + + def _ambiguous_save(ws_id: str, role: str, content: str, **kwargs: object) -> int: + nonlocal assistant_attempts + commit_key = kwargs.get("commit_key") + # Participant-join operator context is intentionally outside the + # conversation-row journal. It may precede the keyed user turn in a + # shared workstream and is irrelevant to this lost-ACK seam. + if not isinstance(commit_key, str) or not commit_key: + return 1 + identity = (ws_id, commit_key) + if identity not in ids_by_commit: + ids_by_commit[identity] = len(ids_by_commit) + 1 + row: dict[str, Any] = { + "role": role, + "content": content, + "_commit_key": commit_key, + } + raw_meta = kwargs.get("meta") + if role == "assistant": + assert isinstance(raw_meta, str) + row["_provenance"] = json.loads(raw_meta)[PROVENANCE_META_KEY] + durable_rows.append(row) + if role == "assistant": + assistant_attempts += 1 + return 0 + return ids_by_commit[identity] + + with ( + patch("turnstone.core.session.save_message", side_effect=_ambiguous_save), + pytest.raises(ConversationPersistenceError), + ): + session.send("hello", acting_user_id="alice") + + expected = { + "model_alias": "main", + "backend_model_id": "test-model", + "registry_generation": 5, + "acting_principal_id": "alice", + } + assert assistant_attempts == 1 + physical_assistants = [row for row in durable_rows if row["role"] == "assistant"] + assert len(physical_assistants) == 1 + assert physical_assistants[0]["_provenance"] == expected + + # A storage outage leaves the immutable pending projection authoritative. + pending, _token = session.capture_history_handoff(lambda _overscan: []) + pending_assistants = [row for row in pending if row.get("role") == "assistant"] + assert len(pending_assistants) == 1 + assert pending_assistants[0]["_provenance"] == expected + + # Once the durable keyed row is visible it reconciles in place, still once + # and with the exact same request-time identity. + reconciled, _token = session.capture_history_handoff(lambda _overscan: durable_rows) + reconciled_assistants = [row for row in reconciled if row.get("role") == "assistant"] + assert len(reconciled_assistants) == 1 + assert reconciled_assistants[0]["_provenance"] == expected + + +def test_public_and_model_facing_projections_do_not_expose_principal() -> None: + provenance = TurnProvenance( + model_alias="main", + backend_model_id="kernel", + registry_generation=8, + acting_principal_id="private-user-id", + ) + turn = Turn.assistant("accepted") + turn.meta.extra[PROVENANCE_META_KEY] = provenance.to_meta() + internal = turn_to_dict(turn) + + history = project_history_messages([internal]) + assert history == [{"role": "assistant", "content": "accepted"}] + assert "private-user-id" not in json.dumps(history) + assert "_provenance" not in _serialize_messages([internal])[0] + assert "_provenance" not in _serialize_messages([internal], include_provider_content=True)[0] + + storage = MagicMock() + storage.get_workstream.return_value = {"state": "idle"} + storage.load_message_turns.return_value = [turn] + storage.get_attachments.return_value = [] + exported = export_workstream(storage, "ws") + payload = json.loads(exported.data) + assert payload == {"messages": [{"role": "assistant", "content": "accepted"}]} + assert b"private-user-id" not in exported.data + + +def test_history_projection_failure_returns_503_without_private_row_fields( + tmp_db: str, +) -> None: + """No public decoration/projection failure can authorize raw history.""" + from tests._coord_test_helpers import _fake_registry + from tests.test_coordinator_endpoints import ( + _COORD_HEADERS, + _build_history_mgr, + _make_client, + ) + from turnstone.core.storage import get_storage + + storage = get_storage() + assert storage is not None + # A 200 history response is authoritative only when a concrete live + # session captures the durable rows and returns a handoff token. Use the + # endpoint suite's token-bearing session fixture; the generic coordinator + # stub intentionally has no history-handoff protocol and must now fail + # closed with 503. + manager = _build_history_mgr(storage) + workstream = manager.create(user_id="user-1") + private_principal = "PRIVATE-PRINCIPAL-SENTINEL" + private_reasoning = "PRIVATE-REASONING-SENTINEL" + private_signature = "PRIVATE-SIGNATURE-SENTINEL" + private_producer = "PRIVATE-PRODUCER-SENTINEL" + private_commit_key = "PRIVATE-COMMIT-KEY-SENTINEL" + raw = TurnProvenance( + model_alias="main", + backend_model_id="kernel", + registry_generation=8, + acting_principal_id=private_principal, + ).to_meta() + storage.save_message( + workstream.id, + "assistant", + "accepted", + provider_data=json.dumps( + { + "producer": private_producer, + "blocks": [ + { + "type": "reasoning_text", + "text": private_reasoning, + "signature": private_signature, + } + ], + } + ), + meta=json.dumps({PROVENANCE_META_KEY: raw}), + commit_key=private_commit_key, + ) + client = _make_client(storage, coord_mgr=manager, registry=_fake_registry()) + + for projection_seam in ( + "decorate_history_messages", + "extract_reasoning_for_history", + "project_history_messages", + ): + with patch( + f"turnstone.core.history_decoration.{projection_seam}", + side_effect=RuntimeError("public projection unavailable"), + ): + response = client.get( + f"/v1/api/workstreams/{workstream.id}/history", + headers=_COORD_HEADERS, + ) + + assert response.status_code == 503 + assert response.json() == {"error": "History temporarily unavailable"} + assert "messages" not in response.json() + assert "cursor" not in response.json() + assert "handoff_token" not in response.json() + for private_value in ( + private_principal, + private_reasoning, + private_signature, + private_producer, + private_commit_key, + "_provider_content", + "_producer", + "_provenance", + "_commit_key", + ): + assert private_value not in response.text diff --git a/tests/test_webui_content.py b/tests/test_webui_content.py index 7b165e43..5f4f4d42 100644 --- a/tests/test_webui_content.py +++ b/tests/test_webui_content.py @@ -127,6 +127,50 @@ class TestContentAccumulation: assert ui._ws_turn_content == [] assert ui._ws_turn_content_size == 0 + def test_persistence_refresh_is_operator_only_and_non_consuming(self): + ui = _make_ui() + ui._ws_turn_content = ["preserve me"] + ui._ws_turn_content_size = len("preserve me") + session = MagicMock() + session.conversation_persistence_status = lambda: {"state": "conflict"} + ui.bind_session(session) + ws = MagicMock() + ws.state.value = "error" + # The registry row deliberately disagrees: the persistence field + # must come from the BOUND session (an id-reuse replacement row + # reporting healthy must not launder the badge). + ws.session.conversation_persistence_status = lambda: {"state": "healthy"} + mgr = MagicMock() + mgr.get.return_value = ws + WebUI._workstream_mgr = mgr + try: + ui.on_persistence_state_changed() + finally: + WebUI._workstream_mgr = None + + event = _drain_global()[0] + assert event["type"] == "ws_state" + assert event["state"] == "error" + assert event["persistence_state"] == "conflict" + assert "content" not in event + assert ui._ws_turn_content == ["preserve me"] + assert ui._ws_turn_content_size == len("preserve me") + + def test_persistence_state_derives_from_bound_session_not_registry(self): + """The badge keeps telling the truth while the row is out of the + map: failed-delete tombstone retention and retirement pop the + registry entry exactly when the journal needs the operator, and + the old by-id lookup laundered that window into "healthy".""" + ui = _make_ui() + session = MagicMock() + session.conversation_persistence_status = lambda: {"state": "conflict"} + ui.bind_session(session) + # No manager wired at all — the registry cannot be consulted. + assert WebUI._workstream_mgr is None + assert ui._current_persistence_state() == "conflict" + # Unbound (or collected) still fails closed to healthy. + assert _make_ui()._current_persistence_state() == "healthy" + def test_thinking_broadcast_does_not_touch_accumulator(self): """_broadcast_state('thinking') should not affect the accumulator.""" ui = _make_ui() diff --git a/tests/test_workstream_endpoints.py b/tests/test_workstream_endpoints.py index 12cddc84..5c2b6bcb 100644 --- a/tests/test_workstream_endpoints.py +++ b/tests/test_workstream_endpoints.py @@ -7,6 +7,7 @@ import json import logging import queue import threading +import time from types import SimpleNamespace from typing import TYPE_CHECKING, Any from unittest.mock import MagicMock, patch @@ -43,7 +44,7 @@ from turnstone.core.session_routes import ( make_set_title_handler, ) from turnstone.core.storage._sqlite import SQLiteBackend -from turnstone.core.workstream import WorkstreamKind +from turnstone.core.workstream import Workstream, WorkstreamKind from turnstone.server import ( _interactive_tenant_check, delete_workstream_endpoint, @@ -236,19 +237,23 @@ def _rewind_retry_mocks(*, worker_running=False, rewind_return=4, retry_return=" """Mocked ``(manager, session, enqueued-events)`` for the lifted rewind/retry handlers. ``ws._lock`` is a real lock so the handler's busy-gate ``with ws._lock`` works; ``ui._enqueue`` records events.""" - import threading - mock_session = MagicMock() - mock_session.rewind.return_value = rewind_return - mock_session.retry.return_value = retry_return enqueued: list[dict[str, Any]] = [] mock_ui = MagicMock() mock_ui._enqueue.side_effect = lambda ev: enqueued.append(ev) - mock_ws = MagicMock() - mock_ws.session = mock_session - mock_ws.ui = mock_ui - mock_ws._lock = threading.Lock() + mock_ws = Workstream(id="ws1", session=mock_session, ui=mock_ui) mock_ws._worker_running = worker_running + + def _rewind(_turns: int, *, publish_reset: Callable[[], None]) -> int: + publish_reset() + return rewind_return + + def _retry(*, publish_reset: Callable[[], None]) -> str | None: + publish_reset() + return retry_return + + mock_session.rewind.side_effect = _rewind + mock_session.retry.side_effect = _retry mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws return mock_mgr, mock_session, enqueued @@ -279,8 +284,10 @@ def test_rewind_returns_removed_and_emits_clear_ui(): resp = client.post("/v1/api/workstreams/ws1/rewind", json={"turns": 2}) assert resp.status_code == 200 assert resp.json() == {"status": "ok", "removed": 4} - mock_session.rewind.assert_called_once_with(2) - assert {"type": "clear_ui"} in enqueued + mock_session.rewind.assert_called_once() + assert mock_session.rewind.call_args.args == (2,) + assert callable(mock_session.rewind.call_args.kwargs["publish_reset"]) + assert enqueued == [{"type": "clear_ui"}] def test_rewind_rejects_non_positive_or_non_int_turns(): @@ -305,46 +312,194 @@ def test_rewind_while_busy_returns_busy_and_skips_mutation(): assert any(e.get("type") == "busy_error" for e in enqueued) -def test_retry_dispatches_and_emits_clear_ui(): - mock_mgr, _session, enqueued = _rewind_retry_mocks(retry_return="hello") - dispatched: list[str] = [] - handler = make_retry_handler( - _verb_cfg(mock_mgr), dispatch_retry=lambda _ws, msg: dispatched.append(msg) - ) +def test_retry_refused_409_when_foreign_queued_input_retained(): + """Round-4 review pin: /retry runs the same foreign-queue admission gate + as ordinary sends — another participant's persistence-retained input + refuses the retry up-front with 409 cross_user_interjection, instead of + claiming the slot, truncating history, and dying mid-turn on the + advisory seam's ownership assert after the HTTP response already said ok. + """ + mock_mgr, mock_session, _enqueued = _rewind_retry_mocks(retry_return="hello") + mock_session.has_foreign_queued_messages = lambda pid: True + handler = make_retry_handler(_verb_cfg(mock_mgr)) + client = _verb_client("/api/workstreams/{ws_id}/retry", handler) + resp = client.post("/v1/api/workstreams/ws1/retry") + assert resp.status_code == 409 + assert resp.json()["status"] == "cross_user_interjection" + mock_session.retry.assert_not_called() + mock_session.send.assert_not_called() + + +def _wait_until(predicate: Callable[[], bool], timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +def test_retry_closure_sanitizes_error_display(): + """Resurrected HEAD pin (round-5 review): a raise from the replacement + send BEFORE ChatSession's error-convergence envelope — after the + destructive cut succeeded and the HTTP response already said ok — must + converge the pane: sanitized ``on_error`` (a credential-bearing backend + URL never crosses into SSE), ``stream_end``, and ``state=error``. + Deliberately NOT routed through ensure_error_recorded (the reused-session + stale-flag hazard — #865); the display-sanitize half is fixed at the site. + """ + mock_mgr, session, _enqueued = _rewind_retry_mocks(retry_return="retry this") + ws = mock_mgr.get.return_value + ui = ws.ui + converged = threading.Event() + + def _state(state: str) -> None: + if state == "error": + converged.set() + + ui.on_state_change.side_effect = _state + + def _boom(_msg: str, *, acting_user_id: str | None = None) -> None: + raise RuntimeError("cannot reach https://user:pass@host:8000/v1 for model=x") + + session.send.side_effect = _boom + handler = make_retry_handler(_verb_cfg(mock_mgr)) client = _verb_client("/api/workstreams/{ws_id}/retry", handler) resp = client.post("/v1/api/workstreams/ws1/retry") assert resp.status_code == 200 assert resp.json() == {"status": "ok", "retried": True} - assert dispatched == ["hello"] - assert {"type": "clear_ui"} in enqueued + + assert converged.wait(5), "retry failure never converged the pane" + errors = [c.args[0] for c in ui.on_error.call_args_list] + assert errors, "retry closure emitted no on_error" + assert all("user:pass" not in m for m in errors), errors + assert any("REDACTED" in m for m in errors) + ui.on_stream_end.assert_called() + + +def test_retry_closure_converges_idle_on_pre_envelope_cancel(): + """The GenerationCancelled arm (driven synthetically — send self-handles + in-turn cancels, so this is the raced-Stop residue): stream_end + idle, + never an error banner.""" + from turnstone.core.session import GenerationCancelled + + mock_mgr, session, _enqueued = _rewind_retry_mocks(retry_return="retry this") + ws = mock_mgr.get.return_value + ui = ws.ui + idled = threading.Event() + + def _state(state: str) -> None: + if state == "idle": + idled.set() + + ui.on_state_change.side_effect = _state + session.send.side_effect = GenerationCancelled() + handler = make_retry_handler(_verb_cfg(mock_mgr)) + client = _verb_client("/api/workstreams/{ws_id}/retry", handler) + resp = client.post("/v1/api/workstreams/ws1/retry") + assert resp.status_code == 200 + + assert idled.wait(5), "cancelled retry never converged to idle" + ui.on_stream_end.assert_called() + ui.on_error.assert_not_called() + + +def test_retry_closure_stays_silent_when_the_slot_was_superseded(): + """The non-owner negative HEAD's pin lacked: a force-cancel that swapped + the worker slot between the raise and the emissions must suppress them — + no error banner stamped over a live successor.""" + mock_mgr, session, _enqueued = _rewind_retry_mocks(retry_return="retry this") + ws = mock_mgr.get.return_value + ui = ws.ui + worker: list[threading.Thread] = [] + + def _boom(_msg: str, *, acting_user_id: str | None = None) -> None: + worker.append(threading.current_thread()) + ws.worker_thread = None # the successor claimed / force-cancel swapped + raise RuntimeError("late failure") + + session.send.side_effect = _boom + handler = make_retry_handler(_verb_cfg(mock_mgr)) + client = _verb_client("/api/workstreams/{ws_id}/retry", handler) + resp = client.post("/v1/api/workstreams/ws1/retry") + assert resp.status_code == 200 + + assert _wait_until(lambda: bool(worker)) + worker[0].join(5) + assert not worker[0].is_alive() + ui.on_error.assert_not_called() + ui.on_stream_end.assert_not_called() + + +def test_rewind_commit_refusal_returns_503_not_ok(): + """A GenerationCancelled escaping rewind() (commit admission refused by a + racing close/delete/poison) must take the dispatcher's 503 error arm — + round-3 review: it previously escaped ``except Exception``, killed the + worker thread, and the handler answered ``200 ok/removed=0`` with no + ``clear_ui`` for a rewind that never ran. + """ + from turnstone.core.session import GenerationCancelled + + mock_mgr, mock_session, enqueued = _rewind_retry_mocks() + mock_session.rewind.side_effect = GenerationCancelled() + handler = make_rewind_handler(_verb_cfg(mock_mgr)) + client = _verb_client("/api/workstreams/{ws_id}/rewind", handler) + resp = client.post("/v1/api/workstreams/ws1/rewind", json={"turns": 1}) + assert resp.status_code == 503 + assert "history was unchanged" in resp.json()["error"] + assert not any(e.get("type") == "clear_ui" for e in enqueued) + + +def test_retry_dispatches_and_emits_clear_ui(): + mock_mgr, session, enqueued = _rewind_retry_mocks(retry_return="hello") + ws = mock_mgr.get.return_value + send_called = threading.Event() + send_claims: list[tuple[bool, str, str]] = [] + + def capture_send(_message: str, *, acting_user_id: str | None = None) -> None: + send_claims.append( + ( + threading.current_thread() is ws.worker_thread, + ws.worker_kind, + ws._worker_principal_id, + ) + ) + send_called.set() + + session.send.side_effect = capture_send + handler = make_retry_handler(_verb_cfg(mock_mgr)) + client = _verb_client("/api/workstreams/{ws_id}/retry", handler) + resp = client.post("/v1/api/workstreams/ws1/retry") + assert resp.status_code == 200 + assert resp.json() == {"status": "ok", "retried": True} + assert send_called.wait(timeout=5), "replacement send did not start" + session.send.assert_called_once_with("hello", acting_user_id="test-user") + assert send_claims == [(True, "turn", "test-user")] + # The compatibility callback must not claim a second worker after the + # atomic retry worker has already cut history and started its replacement. + assert enqueued == [{"type": "clear_ui"}] def test_retry_nothing_to_retry_skips_dispatch(): - mock_mgr, _session, enqueued = _rewind_retry_mocks(retry_return=None) - dispatched: list[str] = [] - handler = make_retry_handler( - _verb_cfg(mock_mgr), dispatch_retry=lambda _ws, msg: dispatched.append(msg) - ) + mock_mgr, session, enqueued = _rewind_retry_mocks(retry_return=None) + handler = make_retry_handler(_verb_cfg(mock_mgr)) client = _verb_client("/api/workstreams/{ws_id}/retry", handler) resp = client.post("/v1/api/workstreams/ws1/retry") assert resp.status_code == 200 assert resp.json() == {"status": "ok", "retried": False} - assert dispatched == [] - assert {"type": "clear_ui"} in enqueued + session.send.assert_not_called() + assert enqueued == [{"type": "clear_ui"}] def test_retry_while_busy_returns_busy_and_skips_dispatch(): mock_mgr, mock_session, enqueued = _rewind_retry_mocks(worker_running=True) - dispatched: list[str] = [] - handler = make_retry_handler( - _verb_cfg(mock_mgr), dispatch_retry=lambda _ws, msg: dispatched.append(msg) - ) + handler = make_retry_handler(_verb_cfg(mock_mgr)) client = _verb_client("/api/workstreams/{ws_id}/retry", handler) resp = client.post("/v1/api/workstreams/ws1/retry") assert resp.status_code == 200 assert resp.json()["status"] == "busy" mock_session.retry.assert_not_called() - assert dispatched == [] + mock_session.send.assert_not_called() assert any(e.get("type") == "busy_error" for e in enqueued) @@ -368,7 +523,6 @@ def test_retry_invokes_audit_emit(): captured: list[str] = [] handler = make_retry_handler( _verb_cfg(mock_mgr), - dispatch_retry=lambda _ws, _msg: None, audit_emit=lambda _req, ws_id, _ws: captured.append(ws_id), ) client = _verb_client("/api/workstreams/{ws_id}/retry", handler) @@ -390,7 +544,8 @@ def test_rewind_swallows_audit_emit_exception(): resp = client.post("/v1/api/workstreams/ws1/rewind", json={"turns": 1}) assert resp.status_code == 200 assert resp.json() == {"status": "ok", "removed": 2} - mock_session.rewind.assert_called_once_with(1) + mock_session.rewind.assert_called_once() + assert mock_session.rewind.call_args.args == (1,) # =========================================================================== @@ -1102,6 +1257,31 @@ class TestUpdateInterfaceSetting: # These tests pin the interactive wiring against the same factory. +class _HistoryHandoffSession: + """Minimal concrete session implementing the authoritative history seam.""" + + def __init__(self) -> None: + self._history_generation = 0 + + def capture_history_handoff( + self, + load_messages: Callable[[int], list[dict[str, Any]]], + ) -> tuple[list[dict[str, Any]], str]: + return load_messages(0), f"test-history.{self._history_generation}" + + +def _live_history_workstream(ws_id: str) -> SimpleNamespace: + ui = MagicMock() + ui._pending_approval = None + ui.can_replay_from.return_value = False + ui.get_agent_trajectory.return_value = None + return SimpleNamespace( + id=ws_id, + session=_HistoryHandoffSession(), + ui=ui, + ) + + def _interactive_endpoint_cfg( mock_mgr: Any, tenant_check: Any = None, @@ -1341,9 +1521,7 @@ class TestHistoryAgentStepsOverlay: "is_error": False, } ] - mock_ws = MagicMock() - mock_ws.id = ws_id - mock_ws.ui._pending_approval = None + mock_ws = _live_history_workstream(ws_id) mock_ws.ui.get_agent_trajectory = lambda cid: steps if cid == "task1" else None mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws @@ -1362,9 +1540,7 @@ class TestHistoryAgentStepsOverlay: # so the client renders the flat parent record (never a 0-step card). ws_id = "ws-recall-cold" self._save_task_agent_turn(_inject_storage, ws_id) - mock_ws = MagicMock() - mock_ws.id = ws_id - mock_ws.ui._pending_approval = None + mock_ws = _live_history_workstream(ws_id) mock_ws.ui.get_agent_trajectory = lambda cid: None mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws @@ -1384,8 +1560,7 @@ class TestHistoryInteractive: ws_id = "ws-int-1" _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") _inject_storage.save_message(ws_id, "user", "hello interactive") - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws client = _build_history_app(mock_mgr, _inject_storage) @@ -1394,25 +1569,122 @@ class TestHistoryInteractive: assert r.status_code == 200 body = r.json() assert body["ws_id"] == ws_id + assert body["handoff_token"] assert any( m.get("role") == "user" and m.get("content") == "hello interactive" for m in body["messages"] ) - def test_serves_storage_only_workstream(self, _inject_storage): - """Persisted-but-not-loaded interactives serve history without - rehydrating — same shape as coord. Pre-lift interactive had no - history endpoint at all, so this is a feature gain.""" - ws_id = "ws-cold" + def test_gone_latched_live_session_history_returns_503(self, _inject_storage): + """Round-3 review pin (the silent-wipe mechanism): once a session's + workstream-gone latch is set, ``capture_history_handoff`` refuses to + mint, and /history answers the fail-closed 503 — never a + token-bearing 200 over an empty transcript that would authorize the + pane to wipe its stale-but-real view.""" + from turnstone.core.storage import ConversationCommitWorkstreamGoneError + + ws_id = "ws-int-gone" _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") - _inject_storage.save_message(ws_id, "assistant", "from cold storage") + _inject_storage.save_message(ws_id, "user", "hello") + mock_ws = _live_history_workstream(ws_id) + + def _refuse(load_messages): + raise ConversationCommitWorkstreamGoneError("workstream was deleted") + + mock_ws.session.capture_history_handoff = _refuse mock_mgr = MagicMock() - mock_mgr.get.return_value = None # not loaded + mock_mgr.get.return_value = mock_ws + + r = _build_history_app(mock_mgr, _inject_storage).get( + f"/v1/api/workstreams/{ws_id}/history" + ) + assert r.status_code == 503 + assert r.json()["error"] == "History temporarily unavailable" + + def test_live_session_without_route_storage_downgrades_to_tokenless(self): + """Round-3 review pin: with no route-visible storage the durable + prefix is unreadable, so the live-session arm must NOT mint a handoff + token over the journal/live view — the 200 downgrades to the + deliberate tokenless render (the client bootstrap owns convergence + via ``clear_ui``), never a token-authorized splice of incomplete + history and never a 503 repair loop.""" + ws_id = "ws-int-no-storage" + mock_ws = _live_history_workstream(ws_id) + mock_mgr = MagicMock() + mock_mgr.get.return_value = mock_ws + + r = _build_history_app(mock_mgr, None).get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + assert r.json()["handoff_token"] is None + + def test_tool_row_audit_principal_never_reaches_the_history_payload(self, _inject_storage): + """The route's public schema stays free of the tool row's audit identity. + + A TOOL row records the principal its turn executed under so revocation + can read the row directly. /history is the widest consumer of those + same rows, so the whole ladder — reconstruct, decorate, project, scrub + — has to drop it: the response body must not name the principal at + all, and the tool entry must gain no display ``meta`` from it. + """ + ws_id = "ws-tool-principal" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + _inject_storage.save_message(ws_id, "user", "run the probe") + _inject_storage.save_message( + ws_id, + "assistant", + "", + tool_calls=json.dumps( + [ + { + "id": "call-audit", + "type": "function", + "function": {"name": "read_only_probe", "arguments": "{}"}, + } + ] + ), + ) + _inject_storage.save_message( + ws_id, + "tool", + "probe output", + "read_only_probe", + tool_call_id="call-audit", + meta=json.dumps({"effect_status": "committed", "acting_principal": "private-user-id"}), + ) + mock_mgr = MagicMock() + mock_mgr.get.return_value = None client = _build_history_app(mock_mgr, _inject_storage) r = client.get(f"/v1/api/workstreams/{ws_id}/history") assert r.status_code == 200 - assert any(m.get("content") == "from cold storage" for m in r.json()["messages"]) + assert "private-user-id" not in r.text + assert "acting_principal" not in r.text + tool_entry = next(m for m in r.json()["messages"] if m.get("role") == "tool") + assert tool_entry["content"] == "probe output" + assert "meta" not in tool_entry + + def test_serves_storage_only_workstream(self, _inject_storage): + """Persisted-but-not-loaded interactives serve history without rehydrating. + + Deliberate pin update (back to the pre-handoff assertion, plus the + token contract): a cold row has no live writer and no splice to + witness, so /history is a pure storage read — tokenless, never + constructing a ChatSession into the bounded pool. The tokenless 200 + seeds a render plus the tokenless stream bootstrap. + """ + ws_id = "ws-cold" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + _inject_storage.save_message(ws_id, "assistant", "from cold storage") + mock_mgr = MagicMock() + mock_mgr.get.return_value = None # not loaded — and it stays that way + client = _build_history_app(mock_mgr, _inject_storage) + + r = client.get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + body = r.json() + assert any(m.get("content") == "from cold storage" for m in body["messages"]) + assert body["handoff_token"] is None + mock_mgr.open.assert_not_called() def test_404_on_missing_ws_id(self, _inject_storage): mock_mgr = MagicMock() @@ -1448,8 +1720,7 @@ class TestHistoryInteractive: _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") for i in range(4): _inject_storage.save_message(ws_id, "user", f"msg-{i}") - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws client = _build_history_app(mock_mgr, _inject_storage) @@ -1500,8 +1771,7 @@ class TestHistoryInteractive: _inject_storage.save_message(ws_id, "assistant", "Working", tool_calls=tc_json) _inject_storage.save_message(ws_id, "tool", "file.txt", tool_call_id="call_1") # call_2 result not yet persisted — operator refreshes here. - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) # Mid-EXECUTION, not awaiting approval: any approval already # resolved, so ``_pending_approval`` is None. The trailing orphan # tool turn must therefore RENDER (``pending`` absent) — marking it @@ -1554,8 +1824,7 @@ class TestHistoryInteractive: ) _inject_storage.save_message(ws_id, "assistant", "Working", tool_calls=tc_json) # No tool result yet AND the session is parked awaiting approval. - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_ws.ui._pending_approval = {"type": "approve_request", "items": []} mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws @@ -1584,8 +1853,7 @@ class TestHistoryInteractive: ) _inject_storage.save_message(ws_id, "assistant", "Working", tool_calls=tc_json, event_id=12) # call_1 result not yet persisted — executing in-flight orphan. - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_ws.ui._pending_approval = None # executing, not awaiting mock_ws.ui.can_replay_from.return_value = True # buffer can fast-forward mock_mgr = MagicMock() @@ -1617,8 +1885,7 @@ class TestHistoryInteractive: [{"id": "call_1", "type": "function", "function": {"name": "bash", "arguments": "{}"}}] ) _inject_storage.save_message(ws_id, "assistant", "Working", tool_calls=tc_json, event_id=12) - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_ws.ui._pending_approval = None mock_ws.ui.can_replay_from.return_value = False # empty/evicted buffer mock_mgr = MagicMock() @@ -1659,8 +1926,7 @@ class TestHistoryInteractive: # Cancel landed before any tool result — next turn happens. _inject_storage.save_message(ws_id, "user", "second") _inject_storage.save_message(ws_id, "assistant", "ok") - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_ws.ui._pending_approval = None # cancelled, not awaiting approval mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws @@ -1740,9 +2006,7 @@ class TestHistoryCoalescing: storage.save_message(ws_id, "assistant", "hi there") def _live_mgr(self, ws_id: str) -> MagicMock: - mock_ws = MagicMock() - mock_ws.id = ws_id - mock_ws.ui._pending_approval = None + mock_ws = _live_history_workstream(ws_id) mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws return mock_mgr @@ -1823,6 +2087,8 @@ class TestHistoryCoalescing: r1, r2 = asyncio.run(self._drive_two_joiners(app, url, gated, caplog)) assert r1.status_code == 200 assert r2.status_code == 200 + assert r1.json()["handoff_token"] + assert r2.json()["handoff_token"] assert r1.json() == r2.json() contents = [m.get("content") for m in r1.json()["messages"]] assert contents == ["hello", "hi there"] @@ -1886,20 +2152,41 @@ class TestHistoryCoalescing: self, _inject_storage: Any, caplog: Any ) -> None: """A transient ``load_messages`` failure in the shared draw - yields the owner's 200-empty (same as a lone request today) but - the JOINER retries independently and gets the real rows — one - storage blip must not wipe every coalesced pane.""" + yields a bounded 503 for the owner, but the JOINER retries + independently and gets the real rows — one storage blip must not + fail every coalesced pane or authorize an incomplete render.""" gated, _tenant_calls, app, url = self._scaffold(_inject_storage, "ws-flight-fail") gated.fail_next = 1 r1, r2 = asyncio.run(self._drive_two_joiners(app, url, gated, caplog)) - assert r1.status_code == 200 - assert r1.json()["messages"] == [] + assert r1.status_code == 503 + assert r1.json() == {"error": "History temporarily unavailable"} assert r2.status_code == 200 assert [m.get("content") for m in r2.json()["messages"]] == ["hello", "hi there"] # Owner draw + joiner retry — never a third. assert gated.load_calls == 2 + def test_failed_shared_draw_and_joiner_retry_both_return_bounded_503( + self, _inject_storage: Any, caplog: Any + ) -> None: + """The joiner's independent retry must preserve its final failure bit. + + Discarding that bit would turn the second failed reconstruction into a + 200-empty response (or, with a live journal, pending-only history plus + a handoff token), while only the flight owner correctly reported the + outage. + """ + gated, _tenant_calls, app, url = self._scaffold(_inject_storage, "ws-flight-double-fail") + gated.fail_next = 2 + + r1, r2 = asyncio.run(self._drive_two_joiners(app, url, gated, caplog)) + expected = {"error": "History temporarily unavailable"} + assert r1.status_code == 503 + assert r1.json() == expected + assert r2.status_code == 503 + assert r2.json() == expected + assert gated.load_calls == 2 + def test_gates_run_per_request_before_join(self, _inject_storage: Any) -> None: """A caller failing its own tenant gate gets its 404 while the flight is still in the air — it never joins and never triggers @@ -1965,35 +2252,38 @@ class TestHistoryCoalescing: assert r1.json() == r2.json() assert gated.load_calls == 2 - def test_decoration_failure_is_shared_not_retried( + def test_decoration_failure_is_unavailable_and_joiner_retries_once( self, _inject_storage: Any, caplog: Any ) -> None: - """The inverse of the load-failure test: a decoration/projection - failure AFTER a successful load is degraded-but-non-empty, so it - is shared to joiners as-is — ``load_failed`` stays False and no - joiner retry fires (``load_calls`` stays 1, vs the load-failure - test's 2). Pins the two failure modes apart: conflating them - (e.g. setting ``load_failed`` in the decoration except) would - turn every shared degraded draw into a double reconstruction.""" + """A public projection failure is non-authoritative like load failure. + + The owner gets the shared draw's bounded 503. Its coalesced follower + gets exactly one independent reconstruction attempt, but a persistent + projection fault still returns 503 without messages or a handoff + token; raw pre-projection content must never escape. + """ gated, _tenant_calls, app, url = self._scaffold(_inject_storage, "ws-flight-decor") + private_sentinel = "PRIVATE-UNPROJECTED-HISTORY-SENTINEL" + _inject_storage.save_message("ws-flight-decor", "assistant", private_sentinel) with patch( "turnstone.core.history_decoration.project_history_messages", side_effect=RuntimeError("projection failure (test)"), ) as fake_project: r1, r2 = asyncio.run(self._drive_two_joiners(app, url, gated, caplog)) - # The degraded path must actually have run — without this, a - # fixture whose cursor is None either way would let an - # ineffective patch pass the assertions below vacuously. - assert fake_project.called - assert r1.status_code == 200 - assert r2.status_code == 200 - # Degraded (un-projected, cursor=None) but NON-empty and identical — - # the joiner shared the draw instead of re-reconstructing. - assert r1.json() == r2.json() - assert r1.json()["messages"] - assert r1.json()["cursor"] is None - assert gated.load_calls == 1 + expected = {"error": "History temporarily unavailable"} + assert r1.status_code == 503 + assert r2.status_code == 503 + assert r1.json() == expected + assert r2.json() == expected + assert private_sentinel not in r1.text + assert private_sentinel not in r2.text + assert "messages" not in r1.json() + assert "cursor" not in r1.json() + assert "handoff_token" not in r1.json() + # Shared owner reconstruction + one follower retry, never a third. + assert fake_project.call_count == 2 + assert gated.load_calls == 2 def test_owner_disconnect_leaves_joiner_completing( self, _inject_storage: Any, caplog: Any @@ -2122,10 +2412,36 @@ class TestDetailInteractive: "state": "idle", "user_id": "test-user", "kind": "interactive", + "persistence_state": "healthy", "pending_approval": False, "pending_approval_details": [], } + def test_projects_sanitized_persistence_state(self): + ws_id = "ws-detail-persistence" + ws_state = MagicMock() + ws_state.value = "error" + loaded_ws = MagicMock() + loaded_ws.id = ws_id + loaded_ws.name = "needs-history-repair" + loaded_ws.state = ws_state + loaded_ws.user_id = "test-user" + loaded_ws.kind = "interactive" + loaded_ws.session.conversation_persistence_status = lambda: { + "state": "conflict", + "attempts": 1, + "last_failure_at": "sanitized-away", + } + mock_mgr = MagicMock() + mock_mgr.get.return_value = loaded_ws + client = _build_detail_app(mock_mgr) + + body = client.get(f"/v1/api/workstreams/{ws_id}").json() + + assert body["persistence_state"] == "conflict" + assert "attempts" not in body + assert "last_failure_at" not in body + def test_pending_approval_fields_propagate_from_ui(self): """When the workstream's UI is parked on an approval, the detail response surfaces ``pending_approval=True`` + the serialized @@ -2389,8 +2705,7 @@ class TestTenantCheckOnReadEndpoints: ws_id = "ws-mine-hist" _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") _inject_storage.save_message(ws_id, "user", "hello") - mock_ws = MagicMock() - mock_ws.id = ws_id + mock_ws = _live_history_workstream(ws_id) mock_mgr = MagicMock() mock_mgr.get.return_value = mock_ws @@ -2424,8 +2739,9 @@ class TestTenantCheckOnReadEndpoints: ws_id = "ws-cold-cache-hist" _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") _inject_storage.save_message(ws_id, "user", "from cold storage") + # Cold cache: nothing in memory, owner row only in storage — and it + # stays cold; /history never rehydrates. mock_mgr = MagicMock() - # Cold cache: nothing in memory, owner row only in storage. mock_mgr.get.return_value = None def cold_check(request: Any, ws_id: str, mgr: Any) -> JSONResponse | None: @@ -2446,7 +2762,10 @@ class TestTenantCheckOnReadEndpoints: r = client.get(f"/v1/api/workstreams/{ws_id}/history") assert r.status_code == 200 - assert any(m.get("content") == "from cold storage" for m in r.json()["messages"]) + body = r.json() + assert any(m.get("content") == "from cold storage" for m in body["messages"]) + assert body["handoff_token"] is None + mock_mgr.open.assert_not_called() # Pin the offload — reverting ``await asyncio.to_thread(cfg.tenant_check, ...)`` # to ``cfg.tenant_check(...)`` leaves the response shape intact # but drops ``cold_check`` from the spy's call list. @@ -2533,15 +2852,18 @@ class TestHistoryReasoningRehydration: _inject_storage.save_message( ws_id, "assistant", "Final answer.", provider_data=provider_data ) - # No live session — exercises the storage-only path which - # falls back to default surface_persisted_reasoning=True. + # Cold rows are served storage-only (no rehydration), so reasoning + # exercises the storage fallback and the conservative + # surface_persisted_reasoning=True default. mock_mgr = MagicMock() mock_mgr.get.return_value = None client = _build_history_app(mock_mgr, _inject_storage) r = client.get(f"/v1/api/workstreams/{ws_id}/history") assert r.status_code == 200 - msgs = r.json()["messages"] + body = r.json() + assert body["handoff_token"] is None + msgs = body["messages"] assistant = next(m for m in msgs if m.get("role") == "assistant") assert assistant["reasoning"] == "let me reason" @@ -2568,15 +2890,24 @@ class TestHistoryReasoningRehydration: _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") provider_data = json.dumps([{"type": "thinking", "thinking": "hidden", "signature": "s"}]) _inject_storage.save_message(ws_id, "assistant", "Answer.", provider_data=provider_data) + # A stored alias that would resolve to the permissive default: the + # live session's registry must win the tier order (round-3 review — + # the handler once read these attrs off the Workstream wrapper, which + # never carries them in production, so tier 1 silently never ran). + _inject_storage.save_workstream_config(ws_id, {"model_alias": "stored-stale-alias"}) + + def _live_only_reasoning_config(alias: str) -> SimpleNamespace: + assert alias == "claude-opus-4-7", alias + return SimpleNamespace(surface_persisted_reasoning=False) + + session = _HistoryHandoffSession() + session._registry = SimpleNamespace(get_config=_live_only_reasoning_config) + session._model_alias = "claude-opus-4-7" live_session = SimpleNamespace( # The real Workstream carries .session (ChatSession | None); - # the flight key's typed generation read requires the shape. - session=None, + # registry and alias live on the ChatSession, never the wrapper. + session=session, id=ws_id, - _registry=SimpleNamespace( - get_config=lambda alias: SimpleNamespace(surface_persisted_reasoning=False) - ), - _model_alias="claude-opus-4-7", ) mock_mgr = MagicMock() mock_mgr.get.return_value = live_session @@ -2606,7 +2937,8 @@ class TestHistoryReasoningRehydration: [{"type": "thinking", "thinking": "should not surface", "signature": "s"}] ) _inject_storage.save_message(ws_id, "assistant", "Answer.", provider_data=provider_data) - # No live session — handler falls back to workstream_config + registry. + # Cold storage-only read: the handler resolves the persisted alias + # through storage and the kind-appropriate registry on app.state. mock_mgr = MagicMock() mock_mgr.get.return_value = None diff --git a/turnstone/api/console_schemas.py b/turnstone/api/console_schemas.py index 7b4031c0..49c8b7ef 100644 --- a/turnstone/api/console_schemas.py +++ b/turnstone/api/console_schemas.py @@ -16,6 +16,9 @@ from turnstone.core.model_registry import MAX_MODEL_CONCURRENCY from turnstone.core.skill_kind import SkillKind from turnstone.core.skill_parser import MAX_SKILL_DESCRIPTION_LEN +# Pydantic resolves this annotation while constructing the model schema. +from turnstone.core.workstream import ConversationPersistenceState # noqa: TC001 + # --------------------------------------------------------------------------- # Cluster overview # --------------------------------------------------------------------------- @@ -85,6 +88,13 @@ class ClusterWorkstreamInfo(BaseModel): activity: str = "" activity_state: str = "" tool_calls: int = 0 + persistence_state: ConversationPersistenceState = Field( + default="healthy", + description=( + "Sanitized durable-history status for a live row. Older nodes and " + "unloaded persisted-only rows default to healthy." + ), + ) class ClusterWorkstreamsResponse(BaseModel): @@ -1508,6 +1518,16 @@ class CoordinatorSendRequest(BaseModel): """Body for POST /v1/api/workstreams/{ws_id}/send.""" message: str = Field(description="User message to queue onto the coordinator's worker.") + client_send_id: str | None = Field( + default=None, + min_length=1, + max_length=128, + pattern=r"^[A-Za-z0-9_-]+$", + description=( + "Opaque browser correlation token echoed in the accepted `user_turn` " + "event and history row. It is not an idempotency key." + ), + ) attachment_ids: list[str] | None = Field( default=None, description=( @@ -1736,7 +1756,8 @@ class ClusterWsDetailResponse(BaseModel): live: dict[str, Any] | None = Field( default=None, description=( - "Live in-flight counters (state, tokens, activity, pending_approval) when " + "Live in-flight counters and sanitized durable-history status " + "(state, tokens, activity, pending_approval, persistence_state) when " "the owning node returns them; null on degrade." ), ) diff --git a/turnstone/api/console_spec.py b/turnstone/api/console_spec.py index 720c345b..1eaa6681 100644 --- a/turnstone/api/console_spec.py +++ b/turnstone/api/console_spec.py @@ -1494,9 +1494,10 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ "POST", "Drop the last N conversation turns on the coordinator (emits clear_ui)", description=( - "Truncates the coordinator conversation by N turns via the shared " - "rewind handler and emits ``clear_ui`` so the dashboard re-fetches " - "the truncated history. Gated on ``admin.coordinator``." + "Claims the coordinator mutation slot, durably truncates N turns, " + "and emits ``clear_ui`` so the dashboard re-fetches the truncated " + "history. Concurrent sends are ordered after the cut; storage failure " + "returns 503 without changing live history. Gated on ``admin.coordinator``." ), request_model=RewindRequest, response_model=StatusResponse, @@ -1508,9 +1509,9 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ "POST", "Re-send the last user message on the coordinator for a fresh response", description=( - "Drops the last response and re-sends the last user message via the " - "shared worker dispatch, emitting ``clear_ui``. Gated on " - "``admin.coordinator``." + "Uses one shared worker claim to drop the last response and start the " + "replacement generation, emitting ``clear_ui``. Another send cannot " + "enter between those operations. Gated on ``admin.coordinator``." ), response_model=StatusResponse, error_codes=[400, 403, 404, 503], @@ -1524,10 +1525,12 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ "Releases the worker thread + UI listeners and marks the row " "``state=closed`` in storage. The row remains queryable (audit " "/ history) but cannot be reopened — a closed coordinator is " - "terminal from the manager's perspective." + "terminal from the manager's perspective. Returns 409 while an " + "accepted live conversation row still requires persistence reconciliation; " + "the coordinator remains loaded and its history journal is retained." ), response_model=StatusResponse, - error_codes=[403, 404, 500, 503], + error_codes=[403, 404, 409, 500, 503], tags=["Coordinator"], ), EndpointSpec( @@ -1537,11 +1540,42 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ description=( "Server-Sent Events stream carrying ``status``, ``message``, " "``tool_call``, ``tool_result``, ``approval``, ``error``, and " - "the phase-3 ``child_ws_*`` fan-out events. Pings every 5s. " + "the phase-3 ``child_ws_*`` fan-out events. After rendering REST history, " + "pass its opaque handoff_token once as ?history_token=; it names the exact " + "accepted conversation-row prefix used for that render. history_resync " + "closes this stream and requires a fresh history read; numeric replay is " + "not a substitute. Native Last-Event-ID reconnects take priority. " + "Pass ?user_turn=1 to receive typed accepted-user events; without it, " + "those rows use the backward-compatible strong-repair projection. " + "Pass ?tool_turn=1 to receive final accepted tool rows as typed " + "tool_result events with accepted=true; without it, accepted tool rows " + "use the same pre-row strong-repair projection. " + "Pings every 5s. " "Body is text/event-stream — the response schema is omitted " "from the catalog because OpenAPI 3.1 has no first-class SSE " "type." ), + query_params=[ + QueryParam( + "last_event_id", + "Numeric per-workstream event cursor for manual reconnects.", + schema_type="integer", + ), + QueryParam( + "history_token", + "Opaque one-shot token naming the accepted prefix rendered from REST history.", + ), + QueryParam( + "user_turn", + "Set to 1 to receive typed user_turn events instead of history-repair frames.", + schema_type="integer", + ), + QueryParam( + "tool_turn", + "Set to 1 to receive final accepted tool_result projections.", + schema_type="integer", + ), + ], error_codes=[403, 404, 409, 503], tags=["Coordinator"], ), @@ -1552,7 +1586,19 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ description=( "Returns the tail of the conversation in OpenAI-like message " "format. Used by the page-load handshake; SSE handles updates " - "after that. Bounded by the ``limit`` query parameter." + "after that. Cold coordinators are rehydrated before history is served, " + "so every successful response participates in the REST-to-SSE handoff. " + "Messages are the requested tail " + "of one authoritative total accepted conversation-row prefix: user, " + "assistant, tool, and system rows, including projected compaction " + "checkpoints and " + "cancellation markers. The opaque handoff_token names the exact prefix " + "used for the render and is passed once on initial SSE registration. " + "Admission of a later row changes the token; durable acknowledgement does " + "not. If the durable prefix cannot be loaded, the endpoint returns 503 " + "with `History temporarily unavailable`; that response is not authoritative " + "and supplies no usable handoff token. Bounded by the ``limit`` query " + "parameter." ), response_model=WorkstreamHistoryResponse, query_params=[ diff --git a/turnstone/api/server_schemas.py b/turnstone/api/server_schemas.py index f194dab6..9d2d7699 100644 --- a/turnstone/api/server_schemas.py +++ b/turnstone/api/server_schemas.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, Field, model_validator # must remain available at runtime rather than behind TYPE_CHECKING. from pydantic.json_schema import SkipJsonSchema # noqa: TC002 -from turnstone.core.workstream import WorkstreamKind +from turnstone.core.workstream import ConversationPersistenceState, WorkstreamKind # --------------------------------------------------------------------------- # Workstream management @@ -19,6 +19,17 @@ from turnstone.core.workstream import WorkstreamKind class SendRequest(BaseModel): message: str = Field(description="User message text") + client_send_id: str | None = Field( + default=None, + min_length=1, + max_length=128, + pattern=r"^[A-Za-z0-9_-]+$", + description=( + "Opaque browser correlation token echoed in the accepted `user_turn` " + "event and history row. It is not an idempotency key; repeated sends " + "with the same value remain distinct turns." + ), + ) attachment_ids: list[str] | None = Field( default=None, description=( @@ -369,6 +380,15 @@ class WorkstreamInfo(BaseModel): parent_ws_id: str | None = None user_id: str = "" project_id: str | None = None + persistence_state: ConversationPersistenceState = Field( + default="healthy", + description=( + "Sanitized durable-history status for the loaded workstream: " + "healthy, pending its first save, retrying automatically, or blocked " + "by a permanent commit conflict. Older servers and unloaded rows " + "default to healthy." + ), + ) class ListWorkstreamsResponse(BaseModel): @@ -499,6 +519,13 @@ class DashboardWorkstream(BaseModel): parent_ws_id: str | None = None user_id: str = "" project_id: str | None = None + persistence_state: ConversationPersistenceState = Field( + default="healthy", + description=( + "Sanitized durable-history status for this live row. Contains no " + "storage error, commit key, retry count, or conversation content." + ), + ) pending_approval_details: list[PendingApprovalDetail] = Field( default_factory=list, description=( @@ -595,6 +622,13 @@ class WorkstreamDetailResponse(BaseModel): state: str user_id: str kind: WorkstreamKind = WorkstreamKind.INTERACTIVE + persistence_state: ConversationPersistenceState = Field( + default="healthy", + description=( + "Sanitized durable-history status: healthy, pending its first save, " + "retrying automatically, or blocked by a permanent commit conflict." + ), + ) pending_approval: bool = Field( default=False, description=( @@ -635,12 +669,16 @@ class WorkstreamHistoryResponse(BaseModel): messages: list[dict[str, Any]] = Field( default_factory=list, description=( - "Tail of the workstream's message history, projected to the " - "canonical render shape (``role`` may be ``system`` for " - "operator-context turns; flat tool_calls with verdict / " - "output_assessment; top-level source / attachments / reasoning; " - "derived denied / is_error / pending). Bounded " - "by the ``limit`` query parameter (default 100, max 500)." + "Requested limit-bounded tail of one authoritative total accepted " + "conversation-row prefix, projected to the canonical render shape. " + "Roles include " + "``user``, ``assistant``, ``tool``, and ``system``; compaction " + "checkpoints project as ``role=system, source=compaction`` and " + "cancellation-generated assistant/tool markers appear when present. " + "The projection also carries flat tool_calls with verdict / " + "output_assessment; top-level source / attachments / reasoning; and " + "derived denied / is_error / pending. Bounded by the ``limit`` query " + "parameter (default 100, max 500)." ), ) cursor: int | None = Field( @@ -656,6 +694,18 @@ class WorkstreamHistoryResponse(BaseModel): "client connects fresh." ), ) + handoff_token: str | None = Field( + default=None, + description=( + "Opaque token naming the exact accepted conversation-row prefix used " + "for this render. Present only while the workstream is loaded. A " + "client that renders this response passes the token once as the " + "initial event stream's ``history_token`` query parameter; the server " + "atomically validates it while registering the listener. Admission of " + "a later row changes the token; durable acknowledgement does not. " + "Clients must not inspect, persist, or reuse it for later reconnects." + ), + ) # --------------------------------------------------------------------------- diff --git a/turnstone/api/server_spec.py b/turnstone/api/server_spec.py index e3014ffa..cd0cc628 100644 --- a/turnstone/api/server_spec.py +++ b/turnstone/api/server_spec.py @@ -97,9 +97,15 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ "/v1/api/workstreams/{ws_id}/close", "POST", "Close a workstream", + description=( + "Unloads the live workstream while preserving storage. Returns 409 " + "when any accepted live conversation row still requires persistence " + "reconciliation; the workstream remains loaded and its history journal " + "is retained." + ), request_model=CloseWorkstreamRequest, response_model=StatusResponse, - error_codes=[400, 404], + error_codes=[400, 404, 409], tags=["Workstreams"], ), # --- Chat --- @@ -163,17 +169,27 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ "/v1/api/workstreams/{ws_id}/rewind", "POST", "Drop the last N conversation turns (emits clear_ui)", + description=( + "Claims the workstream mutation slot, durably truncates the requested " + "tail, then emits clear_ui. Concurrent sends are ordered after the " + "cut; a storage failure returns 503 without changing live history." + ), request_model=RewindRequest, response_model=StatusResponse, - error_codes=[400, 404], + error_codes=[400, 404, 503], tags=["Chat"], ), EndpointSpec( "/v1/api/workstreams/{ws_id}/retry", "POST", "Drop the last response and re-send the last user message", + description=( + "Uses one workstream worker claim for the durable truncation and the " + "replacement generation, so another send cannot enter between them. " + "A storage failure returns 503 without changing live history." + ), response_model=StatusResponse, - error_codes=[400, 404], + error_codes=[400, 404, 503], tags=["Chat"], ), # --- Streaming --- @@ -182,7 +198,37 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ "GET", "Per-workstream SSE event stream", description="Opens a Server-Sent Events stream scoped to a single workstream. " - "Returns text/event-stream. See API reference for event types.", + "After rendering REST history, pass its opaque handoff_token once as " + "?history_token=; it names the exact accepted conversation-row prefix used " + "for that render. A history_resync event closes this stream and requires a " + "fresh history read; numeric event replay is not a substitute. Native " + "Last-Event-ID reconnects take priority. Pass ?user_turn=1 to opt into " + "typed accepted-user events; otherwise those rows become a backward-compatible " + "strong-repair frame. Pass ?tool_turn=1 to receive the final accepted tool row " + "as a typed tool_result with accepted=true; without it, accepted tool rows use " + "the same pre-row strong-repair projection. Returns " + "text/event-stream. See API reference for event types.", + query_params=[ + QueryParam( + "last_event_id", + "Numeric per-workstream event cursor for manual reconnects.", + schema_type="integer", + ), + QueryParam( + "history_token", + "Opaque one-shot token naming the accepted prefix rendered from REST history.", + ), + QueryParam( + "user_turn", + "Set to 1 to receive typed user_turn events instead of history-repair frames.", + schema_type="integer", + ), + QueryParam( + "tool_turn", + "Set to 1 to receive final accepted tool_result projections.", + schema_type="integer", + ), + ], error_codes=[404], tags=["Streaming"], ), @@ -254,10 +300,20 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ description=( "Returns the tail of the conversation in OpenAI-like message " "format. Persisted-but-not-loaded workstreams (closed / " - "evicted) serve history without rehydrating. Lifted from " + "evicted) are rehydrated before history is served so every " + "successful response participates in the REST-to-SSE handoff. Lifted from " "the coord-only surface in the Stage 2 history/detail verb " "lift — interactive previously only exposed history through " - "the SSE replay on ``/events``." + "the SSE replay on ``/events``. Messages are the " + "requested limit-bounded tail of one authoritative total accepted " + "conversation-row prefix: " + "user, assistant, tool, and system rows, including projected compaction " + "checkpoints and cancellation markers. The opaque handoff_token names " + "the exact prefix used for the render and is passed once on initial SSE " + "registration. Admission of a later row changes the token; durable " + "acknowledgement does not. If the durable prefix cannot be loaded, the " + "endpoint returns 503 with `History temporarily unavailable`; that " + "response is not authoritative and supplies no usable handoff token." ), response_model=WorkstreamHistoryResponse, query_params=[ diff --git a/turnstone/channels/_sse.py b/turnstone/channels/_sse.py index 68a7a422..105ffbc3 100644 --- a/turnstone/channels/_sse.py +++ b/turnstone/channels/_sse.py @@ -76,6 +76,7 @@ async def run_sse_stream( "GET", url, headers=sse_headers, + params={"user_turn": 1}, ) as event_source: status = event_source.response.status_code if status == 404: diff --git a/turnstone/console/collector.py b/turnstone/console/collector.py index d190967e..8299e087 100644 --- a/turnstone/console/collector.py +++ b/turnstone/console/collector.py @@ -22,7 +22,10 @@ from typing import TYPE_CHECKING, Any import httpx import httpx_sse -from turnstone.core.workstream import WorkstreamKind +from turnstone.core.workstream import ( + WorkstreamKind, + normalize_conversation_persistence_state, +) if TYPE_CHECKING: from collections.abc import Callable @@ -539,6 +542,9 @@ class ClusterCollector: continue ws["node"] = node_id ws["server_url"] = node.server_url + ws["persistence_state"] = normalize_conversation_persistence_state( + ws.get("persistence_state") + ) new_ws[ws_id] = ws new_ids = set(new_ws.keys()) # Additions @@ -555,6 +561,7 @@ class ClusterCollector: "parent_ws_id": ws.get("parent_ws_id"), "project_id": ws.get("project_id", "") or "", "persona": ws.get("persona", "") or "", + "persistence_state": ws["persistence_state"], # Mirror the SSE-relay path: the tenancy filter's # ws-creator shortcut reads this. "user_id": ws.get("user_id", "") or "", @@ -569,7 +576,11 @@ class ClusterCollector: new_w = new_ws[ws_id] old_state = old_ws.get("state", "") new_state = new_w.get("state", "") - if old_state != new_state: + old_persistence = normalize_conversation_persistence_state( + old_ws.get("persistence_state") + ) + new_persistence = new_w["persistence_state"] + if old_state != new_state or old_persistence != new_persistence: pending.append( { "type": "cluster_state", @@ -581,6 +592,7 @@ class ClusterCollector: "kind": WorkstreamKind.from_raw(new_w.get("kind")), "parent_ws_id": new_w.get("parent_ws_id"), "activity_state": new_w.get("activity_state", ""), + "persistence_state": new_persistence, } ) old_name = old_ws.get("title", "") or old_ws.get("name", "") @@ -632,6 +644,10 @@ class ClusterCollector: ws["context_ratio"] = data.get("context_ratio", ws.get("context_ratio", 0)) ws["activity"] = data.get("activity", ws.get("activity", "")) ws["activity_state"] = data.get("activity_state", ws.get("activity_state", "")) + if "persistence_state" in data: + ws["persistence_state"] = normalize_conversation_persistence_state( + data["persistence_state"] + ) # kind/parent_ws_id: defensive update from ws_state event. # These rarely change but the event carries them so the # collector's entry stays authoritative even if a delta @@ -651,6 +667,9 @@ class ClusterCollector: "kind": WorkstreamKind.from_raw(ws.get("kind")), "parent_ws_id": ws.get("parent_ws_id"), "activity_state": ws.get("activity_state", ""), + "persistence_state": normalize_conversation_persistence_state( + ws.get("persistence_state") + ), } ) @@ -694,6 +713,9 @@ class ClusterCollector: "user_id": ws_user, "project_id": ws_project, "persona": ws_persona, + "persistence_state": normalize_conversation_persistence_state( + data.get("persistence_state") + ), } pending_events.append( { @@ -1233,6 +1255,7 @@ class ClusterCollector: # gates on this — a missing project_id fails open. "project_id": project_id or "", "persona": persona or "", + "persistence_state": "healthy", "updated": now, } pending.append( @@ -1247,6 +1270,7 @@ class ClusterCollector: "user_id": user_id or "", "project_id": project_id or "", "persona": persona or "", + "persistence_state": "healthy", } ) for event in pending: @@ -1271,6 +1295,7 @@ class ClusterCollector: activity: str = "", activity_state: str = "", content: str = "", + persistence_state: str = "healthy", ) -> None: """Update the coordinator row's state on the console pseudo-node + fan out. @@ -1303,6 +1328,7 @@ class ClusterCollector: entry["context_ratio"] = context_ratio entry["activity"] = activity entry["activity_state"] = activity_state + entry["persistence_state"] = normalize_conversation_persistence_state(persistence_state) self._fanout( { "type": "cluster_state", @@ -1314,6 +1340,7 @@ class ClusterCollector: "kind": WorkstreamKind.COORDINATOR.value, "parent_ws_id": None, "activity_state": activity_state, + "persistence_state": normalize_conversation_persistence_state(persistence_state), } ) diff --git a/turnstone/console/coordinator_adapter.py b/turnstone/console/coordinator_adapter.py index 75d16e08..9d562978 100644 --- a/turnstone/console/coordinator_adapter.py +++ b/turnstone/console/coordinator_adapter.py @@ -16,6 +16,7 @@ for the storage-seeded children rebuild. from __future__ import annotations import queue +import threading from typing import TYPE_CHECKING, Any from turnstone.core import session_worker @@ -26,7 +27,12 @@ from turnstone.core.children_registry import ChildrenRegistry from turnstone.core.log import get_logger from turnstone.core.memory import get_workstream_display_name, get_workstream_display_names from turnstone.core.storage import is_storage_initialized -from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState +from turnstone.core.workstream import ( + Workstream, + WorkstreamKind, + WorkstreamState, + workstream_persistence_state, +) if TYPE_CHECKING: from collections.abc import Callable @@ -40,6 +46,30 @@ if TYPE_CHECKING: log = get_logger(__name__) +def _emit_coord_ui(ui: Any, ws: Workstream, hook: str, *args: Any) -> None: + """Fire ONE coordinator UI hook in isolation. + + The coord counterpart of the interactive route's ``_emit_send_ui``, + with the same contract for the same reason: convergence emits come in + pairs, and a failure in one hook (a listener-queue overflow inside the + stream-end flush, say) must not suppress the sibling that follows — + the missed one is invariably the ``idle`` state change whose absence + leaves the dashboard row stuck busy. + """ + method = getattr(ui, hook, None) + if not callable(method): + return + try: + method(*args) + except Exception: + log.debug( + "coord_adapter.ui_hook_failed ws=%s hook=%s", + ws.id[:8], + hook, + exc_info=True, + ) + + def _coord_display_name(ws: Workstream) -> str: """Resolve a coordinator's display name (``alias > title > name``). @@ -204,6 +234,7 @@ class CoordinatorAdapter: } ws_id = ws.id state_value = state.value + persistence_state = workstream_persistence_state(ws) def _emit() -> None: try: @@ -215,6 +246,7 @@ class CoordinatorAdapter: activity=payload["activity"], activity_state=payload["activity_state"], content=payload["content"], + persistence_state=persistence_state, ) except Exception: log.debug("coord_adapter.state_fanout_failed ws=%s", ws_id[:8], exc_info=True) @@ -302,10 +334,17 @@ class CoordinatorAdapter: ) -> bool: """Queue a message onto a coordinator session's ChatSession. - Returns False if the coordinator isn't loaded in the manager or - if the worker's pending-message queue is full (caller should - surface 429 / backpressure). Priority is parsed from the message - prefix (``/high``, ``/urgent``, etc.) by :meth:`ChatSession.queue_message`. + Returns False when the dispatch is refused; the caller should surface + 429 / backpressure. ``False`` is one undifferentiated signal covering + every refusal — the coordinator isn't loaded in the manager, the + worker's pending-message queue is full, a command window or the + order barrier holds the slot, or the sender's fresh turn was blocked + by another participant's retained queued input. The interactive routes + distinguish that last case as ``cross_user_interjection`` (HTTP 409) + because they have a per-refusal return channel; this signature has + none, so each refusal names its reason in a ``coord_adapter.*`` log + instead. Priority is parsed from the message prefix (``/high``, + ``/urgent``, etc.) by :meth:`ChatSession.queue_message`. ``acting_user_id`` is the authenticated sender. On a fresh turn it is bound as the coordinator's acting user — with coordinator MCP enabled @@ -335,6 +374,11 @@ class CoordinatorAdapter: ``None`` so the steady-state ``coord_adapter.send`` call sites (no attachments) keep working unchanged. """ + # Function-local like CrossUserInterjectionError below: the session + # module imports console-free core only, but the console package + # must not import it at module load. + from turnstone.core.session import GenerationCancelled + mgr = self._manager if mgr is None: raise RuntimeError( @@ -352,6 +396,7 @@ class CoordinatorAdapter: _send_id = send_id if _attachments else None def _run() -> None: + me = threading.current_thread() try: # Fresh turn: bind the authenticated sender so any MCP tools run # under their credentials and the acting-user signal is correct @@ -362,6 +407,25 @@ class CoordinatorAdapter: if callable(bind): bind(acting_user_id) session.send(message, attachments=_attachments, send_id=_send_id) + except GenerationCancelled: + # Safety net, the coord sibling of the /send route's arm: a + # Stop that lands between the slot claim and send()'s own + # convergence envelope raises out of send(), and + # ``session_worker._runner`` catches only ``Exception`` — a + # BaseException here would kill the worker thread via + # ``threading.excepthook`` and leave the dashboard's child + # row stuck busy/streaming. If this thread was + # force-abandoned, ``ws.worker_thread`` was cleared — don't + # emit spurious events over the successor's stream. + if ws_ref.worker_thread is me: + # Both hooks, like the interactive /send and retry + # siblings: ConsoleCoordinatorUI INHERITS a concrete + # on_stream_end from SessionUIBase (finishAssistantStream + # + the tool-output batcher flush live behind it) — a + # state_change alone leaves an unfinalized streaming + # bubble and unflushed chunks on the pane. + _emit_coord_ui(session.ui, ws_ref, "on_stream_end") + _emit_coord_ui(session.ui, ws_ref, "on_state_change", "idle") except Exception: # Attachments were resolved (peeked) from the per-node upload # buffer, not soft-locked — there is no reservation to release @@ -376,7 +440,13 @@ class CoordinatorAdapter: # error for the cluster fan-out / dashboard via # :meth:`ChatSession._record_fatal_error`. The adapter # owns ONLY the worker-level cleanup (attachments, - # logging) above. + # logging) above — plus the one thing send() cannot emit + # for a raise that escaped its envelope: the stream-end + # hook, whose absence leaves the pane's assistant bubble + # unfinalized and its tool-output batch unflushed (the + # interactive sibling has always emitted it here). + if ws_ref.worker_thread is me: + _emit_coord_ui(session.ui, ws_ref, "on_stream_end") def _enqueue() -> None: # Queued user turns can't carry attachments (see @@ -399,13 +469,32 @@ class CoordinatorAdapter: # a message that would be capped and could cross a /resume # identity swap. raise queue.Full() + admission = session_worker.claimed_slot_queue_admission(ws, acting_user_id) + if admission is None: + from turnstone.core.session import CrossUserInterjectionError + + raise CrossUserInterjectionError("Another participant's turn is in flight") + _claimed, queue_kwargs = admission att_ids = [a.attachment_id for a in _attachments] if _attachments else None - session.queue_message( - message, - attachment_ids=att_ids, - queue_msg_id=_send_id, - interjector_user_id=acting_user_id, + queue_kwargs.update({"attachment_ids": att_ids, "queue_msg_id": _send_id}) + session.queue_message(message, **queue_kwargs) + + def _before_spawn() -> bool: + if not session_worker.foreign_queue_conflict(session, acting_user_id): + return True + # This refusal reaches the caller as a bare ``False``, which carries + # none of the classification the route layer's twin surfaces as + # ``rejected="cross_user_interjection"`` (HTTP 409). Name the + # reason here — this closure is the only place the cross-user + # judgement exists on the coordinator path, so an unlogged refusal + # would be indistinguishable from a full queue or an unloaded + # workstream when an operator asks why a message vanished. + log.warning( + "coord_adapter.send_refused_cross_user_queued_input ws=%s user=%s", + ws.id[:8], + acting_user_id, ) + return False # Order-barrier yield, mirroring the /send route's pre-check: once # deferred sends are pending (or a claimed entry's dispatch is in @@ -430,7 +519,10 @@ class CoordinatorAdapter: ws, enqueue=_enqueue, run=_run, + expected_session=session, + before_spawn=_before_spawn, thread_name=f"coord-worker-{ws.id[:8]}", + principal_id=acting_user_id, ) # ------------------------------------------------------------------ diff --git a/turnstone/console/coordinator_client.py b/turnstone/console/coordinator_client.py index fd3091ef..27a3ab60 100644 --- a/turnstone/console/coordinator_client.py +++ b/turnstone/console/coordinator_client.py @@ -2152,6 +2152,7 @@ class CoordinatorClient: _PROVIDER_FIDELITY_KEYS: frozenset[str] = frozenset({"_provider_content", "provider_blocks"}) +_PRIVATE_MESSAGE_KEYS: frozenset[str] = frozenset({"_commit_key", "_provenance"}) def _serialize_messages( @@ -2173,9 +2174,15 @@ def _serialize_messages( for r in rows: if isinstance(r, dict): if include_provider_content: - out.append(r) + out.append({k: v for k, v in r.items() if k not in _PRIVATE_MESSAGE_KEYS}) else: - out.append({k: v for k, v in r.items() if k not in _PROVIDER_FIDELITY_KEYS}) + out.append( + { + k: v + for k, v in r.items() + if k not in _PROVIDER_FIDELITY_KEYS and k not in _PRIVATE_MESSAGE_KEYS + } + ) else: # Fall back to a string repr so at least something lands. out.append({"raw": str(r)}) diff --git a/turnstone/console/coordinator_ui.py b/turnstone/console/coordinator_ui.py index 8d3f3907..fbb4eefb 100644 --- a/turnstone/console/coordinator_ui.py +++ b/turnstone/console/coordinator_ui.py @@ -276,6 +276,44 @@ class ConsoleCoordinatorUI(SessionUIBase): evt["acting_user_id"] = self._acting_user_id self._enqueue(evt) + def on_persistence_state_changed(self) -> None: + """Refresh the cluster row after journal failure or recovery. + + The update stays on the operator cluster bus; it does not enter the + coordinator conversation event stream. It snapshots counters without + consuming terminal turn content and does not persist a synthetic + workstream-state transition. + """ + # The registry read serves ONLY the row-state field (``ws.state`` + # lives on the manager's row); the persistence field derives through + # the session bound at construction, so a miss — or an id-reuse + # replacement — can no longer report another session's journal. A + # miss means the row left the cluster roster: nothing to refresh. + mgr = ConsoleCoordinatorUI._coord_mgr + collector = ConsoleCoordinatorUI._collector + if mgr is None or collector is None: + return + ws = mgr.get(self.ws_id) + if ws is None: + return + payload = self.snapshot_state_payload_non_consuming() + try: + collector.emit_console_ws_state( + self.ws_id, + ws.state.value, + tokens=payload["tokens"], + context_ratio=payload["context_ratio"], + activity=payload["activity"], + activity_state=payload["activity_state"], + persistence_state=self._current_persistence_state(), + ) + except Exception: + log.debug( + "coord_ui.persistence_state_fanout_failed ws=%s", + self.ws_id, + exc_info=True, + ) + def on_state_change_deferred( self, state: str, diff --git a/turnstone/console/server.py b/turnstone/console/server.py index f59f3473..5a920b23 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -44,7 +44,7 @@ from turnstone.api.console_spec import build_console_spec from turnstone.api.docs import make_docs_handler, make_openapi_handler from turnstone.console.collector import ClusterCollector from turnstone.console.coordinator_alias import resolve_coordinator_alias -from turnstone.console.coordinator_client import load_task_envelope +from turnstone.console.coordinator_client import _serialize_messages, load_task_envelope from turnstone.console.metrics import ConsoleMetrics from turnstone.console.router import ConsoleRouter from turnstone.core.audit import record_audit @@ -110,7 +110,11 @@ from turnstone.core.web_helpers import ( read_json_or_400, require_storage_or_503, ) -from turnstone.core.workstream import Workstream, WorkstreamKind +from turnstone.core.workstream import ( + Workstream, + WorkstreamKind, + workstream_persistence_state, +) if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Iterable @@ -633,6 +637,7 @@ def _coordinator_rows(request: Request) -> list[dict[str, Any]]: "user_id": ws.user_id or "", "project_id": ws.project_id or "", "persona": ws.persona or "", + "persistence_state": workstream_persistence_state(ws), } ) seen.add(ws.id) @@ -663,6 +668,8 @@ def _coordinator_rows(request: Request) -> list[dict[str, Any]]: "user_id": row_owner, "project_id": m.get("project_id") or "", "persona": m.get("persona") or "", + # Persisted-only rows have no in-memory journal to inspect. + "persistence_state": "healthy", } ) return rows @@ -719,6 +726,7 @@ _CLUSTER_WS_LIVE_KEYS = ( "model_alias", "title", "name", + "persistence_state", # Carries the inline approve/deny payloads (one per live cycle, # items + judge_verdict each) so coord live-bulk callers can render # row-level UI without a per-child round-trip. ``[]`` when no @@ -870,6 +878,7 @@ def _coordinator_live_snapshot(ws: Any) -> dict[str, Any]: "pending_approval": pending_approval, "pending_approval_details": pending_approval_details, "recent_auto_approvals": recent_auto_approvals, + "persistence_state": workstream_persistence_state(ws), } @@ -954,6 +963,7 @@ async def _fetch_live_block( for entry in payload.get("workstreams", []) or []: if isinstance(entry, dict) and entry.get("ws_id") == ws_id: live = {k: entry.get(k) for k in _CLUSTER_WS_LIVE_KEYS if k in entry} + live.setdefault("persistence_state", "healthy") # Derived field — kept in lockstep with # _coordinator_live_snapshot so both origins produce the # same keys. ``state="attention"`` is the canonical signal; @@ -1082,8 +1092,8 @@ async def cluster_ws_detail(request: Request) -> JSONResponse: # Tail-N bound pushed into SQL (load_messages supports limit # on both backends). Offloaded to the default executor so # the async SSE loop stays unblocked under rapid fan-out. - messages = await asyncio.to_thread( - storage.load_messages, ws_id, limit=limit, repair=False + messages = _serialize_messages( + await asyncio.to_thread(storage.load_messages, ws_id, limit=limit, repair=False) ) except Exception: log.debug("cluster_ws_detail.load_messages_failed", exc_info=True) @@ -3674,40 +3684,6 @@ def _audit_retry_coordinator( ) -def _coord_dispatch_retry(ws: Workstream, user_msg: str) -> None: - """Re-send ``user_msg`` on a coordinator workstream after ``/retry``. - - Passed to :func:`make_retry_handler` as ``dispatch_retry``. Mirrors - :meth:`CoordinatorAdapter.send`'s worker shape — drives the shared - :func:`turnstone.core.session_worker.send` dispatcher — but without - attachment handling (a retry re-sends an existing text turn). The - ``run`` closure's error handling is intentionally light: - :meth:`ChatSession.send` already surfaces failures to SSE, persists - ``last_error`` and emits state=error via ``_record_fatal_error``, and - the shared dispatcher owns the ``_worker_running`` lifecycle — so the - worker only logs. The ``enqueue`` closure hard-rejects (a retry must - not silently queue behind an in-flight turn). - """ - from turnstone.core import session_worker - - session = ws.session - ui = ws.ui - if session is None: - return - - def _run() -> None: - try: - session.send(user_msg) - except Exception: - log.exception("coord.retry.worker_failed ws=%s", ws.id[:8]) - - def _enqueue() -> None: - if ui is not None and hasattr(ui, "on_error"): - ui.on_error("Cannot retry: workstream is busy") - - session_worker.send(ws, enqueue=_enqueue, run=_run, thread_name=f"coord-retry-{ws.id[:8]}") - - def _coord_events_replay( ws: Workstream, ui: Any, @@ -3858,7 +3834,7 @@ async def _coord_create_post_install( Wired onto :attr:`SessionEndpointConfig.create_post_install`. When an ``initial_message`` is provided, dispatches via :meth:`CoordinatorAdapter.send`; any uploaded ``attachment_ids`` - are resolved from the buffer onto the first turn (and drained) so + are resolved from the buffer onto the first turn so the worker picks them up exactly the way interactive's ``post_install`` worker thread does. @@ -3876,24 +3852,13 @@ async def _coord_create_post_install( if coord_adapter is None: return {} - # Resolve (peek) the staged uploads for the dispatched first turn; the - # committing ``ChatSession.send`` drains them from the per-node buffer and - # persists them content-addressed. ``send_id`` is a tracking token only — - # no DB reservation to release on worker failure. + # Resolve (peek) the staged uploads for the dispatched first turn. The + # accepted USER journal admission atomically transfers their buffer + # ownership; a refusal before admission leaves them staged for retry. send_id = _uuid.uuid4().hex resolved_atts: list[Any] = [] if attachment_ids: resolved_atts, _ord, _drop = resolve_staged_attachments(attachment_ids, ws.id, uid) - # Drain the staged uploads now: the create-time dispatch is their only - # consumer, so leaving them staged would let the new coord pane's - # rehydrate race the worker's write-time drain and show them as still- - # pending composer chips (the committing send's discard then no-ops). - if _ord: - from turnstone.core.attachment_buffer import get_attachment_buffer - - _buf = get_attachment_buffer() - for _aid in _ord: - _buf.discard(_aid, ws_id=ws.id, user_id=uid) coord_adapter.send( ws.id, initial_message, @@ -5031,14 +4996,14 @@ def _coord_idle_cleanup_thread( behind by prior console process incarnations. Runs an initial sweep BEFORE the first wait so cold-start orphans are - reaped immediately. ``timeout_sec == 0`` disables ordinary idle eviction, - but the independent provisional-create recovery still runs with its fixed - conservative grace and cadence. + reaped immediately. A short tick also reconciles due accepted-row writes. + ``timeout_sec == 0`` disables ordinary idle eviction, but both persistence + and provisional-create recovery remain active at their independent + cadences. When idle eviction is enabled, the wait subscribes to manager state and a - transition can wake the next sweep early. With idle eviction disabled, the - thread uses only the fixed provisional-create cadence so ordinary turn - transitions do not cause redundant storage scans. + transition can request the next idle sweep early. The persistence tick does + not accelerate those idle/orphan storage scans. ``min_sweep_interval`` is the hard floor between successive ``close_idle`` calls (default 5 s) — without it, sustained @@ -5061,19 +5026,21 @@ def _coord_idle_cleanup_thread( ``stop_event`` is the lifecycle shutdown signal and ``wake_event`` is the shared state-change/shutdown wake path. Lifecycle owners set both so an - idle-enabled thread cannot remain blocked in its long heartbeat wait. + idle-enabled thread exits immediately. """ from turnstone.core.session_manager import ( + PERSISTENCE_RECONCILE_INTERVAL_SECONDS, STALE_CREATE_GRACE_SECONDS, STALE_CREATE_SWEEP_INTERVAL_SECONDS, ) idle_enabled = timeout_sec > 0 - check_every = ( + lifecycle_check_every = ( min(STALE_CREATE_SWEEP_INTERVAL_SECONDS, timeout_sec / 4) if idle_enabled else float(STALE_CREATE_SWEEP_INTERVAL_SECONDS) ) + check_every = min(PERSISTENCE_RECONCILE_INTERVAL_SECONDS, lifecycle_check_every) tick_now = wake_event if wake_event is not None else threading.Event() def _on_state_change(_ws_id: str, _state: Any) -> None: @@ -5110,6 +5077,10 @@ def _coord_idle_cleanup_thread( log.debug("console.coord_stale_create_cleanup_failed", exc_info=True) try: + try: + mgr.reconcile_unresolved_persistence() + except Exception: + log.debug("console.coord_persistence_reconcile_initial_failed", exc_info=True) # Initial sweep — runs once before entering the wait loop. # ``tick_now`` is intentionally not cleared here: any # state-change event that arrives between subscribe and the @@ -5117,38 +5088,36 @@ def _coord_idle_cleanup_thread( # discarded. _sweep(initial=True) last_sweep_at = time.monotonic() + early_sweep_requested = False while True: if stop_event is not None and stop_event.is_set(): return if idle_enabled: - tick_now.wait(check_every) + if tick_now.wait(check_every): + early_sweep_requested = True elif stop_event is not None: stop_event.wait(check_every) else: time.sleep(check_every) if stop_event is not None and stop_event.is_set(): return - # Clear BEFORE the cadence floor so any state-change event - # arriving during the cooldown (or during the close_idle - # below) leaves ``tick_now`` set — the next loop iteration - # then re-enters ``wait`` already-set and re-evaluates - # promptly. close_idle is idempotent so a spurious extra - # tick is just one redundant scan. tick_now.clear() - # Cadence floor — see docstring for the tight-spin - # hazard rationale. Cooldown uses ``stop_event.wait`` - # (not ``time.sleep``) so the test stop hook still - # terminates promptly during the cooldown window. + try: + mgr.reconcile_unresolved_persistence() + except Exception: + log.debug("console.coord_persistence_reconcile_failed", exc_info=True) + # Persistence repair has a one-second heartbeat, but expensive + # close-idle/orphan scans retain their original heartbeat and + # state-wake floor. Keep an early request latched instead of + # sleeping through persistence ticks during the cooldown. since_last = time.monotonic() - last_sweep_at - if since_last < min_sweep_interval: - gap = min_sweep_interval - since_last - if stop_event is not None: - if stop_event.wait(gap): - return - else: - time.sleep(gap) + heartbeat_due = since_last >= lifecycle_check_every + early_due = idle_enabled and early_sweep_requested and since_last >= min_sweep_interval + if not heartbeat_due and not early_due: + continue _sweep() last_sweep_at = time.monotonic() + early_sweep_requested = False finally: if idle_enabled: mgr.unsubscribe_from_state(_on_state_change) @@ -15910,7 +15879,6 @@ def create_app( ), retry=make_retry_handler( # lifted: shared body (#549) coord_endpoint_config, - dispatch_retry=_coord_dispatch_retry, audit_emit=_audit_retry_coordinator, ), events=make_events_handler(coord_endpoint_config), # lifted: shared body diff --git a/turnstone/console/static/app.js b/turnstone/console/static/app.js index d7d36630..d41a2637 100644 --- a/turnstone/console/static/app.js +++ b/turnstone/console/static/app.js @@ -100,6 +100,36 @@ const STATE_DISPLAY = { error: { symbol: "\u2716", label: "err" }, }; +const PERSISTENCE_DISPLAY = { + pending: { + label: "History save pending", + tooltip: "An accepted conversation turn has not reached durable history yet.", + }, + retrying: { + label: "History save retrying", + tooltip: + "An accepted conversation turn has not reached durable history yet. Automatic recovery is in progress.", + }, + conflict: { + label: "History save blocked", + tooltip: + "An accepted conversation turn cannot reach durable history automatically. Operator intervention is required.", + }, +}; + +function appendPersistenceStatus(container, ws) { + const display = PERSISTENCE_DISPLAY[ws.persistence_state]; + if (!display) return; + const badge = document.createElement("span"); + badge.className = "dash-persistence-badge"; + badge.dataset.state = ws.persistence_state; + badge.textContent = display.label; + badge.title = display.tooltip; + badge.setAttribute("role", "status"); + badge.setAttribute("aria-label", display.label + ". " + display.tooltip); + container.appendChild(badge); +} + // --- Cluster State Model --- function applySnapshot(data) { clusterState = { @@ -126,6 +156,8 @@ function patchClusterState(data) { if ("context_ratio" in data) ws.context_ratio = data.context_ratio; if ("activity" in data) ws.activity = data.activity; if ("activity_state" in data) ws.activity_state = data.activity_state; + if ("persistence_state" in data) + ws.persistence_state = data.persistence_state; } }); } @@ -145,6 +177,7 @@ function patchClusterState(data) { activity: "", activity_state: "", tool_calls: 0, + persistence_state: data.persistence_state || "healthy", // ws_created SSE events carry kind / parent_ws_id / user_id / // project_id / persona; preserve them on the in-memory ws so the // home-landing active-coordinators list and the tree grouping both @@ -743,6 +776,9 @@ function _renderWsRow(ws, opts, container) { if (ws.tokens) ariaLabel += ", " + formatTokens(ws.tokens) + " tokens"; if (ws.context_ratio > 0) ariaLabel += ", " + Math.round(ws.context_ratio * 100) + "% context"; + const persistenceDisplay = PERSISTENCE_DISPLAY[ws.persistence_state]; + if (persistenceDisplay) + ariaLabel += ", " + persistenceDisplay.label.toLowerCase(); if (opts.isCoordinator && opts.childCount != null) ariaLabel += ", " + opts.childCount + " children"; if (opts.isOrphan) ariaLabel += ", orphan"; @@ -913,7 +949,11 @@ function _renderWsRow(ws, opts, container) { const sub = document.createElement("div"); sub.className = "dash-row-sub"; if (ws.activity_state === "approval") sub.classList.add("sub-attention"); - sub.textContent = ws.activity || ""; + appendPersistenceStatus(sub, ws); + if (ws.activity) { + if (sub.childNodes.length) sub.append(" \u00b7 "); + sub.append(ws.activity); + } row.appendChild(sub); // Deep link: click opens proxied server UI at this workstream. diff --git a/turnstone/console/static/coordinator/coordinator.js b/turnstone/console/static/coordinator/coordinator.js index 80014a1b..5e056773 100644 --- a/turnstone/console/static/coordinator/coordinator.js +++ b/turnstone/console/static/coordinator/coordinator.js @@ -40,15 +40,21 @@ import { buildConvActions, buildConvStatus, buildConvResult, + buildPreviewChip, batchKicker, indexLabel, } from "/shared/conversation.js"; import { redactCredentials } from "/shared/redact_credentials.js"; import { tryParseMcpError, buildMcpErrorEmbed } from "/shared/mcp_error.js"; import { + acceptUserTurnEvent, + clientSendMaySettleForViewer, createQueueController, + mintClientSendId, parsePriority, - settleSendResponse, + postAndSettleSend, + settleAcceptedClientSends, + viewerUserId, } from "/shared/composer_queue.js"; import { OVERFLOW_TRIP_COUNT, @@ -62,6 +68,20 @@ import { overflowWindowTripped, degradedCooldownStep, } from "/shared/sse_overflow.js"; +import { + createHistoryHandoffDeadline, + createHistoryHandoffRepair, + HISTORY_HANDOFF_FETCH_TIMEOUT_MS, +} from "/shared/history_handoff.js"; +import { + acceptedToolEventAlreadyRendered, + enqueueToolOccurrence, + indexHistoryToolOutcomes, + indexLatestToolRow, + recordAcceptedToolEvent, + shiftToolOccurrence, + shouldRefreshTasksForToolResult, +} from "/shared/tool_projection.js"; // Standalone-page pending-consent chip (#874): the rail-less coordinator // page's counterpart of the L-shell rail badge. The pending set is @@ -679,13 +699,20 @@ function createCoordinatorPane(root, wsId, opts) { // an older snapshot landing late can neither double-render nor clear // the staleness latch over a newer truth. let refetchSeq = 0; - // EVERY in-flight /history AbortController — destroy() aborts them all - // so a slow fetch cannot pin the destroyed pane's closure for the - // bound's full 15s (the same dead-not-inert ruling destroy() applies - // to staleRetryTimer). A Set, not a single slot (r9): overlapping - // dispatches settle in any order, and a newest-wins slot nulled by - // the newer dispatch's finally left the OLDER fetch unabortable. - const histCtrls = new Set(); + // EVERY in-flight /history attempt as ONE composite { ctrl, deadline } + // record — destroy() aborts the fetch AND disposes the logical deadline + // from the same entry, so a future attempt site cannot register into + // one bookkeeping structure and not the other (an unaborted fetch pins + // the destroyed pane's closure for the bound's full 15s; an undisposed + // deadline leaves its timer + settle slot alive past destroy — the + // dead-not-inert ruling either way). A Set, not a single slot (r9): + // overlapping dispatches settle in any order, and a newest-wins slot + // nulled by the newer dispatch's finally left the OLDER fetch + // unabortable. `ctrl` is null on runtimes without AbortController — + // an optimization, not the settle guarantee; the deadline is + // unconditional and retirement always goes through handle.dispose(), + // never direct state-slot pokes. + const histAttempts = new Set(); // call_ids of tool calls whose results are still ARRIVING ON THE LIVE // STREAM — the render-time gate's tool-phase liveness signal. Fed // ONLY by live SSE events (tool_pending / tool_info add, tool_result @@ -747,6 +774,36 @@ function createCoordinatorPane(root, wsId, opts) { // brand-new EventSource (initial connect, scheduleReconnect after // close). let lastEventId = null; + // One-shot token tying a rendered REST history revision to its initial SSE + // listener registration. connectSSE consumes it once; subsequent reconnects + // are driven solely by Last-Event-ID / lastEventId. + let historyHandoffToken = null; + // A history_resync means the rendered transcript is not authoritative. + // Keep that repair intent across failed /history calls and every transport + // lifecycle edge; no cursorless/tokenless EventSource may open until a + // fresh response renders and supplies its handoff token. One capped + // exponential timer owns retries. The latch, budget, backoff, and parked + // prompt live in the shared controller so this pane and the interactive one + // cannot drift on them. + const historyRepair = createHistoryHandoffRepair({ + baseDelayMs: STALE_RETRY_BASE_MS, + jitterMs: STALE_RETRY_JITTER_MS, + maxMs: DEGRADED_COOLDOWN_MAX_MS, + isAlive: () => !!visHandler, + load: (_scope, manualAttempt) => loadHistoryThenReconnect(manualAttempt), + connect: () => connectSSE(), + setStale: (stale) => { + historyStale = stale; + }, + deferToShowEdge: () => { + hiddenDisconnect = true; + }, + showPaused: () => setSseStatus("Live updates paused", "err"), + placePrompt: (prompt) => { + messagesEl.appendChild(prompt); + _scheduleScroll(); + }, + }); let reconnectTimer = null; // --- SSE overflow-recovery state (client half — mirrors interactive.js) --- // Field instrumentation for the two distinct "output stops while the backend @@ -804,6 +861,8 @@ function createCoordinatorPane(root, wsId, opts) { // by the system_turn handler — reset per refetchHistory. Mirrors // ui/static/app.js's per-pane _renderedSystemEventIds. const renderedSystemEventIds = new Set(); + const renderedUserEventIds = new Set(); + const renderedToolEventIds = new Set(); // Compaction lifecycle holder for the shared reducer // (conversation.applyCompactionEvent); `card` is the in-progress card @@ -1363,7 +1422,12 @@ function createCoordinatorPane(root, wsId, opts) { // composer staged on submit. Attachments is a list of // {kind, filename}; falsy/empty falls through to plain text. function appendUserMessageWithAttachments(text, attachments, opts) { + opts = opts || {}; const el = appendText("user", text, opts); + if (opts.clientSendId) el.dataset.clientSendId = opts.clientSendId; + if (opts.eventId != null) el.dataset.eventId = String(opts.eventId); + if (opts.sender) el.dataset.sender = opts.sender; + if (opts.source) el.dataset.source = opts.source; // Per-message edit + rewind affordance (#549) on every user turn, // matching the interactive pane. Attached before the early-return so // image-only sends (no attachments) still get the action bar. @@ -1419,6 +1483,37 @@ function createCoordinatorPane(root, wsId, opts) { return el; } + function markAcceptedClientSends( + clientSendIds, + remove, + skipAlreadyAccepted = false, + ) { + return settleAcceptedClientSends( + messagesEl, + queue, + clientSendIds, + remove, + skipAlreadyAccepted, + ); + } + + function acceptUserTurn(ev) { + acceptUserTurnEvent(ev, { + renderedEventIds: renderedUserEventIds, + messagesEl: messagesEl, + queue: queue, + consumeAttachments: (ids) => attachments.consume(ids, []), + renderNudgeMarker: appendSystemNudgeMarker, + renderUserTurn: (content, atts, opts) => + appendUserMessageWithAttachments(content, atts || [], { + label: opts.sender && opts.sender === opts.viewer ? "you" : "user", + eventId: opts.eventId, + sender: opts.sender, + source: opts.source, + }), + }); + } + // Thin ``.msg.user.system-nudge`` marker rendered for a wake-driven // empty user turn. Replaces the previously-invisible synthetic empty // user turn with a visible-but-subtle DOM element; the nudges it @@ -1492,7 +1587,12 @@ function createCoordinatorPane(root, wsId, opts) { function appendToolResult(name, callId, output, isError, opts) { if (callId && toolRows.has(callId)) { const entry = toolRows.get(callId); - _appendResultToRow(entry.row, output, isError, opts); + const prior = toolResultNodes.get(callId); + if (prior && prior.row == null && prior.node && prior.node.isConnected) { + prior.node.remove(); + } + const resultNode = _appendResultToRow(entry.row, output, isError, opts); + toolResultNodes.set(callId, { row: entry.row, node: resultNode }); // The batch may have been --running (live tool_info auto path, // approval_resolved approved path, or replay-time orphan). // Drop --running once every row in the batch has a result so @@ -1509,6 +1609,10 @@ function createCoordinatorPane(root, wsId, opts) { } // Orphan result (no live row — replay edge): still render the MCP // error card rather than the raw envelope (#725). + const prior = callId ? toolResultNodes.get(callId) : null; + if (prior && prior.row == null && prior.node && prior.node.isConnected) { + prior.node.remove(); + } const orphanCard = _tryMcpErrorBlock(isError, output); if (orphanCard) { const el = appendMsg("error", "", { @@ -1516,6 +1620,7 @@ function createCoordinatorPane(root, wsId, opts) { callId: callId, }); el.querySelector(".msg-body").appendChild(orphanCard); + if (callId) toolResultNodes.set(callId, { row: null, node: el }); return el; } const html = renderToolOutput(output); @@ -1523,6 +1628,7 @@ function createCoordinatorPane(root, wsId, opts) { label: (isError ? "error · " : "") + (name || "tool"), callId: callId, }); + if (callId) toolResultNodes.set(callId, { row: null, node: el }); return el; } @@ -1543,6 +1649,10 @@ function createCoordinatorPane(root, wsId, opts) { // item payload is intentionally not retained (long sessions would // pin per-call preview / parsed-args memory for the page lifetime). const toolRows = new Map(); + const latestToolRowElements = new Map(); + // Tracks the result node currently owned by a call id. Row-backed results + // are already replace-in-place; orphan bubbles need explicit removal. + const toolResultNodes = new Map(); // Most-recently-rendered batch with an open approval gate. Used for // keyboard focus claiming and approval_resolved fallbacks. @@ -1823,9 +1933,17 @@ function createCoordinatorPane(root, wsId, opts) { } function _appendResultToRow(row, output, isError, opts) { - if (!row) return; - const existing = row.querySelector(".conv-row-result"); - if (existing) existing.remove(); + if (!row) return null; + row + .querySelectorAll(".conv-row-result") + .forEach((existing) => existing.remove()); + if (opts && opts.accepted) { + if (opts.effectStatus) { + row.dataset.effectStatus = String(opts.effectStatus); + } else { + delete row.dataset.effectStatus; + } + } if (isError) { row.classList.add("error"); // Lift the row's error onto the enclosing batch so the left @@ -1848,6 +1966,34 @@ function createCoordinatorPane(root, wsId, opts) { // across upgrade-in-place when the error came from a tool_result. if (isError) block.classList.add("conv-row-result--error"); row.appendChild(block); + // Accepted and replayed preview rows retain the transcript affordance even + // when the result is an error/cancellation. Only the executor's + // preliminary event may auto-open; this final projection is chip-only. + if (opts && opts.preview) { + const chip = buildPreviewChip(opts.preview, (descriptor) => { + const shell = window.TS_SHELL; + if (shell && typeof shell.openPreview === "function") { + shell.openPreview(descriptor, { base: "", wsId: wsId }); + return; + } + // The standalone coordinator has no split-pane host. Keep the chip + // useful by opening its same-origin stored preview in a new tab. + if (descriptor && descriptor.attachment_id) { + window.open( + "/v1/api/workstreams/" + + encodeURIComponent(wsId) + + "/attachments/" + + encodeURIComponent(descriptor.attachment_id) + + "/preview", + "_blank", + "noopener", + ); + } + }); + chip.classList.add("conv-row-result"); + row.appendChild(chip); + } + return block; } // Concise, screen-reader friendly summary of a pending batch — used @@ -2048,9 +2194,23 @@ function createCoordinatorPane(root, wsId, opts) { if (items.length === 0) return null; opts = opts || {}; - const allMapped = items.every( - (it) => it.call_id && toolRows.has(it.call_id), + const mappedEntries = items.map((it) => + it.call_id ? toolRows.get(it.call_id) : null, ); + const allMapped = mappedEntries.every(Boolean); + const mappedBatch = allMapped ? mappedEntries[0].batch : null; + const sameMappedBatch = + allMapped && mappedEntries.every((entry) => entry.batch === mappedBatch); + const mappedRowsUnresolved = + sameMappedBatch && + mappedEntries.every( + (entry) => !entry.row.querySelector(".conv-row-result"), + ); + // Only an unresolved shell can be the same occurrence replayed with more + // authoritative state (early paint/history orphan -> tool_info/approval). + // A mapped row with a result is complete: the provider may legitimately + // reuse its call id in a later turn, which must create a fresh batch. + const canUpgradeMappedBatch = sameMappedBatch && mappedRowsUnresolved; // Partial-overlap guard. If only SOME of the incoming call_ids // are mapped (i.e. they belong to a different prior batch), // overwriting toolRows in the create-new path below would orphan @@ -2059,21 +2219,23 @@ function createCoordinatorPane(root, wsId, opts) { // the new batch. This shape doesn't occur in normal operation // (the server never sends overlapping envelopes), so we log it + // unmap the stale entries before the new batch claims them. - if (!allMapped) { + if (!canUpgradeMappedBatch) { const partial = items.filter( (it) => it.call_id && toolRows.has(it.call_id), ); if (partial.length > 0) { - console.warn( - "coord_ui: partial-overlap envelope — unmapping", - partial.length, - "stale call_ids before new batch claims them", - ); + if (!allMapped || !sameMappedBatch) { + console.warn( + "coord_ui: partial-overlap envelope — unmapping", + partial.length, + "stale call_ids before new batch claims them", + ); + } partial.forEach((it) => toolRows.delete(it.call_id)); } } - if (allMapped) { - const existing = toolRows.get(items[0].call_id).batch; + if (canUpgradeMappedBatch) { + const existing = mappedBatch; // Late cycle identity (SSE approve_request upgrading a replay / // early-paint shell) — stamp it so the approve POST can route. if (opts.cycleId) existing.dataset.cycleId = opts.cycleId; @@ -2216,6 +2378,12 @@ function createCoordinatorPane(root, wsId, opts) { batch.appendChild(row); renderedRows.push(row); if (it.call_id) { + indexLatestToolRow( + latestToolRowElements, + toolResultNodes, + it.call_id, + row, + ); toolRows.set(it.call_id, { batch, row }); } if (row.classList.contains("error")) anyRowError = true; @@ -2495,12 +2663,13 @@ function createCoordinatorPane(root, wsId, opts) { let queuedEl = null; let optimisticEl = null; const isBusy = busy; + const clientSendId = mintClientSendId(); // Display-only strip of the !!! prefix (the server re-parses it // authoritatively); shared parse so the settle helper's retro-convert // renders the same chip either pane would have built pre-POST. const { displayText, priority } = parsePriority(trimmed); if (isBusy) { - queuedEl = queue.addQueuedMessage(displayText, priority); + queuedEl = queue.addQueuedMessage(displayText, priority, clientSendId); } else { // "optimistic": no server state event asserted this — the settle // arms may undo it if the send turns out deferred/refused (see @@ -2515,6 +2684,7 @@ function createCoordinatorPane(root, wsId, opts) { snap.attachments, { label: "you", + clientSendId: clientSendId, }, ); } @@ -2533,6 +2703,7 @@ function createCoordinatorPane(root, wsId, opts) { body: JSON.stringify({ message: trimmed, attachment_ids: snap.attachment_ids, + client_send_id: clientSendId, }), }; let sendTimer = null; @@ -2549,70 +2720,24 @@ function createCoordinatorPane(root, wsId, opts) { sendInit, ); if (sendTimer) sendReq = sendReq.finally(() => clearTimeout(sendTimer)); - sendReq - .then((r) => { - // A rejected send (4xx/5xx) carries {error}, not {status}; without - // this guard it falls through to the "unknown status" branch and gets - // promote()'d — a server-refused message shown as delivered (with a - // false "already sent" notice if it was dismissed). Route it to the - // .catch (removes the bubble + shows the error) instead, surfacing the - // server's {error} text ("No session", a rate-limit reason, etc.) - // rather than a bare status code. A wedged proxy can answer non-JSON - // (502/504 HTML); the parse-failure arm falls back to the status code - // so that can't surface as an "Unexpected token <" error. - if (!r.ok) { - // 409 = the server-side cross-user interjection block; convert to a - // handled status so it routes to the clean branch below (not the - // generic error). Reactive fallback for the race where the send - // button wasn't yet disabled. - if (r.status === 409) { - return r.json().then( - (b) => ({ - status: "cross_user_interjection", - error: (b && b.error) || "", - }), - () => ({ status: "cross_user_interjection", error: "" }), - ); - } - return r.json().then( - (b) => { - throw new Error((b && b.error) || "send_http_" + r.status); - }, - () => { - throw new Error("send_http_" + r.status); - }, - ); - } - return r.json(); - }) - .then((data) => { - // The full status dispatch (queued/retro-convert, busy, - // queue_full, attachments_busy, cross_user, unknown-ok) lives in - // the shared helper — ONE settle matrix for both panes; see - // settleSendResponse's contract for the arm semantics. - settleSendResponse(queue, data, { - queuedEl, - optimisticEl, - isBusy, - displayText, - priority, - setBusy: (b) => setBusy(b), - busyIsOptimistic: () => busy && busySource === "optimistic", - paneIsBusy: () => busy, - renderError: (msg) => appendText("error", msg, { label: "error" }), - consumeAttachments: (attached, droppedIds) => - attachments.consume(attached, droppedIds), - }); - }) - .catch((e) => { - if (queuedEl) queue.remove(queuedEl); - appendText( - "error", - "Connection error: " + (e && e.message ? e.message : e), - { label: "error" }, - ); - if (!queuedEl) setBusy(false); - }); + // Response normalization, the full status dispatch (queued/retro-convert, + // busy, queue_full, attachments_busy, cross_user, unknown-ok) and the + // accepted-guarded transport catch all live in the shared helper — ONE + // send settle for both panes and both of each pane's send flows. + postAndSettleSend(queue, sendReq, { + queuedEl, + optimisticEl, + isBusy, + displayText, + priority, + clientSendId, + setBusy: (b) => setBusy(b), + busyIsOptimistic: () => busy && busySource === "optimistic", + paneIsBusy: () => busy, + renderError: (msg) => appendText("error", msg, { label: "error" }), + consumeAttachments: (attached, droppedIds) => + attachments.consume(attached, droppedIds), + }); return false; } @@ -2705,6 +2830,14 @@ function createCoordinatorPane(root, wsId, opts) { return; } if (!resp.ok) { + if (resp.status === 409) { + const msg = + "Conversation history is still being saved. Try ending the session again shortly."; + if (typeof toast !== "undefined" && toast.error) toast.error(msg); + else window.alert(msg); + resumeSse(); + return; + } let detail = "HTTP " + resp.status; try { const body = await resp.json(); @@ -2868,6 +3001,13 @@ function createCoordinatorPane(root, wsId, opts) { if (connectCursor != null) { url += "?last_event_id=" + encodeURIComponent(connectCursor); } + // Declare typed user-turn support on every manual URL. Native + // EventSource reconnects reuse this URL, so the capability survives + // transport churn without another client-side hook. + url += (url.includes("?") ? "&" : "?") + "user_turn=1"; + // The accepted TOOL projection is likewise URL-sticky across native + // reconnects; SDK and channel consumers intentionally do not opt in. + url += "&tool_turn=1"; // Close-on-hide / replay-on-show: install once per pane, removed by // destroy(). A hidden tab's throttled drain is the likeliest slow consumer // behind a server-side queue overflow, and an idle hidden tab holds a node @@ -2898,8 +3038,25 @@ function createCoordinatorPane(root, wsId, opts) { setSseStatus("paused — tab hidden", ""); return; } + if (historyRepair.isRepairing(wsId)) { + // A history mismatch is not a numeric replay gap. Refuse every + // transport redial until /history has rendered a fresh proof; this + // also makes hide/show, login, and degraded-recovery paths fail closed. + setSseStatus("history out of date — retrying…", "err"); + statusBarEl.classList.add("ws-sb-disconnected"); + sbTokensEl.textContent = "History out of date — retrying…"; + historyRepair.schedule(); + return; + } + if (historyHandoffToken != null) { + url += + (url.includes("?") ? "&" : "?") + + "history_token=" + + encodeURIComponent(historyHandoffToken); + } setSseStatus("connecting…", ""); evtSource = new EventSource(url, { withCredentials: true }); + historyHandoffToken = null; evtSource.onopen = function () { reconnectAttempts = 0; // Measure the gap this open just closed, then clear it (disconnectedAt is @@ -3332,7 +3489,13 @@ function createCoordinatorPane(root, wsId, opts) { // closed stream keeps this async tail from reopening one on a // destroyed pane (close-session's own resumeSse re-arms on its // failure paths). - function loadHistoryThenReconnect() { + function loadHistoryThenReconnect(manualAttempt = false) { + const repairingHistoryHandoff = historyRepair.isRepairing(wsId); + let repairAttemptId = null; + if (repairingHistoryHandoff) { + if (!historyRepair.admitAttempt(manualAttempt)) return; + repairAttemptId = historyRepair.startAttempt(manualAttempt); + } suspendStream(); currentAssistantEl = null; currentAssistantBuf = ""; @@ -3353,12 +3516,13 @@ function createCoordinatorPane(root, wsId, opts) { // reconnect still retries correctly: truncatedFromCursor (not // lastEventId) is the durable repair state the chokepoint presents. lastEventId = null; + historyHandoffToken = null; // A pending deferred resync is superseded by this full refetch — without // this clear, the next idle edge would run a second, pointless full // resync. pendingTruncatedResync = false; refetchHistory(true) - .then(() => { + .then((outcome) => { // Successful heal only (a failed fetch resolves too, but leaves the // record set; a render throw skips .then entirely): refresh the // sidebar once. The envelope refreshed it at gap START; this @@ -3403,8 +3567,26 @@ function createCoordinatorPane(root, wsId, opts) { ) { refreshSidebarAfterGap(); } + return outcome; }) - .finally(() => { + .catch((err) => { + // Normalize to a fail-closed settle below, but never silently: a + // render throw on the ordinary heal path used to surface as an + // unhandled rejection — keep the diagnostic loud. + console.error("history load/render failed", err); + return undefined; + }) + .then((outcome) => { + if (repairingHistoryHandoff) { + historyRepair.endAttempt(repairAttemptId); + if (!visHandler) return; + historyRepair.settle({ + outcome, + hasToken: historyHandoffToken != null, + manualAttempt, + }); + return; + } if (visHandler) connectSSE(); }); } @@ -3546,20 +3728,39 @@ function createCoordinatorPane(root, wsId, opts) { noteStreamOverflow(); break; case "tool_result": - liveToolCalls.delete(ev.call_id || ""); - appendToolResult( - ev.name || "tool", - ev.call_id || "", - ev.output || "", - !!ev.is_error, - ); - // tasks mutations change persisted state the sidebar reads - // from GET /tasks — re-fetch so the operator sees - // add/update/remove/reorder without clicking the refresh icon. - // list is a read-only action; skip to avoid redundant fetches. - // Debounced so a burst of mutations coalesces into one fetch. - if (ev.name === "tasks" && !ev.is_error) { - loadTasksDebounced(); + if (acceptedToolEventAlreadyRendered(renderedToolEventIds, ev)) { + break; + } + { + const callId = ev.call_id || ""; + const mapped = callId ? toolRows.get(callId) : null; + const hadResult = !!( + (mapped && mapped.row.querySelector(".conv-row-result")) || + (callId && toolResultNodes.has(callId)) + ); + liveToolCalls.delete(callId); + appendToolResult( + ev.name || "tool", + callId, + ev.output || "", + !!ev.is_error, + { + accepted: ev.accepted === true, + effectStatus: ev.effect_status, + preview: ev.preview, + }, + ); + recordAcceptedToolEvent(renderedToolEventIds, ev); + // tasks mutations change persisted state the sidebar reads + // from GET /tasks — re-fetch so the operator sees + // add/update/remove/reorder without clicking the refresh icon. + // list is a read-only action; skip to avoid redundant fetches. + // Debounced so a burst of mutations coalesces into one fetch. The + // accepted replacement does not repeat that side effect when the + // provisional receipt was already rendered. + if (shouldRefreshTasksForToolResult(ev, hadResult)) { + loadTasksDebounced(); + } } break; case "approve_request": @@ -3695,6 +3896,9 @@ function createCoordinatorPane(root, wsId, opts) { // styling which mis-categorised them as tool calls. appendText("info", ev.message || "", { label: "info" }); break; + case "user_turn": + acceptUserTurn(ev); + break; case "system_turn": { // First-class operator-context system turn (output-guard finding, // user interjection, metacognitive nudge, watch result — see @@ -3708,6 +3912,14 @@ function createCoordinatorPane(root, wsId, opts) { // /history+replay seam idempotent. Mirrors ui/static/app.js. const sysEid = ev._event_id != null ? String(ev._event_id) : null; if (sysEid && renderedSystemEventIds.has(sysEid)) break; + if ( + ev.source === "user_interjection" && + ev.meta && + ev.meta.client_send_id && + clientSendMaySettleForViewer(ev.meta.sender, viewerUserId()) + ) { + markAcceptedClientSends([ev.meta.client_send_id], true); + } renderSystemTurn(ev.source || "", ev.content || "", ev.meta); if (sysEid) renderedSystemEventIds.add(sysEid); break; @@ -3874,6 +4086,11 @@ function createCoordinatorPane(root, wsId, opts) { // already showed it; nothing to render here. (Earlier this // surfaced an extra info row, which doubled up with the // queued bubble once the composer started rendering one.) + if ( + ev.client_send_id && + clientSendMaySettleForViewer(ev.sender, viewerUserId()) + ) + markAcceptedClientSends([ev.client_send_id], false, true); break; case "message_dispatched": // A deferred send left the parked list: fresh spawn (promote the @@ -4030,20 +4247,42 @@ function createCoordinatorPane(root, wsId, opts) { if (!_pendingEditSend) return; const editText = _pendingEditSend; _pendingEditSend = null; - setBusy(true); - appendUserMessageWithAttachments(editText, [], { label: "you" }); - authFetch( - "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/send", - { - method: "POST", - credentials: "include", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ message: editText }), - }, - ).catch((err) => { - appendText("error", "Connection error: " + err.message); - setBusy(false); + const editClientSendId = mintClientSendId(); + const editPriority = parsePriority(editText); + setBusy(true, "optimistic"); + const editEl = appendUserMessageWithAttachments(editText, [], { + label: "you", + clientSendId: editClientSendId, }); + postAndSettleSend( + queue, + authFetch( + "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/send", + { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + message: editText, + client_send_id: editClientSendId, + }), + }, + ), + { + queuedEl: null, + optimisticEl: editEl, + isBusy: false, + displayText: editPriority.displayText, + priority: editPriority.priority, + clientSendId: editClientSendId, + setBusy: (value) => setBusy(value), + busyIsOptimistic: () => busy && busySource === "optimistic", + paneIsBusy: () => busy, + renderError: (message) => + appendText("error", message, { label: "error" }), + consumeAttachments: () => {}, + }, + ); }) .catch((err) => { // Render runs outside refetchHistory's fetch try/catch by design; @@ -4054,6 +4293,13 @@ function createCoordinatorPane(root, wsId, opts) { }); break; } + case "history_resync": + // A conversation commit crossed the REST-history -> listener handoff. + // Ring replay cannot prove that a committed row was rendered. The + // explicit repair mode survives /history failure and refuses every + // cursorless/tokenless transport redial until a fresh proof renders. + historyRepair.begin(wsId); + break; case "replay_truncated": { // The stream just admitted losing events past recovery — treat the // connection as DEAD and run the full fresh-connect flow @@ -6119,21 +6365,25 @@ function createCoordinatorPane(root, wsId, opts) { // retry on later organic edges against a fresh attempt. const histCtrl = typeof AbortController === "function" ? new AbortController() : null; - if (histCtrl) histCtrls.add(histCtrl); - const histTimer = histCtrl - ? setTimeout(() => histCtrl.abort(), 15000) - : null; + const deadlineHandle = createHistoryHandoffDeadline(() => { + if (histCtrl) histCtrl.abort(); + }, HISTORY_HANDOFF_FETCH_TIMEOUT_MS); + const attempt = { ctrl: histCtrl, deadline: deadlineHandle }; + histAttempts.add(attempt); try { - hist = await getJSON( - "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/history", - histCtrl ? { signal: histCtrl.signal } : undefined, - ); + hist = await Promise.race([ + getJSON( + "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/history", + histCtrl ? { signal: histCtrl.signal } : undefined, + ), + deadlineHandle.promise, + ]); } catch (e) { console.warn("coord history fetch failed", e); hist = null; } finally { - if (histTimer) clearTimeout(histTimer); - if (histCtrl) histCtrls.delete(histCtrl); + deadlineHandle.dispose(); + histAttempts.delete(attempt); refetchesInFlight--; } // A FAILED fetch keeps the pane intact: the wipe + tracking resets @@ -6269,8 +6519,12 @@ function createCoordinatorPane(root, wsId, opts) { staleRetryTimer = null; } toolRows.clear(); + latestToolRowElements.clear(); + toolResultNodes.clear(); activeBatch = null; renderedSystemEventIds.clear(); + renderedUserEventIds.clear(); + renderedToolEventIds.clear(); resetCompactionHolder(compactionHolder); // Fresh-connect fast-forward: when the trailing turn is an executing // in-flight tool batch the server can replay, /history returns a @@ -6291,9 +6545,17 @@ function createCoordinatorPane(root, wsId, opts) { // tool result rendered with the literal label "tool", which // looked like the tool calls had been replaced by raw JSON. const toolNameByCallId = new Map(); + // Core TOOL rows must pair correctly even when a provider reuses call ids + // across turns or emits malformed duplicates inside one assistant batch. + // (Verdict/output-assessment decoration remains call-id keyed upstream.) + const pendingHistoryToolRows = new Map(); - // Pre-scan every tool message's tool_call_id so the - // assistant.tool_calls branch below knows whether each call_id + // Structurally pair each assistant batch with only its immediately + // following TOOL rows. A bounded /history slice may begin with a TOOL whose + // assistant was cut off; a global call-id queue would let that leading + // orphan poison a later reused id's outcome. + // + // The assistant.tool_calls branch below needs to know whether each call_id // already has a result persisted. An assistant tool_calls turn // with NO matching tool result for some call_ids = orphan: the // tool was dispatched but didn't complete before the reload @@ -6307,26 +6569,8 @@ function createCoordinatorPane(root, wsId, opts) { // for case c. Without this neutral state, painting Approve // buttons on a non-pending orphan was misleading and could // 409-on-submit because the call_id wasn't in pending_items. - const callOutcomes = new Map(); - (hist.messages || []).forEach((m) => { - if ((m.role || "tool") !== "tool" || !m.tool_call_id) return; - // The server-side /history projection derives denied / is_error on - // each tool message (content-prefix heuristic + persisted flags), - // so we read those fields directly rather than re-sniffing content - // here. "Denied by user" / "Blocked" -> denied; the error-prefix - // set (Error, Command timed out, ...) -> is_error. The - // assistant.tool_calls render below reads this map to mark a batch - // resolved-denied (vs the default resolved-approved) and to - // propagate the error flag to appendToolResult; a call_id absent - // from the map is an orphan (no result yet) -> --running. - let outcome = "ok"; - if (m.denied) { - outcome = "denied"; - } else if (m.is_error) { - outcome = "error"; - } - callOutcomes.set(m.tool_call_id, outcome); - }); + const historyMessages = hist.messages || []; + const historyBatchOutcomes = indexHistoryToolOutcomes(historyMessages); // Render an assistant turn's tool_calls as a single batch // construct. Synthesises one batch per assistant turn so a @@ -6337,7 +6581,8 @@ function createCoordinatorPane(root, wsId, opts) { // resolvedCallIds rationale above). SSE upgrades --running in // place when it knows more. function renderAssistantToolBatch(m) { - const items = m.tool_calls.map((tc) => { + const batchOutcomes = historyBatchOutcomes.get(m) || []; + const items = m.tool_calls.map((tc, callIndex) => { // tool_calls arrive flattened by the server /history projection: // {id, name, arguments} (no nested `function` wrapper). const name = String((tc && tc.name) || "tool"); @@ -6356,6 +6601,7 @@ function createCoordinatorPane(root, wsId, opts) { parsedArgs, argsRaw, ); + item._historyOutcome = batchOutcomes[callIndex]; // Server attaches the persisted intent_verdict to each // tc on /history (newest-wins per call_id; LLM upgrade // beats heuristic when both exist). Stamp on the item @@ -6399,17 +6645,33 @@ function createCoordinatorPane(root, wsId, opts) { // - else → resolved-approved (a runtime error doesn't // change the approval verdict; the per-row .error class // comes from the tool_result branch below) - const outcomes = items.map((it) => - it.call_id ? callOutcomes.get(it.call_id) : "ok", - ); + const outcomes = items.map((it) => it._historyOutcome); const allResolved = outcomes.every((o) => o !== undefined); + let renderedBatch = null; if (!allResolved) { - appendToolBatch(items, { running: true }); + renderedBatch = appendToolBatch(items, { running: true }); } else if (outcomes.some((o) => o === "denied")) { - appendToolBatch(items, { resolved: { approved: false } }); + renderedBatch = appendToolBatch(items, { + resolved: { approved: false }, + }); } else { - appendToolBatch(items, { resolved: { approved: true } }); + renderedBatch = appendToolBatch(items, { + resolved: { approved: true }, + }); } + const renderedRows = renderedBatch + ? Array.from(renderedBatch.children).filter((child) => + child.classList.contains("conv-row"), + ) + : []; + items.forEach((item, index) => { + if (!item.call_id || !renderedRows[index]) return; + enqueueToolOccurrence(pendingHistoryToolRows, item.call_id, { + row: renderedRows[index], + name: item.func_name || "tool", + outcome: item._historyOutcome, + }); + }); // Output-guard findings — render each one as a chip // anchored to the .conv-row that tripped the guard // rather than a generic "[output guard]" chat line. @@ -6422,9 +6684,9 @@ function createCoordinatorPane(root, wsId, opts) { if (!oa || !oa.risk_level || oa.risk_level === "none") continue; const cid = items[oi].call_id || ""; if (!cid) continue; - const entry = toolRows.get(cid); - if (!entry || !entry.row) continue; - _attachOutputWarningChip(entry.row, oa); + const row = renderedRows[oi]; + if (!row) continue; + _attachOutputWarningChip(row, oa); } } @@ -6461,10 +6723,52 @@ function createCoordinatorPane(root, wsId, opts) { // batch state (no per-row error needed there; the row's // content reads "Denied by user"). const callId = m.tool_call_id || ""; + const occurrence = callId + ? shiftToolOccurrence(pendingHistoryToolRows, callId) + : null; const toolName = - (callId && toolNameByCallId.get(callId)) || m.tool_name || "tool"; - const isError = callOutcomes.get(callId) === "error"; - appendToolResult(toolName, callId, content || "", isError); + (occurrence && occurrence.name) || + (callId && toolNameByCallId.get(callId)) || + m.tool_name || + "tool"; + // Orphan tool rows (their assistant batch fell outside the bounded + // /history window) have no occurrence; the row's own projected flag + // is the authority there — without it a failed tool reads on reload + // as a normal successful result. + const isError = occurrence + ? occurrence.outcome === "error" + : m.is_error === true; + const resultOpts = { + accepted: true, + effectStatus: m.effect_status, + preview: m.preview, + }; + if (occurrence && occurrence.row) { + const resultNode = _appendResultToRow( + occurrence.row, + content || "", + isError, + resultOpts, + ); + if (callId) { + toolResultNodes.set(callId, { + row: occurrence.row, + node: resultNode, + }); + } + _unsetBatchRunningIfAllResults(occurrence.row.closest(".conv-batch")); + } else { + appendToolResult( + toolName, + callId, + content || "", + isError, + resultOpts, + ); + } + if (m.event_id != null) { + renderedToolEventIds.add(String(m.event_id)); + } // Tool-channel metacog nudges + queued interjections that used to // splice into the tool result now follow it as first-class // operator-context ``system`` rows and render via the ``system`` @@ -6538,12 +6842,19 @@ function createCoordinatorPane(root, wsId, opts) { // The nudges it carried are now first-class operator-context // ``system`` rows that follow it and render below. appendSystemNudgeMarker(); + if (m.event_id != null) + renderedUserEventIds.add(String(m.event_id)); return; } if (!content && userAttachments.length === 0) return; + const viewer = viewerUserId(); appendUserMessageWithAttachments(content, userAttachments, { - label: role, + label: m.sender && m.sender === viewer ? "you" : "user", + eventId: m.event_id, + sender: m.sender || "", + source: m.source || "", }); + if (m.event_id != null) renderedUserEventIds.add(String(m.event_id)); } else if (role === "system") { // First-class operator-context system turn — ``renderSystemTurn`` // routes by ``m.source`` to the structured card (watch / guard / @@ -6566,6 +6877,19 @@ function createCoordinatorPane(root, wsId, opts) { // live-turn-ends case — without this a reloaded or rewound coordinator // showed assistant turns with no retry button. _refreshRetryButton(); + // The token asserts that this exact REST revision was fully rendered. + // Arm it only after every synchronous render step succeeds; otherwise the + // caller's rejected promise must not reconnect under a false assertion. + if (seedCursor) { + historyHandoffToken = + typeof hist.handoff_token === "string" && hist.handoff_token + ? hist.handoff_token + : null; + } + // The outcome tells the repair settle whether a TOKENLESS response was a + // completed render (the server's deliberate cold storage-only read — + // downgrade to the tokenless bootstrap) or a failure (fail closed). + return "rendered"; } // Re-arm the stream after a 401 re-auth: reset backoff + reconnect now. @@ -6586,6 +6910,7 @@ function createCoordinatorPane(root, wsId, opts) { // Stream + the retry timers (reconnect backoff, degraded catch-up, // truncated resync). closeStreamTransport(); + historyRepair.clear(); // The clear_ui-failure retry deliberately survives closeStreamTransport // (transport-only redials keep the heal intent — see its decl), so it // must die HERE, the terminal path: destroy() bumps no generation, and @@ -6599,14 +6924,17 @@ function createCoordinatorPane(root, wsId, opts) { // Abort every in-flight /history for the same reason: the 15s bound // alone would keep the detached pane's closure alive until it fired // (the settled fetches' renders then discard via the !hist path). - histCtrls.forEach((c) => { - try { - c.abort(); - } catch (_) { - /* noop */ + histAttempts.forEach((attempt) => { + if (attempt.ctrl) { + try { + attempt.ctrl.abort(); + } catch (_) { + /* noop */ + } } + attempt.deadline.dispose({ expire: true, resolve: true }); }); - histCtrls.clear(); + histAttempts.clear(); [ cancelTimeoutId, forceTimeoutId, diff --git a/turnstone/core/adapters/_ui_cleanup.py b/turnstone/core/adapters/_ui_cleanup.py index 044a160c..d84ce099 100644 --- a/turnstone/core/adapters/_ui_cleanup.py +++ b/turnstone/core/adapters/_ui_cleanup.py @@ -83,6 +83,12 @@ def _broadcast_ws_closed_to_listeners(ui: SessionUI) -> None: if listeners is None or listeners_lock is None: return with listeners_lock: + # Fence stale events requests that already captured the Workstream/UI + # but have not yet reached listener registration. SessionUIBase's + # registration helpers observe this under the same lock and return a + # pre-closed, non-retained queue. + ui_any: Any = ui + ui_any._listeners_terminal = True for lq in listeners: # Mark the stream closing BEFORE attempting the sentinel: a # poisoned/full ``_ListenerQueue`` rejects every put (the diff --git a/turnstone/core/attachment_buffer.py b/turnstone/core/attachment_buffer.py index b7fe0242..4c4bf643 100644 --- a/turnstone/core/attachment_buffer.py +++ b/turnstone/core/attachment_buffer.py @@ -170,6 +170,47 @@ class AttachmentBuffer: del self._blobs[handle] return True + def consume_all( + self, + handles: list[str] | tuple[str, ...], + *, + ws_id: str, + user_id: str, + ) -> frozenset[str]: + """Atomically transfer every present scoped reference to a send. + + Returns the set of handles actually consumed. A handle no longer + staged for ``(ws_id, user_id)`` (TTL expiry, an earlier drain) is + simply absent from the result — the survivors are still consumed in + the same critical section, so no reference outlives the admission + that owns its bytes. Duplicate handles represent repeated ordered + message references, but consume the one staged ownership reference + only once. Blobs with no remaining scope are evicted. + + The accepted conversation journal already owns immutable copies of + the bytes when this is called. Any staged owner surviving admission + would let a rapid omitted-id send attach the same upload again + before durable acknowledgement — including when a sibling handle + already expired, which is why this is per-handle, never + all-or-nothing. + """ + unique_handles = tuple(dict.fromkeys(handles)) + if not unique_handles: + return frozenset() + scope = (ws_id, user_id) + consumed: set[str] = set() + with self._lock: + self._evict_expired_locked() + for handle in unique_handles: + blob = self._blobs.get(handle) + if blob is None or scope not in blob.refs: + continue + del blob.refs[scope] + if not blob.refs: + del self._blobs[handle] + consumed.add(handle) + return frozenset(consumed) + def clear(self) -> None: """Drop all pending uploads (every reference and blob). diff --git a/turnstone/core/attachments.py b/turnstone/core/attachments.py index 00083f5a..2807b319 100644 --- a/turnstone/core/attachments.py +++ b/turnstone/core/attachments.py @@ -353,9 +353,9 @@ def resolve_staged_attachments( This is a *peek*, not a drain: the entries stay in the buffer so a send that resolves them but doesn't commit (e.g. the queue rejects an attachment-bearing turn → ``attachments_busy``, and the client retries) can - still find them. The committing path drains them at write time via - :meth:`ChatSession._append_user_turn` (``buffer.discard`` per persisted - id); anything left over expires on the buffer's TTL. + still find them. The committing path transfers all referenced entries + atomically when :meth:`ChatSession._append_user_turn` admits the immutable + pending row; anything left over expires on the buffer's TTL. Kind-agnostic: both create-with-attachments and ``/send`` paths call this. The old ``send_id`` reservation token is gone — the buffer is the pending diff --git a/turnstone/core/history_decoration.py b/turnstone/core/history_decoration.py index b878a6c7..e26b99bb 100644 --- a/turnstone/core/history_decoration.py +++ b/turnstone/core/history_decoration.py @@ -27,6 +27,7 @@ from typing import TYPE_CHECKING, Any from turnstone.core import fence from turnstone.core.log import get_logger +from turnstone.core.trajectory import sanitize_client_send_ids log = get_logger(__name__) @@ -537,6 +538,7 @@ def project_history_messages( d = part.get("document", {}) attachments_meta.append( { + "attachment_id": str(part.get("attachment_id") or ""), "kind": "text", "filename": str(d.get("name", "")), "mime_type": str(d.get("media_type", "")), @@ -551,6 +553,7 @@ def project_history_messages( if isinstance(side_meta, list) and side_meta: attachments_meta = [ { + "attachment_id": str(m.get("attachment_id") or ""), "kind": str(m.get("kind") or ""), "filename": str(m.get("filename") or ""), "mime_type": str(m.get("mime_type") or ""), @@ -584,6 +587,17 @@ def project_history_messages( if msg.get("_source"): entry["source"] = str(msg["_source"]) + # Ordinary accepted user rows additionally carry participant + # attribution and one-shot optimistic-send correlation. Both are + # role-local metadata and never enter the provider wire payload. + if role == "user": + sender = msg.get("_sender") + if isinstance(sender, str) and sender: + entry["sender"] = sender + stable_client_send_ids = sanitize_client_send_ids(msg.get("_client_send_ids")) + if stable_client_send_ids: + entry["client_send_ids"] = stable_client_send_ids + # (3b) ``_source_meta`` side-channel → top-level ``meta``. The # operator turn's structured per-kind fields (``watch_triggered``'s # ``watch_name`` / command / poll counters) — the FE branches on diff --git a/turnstone/core/idle_nudge_watcher.py b/turnstone/core/idle_nudge_watcher.py index eecacc6c..8dd2cdcc 100644 --- a/turnstone/core/idle_nudge_watcher.py +++ b/turnstone/core/idle_nudge_watcher.py @@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Any from turnstone.core import session_worker from turnstone.core.log import get_logger from turnstone.core.nudge_queue import WAKE_PENDING, NudgeQueue -from turnstone.core.workstream import WorkstreamState +from turnstone.core.workstream import WorkstreamState, concrete_method if TYPE_CHECKING: from collections.abc import Callable @@ -38,7 +38,13 @@ if TYPE_CHECKING: log = get_logger(__name__) -def wake_workstream_if_pending(ws: Workstream, *, trigger: str = "unspecified") -> bool: +def wake_workstream_if_pending( + ws: Workstream, + *, + trigger: str = "unspecified", + include_interjections: bool = False, + exclude_interjection_signature: object | None = None, +) -> bool: """Spawn a wake send for *ws* when it is idle with drainable nudges. The shared gate behind both wake triggers: @@ -52,6 +58,13 @@ def wake_workstream_if_pending(ws: Workstream, *, trigger: str = "unspecified") *trigger* is a short label naming which path requested the wake (``"idle-transition"``, ``"watch-fire"``, ``"worker-exit"``); it is used only to tag the log lines below and never affects control flow. + ``include_interjections`` is reserved for the ownership-clear worker-exit + backstop. It lets a user row enqueued after the outgoing turn's final + flush claim the existing wake delivery path; ordinary idle/watch/drain + calls retain their nudge-only gate. + ``exclude_interjection_signature`` is carried only by a queue-only wake + worker: if its failed preamble restored that exact queue snapshot, its own + exit does not immediately retry it. A changed snapshot remains eligible. Gates, in order: @@ -81,11 +94,14 @@ def wake_workstream_if_pending(ws: Workstream, *, trigger: str = "unspecified") exit backstops (or the drain's clean exit when everything was retracted). See the predicate's docstring for the staleness argument. - * nothing gate-eligible under ``WAKE_PENDING`` — tool-only/quiet entries - belong to the next tool-result seam, not a synthetic empty user + * nothing gate-eligible under ``WAKE_PENDING`` and no compatible + interjection claimed by the worker-exit backstop — tool-only/quiet + entries belong to the next tool-result seam, not a synthetic empty user turn (``deliver_wake_nudge_from_queue`` would no-op on them). ``"wake"``-channel entries (the coordinator idle nudges) ARE - gate-eligible: the wake is the only seam that can deliver them. + gate-eligible: the wake is the only seam that can deliver them. The + interjection claim is session-owned and lock-safe; it refuses budget, + abandonment, persistence-poison, and repeated restored-queue attempts. Past the gates, exactly one info line is emitted per call: @@ -109,6 +125,19 @@ def wake_workstream_if_pending(ws: Workstream, *, trigger: str = "unspecified") session = ws.session if session is None or ws._closed or ws.state is not WorkstreamState.IDLE: return False + gone_probe = concrete_method(session, "is_workstream_gone") + if gone_probe is not None and gone_probe(): + # A hard-deleted workstream (the terminal gone latch) cannot accept + # ANY unattended turn: send admission refuses under the latch and + # the cancel finalizer converges that refusal internally, so a wake + # spawned past this point burns its drained nudges for nothing — + # and the latch does not paint ERROR, so the IDLE gate above stays + # open without this arm. Refuse the spawn; queued interjections + # stay retained (their disposition on a deleted workstream is + # #1001's). Logged because every other refusal on this lane is + # silent and "my queued message never delivered" needs one trace. + log.info("nudge_wake.refused_workstream_gone ws=%s trigger=%s", ws.id[:8], trigger) + return False if ws.send_barrier_active(): # Order-barrier yield: deferred sends (acknowledged "queued" — # see _PendingSend) are older than any nudge, and a wake worker @@ -130,7 +159,29 @@ def wake_workstream_if_pending(ws: Workstream, *, trigger: str = "unspecified") # Gate on WAKE_PENDING, not USER_DRAIN: ``"quiet"`` entries (external # events demoted by a user cancel) deliver at the next legitimate seam # but must never themselves wake the workstream the user just stopped. - if not isinstance(nudge_queue, NudgeQueue) or not nudge_queue.has_pending(WAKE_PENDING): + if not isinstance(nudge_queue, NudgeQueue): + return False + nudge_pending = nudge_queue.has_pending(WAKE_PENDING) + interjection_signature: object | None = None + if include_interjections and not nudge_pending: + # Concrete-hook lookup keeps this extension on the real ChatSession + # queue contract. The method owns its queue lock and returns an + # exact-snapshot retry token; no queue state is read or interpreted by + # this orchestration module. + claim_interjection = concrete_method(session, "claim_pending_interjection_wake") + if claim_interjection is not None: + try: + interjection_signature = claim_interjection( + exclude_signature=exclude_interjection_signature, + ) + except Exception: + log.warning( + "nudge_wake.interjection_claim_failed ws=%s trigger=%s", + ws.id[:8], + trigger, + exc_info=True, + ) + if not nudge_pending and interjection_signature is None: return False deferred = False @@ -143,7 +194,9 @@ def wake_workstream_if_pending(ws: Workstream, *, trigger: str = "unspecified") ws, enqueue=_noop_enqueue, run=session.deliver_wake_nudge_from_queue, + expected_session=session, thread_name=f"wake-nudge-{ws.id[:8]}", + interjection_wake_signature=interjection_signature, ) if deferred: log.info("nudge_wake.deferred_worker_busy ws=%s trigger=%s", ws.id[:8], trigger) diff --git a/turnstone/core/memory.py b/turnstone/core/memory.py index 218e727d..b2b5903e 100644 --- a/turnstone/core/memory.py +++ b/turnstone/core/memory.py @@ -4,9 +4,11 @@ All functions maintain their existing signatures for consumers (session.py, server.py, cli.py). The actual storage implementation lives in ``turnstone.core.storage``. -The no-raise contract is preserved — callers never see exceptions from this -module. All failures are logged so storage issues are visible in logs -rather than silently swallowed. +The established best-effort contract is preserved for operational failures. +Operations whose callers require positive durability return an explicit +failure sentinel rather than swallowing an exception into an indistinguishable +successful ``None``. Typed invariant conflicts remain exceptions: callers +must never mistake a different immutable commit for a transient storage blip. """ from __future__ import annotations @@ -14,7 +16,12 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any from turnstone.core.log import get_logger -from turnstone.core.storage import get_storage +from turnstone.core.storage import ( + AttachmentWrite, + ConversationCommitConflictError, + ConversationCommitWorkstreamGoneError, + get_storage, +) from turnstone.core.workstream import WorkstreamKind if TYPE_CHECKING: @@ -33,6 +40,27 @@ def normalize_key(key: str) -> str: # -- Core conversation operations --------------------------------------------- +_TYPED_COMMIT_ERRORS = (ConversationCommitConflictError, ConversationCommitWorkstreamGoneError) + + +def _keyed_save(operation: Callable[[], int], describe: str) -> int: + """Run one keyed save with the shared typed-passthrough frame. + + Typed commit outcomes must reach the session journal un-coerced — the + next typed class belongs in ``_TYPED_COMMIT_ERRORS`` ONCE, for every + wrapper (a missed wrapper would convert a permanent invariant conflict + into a logged return-0 the journal retries forever). Operational + failures log and return ``0``; the journal classifies and retries them. + """ + try: + return operation() + except _TYPED_COMMIT_ERRORS: + raise + except Exception: + log.warning("%s", describe, exc_info=True) + return 0 + + def save_message( ws_id: str, role: str, @@ -46,11 +74,14 @@ def save_message( is_error: bool = False, producer: str | None = None, meta: str | None = None, + commit_key: str | None = None, ) -> int: """Log a message to the conversations table. - Returns the inserted row id, or ``0`` on failure (preserving the - module's no-raise contract). + Returns the inserted row id, or ``0`` on an operational failure. A keyed + retry whose immutable payload conflicts with the committed row raises + :class:`ConversationCommitConflictError` so the durability journal can + classify the permanent invariant failure without retrying it. ``source`` is the persisted twin of the in-memory ``_source`` side-channel (which producer synthesised the row); ``None`` for the @@ -61,15 +92,18 @@ def save_message( passes ``self.ui._event_id`` so ``/history`` can return it as the ``Last-Event-ID`` resume cursor. ``None`` for offline / bulk saves. - ``meta`` is the pre-serialized JSON of a first-class ``system`` turn's - structured per-kind operator-context fields (e.g. ``watch_triggered``'s - ``watch_name`` / ``command`` / poll counters) — the persisted twin of the - in-memory ``Turn.meta.extra["source_meta"]`` / ``_source_meta`` side - channel. ``None`` for ordinary rows and operator turns with no extra - fields. Opaque to storage (like ``tool_calls`` / ``provider_data``). + ``meta`` is pre-serialized role-specific conversation metadata: structured + operator context on system turns, effect/preview fields plus the acting + principal on tool turns, sender identity on shared-workstream user turns, + or the immutable model provenance envelope on accepted assistant turns. + It is opaque to the backend and decoded only at the row-to-Turn boundary. + + ``commit_key`` is the per-workstream idempotency identity for one admitted + conversation row. Retrying the same non-NULL key returns the original row + id without appending a duplicate. """ - try: - return get_storage().save_message( + return _keyed_save( + lambda: get_storage().save_message( ws_id, role, content, @@ -82,10 +116,79 @@ def save_message( is_error=is_error, producer=producer, meta=meta, - ) - except Exception: - log.warning("Failed to save message for ws=%s role=%s", ws_id, role, exc_info=True) - return 0 + commit_key=commit_key, + ), + f"Failed to save message for ws={ws_id} role={role}", + ) + + +def save_user_message_with_attachments( + ws_id: str, + content: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + source: str | None = None, + event_id: int | None = None, + meta: str | None = None, + commit_key: str, +) -> int: + """Atomically persist a keyed USER row and its attachment ownership. + + Returns the positively acknowledged row id, or ``0`` on an operational + failure. An immutable commit mismatch raises + :class:`ConversationCommitConflictError`. The session handoff journal may + acknowledge the row only when the backend has confirmed the row, blobs, + exact refcount increments, and ordered ref-list as one transaction. + Retrying the same immutable commit is safe. + """ + return _keyed_save( + lambda: get_storage().save_user_message_with_attachments( + ws_id, + content, + attachments, + source=source, + event_id=event_id, + meta=meta, + commit_key=commit_key, + ), + f"Failed atomic user attachment commit for ws={ws_id} commit_key={commit_key}", + ) + + +def save_tool_message_with_attachments( + ws_id: str, + content: str, + tool_name: str, + tool_call_id: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + event_id: int | None = None, + is_error: bool = False, + meta: str | None = None, + commit_key: str, +) -> int: + """Atomically persist a keyed TOOL row and its attachment ownership. + + Returns the positively acknowledged row id, or ``0`` on an operational + failure. An immutable commit mismatch raises + :class:`ConversationCommitConflictError`. This is the storage seam for + ordinary tool-image rows and cancelled rows whose already-published preview + blob must survive with the synthesized result. + """ + return _keyed_save( + lambda: get_storage().save_tool_message_with_attachments( + ws_id, + content, + tool_name, + tool_call_id, + attachments, + event_id=event_id, + is_error=is_error, + meta=meta, + commit_key=commit_key, + ), + f"Failed atomic tool attachment commit for ws={ws_id} commit_key={commit_key}", + ) def save_messages_bulk(rows: list[dict[str, Any]]) -> bool: diff --git a/turnstone/core/model_turn.py b/turnstone/core/model_turn.py index d61f4bdf..2534f408 100644 --- a/turnstone/core/model_turn.py +++ b/turnstone/core/model_turn.py @@ -42,7 +42,7 @@ import contextlib import random import time import uuid -from dataclasses import dataclass, fields, replace +from dataclasses import dataclass, field, fields, replace from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -106,9 +106,11 @@ from turnstone.core.storage._utils import ( strip_orphan_client_tool_blocks, ) from turnstone.core.trajectory import ( + PROVENANCE_META_KEY, ProviderNative, ToolCall, Turn, + TurnProvenance, dicts_from_turns, materialize_attachments, ) @@ -452,7 +454,10 @@ class ModelLane: *alias* is the registry alias used for config resolution, ``""`` when the lane runs outside the registry (then every registry-backed pass - degrades to its documented miss behavior). + degrades to its documented miss behavior). ``registry_generation`` is + captured with the binding by :func:`resolve_model_binding`; zero is the + explicit direct-lane value. Together with ``model`` those fields are the + immutable serving identity stamped on a successful turn. *temperature* / *reasoning_effort* are the lane's OPERATOR-resolved sampling knobs — the assignment scheme's operator rungs only @@ -477,6 +482,9 @@ class ModelLane: capabilities: ModelCapabilities | None = None extra_params: dict[str, Any] | None = None registry: ModelRegistry | None = None + # Registry snapshot paired atomically with alias/client/model/config by + # ``resolve_model_binding``. Zero is the explicit non-registry value. + registry_generation: int = 0 temperature: float | None = None reasoning_effort: str | None = None # Runtime credential resolver supplied by the host that owns OAuth state. @@ -680,6 +688,7 @@ def resolve_lane( *, alias: str = "", registry: ModelRegistry | None = None, + registry_generation: int = 0, capabilities: ModelCapabilities | None = None, extra_params: dict[str, Any] | None | EllipsisType = ..., cfg: ModelConfig | None | EllipsisType = ..., @@ -736,6 +745,7 @@ def resolve_lane( capabilities=caps, extra_params=extra, registry=registry, + registry_generation=registry_generation, temperature=resolve_temperature_setting(resolved_cfg, config_store), reasoning_effort=resolve_effort_setting(resolved_cfg, config_store), backend_auth_resolver=backend_auth_resolver, @@ -771,6 +781,7 @@ def resolve_model_binding( model, alias=effective_alias, registry=registry, + registry_generation=generation, cfg=cfg, config_store=config_store, backend_auth_resolver=backend_auth_resolver, @@ -955,11 +966,12 @@ class ModelTurnResult: computed against what the provider actually counted, lowerings the caller cannot see included. - *producer* and *serving_model* identify the SERVING lane (the storage row's - ``producer`` column) — the identity stamped on ``turn.native`` when a - native lane exists, carried separately so a native-less turn still - records who produced it and a fallback-served turn is not labeled - with the primary binding. + *provenance* is the immutable serving alias / backend model id / registry + generation / acting-principal tuple. The same JSON-safe value is stamped + on ``turn.meta.extra["provenance"]`` before this result leaves the plant + call, so a later registry or shared-workstream rebind cannot relabel an + accepted turn at commit time. *producer* and *serving_model* remain the + provider-native and compatibility projections of that serving lane. *tool_def_chars* is the serialized size of the final provider-native tool definitions handed to that serving lane. The session's token calibration @@ -972,6 +984,7 @@ class ModelTurnResult: finish_reason: str usage: UsageInfo | None tool_calls: list[dict[str, Any]] + provenance: TurnProvenance = field(default_factory=TurnProvenance) wire_msgs: list[dict[str, Any]] | None = None producer: str = "" serving_model: str = "" @@ -994,7 +1007,11 @@ def cap_tool_calls(result: ModelTurnResult, max_calls: int) -> tuple[list[dict[s capped = result.tool_calls[:max_calls] turn = result.turn if len(result.tool_calls) > len(capped): - turn = Turn.assistant(result.content, tool_calls=turn.tool_calls[: len(capped)]) + turn = replace( + turn, + tool_calls=turn.tool_calls[: len(capped)], + native=None, + ) return capped, turn @@ -1090,6 +1107,7 @@ def model_turn( resolve_attachments: Callable[[list[str]], dict[str, Any]] | None = None, cancel_ref: list[Any] | None = None, backend_auth_token: str | None = None, + acting_principal_id: str = "", deferred_names: frozenset[str] | None = None, prepare_wire: Callable[[list[dict[str, Any]], ModelLane], list[dict[str, Any]]] | None = None, on_chunk: Callable[[StreamChunk], None] | None = None, @@ -1224,6 +1242,13 @@ def model_turn( explicit argument is absent, ``lane.backend_auth_resolver`` resolves it after admission for each transport attempt, so a queued call cannot age a minted credential before it reaches the wire. + + *acting_principal_id* is the caller's already-pinned effective principal, + not a live session lookup. It is never used to mint a credential here; it + only joins the immutable serving identity stamped after a successful + result. Session-backed CLI, eval, scheduled, and internal lanes pass their + effective owner credential principal; truly ownerless direct calls pass + the empty string. """ if mint is not None and wire_id_map is None: raise ValueError( @@ -1346,6 +1371,8 @@ def model_turn( error_type=type(drain_error).__name__, attempt=attempt, model=lane.model, + alias=lane.alias, + registry_generation=lane.registry_generation, retry_in=round(delay, 2), ) if delay > 0: @@ -1410,11 +1437,20 @@ def model_turn( if native_blocks else None ) + provenance = TurnProvenance( + model_alias=lane.alias, + backend_model_id=lane.model, + registry_generation=lane.registry_generation, + acting_principal_id=acting_principal_id, + ) + turn = Turn.assistant(result.content or "", tool_calls=tool_calls, native=native) + turn.meta.extra[PROVENANCE_META_KEY] = provenance.to_meta() return ModelTurnResult( - turn=Turn.assistant(result.content or "", tool_calls=tool_calls, native=native), + turn=turn, finish_reason=result.finish_reason, usage=result.usage, tool_calls=raw_calls, + provenance=provenance, wire_msgs=served_wire, producer=lane.provider.provider_name, serving_model=lane.model, diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 66310ac7..267371b1 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -22,6 +22,7 @@ import json import mimetypes import os import queue +import random import re import shlex import shutil @@ -33,9 +34,9 @@ import threading import time import traceback import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from html import escape as _html_escape -from typing import TYPE_CHECKING, Any, ClassVar, Protocol +from typing import TYPE_CHECKING, Any, ClassVar, Protocol, cast import httpx @@ -77,15 +78,11 @@ from turnstone.core.lowering import ( ) from turnstone.core.mcp_client import try_prime_user_pools from turnstone.core.memory import ( - count_messages, count_structured_memories, - delete_messages_after, delete_structured_memory_by_id, delete_workstream, get_attachments, get_compaction_checkpoint, - get_compaction_floor, - get_compaction_watermark, get_skill_by_name, get_structured_memory_by_name, get_workstream_display_name, @@ -98,16 +95,16 @@ from turnstone.core.memory import ( load_workstream_config, normalize_key, resolve_workstream, - save_attachment, save_message, save_messages_bulk, save_structured_memory, + save_tool_message_with_attachments, + save_user_message_with_attachments, save_workstream_config, search_history, search_history_recent, search_structured_memories, search_visible_structured_memories, - set_message_attachments, set_workstream_alias, touch_structured_memories, update_workstream_title, @@ -195,9 +192,15 @@ from turnstone.core.preview import ( ) from turnstone.core.ratelimit import TokenBucket from turnstone.core.safety import is_command_blocked, sanitize_command +from turnstone.core.session_worker import WorkerClaim, current_worker_claim from turnstone.core.settings_registry import DEFAULT_AUTO_COMPACT_PCT from turnstone.core.skill_field_validation import SKILL_RUNTIME_CONFIG_FIELDS from turnstone.core.skill_parser import MAX_SKILL_DESCRIPTION_LEN +from turnstone.core.storage import ( + AttachmentWrite, + ConversationCommitConflictError, + ConversationCommitWorkstreamGoneError, +) from turnstone.core.storage._registry import get_storage from turnstone.core.storage._utils import ( COMPACTION_SOURCE, @@ -224,6 +227,7 @@ from turnstone.core.tools import ( merge_mcp_tools, ) from turnstone.core.trajectory import ( + PROVENANCE_META_KEY, AttachmentRef, EffectStatus, ProviderNative, @@ -231,6 +235,7 @@ from turnstone.core.trajectory import ( TextBlock, ToolCall, Turn, + TurnProvenance, dicts_from_turns, last_assistant_text, turn_from_dict, @@ -243,6 +248,7 @@ from turnstone.core.workstream import ( INTERJECTION_CAP_CHARS, PENDING_SENDS_MAX, WorkstreamKind, + concrete_method, ) from turnstone.prompts import ( INTERACTIVE_CONSENT_CLIENT_TYPES, @@ -326,6 +332,27 @@ class CrossUserInterjectionError(Exception): """ +class ConversationPersistenceError(Exception): + """An accepted conversation row could not be durably reconciled. + + The row remains in the live history handoff journal. Later sends must stop + before appending a causal suffix until the idempotent commit succeeds (or a + history read proves that an in-row-only commit already landed). This is + distinct from an ordinary provider/tool failure: generic failure cleanup + flushes queued user messages, which would create a durable row *after* the + missing boundary. + """ + + +class _MalformedToolBatchError(RuntimeError): + """Provider tool calls cannot be executed without unambiguous identities.""" + + +_CONVERSATION_PERSISTENCE_RETRY_BASE_SECONDS = 1.0 +_CONVERSATION_PERSISTENCE_RETRY_CAP_SECONDS = 60.0 +_SOFT_CLOSE_STRUCTURAL_WAIT_SECONDS = 1.0 + + class _CompactionIrreducibleError(Exception): """Raised by ``ChatSession._summarize_blocks`` when chunked summarisation cannot shrink the input — a recursion level fails to reduce the block count, @@ -340,10 +367,39 @@ class _CompactionIrreducibleError(Exception): @dataclasses.dataclass(frozen=True) class _SummaryResult: - """Compacted text plus the provider label from its final model turn.""" + """Compacted text plus the identity of its final model turn.""" text: str producer: str | None + provenance: TurnProvenance = dataclasses.field(default_factory=TurnProvenance) + + +_ModelCalibrationKey = tuple[str, str, int] + + +@dataclasses.dataclass(frozen=True, slots=True) +class _TokenCalibration: + """One producing model binding's prompt estimator. + + ``prompt_tokens`` anchors the exact prefix whose object identities ride in + ``message_prefix_ids``. ``None`` retains the learned tokenizer ratio while + explicitly invalidating an anchor after a structural history replacement. + """ + + chars_per_token: float + prompt_tokens: int | None = None + message_prefix_ids: tuple[int, ...] = () + + +@dataclasses.dataclass(frozen=True, slots=True) +class _ServingFailureContext: + """Secret-free identity of the lane that raised a terminal stream error.""" + + model_alias: str + backend_model_id: str + registry_generation: int + provider_label: str + base_url: str @dataclasses.dataclass(frozen=True) @@ -365,6 +421,31 @@ class _CancelledToolResult: live_emitted: bool +@dataclasses.dataclass(frozen=True) +class _PendingConversationCommit: + """One accepted conversation row awaiting a complete durable ACK. + + ``ack_from_durable_row`` is normally true because every accepted row kind + uses one keyed transaction (including user/tool attachments). It is + switched off after an immutable-key conflict: seeing a different durable + row with the same key must not acknowledge or hide the intended pending + operation. + """ + + commit_key: str + message: dict[str, Any] + persist: Callable[[], int] + ack_from_durable_row: bool = True + + +@dataclasses.dataclass(frozen=True, slots=True) +class _ToolStructuralDebt: + """One accepted assistant tool-call block awaiting matching TOOL rows.""" + + generation: int + call_ids: tuple[str, ...] + + def _cancelled_observed_result_detail( status: EffectStatus | None, *, @@ -910,7 +991,21 @@ class _StreamTurnConsumer: content = self.partial_content() self._flush_terminal_carries() self._session.ui.on_stream_end() - self._session._cancelled_partial_msg = {"role": "assistant", "content": content} + msg: dict[str, Any] = {"role": "assistant", "content": content} + lane = self.lane + if lane is not None: + principal_id = ( + self._session._generation_principals.get(self._my_generation, "") + if self._my_generation + else "" + ) + msg["_provenance"] = TurnProvenance( + model_alias=lane.alias, + backend_model_id=lane.model, + registry_generation=lane.registry_generation, + acting_principal_id=principal_id, + ).to_meta() + self._session._cancelled_partial_msg = msg self._session._publish_for_generation( self._my_generation, @@ -2110,18 +2205,56 @@ def _screen_tool_url(url: str, allow_private_network: bool) -> tuple[str | None, return f"Error: {ssrf_err}.{hint}", False, is_private_block +def _attachment_writes(attachments: Iterable[Attachment]) -> tuple[AttachmentWrite, ...]: + """Project resolved attachments onto the atomic conversation-write shape. + + ``size_bytes`` is derived from the content actually being written rather + than carried alongside it, so the durable row can never disagree with the + blob. Shared by the user-turn path and both accepted-TOOL-row paths. + """ + return tuple( + AttachmentWrite( + attachment_id=attachment.attachment_id, + filename=attachment.filename, + mime_type=attachment.mime_type, + size_bytes=len(attachment.content), + kind=attachment.kind, + content=attachment.content, + ) + for attachment in attachments + ) + + def _tool_turn_meta( - status: EffectStatus | None, preview: dict[str, Any] | None = None + status: EffectStatus | None, + preview: dict[str, Any] | None = None, + *, + acting_principal: str = "", ) -> str | None: """Serialize a tool turn's typed side-channels to the ``conversations.meta`` - JSON envelope: the effect disposition and/or the preview-pane descriptor. - Role-exclusive with ``source_meta`` (which rides SYSTEM turns); the decode + - role routing lives in ``reconstruct_turns``. No channels → no meta.""" + JSON envelope: the effect disposition, the preview-pane descriptor, and/or + the principal the turn executed under. Role-exclusive with ``source_meta`` + (which rides SYSTEM turns); the decode + role routing lives in + ``reconstruct_turns``. No channels → no meta. + + ``acting_principal`` is the generation's bound acting principal — the same + identity the accepted ASSISTANT row records as its provenance + ``acting_principal_id`` axis — so revocation and audit queries can read a + TOOL row's principal directly instead of joining back to the assistant row + that opened its batch. It is deliberately NOT the four-axis provenance + envelope: a tool row is an effect receipt, not a model attempt, and empty + kernel axes would falsify what that envelope promises. Empty (wake, + internal, and CLI lanes) omits the key rather than persisting a + meaningless empty string, matching the USER row's ``sender`` convention. + Private audit metadata: no projection lowers it to a wire or public + payload.""" envelope: dict[str, Any] = {} if status is not None: envelope["effect_status"] = status.value if preview: envelope["preview"] = preview + if acting_principal: + envelope["acting_principal"] = acting_principal return json.dumps(envelope) if envelope else None @@ -2152,6 +2285,29 @@ def _speaks_for_backend(err: BaseException) -> bool: return not isinstance(err, _NON_BACKEND_ERRORS) +def _queued_row_owner(row: tuple[str, ...]) -> str: + """Owner principal of a queued-message row; ``""`` = unowned/legacy. + + THE single reading of the tuple layout "owner is index 2 when + present" — the partition pop, the foreign-row predicate, the + identity-swap notice counting, and the advisory sender stamp must + all agree, or the before_spawn gate and the retention behavior + contradict each other. Deliberately no ``.strip()``: owners are + stripped at write time and legacy rows injected by tests must read + back verbatim. + """ + return row[2] if len(row) >= 3 and row[2] else "" + + +def _queued_row_client_send_id(row: tuple[str, ...]) -> str: + """Browser correlation id of a queued row; ``""`` when absent. + + The ``row[3]`` sibling of :func:`_queued_row_owner` — both fields' + layout knowledge is single-sourced together. + """ + return row[3] if len(row) >= 4 and row[3] else "" + + class ChatSession: # The mid-turn interjection queue's cap — an ALIAS of the shared # per-workstream backpressure bound (see workstream.PENDING_SENDS_MAX): @@ -2283,6 +2439,7 @@ class ChatSession: model_binding, lane=dataclasses.replace( model_binding.lane, + registry_generation=model_binding.registry_generation, temperature=temperature, reasoning_effort=reasoning_effort or None, backend_auth_resolver=self._model_backend_auth_token, @@ -2308,8 +2465,21 @@ class ChatSession: # cannot improve until an admin edits it. self._rebind_failed_key: tuple[str, int] | None = None self._rebind_failed_cause: str | None = None + # Exception objects are not uniformly attribute- or weakref-capable. + # Retain the value-only lane snapshot out-of-band until the fatal + # formatter consumes it; no client, resolver, credential, or principal + # crosses this boundary. + self._serving_failure_context_lock = threading.Lock() + self._serving_failure_contexts: dict[int, tuple[BaseException, _ServingFailureContext]] = {} self._health_registry = health_registry self.ui = ui + # Give the UI a self-derivation path for session-owned reporting + # (persistence state) so it never resolves this session through a + # registry by id — that lookup misses (or hits an id-reuse + # replacement) exactly during tombstone retention and retirement. + bind_session = concrete_method(ui, "bind_session") + if bind_session is not None: + bind_session(self) self.instructions = instructions self.temperature = temperature self.max_tokens = max_tokens @@ -2490,6 +2660,12 @@ class ChatSession: # Turns→dicts at that boundary until those layers migrate. self.messages: list[Turn] = [] self._last_usage: dict[str, int] | None = None + # Provider tokenization varies by coherent serving identity. A + # fallback's ratio and prompt anchor must not replace the primary's: + # the next A dispatch after A -> B fallback resumes A's estimator. + self._token_calibrations: dict[_ModelCalibrationKey, _TokenCalibration] = {} + self._active_token_calibration_key: _ModelCalibrationKey | None = None + self._last_usage_calibration_key: _ModelCalibrationKey | None = None self._msg_tokens: list[int] = [] # parallel to self.messages self._system_tokens = 0 # tokens for system_messages # Workstream template metadata @@ -2558,17 +2734,30 @@ class ChatSession: # OrderedDict preserves FIFO order and supports O(1) removal by ID. # Queued user turns never carry attachments — see # ``AttachmentsNotQueueableError`` for the role-ordering reason — - # so the entry tuple is just ``(cleaned, priority)``. + # so the entry tuple is ``(cleaned, priority, owner_principal)``. + # The immutable owner prevents a retained interjection from being + # drained under another participant after a persistence failure. # Ids retracted while a dispatcher held the popped items — the # DELETE route can land during the handoff's in-flight send, find # the id already popped, and answer "already sent"; a restore # that resurrected it would deliver a message the user explicitly - # cancelled. Written under ``_queued_lock``; cleared at each pop - # (a new window) and consumed by the restore. + # cancelled. Written under ``_queued_lock``; per-id: each pop + # discards the stale records for the ids it pops, the restore + # consumes the records for the ids it considered. self._retracted_while_popped: set[str] = set() - self._queued_messages: collections.OrderedDict[str, tuple[str, str]] = ( - collections.OrderedDict() - ) + # Ids currently held by an OPEN pop window (popped, not yet + # committed/restored/discarded). ``dequeue_message`` records a + # retraction into the ledger above ONLY for these ids — a miss + # for any other id (already delivered, never queued) records + # nothing, which is what bounds the ledger: with per-id + # discipline nothing would ever prune such an entry. All + # mutation under ``_queued_lock``; every window close removes + # its ids via ``_restore_queued_messages`` (atomic with its + # ledger consume) or ``_close_pop_window``. + self._popped_in_flight: set[str] = set() + self._queued_messages: collections.OrderedDict[ + str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str] + ] = collections.OrderedDict() self._queued_lock = threading.Lock() # Repeat detection: streak counter over tool-call signatures. # Fires when a (name, args) signature has been seen N times in @@ -2605,6 +2794,21 @@ class ChatSession: # outside this lock and cooperatively cancellable. self._generation_transition_lock = threading.Lock() self._generation_lock = threading.RLock() + # An accepted assistant row with provider tool calls creates a + # generation-owned structural obligation: one journaled TOOL row per + # call must exist before a terminal close or successor generation can + # cross it. The condition lets soft close cooperatively cancel and + # wait briefly without holding the generation lock needed by the + # worker's cancellation/failure finalizer. + self._tool_structural_condition = threading.Condition(self._generation_lock) + self._tool_structural_debt: _ToolStructuralDebt | None = None + self._soft_close_preparing = False + # Acting-user changes touch several coupled MCP listener/catalog + # projections. Serialize the whole rebind independently of generation + # transitions: Stop/force remains responsive, while a stale predecessor + # must finish (and fail its owner check) before the successor installs + # the final actor projection. + self._acting_user_bind_lock = threading.Lock() self._publication_shutdown = False # Durable writes admitted by generation commits execute in the same # order as their in-memory/live commits, but never while @@ -2616,6 +2820,52 @@ class ChatSession: self._durability_cond = threading.Condition(threading.Lock()) self._durability_next_ticket = 0 self._durability_serving_ticket = 0 + # A destructive history cut is admitted onto the durability lane, but + # applies its live slice only after the storage transaction commits. + # While that ticket is active, later generation/direct commits wait on + # this condition before mutating conversation state. Stop and terminal + # generation claims use ``_generation_lock`` directly and therefore + # remain responsive while storage is blocked. + self._history_truncation_condition = threading.Condition(self._generation_lock) + self._history_truncation_active = False + # Conversation rows cross two independently observed surfaces: + # storage history and the live SSE stream. One ordered journal keeps + # their immutable, idempotently keyed rows from bounded generation + # admission until storage has positively acknowledged them. + # ``_history_handoff_lock`` owns the journal + monotonic accepted + # revision and is also the atomic bridge to initial listener + # registration. ``_history_visibility_lock`` is deliberately separate + # from ``_generation_lock`` and spans only a history load or one + # conversation save/ack, making DB visibility and the pending overlay + # linearizable without blocking Stop/force-claim. + self._history_handoff_lock = threading.RLock() + # Reentrant because reconciliation owns the visibility lane across its + # publication-shutdown recheck and calls the single-entry persistence + # helper, which uses the same lane. Lock order is visibility -> + # generation -> handoff; generation commit paths never acquire + # visibility while holding the generation lock. + self._history_visibility_lock = threading.RLock() + self._history_handoff_epoch = uuid.uuid4().hex + self._history_handoff_revision = 0 + # KEYED BY WORKSTREAM IDENTITY: set to the ws_id whose durable parent + # a keyed save observed hard-deleted (cross-node delete or prune), + # never cleared. Readers compare against the CURRENT self._ws_id, so + # an identity swap (/new, resume) structurally un-poisons the session + # object while the latch stays permanent for the workstream that died + # — no reset choreography to forget (round-4 review). + self._workstream_gone_ws: str | None = None + self._pending_conversation_commits: collections.OrderedDict[ + str, _PendingConversationCommit + ] = collections.OrderedDict() + self._conversation_persistence_error: ConversationPersistenceError | None = None + self._conversation_persistence_failure_kind: str | None = None + self._conversation_persistence_attempts = 0 + self._conversation_persistence_first_failure_at: datetime | None = None + self._conversation_persistence_last_failure_at: datetime | None = None + self._conversation_persistence_next_retry_at: float | None = None + self._conversation_persistence_next_retry_wall_at: datetime | None = None + self._conversation_persistence_resync_commit_key: str | None = None + self._conversation_persistence_fatal_revision: int | None = None # Monotonic edge for approval gates that exist before a generation is # claimed (the token-budget override). Unlike ``_cancel_event``, a # snapshot can distinguish a new Stop from a harmless set event left @@ -2909,9 +3159,14 @@ class ChatSession: """ lane = self._model_binding.lane effort = self.reasoning_effort or None - if lane.temperature != self.temperature or lane.reasoning_effort != effort: + if ( + lane.registry_generation != self._model_binding.registry_generation + or lane.temperature != self.temperature + or lane.reasoning_effort != effort + ): lane = dataclasses.replace( lane, + registry_generation=self._model_binding.registry_generation, temperature=self.temperature, reasoning_effort=effort, ) @@ -2945,10 +3200,10 @@ class ChatSession: cs = getattr(self, "_config_store", None) if cs is None: return jc - snapshot = getattr(type(cs), "effective_snapshot", None) - if callable(snapshot): + snapshot = concrete_method(cs, "effective_snapshot") + if snapshot is not None: try: - _version, values = snapshot(cs) + _version, values = snapshot() if isinstance(values, dict): return self._compose_judge_cfg(values.get) except Exception: @@ -2997,10 +3252,10 @@ class ChatSession: if self._judge_config is None: return None, None cs = getattr(self, "_config_store", None) - snapshot = getattr(type(cs), "effective_snapshot", None) - if callable(snapshot): + snapshot = concrete_method(cs, "effective_snapshot") + if snapshot is not None: try: - version, values = snapshot(cs) + version, values = snapshot() if type(version) is int and isinstance(values, dict): return self._compose_judge_cfg(values.get), version except Exception: @@ -4580,6 +4835,75 @@ class ChatSession: """Estimated token cost of the active tool definitions.""" return int(self._tool_def_chars(caps) / self._chars_per_token) + @staticmethod + def _token_calibration_key(lane: ModelLane) -> _ModelCalibrationKey: + """Return the coherent registry identity whose tokenizer is in use.""" + return (lane.alias, lane.model, lane.registry_generation) + + @staticmethod + def _provenance_calibration_key( + provenance: TurnProvenance | None, + ) -> _ModelCalibrationKey | None: + """Project accepted-turn provenance onto its calibration identity.""" + if provenance is None: + return None + key = ( + provenance.model_alias, + provenance.backend_model_id, + provenance.registry_generation, + ) + return key if any((key[0], key[1], key[2])) else None + + def _calibration_anchor_valid(self, calibration: _TokenCalibration) -> bool: + """Whether a provider prompt count still names the live prefix.""" + prefix = calibration.message_prefix_ids + if calibration.prompt_tokens is None or len(prefix) > len(self.messages): + return False + return prefix == tuple(id(turn) for turn in self.messages[: len(prefix)]) + + def _activate_token_calibration(self, lane: ModelLane) -> None: + """Activate *lane*'s ratio and coherent provider prompt anchor. + + Lane activation happens before every main dispatch and before pre-send + budget checks. Switching identities re-estimates the whole live + trajectory with that tokenizer ratio; staying on one identity preserves + exact completion-token counts already appended by the usage path. + """ + key = self._token_calibration_key(lane) + calibration = self._token_calibrations.get(key) + ratio = calibration.chars_per_token if calibration is not None else 4.0 + identity_changed = ( + self._active_token_calibration_key != key or self._chars_per_token != ratio + ) + self._active_token_calibration_key = key + self._chars_per_token = ratio + + sys_chars = sum(self._msg_char_count(message) for message in self.system_messages) + self._system_tokens = max(1, int(sys_chars / ratio)) + if identity_changed: + self._msg_tokens = [ + max(1, int(self._msg_char_count(message) / ratio)) for message in self.messages + ] + if calibration is not None and self._calibration_anchor_valid(calibration): + self._calibrated_msg_count = len(calibration.message_prefix_ids) + else: + self._calibrated_msg_count = 0 + if not self._manual_tool_truncation: + self.tool_truncation = int(self.context_window * ratio * 0.5) + + def _invalidate_token_calibration_anchors(self) -> None: + """Forget prompt-prefix counts after replacing/removing history. + + Tokenizer ratios remain useful per model binding; only the provider + counts tied to the old Turn identities become invalid. + """ + self._token_calibrations = { + key: dataclasses.replace(value, prompt_tokens=None, message_prefix_ids=()) + for key, value in self._token_calibrations.items() + } + self._last_usage_calibration_key = None + self._calibrated_msg_count = 0 + def _estimated_prompt_tokens(self) -> int: """Best estimate of the current prompt size, in tokens. @@ -4595,9 +4919,14 @@ class ChatSession: auto-compaction triggers all read it, so they cannot disagree about the fullness of the same state. """ - if self._last_usage: - # Clamp the index: a stale _calibrated_msg_count must not - # over-slice after compaction or message-list mutations. + active_key = self._active_token_calibration_key + calibration = self._token_calibrations.get(active_key) if active_key is not None else None + if calibration is not None and self._calibration_anchor_valid(calibration): + start = len(calibration.message_prefix_ids) + return int(calibration.prompt_tokens or 0) + sum(self._msg_tokens[start:]) + if self._last_usage and self._last_usage_calibration_key is None: + # Compatibility for direct seam callers/tests that populate the + # legacy usage slot without going through _update_token_table. start = min(self._calibrated_msg_count, len(self._msg_tokens)) return self._last_usage["prompt_tokens"] + sum(self._msg_tokens[start:]) # No provider anchor yet (e.g. a just-resumed session before its first @@ -4939,6 +5268,7 @@ class ChatSession: ], max_tokens=_TITLE_MAX_TOKENS, lane=title_lane, + principal_id=captured_principal, ) raw = result.content or "" log.info("ws.title.llm_response", ws_id=ws_id[:8], raw=raw[:200]) @@ -5353,11 +5683,13 @@ class ChatSession: self._read_files.clear() self._repeat_detector.clear() self._last_usage = None - self._calibrated_msg_count = 0 + self._invalidate_token_calibration_anchors() self._title_generated = True # don't re-title resumed workstreams self._msg_tokens = [ max(1, int(self._msg_char_count(m) / self._chars_per_token)) for m in self.messages ] + if not self._manual_tool_truncation: + self.tool_truncation = int(self.context_window * self._chars_per_token * 0.5) resume_diagnostics = lane_diagnostics(self._primary_lane()) log.info( "Resuming ws=%s: %d messages, provider=%s, model=%s", @@ -5518,6 +5850,7 @@ class ChatSession: if not fork: self._follow_watch_registration(old_ws_id) self._init_system_messages() + self._activate_token_calibration(self._primary_lane()) return True def _follow_watch_registration(self, old_ws_id: str) -> None: @@ -6752,6 +7085,7 @@ class ChatSession: if deferred_persistence is None: clear_last_error(persist_ws_id) self._has_persisted_error = False + self._conversation_persistence_fatal_revision = None else: def _clear_if_owned() -> None: @@ -6769,6 +7103,7 @@ class ChatSession: and self._persisted_error_revision == error_revision ): self._has_persisted_error = False + self._conversation_persistence_fatal_revision = None deferred_persistence.append(_clear_if_owned) # Surface the acting user (turn initiator) to the UI so web clients can @@ -6803,6 +7138,80 @@ class ChatSession: else: self.ui.on_state_change(state) + def _remember_serving_failure_context( + self, + exc: BaseException, + lane: ModelLane, + ) -> None: + """Associate *exc* with the value-only lane that actually failed.""" + model_alias = lane.alias + backend_model_id = lane.model + registry_generation = lane.registry_generation + provider_label = "?" + base_url = "?" + try: + diagnostics = lane_diagnostics(lane) + provider_label = diagnostics.provider_name or diagnostics.provider_type + # Query parameters are never diagnostic and can carry API keys. + base_url = diagnostics.base_url.split("?", 1)[0].rstrip("/") + except Exception: + log.debug("session.fatal.serving_lane_snapshot_failed", exc_info=True) + context = _ServingFailureContext( + model_alias=model_alias, + backend_model_id=backend_model_id, + registry_generation=registry_generation, + provider_label=provider_label, + base_url=base_url, + ) + with self._serving_failure_context_lock: + # Bound abandoned/recovered retry objects. Entries are normally + # consumed synchronously by _record_fatal_error. + while len(self._serving_failure_contexts) >= 32: + self._serving_failure_contexts.pop(next(iter(self._serving_failure_contexts))) + # Retain object identity beside the id: a recovered exception can + # otherwise be collected and its integer id reused before a stale + # bounded entry is evicted, misattributing an unrelated failure. + self._serving_failure_contexts[id(exc)] = (exc, context) + + def _take_serving_failure_context( + self, + exc: BaseException, + ) -> _ServingFailureContext | None: + """Consume the lane snapshot associated with *exc*, if any.""" + with self._serving_failure_context_lock: + stored = self._serving_failure_contexts.pop(id(exc), None) + if stored is None or stored[0] is not exc: + return None + return stored[1] + + def _forget_serving_failure_context(self, exc: BaseException | None) -> None: + """Drop a recovered retry's no-longer-actionable lane snapshot.""" + if exc is None: + return + with self._serving_failure_context_lock: + stored = self._serving_failure_contexts.get(id(exc)) + if stored is not None and stored[0] is exc: + self._serving_failure_contexts.pop(id(exc), None) + + @contextlib.contextmanager + def _recovered_serving_failures(self) -> Iterator[list[BaseException]]: + """Own the lane snapshots a retry ladder takes, and drop the dead ones. + + The ladder appends every failure it remembered to the yielded list. + Leaving the block forgets all of them EXCEPT an exception that is still + escaping: that one's snapshot is what ``_record_fatal_error`` consumes + to name the lane that actually failed. + """ + recovered: list[BaseException] = [] + try: + yield recovered + except BaseException as escaping: + recovered = [failure for failure in recovered if failure is not escaping] + raise + finally: + for failure in recovered: + self._forget_serving_failure_context(failure) + def _record_fatal_error( self, exc: BaseException, @@ -6839,7 +7248,13 @@ class ChatSession: """ from turnstone.core.memory import persist_last_error, sanitize_error_text - raw = self._format_backend_error(exc) or f"{type(exc).__name__}: {exc}" + take_context = getattr(self, "_take_serving_failure_context", None) + serving_context = take_context(exc) if callable(take_context) else None + raw = ( + self._format_backend_error(exc) + if serving_context is None + else self._format_backend_error(exc, serving_context=serving_context) + ) or f"{type(exc).__name__}: {exc}" safe = sanitize_error_text(raw) # The one journal trace of a fatal turn — UI sinks and the config row # are invisible to log scrapers. Sanitized text only (same @@ -6847,12 +7262,18 @@ class ChatSession: # through this chokepoint too; it is a user action, not a fault, so it # logs at INFO instead of ERROR. fatal_log = log.info if isinstance(exc, KeyboardInterrupt) else log.error - fatal_log( - "session.fatal.recorded", - ws=self._ws_id, - error_type=type(exc).__name__, - error=safe, - ) + fatal_fields: dict[str, Any] = { + "ws": self._ws_id, + "error_type": type(exc).__name__, + "error": safe, + } + if serving_context is not None: + fatal_fields.update( + model_alias=serving_context.model_alias, + backend_model_id=serving_context.backend_model_id, + registry_generation=serving_context.registry_generation, + ) + fatal_log("session.fatal.recorded", **fatal_fields) # Frames only — ``exc_info=True`` would render the raw exception # message, which can carry credentials verbatim (the sanitize floor # the lines above exist to hold); format_tb renders the stack @@ -6875,8 +7296,18 @@ class ChatSession: persist_last_error(persist_ws_id, safe) else: deferred_persistence.append(functools.partial(persist_last_error, persist_ws_id, safe)) - self._persisted_error_revision = getattr(self, "_persisted_error_revision", 0) + 1 - self._has_persisted_error = True + # Manager-owned persistence recovery reads and retires this exact + # revision under the generation lock. Publish the revision and both + # ownership latches as one atomic snapshot so a concurrent maintenance + # pass cannot observe a new revision with stale ownership metadata. + with self._generation_lock: + self._persisted_error_revision = getattr(self, "_persisted_error_revision", 0) + 1 + self._has_persisted_error = True + self._conversation_persistence_fatal_revision = ( + self._persisted_error_revision + if isinstance(exc, ConversationPersistenceError) + else None + ) self._emit_state("error", deferred_persistence=deferred_persistence) def ensure_error_recorded(self, exc: BaseException) -> None: @@ -6916,7 +7347,12 @@ class ChatSession: return self._record_fatal_error(exc) - def _format_backend_error(self, exc: BaseException) -> str | None: + def _format_backend_error( + self, + exc: BaseException, + *, + serving_context: _ServingFailureContext | None = None, + ) -> str | None: """Return an enriched message for known backend boundary errors. Returns ``None`` for exceptions outside the recognised set so the @@ -6946,8 +7382,14 @@ class ChatSession: # The backend id (what the server was actually asked for) rides as a # labeled annotation for the operator, collapsing to one token when the # two coincide. Both labels derive from the frozen session binding. - alias = self._model_alias or "" - backend_id = self.model or "" + alias = ( + serving_context.model_alias + if serving_context is not None + else (self._model_alias or "") + ) + backend_id = ( + serving_context.backend_model_id if serving_context is not None else (self.model or "") + ) if alias and backend_id and alias != backend_id: model_label = f"{alias} (id={backend_id})" else: @@ -7007,8 +7449,13 @@ class ChatSession: # the turn. Remediation is PER-LANE: /model is routable only on the # CLI and node-interactive command lanes, so the console coordinator # — which routes no slash commands — gets recreate-or-adjust wording. - if self._registry_alias_removed or ( - self._rebind_failed_key is not None and self._rebind_failed_key[0] == self._model_alias + context_is_primary = serving_context is None or alias == (self._model_alias or "") + if context_is_primary and ( + self._registry_alias_removed + or ( + self._rebind_failed_key is not None + and self._rebind_failed_key[0] == self._model_alias + ) ): available = "" if self._registry is not None and self._registry.count: @@ -7038,17 +7485,18 @@ class ChatSession: # Pull backend identity — every branch swallows so a bad # accessor on a partially-initialised session can't hide the # original exception behind a NoneType error. - base_url = "?" - provider_label = "?" - try: - # The PRIMARY binding, deliberately: endpoint, model label, and - # provider must describe one coherent lane even if a fallback - # produced the terminal error. - diagnostics = lane_diagnostics(self._primary_lane()) - base_url = diagnostics.base_url.split("?")[0].rstrip("/") - provider_label = diagnostics.provider_name or diagnostics.provider_type - except Exception: - log.debug("session.fatal.lane_diagnostics_failed", exc_info=True) + if serving_context is not None: + base_url = serving_context.base_url + provider_label = serving_context.provider_label + else: + base_url = "?" + provider_label = "?" + try: + diagnostics = lane_diagnostics(self._primary_lane()) + base_url = diagnostics.base_url.split("?")[0].rstrip("/") + provider_label = diagnostics.provider_name or diagnostics.provider_type + except Exception: + log.debug("session.fatal.lane_diagnostics_failed", exc_info=True) if name in _BACKEND_TIMEOUT_EXC_NAMES: return ( f"Backend timeout ({name}): no response from {provider_label} " @@ -7155,14 +7603,18 @@ class ChatSession: resolved dict rather than resolved separately — one config generation, as ``resolve_lane`` intends. - ``principal_id`` pins dynamic backend authentication for work that - outlives the caller's mutable session binding (compaction and parallel - tool extraction). ``None`` preserves the lane's existing resolver for - direct and already-pinned callers such as title generation. + ``principal_id`` pins dynamic backend authentication and accepted-turn + audit identity for work that outlives the caller's mutable session + binding (compaction, title generation, and parallel tool extraction). + ``None`` snapshots the session's effective principal at entry. """ lane = lane or self._primary_lane() - if principal_id is not None: - lane = self._lane_for_backend_auth_principal(lane, principal_id) + effective_principal_id = ( + (self._mcp_effective_user_id or "").strip() + if principal_id is None + else principal_id.strip() + ) + lane = self._lane_for_backend_auth_principal(lane, effective_principal_id) caps = require_lane_capabilities(lane) clamped = min(max_tokens, caps.max_output_tokens) if caps.max_output_tokens else max_tokens suppress_reasoning = lane_thinking_suppressed(lane) @@ -7180,6 +7632,7 @@ class ChatSession: # parallel web-fetch calls each pass an independent # StreamAbortRef that never publishes into the main stream slot. cancel_ref=cancel_ref, + acting_principal_id=effective_principal_id, ) # Utility completions (title gen, compaction, web-fetch extraction) # bypass the streaming on_status path — record their usage so the @@ -7635,6 +8088,10 @@ class ChatSession: """ if principal_id is not None: lane = self._lane_for_backend_auth_principal(lane, principal_id) + # Wire preparation, attachment materialization, and tool-output budget + # helpers all read the session estimator. Activate the exact lane + # before any of them can run. + self._activate_token_calibration(lane) diagnostics = lane_diagnostics(lane) safe_url = diagnostics.base_url.split("?")[0] # query params may contain keys caps = require_lane_capabilities(lane) @@ -7669,6 +8126,7 @@ class ChatSession: principal_id=principal_id, ), cancel_ref=ref, + acting_principal_id=principal_id or "", on_chunk=consumer, ) except Exception as e: @@ -7770,6 +8228,84 @@ class ChatSession: for scope in scopes: scope.abort() + def _capture_worker_claim(self, principal_id: str = "") -> WorkerClaim | None: + """Capture one fresh worker slot's immutable session admission. + + ``session_worker.send`` calls this immediately before entering the + workstream slot lock. Capturing in that order avoids the inverse of a + generation commit's generation-lock -> UI -> workstream-lock path. + Stop/close publish their monotonic edge under the same generation lock + used here; if one lands between this snapshot and slot installation, + send entry rejects the conservatively stale epoch. + + Internal workers omit an authenticated principal; pin the current + effective actor at claim time so wake/init continuations cannot later + borrow a mutable actor changed by another request. + """ + + with self._generation_lock: + if self._publication_shutdown: + raise RuntimeError("Cannot claim a worker on a closed session") + if self._soft_close_preparing: + raise RuntimeError("Cannot claim a worker while the session is closing") + if self._tool_structural_debt is not None and self._cancel_event.is_set(): + # An exceptional cancellation/close window still owes TOOL + # receipts. Refuse a fresh/reuse dispatch until its owner has + # journaled them; normal in-flight tool execution keeps the + # event clear and therefore retains interjection behavior. + raise RuntimeError("Cannot claim a worker while tool cleanup is pending") + principal = principal_id.strip() or (self._mcp_effective_user_id or "").strip() + cancel_event = self._cancel_event + return WorkerClaim( + session=self, + principal_id=principal, + cancel_epoch=self._approval_cancel_epoch, + cancel_event=cancel_event, + cancel_event_was_set=cancel_event.is_set(), + ) + + def _worker_claim_is_current(self, claim: WorkerClaim) -> bool: + """Revalidate a captured worker witness without acquiring any lock. + + ``session_worker.send`` calls this while holding the workstream lock, + so this method MUST remain a bounded read of fields published under + ``_generation_lock``. The predecessor releases its workstream slot + only after those writes, making the enqueue/spawn crossing coherent + without introducing a workstream -> generation lock inversion. + + Event identity deliberately is not compared: a normal first worker + rotates the session Event when it claims its generation, while a + concurrent sender captured just before that rotation must still be + allowed to enqueue. An unset-to-set transition on the exact captured + Event is the monotonic evidence that cancellation/structural poison + overtook the claim. An Event already set at capture can be the valid + post-truncation state that the fresh worker is expected to rotate. + """ + return bool( + claim.session is self + and claim.cancel_epoch == getattr(self, "_approval_cancel_epoch", -1) + and (claim.cancel_event_was_set or not claim.cancel_event.is_set()) + and not getattr(self, "_publication_shutdown", False) + and not self._soft_close_preparing + and not (self._tool_structural_debt is not None and self._cancel_event.is_set()) + ) + + def _poison_tool_structural_debt_locked(self, generation: int | None = None) -> bool: + """Publish one fail-stop edge for an incomplete accepted tool block. + + Caller holds ``_generation_lock``. Advancing the existing monotonic + worker epoch only on the Event's unset-to-set transition invalidates a + claim captured before the poison without repeatedly advancing it when + force and send-finally observe the same retained debt. + """ + debt = self._tool_structural_debt + if debt is None or (generation is not None and debt.generation != generation): + return False + if not self._cancel_event.is_set(): + self._approval_cancel_epoch = getattr(self, "_approval_cancel_epoch", 0) + 1 + self._cancel_event.set() + return True + def cancel(self) -> None: """Request cancellation of the current generation. @@ -7857,7 +8393,76 @@ class ChatSession: if _generation_superseded(self, my_generation): raise GenerationCancelled() - def _claim_generation(self, *, principal_id: str | None = None) -> int: + def _check_generation_admission(self, my_generation: int) -> None: + """Fence pre-turn setup against Stop, force-successor, and close. + + This deliberately does not dispatch through ``_check_cancelled``. + That later cooperative probe is an established lightweight-session + test seam and also includes task-agent-local cancellation. The setup + path needs the narrower, non-overridable session ownership invariant: + once a generation loses its event, number, or publication latch it may + not bind an actor, refresh configuration, or admit a user row. + """ + + if ( + self._publication_shutdown + or self._cancel_event.is_set() + or _generation_superseded(self, my_generation) + ): + raise GenerationCancelled() + + def _admit_tool_structural_debt_locked( + self, + generation: int, + call_ids: tuple[str, ...], + ) -> None: + """Record an accepted assistant tool block under generation ownership.""" + if not call_ids: + return + if self._tool_structural_debt is not None: + raise RuntimeError("Cannot accept a new tool block before completing the prior block") + self._tool_structural_debt = _ToolStructuralDebt( + generation=generation, + call_ids=call_ids, + ) + + def has_tool_structural_debt(self) -> bool: + """Whether an accepted assistant block still needs TOOL receipts.""" + with self._generation_lock: + return self._tool_structural_debt is not None + + def _complete_tool_structural_debt_locked( + self, + generation: int, + completed_call_ids: list[str] | tuple[str, ...], + ) -> bool: + """Clear exact generation debt only after every TOOL row is journaled.""" + debt = self._tool_structural_debt + if debt is None: + return False + if debt.generation != generation: + raise RuntimeError("Tool structural debt belongs to another generation") + expected = collections.Counter(debt.call_ids) + observed = collections.Counter(completed_call_ids) + if observed != expected: + missing = expected - observed + unexpected = observed - expected + details: list[str] = [] + if missing: + details.append(f"missing: {', '.join(sorted(missing.elements()))}") + if unexpected: + details.append(f"unexpected: {', '.join(sorted(unexpected.elements()))}") + raise RuntimeError(f"Tool structural debt mismatch ({'; '.join(details)})") + self._tool_structural_debt = None + self._tool_structural_condition.notify_all() + return True + + def _claim_generation( + self, + *, + principal_id: str | None = None, + expected_cancel_epoch: int | None = None, + ) -> int: """Claim the next generation and install its fresh cancel event. The entry half of the per-generation cancel discipline shared by @@ -7882,6 +8487,18 @@ class ChatSession: with self._generation_transition_lock, self._generation_lock: if self._publication_shutdown: raise RuntimeError("Cannot claim a generation on a closed session") + if self._soft_close_preparing: + raise RuntimeError("Cannot claim a generation while the session is closing") + if self._tool_structural_debt is not None: + raise RuntimeError("Cannot claim a generation before tool cleanup completes") + if ( + expected_cancel_epoch is not None + and self._approval_cancel_epoch != expected_cancel_epoch + ): + # This worker slot predates a Stop/force/terminal admission. + # Never replace the edge it was meant to observe with a fresh + # event: a successor may already own the workstream slot. + raise GenerationCancelled() self._generation += 1 generation = self._generation self._cancel_event = threading.Event() @@ -7931,9 +8548,11 @@ class ChatSession: zero generation is the legacy/direct-call unscoped form and remains publishable. """ - with self._generation_lock: + with self._history_truncation_condition: + self._history_truncation_condition.wait_for(lambda: not self._history_truncation_active) if ( self._publication_shutdown + or (not origin_generation and self._soft_close_preparing) or (origin_generation and self._generation != origin_generation) or (not allow_cancelled and self._cancel_event.is_set()) ): @@ -7941,12 +8560,904 @@ class ChatSession: publish() return True + def _history_handoff_token_locked(self) -> str: + """Return the opaque token for the currently accepted row prefix.""" + return f"{self._history_handoff_epoch}.{self._history_handoff_revision}" + + def capture_history_handoff( + self, + load_messages: Callable[[int], list[dict[str, Any]]], + ) -> tuple[list[dict[str, Any]], str]: + """Load durable history and merge accepted conversation rows once. + + ``load_messages`` runs while the per-session visibility lane is held. + Every conversation-row persistence closure uses the same lane through + durable commit and journal acknowledgement, so the result linearizes on one side of + that transition: + + * reader first: old durable prefix plus the pending immutable row; + * writer first: new durable prefix with the journal entry removed. + + A backend commit whose acknowledgement was lost is identified by its + stable ``_commit_key`` and reconciled during this read. The accepted + revision does not change when a row moves from journal to storage — the + two representations are visibility-equivalent. + + The loader receives the pending-journal size and must widen any + tail-bounded window by that many rows. Within one live session every + durable write flows through the journal in admission order and the + visibility lane freezes the durable snapshot across this whole + capture, so a pending key's committed twin — acknowledged or lost — + is always among the newest ``journal size`` durable rows: the widened + window necessarily contains it, and a journal row missing from the + window is genuinely not yet durable and appends at the tail in + admission order. (A second writer — cross-node re-home overlap or a + legacy NULL-key append — can in principle push a twin beyond the + widened window; that residual degrades to the same tail append and + self-heals on reconcile.) The caller keeps ownership of the loaded + rows — they are merged without copying, so the loader must return + rows nothing else retains; only journal overlay entries are copied. + No storage call ever runs under the handoff lock here: the journal + size is sampled first, the load runs outside it, and the merge is + pure in-memory work. + """ + notify_state_change = False + with self._history_visibility_lock: + with self._history_handoff_lock: + if self._workstream_gone_ws == self._ws_id: + # The durable parent is deleted: there is no authoritative + # transcript to verify a handoff against, and minting a + # token over the empty load is exactly the silent-wipe + # mechanism (round-3 review). Raising routes /history to + # its fail-closed 503 arm, which keeps the pane's stale + # transcript visible. + raise ConversationCommitWorkstreamGoneError( + "workstream was deleted; no history handoff can be verified" + ) + overscan = len(self._pending_conversation_commits) + loaded = load_messages(overscan) + if not isinstance(loaded, list): + raise TypeError("history loader must return a list") + with self._history_handoff_lock: + previous_state = self._conversation_persistence_state_locked() + durable_indexes = { + str(message.get("_commit_key")): index + for index, message in enumerate(loaded) + if isinstance(message, dict) and message.get("_commit_key") + } + for commit_key, pending in list(self._pending_conversation_commits.items()): + if commit_key in durable_indexes and pending.ack_from_durable_row: + self._pending_conversation_commits.pop(commit_key, None) + if not self._pending_conversation_commits: + self._clear_conversation_persistence_failure_locked() + notify_state_change = self._conversation_persistence_state_needs_notification( + previous_state, + self._conversation_persistence_state_locked(), + ) + + merged = list(loaded) + for commit_key, pending in self._pending_conversation_commits.items(): + durable_index = durable_indexes.get(commit_key) + if durable_index is not None: + # A non-atomic extension may expose a keyed row before + # its full closure ACKs — and a conflicted entry + # (``ack_from_durable_row`` False) deliberately renders + # the journal's version in place, fail-visible beside + # its persistence banner. + merged[durable_index] = copy.deepcopy(pending.message) + else: + merged.append(copy.deepcopy(pending.message)) + result = merged, self._history_handoff_token_locked() + if notify_state_change: + self._notify_conversation_persistence_state_changed() + return result + + def register_listener_for_history_handoff( + self, + token: str, + *, + last_event_id: int | None = None, + maxsize: int = 500, + ) -> tuple[Any, list[dict[str, Any]], str, int, int, dict[str, Any]] | None: + """Validate a history token and atomically register its SSE listener. + + Every conversation-row admission takes the same short handoff lock + while journaling a complete row and publishing the corresponding live + transition. Therefore a commit either changes the token before this + comparison or reaches the listener registered here; it cannot fall + between those operations. A mismatched epoch/revision returns ``None`` + and callers must refetch history rather than trusting numeric ring + coverage. + """ + with self._history_handoff_lock: + if not token or token != self._history_handoff_token_locked(): + return None + ui_base: Any = self.ui + if last_event_id is not None: + return cast( + "tuple[Any, list[dict[str, Any]], str, int, int, dict[str, Any]]", + ui_base.register_listener_with_replay(last_event_id, maxsize=maxsize), + ) + client_queue, snapshot = ui_base.register_listener_with_in_progress_snapshot( + maxsize=maxsize + ) + return client_queue, [], "fresh", 0, 0, snapshot + + def has_unresolved_conversation_persistence(self) -> bool: + """Whether an accepted row lacks a confirmed complete durable commit. + + Blocking form — only for callers holding NO workstream or manager + lock (reconcile scans, delete-ambiguity arms). Scans that probe while + holding ``ws._lock`` / the manager lock must use + :meth:`has_unresolved_conversation_persistence_nowait` instead: this + method acquires the generation and handoff locks, which inverts the + generation→workstream/manager order those scans hold. + """ + with self._generation_lock: + if self._tool_structural_debt is not None: + return True + with self._history_handoff_lock: + return bool(self._pending_conversation_commits) + + def has_unresolved_conversation_persistence_nowait(self) -> bool | None: + """Non-blocking probe for retirement scans holding ws/manager locks. + + Returns ``None`` when either session lock is momentarily held — + callers treat that as "busy, skip this candidate this sweep": a busy + session is never closed or evicted on a stale answer, and never + blocked on. Blocking here from under ``ws._lock`` deadlocks against + force-cancel's finalizer, which publishes under the generation lock + and then takes the workstream lock (round-4 review). + """ + if not self._generation_lock.acquire(blocking=False): + return None + try: + if self._tool_structural_debt is not None: + return True + finally: + self._generation_lock.release() + if not self._history_handoff_lock.acquire(blocking=False): + return None + try: + return bool(self._pending_conversation_commits) + finally: + self._history_handoff_lock.release() + + def is_workstream_gone(self) -> bool: + """True once a keyed save observed THIS workstream's parent deleted. + + The latch records the ws_id that died and is never cleared; readers + compare it against the current identity, so an identity swap (/new, + ``resume``) to a different workstream reads False structurally. + While it matches, new conversation admissions refuse (the + finalizer/force-abandon lanes still converge via + ``allow_workstream_gone``) and ``capture_history_handoff`` refuses to + mint tokens, so a pane keeps its stale transcript rather than + rendering a silently wiped one. + """ + with self._history_handoff_lock: + return self._workstream_gone_ws == self._ws_id + + def conversation_persistence_status(self) -> dict[str, object]: + """Return a content-free live projection of durable journal health.""" + with self._generation_lock: + structural_pending = ( + len(self._tool_structural_debt.call_ids) + if self._tool_structural_debt is not None + else 0 + ) + with self._history_handoff_lock: + pending_rows = len(self._pending_conversation_commits) + structural_pending + # One classifier, shared with the notification edge. Structural + # tool debt widens an otherwise-healthy journal to ``pending``: + # its receipts will re-journal but are not yet in the commit map. + state = self._conversation_persistence_state_locked() + if state == "healthy" and structural_pending: + state = "pending" + return { + "state": state, + "pending_rows": pending_rows, + "attempts": self._conversation_persistence_attempts, + "first_failure_at": ( + self._conversation_persistence_first_failure_at.isoformat() + if self._conversation_persistence_first_failure_at is not None + else None + ), + "last_failure_at": ( + self._conversation_persistence_last_failure_at.isoformat() + if self._conversation_persistence_last_failure_at is not None + else None + ), + "next_retry_at": ( + self._conversation_persistence_next_retry_wall_at.isoformat() + if self._conversation_persistence_next_retry_wall_at is not None + else None + ), + } + + def conversation_persistence_fatal_revision(self) -> int | None: + """Return the current fatal-error revision only when persistence owns it.""" + with self._generation_lock: + revision = self._conversation_persistence_fatal_revision + if ( + revision is None + or not self._has_persisted_error + or revision != self._persisted_error_revision + ): + return None + return revision + + def acknowledge_conversation_persistence_state_recovery(self, revision: int) -> bool: + """Retire one exact in-memory fatal latch after manager recovery. + + The sanitized durable ``last_error`` remains as an audit record; public + readers expose it only while the workstream state is ``error``. A later + fatal error therefore cannot be deleted by a stale repair callback. + """ + with self._generation_lock: + if ( + self._conversation_persistence_fatal_revision != revision + or self._persisted_error_revision != revision + or not self._has_persisted_error + ): + return False + self._conversation_persistence_fatal_revision = None + self._has_persisted_error = False + return True + + def _clear_conversation_persistence_failure_locked(self) -> None: + """Reset failure metadata after the complete accepted prefix ACKs.""" + self._conversation_persistence_error = None + self._conversation_persistence_failure_kind = None + self._conversation_persistence_attempts = 0 + self._conversation_persistence_first_failure_at = None + self._conversation_persistence_last_failure_at = None + self._conversation_persistence_next_retry_at = None + self._conversation_persistence_next_retry_wall_at = None + self._conversation_persistence_resync_commit_key = None + + def _conversation_persistence_retry_delay(self, attempt: int) -> float: + """Return equal-jitter exponential backoff for one transient failure.""" + ceiling = min( + _CONVERSATION_PERSISTENCE_RETRY_CAP_SECONDS, + _CONVERSATION_PERSISTENCE_RETRY_BASE_SECONDS * (2.0 ** min(30, max(0, attempt - 1))), + ) + return random.uniform(ceiling / 2.0, ceiling) + + def _conversation_persistence_state_locked(self) -> str: + if not self._pending_conversation_commits: + return "healthy" + return self._conversation_persistence_failure_kind or "pending" + + @staticmethod + def _conversation_persistence_state_needs_notification( + previous_state: str, + current_state: str, + ) -> bool: + failure_states = {"retrying", "conflict"} + return previous_state != current_state and ( + previous_state in failure_states or current_state in failure_states + ) + + def _notify_conversation_persistence_state_changed(self) -> None: + """Best-effort operator projection refresh, always outside journal locks.""" + callback = concrete_method(self.ui, "on_persistence_state_changed") + if callback is None: + return + try: + callback() + except Exception: + log.debug("ui.on_persistence_state_changed raised", exc_info=True) + + def _prepare_direct_conversation_mutation( + self, + deferred_persistence: list[Callable[[], None]] | None, + ) -> None: + """Fence an unbatched row mutation behind the accepted FIFO prefix. + + Generation commits already serialize their durable closures through + the ticket lane. Direct command/test helpers have no such batch, so + they must reconcile an older pending row *before* changing in-memory + history. Otherwise a successful direct save could persist a causal + suffix after an unresolved accepted predecessor. + """ + if deferred_persistence is None: + try: + self._reconcile_pending_conversation_commits() + except GenerationCancelled as exc: + raise RuntimeError("Cannot mutate a closed session") from exc + + def _journal_conversation_row_locked( + self, + *, + commit_key: str, + message: dict[str, Any], + persist: Callable[[], int], + event_id: int | None, + ) -> _PendingConversationCommit: + """Admit one immutable row while the history handoff lock is held.""" + pending_message = copy.deepcopy(message) + pending_message["_commit_key"] = commit_key + pending_message["_event_id"] = event_id + pending_message["_pending_durability"] = True + pending = _PendingConversationCommit( + commit_key=commit_key, + message=pending_message, + persist=persist, + ) + self._pending_conversation_commits[commit_key] = pending + self._history_handoff_revision += 1 + return pending + + def _rollback_live_journal_admission_locked( + self, + *, + turn: Turn, + commit_key: str, + history_revision_before: int, + reason: str, + ) -> None: + """Undo a live-tail append whose journal admission raised. + + Shared by the USER, SYSTEM, and TOOL admission sites so a raise inside + ``_journal_conversation_row_locked`` can never leave a live turn that + no journal entry or durable row will ever represent. + """ + if not self.messages or self.messages[-1] is not turn: + raise RuntimeError("journal rollback lost live-tail ownership") + self.messages.pop() + self._msg_tokens.pop() + self._pending_conversation_commits.pop(commit_key, None) + self._history_handoff_revision = history_revision_before + self._request_history_resync_locked(reason) + + def _schedule_conversation_persistence( + self, + pending: _PendingConversationCommit, + deferred_persistence: list[Callable[[], None]] | None, + ) -> None: + """Put a journal entry on its generation batch or persist directly.""" + if deferred_persistence is not None: + deferred_persistence.append(lambda: self._persist_pending_conversation_commit(pending)) + return + self._persist_pending_conversation_commit(pending) + + def _request_history_resync_locked(self, reason: str) -> int | None: + """Best-effort repair event for an accepted row handoff. + + Current UIs return the exact enqueued event id, which callers persist + directly. A custom legacy hook returning ``None`` retains the previous + immediate high-water sample as a best-effort compatibility fallback. + """ + request_resync = concrete_method(self.ui, "on_history_resync") + if request_resync is None: + return None + try: + event_id = _coerce_event_id(request_resync(reason)) + except Exception: + log.debug("ui.on_history_resync raised", exc_info=True) + return None + return event_id if event_id is not None else self._ui_event_id() + + def _publish_user_turn_locked( + self, + *, + content: str, + attachments: list[dict[str, Any]], + sender: str | None, + source: str | None, + client_send_ids: list[str], + ) -> int | None: + """Publish one accepted user row, falling back to a repair refetch. + + The caller holds ``_history_handoff_lock`` across the in-memory append, + journal admission, and this publication. A listener registered before + that boundary therefore receives the complete ``user_turn`` event; + one attempting to register afterwards must present the new history + revision. Older/custom UIs without the hook retain the exceptional + ``history_resync`` repair path instead of silently missing the row. + """ + publish = concrete_method(self.ui, "on_user_turn") + if publish is not None: + try: + event_id = _coerce_event_id( + publish( + content, + attachments=attachments, + sender=sender, + source=source, + client_send_ids=client_send_ids, + ) + ) + if event_id is not None: + return event_id + log.debug("ui.on_user_turn returned no event id; using repair fallback") + except Exception: + log.debug("ui.on_user_turn raised", exc_info=True) + return self._request_history_resync_locked("user_turn_accepted") + + def _publish_tool_turn_locked( + self, + *, + call_id: str, + name: str, + output: str, + is_error: bool, + preview: dict[str, Any] | None, + effect_status: str | None, + projection_safe: bool, + ) -> int | None: + """Publish one accepted TOOL row or request exceptional repair. + + The executor's earlier ``on_tool_result`` is deliberately provisional: + output guarding and context truncation can still change the text that + enters history. A capable UI receives this final scalar projection and + upserts it by provider call id. Duplicate or blank ids within one + accepted assistant batch are not an unambiguous browser key, so they + retain the strong history-repair path. + + The caller holds the history handoff lock across append, publication, + and journal admission. A listener therefore sees either the canonical + event or a newer REST revision, never an unrepresented accepted row. + """ + if projection_safe: + publish = concrete_method(self.ui, "on_tool_turn_accepted") + if publish is not None: + try: + event_id = _coerce_event_id( + publish( + call_id, + name, + output, + is_error=is_error, + preview=preview, + effect_status=effect_status, + ) + ) + if event_id is not None: + return event_id + log.debug( + "ui.on_tool_turn_accepted returned no event id; using repair fallback" + ) + except Exception: + log.debug("ui.on_tool_turn_accepted raised", exc_info=True) + repair_reason = "tool_turn_accepted" + else: + repair_reason = "tool_turn_projection_ambiguous" + return self._request_history_resync_locked(repair_reason) + + def _persist_pending_conversation_commit( + self, + entry: _PendingConversationCommit, + *, + _retry_admitted: bool = False, + ) -> None: + """Attempt one idempotent save and either ACK or classify the head.""" + error: ConversationPersistenceError | None = None + error_cause: Exception | None = None + notify_state_change = False + request_resync = False + resync_reason = "conversation_persistence_unresolved" + with self._history_visibility_lock: + with self._history_handoff_lock: + if entry.commit_key not in self._pending_conversation_commits: + return + if ( + self._conversation_persistence_failure_kind == "conflict" + and self._conversation_persistence_error is not None + ): + raise self._conversation_persistence_error + if ( + not _retry_admitted + and self._conversation_persistence_failure_kind == "retrying" + and self._conversation_persistence_error is not None + ): + raise self._conversation_persistence_error + previous_state = self._conversation_persistence_state_locked() + + row_id = 0 + conflict: ConversationCommitConflictError | None = None + gone: ConversationCommitWorkstreamGoneError | None = None + try: + row_id = int(entry.persist() or 0) + except ConversationCommitConflictError as exc: + conflict = exc + error_cause = exc + except ConversationCommitWorkstreamGoneError as exc: + gone = exc + error_cause = exc + except Exception as exc: + error_cause = exc + + with self._history_handoff_lock: + if row_id > 0: + self._pending_conversation_commits.pop(entry.commit_key, None) + if not self._pending_conversation_commits: + self._clear_conversation_persistence_failure_locked() + notify_state_change = self._conversation_persistence_state_needs_notification( + previous_state, + self._conversation_persistence_state_locked(), + ) + elif gone is not None: + # The durable parent row was hard-deleted after these rows + # were accepted; keyed saves never recreate it, so no retry + # can succeed. The delete is the newer authoritative ruling + # on this workstream: discard the journal so close and + # eviction stop waiting on rows that can never land, and + # return NORMALLY — the deletion, not this persist, is the + # user-facing event, and raising here would only re-latch a + # phantom error over an empty journal. The revision bump + # invalidates every token minted over the discarded rows. + # The latch makes the discovery terminal: new conversation + # admissions refuse (finalizer lanes still converge) and + # capture_history_handoff stops minting tokens, so panes + # keep their stale-but-real transcript instead of a + # silently wiped one. Re-entry with the latch already set + # (further in-flight saves of the same generation) is + # idempotent. The forensic log carries commit keys and + # roles only — never message content. + discarded = [ + (key, str(entry.message.get("role", ""))) + for key, entry in self._pending_conversation_commits.items() + ] + self._pending_conversation_commits.clear() + self._clear_conversation_persistence_failure_locked() + self._history_handoff_revision += 1 + self._workstream_gone_ws = self._ws_id + request_resync = True + resync_reason = "workstream_gone" + log.error( + "session.conversation_rows_discarded_workstream_gone ws=%s rows=%d discarded=%s", + self._ws_id[:8], + len(discarded), + discarded, + ) + notify_state_change = self._conversation_persistence_state_needs_notification( + previous_state, + self._conversation_persistence_state_locked(), + ) + else: + failed_at = datetime.now(UTC) + self._conversation_persistence_attempts += 1 + if self._conversation_persistence_first_failure_at is None: + self._conversation_persistence_first_failure_at = failed_at + self._conversation_persistence_last_failure_at = failed_at + if conflict is not None: + current = self._pending_conversation_commits.get(entry.commit_key) + if current is not None and current.ack_from_durable_row: + self._pending_conversation_commits[entry.commit_key] = ( + dataclasses.replace(current, ack_from_durable_row=False) + ) + self._conversation_persistence_failure_kind = "conflict" + self._conversation_persistence_next_retry_at = None + self._conversation_persistence_next_retry_wall_at = None + error = ConversationPersistenceError( + "Accepted conversation row has an immutable durable commit conflict" + ) + else: + self._conversation_persistence_failure_kind = "retrying" + delay = self._conversation_persistence_retry_delay( + self._conversation_persistence_attempts + ) + self._conversation_persistence_next_retry_at = time.monotonic() + delay + self._conversation_persistence_next_retry_wall_at = failed_at + timedelta( + seconds=delay + ) + error = ConversationPersistenceError( + "Accepted conversation row is awaiting durable storage reconciliation" + ) + if error_cause is not None: + error.__cause__ = error_cause + self._conversation_persistence_error = error + request_resync = ( + self._conversation_persistence_resync_commit_key != entry.commit_key + ) + if request_resync: + self._conversation_persistence_resync_commit_key = entry.commit_key + notify_state_change = self._conversation_persistence_state_needs_notification( + previous_state, + self._conversation_persistence_state_locked(), + ) + if notify_state_change: + self._notify_conversation_persistence_state_changed() + # A listener can atomically register immediately before a tool-only + # assistant is accepted. Its inflight snapshot is empty, and failure + # prevents the later tool-info event that would reconstruct the row. + # Force those already-connected panes through the authoritative + # history overlay. Keep this outside both history locks: enqueue takes + # UI locks and browser callbacks may immediately start a new request. + # The repair fires for the non-raising workstream-gone discard too — + # its panes must leave the discarded rows for the (deleted) + # authoritative history. + if request_resync: + request_resync_hook = concrete_method(self.ui, "on_history_resync") + if request_resync_hook is not None: + try: + request_resync_hook(resync_reason) + except Exception: + log.debug("ui.on_history_resync raised", exc_info=True) + if error is not None: + if error_cause is not None: + raise error from error_cause + raise error + return + + def _reconcile_pending_conversation_commits( + self, + *, + _terminal_admitted_repair: bool = False, + _force_retry: bool = False, + ) -> None: + """Retry every unresolved row in total acceptance order. + + Ordinary callers enter the history-visibility lane and then recheck + publication admission. This is the un-ticketed repair equivalent of + durability admission: if hard-delete's terminal barrier wins first, + no repair write can start after it; if reconciliation wins first, the + barrier waits for its acknowledgement before deletion proceeds. + + ``_terminal_admitted_repair`` is deliberately private and narrow. A + soft close must repair the accepted prefix after installing its latch + so it can decide whether unloading is safe; an already-admitted + durability ticket may likewise finish its ordered predecessor repair + while hard-delete waits for that ticket. No unadmitted caller may + write conversation storage after publication shutdown. + """ + + with self._history_visibility_lock: + with self._generation_lock: + if self._publication_shutdown and not _terminal_admitted_repair: + raise GenerationCancelled() + with self._history_handoff_lock: + persistence_error = self._conversation_persistence_error + failure_kind = self._conversation_persistence_failure_kind + next_retry_at = self._conversation_persistence_next_retry_at + if failure_kind == "conflict" and persistence_error is not None: + raise persistence_error + if ( + not _force_retry + and failure_kind == "retrying" + and next_retry_at is not None + and time.monotonic() < next_retry_at + and persistence_error is not None + ): + raise persistence_error + pending = list(self._pending_conversation_commits.values()) + for entry in pending: + self._persist_pending_conversation_commit(entry, _retry_admitted=True) + + def reconcile_unresolved_persistence_if_due(self, *, now: float | None = None) -> bool: + """Run one due transient repair pass for manager-owned maintenance. + + Returns whether a due pass was admitted. Expected persistence failures + remain represented by the journal state and are not allowed to tear + down the shared maintenance thread. + """ + check_at = time.monotonic() if now is None else now + with self._history_visibility_lock: + with self._generation_lock: + if self._publication_shutdown: + return False + with self._history_handoff_lock: + failure_kind = self._conversation_persistence_failure_kind + next_retry_at = self._conversation_persistence_next_retry_at + if not self._pending_conversation_commits or failure_kind == "conflict": + return False + if failure_kind == "retrying" and ( + next_retry_at is None or check_at < next_retry_at + ): + return False + try: + self._reconcile_pending_conversation_commits(_force_retry=True) + except GenerationCancelled: + return False + except ConversationPersistenceError: + return True + return True + + def _await_prior_conversation_persistence(self, my_generation: int) -> None: + """Fence a new user turn behind all prior accepted durability batches.""" + with self._durability_cond: + high_water = self._durability_next_ticket + already_drained = self._durability_serving_ticket >= high_water + with self._history_handoff_lock: + needs_reconciliation = bool( + self._pending_conversation_commits or self._conversation_persistence_error + ) + # Preserve the historical first-send sequencing (including lightweight + # test/CLI sessions whose cancellation probe begins after user append). + # The fence is needed only when a predecessor ticket or unresolved row + # actually exists. + if already_drained and not needs_reconciliation: + return + while True: + with self._durability_cond: + if self._durability_serving_ticket >= high_water: + break + self._durability_cond.wait(timeout=0.05) + self._check_cancelled(my_generation) + self._check_cancelled(my_generation) + self._reconcile_pending_conversation_commits() + self._check_cancelled(my_generation) + + def prepare_soft_close(self) -> bool: + """Cancel, close structural debt, then reconcile the durable prefix. + + Preparing is a short admission latch distinct from terminal shutdown: + it blocks fresh generations and unscoped mutations while the accepted + worker retains ownership long enough to journal any owed TOOL receipts. + A wedged worker gets a bounded grace period; refusal keeps the live + session loaded for an operator force-cancel instead of unloading an + invalid USER/ASSISTANT prefix. + """ + with self._generation_transition_lock, self._generation_lock: + if self._publication_shutdown: + return not self.has_unresolved_conversation_persistence() + if self._soft_close_preparing: + return False + self._soft_close_preparing = True + + # Use the complete cooperative cancellation path so provider streams, + # tool subprocesses, judges, and task-agent scopes all receive the + # close edge. Fresh claims remain fenced by the preparing latch while + # cancel() briefly reacquires the transition lock. + try: + self.cancel() + except BaseException: + with self._tool_structural_condition: + self._soft_close_preparing = False + self._tool_structural_condition.notify_all() + raise + + deadline = time.monotonic() + _SOFT_CLOSE_STRUCTURAL_WAIT_SECONDS + with self._tool_structural_condition: + while self._tool_structural_debt is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + self._soft_close_preparing = False + self._tool_structural_condition.notify_all() + return False + self._tool_structural_condition.wait(timeout=remaining) + + self._publication_shutdown = True + self._soft_close_preparing = False + # Invalidate worker slots captured before this close attempt even + # when durability reconciliation refuses the close and the terminal + # latch is rolled back. Fresh slots admitted after rollback capture + # this newer edge and remain usable. + durability_high_water = self._durability_next_ticket + + with self._durability_cond: + self._durability_cond.wait_for( + lambda: self._durability_serving_ticket >= durability_high_water + ) + + try: + self._reconcile_pending_conversation_commits( + _terminal_admitted_repair=True, + _force_retry=True, + ) + except ConversationPersistenceError: + durable_prefix_settled = False + else: + durable_prefix_settled = not self.has_unresolved_conversation_persistence() + if durable_prefix_settled: + return True + with self._generation_lock: + self._generation += 1 + self._cancel_event = threading.Event() + self._publication_shutdown = False + self._soft_close_preparing = False + return False + + def force_abandon_generation( + self, + *, + target_is_current: Callable[[], bool], + clear_target: Callable[[], bool], + publish_abandoned: Callable[[], None], + ) -> tuple[bool, ConversationPersistenceError | None]: + """Supersede one pinned worker after closing its accepted tool block. + + The caller supplies two tiny workstream-slot callbacks so this method + can preserve the global lock order (generation -> workstream) without + importing manager state. While the generation lock is held, + ``session_worker`` cannot capture a successor claim: an exact target + check therefore stays authoritative through supersession, UNKNOWN + receipt journaling, and the owner-conditional slot clear. TOOL storage + closures enter the ordinary FIFO ticket and remain poison-gated behind + any unresolved predecessor row. The terminal UI callback then runs + after that FIFO settles but before the transition lock releases, so a + successor cannot publish thinking ahead of the predecessor's + stream-end/idle edge. + """ + abandoned = False + persistence_error: ConversationPersistenceError | None = None + + def _finalize_force_abandon(durable: list[Callable[[], None]]) -> None: + nonlocal abandoned + if not target_is_current(): + return + debt = self._tool_structural_debt + self._cancel_event.set() + self._generation += 1 + self._cancel_event = threading.Event() + if debt is not None: + try: + self._synthesize_cancelled_results( + "Force-cancelled before the tool outcome could be observed.", + deferred_persistence=durable, + structural_generation=debt.generation, + ) + except BaseException: + # Force installed a fresh generation event above. If its + # own structural finalizer fails, no send-finally hook will + # poison that new event for us; keep dispatcher admission + # closed until a later force retry repairs the same debt. + self._poison_tool_structural_debt_locked(debt.generation) + raise + try: + self._drain_pending_advisories() + except Exception: + # Advisory cleanup is best-effort and historically could not + # defeat the force escape hatch. Structural completion above + # remains fail-closed; only this ancillary latch is demoted. + log.debug( + "session.force_abandon.advisory_cleanup_failed ws=%s", + self._ws_id[:8], + exc_info=True, + ) + abandoned = clear_target() + + with self._generation_transition_lock: + # Destructive history/lifecycle workers advertise a non-abandonable + # slot. Refuse before entering the truncation/durability admission + # path so force remains a prompt cooperative cancel instead of + # waiting on the very transaction whose ownership must be kept. + if not target_is_current(): + return False, None + try: + admitted = self._commit_for_generation( + 0, + _finalize_force_abandon, + allow_cancelled=True, + allow_soft_close_preparing=True, + allow_workstream_gone=True, + allow_tool_structural_debt=True, + ) + except ConversationPersistenceError as exc: + # The structural rows were journaled and the exact slot was + # already cleared before the FIFO batch observed its poisoned + # predecessor. Preserve the typed result for the route/log and + # let the normal retry/conflict state own later reconciliation. + persistence_error = exc + else: + if not admitted: + return False, None + if abandoned: + try: + publish_abandoned() + except Exception: + # Structural and slot ownership are already settled. The + # terminal event remains best-effort, but attempting it on + # this side of generation admission preserves ordering. + log.debug( + "session.force_abandon.terminal_publish_failed ws=%s", + self._ws_id[:8], + exc_info=True, + ) + return abandoned, persistence_error + def _commit_for_generation( self, origin_generation: int, commit: Callable[[list[Callable[[], None]]], None], *, allow_cancelled: bool = True, + allow_persistence_poison: bool = False, + allow_soft_close_preparing: bool = False, + allow_tool_structural_debt: bool = False, + allow_workstream_gone: bool = False, ) -> bool: """Commit live state atomically, then run its durable batch in FIFO order. @@ -7959,19 +9470,65 @@ class ChatSession: The caller retains the historical synchronous durability contract: it returns only after its batch finishes (or raises), but Stop/close/claim remain free to acquire the lifecycle locks while storage is blocked. + ``allow_soft_close_preparing`` and ``allow_tool_structural_debt`` are + reserved for force-abandon's exact structural finalizer. Ordinary + origin-zero mutations remain fenced by both latches. + + ``allow_workstream_gone`` marks the convergence lanes (the + cancel/interrupt/failure finalizers and force-abandon) that must still + run after a keyed save observed the durable parent hard-deleted; every + other admission refuses so a deleted workstream cannot keep accepting + turns whose rows silently discard. """ durable: list[Callable[[], None]] = [] ticket: int | None = None - with self._generation_lock: + commit_error: BaseException | None = None + # This method and the terminal drain below are the two ChatSession + # entry points BORROWED UNBOUND by resource shells that own only the + # durability lane and none of the admission fields, so their lifecycle + # reads stay defensive where the rest of the file reads directly. + truncation_condition = getattr(self, "_history_truncation_condition", None) + commit_guard = truncation_condition or self._generation_lock + with commit_guard: + if truncation_condition is not None: + truncation_condition.wait_for(lambda: not self._history_truncation_active) if ( self._publication_shutdown + or ( + not origin_generation + and getattr(self, "_soft_close_preparing", False) + and not allow_soft_close_preparing + ) + or ( + not origin_generation + and getattr(self, "_tool_structural_debt", None) is not None + and not allow_tool_structural_debt + ) or (origin_generation and self._generation != origin_generation) or (not allow_cancelled and self._cancel_event.is_set()) + or ( + not allow_workstream_gone + and getattr(self, "_workstream_gone_ws", None) is not None + and self._workstream_gone_ws == self._ws_id + ) ): return False owner_token = _active_commit_origin_generation.set(origin_generation) try: commit(durable) + except BaseException as exc: + # Journal admission can precede a later callback failure. Its + # already-collected persistence closures still own a place in + # the FIFO; dropping them here leaves a pending-only overlay + # with no due retry and can expose an invalid durable prefix. + commit_error = exc + if getattr(self, "_tool_structural_debt", None) is not None: + # Poison dispatcher admission before releasing the same + # generation lock that protects structural debt. Relying + # on send()'s later _consume_cancel finally leaves a race + # where another request captures a nominal worker claim + # against an invalid ASSISTANT-without-TOOL prefix. + self._poison_tool_structural_debt_locked() finally: _active_commit_origin_generation.reset(owner_token) if durable: @@ -7979,21 +9536,49 @@ class ChatSession: self._durability_next_ticket += 1 if ticket is None: + if commit_error is not None: + raise commit_error return True with self._durability_cond: self._durability_cond.wait_for(lambda: self._durability_serving_ticket == ticket) + durability_error: BaseException | None = None try: + history_lock = getattr(self, "_history_handoff_lock", None) + if history_lock is None: + poison = None + else: + with history_lock: + poison = self._conversation_persistence_error + if poison is not None and not allow_persistence_poison: + # Tickets admitted behind an unresolved conversation boundary + # must settle without executing. Waiting for a future repair + # would pin their owner threads indefinitely; executing would + # persist a causal suffix after a missing assistant row. + raise poison for persist in durable: persist() + except BaseException as exc: + # ConversationPersistenceError needs no special arm: the retrying + # and conflict classifiers latch ``_conversation_persistence_error`` + # before raising, and the workstream-gone discard returns without + # raising — re-latching here could only recreate a phantom error + # over an empty journal. + durability_error = exc finally: with self._durability_cond: self._durability_serving_ticket += 1 self._durability_cond.notify_all() + if durability_error is not None: + if commit_error is not None: + raise durability_error from commit_error + raise durability_error + if commit_error is not None: + raise commit_error return True def shutdown_publication_and_drain_durability(self) -> None: - """Close durable admission and wait for every accepted batch. + """Close admission, cancel resources, and drain admitted batches. Hard deletion needs a stronger boundary than cooperative worker cancellation: a generation commit may already have published its @@ -8009,15 +9594,51 @@ class ChatSession: after the lifecycle outcome is known. """ with self._generation_lock: + structural_debt = getattr(self, "_tool_structural_debt", None) + # Install the terminal latch before cooperative cancellation. + # A worker that observes the cancel edge must not synthesize a + # new ws-id-only TOOL storage closure: the authorized durable + # incarnation may already have been replaced on another node. + # Successful exact deletion erases the debt; an ambiguous + # same/unknown outcome retains it in the manager tombstone. self._publication_shutdown = True self._cancel_event.set() + self._approval_cancel_epoch = getattr(self, "_approval_cancel_epoch", 0) + 1 + self._soft_close_preparing = False + structural_condition = getattr(self, "_tool_structural_condition", None) + if structural_condition is not None: + structural_condition.notify_all() durability_high_water = self._durability_next_ticket + terminal_error: BaseException | None = None + if structural_debt is not None: + try: + # Abort provider streams, task scopes, judges, and subprocesses + # after the latch. Their owner can unwind but cannot publish a + # fresh TOOL suffix across a remote same-id replacement. + self.cancel() + except BaseException as exc: + # The terminal admission edge above remains authoritative. + terminal_error = exc + with self._durability_cond: self._durability_cond.wait_for( lambda: self._durability_serving_ticket >= durability_high_water ) + # Un-ticketed lost-ACK reconciliation uses the visibility lane. Take + # it once after the terminal latch and ticket drain: a repair that won + # before the latch completes first, while a waiter that lost the latch + # rechecks publication shutdown and exits without writing. The caller + # may now delete storage without a late reconciliation resurrecting a + # conversation row. + history_visibility_lock = getattr(self, "_history_visibility_lock", None) + if history_visibility_lock is not None: + with history_visibility_lock: + pass + if terminal_error is not None: + raise terminal_error + def _consume_cancel(self, my_generation: int) -> bool: """Clear this generation's cancel signal on exit; report if one landed. @@ -8031,6 +9652,13 @@ class ChatSession: with self._generation_transition_lock, self._generation_lock: if self._generation != my_generation: return False + if self._poison_tool_structural_debt_locked(my_generation): + # An abnormal structural finalizer can itself fail (for + # example, journal admission rejects its TOOL receipt). Keep + # this generation poisoned at the dispatcher boundary: a new + # worker must not report successful admission only to fail its + # later generation claim behind the still-incomplete prefix. + return True landed = self._cancel_event.is_set() if not self._publication_shutdown: self._cancel_event.clear() @@ -8061,6 +9689,33 @@ class ChatSession: raise GenerationCancelled() from None self._check_cancelled(my_generation) + def _direct_commit_reentry( + self, + closed_error: str, + reenter: Callable[[list[Callable[[], None]]], Any], + ) -> Any: + """THE direct-mutation admission frame, expressed once. + + Wraps a re-entrant call in the full direct-commit discipline: + ``_prepare_direct_conversation_mutation`` (the reconcile poison + gate), admission through ``_commit_for_generation(0, ...)``, and + the closed-session refusal (``closed_error`` verbatim). Every + direct path (``deferred_persistence is None``) takes this frame, + so an admission precondition added here reaches ALL direct + mutations at once instead of three of four hand-synced copies — + a path skipping a tightened gate would mutate conversation state + outside the discipline. Returns the re-entrant call's value. + """ + self._prepare_direct_conversation_mutation(None) + result: list[Any] = [] + + def _admit_direct(durable: list[Callable[[], None]]) -> None: + result.append(reenter(durable)) + + if not self._commit_for_generation(0, _admit_direct): + raise RuntimeError(closed_error) + return result[0] if result else None + def _append_user_turn( self, user_input: str, @@ -8070,7 +9725,9 @@ class ChatSession: from_wake: bool = False, source: str | None = None, deferred_persistence: list[Callable[[], None]] | None = None, - ) -> int: + sender_user_id: str | None = None, + client_send_ids: tuple[str, ...] = (), + ) -> None: """Append a user turn (plain or multipart) and persist it. When ``attachments`` is non-empty the in-memory message carries @@ -8079,17 +9736,31 @@ class ChatSession: bytes are written content-addressed + reference-counted into ``workstream_attachments`` (``attachment_id`` = the content hash) and the ordered id-list is recorded on the row's ``attachments`` ref-list - column — the sole message->blob link. Returns the saved conversations - row id (0 on save failure, per the storage wrapper's no-raise - contract). + column — the sole message->blob link. An unresolved save raises + :class:`ConversationPersistenceError` while the exact row remains in + the live handoff journal. - ``send_id`` (when provided) is the end-to-end send token; it no longer - gates a DB reservation (the upload buffer is the pending store — the - bytes were already drained from it before this call). + ``send_id`` identifies a web-staged attachment send. Its scoped buffer + references transfer atomically at journal admission, after every + pre-admission rejection boundary and before any later send can select + them again. Direct/internal callers without ``send_id`` may supply + immutable unstaged attachments. """ - # New user content invalidates the per-turn memory-search cache - # (composition will see a different recent-context string). - self._invalidate_memory_cache() + if deferred_persistence is None: + self._direct_commit_reentry( + "Cannot append a user turn to a closed session", + lambda durable: self._append_user_turn( + user_input, + attachments, + send_id=send_id, + from_wake=from_wake, + source=source, + deferred_persistence=durable, + sender_user_id=sender_user_id, + client_send_ids=client_send_ids, + ), + ) + return user_content: str | list[dict[str, Any]] if attachments: # Attachments ride by reference (``{type: kind, attachment_id}`` → @@ -8117,6 +9788,10 @@ class ChatSession: else: user_content = user_input + # Every live user row receives the same idempotent storage identity as + # an accepted assistant row. It is independent of ``send_id`` (a UI + # tracking token) and survives ambiguous storage acknowledgement. + commit_key = uuid.uuid4().hex user_msg: dict[str, Any] = {"role": "user", "content": user_content} if from_wake and self._wake_source_tag: # Sibling tag for audit / replay: the synthetic empty user @@ -8143,9 +9818,17 @@ class ChatSession: # / ``from_wake`` and stay unstamped so they never get a speaker label. # Rides the wire-invisible ``_sender`` side channel and, below, the # persisted ``meta`` column so history replay re-attributes correctly. - sender = "" if (from_wake or source) else (self._mcp_effective_user_id or "").strip() + effective_sender = ( + (self._mcp_effective_user_id or "") if sender_user_id is None else sender_user_id + ) + sender = "" if (from_wake or source) else effective_sender.strip() if sender: user_msg["_sender"] = sender + stable_client_send_ids = tuple( + client_send_id for client_send_id in client_send_ids if client_send_id + ) + if stable_client_send_ids: + user_msg["_client_send_ids"] = list(stable_client_send_ids) if attachments: # Sibling metadata so live history replay has the same shape # as reloaded-from-DB (filenames are not recoverable from an @@ -8153,6 +9836,7 @@ class ChatSession: # underscore keys before the wire call so this is safe. user_msg["_attachments_meta"] = [ { + "attachment_id": a.attachment_id, "kind": a.kind, "filename": a.filename, "mime_type": a.mime_type, @@ -8162,18 +9846,17 @@ class ChatSession: } for a in attachments ] - self.messages.append(turn_from_dict(user_msg)) - if sender: - # A newly recorded sender can change shared-workstream state; let - # the next system-prompt compose re-derive it (memoized otherwise). - self._invalidate_shared_state() - self._msg_tokens.append(max(1, int(self._msg_char_count(user_msg) / self._chars_per_token))) + user_turn = turn_from_dict(user_msg) + user_token_estimate = max( + 1, + int(self._msg_char_count(user_msg) / self._chars_per_token), + ) # DB row stores the raw text only; attachment bytes are written # content-addressed into workstream_attachments and the ordered id-list # is recorded on this row's ``attachments`` ref-list column (the sole - # message->blob link), joined back in on load. ``send_id`` no longer - # gates a reservation — the bytes were drained from the upload buffer - # before this call. + # message->blob link), joined back in on load. A web-staged send moves + # the matching upload-buffer ownership into this immutable closure at + # admission; direct/internal calls may supply unstaged bytes. # # The wake's synthesised empty turn carries ``_source`` onto the # row so reconnecting tabs render the marker instead of an @@ -8185,78 +9868,123 @@ class ChatSession: # column already carries opaque per-row metadata). ``reconstruct_turns`` # restores it to ``Turn.meta.extra["sender"]`` so a worker rehydrating a # shared workstream re-attributes each user turn. - meta_json = json.dumps({"sender": sender}) if sender else None + meta_envelope: dict[str, Any] = {} + if sender: + meta_envelope["sender"] = sender + if stable_client_send_ids: + meta_envelope["client_send_ids"] = list(stable_client_send_ids) + meta_json = json.dumps(meta_envelope) if meta_envelope else None persist_ws_id = self._ws_id persist_user_id = self._user_id - persist_event_id = self._ui_event_id() - persist_attachments = tuple(attachments) + attachment_writes = _attachment_writes(attachments) + persist_event_id: int | None = None def _persist_user_turn() -> int: - message_id = save_message( - persist_ws_id, - "user", - user_input, - source=source if isinstance(source, str) and source else None, - event_id=persist_event_id, - meta=meta_json, - ) - if persist_attachments and message_id: - self._persist_attachment_refs( - message_id, - persist_attachments, - ws_id=persist_ws_id, + persist_source = source if isinstance(source, str) and source else None + if attachment_writes: + message_id = save_user_message_with_attachments( + persist_ws_id, + user_input, + attachment_writes, + source=persist_source, + event_id=persist_event_id, + meta=meta_json, + commit_key=commit_key, + ) + else: + message_id = save_message( + persist_ws_id, + "user", + user_input, + source=persist_source, + event_id=persist_event_id, + meta=meta_json, + commit_key=commit_key, ) - # Drain the now-committed handles from the per-node upload - # buffer. Capture both identities at admission: a force - # successor or resume may rebind the live session before this - # ordered durable closure runs. - buffer = get_attachment_buffer() - for att in persist_attachments: - buffer.discard( - att.attachment_id, - ws_id=persist_ws_id, - user_id=persist_user_id, - ) return message_id - if deferred_persistence is not None: + event_attachments = [ + { + "attachment_id": write.attachment_id, + "kind": write.kind, + "filename": write.filename, + "mime_type": write.mime_type, + } + for write in attachment_writes + ] - def _persist_user_turn_deferred() -> None: - _persist_user_turn() - - deferred_persistence.append(_persist_user_turn_deferred) - return 0 - return _persist_user_turn() - - def _persist_attachment_refs( - self, - message_id: int, - attachments: list[Attachment] | tuple[Attachment, ...], - *, - origin: str = "upload", - ws_id: str | None = None, - ) -> None: - """Write each attachment's bytes content-addressed and record the ref-list. - - ``attachment_id`` is the content hash, so identical bytes dedupe to one - blob and each reference bumps its refcount; the ordered id-list is - recorded on the conversations row's ``attachments`` column. Used by - the user-turn commit (``origin='upload'``) and the tool-image persist - (``origin='tool'``). - """ - ref_ids: list[str] = [] - for att in attachments: - save_attachment( - att.attachment_id, - att.filename, - att.mime_type, - len(att.content), - att.kind, - att.content, - origin, + # Admission, journal/revision advance, and event publication are one atomic + # REST-history -> SSE transition. A listener registered before this + # lock receives ``user_turn``; an old-token registration after it is + # rejected. Journal the immutable accepted row before making it + # externally visible, then stamp that same row with the event cursor. + with self._history_handoff_lock: + # The buffer transfer and all accepted-row mutations share this + # admission point. A failed claim leaves both staging and history + # untouched; a successful claim is backed by the immutable writes + # captured in ``_persist_user_turn``. + self._invalidate_memory_cache() + history_revision_before = self._history_handoff_revision + self.messages.append(user_turn) + if sender: + # A newly recorded sender can change shared-workstream state; + # let the next system-prompt compose re-derive it. + self._invalidate_shared_state() + self._msg_tokens.append(user_token_estimate) + try: + pending = self._journal_conversation_row_locked( + commit_key=commit_key, + message=user_msg, + persist=_persist_user_turn, + event_id=None, + ) + except BaseException: + self._rollback_live_journal_admission_locked( + turn=user_turn, + commit_key=commit_key, + history_revision_before=history_revision_before, + reason="user_turn_admission_rolled_back", + ) + raise + if send_id and attachment_writes: + # Drain staging only AFTER the journal claim succeeded: + # ``attachment_writes`` already carries the resolved bytes; the + # staged buffer is only the double-spend guard for a second + # send racing this one, and both sends serialize on this lock. + # A raise above therefore leaves the handles staged, so the + # client's retry of the same send resolves them again instead + # of being rejected as unknown/expired (round-3 review). + # Entries can legitimately be gone by now (TTL expiry while + # queued behind the durability fence), so a missing handle + # must not fail a turn whose payload is fully in hand — but + # every surviving sibling is still consumed so no staged owner + # outlives this admission. + unique_ids = tuple( + dict.fromkeys(write.attachment_id for write in attachment_writes) + ) + transferred = get_attachment_buffer().consume_all( + unique_ids, + ws_id=persist_ws_id, + user_id=persist_user_id, + ) + if len(transferred) < len(unique_ids): + log.debug( + "session.user_turn.staged_attachments_partially_released ws=%s missing=%d", + persist_ws_id[:8], + len(unique_ids) - len(transferred), + ) + persist_event_id = self._publish_user_turn_locked( + content=user_input, + attachments=event_attachments, + sender=sender or None, + source=source if isinstance(source, str) and source else None, + client_send_ids=list(stable_client_send_ids), ) - ref_ids.append(att.attachment_id) - set_message_attachments(ws_id or self._ws_id, message_id, ref_ids) + user_msg["_event_id"] = persist_event_id + user_turn.meta.event_id = persist_event_id + pending.message["_event_id"] = persist_event_id + + self._schedule_conversation_persistence(pending, deferred_persistence) @staticmethod def _decode_image_part(part: Any, tool_name: str) -> Attachment | None: @@ -8355,40 +10083,77 @@ class ChatSession: live mirror both rebuild the same per-kind bubble. It is stripped before the LLM wire (``_source_meta`` is a ``_``-prefixed key). """ - turn = make_system_turn(source, content, **meta) - self.messages.append(turn_from_dict(turn)) - self._msg_tokens.append(max(1, int(self._msg_char_count(turn) / self._chars_per_token))) - meta_json = json.dumps(meta) if meta else None - # Fire the live SSE hook BEFORE persisting so the row carries the SAME - # event_id its ``system_turn`` event carries. ``on_system_turn`` - # returns the id ``_enqueue`` assigned (``None`` for non-SSE UIs / test - # doubles). The hook stays best-effort: on failure the persist below - # still runs and the row falls back to the current cursor (no live - # event was delivered to double anyway). - emitted_event_id: int | None = None - try: - emitted_event_id = _coerce_event_id( - self.ui.on_system_turn(content, source, meta or None) + if deferred_persistence is None: + self._direct_commit_reentry( + "Cannot append a system turn to a closed session", + lambda durable: self._append_system_turn( + source, + content, + deferred_persistence=durable, + **meta, + ), ) - except Exception: - log.warning("ui.on_system_turn failed; system turn still appended", exc_info=True) + return + turn = make_system_turn(source, content, **meta) + system_turn = turn_from_dict(turn) + token_estimate = max(1, int(self._msg_char_count(turn) / self._chars_per_token)) + meta_json = json.dumps(meta) if meta else None persist_ws_id = self._ws_id - persist_event_id = emitted_event_id if emitted_event_id is not None else self._ui_event_id() + commit_key = uuid.uuid4().hex + persist_event_id: int | None = None - def _persist_system_turn() -> None: - save_message( + def _persist_system_turn() -> int: + return save_message( persist_ws_id, "system", content, source=source, event_id=persist_event_id, meta=meta_json, + commit_key=commit_key, ) - if deferred_persistence is None: - _persist_system_turn() - else: - deferred_persistence.append(_persist_system_turn) + # The semantic event and row admission are one handoff transition. A + # throwing hook falls back to an explicit history repair before the + # revision changes, so already-connected panes cannot miss the row. + with self._history_handoff_lock: + history_revision_before = self._history_handoff_revision + self.messages.append(system_turn) + self._msg_tokens.append(token_estimate) + hook_completed = False + emitted_event_id: int | None = None + try: + emitted_event_id = _coerce_event_id( + self.ui.on_system_turn(content, source, meta or None) + ) + hook_completed = True + except Exception: + log.warning( + "ui.on_system_turn failed; requesting authoritative history", + exc_info=True, + ) + if not hook_completed: + self._request_history_resync_locked("system_turn_accepted") + persist_event_id = ( + emitted_event_id if emitted_event_id is not None else self._ui_event_id() + ) + try: + pending = self._journal_conversation_row_locked( + commit_key=commit_key, + message=turn, + persist=_persist_system_turn, + event_id=persist_event_id, + ) + except BaseException: + self._rollback_live_journal_admission_locked( + turn=system_turn, + commit_key=commit_key, + history_revision_before=history_revision_before, + reason="system_turn_admission_rolled_back", + ) + raise + + self._schedule_conversation_persistence(pending, deferred_persistence) # -- Main generation loop ------------------------------------------------ @@ -8492,6 +10257,39 @@ class ChatSession: return self._acting_user_id or self._user_id or None def bind_acting_user(self, user_id: str) -> None: + """Bind directly, or defer a worker-thread bind until generation claim. + + HTTP/coordinator worker closures historically call this immediately + before :meth:`send`. A slot-time :class:`WorkerClaim` is authoritative + in that window: applying the bind here would let an abandoned worker + overwrite its successor before the old claim is rejected. ``send`` + performs the generation-owned bind after validating that claim. + + Direct CLI/tests have no active worker claim and retain the synchronous + behavior. + """ + + requested = user_id.strip() + claim = current_worker_claim(self) + if claim is not None: + if claim.principal_id and requested and requested != claim.principal_id: + raise RuntimeError("Worker principal does not match acting user") + return + with self._acting_user_bind_lock: + self._bind_acting_user_unfenced(requested) + + def _bind_acting_user_for_generation(self, user_id: str, my_generation: int) -> None: + """Install one actor projection while *my_generation* still owns it.""" + + with self._acting_user_bind_lock: + self._check_generation_admission(my_generation) + self._bind_acting_user_unfenced(user_id) + # A force successor can claim while the listener/catalog work above + # runs. It waits on this bind lock and will install the final actor; + # the stale worker must stop before any model/tool work of its own. + self._check_generation_admission(my_generation) + + def _bind_acting_user_unfenced(self, user_id: str) -> None: """Bind the authenticated initiator of the current turn. Called from the HTTP send path with the caller's authenticated @@ -8566,6 +10364,8 @@ class ChatSession: send_id: str | None, from_wake: bool, turn_principal_id: str, + client_send_ids: tuple[str, ...] = (), + acting_principal_id: str | None = None, wire_part_cache: dict[ tuple[str, tuple[bool, bool, bool]], dict[str, Any] | list[dict[str, Any]], @@ -8589,9 +10389,14 @@ class ChatSession: else 0 ) shared_state_plan = self._plan_shared_state() + acting_principal = ( + (self._mcp_effective_user_id or "").strip() + if acting_principal_id is None + else acting_principal_id.strip() + ) participant_name: str | None = None if not from_wake: - participant_id = (self._mcp_effective_user_id or "").strip() + participant_id = acting_principal owner_id = (self._mcp_user_id or "").strip() if participant_id and participant_id != owner_id: participant_name = self._resolve_display_name(participant_id) @@ -8622,10 +10427,12 @@ class ChatSession: send_id=send_id, from_wake=from_wake, deferred_persistence=durable, + sender_user_id=acting_principal, + client_send_ids=client_send_ids, ) if not from_wake: became_shared = self._maybe_note_new_participant( - self._mcp_effective_user_id, + acting_principal, deferred_persistence=durable, recompose_system=False, resolved_name=participant_name, @@ -8682,7 +10489,7 @@ class ChatSession: ): raise GenerationCancelled() if recompose_out and recompose_out[0]: - self._check_cancelled(my_generation) + self._check_generation_admission(my_generation) if not self._init_system_messages(origin_generation=my_generation): raise GenerationCancelled() @@ -8694,6 +10501,7 @@ class ChatSession: *, from_wake: bool = False, acting_user_id: str | None = None, + client_send_ids: tuple[str, ...] = (), ) -> None: """Send user input and handle the response loop (including tool calls). @@ -8705,56 +10513,98 @@ class ChatSession: column. ``send_id`` is an end-to-end tracking token only; it no longer gates a - DB reservation (the upload buffer is the pending store, and the bytes - in ``attachments`` were already drained/peeked from it by the caller). + DB reservation. The caller only peeks at the upload buffer; admission + of the immutable pending user row atomically transfers staged bytes. ``acting_user_id`` is the authenticated caller who initiated this turn (HTTP send / retry paths). It rebinds per-user MCP credential resolution to that user for this and subsequent turns — see :meth:`bind_acting_user`. ``None`` (internal callers: wake nudges, auto-resume, CLI) leaves the current binding untouched. + + ``client_send_ids`` are browser-generated correlation tokens for the + accepted user-row event. They are deliberately not idempotency keys: + two sends carrying the same token remain two distinct conversation + rows and are distinguished by their monotonic event ids. """ - if acting_user_id is not None: - self.bind_acting_user(acting_user_id) - turn_principal_id = (self._mcp_effective_user_id or "").strip() - # A dead binding diagnosed by the refresh does NOT fail fast here: - # the send proceeds so the fallback chain can carry the turn, and - # ``_format_backend_error`` surfaces the latched cause if it cannot. - self._refresh_model_from_registry() - # Token budget approval gate - if self._budget_exhausted: - budget_cancel_witness = _ApprovalCancelWitness(self) - approved, _ = self.ui.approve_tools( - [ - { - "func_name": "__budget_override__", - "preview": ( - f"Token budget ({self._token_budget:,}) exhausted. Approve to continue." - ), - "needs_approval": True, - # Synchronization-only state. SessionUIBase projects - # approval items through an explicit wire allowlist, so - # this witness never crosses SSE / persistence. - "_approval_cancel_witness": budget_cancel_witness, - } - ] - ) - if not approved: - # A Stop is not a budget-policy rejection. The approval cycle - # already emitted its cancelled resolution; return quietly and - # let the worker's normal convergence restore idle state. - if budget_cancel_witness.aborted: - return - self.ui.on_error("Token budget exhausted. Approval required to continue.") - return - self._budget_exhausted = False - self._budget_warned = False - my_generation = self._claim_generation() + worker_claim = current_worker_claim(self) + requested_principal = (acting_user_id or "").strip() + if worker_claim is not None and worker_claim.principal_id: + if requested_principal and requested_principal != worker_claim.principal_id: + raise RuntimeError("Worker principal does not match acting user") + turn_principal_id = worker_claim.principal_id + else: + turn_principal_id = requested_principal or (self._mcp_effective_user_id or "").strip() + # Worker-slot admission is the first lifecycle action. In particular, + # do not bind an actor, refresh a registry, or enter an approval gate + # before validating the slot-time cancellation edge: a force successor + # may already own this same ChatSession. + my_generation = self._claim_generation( + principal_id=turn_principal_id or None, + expected_cancel_epoch=(worker_claim.cancel_epoch if worker_claim is not None else None), + ) wire_part_cache: dict[ tuple[str, tuple[bool, bool, bool]], dict[str, Any] | list[dict[str, Any]], ] = {} try: + with self._generation_lock: + if self._generation != my_generation: + raise GenerationCancelled() + generation_cancel_event = self._cancel_event + if turn_principal_id: + self._bind_acting_user_for_generation(turn_principal_id, my_generation) + self._check_generation_admission(my_generation) + # A dead binding diagnosed by the refresh does NOT fail fast here: + # the send proceeds so the fallback chain can carry the turn, and + # ``_format_backend_error`` surfaces the latched cause if it cannot. + self._refresh_model_from_registry() + self._check_generation_admission(my_generation) + # Context checks and the user-row estimate below must use the primary + # binding's tokenizer even when the preceding accepted turn came from + # a fallback lane. + self._activate_token_calibration(self._primary_lane()) + self._check_generation_admission(my_generation) + # Token budget approval gate. The generation-local event is captured + # at claim time; consulting ``self._cancel_event`` here could pick up a + # force successor's fresh event and strand this stale approval. + if self._budget_exhausted: + budget_cancel_witness = _ApprovalCancelWitness( + self, + generation_cancel_event, + ) + approved, _ = self.ui.approve_tools( + [ + { + "func_name": "__budget_override__", + "preview": ( + f"Token budget ({self._token_budget:,}) exhausted. " + "Approve to continue." + ), + "needs_approval": True, + # Synchronization-only state. SessionUIBase projects + # approval items through an explicit wire allowlist, + # so this witness never crosses SSE / persistence. + "_approval_cancel_witness": budget_cancel_witness, + } + ] + ) + if not approved: + # A Stop is not a budget-policy rejection. The approval cycle + # already emitted its cancelled resolution; return quietly. + if budget_cancel_witness.aborted: + return + self.ui.on_error("Token budget exhausted. Approval required to continue.") + return + self._budget_exhausted = False + self._budget_warned = False + # A force successor may claim while its predecessor's accepted + # assistant row is still waiting on storage. Claim/cancel remains + # responsive, but the successor may not append its user row until + # that causal boundary is durably reconciled. Otherwise a failed + # assistant save can leave a later user/tool row behind a permanent + # hole in the transcript. + self._await_prior_conversation_persistence(my_generation) # Install every post-claim, pre-stream mutation only after this # cleanup bracket is live. The helper publishes the user turn, # nudges/context, principal, and send-owned attachment cache as one @@ -8766,6 +10616,8 @@ class ChatSession: send_id=send_id, from_wake=from_wake, turn_principal_id=turn_principal_id, + client_send_ids=client_send_ids, + acting_principal_id=turn_principal_id, wire_part_cache=wire_part_cache, ) # Bail an orphaned/superseded send BEFORE the pre-send compaction below @@ -8851,73 +10703,78 @@ class ChatSession: ): raise GenerationCancelled() try: - try: - result = self._stream_response(my_generation) - except Exception as ctx_err: - # Context overflow recovery: if the API rejects the - # request due to exceeding the context window, compact - # the conversation and retry once. - if not _is_ctx_overflow(ctx_err): - raise - log.warning( - "Context overflow detected (%s), compacting and retrying", - type(ctx_err).__name__, - ) - # The notice and spinner handoff are one generation - # publication. A force successor/close/Stop that wins - # after the overflow must not let this old worker paint - # over the live UI before compaction's own owner fence. - with self._generation_lock: - if ( - self._publication_shutdown - or self._generation != my_generation - or self._cancel_event.is_set() - ): - raise GenerationCancelled() from None - self.ui.on_info("\n[Context overflow — auto-compacting and retrying]") - # Stop thinking indicator before compact (which has - # its own start/stop) to avoid nested spinners. - self.ui.on_thinking_stop() - try: - # my_generation: without it a stale send that hits - # overflow here could compact-and-swap the LIVE - # generation's history after a force-cancel started - # a newer one — the same race every other compaction - # site already guards. - self._compact_messages(auto=True, my_generation=my_generation) - except Exception: - # RECOVERY-machinery failure: the overflow error - # is still the actionable one, and its wording - # already anticipates this case ("compaction - # could not reduce it enough"). - log.warning( - "Compact-and-retry failed, raising original error", - exc_info=True, - ) - raise ctx_err from None - if not self._publish_for_generation( - my_generation, - self.ui.on_thinking_start, - allow_cancelled=False, - ): - raise GenerationCancelled() from None + with self._recovered_serving_failures() as recovered_failures: try: result = self._stream_response(my_generation) - except Exception as retry_err: - if not _is_ctx_overflow(retry_err): - # A post-compaction failure that is NOT a - # recurring overflow surfaces as ITSELF - # (implicitly chained to ctx_err): masking - # a dead wire or bad credentials behind - # "Context window exceeded" sends the - # operator to shrink a conversation that - # already compacted fine. + except Exception as ctx_err: + recovered_failures.append(ctx_err) + # Context overflow recovery: if the API rejects the + # request due to exceeding the context window, compact + # the conversation and retry once. + if not _is_ctx_overflow(ctx_err): raise log.warning( - "Compact-and-retry re-overflowed, raising original error", - exc_info=True, + "Context overflow detected (%s), compacting and retrying", + type(ctx_err).__name__, ) - raise ctx_err from None + # The notice and spinner handoff are one generation + # publication. A force successor/close/Stop that wins + # after the overflow must not let this old worker paint + # over the live UI before compaction's own owner fence. + with self._generation_lock: + if ( + self._publication_shutdown + or self._generation != my_generation + or self._cancel_event.is_set() + ): + raise GenerationCancelled() from None + self.ui.on_info( + "\n[Context overflow — auto-compacting and retrying]" + ) + # Stop thinking indicator before compact (which has + # its own start/stop) to avoid nested spinners. + self.ui.on_thinking_stop() + try: + # my_generation: without it a stale send that hits + # overflow here could compact-and-swap the LIVE + # generation's history after a force-cancel started + # a newer one — the same race every other compaction + # site already guards. + self._compact_messages(auto=True, my_generation=my_generation) + except Exception: + # RECOVERY-machinery failure: the overflow error + # is still the actionable one, and its wording + # already anticipates this case ("compaction + # could not reduce it enough"). + log.warning( + "Compact-and-retry failed, raising original error", + exc_info=True, + ) + raise ctx_err from None + if not self._publish_for_generation( + my_generation, + self.ui.on_thinking_start, + allow_cancelled=False, + ): + raise GenerationCancelled() from None + try: + result = self._stream_response(my_generation) + except Exception as retry_err: + recovered_failures.append(retry_err) + if not _is_ctx_overflow(retry_err): + # A post-compaction failure that is NOT a + # recurring overflow surfaces as ITSELF + # (implicitly chained to ctx_err): masking + # a dead wire or bad credentials behind + # "Context window exceeded" sends the + # operator to shrink a conversation that + # already compacted fine. + raise + log.warning( + "Compact-and-retry re-overflowed, raising original error", + exc_info=True, + ) + raise ctx_err from None finally: # Only clear if this generation is still active — # an orphaned thread must not clobber a newer stream. @@ -8966,31 +10823,29 @@ class ChatSession: self._update_token_table( msgs=completed_result.wire_msgs, tool_def_chars=completed_result.tool_def_chars, + provenance=completed_result.provenance, ) # Report usage for every completed API call that this # generation still owns. self._print_status_line( - model=completed_result.serving_model or None, + model=( + completed_result.provenance.backend_model_id + or completed_result.serving_model + or None + ), deferred_persistence=durable, ) # The canonical Turn: minted tool ids, finalized native # lane, and the SERVING lane's producer, so a # fallback-served turn is not labeled with the primary's. - self.messages.append(completed_result.turn) - # Clear per-turn inflight buffers — the assistant message - # is now in history, so the next execution/stream window - # must not replay the same in-progress text. - self.ui.on_turn_committed() - self._msg_tokens.append( - self._assistant_pending_tokens - or max( - 1, - int( - self._msg_char_count(completed_result.turn) / self._chars_per_token - ), - ) + accepted_tool_call_ids = tuple( + call.id for call in completed_result.turn.tool_calls if call.id ) + assistant_token_estimate = self._assistant_pending_tokens or max( + 1, + int(self._msg_char_count(completed_result.turn) / self._chars_per_token), + ) # Save assistant message atomically (content + tool_calls # in one row). The persisted mirror and executed call list # are derived from the same completed result above. @@ -9000,20 +10855,125 @@ class ChatSession: or completed_tool_calls_json ): persist_ws_id = self._ws_id - persist_event_id = self._ui_event_id() + persist_producer = completed_result.producer or None + provenance = TurnProvenance.from_meta( + completed_result.turn.meta.extra.get(PROVENANCE_META_KEY) + ) + persist_meta = ( + json.dumps({PROVENANCE_META_KEY: provenance.to_meta()}) + if provenance is not None + else None + ) + commit_key = uuid.uuid4().hex + # The journal takes the one isolating copy and stamps + # it: its pending-durability mark is what keeps the + # history orphan trimmer from dropping a complete + # tool-call-only row whose storage write is still in + # flight (the ring does not carry its native structure + # until later tool-info events). + pending_message = turn_to_dict(completed_result.turn) - def _persist_assistant_turn() -> None: - save_message( - persist_ws_id, - "assistant", - completed_content, - provider_data=completed_provider_data, - tool_calls=completed_tool_calls_json, - event_id=persist_event_id, - producer=completed_result.producer or None, + # Journal admission, accepted-revision advance, and + # inflight-buffer clear are one bootstrap transition. + # An initial SSE registration sharing this lock either + # precedes the transition (and receives the live tail) + # or observes the new token/pending row. + with self._history_handoff_lock: + try: + self.ui.on_turn_committed() + except Exception: + log.warning( + "ui.on_turn_committed failed; requesting authoritative history", + exc_info=True, + ) + self._request_history_resync_locked("assistant_turn_accepted") + # ``on_turn_committed`` flushes a final pending + # content batch before clearing the inflight + # buffers. Stamp the row *after* that flush so its + # durable cursor covers every event representing + # this assistant turn. + persist_event_id = self._ui_event_id() + + def _save_assistant_turn() -> int: + return save_message( + persist_ws_id, + "assistant", + completed_content, + provider_data=completed_provider_data, + tool_calls=completed_tool_calls_json, + event_id=persist_event_id, + producer=persist_producer, + meta=persist_meta, + commit_key=commit_key, + ) + + # Structural acceptance is inseparable from journal + # admission. If the in-memory journal itself rejects + # the assistant row, roll back both the live turn and + # its TOOL debt before generic exception cleanup can + # manufacture a durable suffix without its parent. + history_revision_before = self._history_handoff_revision + self._admit_tool_structural_debt_locked( + my_generation, + accepted_tool_call_ids, ) + self.messages.append(completed_result.turn) + self._msg_tokens.append(assistant_token_estimate) + try: + pending = self._journal_conversation_row_locked( + commit_key=commit_key, + message=pending_message, + persist=_save_assistant_turn, + event_id=persist_event_id, + ) + except BaseException: + # The shared rollback undoes the live tail; the + # structural-debt admission above is the one + # assistant-specific extra it cannot know about. + self._rollback_live_journal_admission_locked( + turn=completed_result.turn, + commit_key=commit_key, + history_revision_before=history_revision_before, + reason="assistant_turn_admission_rolled_back", + ) + if accepted_tool_call_ids: + expected_debt = _ToolStructuralDebt( + generation=my_generation, + call_ids=accepted_tool_call_ids, + ) + if self._tool_structural_debt != expected_debt: + raise RuntimeError( + "Assistant journal rollback lost structural-debt " + "ownership" + ) from None + self._tool_structural_debt = None + self._tool_structural_condition.notify_all() + raise - durable.append(_persist_assistant_turn) + def _persist_assistant_turn() -> None: + self._persist_pending_conversation_commit(pending) + + durable.append(_persist_assistant_turn) + else: + # No durable assistant row exists for a structurally + # empty result, but the live inflight buffers still end + # at this accepted turn boundary. Record that decision + # on the turn itself: a later tail cut must not count + # this live-only row when translating removed turns + # into durable rows (a raw count would delete an older + # committed row in its place). + completed_result.turn.meta.extra["no_durable_row"] = True + with self._history_handoff_lock: + self._admit_tool_structural_debt_locked( + my_generation, + accepted_tool_call_ids, + ) + self.messages.append(completed_result.turn) + self._msg_tokens.append(assistant_token_estimate) + try: + self.ui.on_turn_committed() + except Exception: + log.warning("ui.on_turn_committed failed", exc_info=True) if not completed_tool_calls: compaction_stop_out.append(self._compaction_advised) self._compaction_advised = False @@ -9034,6 +10994,11 @@ class ChatSession: allow_cancelled=True, ): return + # Fallback B produced the accepted result, but every new + # agentic round starts at primary A. Restore A's estimator + # before tool-result admission or end-of-turn compaction reads + # the remaining budget. + self._activate_token_calibration(self._primary_lane()) stopped_to_compact = stopped_to_compact_out[0] if stopped_to_compact_out else False if not tool_calls: @@ -9128,6 +11093,25 @@ class ChatSession: continue break + # Keyed executor/guard side maps intentionally model one live + # occurrence per provider call id. Same-batch duplicate + # non-empty ids would cross-attribute names, output guards, + # effects, and previews. Fail before state publication or any + # executor can populate those maps; the generic structural + # finalizer below emits one definitely-unstarted TOOL receipt + # per assistant-call occurrence. + call_id_counts = collections.Counter( + call.get("id", "") for call in tool_calls if call.get("id") + ) + duplicate_call_ids = sorted( + call_id for call_id, count in call_id_counts.items() if count > 1 + ) + if duplicate_call_ids: + raise _MalformedToolBatchError( + "Provider emitted duplicate tool call ids in one assistant batch: " + + ", ".join(duplicate_call_ids) + ) + # Execute tool calls (potentially in parallel) def _start_tools(durable: list[Callable[[], None]]) -> None: self._emit_state( @@ -9151,6 +11135,30 @@ class ChatSession: if self._generation != my_generation: return + # Same construction as the duplicate-id gate above; one + # comprehension serves both so the two checks can never + # disagree about the batch. + expected_result_counts = call_id_counts + observed_result_counts = collections.Counter(call_id for call_id, _ in results) + unexpected_results = observed_result_counts - expected_result_counts + if unexpected_results: + # An executor receipt may answer only an occurrence in the + # accepted assistant block. Fail before any TOOL row is + # admitted; generic structural cleanup will conservatively + # close every expected call. Clear keyed side maps so an + # internal malformed receipt cannot bleed into a later + # provider generation that reuses the same id. + for unexpected_id in unexpected_results: + self._cancelled_tool_results.pop(unexpected_id, None) + self._tool_error_flags.pop(unexpected_id, None) + self._tool_status.pop(unexpected_id, None) + self._tool_previews.pop(unexpected_id, None) + unexpected_ids = ", ".join(sorted(unexpected_results.elements())) + raise RuntimeError( + f"Tool executor returned receipts outside the accepted block: " + f"{unexpected_ids}" + ) + # Repeat-detection + tool-error nudge. Mutates *results* # in place to inject inline warning text on identical # repeats; queues advisories for the next drain pass. @@ -9401,6 +11409,7 @@ class ChatSession: tc_names: dict[str, Any], last_idx: int, feedback: str, + accepted_call_id_counts: collections.Counter[str], durable: list[Callable[[], None]], ) -> None: # Operator-context system turns are accumulated across the @@ -9446,8 +11455,8 @@ class ChatSession: # provider tool-call could still need cancellation # repair; otherwise a reused call id in a later loop # could inherit stale output. - self._cancelled_tool_results.pop(tc_id, None) - self.messages.append(turn_from_dict(tool_msg)) + staged_result = self._cancelled_tool_results.pop(tc_id, None) + tool_turn = turn_from_dict(tool_msg) # Token estimation — image content uses a fixed # heuristic while text follows the calibrated ratio. @@ -9460,7 +11469,11 @@ class ChatSession: 1, int(text_chars / self._chars_per_token) + image_count * 1000, ) - store_text = " ".join( + # One scalar is shared by pending /history, + # accepted SSE, and the durable TEXT column. The + # history projector joins multipart text blocks + # with newlines, so use the same delimiter here. + store_text = "\n".join( p.get("text", "") for p in output if isinstance(p, dict) and p.get("type") == "text" @@ -9468,51 +11481,76 @@ class ChatSession: else: tok_est = max(1, int(len(output) / self._chars_per_token)) store_text = output - self._msg_tokens.append(tok_est) - tool_atts = list(tool_image_atts) if tool_preview is not None: tool_atts.append(tool_preview[1]) persist_ws_id = self._ws_id - persist_event_id = self._ui_event_id() + commit_key = uuid.uuid4().hex + event_id_ref: list[int | None] = [None] persist_meta = _tool_turn_meta( tool_status, tool_preview[0] if tool_preview else None, + acting_principal=self._generation_principals.get(my_generation, ""), ) - persist_atts = tuple(tool_atts) + attachment_writes = _attachment_writes(tool_atts) - def _persist_tool_result( - *, - ws_id: str = persist_ws_id, - event_id: int | None = persist_event_id, - text: str = store_text, - tool_name: str = _tname, - call_id: str = tc_id, - is_error: bool = tool_is_error, - meta_json: str | None = persist_meta, - attachments: tuple[Attachment, ...] = persist_atts, - ) -> None: - tool_message_id = save_message( - ws_id, - "tool", - text, - tool_name, - tool_call_id=call_id, - event_id=event_id, - is_error=is_error, - meta=meta_json, - ) - if attachments and tool_message_id: - self._persist_attachment_refs( - tool_message_id, - attachments, - origin="tool", - ws_id=ws_id, - ) + restore_tool_side_state = functools.partial( + self._restore_tool_side_maps, + call_id=tc_id, + staged=staged_result, + was_error=tool_is_error, + status=tool_status, + preview=tool_preview, + ) - durable.append(_persist_tool_result) + # The detailed live result was emitted before output + # evaluation. Publish the final guarded scalar while + # sharing the handoff lock with row admission so a + # listener joining after the receipt still sees the + # accepted row. Duplicate ids within this assistant + # batch cannot key a browser upsert and retain strong + # repair instead. + pending = self._admit_and_publish_tool_row( + turn=tool_turn, + message=tool_msg, + token_estimate=tok_est, + commit_key=commit_key, + persist=self._tool_row_persist_closure( + ws_id=persist_ws_id, + text=store_text, + tool_name=_tname, + call_id=tc_id, + is_error=tool_is_error, + meta_json=persist_meta, + attachments=attachment_writes, + commit_key=commit_key, + event_id_ref=event_id_ref, + ), + restore_on_rollback=restore_tool_side_state, + event_id_ref=event_id_ref, + call_id=tc_id, + name=_tname, + output=store_text, + is_error=tool_is_error, + preview=tool_preview[0] if tool_preview else None, + effect_status=(tool_status.value if tool_status is not None else None), + projection_safe=(bool(tc_id) and accepted_call_id_counts[tc_id] == 1), + ) + self._schedule_conversation_persistence(pending, durable) pending_system_turns.extend(result_advisories) + # Every ordinary result row above is now present in both + # the live trajectory and handoff journal. A malformed + # executor result set is completed conservatively here; + # the shared synthesizer then retires the exact assistant + # block before this callback returns and the durability + # ticket is reserved. Storage may still fail, but the + # accepted overlay cannot expose an unmatched prefix. + self._synthesize_cancelled_results( + "Tool execution returned no observable outcome.", + deferred_persistence=durable, + ) + # Emit accumulated operator context only after the complete # tool block, preserving the native-system placement rule. for source, content, meta in pending_system_turns: @@ -9535,6 +11573,7 @@ class ChatSession: _tc_names, _last_idx, user_feedback or "", + expected_result_counts, ) if not self._commit_for_generation( @@ -9558,6 +11597,46 @@ class ChatSession: # re-checks. if not pre_attempted_compact: self._maybe_compact_midturn(my_generation) + except ConversationPersistenceError as exc: + # Do not run the generic failed-generation cleanup here: it drains + # queued user messages into a new conversation row, which would + # persist *after* the unresolved assistant boundary. Keep queued + # input queued, prevent tool/provider continuation, and surface a + # bounded error/state update only. The pending full assistant row + # and any synthesized unstarted TOOL suffix remain visible through + # /history and reconcile in order before a later send can mutate + # history. + def _finalize_persistence_failure( + error: BaseException, + durable: list[Callable[[], None]], + ) -> None: + # The assistant turn is already accepted in memory before its + # durable closure runs. If it contains tool calls, the failed + # closure prevents control from ever reaching _execute_tools; + # complete that provider-visible block with definitive + # unstarted TOOL rows. Keep their scheduled wrappers in a + # scratch list: each journal entry retains its own persistence + # closure for the manager's later ordered reconciliation, while + # executing a wrapper here would bypass the retry due time and + # stop the fatal-state closure behind the existing poison. + deferred_tool_repairs: list[Callable[[], None]] = [] + self._synthesize_cancelled_results( + "Cancelled before tool execution because conversation persistence " + "was interrupted; no side effects.", + deferred_persistence=deferred_tool_repairs, + definitely_unstarted=True, + ) + self._drain_pending_advisories() + self._record_fatal_error(error, deferred_persistence=durable) + + self._commit_for_generation( + my_generation, + functools.partial(_finalize_persistence_failure, exc), + allow_cancelled=True, + allow_persistence_poison=True, + allow_workstream_gone=True, + ) + raise except GenerationCancelled: def _finalize_cancelled_generation( @@ -9595,24 +11674,42 @@ class ChatSession: else: msg["content"] = "[generation cancelled before completion]" persist_ws_id = self._ws_id - persist_event_id = self._ui_event_id() + commit_key = uuid.uuid4().hex + persist_event_id: int | None = None persist_content = msg["content"] + partial_provenance = TurnProvenance.from_meta(msg.get("_provenance")) + persist_meta = ( + json.dumps({PROVENANCE_META_KEY: partial_provenance.to_meta()}) + if partial_provenance is not None + else None + ) - def _persist_cancelled_partial() -> None: - save_message( + def _persist_cancelled_partial() -> int: + return save_message( persist_ws_id, "assistant", persist_content, event_id=persist_event_id, + meta=persist_meta, + commit_key=commit_key, ) - durable.append(_persist_cancelled_partial) - self.messages.append(turn_from_dict(msg)) tok_est = max( 1, int(self._msg_char_count(msg) / self._chars_per_token), ) - self._msg_tokens.append(tok_est) + with self._history_handoff_lock: + self.messages.append(turn_from_dict(msg)) + self._msg_tokens.append(tok_est) + self._request_history_resync_locked("cancelled_assistant_turn_accepted") + persist_event_id = self._ui_event_id() + pending = self._journal_conversation_row_locked( + commit_key=commit_key, + message=msg, + persist=_persist_cancelled_partial, + event_id=persist_event_id, + ) + self._schedule_conversation_persistence(pending, durable) else: # Cancelled during tool execution — synthesize cancelled # tool_result for any tool_calls that lack a matching result. @@ -9638,6 +11735,7 @@ class ChatSession: if not self._commit_for_generation( my_generation, _finalize_cancelled_generation, + allow_workstream_gone=True, ): return # Do NOT re-raise — return normally so server worker thread @@ -9661,6 +11759,7 @@ class ChatSession: self._commit_for_generation( my_generation, functools.partial(_finalize_interrupted_generation, exc), + allow_workstream_gone=True, ) raise except Exception as exc: @@ -9675,6 +11774,21 @@ class ChatSession: error: BaseException, durable: list[Callable[[], None]], ) -> None: + # Any exception after assistant acceptance still owes the + # provider one TOOL receipt per call. The central synthesizer + # is a no-op when this generation has no structural debt. + definitely_unstarted = isinstance(error, _MalformedToolBatchError) + repair_reason = ( + "Tool batch was rejected before execution because provider tool call " + "ids were duplicated; no side effects." + if definitely_unstarted + else "Tool execution ended before the outcome could be observed." + ) + self._synthesize_cancelled_results( + repair_reason, + deferred_persistence=durable, + definitely_unstarted=definitely_unstarted, + ) self._flush_queued_messages(deferred_persistence=durable) self._drain_pending_advisories() self._record_fatal_error(error, deferred_persistence=durable) @@ -9682,6 +11796,7 @@ class ChatSession: self._commit_for_generation( my_generation, functools.partial(_finalize_failed_generation, exc), + allow_workstream_gone=True, ) raise finally: @@ -9761,6 +11876,8 @@ class ChatSession: reason: str, *, deferred_persistence: list[Callable[[], None]] | None = None, + definitely_unstarted: bool = False, + structural_generation: int | None = None, ) -> None: """Synthesize tool_result messages for orphaned tool_calls after cancel. @@ -9769,7 +11886,23 @@ class ChatSession: cancelled results for any that don't. This keeps the conversation valid (both providers require matching tool_results) while preserving the full tool call structure so the model knows what was attempted. + + ``definitely_unstarted`` is reserved for the assistant-persistence + failure seam, which runs before tool execution is reachable. It records + a definitive ``NONE`` disposition instead of the ordinary conservative + ``UNKNOWN`` cancellation outcome. """ + if deferred_persistence is None: + self._direct_commit_reentry( + "Cannot append cancelled results to a closed session", + lambda durable: self._synthesize_cancelled_results( + reason, + deferred_persistence=durable, + definitely_unstarted=definitely_unstarted, + structural_generation=structural_generation, + ), + ) + return # Find the last assistant message with tool_calls assistant_idx = None for i in range(len(self.messages) - 1, -1, -1): @@ -9780,11 +11913,30 @@ class ChatSession: if assistant_idx is None: return - # Collect tool_call IDs that already have results - answered_ids: set[str] = set() + # Count matching results by occurrence, not just by ID. Provider IDs + # are normally unique, but compatibility servers may repeat a + # non-empty ID (``ensure_tool_call_ids`` repairs blanks only). In that + # case one observed TOOL row answers exactly one call; every remaining + # occurrence still needs its own structural receipt. + remaining_answered = collections.Counter[str]() for msg in self.messages[assistant_idx + 1 :]: if msg.role is Role.TOOL: - answered_ids.add(msg.tool_call_id or "") + remaining_answered[msg.tool_call_id or ""] += 1 + assistant_call_id_counts = collections.Counter( + tc.id for tc in self.messages[assistant_idx].tool_calls if tc.id + ) + # The generation these receipts belong to: the exact one force-abandon's + # structural finalizer names, otherwise the bounded live commit staging + # on this thread. One derivation feeds both the acting principal + # stamped on every synthesized row and the structural debt completed + # below, so a synthesized receipt and the ordinary fold that shares its + # batch can never disagree about whose turn executed it. + origin_generation = ( + structural_generation + if structural_generation is not None + else _active_commit_origin_generation.get() + ) + acting_principal = self._generation_principals.get(origin_generation, "") # An orphaned tool_call usually has no observable result when cancel # lands, so the generic disposition is UNKNOWN rather than a bare @@ -9796,12 +11948,17 @@ class ChatSession: for tc in self.messages[assistant_idx].tool_calls: tc_id = tc.id func_name = tc.name - if tc_id and tc_id not in answered_ids: + if tc_id and remaining_answered[tc_id]: + remaining_answered[tc_id] -= 1 + continue + if tc_id: staged_cancel = self._cancelled_tool_results.pop(tc_id, None) effect_status: EffectStatus | None if staged_cancel is None: - detail = generic_detail - effect_status = EffectStatus.UNKNOWN + detail = reason if definitely_unstarted else generic_detail + effect_status = ( + EffectStatus.NONE if definitely_unstarted else EffectStatus.UNKNOWN + ) result_is_error = True live_preview = None live_result_already_emitted = False @@ -9815,8 +11972,8 @@ class ChatSession: # tool may have self-reported an error/status immediately # before Stop; leaving either keyed by a reusable provider id # would stamp it onto a later generation's unrelated result. - self._tool_error_flags.pop(tc_id, None) - self._tool_status.pop(tc_id, None) + prior_error_flag = self._tool_error_flags.pop(tc_id, False) + prior_tool_status = self._tool_status.pop(tc_id, None) # A staged preview means _exec_open_preview COMPLETED and its # descriptor already reached the frontend (live SSE at exec # time) — the pane is open on it. Discarding the blob here @@ -9834,63 +11991,50 @@ class ChatSession: ) if preview_entry is not None: cancelled_turn.meta.extra["preview"] = preview_entry[0] - self.messages.append(cancelled_turn) - self._msg_tokens.append(1) persist_ws_id = self._ws_id - persist_event_id = self._ui_event_id() + commit_key = uuid.uuid4().hex + event_id_ref: list[int | None] = [None] persist_meta = _tool_turn_meta( effect_status, preview_entry[0] if preview_entry else None, + acting_principal=acting_principal, + ) + attachment_writes = _attachment_writes( + () if preview_entry is None else (preview_entry[1],) ) - persist_preview = preview_entry[1] if preview_entry is not None else None - def _persist_cancelled_result( + restore_cancelled_side_state = functools.partial( + self._restore_tool_side_maps, + call_id=tc_id, + staged=staged_cancel, + was_error=prior_error_flag, + status=prior_tool_status, + preview=preview_entry, + ) + + # Complete the in-DOM tool batch when Stop won before the + # ordinary result publication. The accepted projection + # follows even when an executor receipt was already sent: + # a listener may have registered after that receipt, and a + # committed preview must remain reopenable on an error row. + def _emit_live_result_once( *, - ws_id: str = persist_ws_id, - event_id: int | None = persist_event_id, - result_detail: str = detail, - tool_name: str = func_name, - call_id: str = tc_id, - is_error: bool = result_is_error, - meta_json: str | None = persist_meta, - preview_attachment: Attachment | None = persist_preview, + _already_emitted: bool = live_result_already_emitted, + _call_id: str = tc_id, + _name: str = func_name, + _detail: str = detail, + _was_error: bool = result_is_error, + _preview: Any = live_preview, ) -> None: - cancelled_row_id = save_message( - ws_id, - "tool", - result_detail, - tool_name, - tool_call_id=call_id, - event_id=event_id, - is_error=is_error, - meta=meta_json, - ) - if preview_attachment is not None and cancelled_row_id: - self._persist_attachment_refs( - cancelled_row_id, - [preview_attachment], - origin="tool", - ws_id=ws_id, - ) - - if deferred_persistence is None: - _persist_cancelled_result() - else: - deferred_persistence.append(_persist_cancelled_result) - # Emit synthetic tool_result so live SSE listeners can - # complete the in-DOM tool batch — without this the - # coord ``--running`` indicator (added by SSE - # tool_info) would spin forever on cancelled batches. - # Defensive: we're already on a cancel/error path, so - # a UI hook failure must not compound the problem. - if not live_result_already_emitted: + if _already_emitted: + return try: self.ui.on_tool_result( - tc_id, - func_name, - detail, - is_error=result_is_error, - preview=live_preview, + _call_id, + _name, + _detail, + is_error=_was_error, + preview=_preview, ) except Exception: log.debug( @@ -9899,6 +12043,192 @@ class ChatSession: exc_info=True, ) + pending_message = turn_to_dict(cancelled_turn) + pending = self._admit_and_publish_tool_row( + turn=cancelled_turn, + message=pending_message, + token_estimate=1, + commit_key=commit_key, + persist=self._tool_row_persist_closure( + ws_id=persist_ws_id, + text=detail, + tool_name=func_name, + call_id=tc_id, + is_error=result_is_error, + meta_json=persist_meta, + attachments=attachment_writes, + commit_key=commit_key, + event_id_ref=event_id_ref, + ), + restore_on_rollback=restore_cancelled_side_state, + event_id_ref=event_id_ref, + call_id=tc_id, + name=func_name, + output=detail, + is_error=result_is_error, + preview=preview_entry[0] if preview_entry else None, + effect_status=(effect_status.value if effect_status is not None else None), + projection_safe=(assistant_call_id_counts[tc_id] == 1), + pre_publish=_emit_live_result_once, + ) + self._schedule_conversation_persistence( + pending, + deferred_persistence, + ) + + if origin_generation: + completed_ids = [ + msg.tool_call_id or "" + for msg in self.messages[assistant_idx + 1 :] + if msg.role is Role.TOOL + ] + self._complete_tool_structural_debt_locked( + origin_generation, + completed_ids, + ) + + def _restore_tool_side_maps( + self, + *, + call_id: str, + staged: _CancelledToolResult | None, + was_error: bool, + status: EffectStatus | None, + preview: tuple[dict[str, Any], Attachment] | None, + ) -> None: + """Re-stage one call's popped side channels after a rolled-back row. + + Both accepted-TOOL-row paths pop the same four per-call maps while + building their row, so a failed journal admission has to put back + exactly what the fold consumed. Shared so the two rollbacks cannot + drift. + """ + if staged is not None: + self._cancelled_tool_results[call_id] = staged + if was_error: + self._tool_error_flags[call_id] = True + if status is not None: + self._tool_status[call_id] = status + if preview is not None: + self._tool_previews[call_id] = preview + + @staticmethod + def _tool_row_persist_closure( + *, + ws_id: str, + text: str, + tool_name: str, + call_id: str, + is_error: bool, + meta_json: str | None, + attachments: tuple[AttachmentWrite, ...], + commit_key: str, + event_id_ref: list[int | None], + ) -> Callable[[], int]: + """Build the keyed durable write for one accepted TOOL row. + + Shared by the ordinary result path and cancelled-result synthesis so + the two accepted-row shapes cannot drift. + """ + + def _persist() -> int: + if attachments: + return save_tool_message_with_attachments( + ws_id, + text, + tool_name, + call_id, + attachments, + event_id=event_id_ref[0], + is_error=is_error, + meta=meta_json, + commit_key=commit_key, + ) + return save_message( + ws_id, + "tool", + text, + tool_name, + tool_call_id=call_id, + event_id=event_id_ref[0], + is_error=is_error, + meta=meta_json, + commit_key=commit_key, + ) + + return _persist + + def _admit_and_publish_tool_row( + self, + *, + turn: Turn, + message: dict[str, Any], + token_estimate: int, + commit_key: str, + persist: Callable[[], int], + restore_on_rollback: Callable[[], None], + event_id_ref: list[int | None], + call_id: str, + name: str, + output: str, + is_error: bool, + preview: dict[str, Any] | None, + effect_status: str | None, + projection_safe: bool, + pre_publish: Callable[[], None] | None = None, + ) -> _PendingConversationCommit: + """Append, journal, and publish one accepted TOOL row atomically. + + The handoff lock spans live append, journal admission, and event + publication so a listener joining mid-batch still sees the accepted + row. A failed journal admission rolls the live list and revision + back, restores the caller's per-call side maps, and re-raises, + leaving no half-admitted receipt behind. Shared by the ordinary + result path and cancelled-result synthesis so the admission and + rollback choreography cannot drift. + """ + with self._history_handoff_lock: + history_revision_before = self._history_handoff_revision + self.messages.append(turn) + self._msg_tokens.append(token_estimate) + try: + # pre_publish sits inside the rollback envelope: its supplier + # catches Exception only, so a BaseException (a second Ctrl-C + # landing in ui.on_tool_result during cancelled-result + # synthesis on the CLI main thread) must roll the live append + # back rather than leave a turn no journal entry or durable + # row will ever represent (round-4 review). + if pre_publish is not None: + pre_publish() + pending = self._journal_conversation_row_locked( + commit_key=commit_key, + message=message, + persist=persist, + event_id=None, + ) + except BaseException: + self._rollback_live_journal_admission_locked( + turn=turn, + commit_key=commit_key, + history_revision_before=history_revision_before, + reason="tool_turn_admission_rolled_back", + ) + restore_on_rollback() + raise + event_id_ref[0] = self._publish_tool_turn_locked( + call_id=call_id, + name=name, + output=output, + is_error=is_error, + preview=preview, + effect_status=effect_status, + projection_safe=projection_safe, + ) + message["_event_id"] = event_id_ref[0] + turn.meta.event_id = event_id_ref[0] + pending.message["_event_id"] = event_id_ref[0] + return pending + # -- Rewind / retry ------------------------------------------------------- def _find_turn_boundaries(self) -> list[int]: @@ -9920,65 +12250,182 @@ class ChatSession: if m.role is Role.USER and m.source != COMPACTION_SOURCE ] - def _persist_truncation(self, removed_count: int) -> None: - """Mirror a rewind/retry tail-trim into storage, compaction-safe. + def _persist_truncation(self, removed_count: int) -> int: + """Atomically remove a compaction-floored durable tail. - rewind/retry remove turns from the in-memory TAIL, which map 1:1 to the - most recent storage rows. Deleting by *removed-row count* from the - storage end — rather than keeping the first ``len(self.messages)`` rows — - stays correct after a compaction, where ``self.messages`` holds synthetic - summary turns with no 1:1 storage rows while storage still holds the full - pre-compaction transcript plus the checkpoint marker. The keep-count is - floored at the compaction boundary (:func:`get_compaction_floor`) so the - summary's backing rows and the marker are never deleted. Identical to the - old ``keep = len(self.messages)`` when the ws never compacted (floor 0, - total == in-memory length). - - Residual: an over-deep rewind that would cross the summary boundary clamps - at the floor in storage, so the in-memory trim can drop more than storage - does; a later resume rehydrates ``[summary] + [surviving tail]`` and the - two reconcile. + The backend owns parent locking, row-count/floor calculation, exact + deletion, and attachment reference release in one transaction. It + raises on a missing parent or storage failure; callers must not publish + the corresponding live cut until this returns successfully. """ if removed_count <= 0: - return - # The caller already truncated self.messages (rewind/retry both trim - # before calling this), so any sender that only appeared in the - # dropped tail is no longer derivable from live history — unlike - # compaction narrowing (which keeps the full transcript in storage, - # exactly why _recompute_shared_state unions in a persisted-sender - # read), a rewind/retry deletes those very rows below. Force a full, - # fresh re-derivation on the next compose rather than let the - # monotonic union/latch keep a departed sender "known" and the - # workstream stuck "shared" forever after the only evidence of a - # second participant is gone. - self._reset_shared_state() - total = count_messages(self._ws_id) - if total <= 0: - # Count unavailable (storage error) — skip the delete rather than risk - # a wrong truncation; the in-memory trim holds and resume reconciles. - return - floor = get_compaction_floor(self._ws_id) - if floor < 0: - # Floor unavailable (storage error). A 0 here is indistinguishable from - # "never compacted", so an over-deep trim could delete the marker and - # summarized prefix — skip rather than risk it; resume reconciles. - return - delete_messages_after(self._ws_id, max(floor, total - removed_count)) - # History generation (#894/#884 seam): every truncation bumps the - # counter the /history single-flight folds into its flight key, so - # a request dispatched AFTER a rewind/retry can never join a flight - # whose load_messages ran BEFORE it (a joined pre-rewind payload - # rendered as fresh truth on the coordinator and reopened the - # over-rewind window the #894 client latch closes). Bumped AFTER - # the storage delete: flights rebuild from storage, so a flight - # keyed with the OLD generation that reads post-delete rows is the - # harmless spuriously-fresh direction, while a NEW-generation - # flight reading pre-delete rows would be the wrongly-joined one — - # and the early-return error paths above (count/floor unavailable, - # delete skipped) correctly leave the generation unbumped. - self._history_generation += 1 + return 0 + return get_storage().truncate_messages_tail(self._ws_id, removed_count) - def rewind(self, n: int) -> int: + def _commit_history_truncation( + self, + admit: Callable[[list[Callable[[], None]]], None], + ) -> None: + """Admit a destructive cut only when no older durability ticket runs. + + The truncation latch intentionally stays clear while an older ticket is + draining. A deferred UI/storage callback in that ticket may itself + need a bounded generation commit; closing admission first would make + the callback wait for truncation while truncation waits for its ticket. + The generation lock makes the final idle-check, latch, callback, and + new ticket reservation indivisible from every ordinary commit. + """ + + def _durability_reached(high_water: int) -> bool: + return self._durability_serving_ticket >= high_water + + while True: + wait_high_water: int | None = None + + def _admit_if_idle(durable: list[Callable[[], None]]) -> None: + nonlocal wait_high_water + with self._durability_cond: + if self._durability_serving_ticket != self._durability_next_ticket: + wait_high_water = self._durability_next_ticket + return + admit(durable) + + if not self._commit_for_generation( + 0, + _admit_if_idle, + allow_persistence_poison=True, + ): + raise GenerationCancelled() + if wait_high_water is None: + return + high_water = wait_high_water + with self._durability_cond: + self._durability_cond.wait_for(functools.partial(_durability_reached, high_water)) + + def _stage_history_truncation( + self, + durable: list[Callable[[], None]], + cut_index: int, + publish_reset: Callable[[], None] | None = None, + ) -> int: + """Reserve and stage one failure-atomic live/durable tail cut. + + Called only from a ``_commit_for_generation`` callback while the + generation lock and truncation condition are held. The active latch + prevents every later generation/direct commit from changing the live + list before the durability ticket commits and applies this exact slice. + """ + removed_turns = tuple(self.messages[cut_index:]) + removed_count = len(removed_turns) + original_length = len(self.messages) + if removed_count <= 0: + return 0 + if self._history_truncation_active: + raise RuntimeError("history truncation admitted while another cut is active") + self._history_truncation_active = True + # Live turns and durable rows are not one-to-one: a structurally empty + # assistant result lives only in ``self.messages`` (stamped + # ``no_durable_row`` at admission). The durable cut must count only + # turns that own a storage row, else each live-only turn in the span + # deletes one older committed row in its place. + durable_removed = sum( + 1 for turn in removed_turns if not turn.meta.extra.get("no_durable_row") + ) + + def _persist_and_publish_truncation() -> None: + try: + # This ticket was admitted before any terminal latch. Repair an + # older ambiguous row in FIFO order even if close/delete began + # while the ticket was waiting; terminal deletion waits for us. + # Deliberately NOT ``_force_retry``: a destructive mutation + # inside the transient backoff fails fast on the stored poison + # instead of hammering storage (soft close differs — it forces + # one last probe because refusing it keeps the session loaded). + # Pinned by ``test_unresolved_prefix_repair_failure_refuses_ + # truncation_without_cut_publication``. + self._reconcile_pending_conversation_commits(_terminal_admitted_repair=True) + with self._history_visibility_lock: + # No later commit can mutate the list while the active latch + # is set. Validate nevertheless so a future unguarded history + # replacement fails before touching durable state. + with self._generation_lock: + current = self.messages[cut_index : cut_index + removed_count] + if len(current) != removed_count or any( + actual is not expected + for actual, expected in zip( + current, + removed_turns, + strict=True, + ) + ): + raise RuntimeError("conversation changed outside truncation admission") + + self._persist_truncation(durable_removed) + + # Storage has committed. From this point onward only + # infallible list/field updates and best-effort UI repair + # remain: raising on a second validation could leave the DB + # cut committed while the old live tail stayed visible. + # Publish the matching live cut, token revision, and repair + # event as one bounded transition. History readers remain + # behind visibility until both representations agree. + with self._generation_lock, self._history_handoff_lock: + del self.messages[cut_index:original_length] + del self._msg_tokens[cut_index:original_length] + self._last_usage = None + self._invalidate_token_calibration_anchors() + # A sender appearing only in the dropped tail is no + # longer evidence that this workstream is shared. + self._reset_shared_state() + self._history_generation += 1 + self._history_handoff_revision += 1 + self._publish_history_truncation_reset_locked(publish_reset) + + # Fence any generation that prepared provider/tool work against + # the removed trajectory. Do not advance the Stop epoch: the + # retry command's already-captured worker claim must remain + # valid for its replacement send unless a real Stop occurred. + with self._generation_transition_lock, self._generation_lock: + self._cancel_event.set() + self._generation += 1 + finally: + with self._history_truncation_condition: + self._history_truncation_active = False + self._history_truncation_condition.notify_all() + + durable.append(_persist_and_publish_truncation) + return removed_count + + def _publish_history_truncation_reset_locked( + self, + publish_reset: Callable[[], None] | None, + ) -> None: + """Publish exactly one structural-reset signal for a history cut. + + HTTP callers supply their established ``clear_ui`` publisher: that + event owns the live-stream quiesce and edit-and-resend continuation in + both web clients. Direct callers have no route-level follow-up, so the + stronger handoff repair remains their default. A broken callback must + not turn an already-committed storage cut into a raised half-success; + fall back to the generic repair event instead. + + Called with the history handoff lock held so the reset is ordered in + the same accepted revision as the live/durable cut. + """ + if publish_reset is not None: + try: + publish_reset() + return + except Exception: + log.debug("history truncation reset publisher raised", exc_info=True) + self._request_history_resync_locked("history_truncated") + + def rewind( + self, + n: int, + *, + publish_reset: Callable[[], None] | None = None, + ) -> int: """Drop the last *n* complete turns from the conversation. A turn = user message + all assistant/tool messages until the next @@ -9987,39 +12434,61 @@ class ChatSession: """ if n < 1: return 0 - boundaries = self._find_turn_boundaries() - if not boundaries: - return 0 - n = min(n, len(boundaries)) - cut_index = boundaries[-n] - removed_count = len(self.messages) - cut_index - del self.messages[cut_index:] - del self._msg_tokens[cut_index:] - self._persist_truncation(removed_count) + removed_count = 0 + + def _admit_rewind(durable: list[Callable[[], None]]) -> None: + nonlocal removed_count + boundaries = self._find_turn_boundaries() + if not boundaries: + # Web rewind historically emits clear_ui even when there is + # nothing to remove (including an already-empty transcript). + # Direct callers pass no publisher and retain their no-event + # no-op behavior. + if publish_reset is not None: + self._publish_history_truncation_reset_locked(publish_reset) + return + turn_count = min(n, len(boundaries)) + removed_count = self._stage_history_truncation( + durable, + boundaries[-turn_count], + publish_reset, + ) + + self._commit_history_truncation(_admit_rewind) return removed_count - def retry(self) -> str | None: + def retry( + self, + *, + publish_reset: Callable[[], None] | None = None, + ) -> str | None: """Drop the last assistant response and return the user message to re-send. The caller is responsible for calling ``send()`` with the returned message. Returns ``None`` if there is nothing to retry. """ - boundaries = self._find_turn_boundaries() - if not boundaries: - return None - last_user_idx = boundaries[-1] - content = turn_to_dict(self.messages[last_user_idx]).get("content") - # Multipart messages (vision/images) have list-type content; - # retry only supports plain text. - if not isinstance(content, str) or not content: - return None - # Drop everything from (and including) the user message onward; - # send() will re-append the user message. - removed_count = len(self.messages) - last_user_idx - del self.messages[last_user_idx:] - del self._msg_tokens[last_user_idx:] - self._persist_truncation(removed_count) - return content + retry_content: str | None = None + + def _admit_retry(durable: list[Callable[[], None]]) -> None: + nonlocal retry_content + boundaries = self._find_turn_boundaries() + if not boundaries: + if publish_reset is not None: + self._publish_history_truncation_reset_locked(publish_reset) + return + last_user_idx = boundaries[-1] + content = turn_to_dict(self.messages[last_user_idx]).get("content") + # Multipart messages (vision/images) have list-type content; + # retry only supports plain text. + if not isinstance(content, str) or not content: + if publish_reset is not None: + self._publish_history_truncation_reset_locked(publish_reset) + return + self._stage_history_truncation(durable, last_user_idx, publish_reset) + retry_content = content + + self._commit_history_truncation(_admit_retry) + return retry_content def _ui_stream_discarded(self) -> None: """Best-effort dead-segment discard across UI generations. @@ -10073,6 +12542,10 @@ class ChatSession: # phase fails with an unarmed error, the original death is the one # the operator needs to see, not the re-create's. last_stream_death: Exception | None = None + # Provenance of the attempt whose text ``dead_partial`` carries. A + # zero-token re-death must not relabel preserved text from the prior + # attempt's lane. + dead_provenance: TurnProvenance | None = None consumer = _StreamTurnConsumer(self, my_generation) principal_id = self._generation_principals.get(my_generation) @@ -10125,10 +12598,13 @@ class ChatSession: return cur = self._cancelled_partial_msg if cur is None or (not cur.get("content") and dead_partial): - self._cancelled_partial_msg = { + promoted: dict[str, Any] = { "role": "assistant", "content": dead_partial, } + if dead_provenance is not None: + promoted["_provenance"] = dead_provenance.to_meta() + self._cancelled_partial_msg = promoted self._publish_for_generation( my_generation, @@ -10136,118 +12612,190 @@ class ChatSession: allow_cancelled=True, ) - while True: - try: - result = self._model_turn_with_fallback( - consumer, - _prepare, - my_generation, - principal_id=principal_id, - ) - # A Stop that raced the trailing-metadata window: cancel() - # closed the stream and the drain's post-finish tolerance - # ended it CLEANLY, so without this re-check the turn - # commits as complete and its tool calls execute. - self._check_cancelled(my_generation) - consumer.finish_stream() - # Finalization emits terminal warnings/stream_end and may - # legalize a truncated result. Keep that complete policy step - # on the same owner rail as the carry flush above so a force - # successor or close cannot receive the predecessor's terminal - # UI event in the gap between them. A same-generation Stop - # after a completed stream preserves the historical completed- - # result boundary. - with self._generation_lock: - if self._publication_shutdown or _generation_superseded(self, my_generation): - raise GenerationCancelled() - return self._finalize_stream_result(result) - except GenerationCancelled: - # A Stop during an attempt (incl. the re-create/TTFT window - # after a death) — finalize the streamed display if the - # attempt got a stream, then preserve the window's best - # partial. - if consumer.attempt_armed: - consumer.record_cancelled_partial() - _promote_dead_partial() - raise - except KeyboardInterrupt: - # BaseException — the Exception arm below never sees it, - # but send() treats Ctrl-C as a survivable, recorded path, - # so the dead attempt still needs its client-side finalize - # (the CLI's markdown fence resets only in on_stream_end). - self._publish_for_generation( - my_generation, - self.ui.on_stream_end, - allow_cancelled=True, - ) - raise - except Exception as e: - armed = consumer.attempt_armed - new_dead = consumer.partial_content() if armed else "" - dead_partial = new_dead or dead_partial - if armed: - # The partial is captured, so there is no live attempt - # until the next ``begin_attempt``: without this, a - # Stop or a walk-preamble failure landing in the - # re-create window reads the DEAD attempt's armed - # state (see ``end_attempt``). - consumer.end_attempt() - if self._cancel_event.is_set(): - _promote_dead_partial() - raise GenerationCancelled() from None - if _generation_superseded(self, my_generation) or self._publication_shutdown: - # Superseded (force-cancel started a newer generation): - # an orphaned thread must not touch the UI — a finalize - # emitted here would clobber the NEW generation's - # in-flight stream state. - raise - if not armed: - # Creation-phase failure: the walk already ran its full - # ladder + fallbacks. Mid re-issue it must not MASK - # the original stream death (a closed-client re-create - # surfaces as a retryable APIConnectionError and would - # replace the operator-actionable wording) — EXCEPT - # the classes carrying their own remediation: an - # overflow surfaces as ITSELF so send()'s - # compact-and-retry arm can recover the turn, and an - # auth refusal or wire-preparation fault surfaces as - # ITSELF so the fatal formatter's dedicated branch - # renders. Class name only in the log — a - # ConnectError's text can carry a credential-bearing - # base_url verbatim. - if ( - last_stream_death is None - or isinstance(e, _SELF_SURFACING_ERRORS) - or _is_ctx_overflow(e) - ): - raise - log.warning( - "stream.retry.recreate_failed", - error_type=type(e).__name__, + with self._recovered_serving_failures() as recovered_failures: + while True: + try: + result = self._model_turn_with_fallback( + consumer, + _prepare, + my_generation, + principal_id=principal_id, ) - raise last_stream_death from None - # The terminal predicate is the SHARED _stop_retrying, - # capped at _MID_STREAM_RETRIES, judged by the lane that - # ACTUALLY armed this stream (a fallback's retryable set - # can differ, e.g. ResponsesStreamFailedError). The - # overflow arm applies here too — an overflow can surface - # mid-consumption (error-frame lanes), and it must fall - # through to send()'s compact-and-retry arm rather than - # burn re-issues on a deterministic failure. - serving_lane = consumer.lane - if serving_lane is None: - raise RuntimeError("armed stream has no serving model lane") from e - if self._stop_retrying( - e, attempt, serving_lane, max_retries=self._MID_STREAM_RETRIES - ): - # Terminal: finalize AND discard, exactly like the - # retry arm. Keeping the buffers bought nothing — the - # fatal path's _emit_state("error") drains and wipes - # them anyway on every SessionUIBase lane — and an - # overflow that send()'s compact-and-retry RECOVERS - # re-streams into buffers that would otherwise still - # hold the dead attempt's text, concatenating the two - # in the idle payload. + # A Stop that raced the trailing-metadata window: cancel() + # closed the stream and the drain's post-finish tolerance + # ended it CLEANLY, so without this re-check the turn + # commits as complete and its tool calls execute. + self._check_cancelled(my_generation) + consumer.finish_stream() + # Finalization emits terminal warnings/stream_end and may + # legalize a truncated result. Keep that complete policy step + # on the same owner rail as the carry flush above so a force + # successor or close cannot receive the predecessor's terminal + # UI event in the gap between them. A same-generation Stop + # after a completed stream preserves the historical completed- + # result boundary. + with self._generation_lock: + if self._publication_shutdown or _generation_superseded( + self, my_generation + ): + raise GenerationCancelled() + return self._finalize_stream_result(result) + except GenerationCancelled: + # A Stop during an attempt (incl. the re-create/TTFT window + # after a death) — finalize the streamed display if the + # attempt got a stream, then preserve the window's best + # partial. + if consumer.attempt_armed: + consumer.record_cancelled_partial() + _promote_dead_partial() + raise + except KeyboardInterrupt: + # BaseException — the Exception arm below never sees it, + # but send() treats Ctrl-C as a survivable, recorded path, + # so the dead attempt still needs its client-side finalize + # (the CLI's markdown fence resets only in on_stream_end). + self._publish_for_generation( + my_generation, + self.ui.on_stream_end, + allow_cancelled=True, + ) + raise + except Exception as e: + attempt_provenance = None + armed = consumer.attempt_armed + failed_lane = consumer.lane if armed else None + if failed_lane is not None: + self._remember_serving_failure_context(e, failed_lane) + recovered_failures.append(e) + new_dead = consumer.partial_content() if armed else "" + if failed_lane is not None: + attempt_provenance = TurnProvenance( + model_alias=failed_lane.alias, + backend_model_id=failed_lane.model, + registry_generation=failed_lane.registry_generation, + acting_principal_id=principal_id or "", + ) + dead_partial = new_dead or dead_partial + if attempt_provenance is not None and (new_dead or dead_provenance is None): + dead_provenance = attempt_provenance + if armed: + # The partial is captured, so there is no live attempt + # until the next ``begin_attempt``: without this, a + # Stop or a walk-preamble failure landing in the + # re-create window reads the DEAD attempt's armed + # state (see ``end_attempt``). + consumer.end_attempt() + if self._cancel_event.is_set(): + _promote_dead_partial() + raise GenerationCancelled() from None + if _generation_superseded(self, my_generation) or self._publication_shutdown: + # Superseded (force-cancel started a newer generation): + # an orphaned thread must not touch the UI — a finalize + # emitted here would clobber the NEW generation's + # in-flight stream state. send()'s orphan gate absorbs + # the re-raise without recording it, so this snapshot + # has no reader even though its exception escapes. + self._forget_serving_failure_context(e) + raise + if not armed: + # Creation-phase failure: the walk already ran its full + # ladder + fallbacks. Mid re-issue it must not MASK + # the original stream death (a closed-client re-create + # surfaces as a retryable APIConnectionError and would + # replace the operator-actionable wording) — EXCEPT + # the classes carrying their own remediation: an + # overflow surfaces as ITSELF so send()'s + # compact-and-retry arm can recover the turn, and an + # auth refusal or wire-preparation fault surfaces as + # ITSELF so the fatal formatter's dedicated branch + # renders. Class name only in the log — a + # ConnectError's text can carry a credential-bearing + # base_url verbatim. + if ( + last_stream_death is None + or isinstance(e, _SELF_SURFACING_ERRORS) + or _is_ctx_overflow(e) + ): + raise + log.warning( + "stream.retry.recreate_failed", + error_type=type(e).__name__, + ) + raise last_stream_death from None + # The terminal predicate is the SHARED _stop_retrying, + # capped at _MID_STREAM_RETRIES, judged by the lane that + # ACTUALLY armed this stream (a fallback's retryable set + # can differ, e.g. ResponsesStreamFailedError). The + # overflow arm applies here too — an overflow can surface + # mid-consumption (error-frame lanes), and it must fall + # through to send()'s compact-and-retry arm rather than + # burn re-issues on a deterministic failure. + serving_lane = consumer.lane + if serving_lane is None: + raise RuntimeError("armed stream has no serving model lane") from e + if self._stop_retrying( + e, attempt, serving_lane, max_retries=self._MID_STREAM_RETRIES + ): + # Terminal: finalize AND discard, exactly like the + # retry arm. Keeping the buffers bought nothing — the + # fatal path's _emit_state("error") drains and wipes + # them anyway on every SessionUIBase lane — and an + # overflow that send()'s compact-and-retry RECOVERS + # re-streams into buffers that would otherwise still + # hold the dead attempt's text, concatenating the two + # in the idle payload. + with self._generation_lock: + if ( + self._publication_shutdown + or _generation_superseded(self, my_generation) + or self._cancel_event.is_set() + ): + _promote_dead_partial() + raise GenerationCancelled() from None + self.ui.on_stream_end() + self._ui_stream_discarded() + raise # fatal path otherwise unchanged + # This death supersedes the previous one: drop the older + # snapshot here rather than at the ladder's exit, so a long + # re-issue run holds one entry in the bounded map, not one + # per attempt. + self._forget_serving_failure_context(last_stream_death) + last_stream_death = e + # Delay from the PRE-increment attempt index — the same + # convention as the sibling ladders' range loops. + delay = self._RETRY_BASE_DELAY * (2**attempt) + attempt += 1 + cause = type(e.__cause__).__name__ if e.__cause__ else type(e).__name__ + log.warning( + "stream.retry", + error_type=cause, + attempt=attempt, + model=serving_lane.model, + alias=serving_lane.alias, + registry_generation=serving_lane.registry_generation, + retry_in=delay, + # Spend trace for the abandoned generation: the wire + # reports usage only at stream end, so a dead attempt's + # billed tokens are otherwise invisible — dead_usage + # carries what the wire DID deliver (Anthropic's early + # prompt tokens; None on the OpenAI chat lane, whose + # usage chunk trails the finish), and the char count + # lets an operator estimate the discarded completion. + # THIS death's flushed text only — a pre-token re-death + # logs 0, never the Stop-preservation carry from a + # prior attempt (that would double-count spend). + dead_usage=self._last_usage, + dead_content_chars=len(new_dead), + ) + # Finalize the dead attempt client-side, then WAIT before + # discarding: stream_end (browser bubble, CLI markdown + # flush/fence reset) -> notice -> backoff. The + # server-buffer discard runs only AFTER the backoff + # survives the Stop window — a Stop during backoff persists + # the promoted partial to history, and the idle payload + # (drained from the turn buffer) must carry the same text, + # or the dashboard renders the cancelled turn empty while + # the transcript has it. with self._generation_lock: if ( self._publication_shutdown @@ -10257,98 +12805,53 @@ class ChatSession: _promote_dead_partial() raise GenerationCancelled() from None self.ui.on_stream_end() - self._ui_stream_discarded() - raise # fatal path otherwise unchanged - last_stream_death = e - # Delay from the PRE-increment attempt index — the same - # convention as the sibling ladders' range loops. - delay = self._RETRY_BASE_DELAY * (2**attempt) - attempt += 1 - cause = type(e.__cause__).__name__ if e.__cause__ else type(e).__name__ - log.warning( - "stream.retry", - error_type=cause, - attempt=attempt, - model=serving_lane.model, - retry_in=delay, - # Spend trace for the abandoned generation: the wire - # reports usage only at stream end, so a dead attempt's - # billed tokens are otherwise invisible — dead_usage - # carries what the wire DID deliver (Anthropic's early - # prompt tokens; None on the OpenAI chat lane, whose - # usage chunk trails the finish), and the char count - # lets an operator estimate the discarded completion. - # THIS death's flushed text only — a pre-token re-death - # logs 0, never the Stop-preservation carry from a - # prior attempt (that would double-count spend). - dead_usage=self._last_usage, - dead_content_chars=len(new_dead), - ) - # Finalize the dead attempt client-side, then WAIT before - # discarding: stream_end (browser bubble, CLI markdown - # flush/fence reset) -> notice -> backoff. The - # server-buffer discard runs only AFTER the backoff - # survives the Stop window — a Stop during backoff persists - # the promoted partial to history, and the idle payload - # (drained from the turn buffer) must carry the same text, - # or the dashboard renders the cancelled turn empty while - # the transcript has it. - with self._generation_lock: - if ( - self._publication_shutdown - or _generation_superseded(self, my_generation) - or self._cancel_event.is_set() - ): + self.ui.on_info( + f"[stream died mid-response ({cause}) — retrying in " + f"{delay:.0f}s ({attempt}/{self._MID_STREAM_RETRIES})]" + ) + try: + self._backoff_or_cancelled(delay, my_generation) + # Retry preparation is one generation publication. A + # successor can claim immediately after the backoff check; + # without this lock the abandoned worker could discard the + # successor's buffer, restart its spinner, and clear its + # newly registered provider handle before noticing it was + # stale on the next loop iteration. + with self._generation_lock: + if ( + self._publication_shutdown + or _generation_superseded(self, my_generation) + or self._cancel_event.is_set() + ): + raise GenerationCancelled() + # Retry is proceeding: truncate the dead segment from + # the multi-segment turn buffer (the IDLE payload's + # source) and reset the inflight snapshot BEFORE any + # retried token lands, or every consumer appends the + # retried text onto the dead attempt's. + self._ui_stream_discarded() + # Spinner for the recreate+TTFT window, and the fresh + # segment watermark — AFTER the truncate, so a later + # discard cannot resurrect this dead segment. A + # pre-first-token death leaves the spinner RUNNING + # (_stop_spinner_once never fired) — on_thinking_start + # is idempotent at the callee. + self.ui.on_thinking_start() + self._cancel_stream = None # drop the dead SDK handle + # A concurrent ModelRegistry.reload() closes cached + # clients whose connection config changed — the + # in-flight read then dies with a ReadError and + # the old lane's client is CLOSED. Generation-gated (two compares + # when nothing changed); the next attempt's + # ``prepare_wire`` closure re-prepares against whatever + # binding the walk resolves. + self._refresh_model_from_registry() + except GenerationCancelled: + # A Stop landing in the backoff window aborts the turn + # with the dead attempt's partial preserved — the same + # disposition a cancel DURING the attempt gets. _promote_dead_partial() - raise GenerationCancelled() from None - self.ui.on_stream_end() - self.ui.on_info( - f"[stream died mid-response ({cause}) — retrying in " - f"{delay:.0f}s ({attempt}/{self._MID_STREAM_RETRIES})]" - ) - try: - self._backoff_or_cancelled(delay, my_generation) - # Retry preparation is one generation publication. A - # successor can claim immediately after the backoff check; - # without this lock the abandoned worker could discard the - # successor's buffer, restart its spinner, and clear its - # newly registered provider handle before noticing it was - # stale on the next loop iteration. - with self._generation_lock: - if ( - self._publication_shutdown - or _generation_superseded(self, my_generation) - or self._cancel_event.is_set() - ): - raise GenerationCancelled() - # Retry is proceeding: truncate the dead segment from - # the multi-segment turn buffer (the IDLE payload's - # source) and reset the inflight snapshot BEFORE any - # retried token lands, or every consumer appends the - # retried text onto the dead attempt's. - self._ui_stream_discarded() - # Spinner for the recreate+TTFT window, and the fresh - # segment watermark — AFTER the truncate, so a later - # discard cannot resurrect this dead segment. A - # pre-first-token death leaves the spinner RUNNING - # (_stop_spinner_once never fired) — on_thinking_start - # is idempotent at the callee. - self.ui.on_thinking_start() - self._cancel_stream = None # drop the dead SDK handle - # A concurrent ModelRegistry.reload() closes cached - # clients whose connection config changed — the - # in-flight read then dies with a ReadError and - # the old lane's client is CLOSED. Generation-gated (two compares - # when nothing changed); the next attempt's - # ``prepare_wire`` closure re-prepares against whatever - # binding the walk resolves. - self._refresh_model_from_registry() - except GenerationCancelled: - # A Stop landing in the backoff window aborts the turn - # with the dead attempt's partial preserved — the same - # disposition a cancel DURING the attempt gets. - _promote_dead_partial() - raise + raise def _finalize_stream_result(self, result: ModelTurnResult) -> ModelTurnResult: """Post-drain policies for a COMPLETED interactive turn. @@ -10392,7 +12895,7 @@ class ChatSession: native = ProviderNative(producer=old_native.producer, blocks=tuple(blocks)) result = dataclasses.replace( result, - turn=Turn.assistant(result.turn.text, native=native), + turn=dataclasses.replace(result.turn, tool_calls=(), native=native), tool_calls=[], ) elif finish_reason == "content_filter": @@ -10420,12 +12923,16 @@ class ChatSession: tools=[tc["function"]["name"] for tc in result.tool_calls], ) + provenance = result.provenance log.debug( "stream.finished", finish_reason=finish_reason, has_content=bool(result.content), tool_call_count=len(result.tool_calls), content_length=len(result.content), + alias=provenance.model_alias, + model=provenance.backend_model_id, + registry_generation=provenance.registry_generation, ) self.ui.on_stream_end() return result @@ -10586,6 +13093,7 @@ class ChatSession: *, msgs: list[dict[str, Any]] | None = None, tool_def_chars: int | None = None, + provenance: TurnProvenance | None = None, ) -> None: """Update per-message token estimates using API usage data. @@ -10633,6 +13141,19 @@ class ChatSession: elif text_chars > 0: self._chars_per_token = text_chars / text_prompt_tok + calibration_key = ( + self._provenance_calibration_key(provenance) + or self._active_token_calibration_key + or self._token_calibration_key(self._primary_lane()) + ) + self._active_token_calibration_key = calibration_key + self._last_usage_calibration_key = calibration_key + self._token_calibrations[calibration_key] = _TokenCalibration( + chars_per_token=self._chars_per_token, + prompt_tokens=prompt_tok, + message_prefix_ids=tuple(id(message) for message in self.messages), + ) + # Compute system_tokens (stable after first call) sys_chars = sum(self._msg_char_count(m) for m in self.system_messages) self._system_tokens = max(1, int(sys_chars / self._chars_per_token)) @@ -11039,7 +13560,13 @@ class ChatSession: self._compaction_event( my_generation, {"phase": "progress", "warning": "summary_truncated"} ) - return _SummaryResult(text=summary, producer=result.producer) + return _SummaryResult( + text=summary, + producer=result.producer, + # Lightweight seam fakes predating provenance remain valid; every + # production ModelTurnResult carries the field. + provenance=getattr(result, "provenance", TurnProvenance()), + ) def _summarize_blocks( self, @@ -11423,6 +13950,14 @@ class ChatSession: raise GenerationCancelled() try: return self._compact_messages_impl(auto, preserve_tail, my_generation, carry_spill) + except ConversationPersistenceError: + # The compaction state swap and successful END were already + # accepted atomically with a pending marker row. Durability poison + # must propagate to stop causal suffixes, but it is not a second + # compaction outcome: emitting a failed END here would violate the + # exactly-one terminal event contract and contradict the pending + # successful history card. + raise except BaseException as e: # GenerationCancelled is a BaseException — the except above the # message swap lets it propagate so history stays untouched; the @@ -11875,6 +14410,7 @@ class ChatSession: "role": "assistant", "content": summary, "_source": COMPACTION_SOURCE, + "_provenance": summary_result.provenance.to_meta(), } su_tok = max(1, int(self._msg_char_count(summary_user) / self._chars_per_token)) sa_tok = max(1, int(self._msg_char_count(summary_asst) / self._chars_per_token)) @@ -11903,60 +14439,134 @@ class ChatSession: "prompt_tokens": after_tokens, "total_tokens": after_tokens, } + active_key = self._active_token_calibration_key + if active_key is not None: + active = self._token_calibrations.get(active_key) + self._token_calibrations[active_key] = _TokenCalibration( + chars_per_token=( + active.chars_per_token if active is not None else self._chars_per_token + ), + prompt_tokens=after_tokens, + message_prefix_ids=tuple(id(message) for message in compacted_messages), + ) + if self._last_usage is not None: + self._last_usage_calibration_key = active_key - # The successful end event carries everything a UI needs to paint - # the result card; its id stamps the marker row below so /history - # and the live stream stay aligned. - end_event_id = self._compaction_event( - my_generation, - { - "phase": "end", - "ok": True, - "trigger": trigger, - "before_tokens": before_tokens, - "after_tokens": after_tokens, - "summary": summary, - }, - ) - # Persist a checkpoint so reopen rehydrates [summary]+[tail] rather - # than the full transcript. Storage retains the complete audit - # history; this marker governs only the resume slice. + # Persist a checkpoint so reopen rehydrates [summary]+[tail] + # rather than the full transcript. Storage retains the complete + # audit history; this marker governs only the resume slice. The + # live display projection is the same SYSTEM/source=compaction + # shape produced by ``include_compaction=True`` while storage + # retains the canonical ASSISTANT marker role. if self._ws_id: persist_ws_id = self._ws_id - persist_event_id = end_event_id if end_event_id is not None else self._ui_event_id() + commit_key = uuid.uuid4().hex + event_id_ref: list[int | None] = [None] + marker_meta: dict[str, Any] = { + "before_tokens": before_tokens, + "after_tokens": after_tokens, + "trigger": trigger, + } + if summary_result.producer: + marker_meta["summary_producer"] = summary_result.producer + marker_meta[PROVENANCE_META_KEY] = summary_result.provenance.to_meta() + # The public card keeps checkpoint/display fields but never + # exposes the private provenance/principal envelope. + display_meta = { + key: copy.deepcopy(value) + for key, value in marker_meta.items() + if key != PROVENANCE_META_KEY + } + pending_marker = { + "role": "system", + "content": summary, + "_source": COMPACTION_SOURCE, + "_source_meta": display_meta, + } - def _persist_compaction_marker() -> None: - # This read is part of the same ordered durable batch as - # the marker insert. Every earlier generation batch has - # completed and no later batch can overtake it, so the - # watermark names exactly the prefix summarized above. - watermark = get_compaction_watermark(persist_ws_id, preserve_tail) - if watermark is None: - return - marker_meta: dict[str, Any] = { - "watermark": watermark, - "before_tokens": before_tokens, - "after_tokens": after_tokens, - "trigger": trigger, - } - # Compaction has no provider-native block lane, so the - # generic ``producer=`` storage envelope cannot carry this - # attribution. Persist the final merge producer in the - # checkpoint marker's metadata instead; #964 owns the - # broader accepted-turn provenance tuple. - if summary_result.producer: - marker_meta["summary_producer"] = summary_result.producer - save_message( - persist_ws_id, + meta_json_cell: list[str | None] = [None] + + def _persist_compaction_marker( + *, + ws_id: str = persist_ws_id, + row_commit_key: str = commit_key, + row_event_id: list[int | None] = event_id_ref, + ) -> int: + # The watermark is read inside the ordered durable batch: + # every earlier generation batch has completed and no + # later batch can overtake it, so the boundary names + # exactly the prefix summarized above — including rows + # that were still pending in the journal when compaction + # was admitted. Memoized because the marker is a KEYED + # save: a lost-ACK retry must present byte-identical meta + # to the conflict validator, and a re-read after the first + # attempt could observe this marker's own row. The read + # deliberately bypasses the memory wrapper's error-to-None + # coercion: a failed read must abort THIS attempt (no + # bytes fixed, no save tried — the journal classifies it + # retrying and the retry re-reads) rather than memoize + # "absent" and durably commit a checkpoint-less marker on + # the first healthy retry. ``None`` from a healthy read is + # the legitimate no-rows case and persists a display-only + # marker; reopen then resumes from full history. + if meta_json_cell[0] is None: + watermark = get_storage().get_compaction_watermark(ws_id, preserve_tail) + persisted_meta = dict(marker_meta) + if watermark is not None: + persisted_meta["watermark"] = watermark + meta_json_cell[0] = json.dumps(persisted_meta) + return save_message( + ws_id, "assistant", summary, source=COMPACTION_SOURCE, - meta=json.dumps(marker_meta), - event_id=persist_event_id, + meta=meta_json_cell[0], + event_id=row_event_id[0], producer=summary_result.producer, + commit_key=row_commit_key, ) - durable.append(_persist_compaction_marker) + # END publication, fallback repair, marker admission, and + # revision advance are one REST/SSE handoff transition. Stamp + # after every emitted event so the row cursor covers it. + with self._history_handoff_lock: + end_event_id = self._compaction_event( + my_generation, + { + "phase": "end", + "ok": True, + "trigger": trigger, + "before_tokens": before_tokens, + "after_tokens": after_tokens, + "summary": summary, + }, + ) + if end_event_id is None and _active_task_agent_cancel_scope.get() is None: + self._request_history_resync_locked("compaction_checkpoint_accepted") + event_id_ref[0] = ( + end_event_id if end_event_id is not None else self._ui_event_id() + ) + pending = self._journal_conversation_row_locked( + commit_key=commit_key, + message=pending_marker, + persist=_persist_compaction_marker, + event_id=event_id_ref[0], + ) + self._schedule_conversation_persistence(pending, durable) + else: + # Unscoped CLI/eval sessions have no history row, but retain + # the successful lifecycle event. + self._compaction_event( + my_generation, + { + "phase": "end", + "ok": True, + "trigger": trigger, + "before_tokens": before_tokens, + "after_tokens": after_tokens, + "summary": summary, + }, + ) if not self._commit_for_generation( my_generation, @@ -13248,6 +15858,8 @@ class ChatSession: attachment_ids: list[str] | tuple[str, ...] | None = None, queue_msg_id: str | None = None, interjector_user_id: str = "", + turn_principal_id: str | None = None, + client_send_id: str = "", ) -> tuple[str, str, str]: """Queue a user message for injection at the next tool-result seam. @@ -13271,9 +15883,19 @@ class ChatSession: credentials and be misattributed to them. The interjector must wait and send a fresh turn under their own identity. + ``turn_principal_id`` is the immutable owner captured with the active + worker-slot claim. HTTP/coordinator queue paths pass it during the + short window before ``bind_acting_user`` updates the mutable session + actor. Omitting it preserves the direct/CLI behavior and compares + against the current effective actor. + ``queue_msg_id`` lets the caller supply the id (so it matches the ``send_id`` tracking token threaded through the send) — when omitted, an id is generated. + + ``client_send_id`` is retained independently as one-shot browser + correlation. It follows the text through a combined queue flush but + never affects queue/storage identity or delivery semantics. """ from turnstone.core.tool_advisory import parse_priority @@ -13284,10 +15906,21 @@ class ChatSession: ) interjector = (interjector_user_id or "").strip() - if interjector and interjector != self._mcp_effective_user_id: + active_principal = ( + (self._mcp_effective_user_id or "").strip() + if turn_principal_id is None + else (turn_principal_id or "").strip() + ) + if interjector and interjector != active_principal: # Only an authenticated non-acting participant is blocked: the # acting user interjecting their own in-flight turn is fine, and - # unauthenticated lanes (empty id) keep the pre-existing behaviour. + # an unauthenticated interjector (empty id) keeps the pre-existing + # internal-lane behaviour. An EMPTY turn principal fails closed: + # folding an authenticated send into an unowned in-flight turn + # (init worker, internal wake on an ownerless session) would run + # the words under the ambient credential context and misattribute + # them to that turn's initiator — exactly what this guard exists + # to prevent. raise CrossUserInterjectionError( "Another participant's turn is in flight — wait for it to " "finish, then send your message so it runs under your own " @@ -13307,23 +15940,97 @@ class ChatSession: with self._queued_lock: if len(self._queued_messages) >= self._QUEUE_MAX: raise queue.Full() - self._queued_messages[msg_id] = (cleaned, priority) + owner = interjector or (active_principal if turn_principal_id is not None else "") + self._queued_messages[msg_id] = ( + (cleaned, priority, owner, client_send_id) + if client_send_id + else (cleaned, priority, owner) + ) return cleaned, priority, msg_id + def has_foreign_queued_messages(self, principal_id: str) -> bool: + """Whether retained queued text belongs to another participant. + + Called under the workstream worker-slot lock before a fresh worker is + claimed. Empty owners are legacy/internal entries and remain + compatible with every principal; authenticated entries are exact. + """ + principal = (principal_id or "").strip() + if not principal: + return False + with self._queued_lock: + return self._has_foreign_queued_messages_locked(principal) + + def claim_pending_interjection_wake( + self, + *, + exclude_signature: object | None = None, + ) -> tuple[tuple[str, tuple[str, ...]], ...] | None: + """Return one compatible queue snapshot token for an automatic wake. + + Called only from a worker's ownership-clear backstop. Budget refusal, + an abandoned generation, and unresolved durable/structural state all + require a later explicit user seam rather than an unattended retry. + ``exclude_signature`` belongs to the exact wake worker now exiting: if + its failed preamble restored the same popped rows, that attempt is not + immediately repeated. The token is otherwise stateless until a wake + runner successfully spawns, so a deferred/refused dispatch cannot + suppress the competing real worker's own exit handoff. + + The workstream-gone latch is its own blocker arm: the gone + discovery CLEARS the pending-commit journal, so the unresolved- + persistence check reads False exactly when the workstream can no + longer accept any turn — without this arm the claim admits a wake + that is structurally guaranteed to fail, burning wake nudges (and, + via send()'s internally-converged cancel, destroying the popped + rows) at every subsequent worker exit. Refusal keeps the rows + retained; their disposition on a deleted workstream is #1001's. + Checked before ``_queued_lock`` like the persistence probe — both + take other locks and must not nest inside the queue lock. + """ + if ( + self._budget_exhausted + or self._generation_abandoned + or self.is_workstream_gone() + or self.has_unresolved_conversation_persistence() + ): + return None + principal = (self._mcp_effective_user_id or "").strip() + with self._queued_lock: + if not self._queued_messages: + return None + if principal and self._has_foreign_queued_messages_locked(principal): + return None + signature = tuple( + (message_id, tuple(row)) for message_id, row in self._queued_messages.items() + ) + return None if signature == exclude_signature else signature + + def _has_foreign_queued_messages_locked(self, principal_id: str) -> bool: + """Locked half of :meth:`has_foreign_queued_messages`.""" + return any( + (owner := _queued_row_owner(row)) and owner != principal_id + for row in self._queued_messages.values() + ) + def dequeue_message(self, msg_id: str) -> bool: """Remove a queued message by ID. Returns True if removed. - A miss is RECORDED, not just reported: during the wake handoff's - in-flight send the items live in the dispatcher's hands, so the - user's retraction cannot reach the queue — the ledger lets a - failure-path restore honour it instead of resurrecting a message - the user cancelled. (A miss for an id that was simply already - delivered records harmlessly: the ledger is cleared at each pop - and consulted only by the restore.) + A miss for an id an OPEN pop window holds in flight is RECORDED, + not just reported: during the wake handoff's in-flight send the + items live in the dispatcher's hands, so the user's retraction + cannot reach the queue — the ledger lets a failure-path restore + honour it instead of resurrecting a message the user cancelled. + A miss for any other id (already delivered, never queued at all) + records NOTHING: under per-id ledger discipline no pop, restore, + or sweep would ever prune such an entry, so unconditional + recording grew the set for the session's whole lifetime — one + permanent entry per retract-after-delivery, unbounded for any + authenticated writer looping DELETEs with invented ids. """ with self._queued_lock: popped = self._queued_messages.pop(msg_id, None) - if popped is None: + if popped is None and msg_id in self._popped_in_flight: self._retracted_while_popped.add(msg_id) return popped is not None @@ -13341,6 +16048,80 @@ class ChatSession: """ return self._flush_queued_messages() + def _drain_queue_for_identity_swap(self) -> None: + """Settle the queue before /new or /resume swaps this session's ws_id. + + TOTAL: it must be impossible for the escape commands to raise out of + here — the CLI dispatches them uncaught, and an unhealthy journal + must never block leaving a workstream (round-5 review). Stranded OWN + text persists into the CURRENT workstream when the journal can accept + it; everything that cannot land is discarded WITH an accurate notice, + never carried across the swap. The except arm is deliberately + ``(GenerationCancelled, Exception)`` rather than a curated tuple — + the direct-flush raise surface (reconcile poison gate, commit + refusal, journal admission internals, storage drivers) is open-ended, + and a curated tuple is exactly how the last "never raises" version + ended up raising. + """ + with self._queued_lock: + if not self._queued_messages: + # Skip even the flush's direct-mode preamble: the reconcile + # poison gate can raise with nothing queued at all. + self._retracted_while_popped.clear() + self._popped_in_flight.clear() + return + if self.is_workstream_gone(): + with self._queued_lock: + dropped = len(self._queued_messages) + self._queued_messages.clear() + self._retracted_while_popped.clear() + self._popped_in_flight.clear() + if dropped: + self.ui.on_info( + f"Discarded {dropped} queued message(s) — the workstream was " + "deleted and can no longer save them." + ) + return + flushed_ok = True + try: + self._flush_queued_messages() + except (GenerationCancelled, Exception): + flushed_ok = False + log.warning("session.identity_swap_flush_failed ws=%s", self._ws_id[:8], exc_info=True) + principal = (self._mcp_effective_user_id or "").strip() + with self._queued_lock: + own = 0 + foreign = 0 + for row in self._queued_messages.values(): + owner = _queued_row_owner(row) + if principal and owner and owner != principal: + foreign += 1 + else: + own += 1 + self._queued_messages.clear() + self._retracted_while_popped.clear() + # No pop window can be open here (CLI-only path, single REPL + # thread, no CLI wake lane), so this is a belt-and-braces + # invariant, not a live close: a stale in-flight id must not + # outlive the identity swap and suppress a legitimate later + # message on the NEW workstream. + self._popped_in_flight.clear() + # Per-partition notices: after a SUCCESSFUL flush the leftovers are + # provably another participant's; after a failed flush the actor's + # own unsaved rows must never be counted as someone else's. + if own: + self.ui.on_info( + f"Discarded {own} of your queued message(s) — they could not be " + "saved before leaving this workstream." + ) + if foreign: + self.ui.on_info( + f"Discarded {foreign} queued message(s) from another participant — " + "they cannot follow into a different workstream." + ) + if not flushed_ok and not own and not foreign: + self.ui.on_info("Queued messages could not be saved and were discarded.") + def compact_now(self, *, principal_id: str | None = None) -> bool: """Manual compaction with send()'s full generation discipline. @@ -13374,10 +16155,20 @@ class ChatSession: pass it explicitly; local/direct callers capture the session's current effective user once before claiming the compaction generation. """ + worker_claim = current_worker_claim(self) compact_principal = ( - (self._mcp_effective_user_id or "") if principal_id is None else principal_id + ( + worker_claim.principal_id + if worker_claim is not None + else (self._mcp_effective_user_id or "") + ) + if principal_id is None + else principal_id ).strip() - my_generation = self._claim_generation(principal_id=compact_principal) + my_generation = self._claim_generation( + principal_id=compact_principal, + expected_cancel_epoch=(worker_claim.cancel_epoch if worker_claim is not None else None), + ) cancel_landed = False try: compacted = self._compact_messages(my_generation=my_generation) @@ -13433,48 +16224,122 @@ class ChatSession: Returns ``True`` when any user row was appended (prefix or items), ``False`` when both were empty. """ - queued_text = self._pop_queued_messages_text() - if not queued_text and not prefix: - return False + if deferred_persistence is None: + return bool( + self._direct_commit_reentry( + "Cannot flush queued messages on a closed session", + lambda durable: self._flush_queued_messages( + prefix, + deferred_persistence=durable, + ), + ) + ) + queued_items = self._pop_queued_messages() + try: + queued_text = self._render_queued_messages(queued_items) + client_send_ids = self._client_send_ids_from_queued(queued_items) + if not queued_text and not prefix: + return False - if prefix and queued_text: - content = prefix + "\n\n" + queued_text - elif prefix: - content = prefix - else: - content = queued_text - self._append_user_turn( - content, - (), - deferred_persistence=deferred_persistence, - ) - return True + if prefix and queued_text: + content = prefix + "\n\n" + queued_text + elif prefix: + content = prefix + else: + content = queued_text + self._append_user_turn( + content, + (), + deferred_persistence=deferred_persistence, + client_send_ids=client_send_ids, + ) + return True + finally: + # Every flush exit closes its window: appended (committed), + # husks-only (deliberately discarded), or a raise out of the + # append (no restore path exists on the flush seams). No + # restore happens in this frame, so the unconditional close + # cannot strip a marker a restore re-armed. + self._close_pop_window(queued_items) - def _pop_queued_messages(self) -> dict[str, tuple[str, str]]: - """Atomically drain ``_queued_messages``, returning the raw items. + def _pop_queued_messages( + self, + ) -> dict[str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str]]: + """Partition-pop ``_queued_messages`` by owner, returning the raw items. - The pop happens under ``_queued_lock`` and is destructive: the - caller owns delivery of whatever comes back, and a caller whose - delivery can fail restores the SAME mapping via - :meth:`_restore_queued_messages` — ids and priorities intact, so - the queued-id / send-id correspondence the delete route and the - pending rows rely on survives a failed dispatch. + Pops entries owned by the CURRENT effective actor plus unowned/legacy + entries (``owner == ""`` or a pre-owner 2-tuple); another + participant's entries are STRUCTURALLY retained in place — they can + never leave the queue under a different actor's credentials, which is + the security property the deleted pop-side ownership assert enforced + by raising. Every consumer therefore does the right thing on a + mixed-ownership queue with no per-site mode flag: own text flows, + foreign text waits for its owner's next turn. With an EMPTY current + principal everything pops — an owned row cannot coexist with an empty + actor (admission fails closed on empty principals and the effective + id is sticky), so that arm only ever sees all-unowned queues. + + The pop is destructive for the popped subset: the caller owns + delivery, and a caller whose delivery can fail restores the SAME + mapping via :meth:`_restore_queued_messages` — ids and priorities + intact, so the queued-id / send-id correspondence the delete route + and the pending rows rely on survives a failed dispatch. + + Retraction-ledger discipline is PER-ID, never wholesale: this pop + discards suppression records only for the ids it pops (a fresh pop + of a re-queued id starts a clean window), leaving records that guard + an OUTER pop window intact — the wake handoff holds its popped items + across a full ``send()`` whose own flush seams pop again, and a + wholesale clear here would destroy the outer window's retraction and + resurrect a message the user cancelled. """ + principal = (self._mcp_effective_user_id or "").strip() with self._queued_lock: - items = dict(self._queued_messages) - self._queued_messages.clear() - self._retracted_while_popped.clear() - return items + popped: dict[ + str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str] + ] = {} + for mid, row in self._queued_messages.items(): + owner = _queued_row_owner(row) + if principal and owner and owner != principal: + continue + popped[mid] = row + for mid in popped: + del self._queued_messages[mid] + self._retracted_while_popped.discard(mid) + # Window OPEN, atomic with the delete: a gap would let a + # DELETE race in, see the id in neither structure, record + # nothing — and the restore would resurrect the retracted + # message. Exactly one close per window: the restore (for + # the ids it considered) or ``_close_pop_window`` on every + # success / deliberate-discard / exception exit. + self._popped_in_flight.add(mid) + return popped - def _restore_queued_messages(self, items: dict[str, tuple[str, str]]) -> None: + def _restore_queued_messages( + self, + items: dict[str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str]], + ) -> None: """Put popped items back at the FRONT of ``_queued_messages``. The undo half of :meth:`_pop_queued_messages`, for a dispatcher whose delivery failed before any turn was appended. Restores unconditionally — the items were already admitted, so ``_QUEUE_MAX`` (an admission gate, not a storage invariant) does - not re-apply — and ahead of anything queued meanwhile, keeping - arrival order. + not re-apply — and ahead of anything queued meanwhile. With + partitioned pops "meanwhile" can include another participant's + RETAINED rows that arrived earlier; restored rows still merge to the + front (bounded: different owners never render in the same turn, so + cross-owner order is presentation-free). Ledger discipline is + per-id: only the suppression records this restore consumed (the ids + it considered) are discarded, never records guarding another pop + window. The restore IS this window's close: the in-flight + removal covers every id it CONSIDERED (including the retracted + subset it did not re-insert — a restored-subset-only removal + would leak one in-flight id per honoured retraction), atomically + with the ledger consume under one lock acquisition, so a re-pop + by a new window cannot be stripped by the old window's later + cleanup. A caller whose restore ran must NOT also run + ``_close_pop_window`` for the same ids. """ with self._queued_lock: merged = { @@ -13483,10 +16348,33 @@ class ChatSession: merged.update(self._queued_messages) self._queued_messages.clear() self._queued_messages.update(merged) - self._retracted_while_popped.clear() + self._retracted_while_popped.difference_update(items.keys()) + self._popped_in_flight.difference_update(items.keys()) + + def _close_pop_window( + self, + items: dict[str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str]], + ) -> None: + """Close a pop window that ends WITHOUT a restore. + + Success-commit, deliberate-discard (gone latch, cancel + no-restore, content-free husks), and exception escapes all end + here — per-id, so a nested window's close never touches an outer + window's ids. Callers whose failure arm restored must skip this + for the restored window: the restore already closed those ids, + and a later blanket discard could strip a marker a NEW window + now owns (re-enabling the resurrection bug the ledger exists to + prevent). + """ + if not items: + return + with self._queued_lock: + self._popped_in_flight.difference_update(items.keys()) @staticmethod - def _render_queued_messages(items: dict[str, tuple[str, str]]) -> str: + def _render_queued_messages( + items: dict[str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str]], + ) -> str: """The one rendering of popped interjection items as USER-turn content — ``[IMPORTANT]``-prefixed per item priority, items joined by blank lines — shared by :meth:`_flush_queued_messages` @@ -13503,16 +16391,17 @@ class ChatSession: from turnstone.core.tool_advisory import PRIORITY_IMPORTANT return "\n\n".join( - f"[IMPORTANT] {msg}" if pri == PRIORITY_IMPORTANT else msg - for msg, pri in items.values() - if msg.strip() + f"[IMPORTANT] {row[0]}" if row[1] == PRIORITY_IMPORTANT else row[0] + for row in items.values() + if row[0].strip() ) - def _pop_queued_messages_text(self) -> str: - """Pop-and-render in one step, for the flush seams whose delivery - cannot fail between pop and append (:meth:`_flush_queued_messages` - appends the turn immediately).""" - return self._render_queued_messages(self._pop_queued_messages()) + @staticmethod + def _client_send_ids_from_queued( + items: dict[str, tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str]], + ) -> tuple[str, ...]: + """Return non-empty browser correlation ids in queue arrival order.""" + return tuple(csid for row in items.values() if (csid := _queued_row_client_send_id(row))) def _collect_advisories( self, @@ -13565,26 +16454,51 @@ class ChatSession: # and tool/any-channel metacog nudges. Both fire once per batch so a # parallel fan-out doesn't paint the same advisory N times. if is_last_in_batch: - with self._queued_lock: - queued_items = list(self._queued_messages.values()) - self._queued_messages.clear() - for text, priority in queued_items: - # Drop empty/whitespace interjections (e.g. a bare "!!!" whose - # priority prefix ``parse_priority`` strips to "") — an empty - # operator turn would fold to an empty fence / paint a blank - # bubble. Frame the rest as the user's words so the turn keeps - # user (not operator) authority, including on the native path - # where it enters as a real role=system message. - if not text.strip(): - continue - # ``framed`` (preamble + "User message: …") is the model-facing - # content — it keeps the user's authority framing on the wire. - # The structured meta carries the user's RAW words + priority so - # the FE renders a clean "queued message" bubble (the operator - # reads the message, not the model-directed preamble) with - # priority emphasis. Both derive from ``(text, priority)``. - framed = render_user_interjection(text, priority) - specs.append(("user_interjection", framed, {"priority": priority, "message": text})) + # THE partitioned pop — not an inline copy of it: only the acting + # principal's (and unowned) rows become advisory specs; another + # participant's retained rows stay queued and produce NO spec — + # an advisory is a model-facing turn, and announcing another + # participant's pending text would leak their activity into this + # actor's transcript with no sender-scoped rendering on the + # model side. Silence + retention is the correct exit. Sharing + # the method keeps the security-relevant partition predicate and + # the ledger/window bookkeeping single-sourced. + popped = self._pop_queued_messages() + try: + for row in popped.values(): + text, priority = row[:2] + # Drop empty/whitespace interjections (e.g. a bare "!!!" + # whose priority prefix ``parse_priority`` strips to "") + # — an empty operator turn would fold to an empty fence / + # paint a blank bubble. Frame the rest as the user's + # words so the turn keeps user (not operator) authority, + # including on the native path where it enters as a real + # role=system message. + if not text.strip(): + continue + # ``framed`` (preamble + "User message: …") is the + # model-facing content — it keeps the user's authority + # framing on the wire. The structured meta carries the + # user's RAW words + priority so the FE renders a clean + # "queued message" bubble (the operator reads the + # message, not the model-directed preamble) with priority + # emphasis. Both derive from ``(text, priority)``. + framed = render_user_interjection(text, priority) + interjection_meta = {"priority": priority, "message": text} + sender = _queued_row_owner(row) + if sender: + interjection_meta["sender"] = sender + client_send_id = _queued_row_client_send_id(row) + if client_send_id: + interjection_meta["client_send_id"] = client_send_id + specs.append(("user_interjection", framed, interjection_meta)) + finally: + # The advisory lane has NO restore path — a raise between + # here and the caller's ``_append_system_turn`` loses the + # specs with the closure — so the window closes at this + # seam's exit: a longer window buys nothing and only lets + # retractions accumulate against ids nothing will restore. + self._close_pop_window(popped) # Metacognitive tool-channel drain. Queued by # ``_queue_tool_advisory`` from the tool_error / repeat @@ -16000,19 +18914,47 @@ class ChatSession: # queued for the user's next real send, where the approval # prompt has someone in front of it, and falls through to the # wake drain so this worker's exit keeps its convergence. - if not self._budget_exhausted: + # The gone latch takes the same rule for the same reason: the + # admission refusal surfaces as GenerationCancelled, which + # send()'s own cancel finalizer converges INTERNALLY (no + # re-raise), so neither except arm below would run and the + # popped rows would be destroyed with no restore, no log, no + # notice. This delivery-site gate must be at least as strong as + # the claim gate that normally protects it — a nudge-driven wake + # reaches here without ever consulting the claim. + if not self._budget_exhausted and not self.is_workstream_gone(): + # Partitioned pop: only the wake lane's effective principal's + # (and unowned) rows fold into the interjection send; another + # participant's retained rows stay queued for their owner and the + # wake falls through to the nudge drain below. popped = self._pop_queued_messages() interjection = self._render_queued_messages(popped) + client_send_ids = self._client_send_ids_from_queued(popped) if interjection: - dropped = self._nudge_queue.clear_channels({WAKE_CHANNEL}) - log.info( - "wake_nudge.interjection_owns_seam ws=%s dropped_wake_nudges=%d", - self._ws_id[:8], - dropped, - ) + # ``appended_before`` first, then EVERYTHING that can raise + # inside the try: a raise out of ``clear_channels`` lands in + # the BaseException arm with nothing appended and restores — + # outside the try it would destroy the popped rows AND leak + # the open window. appended_before = len(self.messages) + window_closed = False try: - self.send(interjection) + dropped = self._nudge_queue.clear_channels({WAKE_CHANNEL}) + log.info( + "wake_nudge.interjection_owns_seam ws=%s dropped_wake_nudges=%d", + self._ws_id[:8], + dropped, + ) + # Queued items that predate browser correlation ids keep + # the established plain ``send(text)`` handoff: some + # embedded session adapters expose exactly that surface + # and do not accept a redundant empty keyword. Building + # the keyword only when there is something to correlate + # keeps that promise with one call shape. + send_kwargs: dict[str, Any] = ( + {"client_send_ids": client_send_ids} if client_send_ids else {} + ) + self.send(interjection, **send_kwargs) except GenerationCancelled: # Same containment as the wake send below: this # method IS the wake worker's run() closure and @@ -16048,18 +18990,33 @@ class ChatSession: # items); a worker-exit re-check of the interjection # queue would close that window structurally. if len(self.messages) == appended_before: + # The restore is this window's close — it removes + # the in-flight markers for every id it considered, + # atomically with its ledger consume. The finally + # below must then SKIP: a blanket close after the + # restore could strip a marker a new window already + # re-popped (the resurrection re-enabler). self._restore_queued_messages(popped) + window_closed = True raise + finally: + if not window_closed: + # Success-commit, the deliberate no-restore cancel + # arm, and appended-then-raised all end the window + # without a restore. + self._close_pop_window(popped) return if popped: # Everything queued was content-free (a bare priority # marker) — nothing to dispatch, nothing worth a turn. - # Fall through to the normal wake drain below. + # Deliberate discard: close the window, then fall through + # to the normal wake drain below. log.info( "wake_nudge.interjection_empty ws=%s discarded=%d", self._ws_id[:8], len(popped), ) + self._close_pop_window(popped) # Two-pass drain: wake-eligible channels first. ``"quiet"`` entries # (external events demoted by a user cancel) ride a wake earned by @@ -22755,25 +25712,27 @@ class ChatSession: self._read_files.clear() self._repeat_detector.clear() self._last_usage = None - self._calibrated_msg_count = 0 + self._invalidate_token_calibration_anchors() self._msg_tokens = [] + self._activate_token_calibration(self._primary_lane()) self.ui.on_info("Context cleared (messages preserved in database).") elif cmd == "/new": from turnstone.core.memory import register_workstream - # Flush any stranded queued text BEFORE the identity swap so it - # is persisted into the workstream it was ADDRESSED to. Sends - # during the command window itself defer in the /send route + # Settle stranded queued text BEFORE the identity swap so it is + # persisted into the workstream it was ADDRESSED to (or discarded + # with a notice when it cannot land — gone latch / foreign owner). + # Sends during the command window itself defer in the /send route # (ws._pending_sends) and never queue; this covers only a # message stranded by a dying send worker's closing race # before this command started. - self._flush_queued_messages() + self._drain_queue_for_identity_swap() self.messages.clear() self._read_files.clear() self._repeat_detector.clear() self._last_usage = None - self._calibrated_msg_count = 0 + self._invalidate_token_calibration_anchors() self._msg_tokens = [] old_ws_id = self._ws_id self._ws_id = uuid.uuid4().hex @@ -22790,6 +25749,7 @@ class ChatSession: self._sender_label_nonce = fence.mint_nonce() self._follow_watch_registration(old_ws_id) self._title_generated = False + self._activate_token_calibration(self._primary_lane()) # The session keeps its persona across /new (the stamp is # re-written by _save_config below) — carry the display slug # onto the fresh row so projections agree with the config. @@ -22831,10 +25791,10 @@ class ChatSession: elif target_id == self._ws_id: self.ui.on_info("Already in that workstream.") else: - # Same pre-swap flush as /new: stranded queued text is - # persisted into the CURRENT workstream before resume() - # swaps this session's identity to the target. - self._flush_queued_messages() + # Same pre-swap settlement as /new: stranded queued text + # persists into the CURRENT workstream (or is discarded + # with a notice) before resume() swaps identities. + self._drain_queue_for_identity_swap() try: resumed: bool | None = self.resume(target_id) except ValueError as exc: @@ -22962,6 +25922,7 @@ class ChatSession: else (cs.get("model.max_tokens") if cs else self.max_tokens) ) self._init_system_messages() + self._activate_token_calibration(self._primary_lane()) self._save_config() self.ui.on_info(f"Switched to {cyan(arg)}: {self.model}") elif construction_error is not None: @@ -23070,12 +26031,26 @@ class ChatSession: self.ui.on_info("\n".join(mcp_lines)) elif cmd == "/retry": - user_msg = self.retry() - if user_msg is None: - self.ui.on_info("Nothing to retry.") + # retry() raises since the durable truncation became raising: + # GenerationCancelled (BaseException) on a refused admission + # (gone latch, racing close) and storage errors from the tail + # delete. The REPL dispatches commands uncaught, so convert to + # messages here instead of killing the process (round-4 review). + try: + user_msg = self.retry() + except GenerationCancelled: + self.ui.on_error( + "Retry refused — the workstream is busy or no longer accepts history changes." + ) + except Exception as exc: + log.warning("cli.retry_failed ws=%s", self._ws_id[:8], exc_info=True) + self.ui.on_error(f"Retry failed: {exc}") else: - self._pending_retry = user_msg - self.ui.on_info(f"Retrying: {user_msg[:80]}...") + if user_msg is None: + self.ui.on_info("Nothing to retry.") + else: + self._pending_retry = user_msg + self.ui.on_info(f"Retrying: {user_msg[:80]}...") elif cmd == "/rewind": if not arg: @@ -23091,14 +26066,25 @@ class ChatSession: else: turns_available = len(self._find_turn_boundaries()) actual_n = min(n, turns_available) - removed = self.rewind(n) - if removed == 0: - self.ui.on_info("No turns to rewind.") - else: - self.ui.on_info( - f"Rewound {actual_n} turn(s) ({removed} messages removed). " - f"{len(self.messages)} messages remain." + # Same raising contract as /retry above. + try: + removed = self.rewind(n) + except GenerationCancelled: + self.ui.on_error( + "Rewind refused — the workstream is busy or " + "no longer accepts history changes." ) + except Exception as exc: + log.warning("cli.rewind_failed ws=%s", self._ws_id[:8], exc_info=True) + self.ui.on_error(f"Rewind failed: {exc}") + else: + if removed == 0: + self.ui.on_info("No turns to rewind.") + else: + self.ui.on_info( + f"Rewound {actual_n} turn(s) ({removed} messages " + f"removed). {len(self.messages)} messages remain." + ) elif cmd == "/help": self.ui.on_info( diff --git a/turnstone/core/session_manager.py b/turnstone/core/session_manager.py index a1f9688a..c8fddb4c 100644 --- a/turnstone/core/session_manager.py +++ b/turnstone/core/session_manager.py @@ -17,10 +17,16 @@ import uuid from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, Protocol +from turnstone.core.adapters._ui_cleanup import _broadcast_ws_closed_to_listeners from turnstone.core.log import get_logger from turnstone.core.model_registry import ModelClientConstructionError, UnknownModelAliasError from turnstone.core.personas import snapshot_from_config -from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState +from turnstone.core.workstream import ( + Workstream, + WorkstreamKind, + WorkstreamState, + concrete_method, +) if TYPE_CHECKING: from collections.abc import Callable, Iterator @@ -33,6 +39,97 @@ if TYPE_CHECKING: log = get_logger(__name__) +def _session_has_unresolved_persistence(session: Any) -> bool: + """Read the concrete ChatSession hook without MagicMock auto-vivification. + + Blocking probe — callable only from contexts holding no workstream or + manager lock (it acquires the session's generation and handoff locks). + Retirement scans use :func:`_session_persistence_blocks_retirement`. + """ + check = concrete_method(session, "has_unresolved_conversation_persistence") + return bool(check()) if check is not None else False + + +def _session_persistence_blocks_retirement(session: Any) -> bool: + """Non-blocking retirement gate: unresolved OR momentarily unprobeable. + + The idle-close and eviction scans call this while holding ``ws._lock`` + (and, for the eviction comprehension, the manager lock). The probe must + therefore never block on the session's generation/handoff locks — that + inverts the generation→workstream/manager order force-cancel's finalizer + and deferred state publication hold, an AB/BA deadlock (round-4 review). + ``None`` (locks busy) reads as True: a busy session is simply not + retirable this sweep; the next sweep re-probes. + """ + probe = concrete_method(session, "has_unresolved_conversation_persistence_nowait") + if probe is None: + # Compatibility/test doubles carry no real locks; the blocking read + # is safe and preserves their scripted answers. + return _session_has_unresolved_persistence(session) + state = probe() + return state is None or bool(state) + + +def _session_unresolved_persistence_nowait(session: Any) -> bool | None: + """Non-blocking unresolved probe for the per-second reconcile walk. + + The steady-state pass concludes "nothing to do" on almost every + workstream almost every second; taking each session's generation and + handoff locks to learn that contends the very locks turn commits use, + forever, scaling with roster size. Try-acquire instead: ``None`` + (locks busy) means a turn owns the session right now — by definition + not a moment that needs an unattended repair — and the next one-second + pass re-probes. Compatibility/test doubles without the nowait hook + keep the blocking read and their scripted answers. + """ + probe = concrete_method(session, "has_unresolved_conversation_persistence_nowait") + if probe is None: + return _session_has_unresolved_persistence(session) + state = probe() + return None if state is None else bool(state) + + +def _session_prepare_soft_close(session: Any) -> bool: + """Run the concrete close fence; compatibility/test doubles have no hook.""" + prepare = concrete_method(session, "prepare_soft_close") + if prepare is None: + return not _session_has_unresolved_persistence(session) + return bool(prepare()) + + +def _session_reconcile_unresolved_persistence_if_due(session: Any, now: float) -> bool: + """Call the concrete retry seam without MagicMock auto-vivification.""" + reconcile = concrete_method(session, "reconcile_unresolved_persistence_if_due") + return bool(reconcile(now=now)) if reconcile is not None else False + + +def _session_conversation_persistence_fatal_revision(session: Any) -> int | None: + """Return a concrete session's exact persistence-owned fatal revision.""" + read_revision = concrete_method(session, "conversation_persistence_fatal_revision") + revision = read_revision() if read_revision is not None else None + return revision if isinstance(revision, int) and not isinstance(revision, bool) else None + + +def _session_acknowledge_conversation_persistence_recovery( + session: Any, + revision: int, +) -> bool: + """Retire only the exact fatal latch that a manager repair recovered.""" + acknowledge = concrete_method(session, "acknowledge_conversation_persistence_state_recovery") + return bool(acknowledge(revision)) if acknowledge is not None else False + + +def _notify_persistence_state_changed(ui: Any) -> None: + """Refresh a concrete UI projection after manager-owned ERROR recovery.""" + callback = concrete_method(ui, "on_persistence_state_changed") + if callback is None: + return + try: + callback() + except Exception: + log.debug("session_mgr.persistence_state_refresh_failed", exc_info=True) + + class WorkstreamAlreadyExistsError(RuntimeError): """A create request did not acquire a fresh durable workstream id.""" @@ -60,6 +157,7 @@ _KIND_SERVICE_TYPE: dict[WorkstreamKind, str] = { # reservations abandoned for two hours qualify. STALE_CREATE_GRACE_SECONDS = 2 * 60 * 60 STALE_CREATE_SWEEP_INTERVAL_SECONDS = 5 * 60 +PERSISTENCE_RECONCILE_INTERVAL_SECONDS = 1.0 class SessionKindAdapter(Protocol): @@ -230,6 +328,12 @@ class SessionManager: self._model_validator = model_validator self._node_id = node_id self._workstreams: dict[str, Workstream] = {} + # A hard-delete whose storage outcome is ambiguous may be the sole + # owner of an accepted conversation repair journal. Keep that exact + # terminal object off every user/capacity surface until maintenance or + # an explicit delete retry proves it safe to retire. + self._failed_delete_tombstones: dict[str, Workstream] = {} + self._failed_delete_unadvertised: set[str] = set() # Deferred creates are addressable internally before their pre-commit # transaction finishes. Keep the exact reservation object beyond a # racing close/delete so the rollback cannot ABA-delete a successor @@ -970,7 +1074,7 @@ class SessionManager: try: with open_lock: with self._lock: - if ws_id in self._retiring_ids: + if ws_id in self._retiring_ids or ws_id in self._failed_delete_tombstones: return None existing = self._workstreams.get(ws_id) if existing is not None and self._pending_creates.get(ws_id) is existing: @@ -1482,7 +1586,11 @@ class SessionManager: token, so a delete/re-register ABA cannot erase the replacement row. """ with self._lock: - candidate = self._workstreams.get(ws_id) or self._pending_creates.get(ws_id) + candidate = ( + self._workstreams.get(ws_id) + or self._pending_creates.get(ws_id) + or self._failed_delete_tombstones.get(ws_id) + ) if candidate is not None and ( candidate._create_publication_active and candidate._create_publication_thread == threading.get_ident() @@ -1497,7 +1605,10 @@ class SessionManager: with candidate._lifecycle_lock: with self._lock: - if self._workstreams.get(ws_id) is not candidate: + if ( + self._workstreams.get(ws_id) is not candidate + and self._failed_delete_tombstones.get(ws_id) is not candidate + ): return False token_direction_needed = bool( expected_reservation_token @@ -1510,6 +1621,7 @@ class SessionManager: # Resolve that direction without the global manager mutex: the # per-id + object lifecycle lanes stabilize ``candidate`` while a # database row lock may legitimately block. + current_row: dict[str, Any] | None = None current_token = "" if token_direction_needed: current_row = self._storage.ensure_workstream_incarnation_snapshot(ws_id) @@ -1520,7 +1632,10 @@ class SessionManager: ) with self._lock: - if self._workstreams.get(ws_id) is not candidate: + if ( + self._workstreams.get(ws_id) is not candidate + and self._failed_delete_tombstones.get(ws_id) is not candidate + ): return False # * durable == local: request is stale; leave local untouched # * durable == expected: local is stale; retire it, then let @@ -1531,7 +1646,22 @@ class SessionManager: return False if current_token != expected_reservation_token: return False - was_unadvertised = self._pending_creates.get(ws_id) is candidate + deleting_authorized_successor = bool( + token_direction_needed and current_token == expected_reservation_token + ) + was_unadvertised = ( + False + if deleting_authorized_successor + else ( + self._pending_creates.get(ws_id) is candidate + or ws_id in self._failed_delete_unadvertised + ) + ) + delete_event_name = name or ( + str(current_row.get("name") or "") + if deleting_authorized_successor and current_row is not None + else candidate.name + ) candidate._lifecycle_terminal_active = True self._retain_state_tail_locked(candidate) @@ -1552,7 +1682,18 @@ class SessionManager: None, ) if callable(drain_durability): - drain_durability() + try: + drain_durability() + except BaseException: + # A successful exact delete makes an unresolved + # conversation repair irrelevant. Continue to that + # authoritative operation; an ambiguous outcome below + # retains the journal tombstone. + log.warning( + "session_mgr.delete_persisted.terminal_repair_failed ws=%s", + ws_id[:8], + exc_info=True, + ) # Drain every admitted predecessor state write before the hard # delete. The per-id lifecycle lane prevents a successor from @@ -1568,22 +1709,29 @@ class SessionManager: deleted = delete_fn() if not deleted: - # Exact deletion can fail only when the durable row/token - # no longer matches the authorized snapshot. This local - # object is therefore a stale incarnation; reopening it - # would leave the manager serving predecessor state over a - # same-id replacement. Retire it silently (the replacement - # was not deleted, so no terminal event is ours to emit). - self._retire_failed_persisted_delete(candidate) + # A conforming exact-delete false normally proves a + # missing/replaced incarnation. Treat it as an ambiguous + # storage outcome nevertheless: a transient implementation + # or wrapper may return false while the same durable row + # survives. Never discard that row's only structural or + # conversation repair owner in the latter case. + self._dispose_ambiguous_failed_delete( + candidate, + was_unadvertised=was_unadvertised, + ) return False with self._lock: - if self._workstreams.get(ws_id) is not candidate: + if ( + self._workstreams.get(ws_id) is not candidate + and self._failed_delete_tombstones.get(ws_id) is not candidate + ): # The id + object lifecycle lanes make this impossible # for conforming paths; never emit against a replacement. candidate._lifecycle_terminal_active = False return False self._workstreams.pop(ws_id, None) + self._drop_delete_tombstone_locked(ws_id) if self._pending_creates.get(ws_id) is candidate: self._pending_creates.pop(ws_id, None) if ws_id in self._order: @@ -1603,20 +1751,88 @@ class SessionManager: self._event_emitter.emit_closed( ws_id, reason="deleted", - name=name or candidate.name, + name=delete_event_name, ) return True except BaseException: - # The terminal claim and publication latch cannot safely be - # rolled back: a concurrent worker may already have observed - # them. Retire this exact local object without publishing a - # false deleted event; if the durable row survived, a later - # open rehydrates a fresh object/incarnation cleanly. - self._retire_failed_persisted_delete(candidate) + self._dispose_ambiguous_failed_delete( + candidate, + was_unadvertised=was_unadvertised, + ) raise finally: self._release_state_tail(candidate) + def _dispose_ambiguous_failed_delete( + self, + candidate: Workstream, + *, + was_unadvertised: bool, + ) -> None: + """One disposition for every ambiguous exact-delete outcome. + + The false-return and raise paths of ``_delete_persisted`` must + stay behaviorally identical: a policy edit applied to one fork + only would let the rarer path silently retire a tombstone that is + the sole owner of an accepted repair journal. + """ + disposition = self._failed_delete_durable_disposition(candidate) + if disposition in {"missing", "different"}: + # Missing/different proves this object's journal can no + # longer repair the durable row. Retire silently: the + # probe is not atomic with lifecycle fan-out, so a remote + # same-id successor could be created before a tokenless + # close event and be erased from collector/client caches. + self._retire_failed_persisted_delete(candidate) + elif _session_has_unresolved_persistence(candidate.session): + # Same or unreadable durable incarnation plus unresolved + # journal is the one lossless failure state: hide it from + # open/create/capacity, retain it for an idempotent exact- + # delete retry, and emit no false close. Its ws-id-only row + # closures must never background-replay across an ABA. + self._retain_failed_persisted_delete_tombstone( + candidate, + was_unadvertised=was_unadvertised, + ) + else: + # The durable prefix is complete, so the historical + # retire-and-rehydrate behavior remains safe. + self._retire_failed_persisted_delete(candidate) + + def _drop_delete_tombstone_locked( + self, + ws_id: str, + *, + candidate: Workstream | None = None, + ) -> bool: + """Retire the (tombstone, unadvertised-flag) PAIR under ``self._lock``. + + The two structures are only ever mutated together: a pop that missed + the flag discard would leave a stale unadvertised marker that + suppresses a later same-id delete's ``ws_closed`` event, and a + discard that outlived an identity-gated pop would strip a REPLACEMENT + tombstone's flag (round-5 review — both halves of the drift). When + ``candidate`` is supplied and a different object holds the tombstone, + neither half is touched. + """ + if candidate is not None and self._failed_delete_tombstones.get(ws_id) is not candidate: + return False + self._failed_delete_tombstones.pop(ws_id, None) + self._failed_delete_unadvertised.discard(ws_id) + return True + + def _retain_delete_tombstone_locked( + self, + ws_id: str, + candidate: Workstream, + *, + was_unadvertised: bool, + ) -> None: + """Install the (tombstone, unadvertised-flag) PAIR under ``self._lock``.""" + self._failed_delete_tombstones[ws_id] = candidate + if was_unadvertised: + self._failed_delete_unadvertised.add(ws_id) + def _retire_failed_persisted_delete(self, candidate: Workstream) -> None: """Silently retire the exact object after a failed hard-delete.""" ws_id = candidate.id @@ -1625,6 +1841,8 @@ class SessionManager: if self._workstreams.get(ws_id) is candidate: self._workstreams.pop(ws_id, None) retired = True + if self._drop_delete_tombstone_locked(ws_id, candidate=candidate): + retired = True if self._pending_creates.get(ws_id) is candidate: self._pending_creates.pop(ws_id, None) if ws_id in self._order: @@ -1642,6 +1860,56 @@ class SessionManager: exc_info=True, ) + def _retain_failed_persisted_delete_tombstone( + self, + candidate: Workstream, + *, + was_unadvertised: bool, + ) -> None: + """Hide an exact terminal object while preserving its repair journal.""" + ws_id = candidate.id + with self._lock: + if self._workstreams.get(ws_id) is candidate: + self._workstreams.pop(ws_id, None) + if self._pending_creates.get(ws_id) is candidate: + self._pending_creates.pop(ws_id, None) + if ws_id in self._order: + self._order.remove(ws_id) + if self._active_id == ws_id: + self._active_id = self._first_visible_id_locked() + self._retain_delete_tombstone_locked( + ws_id, candidate, was_unadvertised=was_unadvertised + ) + ui = candidate.ui + if ui is not None and hasattr(ui, "_listeners_lock"): + # A retained tombstone is terminal to every user-facing surface, + # but cleanup_ui would destroy the session/journal that makes the + # ambiguous delete lossless. Quiesce only its per-workstream SSE + # transports; the sentinel is consumed internally and is not a + # false lifecycle ws_closed event. + _broadcast_ws_closed_to_listeners(ui) + + def _failed_delete_durable_disposition(self, candidate: Workstream) -> str: + """Classify the exact durable incarnation after an ambiguous delete.""" + snapshot = getattr(self._storage, "ensure_workstream_incarnation_snapshot", None) + if not callable(snapshot): + return "unknown" + try: + row = snapshot(candidate.id) + except Exception: + log.warning( + "session_mgr.delete_persisted.snapshot_failed ws=%s", + candidate.id[:8], + exc_info=True, + ) + return "unknown" + if row is None: + return "missing" + durable_token = str(row.get("fork_reservation_token") or "") + if durable_token != candidate._fork_reservation_token: + return "different" + return "same" + # ------------------------------------------------------------------ # close / set_state / close_idle # ------------------------------------------------------------------ @@ -1668,6 +1936,41 @@ class SessionManager: return False with ws._lifecycle_lock: + with self._lock: + if self._workstreams.get(ws_id) is not ws: + return False + # Close admission must become terminal to dispatch before the + # session fence begins. ``prepare_soft_close`` may wait for an + # admitted durability batch; leaving ``_closed`` false across that + # wait lets a racing session_worker claim a fresh slot and report + # the send accepted even though ChatSession will reject its later + # generation claim. Both sides serialize on ``ws._lock``, making + # this the linearization point for close versus dispatch. + with ws._lock: + if ws._closed: + return False + ws._closed = True + ws._state_revision += 1 + + prepared = False + try: + if not _session_prepare_soft_close(ws.session): + # An accepted conversation row is still unresolved. + # Removing the sole live handoff journal would make that + # row disappear on reopen; retain the exact workstream so + # storage recovery can reconcile it idempotently. + return False + prepared = True + finally: + if not prepared: + # The session refused (or raised during) preparation, so + # this incarnation remains live. Advance rather than + # restoring the old revision: a deferred state write that + # observed the temporary tombstone must not regain + # ownership through a revision ABA. + with ws._lock: + ws._closed = False + ws._state_revision += 1 with self._lock: if self._workstreams.get(ws_id) is not ws: return False @@ -1681,11 +1984,9 @@ class SessionManager: if self._active_id == ws_id: self._active_id = self._first_visible_id_locked() - # Publish the in-memory tombstone immediately. Storage and cleanup - # may block, but unrelated workstreams and manager lookups do not. - with ws._lock: - ws._closed = True - ws._state_revision += 1 + # The dispatch tombstone was published before session preparation. + # Storage and cleanup may block, but unrelated workstreams and + # manager lookups do not. try: self._adapter.cleanup_ui(ws) finally: @@ -2124,6 +2425,161 @@ class SessionManager: ) return reaped + def reconcile_unresolved_persistence( + self, + *, + now: float | None = None, + blocking: bool = False, + ) -> list[str]: + """Attempt every due transient conversation repair without manager locks. + + ``blocking`` selects the probe discipline. The default non-blocking + probe suits the per-second maintenance sweep: a workstream whose + generation/handoff locks are momentarily held is skipped and re-probed + on the next pass, so the steady-state walk never contends the locks + turn commits use. A ONE-SHOT caller has no next pass — + ``_reserve_and_install`` runs this exactly once as a last-chance + repair before refusing a create — and must force a definite answer, + or the workstreams most likely to be skipped (the same contended ones + that emptied its candidate list) never get their due repair and the + create fails. Only safe from callers holding no workstream or + manager lock. + + The maintenance owner is shared per process; sessions retain only a + monotonic due timestamp. Permanent commit conflicts deliberately stay + fail-stopped until explicit deletion instead of consuming retry work. + Returns ids whose prefix became durable this pass or whose previously + reconciled persistence-owned ``ERROR`` state was retired, plus hidden + delete tombstones retired after a probe proved their durable + incarnation missing or different. + """ + check_at = time.monotonic() if now is None else now + with self._lock: + candidates = [ + ws + for ws in self._workstreams.values() + if ws.session is not None and self._pending_creates.get(ws.id) is not ws + ] + delete_tombstones = list(self._failed_delete_tombstones.values()) + + repaired: list[str] = [] + for ws in candidates: + with ws._lock: + session = ws.session + if ( + session is None + or ws._closed + or ws._worker_running + or ws._lifecycle_terminal_active + ): + continue + error_state_revision = ( + ws._state_revision if ws.state is WorkstreamState.ERROR else None + ) + fatal_revision = ( + _session_conversation_persistence_fatal_revision(session) + if error_state_revision is not None + else None + ) + if blocking: + unresolved = _session_has_unresolved_persistence(session) + else: + probed = _session_unresolved_persistence_nowait(session) + if probed is None: + # Probe contended: another caller holds the generation/ + # handoff locks this instant. Nothing on this sweep is + # due enough to block a live commit for — the next + # one-second pass re-probes. One-shot callers pass + # ``blocking`` instead; for them there is no next pass. + continue + unresolved = probed + attempted = False + if unresolved: + try: + attempted = _session_reconcile_unresolved_persistence_if_due( + session, + check_at, + ) + except Exception: + log.warning( + "session_mgr.persistence_reconcile_failed ws=%s", + ws.id[:8], + exc_info=True, + ) + continue + recovery_ready = (attempted and not _session_has_unresolved_persistence(session)) or ( + not unresolved and fatal_revision is not None + ) + if recovery_ready: + repaired.append(ws.id) + idle_revision: int | None = None + if fatal_revision is not None: + # A persistence failure is a fatal turn outcome and leaves + # the workstream ERROR. Once its exact journal boundary is + # repaired, retire only that unchanged error state. A new + # worker, successor session, lifecycle tombstone, or any + # intervening state revision wins and keeps its state. + with self._lock: + still_owned = ( + self._workstreams.get(ws.id) is ws + and self._pending_creates.get(ws.id) is not ws + and not ws._lifecycle_terminal_active + ) + if ( + still_owned + and _session_conversation_persistence_fatal_revision(session) + == fatal_revision + ): + with ws._lock: + if ( + ws.session is session + and not ws._closed + and not ws._worker_running + and ws.state is WorkstreamState.ERROR + and ws._state_revision == error_state_revision + ): + self._apply_live_state(ws, WorkstreamState.IDLE, "") + idle_revision = ws._state_revision + if idle_revision is not None: + assert fatal_revision is not None + _session_acknowledge_conversation_persistence_recovery( + session, + fatal_revision, + ) + try: + published = self._run_state_tail( + ws, + idle_revision, + WorkstreamState.IDLE, + ) + except Exception: + log.warning( + "session_mgr.persistence_recovery_state_failed ws=%s", + ws.id[:8], + exc_info=True, + ) + else: + if published: + _notify_persistence_state_changed(ws.ui) + + # Ambiguous hard-delete objects are intentionally absent from the + # ordinary candidate list. A missing/different durable incarnation is + # proof that their predecessor journal can never be applied and may be + # retired. A same/unknown incarnation remains hidden for an explicit + # token-guarded delete retry: captured row closures are keyed only by + # ws_id, so a snapshot-then-background-save would race a remote same-id + # replacement and write predecessor history into it. + for tombstone in delete_tombstones: + with self._id_lifecycle(tombstone.id), tombstone._lifecycle_lock: + with self._lock: + if self._failed_delete_tombstones.get(tombstone.id) is not tombstone: + continue + disposition = self._failed_delete_durable_disposition(tombstone) + if disposition in {"missing", "different"}: + self._retire_failed_persisted_delete(tombstone) + repaired.append(tombstone.id) + return repaired + def close_idle(self, max_age_seconds: float) -> list[str]: """Close IDLE workstreams inactive for more than ``max_age_seconds``. @@ -2267,6 +2723,7 @@ class SessionManager: ws._closed or ws.state is not WorkstreamState.IDLE or ws._worker_running + or _session_persistence_blocks_retirement(ws.session) or (now - ws.last_active) <= max_age_seconds ): return False @@ -2334,9 +2791,10 @@ class SessionManager: same lock, so an IDLE workstream whose turn/command was admitted cannot be evicted between the hint and the terminal claim. """ + persistence_recovery_attempted = False while True: with self._lock: - if ws_id in self._retiring_ids: + if ws_id in self._retiring_ids or ws_id in self._failed_delete_tombstones: raise WorkstreamAlreadyExistsError(f"workstream {ws_id!r} is retiring") if ws_id in self._workstreams or ws_id in self._pending_creates: raise WorkstreamAlreadyExistsError( @@ -2365,10 +2823,19 @@ class SessionManager: and candidate.state is WorkstreamState.IDLE and not candidate._worker_running and not candidate.send_barrier_active() + and not _session_persistence_blocks_retirement(candidate.session) ), key=lambda candidate: candidate.last_active, ) if not candidates: + if not persistence_recovery_attempted: + persistence_recovery_attempted = True + # One shot, and the latch below makes it the only one: + # force a definite probe rather than skipping the + # contended sessions that are the likeliest reason this + # candidate list came back empty. No lock is held here. + self.reconcile_unresolved_persistence(blocking=True) + continue raise RuntimeError(f"All {self._max_active} slots are active") for victim in candidates: @@ -2389,6 +2856,7 @@ class SessionManager: or victim.state is not WorkstreamState.IDLE or victim._worker_running or victim.send_barrier_active() + or _session_persistence_blocks_retirement(victim.session) ): worker_free_idle = False else: diff --git a/turnstone/core/session_routes.py b/turnstone/core/session_routes.py index 20f9128a..21ad3c24 100644 --- a/turnstone/core/session_routes.py +++ b/turnstone/core/session_routes.py @@ -32,6 +32,8 @@ call the factory during startup and pass the result as from __future__ import annotations import asyncio +import functools +import re from collections.abc import Awaitable, Callable, Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Protocol, TypeAlias, cast @@ -41,11 +43,14 @@ from starlette.routing import Route from turnstone.core.log import get_logger from turnstone.core.session_manager import WorkstreamAlreadyExistsError +from turnstone.core.session_replay import session_replay_preamble from turnstone.core.session_ui_base import AutoApproveReason from turnstone.core.workstream import ( INTERJECTION_CAP_CHARS, PENDING_SENDS_MAX, _PendingSend, + concrete_method, + workstream_persistence_state, ) if TYPE_CHECKING: @@ -74,6 +79,7 @@ log = get_logger(__name__) # error verbatim. Length cap + control-char strip keep the message # actionable for legit alias typos while neutralising hostile payloads. _FACTORY_MISCONFIG_MAX_LEN = 200 +_CLIENT_SEND_ID_RE = re.compile(r"[A-Za-z0-9_-]{1,128}\Z") def _safe_factory_misconfig_message(exc: BaseException) -> str: @@ -441,11 +447,10 @@ class SessionEndpointConfig: # (ws, ui, request) -> Iterable[dict]. Kind-specific initial # SSE replay payload the lifted ``events`` body yields after # registering the per-UI listener queue but before the live - # event loop. Interactive replays connected + status + history - # + pending_approval (with cached intent verdicts). Coord replays - # just pending_approval (its dashboard fetches history via a - # separate ``/history`` endpoint and doesn't render the per-tab - # status bar). Kinds that don't need pre-replay wire ``None``. + # event loop. Both production kinds yield connected + optional status, + # followed by pending approval controls and cached verdicts. Conversation + # history stays on the separate REST ``/history`` bootstrap. Kinds that + # don't need pre-replay wire ``None``. events_replay: EventsReplay | None = None # (request) -> Executor for the SSE live-loop's blocking # ``queue.get`` wait. Interactive returns the dedicated @@ -1107,6 +1112,15 @@ def make_close_handler( if ws_before is None: return JSONResponse({"error": cfg.not_found_label}, status_code=404) if not await asyncio.to_thread(mgr.close, ws_id): + # A durability-poisoned session deliberately refuses soft close so + # its live journal cannot be discarded. Distinguish that conflict + # from the ordinary get-vs-close race where another caller already + # removed the workstream. + if mgr.get(ws_id) is not None: + return JSONResponse( + {"error": "workstream has unresolved persistence"}, + status_code=409, + ) return JSONResponse({"error": cfg.not_found_label}, status_code=404) storage = getattr(request.app.state, "auth_storage", None) @@ -1339,6 +1353,7 @@ def make_cancel_handler( async def cancel(request: Request) -> Response: import asyncio + from turnstone.core import session_worker from turnstone.core.web_helpers import read_json_or_400 if cfg.permission_gate is not None: @@ -1378,7 +1393,23 @@ def make_cancel_handler( if session is None or ui is None: return JSONResponse({"error": "No session"}, status_code=400) - was_running = bool(getattr(ws, "_worker_running", False)) + # Pin the exact worker Stop addresses. A cooperative cancel can make + # that owner exit before the force branch below acquires ``ws._lock``; + # a new send admitted after the cancel edge must not then be mistaken + # for — and force-abandoned as — the predecessor. + with ws._lock: + was_running = bool(ws._worker_running) + cancel_target = ws.worker_thread if was_running else None + structural_debt_method = cast( + "Callable[[], bool] | None", + concrete_method(session, "has_tool_structural_debt"), + ) + idle_structural_force = bool( + force + and not was_running + and structural_debt_method is not None + and structural_debt_method() + ) dropped: dict[str, Any] = {} if cfg.cancel_forensics is not None: try: @@ -1420,11 +1451,11 @@ def make_cancel_handler( exc_info=True, ) - # The remaining steps only matter when a worker is actually - # running: force-recovery has nothing to recover otherwise, - # and the SSE ``cancelled`` event would mislead consumers that - # have no in-flight generation to cancel. - if was_running: + # An ordinary idle cancel stays silent. The one idle force case with + # real work to recover is an abnormal finalizer that left structural + # debt after its worker slot exited; repair that poisoned prefix before + # admitting another worker. + if was_running or idle_structural_force: if force: # Force cancel: abandon the stuck worker thread (daemon, # will die on process exit or stream timeout) and emit @@ -1439,24 +1470,17 @@ def make_cancel_handler( # cancel flag short-circuits the abandoned thread # before it reaches the queue-drain seam, leaving the # queued message orphaned until the next spawn). - # ``session_worker.send`` documents this invariant: - # "readers gating on either flag see a coherent - # (worker_thread, _worker_running) pair." + # ``session_worker.send`` documents this invariant: readers + # gating on the running flag see one coherent worker/thread/ + # principal claim. # - # Documented bet — force-cancelling a wedged QUICK command - # (worker_kind == "command", e.g. /resume stuck in storage - # I/O): clearing the flag releases the pending-send - # drain's park (_drain_pending_sends polls the same - # (_worker_running, worker_kind) pair the parked /send - # used to), so a deferred message's fresh worker can then - # run while the abandoned command thread finishes its - # in-place mutation — quick commands have no generation - # checkpoints to retire them (compact_now does). Same - # blast radius as force-abandoning a send worker - # mid-tool; accepted because force-cancel is the operator - # escape hatch for an already-wedged session, not a - # routine path. Revisit if commands ever gain generation - # discipline. + # A turn worker is no longer released on a documented + # mid-tool consistency bet: ChatSession first supersedes its + # generation and journals conservative UNKNOWN receipts for + # every accepted unanswered tool call. QUICK command workers + # still rely on their established command-specific mutation + # discipline; destructive history workers advertise + # ``_worker_force_abandonable=False`` and remain pinned. # # Abandon machinery FIRST — before the ownership clear # and the idle emission below. The latch and the queue @@ -1471,32 +1495,105 @@ def make_cancel_handler( # operator who had just pressed Stop at its hardest. # The abandoned thread re-running the drain at its # eventual death is idempotent. - try: - session._drain_pending_advisories() - except Exception: - log.debug( - "ws.cancel.abandon_latch_failed ws=%s", - ws_id[:8], - exc_info=True, + force_abandoned = False + + def _publish_force_terminal() -> None: + if hasattr(ui, "_enqueue"): + try: + ui._enqueue({"type": "stream_end"}) + except Exception: + log.debug( + "ws.cancel.stream_end_failed ws=%s", + ws_id[:8], + exc_info=True, + ) + if hasattr(ui, "on_state_change"): + try: + ui.on_state_change("idle") + except Exception: + log.debug( + "ws.cancel.idle_state_failed ws=%s", + ws_id[:8], + exc_info=True, + ) + + def _holds_abandonable_claim_locked() -> bool: + # Caller holds ws._lock. + return bool( + ws._worker_running + and ws.worker_thread is cancel_target + and ws._worker_force_abandonable ) - with ws._lock: - ws.worker_thread = None - ws._worker_running = False - if hasattr(ui, "_enqueue"): - try: - ui._enqueue({"type": "stream_end"}) - except Exception: - log.debug( - "ws.cancel.stream_end_failed ws=%s", + + def _release_worker_claim_locked() -> None: + # Caller holds ws._lock and has verified + # ``_holds_abandonable_claim_locked()``; the field set + # itself is owned by session_worker (slot lifecycle). + session_worker.release_slot_locked(ws) + + force_method = concrete_method(session, "force_abandon_generation") + if force_method is not None: + force_abandon = cast( + "Callable[..., tuple[bool, Any]]", + force_method, + ) + + def _target_is_current() -> bool: + if cancel_target is None: + if structural_debt_method is None or not structural_debt_method(): + return False + with ws._lock: + return bool(not ws._worker_running and ws.worker_thread is None) + with ws._lock: + return _holds_abandonable_claim_locked() + + def _clear_target() -> bool: + if cancel_target is None: + with ws._lock: + return bool(not ws._worker_running and ws.worker_thread is None) + with ws._lock: + if not _holds_abandonable_claim_locked(): + return False + _release_worker_claim_locked() + return True + + force_abandoned, persistence_error = await asyncio.to_thread( + force_abandon, + target_is_current=_target_is_current, + clear_target=_clear_target, + publish_abandoned=_publish_force_terminal, + ) + if persistence_error is not None: + status = session.conversation_persistence_status() + log.warning( + "ws.cancel.force_persistence_unresolved ws=%s state=%s", ws_id[:8], - exc_info=True, + status.get("state", "unknown"), ) - if hasattr(ui, "on_state_change"): + else: + # Compatibility for small external/test session doubles. + # Production ChatSession owns the stronger generation + + # structural journal transaction above. + with ws._lock: + if _holds_abandonable_claim_locked(): + try: + session._drain_pending_advisories() + except Exception: + log.debug( + "ws.cancel.abandon_latch_failed ws=%s", + ws_id[:8], + exc_info=True, + ) + _release_worker_claim_locked() + force_abandoned = True + if force_abandoned: + _publish_force_terminal() + if not force_abandoned and hasattr(ui, "_enqueue"): try: - ui.on_state_change("idle") + ui._enqueue({"type": "cancelled"}) except Exception: log.debug( - "ws.cancel.idle_state_failed ws=%s", + "ws.cancel.cancelled_event_failed ws=%s", ws_id[:8], exc_info=True, ) @@ -1552,14 +1649,151 @@ RetryAuditEmitter = Callable[ ["Request", str, "Workstream"], None, ] -# (ws, user_msg) -> None. Re-sends ``user_msg`` on ``ws`` via the kind's -# worker dispatch (driving :func:`turnstone.core.session_worker.send` -# with the kind's own run / enqueue callbacks). The retry handler calls -# it after :meth:`ChatSession.retry` truncates the last turn. -RetryDispatcher = Callable[ - ["Workstream", str], - None, -] + + +@dataclass(frozen=True) +class _DestructiveCommand: + """The worker-slot channel a destructive command's body writes into. + + ``outcome`` carries the body's result back to the handler: + :func:`_dispatch_destructive_command` reads ``"busy"`` and ``"error"``; + every other key belongs to the verb. ``done`` is set by the body, never by + the dispatcher — retry signals it the moment the *cut* lands and then keeps + running the replacement generation on the same slot, so the two events are + not interchangeable. + """ + + outcome: dict[str, Any] + done: threading.Event + publish_clear_ui: Callable[[], None] + + +async def _dispatch_destructive_command( + *, + mgr: SessionManager, + ws: Workstream, + ws_id: str, + ui: Any, + session: ChatSession, + verb: str, + label: str, + not_found_label: str, + body: Callable[[_DestructiveCommand], None], + principal_id: str = "", +) -> dict[str, Any] | JSONResponse: + """Run one destructive history mutation inside a single worker-slot claim. + + Rewind and retry both cut committed history, so the idle check and the + complete in-memory/storage mutation must be ONE claim: a check followed by + an inline mutation lets a concurrent browser claim a turn in between, after + which the cut can delete that turn's USER row while its worker later + persists an orphan ASSISTANT response. + + ``body`` runs on the claimed slot and writes into the returned command's + ``outcome``; it owns ``done`` (see :class:`_DestructiveCommand`). The + ``clear_ui`` publisher handed to it is passed down into the session's + atomic history revision so the frontend's REST ``/history`` refetch is + triggered from inside the truncation — never from the handler afterwards. + + Returns the outcome dict once the body has signalled ``done``, or the + ``JSONResponse`` the handler must return verbatim: ``{"status": "busy"}`` + when the slot was held, 503 on a spawn or mutation failure, and the + 409/404 split below on a dispatch refusal. + """ + import asyncio + import threading + + from turnstone.core import session_worker + + def _publish_clear_ui() -> None: + enqueue = getattr(ui, "_enqueue", None) + if not callable(enqueue): + raise RuntimeError("Session UI cannot publish a structural reset") + enqueue({"type": "clear_ui"}) + + command = _DestructiveCommand( + outcome={}, + done=threading.Event(), + publish_clear_ui=_publish_clear_ui, + ) + + def _reject_busy() -> None: + command.outcome["busy"] = True + if hasattr(ui, "_enqueue"): + ui._enqueue({"type": "busy_error", "message": f"Cannot {verb} while processing."}) + + foreign_refused = False + + def _before_spawn() -> bool: + # Same admission gate ordinary sends run: a retry installs the caller + # as the slot principal and dispatches a replacement send, which + # would otherwise die mid-turn on the advisory seam's ownership + # assert once another participant's persistence-retained input is + # reached (round-4 review). Rewind passes no principal and skips it. + nonlocal foreign_refused + if session_worker.foreign_queue_conflict(session, principal_id): + foreign_refused = True + return False + return True + + try: + dispatched = session_worker.send( + ws, + enqueue=_reject_busy, + run=functools.partial(body, command), + expected_session=session, + before_spawn=_before_spawn, + thread_name=f"{verb}-{ws.id[:8]}", + worker_kind="command", + principal_id=principal_id, + force_abandonable=False, + ) + except Exception: + log.exception("ws.%s.worker_spawn_failed ws=%s", verb, ws_id[:8]) + return JSONResponse( + {"error": f"{label} could not be started — retry shortly."}, + status_code=503, + ) + if command.outcome.get("busy"): + return JSONResponse({"status": "busy"}) + if not dispatched: + if foreign_refused: + return JSONResponse( + { + "status": "cross_user_interjection", + "error": ( + "Another participant's queued input is still waiting. " + f"They must send or retract it before a {verb} can start." + ), + }, + status_code=409, + ) + # ``session_worker.send`` refuses for more reasons than a closed + # workstream: a Stop that raced the claim capture, a soft close + # preparing, structural tool debt pending after a force-cancel, or + # a session swap. Those leave the workstream alive — answer 409 + # (transient, retryable) and reserve 404 for a row that is + # actually gone. + if mgr.get(ws_id) is not None: + return JSONResponse( + {"error": "Workstream is temporarily busy — retry shortly."}, + status_code=409, + ) + return JSONResponse({"error": not_found_label}, status_code=404) + + await asyncio.to_thread(command.done.wait) + if "error" in command.outcome: + log.warning( + "ws.%s.failed ws=%s error=%r", + verb, + ws_id[:8], + command.outcome["error"], + ) + return JSONResponse( + {"error": f"{label} could not be persisted — history was unchanged."}, + status_code=503, + ) + return command.outcome def make_rewind_handler( @@ -1578,12 +1812,14 @@ def make_rewind_handler( mgr → ws-lookup → busy-gate → ``rewind`` → ``clear_ui`` → audit sequence. - Unlike :func:`make_close_handler`, the body emits a ``clear_ui`` event - after the mutation — **always, including a rewind to zero messages**. - The frontend keys its REST ``/history`` refetch (and any queued - edit-and-resend) off this signal, not an inline history payload; the - unconditional emit carries the PR #503 fix (an ``if history:`` guard - once froze the composer on rewind-to-zero). + Unlike :func:`make_close_handler`, the body supplies a ``clear_ui`` + publisher to the atomic mutation — **always, including a rewind to zero + messages**. The frontend keys its REST ``/history`` refetch (and any + queued edit-and-resend) off this signal, not an inline history payload; + publishing inside the truncation revision prevents a competing generic + repair event from closing the listener before ``clear_ui`` arrives. The + unconditional emit carries the PR #503 fix (an ``if history:`` guard once + froze the composer on rewind-to-zero). Args: cfg: per-kind policy bundle (auth, manager lookup, tenant check, @@ -1646,22 +1882,39 @@ def make_rewind_handler( if session is None or ui is None: return JSONResponse({"error": "No session"}, status_code=400) - # Reject rewind while a generation is in flight — mutating - # ``messages`` under a running worker corrupts history / cursors. - # Gate on ``_worker_running`` (not ``worker_thread.is_alive()``) - # for parity with session_worker.send. - with ws._lock: - if ws._worker_running: - if hasattr(ui, "_enqueue"): - ui._enqueue( - {"type": "busy_error", "message": "Cannot rewind while processing."} - ) - return JSONResponse({"status": "busy"}) + def _run_rewind(command: _DestructiveCommand) -> None: + from turnstone.core.session import GenerationCancelled - removed = session.rewind(raw_turns) + try: + command.outcome["removed"] = session.rewind( + raw_turns, + publish_reset=command.publish_clear_ui, + ) + except (GenerationCancelled, Exception) as exc: + # One arm, not two with identical bodies (the codebase's + # established tuple idiom): rewind() raises the BaseException + # GenerationCancelled when the truncation's commit admission + # is refused by a racing close/delete/poison — it must land + # in the dispatcher's error arm like any other refusal, not + # escape ``except Exception``. + command.outcome["error"] = exc + finally: + command.done.set() - if hasattr(ui, "_enqueue"): - ui._enqueue({"type": "clear_ui"}) + outcome = await _dispatch_destructive_command( + mgr=mgr, + ws=ws, + ws_id=ws_id, + ui=ui, + session=session, + verb="rewind", + label="Rewind", + not_found_label=cfg.not_found_label, + body=_run_rewind, + ) + if isinstance(outcome, JSONResponse): + return outcome + removed = int(outcome.get("removed", 0)) if audit_emit is not None: try: @@ -1681,7 +1934,6 @@ def make_rewind_handler( def make_retry_handler( cfg: SessionEndpointConfig, *, - dispatch_retry: RetryDispatcher, audit_emit: RetryAuditEmitter | None = None, accepted_permissions: tuple[str, ...] = (), ) -> Handler: @@ -1692,23 +1944,18 @@ def make_retry_handler( auth → mgr → ws-lookup → busy-gate → ``retry`` → ``clear_ui`` → audit → re-dispatch sequence across kinds. - The re-send goes through ``dispatch_retry`` (a per-kind closure that - drives :func:`turnstone.core.session_worker.send` with the kind's own - ``run`` / ``enqueue`` callbacks) rather than a hand-rolled thread, so - both kinds converge on the shared worker-dispatch primitive instead - of open-coding a third copy. A retry issued while busy is rejected up - front by the busy-gate below; the dispatcher's ``enqueue`` callback - hard-rejects (rather than queues) so the rare check-then-dispatch - race can't silently defer the resend behind the in-flight turn. + The destructive cut and replacement send run in one shared worker slot. + The slot begins as ``command`` (ordinary sends defer instead of entering a + half-truncated turn), then becomes ``turn`` immediately before the + replacement generation. This closes the old check -> truncate -> second + dispatch race. - ``clear_ui`` fires after ``retry()`` regardless of whether anything - was dropped (idempotent REST refetch on the frontend), matching the - pre-lift interactive handler. + ``clear_ui`` is published by ``retry()``'s atomic history revision + regardless of whether anything was dropped (idempotent REST refetch on + the frontend), matching the pre-lift interactive handler. Args: cfg: per-kind policy bundle. - dispatch_retry: ``(ws, user_msg) -> None`` re-send closure. - Required — retry is meaningless without it. audit_emit: kind's audit emitter; receives ``(request, ws_id, ws)``. **Both kinds hardcode the ``conversation.retry`` action.** Wrapped in try/except. @@ -1719,8 +1966,11 @@ def make_retry_handler( async def retry(request: Request) -> Response: import asyncio + import threading + from turnstone.core import session_worker from turnstone.core.auth import require_any_permission + from turnstone.core.session import GenerationCancelled if cfg.permission_gate is not None: err = cfg.permission_gate(request) @@ -1749,28 +1999,97 @@ def make_retry_handler( if session is None or ui is None: return JSONResponse({"error": "No session"}, status_code=400) - with ws._lock: - if ws._worker_running: - if hasattr(ui, "_enqueue"): - ui._enqueue({"type": "busy_error", "message": "Cannot retry while processing."}) - return JSONResponse({"status": "busy"}) - - # A retry is a fresh turn initiated by the authenticated caller — - # rebind per-user MCP credential resolution to them before the - # re-send dispatches (the per-kind ``dispatch_retry`` closure - # calls ``send()`` without identity kwargs). getattr-guarded so - # per-kind session stubs without the method keep working. + # Retry owns one worker slot across BOTH the destructive cut and the + # replacement generation. A second dispatcher after an inline cut is + # too late: a concurrent send can claim the idle slot in between and + # either be deleted by retry or cause the requested replacement to be + # rejected after history was already changed. from turnstone.core.web_helpers import auth_user_id acting_uid = auth_user_id(request) - bind_acting = getattr(session, "bind_acting_user", None) - if acting_uid and callable(bind_acting): - bind_acting(acting_uid) - retry_msg = session.retry() + def _run_retry(command: _DestructiveCommand) -> None: + try: + retry_msg = session.retry(publish_reset=command.publish_clear_ui) + except (GenerationCancelled, Exception) as exc: + # One arm, not two with identical bodies: retry() raises the + # BaseException GenerationCancelled when the cut's commit + # admission is refused by a racing close/delete/poison — the + # dispatcher's error arm owns both refusal shapes. + command.outcome["error"] = exc + command.done.set() + return - if hasattr(ui, "_enqueue"): - ui._enqueue({"type": "clear_ui"}) + command.outcome["retry_msg"] = retry_msg + if not retry_msg: + command.done.set() + return + + # The slot began as a command so ordinary sends defer rather than + # entering the interjection queue while history is being cut. It + # becomes a turn slot atomically before the replacement send, with + # the authenticated principal installed for queue ownership (the + # field set is session_worker's — the slot-lifecycle owner). + with ws._lock: + if ws.worker_thread is not threading.current_thread(): + command.outcome["error"] = RuntimeError( + "Retry worker lost its workstream claim" + ) + command.done.set() + return + session_worker.reclassify_slot_locked( + ws, + worker_kind="turn", + principal_id=acting_uid, + force_abandonable=True, + ) + command.done.set() + me = threading.current_thread() + try: + session.send(retry_msg, acting_user_id=acting_uid or None) + except GenerationCancelled: + # send() self-handles an in-turn cancel; this arm is the + # pre-envelope raced-Stop residue. Converge the pane if this + # thread still owns the slot. + if ws.worker_thread is me: + _emit_send_ui(ws, ui, "on_stream_end") + _emit_send_ui(ws, ui, "on_state_change", "idle") + except Exception as exc: + # A raise BEFORE send()'s error-convergence envelope (a + # force-cancel or soft close landing between the cut and the + # re-send) emits nothing itself — the destructive cut already + # succeeded and the HTTP response already said ok, so without + # this net the pane shows truncated history and silence. + # Deliberately NOT routed through ensure_error_recorded: on a + # REUSED session a pre-try raise after a prior errored turn + # finds the persisted-error flag stale-True and the recorder + # would no-op (#865 owns the reuse-safe signal); the display + # string is sanitized inline instead. state=error via this + # path carries no fresh last_error row — also #865. + log.exception("ws.retry.worker_failed ws=%s", ws_id[:8]) + if ws.worker_thread is me: + from turnstone.core.memory import sanitize_error_text + + _emit_send_ui(ws, ui, "on_error", f"Error: {sanitize_error_text(str(exc))}") + _emit_send_ui(ws, ui, "on_stream_end") + _emit_send_ui(ws, ui, "on_state_change", "error") + + outcome = await _dispatch_destructive_command( + mgr=mgr, + ws=ws, + ws_id=ws_id, + ui=ui, + session=session, + verb="retry", + label="Retry", + not_found_label=cfg.not_found_label, + body=_run_retry, + principal_id=acting_uid, + ) + if isinstance(outcome, JSONResponse): + return outcome + + retry_msg = outcome.get("retry_msg") if audit_emit is not None: try: @@ -1783,8 +2102,6 @@ def make_retry_handler( ) retried = retry_msg is not None - if retry_msg is not None: - dispatch_retry(ws, retry_msg) return JSONResponse({"status": "ok", "retried": retried}) @@ -2069,18 +2386,34 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: # ``Last-Event-ID`` resume: native EventSource auto-reconnect # sends the header; the manual-reconnect path (which uses # ``new EventSource(url)`` and can't set custom headers) sends - # ``?last_event_id=N``. Accept both; malformed values fall - # back to fresh-connect semantics so a broken intermediary - # can't break replay for a client that genuinely lost no - # events. - last_event_id_raw = request.headers.get("Last-Event-ID") or request.query_params.get( - "last_event_id" - ) - last_event_id: int | None + # ``?last_event_id=N``. Accept both. A syntactically valid native + # reconnect header has priority over every URL bootstrap hint. In + # particular, an EventSource may auto-reconnect the original initial + # URL (which still contains ``history_token``); once it has a numeric + # Last-Event-ID it is a normal ring reconnect and must not revalidate a + # one-shot REST handoff token. Range validity is checked atomically + # against the UI's high-water mark below, so a numeric negative/future + # header takes this native path but fails closed as truncated. A + # malformed native header does not get priority: it may have been + # mangled by an intermediary, so fall back to the URL cursor and, + # critically, retain the initial ``history_token`` validation. + last_event_id_header = request.headers.get("Last-Event-ID") + + native_last_event_id: int | None = None try: - last_event_id = int(last_event_id_raw) if last_event_id_raw else None + native_last_event_id = int(last_event_id_header) if last_event_id_header else None except (TypeError, ValueError): - last_event_id = None + native_last_event_id = None + + native_cursor_numeric = native_last_event_id is not None + if native_cursor_numeric: + last_event_id = native_last_event_id + else: + last_event_id_query = request.query_params.get("last_event_id") + try: + last_event_id = int(last_event_id_query) if last_event_id_query else None + except (TypeError, ValueError): + last_event_id = None # Three replay shapes: # - ``last_event_id is None`` → ``"fresh"`` (today's behaviour): @@ -2105,7 +2438,45 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: earliest_available_id = 0 in_progress_snap: dict[str, Any] snap_seq: int = 0 - if last_event_id is None: + history_token = ( + request.query_params.get("history_token") if not native_cursor_numeric else None + ) + supports_user_turn_projection = request.query_params.get("user_turn") == "1" + supports_tool_turn_projection = request.query_params.get("tool_turn") == "1" + tokenless_fresh_bootstrap = not history_token and last_event_id is None + handoff_mismatch = False + if history_token: + # The ChatSession owns the history-revision lock. Validation and + # listener registration happen in one critical section, closing + # the REST-response -> SSE-open window without coupling this + # shared route to the journal implementation. + session = getattr(ws, "session", None) + register_handoff = getattr(session, "register_listener_for_history_handoff", None) + registration = ( + register_handoff(history_token, last_event_id=last_event_id) + if callable(register_handoff) + else None + ) + if registration is None: + handoff_mismatch = True + replay_status = "history_mismatch" + client_queue = None + in_progress_snap = {"content": "", "reasoning": "", "seq": 0} + else: + ( + client_queue, + replay_events, + replay_status, + lost_count, + earliest_available_id, + snapshot, + ) = registration + if replay_status in {"fresh", "truncated"}: + in_progress_snap = snapshot + snap_seq = snapshot["seq"] + else: + in_progress_snap = {"content": "", "reasoning": "", "seq": 0} + elif last_event_id is None: replay_status = "fresh" client_queue, in_progress_snap = ui_base.register_listener_with_in_progress_snapshot() snap_seq = in_progress_snap["seq"] @@ -2164,14 +2535,13 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: replay_cb = cfg.events_replay async def event_generator() -> Any: - import functools import random _metrics.record_sse_connect() loop = asyncio.get_running_loop() def _format_event(event: dict[str, Any]) -> dict[str, str]: - """Strip internal plumbing fields, attach SSE ``id:`` if present. + """Project one buffered event for this listener. Shallow-copies the dict before any mutation because ``_enqueue`` puts ONE reference into every listener @@ -2189,11 +2559,70 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: # the wire. The fresh-path live drain filters on # ``_seq`` BEFORE calling this helper. ev_copy.pop("_seq", None) + canonical_user_turn = ev_copy.get("type") == "user_turn" + canonical_tool_turn = ( + ev_copy.get("type") == "tool_result" and ev_copy.get("accepted") is True + ) + downgraded_user_turn = canonical_user_turn and not supports_user_turn_projection + downgraded_tool_turn = canonical_tool_turn and not supports_tool_turn_projection + if downgraded_user_turn or downgraded_tool_turn: + # A pre-projection client cannot render this accepted row. + # Give it the strong-repair event it already understands, + # anchored at the cursor immediately BEFORE the row. If + # /history repair fails, its reconnect therefore replays + # this canonical event and receives the repair signal + # again instead of skipping the unrepresented row. + ev_copy = { + "type": "replay_truncated", + "ws_id": ws_id, + "reason": ( + "user_turn_projection_unsupported" + if downgraded_user_turn + else "tool_turn_projection_unsupported" + ), + } + elif (canonical_user_turn or canonical_tool_turn) and isinstance(eid, int): + # Typed consumers need the canonical row identity in the + # decoded payload as well as EventSource's transport-only + # ``lastEventId``. SDK parsers do not otherwise expose the + # SSE id field, and both browser reducers use this value + # for exact render deduplication. + ev_copy["_event_id"] = eid out: dict[str, str] = {"data": json.dumps(ev_copy)} - if eid is not None: + if (downgraded_user_turn or downgraded_tool_turn) and isinstance(eid, int): + out["id"] = str(max(0, eid - 1)) + elif eid is not None: out["id"] = str(eid) return out + def _current_state_event() -> dict[str, Any] | None: + """Build the idempotent current-state bootstrap frame.""" + try: + cur_state = getattr(ws.state, "value", None) + if not isinstance(cur_state, str) or not cur_state: + return None + state_evt: dict[str, Any] = { + "type": "state_change", + "state": cur_state, + "ws_id": ws_id, + } + sess = getattr(ws, "session", None) + acting = ( + getattr(sess, "_acting_user_id", "") or getattr(sess, "_user_id", "") + if sess is not None + else "" + ) + if acting: + state_evt["acting_user_id"] = acting + return state_evt + except Exception: + log.debug( + "ws.events.state_change_replay_failed ws=%s", + ws_id[:8], + exc_info=True, + ) + return None + try: # Per-stream reconnect interval jitter. Without this, # all panes on a workstream disconnect together and @@ -2204,6 +2633,29 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: # below today's ping cadence while staggering peaks. yield {"retry": random.randint(2500, 4500)} + if handoff_mismatch: + # A committed conversation row crossed the REST history + # response -> listener-registration boundary. Numeric ring + # replay is not a substitute for that row (tool-only turns + # need not be reconstructable from the ring), so instruct + # the client to repeat /history and terminate this source. + yield { + "data": json.dumps( + { + "type": "history_resync", + "ws_id": ws_id, + "reason": "handoff_mismatch", + } + ) + } + return + + # Every non-mismatch path registered a real listener above. + # Assign a narrowed local for both the live drain and cleanup. + stream_queue = client_queue + if stream_queue is None: + return + if replay_status == "replay_ok": # Buffered events already cover everything since # the client's ``Last-Event-ID`` — skip the @@ -2214,6 +2666,26 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: # their ``_event_id`` as SSE ``id:`` so a # disconnect mid-replay resumes from the latest # buffered id, not the original ``last_event_id``. + # Reconnect deltas still need the idempotent connection / + # status / current-state bootstrap. Do not invoke the full + # replay callback here: its pending-control cards can also + # exist in ``replay_events`` and appending both duplicates + # operator controls. Called directly — both kinds wired + # byte-equivalent trivial wrappers, so the preamble + # contract lives in session_replay_preamble alone (it + # no-ops on a detached session internally). + try: + for ev in session_replay_preamble(ws.session, ui): + yield {"data": json.dumps(ev)} + except Exception: + log.debug( + "ws.events.preamble_replay_failed ws=%s", + ws_id[:8], + exc_info=True, + ) + state_evt = _current_state_event() + if state_evt is not None: + yield {"data": json.dumps(state_evt)} for ev in replay_events: yield _format_event(ev) else: @@ -2224,6 +2696,29 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: # first so the client knows the buffer couldn't # cover the gap and treats the snapshot below as # the recovery floor. + if replay_status == "fresh" and tokenless_fresh_bootstrap: + # Rolling-upgrade floor for a pre-handoff browser. An + # old tab loads REST history without a token; if a row + # lands before this registration, synthetic replay + # cannot prove that row was in its DOM. The deployed + # reducer understands clear_ui and refetches history + # in-place while this atomically-registered listener + # remains open. Do not use replay_truncated here: an + # old reducer closes and reconnects after that frame, + # but quiescent /history has no numeric cursor, so it + # would return tokenless and repeat this bootstrap + # forever. Current clients always present a handoff + # token on first connect. + yield { + "data": json.dumps( + { + "type": "clear_ui", + "ws_id": ws_id, + "reason": "tokenless_history_bootstrap", + } + ) + } + if replay_status == "truncated": # Node-side visibility for every replay-window # miss (evicted ring AND the empty-ring @@ -2279,33 +2774,9 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: # workstream state and the in-progress snapshot. # Both are best-effort — a ws.state read failure # or empty buffers just yields nothing extra. - try: - cur_state = getattr(ws.state, "value", None) - if isinstance(cur_state, str) and cur_state: - state_evt: dict[str, Any] = { - "type": "state_change", - "state": cur_state, - "ws_id": ws_id, - } - # A client connecting mid-turn learns who holds it, - # so it can gate its send button (matches the live - # state_change emitted from server.WebUI). - sess = getattr(ws, "session", None) - acting = ( - getattr(sess, "_acting_user_id", "") - or getattr(sess, "_user_id", "") - if sess is not None - else "" - ) - if acting: - state_evt["acting_user_id"] = acting - yield {"data": json.dumps(state_evt)} - except Exception: - log.debug( - "ws.events.state_change_replay_failed ws=%s", - ws_id[:8], - exc_info=True, - ) + state_evt = _current_state_event() + if state_evt is not None: + yield {"data": json.dumps(state_evt)} if in_progress_snap["content"] or in_progress_snap["reasoning"]: yield { "data": json.dumps( @@ -2385,12 +2856,12 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: while True: if await request.is_disconnected(): return - if getattr(client_queue, "poisoned", False): + if getattr(stream_queue, "poisoned", False): # The queue overflowed: it latched ``poisoned`` # at the FIRST rejected put, freezing its # contents as a contiguous prefix (see # ``_ListenerQueue``). - if getattr(client_queue, "closing", False): + if getattr(stream_queue, "closing", False): # ws teardown raced the overflow: the queue is # poisoned AND its ws is closing. Unwind as a # CLEAN close — no ``stream_overflow`` frame, @@ -2431,7 +2902,7 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: try: event = await loop.run_in_executor( live_executor, - functools.partial(client_queue.get, timeout=5), + functools.partial(stream_queue.get, timeout=5), ) except queue.Empty: continue # ping keeps the connection alive @@ -2443,7 +2914,8 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler: yield _format_event(event) finally: _metrics.record_sse_disconnect() - unregister(client_queue) + if client_queue is not None: + unregister(client_queue) return EventSourceResponse(event_generator(), ping=5) @@ -3284,6 +3756,7 @@ def make_list_handler(cfg: SessionEndpointConfig) -> Handler: "user_id": ws.user_id, "project_id": project_id or None, "persona": persona or None, + "persistence_state": workstream_persistence_state(ws), } ) return rows @@ -3639,6 +4112,13 @@ def _resume_cursor_and_trim( """ if awaiting_approval or not messages: return messages, None + # A journal row is visible through the live history handoff but has not + # completed its durable acknowledgement yet. It is already part of the + # authoritative REST snapshot; trimming it in favour of ring replay would + # discard exactly the row the handoff journal exists to preserve (and a + # tool-call-only row need not be reconstructable from numeric events). + if any(m.get("_pending_durability") for m in messages): + return messages, None can_replay = getattr(ui, "can_replay_from", None) if not callable(can_replay): return messages, None @@ -3672,11 +4152,19 @@ def _resume_cursor_and_trim( return messages[:orphan_idx], cursor -# One /history flight's result: (messages, resume cursor, load_failed). -# ``load_failed`` is True only when ``load_messages`` RAISED — never for -# a legitimately empty workstream; see ``_reconstruct`` inside -# :func:`make_history_handler`. -_HistoryFlightResult: TypeAlias = tuple[list[dict[str, Any]], int | None, bool] +# One /history flight's result: (messages, resume cursor, load_failed, +# live-handoff token, cold storage-only read). +# ``load_failed`` is True when the durable load or the canonical public +# decoration/projection pipeline raised — never for a legitimately empty +# workstream; see ``_reconstruct`` inside :func:`make_history_handler`. +_HistoryFlightResult: TypeAlias = tuple[list[dict[str, Any]], int | None, bool, str | None, bool] +"""``(messages, cursor, load_failed, handoff_token, storage_fallback)``. + +``storage_fallback`` marks the cold storage-only read — the workstream is +not loaded on this node, so there is no live writer and no splice to +witness. The transcript is authoritative but carries no handoff token: it +can seed a render and a tokenless stream bootstrap, never a cursor handoff. +""" def make_history_handler(cfg: SessionEndpointConfig) -> Handler: @@ -3793,16 +4281,16 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: if err_tenant is not None: return err_tenant - # Existence + kind check. The workstream may live only in - # storage (closed coordinators are still readable via /history - # without rehydrating; persisted-but-not-loaded interactives - # are likewise readable). Mirrors the pre-lift coord - # ``_resolve_coordinator_or_404`` ladder: in-memory mgr.get → - # storage row + kind check → 404. Falling back to storage - # without the kind check would leak interactive rows through - # the coord endpoint (and vice versa) on a process that - # shares storage with the other kind. ``cfg.list_kind`` is - # guaranteed non-None by the misconfig gate above. + # Existence + kind check — a read verb that never (re)hydrates. A + # session is loaded exactly when a pane opened this workstream on + # this node (panes POST /open before /history), which is the only + # case with a live writer whose REST -> SSE splice needs witnessing; + # its capture carries the opaque handoff token. A not-loaded row has + # no live writer and is served straight from storage, tokenless: the + # payload seeds a render plus the tokenless stream bootstrap, never a + # cursor handoff. Constructing a ChatSession here would turn history + # browsing into session-pool churn (and a capacity-refusal 503) for + # consumers that never open a stream. storage = getattr(request.app.state, "auth_storage", None) live_session = mgr.get(ws_id) if live_session is None: @@ -3832,19 +4320,15 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: # dispatched after a rewind/retry must never join a flight whose # load_messages ran before it — the joined pre-rewind payload # reads as fresh truth client-side (the dispatch stamp is - # current) and reopened the over-rewind window. Cold workstream: - # generation 0; the first post-load truncation bumps to 1, so a - # cold flight can never be joined across a rewind either. + # current) and reopened the over-rewind window. # mgr.get returns the Workstream WRAPPER — the counter lives on # its ChatSession (the G7 harness caught a direct getattr # silently defaulting to 0 forever, which re-enabled joining). # Typed access, not getattr chains, so mypy carries the shape. - # A cold/detached workstream keys on None, NEVER 0: an eviction - # or close landing inside a held flight's window would otherwise - # let a post-truncation request join a generation-0 live flight - # (rewinds need a live session, so two COLD flights are always - # mutually safe — and a rehydrated session restarting at 0 can - # never share the manager slot with its evicted predecessor). + # A loaded session (pane-opened) has a concrete generation here; a + # cold row keys its flight on ``None`` — every cold read shares one + # storage-only reconstruction, and a session installed mid-flight + # only changes LATER requests' keys. live_gen: int | None = ( live_session.session._history_generation if live_session is not None and live_session.session is not None @@ -3875,19 +4359,51 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: # add a join-vs-cancel race (a late joiner grabbing a task the # last leaver is cancelling) to a seam that is race-free # precisely because the flight is never cancelled. - messages, cursor, load_failed = await asyncio.shield(task) + messages, cursor, load_failed, handoff_token, storage_fallback = await asyncio.shield(task) if joined and load_failed: - # The shared draw hit a transient ``load_messages`` failure - # and produced the 200-empty payload. Both clients render a - # 200-empty as an authoritative empty pane (the seedless - # clear_ui path has no SSE redelivery to repair it), so - # sharing the failed draw would fan ONE storage blip out as - # a pane wipe across every joiner. Joiners therefore retry - # once, independently and unshared — exactly the blast - # radius the un-coalesced endpoint had. The flight OWNER - # keeps its failed draw (same as a lone request today). + # The shared draw hit a transient durable-load or canonical + # public-projection failure and produced an unavailable result. + # Retry once independently so one transient fault is not fanned + # out to every joined pane. + # + # Before the explicit 503 contract this result was a 200-empty + # payload, which both clients rendered as authoritative history; + # with a pending handoff journal it could instead be a 200 + # containing only the pending suffix plus a valid token. Either + # shape hid the older durable transcript and cleared a repair + # latch on incomplete truth. The final ``load_failed`` bit must + # therefore survive this retry and drive the non-2xx below. log.debug("ws.history.coalesced_retry ws=%s", ws_id[:8]) - messages, cursor, _ = await _reconstruct(mgr, storage, request.app.state, ws_id, limit) + messages, cursor, load_failed, handoff_token, storage_fallback = await _reconstruct( + mgr, storage, request.app.state, ws_id, limit + ) + if load_failed: + # Never authorize a browser to replace its transcript or complete + # a REST -> SSE handoff from an incomplete durable prefix. The + # pending journal merge above remains useful internally (and for + # diagnostics), but a client cannot distinguish "pending-only" + # from complete history. Both panes already treat non-2xx as a + # render no-op; handoff-repair mode retains its latch and retries + # without opening a cursorless/tokenless EventSource. + return JSONResponse( + {"error": "History temporarily unavailable"}, + status_code=503, + ) + if not handoff_token and not storage_fallback: + # A 200 history payload is allowed to seed a live EventSource, so + # tokenless snapshots are not authoritative: a row admitted after + # this response and before fresh listener registration would be + # absent from both halves. Production ChatSession captures always + # return a token; reaching this arm means rehydration raced close or + # a session implementation lacks the handoff protocol. The + # deliberate tokenless 200 is the cold storage-only read for any + # workstream not loaded on this node: its payload may seed a + # render plus a tokenless stream bootstrap, never a cursor handoff. + log.warning("ws.history.handoff_unavailable ws=%s", ws_id[:8]) + return JSONResponse( + {"error": "History temporarily unavailable"}, + status_code=503, + ) # Each awaiter serializes its own JSONResponse from the shared # payload — cost parity with the un-coalesced endpoint (one # render per request). Sharing pre-rendered bytes across @@ -3895,7 +4411,14 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: # result to (bytes, flag), splitting the _HistoryFlightResult # contract the retry path still needs, for a herd-only # serialization micro-win. - return JSONResponse({"ws_id": ws_id, "messages": messages, "cursor": cursor}) + return JSONResponse( + { + "ws_id": ws_id, + "messages": messages, + "cursor": cursor, + "handoff_token": handoff_token, + } + ) async def _run_flight( key: tuple[str, int, int | None], @@ -3935,16 +4458,38 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: client connects, the stream answers ``replay_truncated`` and the client resyncs. - Returns ``(messages, cursor, load_failed)`` — ``load_failed`` - is True only when ``load_messages`` RAISED (transient storage - failure), never for a legitimately empty workstream. The - caller uses it to keep an exception-empty payload from fanning - a pane wipe out to coalesced joiners (see the joiner retry in - ``history``). + Returns ``(messages, cursor, load_failed, handoff_token, + storage_fallback)`` — + ``load_failed`` is True when either the durable load or the canonical + public decoration/projection pipeline failed, never for a legitimately + empty workstream. The caller uses it both to retry a joined shared + failure independently and to return 503 instead of authorizing an + incomplete or unprojected history render. """ - live_session = mgr.get(ws_id) load_failed = False + storage_fallback = False + live_session = mgr.get(ws_id) + if live_session is None or live_session.session is None: + # Cold storage-only read — /history never (re)hydrates. A loaded + # session exists exactly when a pane opened this workstream on + # this node, the only case with a live writer whose splice needs + # witnessing. Tokenless payloads cannot seed a cursor handoff; + # the stream bootstrap re-renders. A vanished durable row keeps + # the 503 path: the caller's existence gates saw the row moments + # ago, so this is a delete race, not an empty workstream. + live_session = None + if storage is None: + return [], None, True, None, False + try: + row_probe = await asyncio.to_thread(storage.get_workstream, ws_id) + except Exception: + log.warning("ws.history.cold_lookup_failed ws=%s", ws_id[:8], exc_info=True) + return [], None, True, None, False + if row_probe is None: + return [], None, True, None, False + storage_fallback = True messages: list[dict[str, Any]] = [] + handoff_token: str | None = None # Fresh-connect resume cursor (the ``Last-Event-ID`` the client # opens its initial SSE with). Non-None only when the trailing # turn is an executing in-flight orphan that the ring buffer can @@ -3952,35 +4497,87 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: # on every other path (and on any decoration failure below) so # the client takes the synthetic-snapshot floor. cursor: int | None = None - if storage is not None: + live_chat = live_session.session if live_session is not None else None + capture_handoff = ( + concrete_method(live_chat, "capture_history_handoff") if live_chat is not None else None + ) + if storage is not None or live_chat is not None: try: # repair=False — display read; include_compaction=True so a # persisted compaction marker projects as an in-place # source="compaction" system row and the UI re-renders its # compaction card after a reload. See the # reconstruct_messages docstring for both flags. - messages = await asyncio.to_thread( - storage.load_messages, - ws_id, - limit=limit, - repair=False, - include_compaction=True, - ) + def _load_messages(overscan: int = 0) -> list[dict[str, Any]]: + # ``overscan`` is the pending-journal size: widening the + # tail window by that many rows guarantees any committed + # twin of a pending key is inside the loaded window, so + # the merge never re-renders an already-committed row at + # the transcript tail (out of order). + if storage is None: + return [] + return cast( + "list[dict[str, Any]]", + storage.load_messages( + ws_id, + limit=limit + max(0, overscan), + repair=False, + include_compaction=True, + ), + ) + + if capture_handoff is not None: + messages, handoff_token = await asyncio.to_thread( + capture_handoff, _load_messages + ) + if storage is None: + # No route-visible storage: the durable prefix is + # unreadable here, so this render must not claim + # transcript authority. Serve the live/journal view + # as a deliberate tokenless render — the client's + # bootstrap downgrade owns convergence via clear_ui. + handoff_token = None + storage_fallback = True + # The storage read was tail-bounded to limit+overscan; + # journal rows append after it, so re-apply the public + # response bound to the merged authoritative view. + messages = messages[-limit:] + else: + messages = await asyncio.to_thread(_load_messages) except Exception: - # Warning, not debug: this 200-empty renders as an - # authoritative pane wipe in both clients, and under - # coalescing it is also what triggers the joiner retry — - # a storage blip here is operationally interesting for - # the same reason ``decoration_failed`` below is. + # Warning, not debug: the caller returns 503 and handoff + # repair remains latched until a later complete read. Under + # coalescing this also triggers the joiner's one independent + # retry, so a storage blip here is operationally interesting + # for the same reason ``decoration_failed`` below is. load_failed = True log.warning("ws.history.load_failed ws=%s", ws_id[:8], exc_info=True) + # Preserve the exact pending journal suffix in the internal + # reconstruction result as a fail-visible floor. It is NOT + # complete browser truth without the durable prefix: the + # caller keeps ``load_failed=True`` and returns 503, so no + # handoff token from this fallback can authorize a render. + if capture_handoff is not None: + try: + messages, handoff_token = await asyncio.to_thread( + capture_handoff, + lambda _overscan: [], + ) + messages = messages[-limit:] + except Exception: + log.warning( + "ws.history.pending_handoff_failed ws=%s", + ws_id[:8], + exc_info=True, + ) # Audit-trail decoration — attach persisted intent_verdict and # output_assessment data to each assistant.tool_calls entry so # the dashboard's history replay paints the same verdict pills # / output-warning bubbles the live SSE path shows. Both - # storage queries are off-loop via ``to_thread``. Best-effort: - # any failure leaves messages undecorated — replay degrades to - # the pre-decoration shape rather than 500-ing. + # storage queries are off-loop via ``to_thread``. This is part of the + # authoritative public-history boundary: an uncaught failure anywhere + # in the decoration/reasoning/projection pass makes the reconstruction + # unavailable rather than authorizing its raw intermediate shape. if messages: try: from turnstone.core.history_decoration import ( @@ -4017,9 +4614,11 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: surface_persisted_reasoning = True resolved_alias = "" resolved_registry: Any = None - if live_session is not None: - resolved_registry = getattr(live_session, "_registry", None) - resolved_alias = getattr(live_session, "_model_alias", "") or "" + if live_chat is not None: + # ``live_session`` is the Workstream wrapper; the registry + # and alias live on the wrapped ChatSession. + resolved_registry = getattr(live_chat, "_registry", None) + resolved_alias = getattr(live_chat, "_model_alias", "") or "" if not resolved_alias and storage is not None: # Off-loop the sync storage call (mirrors get_workstream # / load_messages / load_verdict_indexes / decorate / @@ -4115,21 +4714,33 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: if isinstance(steps, list) and steps: tc["agent_steps"] = steps except Exception: - # Operationally interesting: a persistent decoration - # failure (missing migration, driver mismatch, schema - # drift) silently strips verdict pills + output - # warnings from every reload of every workstream. - # Log at warning so it surfaces in normal log review - # rather than only when DEBUG is on. Reset the cursor so a - # mid-pipeline failure can't pair an un-trimmed orphan with - # a fast-forward cursor (which would double-render it). + # This pipeline is the public-schema and privacy boundary, not + # optional decoration. Raw reconstructed rows may carry + # provider-native reasoning/signatures, producer identity, + # audit provenance, and storage idempotency fields. Never let + # an exception skip that boundary and turn its partially + # transformed input into an authoritative token-bearing 200. + # Clear every output axis and let the caller return 503 (or a + # joined request retry once independently). + messages = [] cursor = None + handoff_token = None + load_failed = True log.warning( "ws.history.decoration_failed ws=%s", ws_id[:8], exc_info=True, ) - return messages, cursor, load_failed + # Internal audit/journal metadata is never part of the public history + # schema. Projection constructs fresh dicts and drops it; this final + # scrub remains defense in depth for any future canonical projection + # that preserves an input mapping. Provenance can carry an acting + # principal id, so that key must always fail closed. + for message in messages: + message.pop("_commit_key", None) + message.pop("_pending_durability", None) + message.pop("_provenance", None) + return messages, cursor, load_failed, handoff_token, storage_fallback return history @@ -4437,6 +5048,7 @@ def make_detail_handler(cfg: SessionEndpointConfig) -> Handler: "state": ws.state.value, "user_id": ws.user_id, "kind": ws.kind, + "persistence_state": workstream_persistence_state(ws), "pending_approval": pending_approval, "pending_approval_details": pending_approval_details, } @@ -4510,6 +5122,7 @@ def _make_dispatch_attempt( ordered_taken: list[str], send_id: str, acting_uid: str, + client_send_id: str = "", defer_fidelity: bool = False, ) -> Callable[[ChatSession], tuple[bool, dict[str, Any]]]: """Build one atomic queue-or-spawn attempt bound to ONE session capture. @@ -4572,13 +5185,21 @@ def _make_dispatch_attempt( if defer_fidelity and (resolved_atts or len(message) > INTERJECTION_CAP_CHARS): queue_outcome["rejected"] = "defer_full_fidelity" return + admission = session_worker.claimed_slot_queue_admission(ws, acting_uid) + if admission is None: + queue_outcome["rejected"] = "cross_user_interjection" + return + _claimed, queue_kwargs = admission try: - cleaned, priority, msg_id = session.queue_message( - message, - attachment_ids=list(ordered_taken), - queue_msg_id=send_id or None, - interjector_user_id=acting_uid, + queue_kwargs.update( + { + "attachment_ids": list(ordered_taken), + "queue_msg_id": send_id or None, + } ) + if client_send_id: + queue_kwargs["client_send_id"] = client_send_id + cleaned, priority, msg_id = session.queue_message(message, **queue_kwargs) except AttachmentsNotQueueableError: queue_outcome["rejected"] = "attachments_busy" return @@ -4594,6 +5215,12 @@ def _make_dispatch_attempt( queue_outcome["priority"] = priority queue_outcome["msg_id"] = msg_id + def _before_spawn() -> bool: + if session_worker.foreign_queue_conflict(session, acting_uid): + queue_outcome["rejected"] = "cross_user_interjection" + return False + return True + def _run() -> None: me = threading.current_thread() try: @@ -4602,6 +5229,8 @@ def _make_dispatch_attempt( kwargs["attachments"] = resolved_atts if send_id: kwargs["send_id"] = send_id + if client_send_id: + kwargs["client_send_ids"] = (client_send_id,) # Fresh turn: rebind per-user MCP credentials to the # authenticated sender. Bound here (not via a send() # kwarg) so per-kind session stubs with explicit send @@ -4634,7 +5263,10 @@ def _make_dispatch_attempt( ws, enqueue=_enqueue, run=_run, + expected_session=session, + before_spawn=_before_spawn, thread_name=f"send-worker-{ws.id[:8]}", + principal_id=acting_uid, ) if ok and not queue_outcome and cfg.spawn_metrics is not None: # Fresh spawn — the kind's per-turn metrics fire exactly once, @@ -4946,6 +5578,18 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: message = (body.get("message") or "").strip() if not message: return JSONResponse({"error": "message is required"}, status_code=400) + raw_client_send_id = body.get("client_send_id") + if raw_client_send_id is None: + client_send_id = "" + elif not isinstance(raw_client_send_id, str) or not _CLIENT_SEND_ID_RE.fullmatch( + raw_client_send_id + ): + return JSONResponse( + {"error": "client_send_id must match [A-Za-z0-9_-]{1,128}"}, + status_code=400, + ) + else: + client_send_id = raw_client_send_id if cfg.tenant_check is not None: err_tenant = await asyncio.to_thread(cfg.tenant_check, request, ws_id, mgr) @@ -5108,6 +5752,7 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: ordered_taken=ordered_taken, send_id=pending_msg_id, acting_uid=acting_uid, + client_send_id=client_send_id, defer_fidelity=True, ), ) @@ -5172,6 +5817,8 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: "message": cleaned_display, "priority": pending_priority, "msg_id": pending_msg_id, + **({"sender": acting_uid} if acting_uid else {}), + **({"client_send_id": client_send_id} if client_send_id else {}), }, ) return JSONResponse( @@ -5204,6 +5851,7 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: ordered_taken=ordered_taken, send_id=send_id, acting_uid=acting_uid, + client_send_id=client_send_id, ) session_now = ws.session if session_now is None: @@ -5214,6 +5862,23 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: if window_resp is None: # unreachable: only the barrier probe returns None return JSONResponse({"error": "defer failed"}, status_code=500) return window_resp + if queue_outcome.get("rejected") == "cross_user_interjection": + # A different participant tried to interject into someone else's + # in-flight turn or to claim a fresh turn while that participant's + # persistence-failure-retained input still awaits delivery. + return JSONResponse( + { + "status": "cross_user_interjection", + "error": ( + "Another participant's turn is in progress. Wait for it " + "to finish, then send your message." + ), + "attached_ids": [], + "dropped_attachment_ids": list(requested_ids), + }, + status_code=409, + ) + if not ok: if ws._closed: # ``send`` refused because the workstream closed between our @@ -5241,24 +5906,6 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: } ) - if queue_outcome.get("rejected") == "cross_user_interjection": - # A different participant tried to interject into someone else's - # in-flight turn (see CrossUserInterjectionError). 409 Conflict so - # the client can surface "wait for the current turn" and resend as - # a fresh turn under their own identity. - return JSONResponse( - { - "status": "cross_user_interjection", - "error": ( - "Another participant's turn is in progress. Wait for it " - "to finish, then send your message." - ), - "attached_ids": [], - "dropped_attachment_ids": list(requested_ids), - }, - status_code=409, - ) - dropped = [aid for aid in requested_ids if aid not in taken_set] if queue_outcome: # Reused a live worker; ``queue_message`` succeeded. Best- @@ -5275,6 +5922,8 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: "message": queue_outcome["cleaned"], "priority": queue_outcome["priority"], "msg_id": queue_outcome["msg_id"], + **({"sender": acting_uid} if acting_uid else {}), + **({"client_send_id": client_send_id} if client_send_id else {}), }, ) return JSONResponse( diff --git a/turnstone/core/session_ui_base.py b/turnstone/core/session_ui_base.py index 294bd305..840ef2f1 100644 --- a/turnstone/core/session_ui_base.py +++ b/turnstone/core/session_ui_base.py @@ -35,6 +35,7 @@ import queue import threading import time import uuid +import weakref from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -42,6 +43,7 @@ if TYPE_CHECKING: from collections.abc import Callable from turnstone.core.log import get_logger +from turnstone.core.workstream import session_persistence_state log = get_logger(__name__) @@ -478,6 +480,14 @@ class SessionUIBase: def __init__(self, ws_id: str = "", user_id: str = "") -> None: self.ws_id = ws_id self._user_id = user_id + # The session this UI projects, bound by ChatSession.__init__. Weak: + # SSE generators can outlive retirement holding the UI, and a strong + # back-reference would pin the whole retired session behind them. + # Self-derivation exists so persistence reporting never resolves the + # session through a registry by id — that lookup fails open to + # "healthy" (or to a replacement after id reuse) exactly while + # tombstone retention or retirement has the row out of the map. + self._session_ref: weakref.ref[Any] | None = None # Acting user of the current/last turn (the ``bind_acting_user`` # initiator, owner fallback) — pushed by ``ChatSession._emit_state`` # so web clients can gate cross-user sends on a shared workstream @@ -489,6 +499,12 @@ class SessionUIBase: # SSE listener fan-out — one queue per connected browser tab. self._listeners: list[queue.Queue[dict[str, Any]]] = [] self._listeners_lock = threading.Lock() + # Terminal registration fence, guarded by ``_listeners_lock``. + # Teardown sets this before snapshotting/clearing the current queues; + # a stale events request that already holds the Workstream reference + # but reaches registration afterward receives an internal ws_closed + # sentinel on a non-retained queue instead of blocking forever. + self._listeners_terminal = False # Per-ws event ring buffer for ``Last-Event-ID`` SSE replay. # Holds ``(event_id, event_dict)`` tuples; deque ``maxlen`` # evicts the oldest automatically when the cap is hit. The @@ -777,6 +793,30 @@ class SessionUIBase: # ``conversations.event_id`` rows, corrupting cursor ordering. self._seed_event_id_from_storage() + # ------------------------------------------------------------------ + # Session binding + # ------------------------------------------------------------------ + + def bind_session(self, session: Any) -> None: + """Bind the owning session (called once by ChatSession.__init__).""" + self._session_ref = weakref.ref(session) + + def _bound_session(self) -> Any: + """The bound session, or None before binding / after collection.""" + ref = self._session_ref + return ref() if ref is not None else None + + def _current_persistence_state(self) -> str: + """Sanitized journal state of the UI's own session. + + Derives through the bound session, never a registry lookup by id: + the registry fails open to "healthy" — or to a replacement + workstream after id reuse — exactly while failed-delete tombstone + retention or retirement has the row out of the map, which is + precisely when the operator badge must keep telling the truth. + """ + return session_persistence_state(self._bound_session()) + # ------------------------------------------------------------------ # Listener plumbing (SSE) # ------------------------------------------------------------------ @@ -1229,9 +1269,24 @@ class SessionUIBase: """Create a per-client queue and register it as a listener.""" client_queue: queue.Queue[dict[str, Any]] = _ListenerQueue(maxsize=maxsize) with self._listeners_lock: - self._listeners.append(client_queue) + self._register_or_close_listener_locked(client_queue) return client_queue + def _register_or_close_listener_locked( + self, + client_queue: queue.Queue[dict[str, Any]], + ) -> None: + """Register live, or pre-close a stale post-terminal request. + + Caller holds ``_listeners_lock``. The terminal queue is deliberately + not retained: the events route consumes ``ws_closed`` internally and + exits, while a nonexistent consumer cannot leak a queue on the dead UI. + """ + if getattr(self, "_listeners_terminal", False): + client_queue.put_nowait({"type": "ws_closed"}) + return + self._listeners.append(client_queue) + def _unregister_listener(self, client_queue: queue.Queue[dict[str, Any]]) -> None: """Remove a client queue from the listener list.""" with self._listeners_lock, contextlib.suppress(ValueError): @@ -1283,7 +1338,7 @@ class SessionUIBase: captured_content = list(self._ws_inflight_content) captured_reasoning = list(self._ws_inflight_reasoning) with self._listeners_lock: - self._listeners.append(client_queue) + self._register_or_close_listener_locked(client_queue) snap_seq = self._event_id return client_queue, { "content": "".join(captured_content), @@ -1349,7 +1404,14 @@ class SessionUIBase: text (prevents double-rendering after a truncated emit). ``last_event_id`` semantics: - - ``< earliest_available_id - 1`` → ``"truncated"``. + - ``< 0`` or ``> _event_id`` at the registration boundary → + ``"truncated"``. Per-workstream ids start at 1 and the captured + counter is their authoritative high-water mark, so neither cursor + can have been issued by this stream. Failing closed forces the + caller through its authoritative snapshot/history recovery floor + instead of accepting an empty replay from a forged or corrupt + future cursor. + - Otherwise, ``< earliest_available_id - 1`` → ``"truncated"``. ``lost_count`` is the minimum gap (the buffer may have evicted strictly more than this — we only know the lower bound from what's still retained). @@ -1365,10 +1427,9 @@ class SessionUIBase: distinguishes them: - ``_event_id == 0`` — genuine cold start (brand-new ws, - nothing ever emitted) → ``"replay_ok"`` with an empty - slice. No false ``replay_truncated`` on freshly-opened - workstreams; the ``> 0`` guard also rejects a malformed - negative cursor (``?last_event_id=-1`` parses as an int). + nothing ever emitted). Cursor 0 is the sole valid bootstrap + cursor and returns ``"replay_ok"`` with an empty slice; negative + or future cursors are rejected by the range check above. - ``_event_id > 0`` — this UI instance was rebuilt over an existing conversation (:meth:`_seed_event_id_from_storage` reseeds the counter from ``MAX(conversations.event_id)`` @@ -1383,11 +1444,14 @@ class SessionUIBase: load-bearing) means the client saw everything before the rebuild — no loss, ``"replay_ok"``. - On the empty-ring truncated path ``lost_count`` is the exact - counter gap and ``earliest_available_id`` is ``_event_id + 1`` - (the next id that will exist; nothing below it is retained). - No production client reads either field — they are envelope - forensics — but tests assert them. + On the ordinary stale empty-ring path ``lost_count`` is the exact + counter gap and ``earliest_available_id`` is ``_event_id + 1`` (the + next id that will exist; nothing below it is retained). For an + out-of-range cursor, ``lost_count`` is a conservative numeric floor + (zero for a future cursor) and ``earliest_available_id`` is the first + retained id, or the next counter id when the ring is empty. No + production client reads either field — they are envelope forensics — + but tests assert them. """ client_queue: queue.Queue[dict[str, Any]] = _ListenerQueue(maxsize=maxsize) # Lock order matches writer: ``_ws_lock`` outer, ``_listeners_lock`` @@ -1403,13 +1467,23 @@ class SessionUIBase: captured_reasoning = list(self._ws_inflight_reasoning) with self._listeners_lock: buffered = list(self._event_buffer) - self._listeners.append(client_queue) + self._register_or_close_listener_locked(client_queue) snap_seq = self._event_id snapshot: dict[str, Any] = { "content": "".join(captured_content), "reasoning": "".join(captured_reasoning), "seq": snap_seq, } + if last_event_id < 0 or last_event_id > snap_seq: + # Event ids start at 1, with cursor 0 reserved for the initial + # bootstrap. A negative or beyond-high-water cursor was never + # issued by this UI. Treat it as an uncovered gap so the route + # emits replay_truncated and the client rebuilds from + # authoritative history instead of silently trusting an empty + # replay slice. + earliest_id = buffered[0][0] if buffered else snap_seq + 1 + lost_count = max(0, (earliest_id - 1) - last_event_id) + return client_queue, [], "truncated", lost_count, earliest_id, snapshot if not buffered: # Derived staleness — see the docstring's empty-buffer # section. ``snap_seq`` (the counter captured under the @@ -3748,6 +3822,46 @@ class SessionUIBase: event["preview"] = preview self._enqueue(event) + def on_tool_turn_accepted( + self, + call_id: str, + name: str, + output: str, + *, + is_error: bool = False, + preview: dict[str, Any] | None = None, + effect_status: str | None = None, + ) -> int: + """Publish the canonical accepted TOOL row without replaying metrics. + + ``on_tool_result`` is the executor receipt: it closes the call's + output stream, increments tool metrics, clears activity, and paints a + provisional result immediately. Output guards and truncation run + afterwards, so the durable row may differ. This second hook carries + that final guarded scalar projection to listeners that missed the + receipt and lets reducers replace the provisional rendering in place. + + Deliberately no chunk, activity, or metric bookkeeping lives here. + Replaying any of it would count one accepted tool twice. Structured + image bytes also stay out of the event ring; ``output`` is the same + text-only projection persisted for ``/history`` and attachments remain + content-addressed in storage. + """ + event: dict[str, Any] = { + "type": "tool_result", + "accepted": True, + "call_id": call_id, + "name": name, + "output": output, + } + if is_error: + event["is_error"] = True + if preview: + event["preview"] = preview + if effect_status: + event["effect_status"] = effect_status + return self._enqueue(event) + def on_tool_output_chunk(self, call_id: str, chunk: str) -> None: """Buffer one tool-output line into the per-call chunk batcher. @@ -3945,6 +4059,45 @@ class SessionUIBase: self._flush_all_chunk_batches_locked() self._enqueue({"type": "error", "message": message}) + def on_history_resync(self, reason: str) -> int: + """Tell connected panes to refetch the authoritative history view. + + Reserved for exceptional repair paths: older/custom UIs without the + typed ``user_turn`` hook, history truncation, or an accepted row that + does not reach an unambiguous acknowledgement. + """ + return self._enqueue({"type": "history_resync", "reason": reason}) + + def on_user_turn( + self, + content: str, + *, + attachments: list[dict[str, Any]], + sender: str | None, + source: str | None, + client_send_ids: list[str], + ) -> int: + """Publish one accepted user row to every connected pane. + + ``client_send_ids`` correlate this canonical event with optimistic + bubbles in the sending browser. They are not idempotency keys; the + event's monotonic ``_event_id`` remains the durable row identity. + Peer panes render the same event directly, including sender, + synthetic-source, and attachment metadata. + """ + event: dict[str, Any] = { + "type": "user_turn", + "content": content, + "client_send_ids": list(client_send_ids), + } + if attachments: + event["attachments"] = [dict(item) for item in attachments] + if sender: + event["sender"] = sender + if source: + event["source"] = source + return self._enqueue(event) + def on_system_turn( self, content: str, source: str, meta: dict[str, Any] | None = None ) -> int | None: @@ -4240,3 +4393,22 @@ class SessionUIBase: "activity_state": activity_state, "content": content, } + + def snapshot_state_payload_non_consuming(self) -> dict[str, Any]: + """Locked counters snapshot for observational operator-row refreshes. + + The non-consuming sibling of :meth:`snapshot_and_consume_state_payload` + for hooks that must NOT touch the terminal turn-content accumulator + (the persistence refresh reports a journal transition, not a state + transition). Both kinds' ``on_persistence_state_changed`` read + through here so the interactive dashboard and the console cluster + row cannot silently disagree after the same journal event — a + snapshot field added here reaches both surfaces at once. + """ + with self._ws_lock: + return { + "tokens": self._ws_prompt_tokens + self._ws_completion_tokens, + "context_ratio": self._ws_context_ratio, + "activity": self._ws_current_activity, + "activity_state": self._ws_activity_state, + } diff --git a/turnstone/core/session_worker.py b/turnstone/core/session_worker.py index 76dbbf2b..09cdcbdc 100644 --- a/turnstone/core/session_worker.py +++ b/turnstone/core/session_worker.py @@ -18,8 +18,9 @@ with no consumer. The flag transitions atomically inside the same lock this module holds, so both coord and interactive callers inherit the fix. -This module owns ONLY the dispatch decision, the ``_worker_running`` -lifecycle, and the ownership-clear wake backstop +This module owns ONLY the dispatch decision, immutable slot-time session +claim, the ``_worker_running``/force-abandonable lifecycle, and the +ownership-clear wake backstop (:func:`_retry_pending_wake`). Per-kind concerns — session resolution, attachment resolution, error surfacing, UI callbacks, ``GenerationCancelled`` handling — live in the caller's @@ -38,9 +39,11 @@ closes the window without ever racing a competing worker. from __future__ import annotations +import contextvars +import dataclasses import queue import threading -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from turnstone.core.log import get_logger @@ -52,8 +55,132 @@ if TYPE_CHECKING: log = get_logger(__name__) -def _retry_pending_wake(ws: Workstream) -> None: - """Deliver nudges that arrived while the exiting worker owned *ws*. +def foreign_queue_conflict(session: object, principal_id: str) -> bool: + """Shared ``before_spawn`` predicate for every fresh-turn dispatch gate. + + True when another participant's persistence-retained queued input must + refuse this spawn. One predicate serves the /send route, the destructive + retry dispatcher, and the coordinator adapter — a change to which owners + count as foreign lands once (round-5 review: three hand copies). + Per-surface refusal REPORTING (409 status, outcome dict, log line) stays + at the call sites. Unauthenticated dispatch (empty principal) passes: the + partitioned pop retains foreign rows structurally, so the gate is a + courtesy refusal for the authenticated lanes, not the enforcement. + """ + from turnstone.core.workstream import concrete_method + + if not principal_id: + return False + conflict = concrete_method(session, "has_foreign_queued_messages") + return conflict is not None and bool(conflict(principal_id)) + + +def claimed_slot_queue_admission( + ws: Workstream, + acting_user_id: str, +) -> tuple[str, dict[str, Any]] | None: + """Slot-owner queue admission shared by /send and the coordinator adapter. + + Called under ``ws._lock``. Returns ``(claimed_principal, base queue + kwargs)`` threading the immutable slot owner as ``turn_principal_id`` — + or ``None`` when a DIFFERENT authenticated participant holds the slot: + the claim captured with the slot is authoritative even before the new + worker thread finishes rebinding the ChatSession, whose sticky actor may + still name the previous turn (round-5 review: two hand copies of this + comparison + threading). + """ + claimed_principal = ws._worker_principal_id + if acting_user_id and claimed_principal and acting_user_id != claimed_principal: + return None + kwargs: dict[str, Any] = {"interjector_user_id": acting_user_id} + if claimed_principal: + kwargs["turn_principal_id"] = claimed_principal + return claimed_principal, kwargs + + +def release_slot_locked(ws: Workstream, *, keep_thread: bool = False) -> None: + """Return the worker slot to its unclaimed state — THE release field set. + + Caller holds ``ws._lock`` and has already verified the release is + legitimate (thread-identity / abandonable checks are per-site policy; + the field-set invariant is owned here). This module declares itself + sole owner of the slot lifecycle: a claim field added to the spawn + branch must land here in the same change, or a released slot keeps a + stale value that the next admission reads as live (e.g. queue + admission comparing against a departed principal) — and the NORMAL + worker exit is the most common release, so a hand-copy there is the + likeliest site to go stale. + + ``keep_thread=True`` is the ordinary ``_runner`` exit: the claim + fields clear but ``ws.worker_thread`` deliberately keeps pointing at + the finished thread (late owner-identity checks in that thread's own + closures compare against it; the next spawn overwrites it under this + same lock). + """ + if not keep_thread: + ws.worker_thread = None + ws._worker_running = False + ws._worker_principal_id = "" + ws._worker_force_abandonable = True + + +def reclassify_slot_locked( + ws: Workstream, + *, + worker_kind: WorkerKind, + principal_id: str, + force_abandonable: bool, +) -> None: + """Reclassify a HELD slot in place — the retry command→turn flip. + + Caller holds ``ws._lock`` and has verified it still owns the slot. + Same single-ownership rule as :func:`release_slot_locked`: the + classification field set lives here so the spawn branch and the + reclassify site cannot drift. + """ + ws.worker_kind = worker_kind + ws._worker_principal_id = principal_id.strip() + ws._worker_force_abandonable = force_abandonable + + +@dataclasses.dataclass(frozen=True, slots=True) +class WorkerClaim: + """Immutable ChatSession admission captured with a fresh worker slot. + + ``session`` is an identity fence, not retained application state: it keeps + a nested/direct send for another session from accidentally consuming this + thread's claim. ``cancel_epoch`` linearizes the slot claim with Stop and + terminal admission. ``cancel_event`` plus its captured state detects an + exceptional structural poison that lands after capture without rejecting + an Event already set by a completed history truncation. + """ + + session: object = dataclasses.field(repr=False) + principal_id: str + cancel_epoch: int + cancel_event: threading.Event = dataclasses.field(repr=False) + cancel_event_was_set: bool + + +_active_worker_claim: contextvars.ContextVar[WorkerClaim | None] = contextvars.ContextVar( + "turnstone_active_worker_claim", + default=None, +) + + +def current_worker_claim(session: object) -> WorkerClaim | None: + """Return this thread's slot-time claim when it belongs to *session*.""" + + claim = _active_worker_claim.get() + return claim if claim is not None and claim.session is session else None + + +def _retry_pending_wake( + ws: Workstream, + *, + exclude_interjection_signature: object | None = None, +) -> None: + """Deliver nudges or a tail-raced interjection after ownership clears. Runs in the worker's ``finally`` immediately after it cleared ``_worker_running`` (owner only — abandoned threads skip it). The @@ -65,6 +192,12 @@ def _retry_pending_wake(ws: Workstream) -> None: The same window covers a watch ``wake_fn`` firing while a worker is mid-exit. + The same seam closes a user-send race: an interjection can enqueue after + ``ChatSession.send`` performed its final flush but before this runner's + ``finally`` clears the slot. The worker-exit call uniquely enables the + gate's session-owned queue snapshot claim; ordinary idle/watch callers + remain nudge-only, and a restored failed snapshot is not retried in a loop. + The wake gate (:func:`~turnstone.core.idle_nudge_watcher.wake_workstream_if_pending`) owns every defensive check — session missing, bare stub without a @@ -82,7 +215,12 @@ def _retry_pending_wake(ws: Workstream) -> None: from turnstone.core.idle_nudge_watcher import wake_workstream_if_pending try: - wake_workstream_if_pending(ws, trigger="worker-exit") + wake_workstream_if_pending( + ws, + trigger="worker-exit", + include_interjections=True, + exclude_interjection_signature=exclude_interjection_signature, + ) except Exception: log.warning("session_worker.wake_retry_failed ws=%s", ws.id[:8], exc_info=True) @@ -92,8 +230,13 @@ def send( *, enqueue: Callable[[], None], run: Callable[[], None], + expected_session: object | None = None, + before_spawn: Callable[[], bool] | None = None, thread_name: str | None = None, worker_kind: WorkerKind = "turn", + principal_id: str = "", + force_abandonable: bool = True, + interjection_wake_signature: object | None = None, ) -> bool: """Dispatch work onto a workstream's worker thread. @@ -103,9 +246,11 @@ def send( ``ws._worker_running`` (set before lock release, cleared in the spawned thread's ``finally`` block). - Both callbacks are no-arg closures — callers close over the - ``ChatSession`` they want to drive, so the worker can't be racing a - concurrent ``ws.session`` swap. + Both callbacks are no-arg closures. Production callers pass the exact + closure-bound ``ChatSession`` as ``expected_session``; identity is checked + under ``ws._lock`` before either callback can run, so a concurrent + ``ws.session`` swap refuses the stale dispatch. The optional default keeps + low-level compatibility sessions and test doubles on their legacy path. ``worker_kind`` classifies what the slot holds — ``"turn"`` (send / retry / wake / init, the default) or ``"command"`` (slash-command @@ -137,6 +282,33 @@ def send( race-free; the cost is the contract change, not atomicity.) If you add a NEW enqueue closure that can queue turn work, copy the guard. + ``principal_id`` is the authenticated owner of a fresh turn slot. It is + installed atomically with the worker claim, before the spawned thread does + the broader ChatSession actor rebind. Queueing callers can therefore + reject a different participant without consulting stale mutable session + identity. Internal and unauthenticated workers leave it empty. + + ``force_abandonable=False`` reserves the slot across operator force-cancel. + Use it for destructive history/lifecycle mutations whose half-completed + transaction cannot safely overlap a successor. The cancel path may still + signal cooperative cancellation, but must leave the worker/thread/slot + claim intact until this runner's owner-conditional ``finally`` clears it. + + ``interjection_wake_signature`` is the exact queue snapshot that caused a + queue-only wake spawn. It is carried only by that successfully spawned + runner; on exit the backstop excludes the same restored snapshot, avoiding + a preamble-failure hot loop while still admitting any changed queue. A + reuse/refusal/spawn-error path starts no runner and therefore installs no + suppression that could hide work from a competing real worker's exit. + + ``before_spawn`` is an optional admission check that runs under the same + ``ws._lock`` acquisition immediately before a fresh slot is claimed. It + may inspect queue-owner state, but must not acquire the session generation + lock or manager state locks: generation commits can publish through the UI + while holding that lock and then acquire ``ws._lock``. Returning ``False`` + refuses the dispatch without spawning. Reuse-path admission remains the + caller's ``enqueue`` responsibility. + Returns: ``True`` on successful enqueue (existing worker accepted) or thread spawn (no live worker). @@ -154,7 +326,30 @@ def send( """ name = thread_name or f"session-worker-{ws.id[:8]}" + worker_claim: WorkerClaim | None = None + + # Capture the ChatSession's monotonic cancellation edge before taking the + # workstream lock. Generation commits can reach UI state publication while + # holding ``_generation_lock`` and then acquire ``ws._lock``; doing this + # capture in the opposite order creates a concrete AB/BA deadlock. A Stop + # or terminal transition between this conservative snapshot and slot + # installation only advances the epoch, so send entry rejects the stale + # witness rather than erasing that lifecycle edge. + claim_session = expected_session if expected_session is not None else ws.session + capture_claim = getattr(claim_session, "_capture_worker_claim", None) + if callable(capture_claim): + try: + worker_claim = capture_claim(principal_id) + except Exception: + log.info( + "session_worker.claim_refused ws=%s", + ws.id[:8], + exc_info=True, + ) + return False + def _runner() -> None: + claim_token = _active_worker_claim.set(worker_claim) try: run() except Exception: @@ -169,6 +364,7 @@ def send( # close style signals if the runtime ever delivers them). log.exception("session_worker.uncaught ws=%s", ws.id[:8]) finally: + _active_worker_claim.reset(claim_token) was_owner = False with ws._lock: # Only clear the flag if THIS thread is still the current @@ -180,13 +376,16 @@ def send( # else a third send sees ``_worker_running=False`` and # spawns a second concurrent worker on the same session. if ws.worker_thread is threading.current_thread(): - ws._worker_running = False + release_slot_locked(ws, keep_thread=True) was_owner = True # Outside the lock (the retry's wake dispatch re-acquires it). # Owner only: an abandoned thread retrying would race the # successor's own exit backstop for no benefit. if was_owner: - _retry_pending_wake(ws) + _retry_pending_wake( + ws, + exclude_interjection_signature=interjection_wake_signature, + ) with ws._lock: if ws._closed: @@ -198,6 +397,33 @@ def send( # writes — on a workstream whose ``ws_closed`` already fired. log.info("session_worker.closed_refused ws=%s", ws.id[:8]) return False + if expected_session is not None and ws.session is not expected_session: + # Callbacks close over ``expected_session``. This identity fence is + # required even for compatibility sessions that expose no + # WorkerClaim: capturing a valid witness from a replacement cannot + # authorize mutations through closures still bound to its detached + # predecessor. + log.info("session_worker.session_swap_refused ws=%s", ws.id[:8]) + return False + if isinstance(worker_claim, WorkerClaim): + # The claim was captured before ``ws._lock`` to preserve the + # generation -> UI -> workstream lock order. A predecessor can + # poison structural cleanup in that gap and then either still own + # this slot (enqueue arm) or release it (spawn arm). Revalidate + # only through ChatSession's explicitly lock-free witness check; + # reacquiring its generation lock here would invert that order. + revalidate = getattr(worker_claim.session, "_worker_claim_is_current", None) + try: + claim_is_current = bool( + worker_claim.session is ws.session + and callable(revalidate) + and revalidate(worker_claim) + ) + except Exception: + claim_is_current = False + if not claim_is_current: + log.info("session_worker.stale_claim_refused ws=%s", ws.id[:8]) + return False if ws._worker_running: try: enqueue() @@ -219,9 +445,20 @@ def send( exc_info=True, ) return False - # Set ``_worker_running`` AND assign ``ws.worker_thread`` under - # the same lock acquisition — readers gating on either flag see - # a coherent (worker_thread, _worker_running) pair. Without + if before_spawn is not None: + try: + if not before_spawn(): + return False + except Exception: + log.warning( + "session_worker.spawn_admission_failed ws=%s", + ws.id[:8], + exc_info=True, + ) + return False + # Set ``_worker_running``, actor identity, and ``ws.worker_thread`` + # under the same lock acquisition — readers gating on the running + # flag see one coherent slot claim. Without # this, a reader could observe ``_worker_running=True`` while # ``ws.worker_thread`` still points at the previous (already- # exited) thread, breaking every ``ws.worker_thread is me`` @@ -234,6 +471,8 @@ def send( # write either way. ws._worker_running = True ws.worker_kind = worker_kind + ws._worker_principal_id = principal_id.strip() + ws._worker_force_abandonable = force_abandonable t = threading.Thread(target=_runner, name=name, daemon=True) ws.worker_thread = t # ``t.start()`` may run user code (worker body) before returning; @@ -258,8 +497,7 @@ def send( # queue-full backpressure. with ws._lock: if ws.worker_thread is t: - ws.worker_thread = None - ws._worker_running = False + release_slot_locked(ws) log.exception("session_worker.spawn_failed ws=%s — slot released", ws.id[:8]) raise return True diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index 959031e8..ff734b4f 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -157,7 +157,8 @@ def _build_registry() -> dict[str, SettingDef]: "session.retention_days", "int", 90, - "Days to retain conversation history (0 = disabled)", + "Days to keep unnamed workstreams (0 = no age pruning; " + "empty unnamed workstreams are still removed after a two-hour grace)", "session", min_value=0, ), diff --git a/turnstone/core/storage/__init__.py b/turnstone/core/storage/__init__.py index af09d983..ca94da78 100644 --- a/turnstone/core/storage/__init__.py +++ b/turnstone/core/storage/__init__.py @@ -4,6 +4,9 @@ Supports SQLite (default, zero-config) and PostgreSQL (multi-node, production). """ from turnstone.core.storage._protocol import ( + AttachmentWrite, + ConversationCommitConflictError, + ConversationCommitWorkstreamGoneError, ForkCloneError, ForkCloneExpectation, ForkCloneSnapshot, @@ -21,6 +24,9 @@ from turnstone.core.storage._registry import ( ) __all__ = [ + "AttachmentWrite", + "ConversationCommitConflictError", + "ConversationCommitWorkstreamGoneError", "StorageBackend", "StorageConflictError", "ForkCloneError", diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index 60f74853..409810cd 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -18,11 +18,14 @@ if TYPE_CHECKING: from turnstone.core.trajectory import Turn import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import insert as postgresql_insert from turnstone.core.log import get_logger from turnstone.core.storage._protocol import ( FORK_RESERVATION_CONFIG_KEY, USER_SCOPED_AUTH_TYPES, + AttachmentWrite, + ConversationCommitWorkstreamGoneError, ForkCloneExpectation, ForkCloneSnapshot, MCPOAuthPendingState, @@ -136,6 +139,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( VERDICT_MUTABLE as _VERDICT_MUTABLE, ) +from turnstone.core.storage._utils import ( + KeyedAttachmentSaveWrappers as _KeyedAttachmentSaveWrappers, +) from turnstone.core.storage._utils import ( assert_single_default_persona as _assert_single_default_persona, ) @@ -146,16 +152,25 @@ from turnstone.core.storage._utils import ( clone_workstream_transaction, find_orphan_conversations, parse_checkpoint_watermark, + prepare_attachment_commit, + prepare_conversation_row_values, prepare_provider_data_for_save, purge_orphan_conversations, release_attachment_refs, retain_attachment_refs, sanitize_text, + save_attachment_commit_transaction, senders_from_user_meta, ) +from turnstone.core.storage._utils import ( + delete_messages_after_core as _delete_messages_after_core, +) from turnstone.core.storage._utils import ( escape_like as _escape_like, ) +from turnstone.core.storage._utils import ( + get_compaction_floor_on_connection as _get_compaction_floor_shared, +) from turnstone.core.storage._utils import ( normalize_search_terms as _normalize_search_terms, ) @@ -165,6 +180,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( persona_row_to_dict as _persona_row_to_dict, ) +from turnstone.core.storage._utils import ( + prune_workstreams_shared as _prune_workstreams_shared, +) from turnstone.core.storage._utils import ( reconstruct_messages as _reconstruct_messages, ) @@ -174,6 +192,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( recover_trajectory as _recover_trajectory, ) +from turnstone.core.storage._utils import ( + resolve_keyed_commit_conflict as _resolve_keyed_commit_conflict, +) from turnstone.core.storage._utils import ( row_to_dict as _row_to_dict, ) @@ -186,6 +207,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( split_perms as _split_perms, ) +from turnstone.core.storage._utils import ( + truncate_messages_tail_core as _truncate_messages_tail_core, +) from turnstone.core.storage._utils import ( validate_and_clear_default_persona as _validate_and_clear_default_persona, ) @@ -289,7 +313,7 @@ class _PostgreSQLNotifyStream: conn.close() -class PostgreSQLBackend: +class PostgreSQLBackend(_KeyedAttachmentSaveWrappers): """PostgreSQL implementation of the StorageBackend protocol.""" def __init__( @@ -365,39 +389,143 @@ class PostgreSQLBackend: is_error: bool = False, producer: str | None = None, meta: str | None = None, + commit_key: str | None = None, ) -> int: now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") - content = sanitize_text(content) - provider_data = prepare_provider_data_for_save( - role, sanitize_text(provider_data), tool_calls, producer + values = prepare_conversation_row_values( + ws_id, + role, + content, + tool_name=tool_name, + tool_call_id=tool_call_id, + provider_data=provider_data, + tool_calls=tool_calls, + source=source, + event_id=event_id, + is_error=is_error, + producer=producer, + meta=meta, + commit_key=commit_key, + now=now, ) - source = sanitize_text(source) with self._conn() as conn: - result = conn.execute( - sa.insert(conversations) - .values( - ws_id=ws_id, - timestamp=now, - role=role, - content=content, - tool_name=tool_name, - tool_call_id=tool_call_id, - provider_data=provider_data, - tool_calls=tool_calls, - _source=source, - event_id=event_id, - is_error=is_error, - meta=meta, + inserted = True + parent_observed = None + if commit_key is None: + # A non-locking MVCC observation distinguishes a call that + # genuinely began parentless from one that arrived while prune + # held (and would shortly delete) an existing parent. At READ + # COMMITTED an uncommitted delete remains visible here. + parent_observed = conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id == ws_id) + ).fetchone() + # Hard delete and every conditional delete lock this durable row + # before scanning/deleting conversations. Every writer takes the + # same parent-first order when the row exists. This closes the + # PostgreSQL READ COMMITTED anomaly where prune's earlier + # NOT EXISTS snapshot could delete the parent while an unlocked + # NULL-key insert became visible only afterwards. + # + # A call that genuinely begins without a parent remains a + # deliberate legacy/offline seam for NULL keys. A call that saw a + # parent but wakes after prune deleted it is refused below rather + # than reclassified as a parentless import. Keyed live admission + # always refuses a missing parent. + parent = conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id == ws_id).with_for_update() + ).fetchone() + if commit_key is None and parent_observed is not None and parent is None: + raise RuntimeError("legacy conversation append crossed workstream deletion") + if parent is None and commit_key is not None: + raise ConversationCommitWorkstreamGoneError( + "keyed conversation commit workstream no longer exists" + ) + statement = postgresql_insert(conversations).values(**values) + if commit_key is not None: + statement = statement.on_conflict_do_nothing( + index_elements=[conversations.c.ws_id, conversations.c.commit_key], + index_where=conversations.c.commit_key.is_not(None), + ) + result = conn.execute(statement.returning(conversations.c.id)) + resolved = result.scalar_one_or_none() + if resolved is None: + if commit_key is None: + raise RuntimeError("save_message: row id missing after insert") + inserted = False + rowid = _resolve_keyed_commit_conflict(conn, ws_id, values) + else: + rowid = int(resolved) + if inserted: + conn.execute( + sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now) ) - .returning(conversations.c.id) - ) - rowid = int(result.scalar_one()) - conn.execute( - sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now) - ) conn.commit() return rowid + def _save_message_with_attachments( + self, + ws_id: str, + role: str, + content: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + tool_name: str | None = None, + tool_call_id: str | None = None, + source: str | None = None, + event_id: int | None = None, + is_error: bool = False, + meta: str | None = None, + commit_key: str, + origin: str, + exact_blob_metadata: bool, + ) -> int: + """Dialect-local transaction shared by keyed USER and TOOL rows.""" + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + attachment_ids, blobs, values = prepare_attachment_commit( + ws_id, + role, + content, + attachments, + tool_name=tool_name, + tool_call_id=tool_call_id, + source=source, + event_id=event_id, + is_error=is_error, + meta=meta, + commit_key=commit_key, + now=now, + ) + with self._conn() as conn: + try: + # Match hard-delete's durable-row-first lock order so no + # attachment-bearing insert can commit behind deletion. Holding + # it for the shared body below makes the parent check and every + # row/blob/refcount mutation one indivisible transaction. + parent = conn.execute( + sa.select(workstreams.c.ws_id) + .where(workstreams.c.ws_id == ws_id) + .with_for_update() + ).fetchone() + if parent is None: + raise ConversationCommitWorkstreamGoneError( + "keyed conversation commit workstream no longer exists" + ) + row_id = save_attachment_commit_transaction( + conn, + postgresql_insert, + values=values, + attachment_ids=attachment_ids, + blobs=blobs, + now=now, + origin=origin, + exact_blob_metadata=exact_blob_metadata, + ) + conn.commit() + return row_id + except Exception: + conn.rollback() + raise + def list_message_senders(self, ws_id: str) -> list[str]: # DISTINCT on the raw meta blob: a user row's meta carries only # {"sender": ...}, so distinct blobs ≈ distinct senders and the JSON @@ -452,21 +580,58 @@ class PostgreSQLBackend: } ) with self._conn() as conn: + self._lock_parents_refusing_crossed_deletion(conn, ws_ids) retain_attachment_refs(conn, attachment_ids) conn.execute(sa.insert(conversations), insert_rows) - for wid in ws_ids: - conn.execute( - sa.update(workstreams).where(workstreams.c.ws_id == wid).values(updated=now) - ) + conn.execute( + sa.update(workstreams) + .where(workstreams.c.ws_id.in_(sorted(ws_ids))) + .values(updated=now) + ) conn.commit() + def _lock_parents_refusing_crossed_deletion( + self, + conn: sa.engine.Connection, + ws_ids: set[str], + ) -> None: + """Batched READ COMMITTED anomaly gate for the bulk import path. + + The set-shaped twin of ``save_message``'s single-row + observed→lock→refuse sequence (its NULL-commit-key arm) — keep the + two semantically in lockstep. Matches prune/delete's parent-first + order; the ``ORDER BY ws_id`` on the locking read preserves the + sorted lock order that keeps concurrent bulk writers from + deadlocking one another. Missing parents retain the historical + import behavior and do not abort the batch; a parent OBSERVED but + not lockable crossed a concurrent deletion and refuses. + """ + ordered = sorted(ws_ids) + observed = { + row[0] + for row in conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id.in_(ordered)) + ).fetchall() + } + locked = { + row[0] + for row in conn.execute( + sa.select(workstreams.c.ws_id) + .where(workstreams.c.ws_id.in_(ordered)) + .order_by(workstreams.c.ws_id) + .with_for_update() + ).fetchall() + } + if observed - locked: + raise RuntimeError("legacy conversation append crossed workstream deletion") + def _conversation_rows( self, ws_id: str, limit: int | None ) -> tuple[list[tuple[Any, ...]], dict[int, list[dict[str, Any]]] | None]: """Fetch a ws's conversation rows + resolved attachment map (shared by - :meth:`load_messages` and :meth:`load_message_turns`). The trailing - ``attachments`` ref-list column is split off and is NOT part of the - positional tuple ``reconstruct_*`` unpacks (id..meta).""" + :meth:`load_messages` and :meth:`load_message_turns`). The trailing + ``attachments`` ref-list and ``commit_key`` columns stay internal to + reconstruction.""" _cols = ( conversations.c.id, conversations.c.role, @@ -480,6 +645,7 @@ class PostgreSQLBackend: conversations.c.is_error, conversations.c.meta, conversations.c.attachments, + conversations.c.commit_key, ) with self._conn() as conn: if limit is not None and limit > 0: @@ -650,27 +816,7 @@ class PostgreSQLBackend: twin). ``0`` when the ws never compacted. """ with self._conn() as conn: - marker_id = conn.execute( - sa.select(sa.func.max(conversations.c.id)).where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c._source == _COMPACTION_SOURCE, - ) - ) - ).scalar() - if marker_id is None: - return 0 - n = conn.execute( - sa.select(sa.func.count()) - .select_from(conversations) - .where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c.id <= marker_id, - ) - ) - ).scalar() - return int(n or 0) + return _get_compaction_floor_shared(conn, ws_id) def get_compaction_checkpoint(self, ws_id: str) -> int | None: """Latest persisted marker's watermark — see the protocol docstring. @@ -689,45 +835,65 @@ class PostgreSQLBackend: ).fetchone() return parse_checkpoint_watermark(row[0]) if row is not None else None + def _delete_messages_after_on_connection( + self, + conn: sa.engine.Connection, + ws_id: str, + keep_count: int, + ) -> int: + """Delete one conversation tail; caller owns the parent lock.""" + return _delete_messages_after_core(conn, ws_id, keep_count) + def delete_messages_after(self, ws_id: str, keep_count: int) -> int: with self._conn() as conn: - cutoff_row = conn.execute( - sa.select(conversations.c.id) - .where(conversations.c.ws_id == ws_id) - .order_by(conversations.c.id) - .limit(1) - .offset(keep_count) + # Keyed commits, hard delete, and prune all lock the durable parent + # before touching conversation rows. Take the same lock for a tail + # truncation. A missing legacy parent has no row to lock; continue + # for orphan-truncation compatibility. Fencing same-id recreation + # across that gap requires a separate incarnation boundary. + conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id == ws_id).with_for_update() ).fetchone() - if cutoff_row is None: - return 0 - cutoff_id = cutoff_row[0] - # Refcount GC: read the doomed rows' content-addressed ref-lists, - # decrement each blob's refcount once per reference, and prune - # blobs that hit 0 — so a deduped blob still referenced by a kept - # turn survives. Replaces the old message_id-cascade delete. - doomed = conn.execute( - sa.select(conversations.c.attachments).where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c.id >= cutoff_id, - conversations.c.attachments.is_not(None), - ) - ) - ).fetchall() - doomed_ids: list[str] = [] - for (refs,) in doomed: - doomed_ids.extend(_parse_attachment_refs(refs)) - release_attachment_refs(conn, doomed_ids) - result = conn.execute( - sa.delete(conversations).where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c.id >= cutoff_id, - ) - ) - ) + deleted = self._delete_messages_after_on_connection(conn, ws_id, keep_count) conn.commit() - return result.rowcount + return deleted + + def truncate_messages_tail(self, ws_id: str, remove_count: int) -> int: + """Atomically remove a compaction-floored number of newest rows.""" + if remove_count < 0: + raise ValueError("remove_count must be non-negative") + with self._conn() as conn: + try: + # A truncation runs on a non-abandonable worker slot: operator + # force-cancel deliberately refuses to supersede it, so an + # unbounded FOR UPDATE wait here would pin the workstream until + # node restart. Bound the wait; timing out surfaces as an + # ordinary persist failure the rewind route reports as + # retryable while the slot is released. + conn.execute(sa.text("SET LOCAL lock_timeout = '10s'")) + # Every keyed conversation commit takes this row lock first. + # Hold it across both count queries and the exact tail delete so + # another process cannot turn ``remove_count`` into an + # over-delete by committing in between them. + parent = conn.execute( + sa.select(workstreams.c.ws_id) + .where(workstreams.c.ws_id == ws_id) + .with_for_update() + ).fetchone() + if parent is None: + raise RuntimeError("tail truncation workstream no longer exists") + + deleted = _truncate_messages_tail_core( + conn, + ws_id, + remove_count, + delete_after=self._delete_messages_after_on_connection, + ) + conn.commit() + return deleted + except Exception: + conn.rollback() + raise # -- Workstream management ------------------------------------------------- @@ -788,61 +954,73 @@ class PostgreSQLBackend: ).fetchall() ) - def prune_workstreams(self, retention_days: int = 90) -> tuple[int, int]: - orphans = stale = 0 + def _delete_prune_candidate( + self, + ws_id: str, + predicates: tuple[Any, ...], + ) -> bool: + """Recheck and delete one prune candidate in its own transaction. + + Candidate admission takes the same durable-row-first lock as all + conversation writers and hard deletion. ``SKIP LOCKED`` leaves a + workstream a writer already owns to the next prune; a writer that + arrives after admission waits for this transaction instead: keyed + admission then fails closed, and a NULL-key writer that observed the + pre-delete parent also refuses rather than becoming an invisible + orphan. + + The exact predicate is rechecked as a second statement so it reads a + fresh READ COMMITTED snapshot rather than the lock statement's — a + commit that landed while this transaction waited for the row is + therefore visible, and its workstream is no longer a candidate. One + transaction per candidate keeps that lock and the per-workstream + attachment GC off every unrelated keyed commit for the rest of the run. + """ with self._conn() as conn: - # 1. Remove workstreams with no messages - orphan_rows = conn.execute( - sa.text( - "SELECT ws_id FROM workstreams " - "WHERE state != 'creating' AND NOT EXISTS " - " (SELECT 1 FROM conversations c " - " WHERE c.ws_id = workstreams.ws_id)" - ) - ).fetchall() - orphan_ids = [r[0] for r in orphan_rows] - if orphan_ids: - chunk_size = 10_000 - for i in range(0, len(orphan_ids), chunk_size): - chunk = orphan_ids[i : i + chunk_size] + try: + locked = conn.execute( + sa.select(workstreams.c.ws_id) + .where(workstreams.c.ws_id == ws_id) + .with_for_update(skip_locked=True) + ).fetchone() + exact = ( conn.execute( - sa.delete(workstream_config).where(workstream_config.c.ws_id.in_(chunk)) - ) - result = conn.execute( - sa.delete(workstreams).where(workstreams.c.ws_id.in_(chunk)) - ) - orphans += result.rowcount - - # 2. Remove old unnamed workstreams - if retention_days > 0: - cutoff = (datetime.now(UTC) - timedelta(days=retention_days)).strftime( - "%Y-%m-%dT%H:%M:%S" + sa.select(workstreams.c.ws_id).where( + workstreams.c.ws_id == ws_id, + *predicates, + ) + ).fetchone() + if locked is not None + else None ) - stale_rows = conn.execute( - sa.select(workstreams.c.ws_id).where( - workstreams.c.state != "creating", - workstreams.c.alias.is_(None), - workstreams.c.updated < cutoff, - ) - ).fetchall() - stale_ids = [r[0] for r in stale_rows] - if stale_ids: - chunk_size = 10_000 - for i in range(0, len(stale_ids), chunk_size): - chunk = stale_ids[i : i + chunk_size] - conn.execute( - sa.delete(conversations).where(conversations.c.ws_id.in_(chunk)) - ) - conn.execute( - sa.delete(workstream_config).where(workstream_config.c.ws_id.in_(chunk)) - ) - result = conn.execute( - sa.delete(workstreams).where(workstreams.c.ws_id.in_(chunk)) - ) - stale += result.rowcount + deleted = bool( + exact is not None and self._delete_workstream_on_connection(conn, ws_id) + ) + conn.commit() + return deleted + except Exception: + conn.rollback() + raise - conn.commit() - return (orphans, stale) + def prune_workstreams(self, retention_days: int = 90) -> tuple[int, int]: + # Discovery holds no row locks: every candidate is relocked and + # rechecked in its own bounded transaction by + # ``_delete_prune_candidate``, so a long prune never blocks keyed + # commits to workstreams it has not reached yet. + def _select_ids(predicates: tuple[Any, ...]) -> list[str]: + with self._conn() as conn: + return [ + str(row[0]) + for row in conn.execute( + sa.select(workstreams.c.ws_id).where(*predicates) + ).fetchall() + ] + + return _prune_workstreams_shared( + retention_days, + select_ids=_select_ids, + delete_candidate=self._delete_prune_candidate, + ) def resolve_workstream(self, alias_or_id: str) -> str | None: with self._conn() as conn: diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 2c299ffd..76d0afe1 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -41,6 +41,43 @@ class StorageConflictError(Exception): """ +class ConversationCommitConflictError(StorageConflictError): + """A keyed conversation retry does not match the committed operation. + + A ``commit_key`` identifies one immutable logical write. Returning the + existing row for a retry with different role-specific fields, metadata, + event cursor, or ordered attachment references would acknowledge data the + caller did not commit, so atomic attachment paths refuse that mismatch. + """ + + +class ConversationCommitWorkstreamGoneError(RuntimeError): + """A keyed conversation commit's durable parent row no longer exists. + + Keyed saves refuse to recreate a hard-deleted workstream, so this failure + is permanent: no retry of the same commit can ever succeed. The durability + journal uses the type to stop retrying instead of classifying the miss as + a transient storage failure. + """ + + +@dataclass(frozen=True, slots=True) +class AttachmentWrite: + """One content-addressed blob reference in an atomic conversation write. + + The ordered input sequence is the message's reference list. Repeated + ``attachment_id`` values therefore represent repeated references and each + contribute one to the global refcount. + """ + + attachment_id: str + filename: str + mime_type: str + size_bytes: int + kind: str + content: bytes + + class ForkCloneError(RuntimeError): """Base class for an atomic workstream-clone refusal.""" @@ -251,6 +288,7 @@ class StorageBackend(Protocol): is_error: bool = False, producer: str | None = None, meta: str | None = None, + commit_key: str | None = None, ) -> int: """Log a message to the conversations table. @@ -268,9 +306,82 @@ class StorageBackend(Protocol): cursor space, distinct from the returned ``id`` PK. NULL when the caller has no live UI counter (offline / bulk / fork re-saves). - ``meta`` is the pre-serialized JSON of an operator-context ``system`` - turn's structured per-kind fields (the ``_source_meta`` side channel); - opaque to storage and NULL for ordinary rows. + ``meta`` is pre-serialized role-specific conversation metadata + (operator context, tool disposition/preview plus the tool turn's acting + principal, shared-workstream sender, or accepted-assistant model + provenance); opaque to storage and NULL when a row has no metadata. + + ``commit_key`` is a caller-generated, non-empty idempotency identity scoped to + ``ws_id``. The first save inserts the row; subsequent saves with the + same non-NULL key and identical normalized fields return that row's + existing ``id`` without appending or replacing content. A mismatched + retry raises :class:`ConversationCommitConflictError`. A keyed save + requires the durable workstream row to exist in the same transaction; + it refuses a retry after hard delete instead of recreating an orphan + conversation. NULL preserves append-only legacy/offline semantics, + including historical parent-less writers. PostgreSQL conditionally + takes the same parent-first lock as prune/delete when that parent + exists. If the call observes a parent before blocking behind deletion, + it refuses the post-delete insert; a call that genuinely begins + parentless may still create an orphan. NULL therefore remains + unsuitable for a live accepted row, which MUST use a key. An empty + string is rejected. + """ + ... + + def save_user_message_with_attachments( + self, + ws_id: str, + content: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + source: str | None = None, + event_id: int | None = None, + meta: str | None = None, + commit_key: str, + ) -> int: + """Atomically commit one keyed USER row and its attachment references. + + The conversation row, new content-addressed blobs, exact per-reference + refcount increments, and ordered ``conversations.attachments`` list are + one database transaction. A retry with the same ``(ws_id, + commit_key)`` and identical normalized payload returns the original row + id without changing refcounts. A retry whose row payload or ordered + attachment ids differ raises + :class:`ConversationCommitConflictError`; no partial mutation survives + any failure. The durable workstream parent is locked/validated in the + same transaction, so a retry after hard delete is refused without + recreating either the row or attachment ownership. + + This operation is intentionally narrower than :meth:`save_message`: + ordinary rows and attachment-free USER rows retain their established + persistence path. + """ + ... + + def save_tool_message_with_attachments( + self, + ws_id: str, + content: str, + tool_name: str, + tool_call_id: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + event_id: int | None = None, + is_error: bool = False, + meta: str | None = None, + commit_key: str, + ) -> int: + """Atomically commit one keyed TOOL row and its attachment references. + + The immutable identity covers content, tool name/call id, event cursor, + error disposition, metadata, and the exact ordered attachment ids. Blob + bytes, size, MIME, and kind are validated under the content-addressed id; + newly inserted blobs carry ``origin='tool'``. An identical retry returns + the original row id without replaying refcount increments. Any mismatch + or partial failure raises and leaves the whole transaction unchanged. + The durable workstream parent is locked/validated in that transaction; + hard delete therefore cannot leave a later retry as an orphan. """ ... @@ -521,6 +632,17 @@ class StorageBackend(Protocol): """ ... + def truncate_messages_tail(self, ws_id: str, remove_count: int) -> int: + """Atomically remove up to *remove_count* newest conversation rows. + + The backend locks the durable workstream, derives both the current row + count and latest compaction floor inside that transaction, and never + deletes rows backing the latest compaction marker. Attachment + refcounts are released for exactly the rows deleted. Missing + workstreams and storage failures raise; a negative count is invalid. + """ + ... + # -- Workstream management ------------------------------------------------- def list_workstreams_with_history( @@ -571,7 +693,13 @@ class StorageBackend(Protocol): ... def prune_workstreams(self, retention_days: int = 90) -> tuple[int, int]: - """Remove orphaned + stale unnamed workstreams. Returns (orphans, stale).""" + """Atomically remove orphaned + stale unnamed workstreams. + + Candidate predicates are rechecked while holding the same parent-row + lock (or SQLite writer reservation) used by keyed conversation commits. + Deletion releases all attachment references transactionally. Returns + ``(orphans, stale)``. + """ ... def resolve_workstream(self, alias_or_id: str) -> str | None: diff --git a/turnstone/core/storage/_schema.py b/turnstone/core/storage/_schema.py index 32e52dc7..8eb98399 100644 --- a/turnstone/core/storage/_schema.py +++ b/turnstone/core/storage/_schema.py @@ -75,6 +75,13 @@ conversations = sa.Table( # Stripped before the LLM wire (it is a ``_``-prefixed key by the time it # reaches a provider). Added in migration 060. sa.Column("meta", sa.Text, nullable=True), + # Stable idempotency identity for an admitted live conversation commit. A + # retry after an ambiguous database acknowledgement can recover the original + # row instead of duplicating it. Nullable keeps every legacy/offline bulk + # writer on the historical append-only path; the partial composite unique + # index therefore constrains only keyed commits on both SQLite and + # PostgreSQL. Added in 071. + sa.Column("commit_key", sa.Text), ) sa.Index("idx_conversations_timestamp", conversations.c.timestamp) @@ -82,6 +89,14 @@ sa.Index("idx_conversations_timestamp", conversations.c.timestamp) # seek, not a row scan) and per-ws event-cursor range queries. See # migration 059. sa.Index("idx_conversations_ws_event", conversations.c.ws_id, conversations.c.event_id) +sa.Index( + "uq_conversations_ws_commit_key", + conversations.c.ws_id, + conversations.c.commit_key, + unique=True, + sqlite_where=conversations.c.commit_key.is_not(None), + postgresql_where=conversations.c.commit_key.is_not(None), +) workstreams = sa.Table( "workstreams", diff --git a/turnstone/core/storage/_sqlite.py b/turnstone/core/storage/_sqlite.py index aa8f31ab..70638781 100644 --- a/turnstone/core/storage/_sqlite.py +++ b/turnstone/core/storage/_sqlite.py @@ -12,6 +12,7 @@ from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any import sqlalchemy as sa +from sqlalchemy.dialects.sqlite import insert as sqlite_insert if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Sequence @@ -23,6 +24,8 @@ from turnstone.core.log import get_logger from turnstone.core.storage._protocol import ( FORK_RESERVATION_CONFIG_KEY, USER_SCOPED_AUTH_TYPES, + AttachmentWrite, + ConversationCommitWorkstreamGoneError, ForkCloneExpectation, ForkCloneSnapshot, MCPOAuthPendingState, @@ -136,6 +139,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( VERDICT_MUTABLE as _VERDICT_MUTABLE, ) +from turnstone.core.storage._utils import ( + KeyedAttachmentSaveWrappers as _KeyedAttachmentSaveWrappers, +) from turnstone.core.storage._utils import ( assert_single_default_persona as _assert_single_default_persona, ) @@ -146,16 +152,25 @@ from turnstone.core.storage._utils import ( clone_workstream_transaction, find_orphan_conversations, parse_checkpoint_watermark, + prepare_attachment_commit, + prepare_conversation_row_values, prepare_provider_data_for_save, purge_orphan_conversations, release_attachment_refs, retain_attachment_refs, sanitize_text, + save_attachment_commit_transaction, senders_from_user_meta, ) +from turnstone.core.storage._utils import ( + delete_messages_after_core as _delete_messages_after_core, +) from turnstone.core.storage._utils import ( escape_like as _escape_like, ) +from turnstone.core.storage._utils import ( + get_compaction_floor_on_connection as _get_compaction_floor_shared, +) from turnstone.core.storage._utils import ( normalize_search_terms as _normalize_search_terms, ) @@ -165,6 +180,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( persona_row_to_dict as _persona_row_to_dict, ) +from turnstone.core.storage._utils import ( + prune_workstreams_shared as _prune_workstreams_shared, +) from turnstone.core.storage._utils import ( reconstruct_messages as _reconstruct_messages, ) @@ -174,6 +192,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( recover_trajectory as _recover_trajectory, ) +from turnstone.core.storage._utils import ( + resolve_keyed_commit_conflict as _resolve_keyed_commit_conflict, +) from turnstone.core.storage._utils import ( row_to_dict as _row_to_dict, ) @@ -186,6 +207,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( split_perms as _split_perms, ) +from turnstone.core.storage._utils import ( + truncate_messages_tail_core as _truncate_messages_tail_core, +) from turnstone.core.storage._utils import ( validate_and_clear_default_persona as _validate_and_clear_default_persona, ) @@ -288,7 +312,7 @@ class _SQLiteNotifyStream: self._backend._notify_unregister(self._channels, self._queue) -class SQLiteBackend: +class SQLiteBackend(_KeyedAttachmentSaveWrappers): """SQLite implementation of the StorageBackend protocol.""" def __init__(self, path: str, *, create_tables: bool = True) -> None: @@ -378,6 +402,22 @@ class SQLiteBackend: # -- Core conversation operations ------------------------------------------ + def _index_conversation_fts(self, conn: Any, row_id: int, content: str | None) -> None: + """Mirror one freshly inserted conversation row into the FTS5 index. + + Search is best-effort: a failing index write disables FTS for the + process rather than failing the durable commit that carries it. + """ + if not self._fts5_available or not content: + return + try: + conn.execute( + sa.text("INSERT INTO conversations_fts(rowid, content) VALUES (:rowid, :content)"), + {"rowid": row_id, "content": content}, + ) + except Exception: + self._fts5_available = False + def save_message( self, ws_id: str, @@ -392,53 +432,140 @@ class SQLiteBackend: is_error: bool = False, producer: str | None = None, meta: str | None = None, + commit_key: str | None = None, ) -> int: now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") - content = sanitize_text(content) - provider_data = prepare_provider_data_for_save( - role, sanitize_text(provider_data), tool_calls, producer + values = prepare_conversation_row_values( + ws_id, + role, + content, + tool_name=tool_name, + tool_call_id=tool_call_id, + provider_data=provider_data, + tool_calls=tool_calls, + source=source, + event_id=event_id, + is_error=is_error, + producer=producer, + meta=meta, + commit_key=commit_key, + now=now, ) - source = sanitize_text(source) with self._conn() as conn: - result = conn.execute( - sa.insert(conversations), - { - "ws_id": ws_id, - "timestamp": now, - "role": role, - "content": content, - "tool_name": tool_name, - "tool_call_id": tool_call_id, - "provider_data": provider_data, - "tool_calls": tool_calls, - "_source": source, - "event_id": event_id, - "is_error": is_error, - "meta": meta, - }, - ) - if result.lastrowid is None: - # Should be unreachable under SQLite + autoincrement PKs. - raise RuntimeError("save_message: lastrowid missing after insert") - rowid = int(result.lastrowid) - # FTS5 indexing - if self._fts5_available and content: - try: - conn.execute( - sa.text( - "INSERT INTO conversations_fts(rowid, content) VALUES (:rowid, :content)" - ), - {"rowid": rowid, "content": content}, + inserted = True + if commit_key is None: + result = conn.execute(sa.insert(conversations), values) + if result.lastrowid is None: + # Should be unreachable under SQLite + autoincrement PKs. + raise RuntimeError("save_message: lastrowid missing after insert") + rowid = int(result.lastrowid) + else: + # Keyed commits are live-session writes, not the legacy + # append-without-parent path. Acquire the same writer + # reservation hard delete uses before checking the durable + # parent, so delete and save have a total order: save first is + # removed by delete; delete first makes this attempt fail. + conn.execute(sa.text("BEGIN IMMEDIATE")) + parent = conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id == ws_id) + ).fetchone() + if parent is None: + raise ConversationCommitWorkstreamGoneError( + "keyed conversation commit workstream no longer exists" ) - except Exception: - self._fts5_available = False - # Bump workstream updated timestamp - conn.execute( - sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now) - ) + # ``RETURNING`` is load-bearing: SQLite leaves lastrowid at a + # previous insert after DO NOTHING, which could acknowledge the + # wrong conversation row on an idempotent retry. + result = conn.execute( + sqlite_insert(conversations) + .values(**values) + .on_conflict_do_nothing( + index_elements=[conversations.c.ws_id, conversations.c.commit_key], + index_where=conversations.c.commit_key.is_not(None), + ) + .returning(conversations.c.id) + ) + resolved = result.scalar_one_or_none() + if resolved is None: + inserted = False + rowid = _resolve_keyed_commit_conflict(conn, ws_id, values) + else: + rowid = int(resolved) + if inserted: + self._index_conversation_fts(conn, rowid, values["content"]) + # Bump workstream updated timestamp + conn.execute( + sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now) + ) conn.commit() return rowid + def _save_message_with_attachments( + self, + ws_id: str, + role: str, + content: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + tool_name: str | None = None, + tool_call_id: str | None = None, + source: str | None = None, + event_id: int | None = None, + is_error: bool = False, + meta: str | None = None, + commit_key: str, + origin: str, + exact_blob_metadata: bool, + ) -> int: + """Dialect-local transaction shared by keyed USER and TOOL rows.""" + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + attachment_ids, blobs, values = prepare_attachment_commit( + ws_id, + role, + content, + attachments, + tool_name=tool_name, + tool_call_id=tool_call_id, + source=source, + event_id=event_id, + is_error=is_error, + meta=meta, + commit_key=commit_key, + now=now, + ) + with self._conn() as conn: + try: + # Match delete_workstream's BEGIN IMMEDIATE ordering. The + # parent check and every row/blob/refcount mutation in the + # shared body below are therefore one indivisible SQLite + # writer transaction. + conn.execute(sa.text("BEGIN IMMEDIATE")) + parent = conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id == ws_id) + ).fetchone() + if parent is None: + raise ConversationCommitWorkstreamGoneError( + "keyed conversation commit workstream no longer exists" + ) + row_id = save_attachment_commit_transaction( + conn, + sqlite_insert, + values=values, + attachment_ids=attachment_ids, + blobs=blobs, + now=now, + origin=origin, + exact_blob_metadata=exact_blob_metadata, + index_content=lambda inserted_id: self._index_conversation_fts( + conn, inserted_id, values["content"] + ), + ) + conn.commit() + return row_id + except Exception: + conn.rollback() + raise + def list_message_senders(self, ws_id: str) -> list[str]: # DISTINCT on the raw meta blob: a user row's meta carries only # {"sender": ...}, so distinct blobs ≈ distinct senders and the JSON @@ -517,9 +644,9 @@ class SQLiteBackend: """Fetch a ws's conversation rows + resolved attachment map. Shared by :meth:`load_messages` (→ dicts, resolved for display) and - :meth:`load_message_turns` (→ canonical Turns for resume). The trailing - ``attachments`` ref-list column is split off to resolve blobs and is NOT - part of the positional tuple ``reconstruct_*`` unpacks (id..meta). + :meth:`load_message_turns` (→ canonical Turns for resume). The trailing + ``attachments`` ref-list and ``commit_key`` columns stay internal to + reconstruction and never reach provider/public projections. """ _cols = ( conversations.c.id, @@ -534,6 +661,7 @@ class SQLiteBackend: conversations.c.is_error, conversations.c.meta, conversations.c.attachments, + conversations.c.commit_key, ) with self._conn() as conn: if limit is not None and limit > 0: @@ -722,27 +850,7 @@ class SQLiteBackend: backing intact. """ with self._conn() as conn: - marker_id = conn.execute( - sa.select(sa.func.max(conversations.c.id)).where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c._source == _COMPACTION_SOURCE, - ) - ) - ).scalar() - if marker_id is None: - return 0 - n = conn.execute( - sa.select(sa.func.count()) - .select_from(conversations) - .where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c.id <= marker_id, - ) - ) - ).scalar() - return int(n or 0) + return _get_compaction_floor_shared(conn, ws_id) def get_compaction_checkpoint(self, ws_id: str) -> int | None: """Latest persisted marker's watermark — see the protocol docstring. @@ -761,59 +869,71 @@ class SQLiteBackend: ).fetchone() return parse_checkpoint_watermark(row[0]) if row is not None else None + def _delete_messages_after_on_connection( + self, + conn: sa.engine.Connection, + ws_id: str, + keep_count: int, + ) -> int: + """Delete one conversation tail; caller owns the writer transaction.""" + return _delete_messages_after_core( + conn, ws_id, keep_count, pre_delete=self._delete_fts_tail + ) + + def _delete_fts_tail(self, conn: sa.engine.Connection, ws_id: str, cutoff_id: Any) -> None: + """Remove FTS5 entries first (external content table doesn't auto-sync).""" + if not self._fts5_available: + return + try: + conn.execute( + sa.text( + "DELETE FROM conversations_fts WHERE rowid IN " + "(SELECT id FROM conversations " + " WHERE ws_id = :ws_id AND id >= :cutoff_id)" + ), + {"ws_id": ws_id, "cutoff_id": cutoff_id}, + ) + except Exception: + self._fts5_available = False + def delete_messages_after(self, ws_id: str, keep_count: int) -> int: with self._conn() as conn: - # Find the id of the first row to delete (the row at offset keep_count) - cutoff_row = conn.execute( - sa.select(conversations.c.id) - .where(conversations.c.ws_id == ws_id) - .order_by(conversations.c.id) - .limit(1) - .offset(keep_count) - ).fetchone() - if cutoff_row is None: - return 0 # nothing to delete - cutoff_id = cutoff_row[0] - # Refcount GC: read the doomed rows' content-addressed ref-lists, - # decrement each blob's refcount once per reference, and prune - # blobs that hit 0 — so a deduped blob still referenced by a kept - # turn survives. Replaces the old message_id-cascade delete. - doomed = conn.execute( - sa.select(conversations.c.attachments).where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c.id >= cutoff_id, - conversations.c.attachments.is_not(None), - ) - ) - ).fetchall() - doomed_ids: list[str] = [] - for (refs,) in doomed: - doomed_ids.extend(_parse_attachment_refs(refs)) - release_attachment_refs(conn, doomed_ids) - # Remove FTS5 entries first (external content table doesn't auto-sync) - if self._fts5_available: - try: - conn.execute( - sa.text( - "DELETE FROM conversations_fts WHERE rowid IN " - "(SELECT id FROM conversations " - " WHERE ws_id = :ws_id AND id >= :cutoff_id)" - ), - {"ws_id": ws_id, "cutoff_id": cutoff_id}, - ) - except Exception: - self._fts5_available = False - result = conn.execute( - sa.delete(conversations).where( - sa.and_( - conversations.c.ws_id == ws_id, - conversations.c.id >= cutoff_id, - ) - ) - ) + # Share the keyed-commit / hard-delete writer boundary. BEGIN + # IMMEDIATE orders the complete cutoff + delete + ref-release + # transaction before or after every keyed commit. + conn.execute(sa.text("BEGIN IMMEDIATE")) + deleted = self._delete_messages_after_on_connection(conn, ws_id, keep_count) conn.commit() - return result.rowcount + return deleted + + def truncate_messages_tail(self, ws_id: str, remove_count: int) -> int: + """Atomically remove a compaction-floored number of newest rows.""" + if remove_count < 0: + raise ValueError("remove_count must be non-negative") + with self._conn() as conn: + try: + # Own the writer slot before observing either count. A keyed + # commit therefore lands wholly before this snapshot or after + # the deletion commits; it cannot inflate ``total`` and then be + # included in a stale caller-computed keep count. + conn.execute(sa.text("BEGIN IMMEDIATE")) + parent = conn.execute( + sa.select(workstreams.c.ws_id).where(workstreams.c.ws_id == ws_id) + ).fetchone() + if parent is None: + raise RuntimeError("tail truncation workstream no longer exists") + + deleted = _truncate_messages_tail_core( + conn, + ws_id, + remove_count, + delete_after=self._delete_messages_after_on_connection, + ) + conn.commit() + return deleted + except Exception: + conn.rollback() + raise # -- Workstream management ------------------------------------------------- @@ -885,77 +1005,56 @@ class SQLiteBackend: ).fetchall() ) - def prune_workstreams(self, retention_days: int = 90) -> tuple[int, int]: - orphans = stale = 0 - with self._conn() as conn: - # 1. Remove workstreams with no messages - orphan_ids = [ - row[0] - for row in conn.execute( - sa.text( - "SELECT ws_id FROM workstreams " - "WHERE state != 'creating' AND NOT EXISTS " - " (SELECT 1 FROM conversations c " - " WHERE c.ws_id = workstreams.ws_id)" - ) - ).fetchall() - ] - if orphan_ids: - chunk_size = 500 - for i in range(0, len(orphan_ids), chunk_size): - chunk = orphan_ids[i : i + chunk_size] - placeholders = ",".join([":p" + str(j) for j in range(len(chunk))]) - params = {f"p{j}": oid for j, oid in enumerate(chunk)} - conn.execute( - sa.text(f"DELETE FROM workstream_config WHERE ws_id IN ({placeholders})"), - params, - ) - result = conn.execute( - sa.text(f"DELETE FROM workstreams WHERE ws_id IN ({placeholders})"), - params, - ) - orphans += result.rowcount + def _delete_prune_candidate( + self, + ws_id: str, + predicates: tuple[Any, ...], + ) -> bool: + """Recheck and delete one prune candidate under a short writer txn. - # 2. Remove old unnamed workstreams - if retention_days > 0: - cutoff = (datetime.now(UTC) - timedelta(days=retention_days)).strftime( - "%Y-%m-%dT%H:%M:%S" + SQLite has no row-level writer lock. One ``BEGIN IMMEDIATE`` around + the complete candidate list would therefore stop every unrelated write + while per-workstream attachment GC runs. Candidate discovery is only + a hint; this exact predicate recheck is the admission point. A keyed + commit either lands before it and makes the row ineligible, or blocks + behind it and observes the parent deletion when it resumes. + """ + with self._conn() as conn: + try: + conn.execute(sa.text("BEGIN IMMEDIATE")) + exact = conn.execute( + sa.select(workstreams.c.ws_id).where( + workstreams.c.ws_id == ws_id, + *predicates, + ) + ).fetchone() + deleted = bool( + exact is not None and self._delete_workstream_on_connection(conn, ws_id) ) - stale_ids = [ - row[0] + conn.commit() + return deleted + except Exception: + conn.rollback() + raise + + def prune_workstreams(self, retention_days: int = 90) -> tuple[int, int]: + # Do not reserve SQLite's database-wide writer slot for discovery. + # Every candidate is rechecked in its own bounded writer transaction + # by ``_delete_prune_candidate``. + def _select_ids(predicates: tuple[Any, ...]) -> list[str]: + with self._conn() as conn: + return [ + str(row[0]) for row in conn.execute( - sa.text( - "SELECT ws_id FROM workstreams " - "WHERE state != 'creating' " - "AND alias IS NULL AND updated < :cutoff" - ), - {"cutoff": cutoff}, + sa.select(workstreams.c.ws_id).where(*predicates) ).fetchall() ] - if stale_ids: - chunk_size = 500 - for i in range(0, len(stale_ids), chunk_size): - chunk = stale_ids[i : i + chunk_size] - placeholders = ",".join([":p" + str(j) for j in range(len(chunk))]) - params = {f"p{j}": sid for j, sid in enumerate(chunk)} - conn.execute( - sa.text( - f"DELETE FROM workstream_config WHERE ws_id IN ({placeholders})" - ), - params, - ) - conn.execute( - sa.text(f"DELETE FROM conversations WHERE ws_id IN ({placeholders})"), - params, - ) - result = conn.execute( - sa.text(f"DELETE FROM workstreams WHERE ws_id IN ({placeholders})"), - params, - ) - stale += result.rowcount - conn.commit() - return (orphans, stale) + return _prune_workstreams_shared( + retention_days, + select_ids=_select_ids, + delete_candidate=self._delete_prune_candidate, + ) def resolve_workstream(self, alias_or_id: str) -> str | None: with self._conn() as conn: diff --git a/turnstone/core/storage/_utils.py b/turnstone/core/storage/_utils.py index bdcbace8..d40a66b3 100644 --- a/turnstone/core/storage/_utils.py +++ b/turnstone/core/storage/_utils.py @@ -6,17 +6,20 @@ import base64 import json import re from collections import Counter +from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any import sqlalchemy as sa if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable from turnstone.core.attachments import AUDIO_MIME_TO_FORMAT, unreadable_placeholder from turnstone.core.log import get_logger from turnstone.core.storage._protocol import ( FORK_RESERVATION_CONFIG_KEY, + AttachmentWrite, + ConversationCommitConflictError, ForkCloneExpectation, ForkCloneSnapshot, ForkDestinationConflictError, @@ -35,6 +38,7 @@ from turnstone.core.storage._schema import ( workstreams, ) from turnstone.core.trajectory import ( + PROVENANCE_META_KEY, AttachmentRef, ContentBlock, ProviderNative, @@ -43,8 +47,10 @@ from turnstone.core.trajectory import ( ToolCall, Turn, TurnMeta, + TurnProvenance, dicts_from_turns, resolve_attachment_parts, + sanitize_client_send_ids, turn_to_dict, ) @@ -158,6 +164,303 @@ def prepare_provider_data_for_save( ) +def prepare_attachment_writes( + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], +) -> tuple[list[str], dict[str, AttachmentWrite]]: + """Validate an atomic write and retain one blob payload per distinct id. + + The returned list preserves every reference (including duplicates). The + mapping is only for inserting/validating the globally deduplicated blob. + Filenames are intentionally not part of duplicate compatibility: the blob + store historically keeps the filename of the first global reference, while + identity and rendering semantics are carried by content, MIME, and kind. + """ + if not attachments: + raise ValueError("atomic attachment commit requires at least one attachment") + + ordered_ids: list[str] = [] + unique: dict[str, AttachmentWrite] = {} + for attachment in attachments: + if not attachment.attachment_id: + raise ValueError("attachment_id must be non-empty") + if not isinstance(attachment.content, bytes): + raise TypeError("attachment content must be bytes") + if attachment.size_bytes != len(attachment.content): + raise ValueError( + f"attachment {attachment.attachment_id!r} size does not match its content" + ) + if not attachment.mime_type or not attachment.kind: + raise ValueError("attachment MIME type and kind must be non-empty") + + prior = unique.get(attachment.attachment_id) + if prior is not None and ( + prior.content != attachment.content + or prior.size_bytes != attachment.size_bytes + or prior.mime_type != attachment.mime_type + or prior.kind != attachment.kind + ): + raise ConversationCommitConflictError( + f"attachment {attachment.attachment_id!r} has conflicting blob payloads" + ) + unique.setdefault(attachment.attachment_id, attachment) + ordered_ids.append(attachment.attachment_id) + return ordered_ids, unique + + +# The immutable identity of a keyed conversation commit: every column the +# conflict re-read projects and :func:`assert_conversation_commit_matches` +# compares. ``id`` rides along as the row the retry is acknowledged with; +# ``timestamp`` is database-generated and deliberately absent. +_COMMIT_IDENTITY_COLUMNS = ( + conversations.c.id, + conversations.c.role, + conversations.c.content, + conversations.c.tool_name, + conversations.c.tool_call_id, + conversations.c.provider_data, + conversations.c.tool_calls, + conversations.c._source, + conversations.c.event_id, + conversations.c.is_error, + conversations.c.attachments, + conversations.c.meta, + conversations.c.commit_key, +) + + +def select_conversation_commit_row(conn: Any, ws_id: str, commit_key: str) -> Any: + """Re-read the row a keyed insert conflicted with, or ``None`` if it vanished. + + Every keyed save path — plain and attachment-bearing, on both dialects — + reads the conflicting row through here, so the identity column list exists + once. A vanished row is the caller's signal never to re-insert: a + concurrent hard delete may have removed it. + """ + return conn.execute( + sa.select(*_COMMIT_IDENTITY_COLUMNS).where( + conversations.c.ws_id == ws_id, + conversations.c.commit_key == commit_key, + ) + ).fetchone() + + +def assert_conversation_commit_matches( + row: Any, + *, + role: str, + content: str | None, + tool_name: str | None, + tool_call_id: str | None, + source: str | None, + event_id: int | None, + is_error: bool, + meta: str | None, + commit_key: str, + provider_data: str | None = None, + tool_calls: str | None = None, + attachment_ids: list[str] | None = None, +) -> int: + """Validate an existing row against one normalized keyed commit. + + ``id`` and ``timestamp`` are database-generated and intentionally excluded. + A plain commit (``attachment_ids`` omitted) requires a NULL ``attachments`` + column, because attachment-bearing USER/TOOL writes use the dedicated atomic + seams; those pass their ordered reference list, which is compared decoded so + a re-ordered retry conflicts while an equivalent encoding does not. + + Every other column compares byte-identically on purpose: the durability + journal reads a mismatch as a permanent conflict, so normalization added + here would turn benign retries into spurious failures. + """ + values = row._mapping + expected = { + "role": role, + "content": content, + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "provider_data": provider_data, + "tool_calls": tool_calls, + "_source": source, + "event_id": event_id, + "is_error": is_error, + **({} if attachment_ids is not None else {"attachments": None}), + "meta": meta, + "commit_key": commit_key, + } + mismatched = [ + field + for field, expected_value in expected.items() + if values[field] != expected_value + and not (field == "is_error" and bool(values[field]) is expected_value) + ] + if ( + attachment_ids is not None + and parse_attachment_refs(values["attachments"]) != attachment_ids + ): + mismatched.append("attachments") + if mismatched: + kind = "attachment" if attachment_ids is not None else "conversation" + raise ConversationCommitConflictError( + f"commit_key already identifies a different {kind} commit " + f"(mismatched fields: {', '.join(mismatched)})" + ) + return int(values["id"]) + + +class KeyedAttachmentSaveWrappers: + """Public keyed attachment-save wrappers shared by both dialects. + + Each dialect implements ``_save_message_with_attachments`` (its lock + prologue, insert constructor, and FTS hook differ); these wrappers own + only the role-shaped public signatures, so the storage API cannot drift + between backends. + """ + + def _save_message_with_attachments(self, *args: Any, **kwargs: Any) -> int: + raise NotImplementedError + + def save_user_message_with_attachments( + self, + ws_id: str, + content: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + source: str | None = None, + event_id: int | None = None, + meta: str | None = None, + commit_key: str, + ) -> int: + """Commit a keyed USER row and all attachment ownership atomically.""" + return self._save_message_with_attachments( + ws_id, + "user", + content, + attachments, + source=source, + event_id=event_id, + meta=meta, + commit_key=commit_key, + origin="upload", + exact_blob_metadata=False, + ) + + def save_tool_message_with_attachments( + self, + ws_id: str, + content: str, + tool_name: str, + tool_call_id: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + event_id: int | None = None, + is_error: bool = False, + meta: str | None = None, + commit_key: str, + ) -> int: + """Commit a keyed TOOL row and all attachment ownership atomically.""" + return self._save_message_with_attachments( + ws_id, + "tool", + content, + attachments, + tool_name=tool_name, + tool_call_id=tool_call_id, + event_id=event_id, + is_error=is_error, + meta=meta, + commit_key=commit_key, + origin="tool", + exact_blob_metadata=True, + ) + + +def resolve_keyed_commit_conflict( + conn: Any, + ws_id: str, + values: dict[str, Any], + *, + attachment_ids: list[str] | None = None, + vanished_message: str = "save_message: conflicting commit_key row vanished", +) -> int: + """Recover a keyed insert whose ``ON CONFLICT DO NOTHING`` matched a row. + + Re-selects by commit key and validates byte-identity, driven from the + SAME ``values`` dict the insert used — a column added to the commit + identity is threaded once, so every idempotent-retry validation (both + dialects' plain saves AND the shared attachment-commit body) compares + every field or none. A conflicting row that vanished must not be + silently re-created (that could resurrect a hard-deleted workstream): + raise instead. + """ + commit_key = values["commit_key"] + row = select_conversation_commit_row(conn, ws_id, commit_key) + if row is None: + raise RuntimeError(vanished_message) + return assert_conversation_commit_matches( + row, + role=values["role"], + content=values["content"], + tool_name=values["tool_name"], + tool_call_id=values["tool_call_id"], + provider_data=values.get("provider_data"), + tool_calls=values.get("tool_calls"), + source=values["_source"], + event_id=values["event_id"], + is_error=values["is_error"], + meta=values["meta"], + commit_key=commit_key, + attachment_ids=attachment_ids, + ) + + +def assert_attachment_blobs_match( + conn: Any, + attachments: dict[str, AttachmentWrite], + *, + exact_metadata: bool = False, +) -> None: + """Require every referenced CAS row to contain the requested bytes. + + The USER path preserves the global store's historical first-writer MIME and + kind behavior. The TOOL path passes ``exact_metadata=True`` because its + replay contract requires bytes, length, MIME, and kind to match exactly. + Filename and origin remain first-writer metadata for both paths. + """ + ids = list(attachments) + rows = conn.execute( + sa.select( + workstream_attachments.c.attachment_id, + workstream_attachments.c.mime_type, + workstream_attachments.c.size_bytes, + workstream_attachments.c.kind, + workstream_attachments.c.content, + ).where(workstream_attachments.c.attachment_id.in_(ids)) + ).fetchall() + stored = {str(row._mapping["attachment_id"]): row._mapping for row in rows} + missing = set(ids) - set(stored) + if missing: + raise RuntimeError(f"atomic attachment insert lost blobs: {sorted(missing)!r}") + + for attachment_id, requested in attachments.items(): + row = stored[attachment_id] + raw_content = row["content"] + content = bytes(raw_content) if not isinstance(raw_content, bytes) else raw_content + if ( + content != requested.content + or int(row["size_bytes"]) != requested.size_bytes + or ( + exact_metadata + and ( + str(row["mime_type"]) != requested.mime_type + or str(row["kind"]) != requested.kind + ) + ) + ): + raise ConversationCommitConflictError( + f"attachment {attachment_id!r} conflicts with its content-addressed blob" + ) + + def retain_attachment_refs(conn: Any, attachment_ids: list[str]) -> None: """Increment existing blobs for a newly inserted batch of references. @@ -229,6 +532,195 @@ def release_attachment_refs(conn: Any, attachment_ids: list[str]) -> None: ) +def prepare_attachment_commit( + ws_id: str, + role: str, + content: str, + attachments: list[AttachmentWrite] | tuple[AttachmentWrite, ...], + *, + tool_name: str | None, + tool_call_id: str | None, + source: str | None, + event_id: int | None, + is_error: bool, + meta: str | None, + commit_key: str, + now: str, +) -> tuple[list[str], dict[str, AttachmentWrite], dict[str, Any]]: + """Validate one keyed attachment commit and build its conversation row. + + Pure by design: a malformed write is refused before either backend opens + its writer transaction, so an invalid payload never holds SQLite's + database-wide writer slot or a PostgreSQL parent row lock. Returns the + ordered reference list, the deduplicated blob payloads, and the row values + both dialects insert. + """ + if not commit_key: + raise ValueError("atomic attachment commit requires a commit_key") + attachment_ids, blobs = prepare_attachment_writes(attachments) + if sanitize_text(content) is None: # ``content`` is typed str; defensive at the boundary. + raise TypeError("atomic attachment content must be text") + # The commit-identity dict comes from the ONE builder both commit lanes + # share — a column added to the commit identity reaches the plain and + # attachment lanes' inserts AND their conflict rechecks together (a + # hand-synced sibling here would let attachment retries validate + # against a stale field set on both dialects). The attachment lane's + # only additions: the native lane is structurally empty (attachments + # never carry provider blocks) and the ordered reference list rides + # the ``attachments`` column. + values = prepare_conversation_row_values( + ws_id, + role, + content, + tool_name=tool_name, + tool_call_id=tool_call_id, + provider_data=None, + tool_calls=None, + source=source, + event_id=event_id, + is_error=is_error, + producer=None, + meta=meta, + commit_key=commit_key, + now=now, + ) + values["attachments"] = json.dumps(attachment_ids) + return attachment_ids, blobs, values + + +def prepare_conversation_row_values( + ws_id: str, + role: str, + content: str | None, + *, + tool_name: str | None, + tool_call_id: str | None, + provider_data: str | None, + tool_calls: str | None, + source: str | None, + event_id: int | None, + is_error: bool, + producer: str | None, + meta: str | None, + commit_key: str | None, + now: str, +) -> dict[str, Any]: + """Build the ``save_message`` row values both dialects insert. + + This dict IS the commit identity that ``resolve_keyed_commit_conflict`` + re-reads and compares on an idempotent retry, so a column added to the + commit identity must thread through here exactly once and reach both + dialects' insert AND both rechecks together. A per-dialect copy lets + one backend's retry validation compare a stale field set — every + legitimate retry false-conflicts (journal latched at "conflict"), or a + genuinely different commit is silently acknowledged — on one backend + only, invisible to single-backend tests. + """ + if commit_key == "": + raise ValueError("conversation commit_key must be non-empty") + return { + "ws_id": ws_id, + "timestamp": now, + "role": role, + "content": sanitize_text(content), + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "provider_data": prepare_provider_data_for_save( + role, sanitize_text(provider_data), tool_calls, producer + ), + "tool_calls": tool_calls, + "_source": sanitize_text(source), + "event_id": event_id, + "is_error": is_error, + "meta": meta, + "commit_key": commit_key, + } + + +def save_attachment_commit_transaction( + conn: Any, + dialect_insert: Callable[[Any], Any], + *, + values: dict[str, Any], + attachment_ids: list[str], + blobs: dict[str, AttachmentWrite], + now: str, + origin: str, + exact_blob_metadata: bool, + index_content: Callable[[int], None] | None = None, +) -> int: + """Execute the backend-neutral body of a keyed USER/TOOL attachment commit. + + The caller owns the connection, the transaction, and its dialect-specific + locking prologue: SQLite enters under ``BEGIN IMMEDIATE``, PostgreSQL under + the parent row's ``FOR UPDATE``, and both refuse a missing parent before + calling. ``dialect_insert`` is that dialect's INSERT constructor — + ``ON CONFLICT DO NOTHING`` only exists on the dialect-specific construct. + ``index_content`` is SQLite's external-content FTS refresh, run where the + row id first exists; PostgreSQL passes nothing. + """ + ws_id = values["ws_id"] + inserted = conn.execute( + dialect_insert(conversations) + .values(**values) + .on_conflict_do_nothing( + index_elements=[conversations.c.ws_id, conversations.c.commit_key], + index_where=conversations.c.commit_key.is_not(None), + ) + .returning(conversations.c.id) + ).scalar_one_or_none() + if inserted is None: + row_id = resolve_keyed_commit_conflict( + conn, + ws_id, + values, + attachment_ids=attachment_ids, + vanished_message="atomic attachment commit conflict row vanished", + ) + # A matching row is the transaction's durable witness. Also reject + # latent CAS corruption instead of acknowledging a history row whose + # blob materialization will fail later. + assert_attachment_blobs_match(conn, blobs, exact_metadata=exact_blob_metadata) + return row_id + + row_id = int(inserted) + written = set( + conn.execute( + dialect_insert(workstream_attachments) + .values( + [ + { + "attachment_id": attachment.attachment_id, + "filename": attachment.filename, + "mime_type": attachment.mime_type, + "size_bytes": attachment.size_bytes, + "kind": attachment.kind, + "content": attachment.content, + "created": now, + "refcount": 0, + "origin": origin, + } + for attachment in blobs.values() + ] + ) + .on_conflict_do_nothing(index_elements=["attachment_id"]) + .returning(workstream_attachments.c.attachment_id) + ).scalars() + ) + # Only ids this statement did NOT write can disagree with the request; the + # rest are the bytes we just supplied. Re-reading those too would pull the + # whole upload back out of the database inside the write transaction, + # doubling its I/O and extending the parent row's lock hold. + conflicted = {aid: blob for aid, blob in blobs.items() if aid not in written} + if conflicted: + assert_attachment_blobs_match(conn, conflicted, exact_metadata=exact_blob_metadata) + retain_attachment_refs(conn, attachment_ids) + if index_content is not None: + index_content(row_id) + conn.execute(sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)) + return row_id + + def find_orphan_conversations(conn: Any) -> list[dict[str, Any]]: """Conversation ws_ids that have no ``workstreams`` row, with row stats. @@ -488,6 +980,7 @@ def _reconstruct_attachment_refs( ) meta.append( { + "attachment_id": str(att.get("attachment_id") or ""), "kind": str(att.get("kind") or ""), "filename": str(att.get("filename") or ""), "mime_type": str(att.get("mime_type") or ""), @@ -969,17 +1462,19 @@ def reconstruct_messages( ) -> list[dict[str, Any]]: """Reconstruct OpenAI message format from stored conversation rows. - Each *row* is an 8-to-11-tuple ``(id, role, content, tool_name, + Each *row* is an 8-to-13-tuple ``(id, role, content, tool_name, tool_call_id, provider_data, tool_calls_json, source [, event_id [, is_error - [, meta]]])``, ordered chronologically by row id. ``source`` is rehydrated + [, meta [, attachments [, commit_key]]]]])``, ordered chronologically by + row id. ``source`` is rehydrated as the ``_source`` side channel. (The legacy ``_reminders`` column that used to ride here was dropped in migration 060 — operator context lives in first-class ``system`` turns now.) The trailing optional elements — ``event_id`` (migration 059, the per-ws SSE ``Last-Event-ID`` resume cursor → ``_event_id``), ``is_error`` (migration 060, the persisted tool-result error flag), and ``meta`` (migration 060, an operator-context turn's structured - ``_source_meta``) — are handled by the defensive unpack so shorter legacy - fixtures stay valid. + ``_source_meta``), raw attachment identities, and storage idempotency + identity are handled by the defensive unpack so shorter legacy fixtures + stay valid. The last two remain internal side channels. When ``attachments_by_msg`` is provided (keyed by row id, each value an ordered list of content-addressed attachment rows resolved from the @@ -1015,7 +1510,23 @@ def reconstruct_messages( # counts) and the frontend renders its compaction card at the point in # the transcript where the compaction actually happened. if include_compaction: - rows = [(r[0], "system", *r[2:]) if _is_compaction_marker(r) else r for r in rows] + display_rows: list[Any] = [] + for row in rows: + if not _is_compaction_marker(row): + display_rows.append(row) + continue + rewritten = list(row) + rewritten[1] = "system" + if len(rewritten) > 10: + marker_meta = _source_meta_from_json(rewritten[10]) + if marker_meta is not None: + # The internal checkpoint turn retains the acting + # principal, but /history's display card is a public + # projection and must not expose audit identity. + marker_meta.pop(PROVENANCE_META_KEY, None) + rewritten[10] = json.dumps(marker_meta) if marker_meta else None + display_rows.append(tuple(rewritten)) + rows = display_rows else: rows = [r for r in rows if not _is_compaction_marker(r)] turns = reconstruct_turns(rows, ws_id, attachments_by_msg) @@ -1094,13 +1605,13 @@ def _native_from_provider_data( def _source_meta_from_json(meta_json: str | None) -> dict[str, Any] | None: - """Decode the stored ``meta`` column into an operator-context meta dict. + """Decode the stored ``meta`` column into a conversation metadata object. - The persisted twin of ``Turn.meta.extra["source_meta"]`` — a first-class - ``system`` turn's structured per-kind fields (e.g. ``watch_triggered``'s - ``watch_name`` / ``command`` / poll counters). A decode failure or a - non-object payload yields ``None`` (the meta is dropped — the human-readable - body still lives in ``content``). + Role-specific routing happens in :func:`reconstruct_turns`: system rows + carry ``source_meta``, tool rows carry effect/preview fields plus the + acting principal their turn executed under, user rows carry their sender, + and assistant rows carry the well-known provenance envelope. A decode + failure or non-object payload yields ``None``. """ if not meta_json: return None @@ -1159,23 +1670,48 @@ def reconstruct_turns( raw_meta = _source_meta_from_json(row[10]) if len(row) > 10 else None if raw_meta is not None: # The ``meta`` column is role-exclusive: a TOOL row carries the - # typed ``{"effect_status": ..., "preview": ...}`` envelope (each - # key optional); a SYSTEM row carries operator-context - # ``source_meta``. Route so a tool's disposition doesn't land - # under source_meta (and vice versa). Legacy SYSTEM rows (bare + # typed ``{"effect_status": ..., "preview": ..., + # "acting_principal": ...}`` envelope (each key optional); a SYSTEM + # row carries operator-context ``source_meta``. Route so a tool's + # disposition doesn't land under source_meta (and vice versa) — + # ``source_meta`` IS a public projection, so an unrouted tool key + # would surface as display metadata. Legacy SYSTEM rows (bare # source_meta dict, no tool keys) fall through. - if role == "tool" and ("effect_status" in raw_meta or "preview" in raw_meta): + if role == "assistant": + assistant_meta = dict(raw_meta) + provenance = TurnProvenance.from_meta(assistant_meta.pop(PROVENANCE_META_KEY, None)) + if provenance is not None: + meta.extra[PROVENANCE_META_KEY] = provenance.to_meta() + # A compaction marker carries checkpoint metadata beside the + # producing model's provenance. Preserve both envelopes. + if assistant_meta: + meta.extra["source_meta"] = assistant_meta + elif role == "tool" and ( + "effect_status" in raw_meta + or "preview" in raw_meta + or "acting_principal" in raw_meta + ): if "effect_status" in raw_meta: meta.extra["effect_status"] = raw_meta["effect_status"] if "preview" in raw_meta: meta.extra["preview"] = raw_meta["preview"] - elif role == "user" and "sender" in raw_meta: - # Per-message sender identity (shared-workstream attribution). - # A USER row's meta blob carries only ``{"sender": ...}`` — route - # it to its own key so history replay re-attributes each turn to - # the human who sent it (source_meta rides SYSTEM turns, never - # user turns, so there is no collision). - meta.extra["sender"] = raw_meta["sender"] + # The principal whose turn executed this effect — an audit + # identity kept OFF the dict bridge (``turn_to_dict`` projects + # no key for it), so it cannot reach /history, export, SSE, or + # a provider payload. Fork reads it from here directly. + acting_principal = raw_meta.get("acting_principal") + if isinstance(acting_principal, str) and acting_principal: + meta.extra["acting_principal"] = acting_principal + elif role == "user": + # User-row metadata is its own role-exclusive envelope. Sender + # attribution and browser send-correlation both survive reload + # and fork; absent keys remain compatible with historical rows. + sender = raw_meta.get("sender") + if isinstance(sender, str) and sender: + meta.extra["sender"] = sender + stable_client_send_ids = sanitize_client_send_ids(raw_meta.get("client_send_ids")) + if stable_client_send_ids: + meta.extra["client_send_ids"] = stable_client_send_ids else: meta.extra["source_meta"] = raw_meta # Canonical storage loads retain the exact ordered ref-list captured @@ -1190,6 +1726,8 @@ def reconstruct_turns( # cannot fall back to a workstream-wide ownership query and borrow # a blob referenced by some other row. meta.extra["storage_attachment_ids"] = parse_attachment_refs(row[11]) + if len(row) > 12 and isinstance(row[12], str) and row[12]: + meta.commit_key = row[12] src = str(source) if source else None if role == "user": @@ -1357,6 +1895,186 @@ def _compaction_watermark(row: Any) -> int | None: return parse_checkpoint_watermark(row[10] if len(row) > 10 else None) +# A workstream registered moments ago is a user mid-first-turn, not debris — +# its first rows may still be held in a serving node's in-memory pending +# journal, which no other node's prune can see. The grace makes youth a +# storage-visible proxy for "possibly live"; 2 h matches the deferred-create +# reaper's STALE_CREATE_GRACE_SECONDS precedent (session_manager.py). +ORPHAN_PRUNE_GRACE_SECONDS = 2 * 60 * 60 + + +def prune_workstreams_shared( + retention_days: int, + *, + select_ids: Callable[[tuple[Any, ...]], list[str]], + delete_candidate: Callable[[str, tuple[Any, ...]], bool], +) -> tuple[int, int]: + """Orchestrate both prune categories; dialects supply only the hooks. + + ``select_ids`` runs one lock-free discovery SELECT over a predicate + tuple; ``delete_candidate`` is the dialect's + ``_delete_prune_candidate`` (PostgreSQL: ``FOR UPDATE SKIP LOCKED`` + + separate-statement recheck; SQLite: ``BEGIN IMMEDIATE`` recheck). The + same predicate tuple drives discovery AND recheck, so every conjunct + must be a pure SQLAlchemy expression with no per-call state. + + Orphan category — zero conversation rows — carries two guards beyond + ``state != 'creating'``: + + - ``alias IS NULL``: a named workstream is explicit user intent and is + never auto-pruned, mirroring the stale category. (Consequence, + accepted: an aliased workstream with zero rows is excluded from both + categories and lives until explicitly deleted.) + - ``updated`` older than :data:`ORPHAN_PRUNE_GRACE_SECONDS`. + + Both cutoffs are formatted ONCE at discovery and ride the tuple into the + recheck as literals. That is the safe direction: eligibility can only + shrink between discovery and recheck (an ``updated`` bump makes the row + ineligible under either reading), whereas recomputing ``now`` at recheck + would admit rows discovery never saw. Do not "freshen" the cutoff. + + A candidate whose delete raises aborts the remaining run (the caller's + wrapper logs and reports ``(0, 0)``) — preserved from the pre-shared + bodies; per-candidate continuation would need its own design pass. + """ + orphans = stale = 0 + orphan_cutoff = (datetime.now(UTC) - timedelta(seconds=ORPHAN_PRUNE_GRACE_SECONDS)).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + orphan_predicate = ( + workstreams.c.state != "creating", + ~sa.exists( + sa.select(conversations.c.id).where(conversations.c.ws_id == workstreams.c.ws_id) + ), + workstreams.c.alias.is_(None), + workstreams.c.updated < orphan_cutoff, + ) + for ws_id in select_ids(orphan_predicate): + if delete_candidate(ws_id, orphan_predicate): + orphans += 1 + + # 2. Remove old unnamed workstreams. Discover after orphan deletion so + # the two categories retain their established disjoint accounting. + if retention_days > 0: + stale_cutoff = (datetime.now(UTC) - timedelta(days=retention_days)).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + stale_predicate = ( + workstreams.c.state != "creating", + workstreams.c.alias.is_(None), + workstreams.c.updated < stale_cutoff, + ) + for ws_id in select_ids(stale_predicate): + if delete_candidate(ws_id, stale_predicate): + stale += 1 + return (orphans, stale) + + +def get_compaction_floor_on_connection(conn: Any, ws_id: str) -> int: + """Latest compaction floor inside the caller's transaction. + + The count of rows with ``id <= the latest marker's id`` — the summarized + prefix plus the marker itself — or ``0`` when the workstream never + compacted. Dialect-neutral; the caller owns the surrounding + transaction/lock discipline. + """ + marker_id = conn.execute( + sa.select(sa.func.max(conversations.c.id)).where( + sa.and_( + conversations.c.ws_id == ws_id, + conversations.c._source == COMPACTION_SOURCE, + ) + ) + ).scalar() + if marker_id is None: + return 0 + n = conn.execute( + sa.select(sa.func.count()) + .select_from(conversations) + .where( + sa.and_( + conversations.c.ws_id == ws_id, + conversations.c.id <= marker_id, + ) + ) + ).scalar() + return int(n or 0) + + +def truncate_messages_tail_core( + conn: Any, + ws_id: str, + remove_count: int, + *, + delete_after: Callable[[Any, str, int], int], +) -> int: + """Count → floor → keep → delete body shared by both dialects. + + The caller owns the transaction and has already taken its dialect's + parent lock (``SELECT … FOR UPDATE`` vs ``BEGIN IMMEDIATE``) and raised + on a missing parent, so a keyed commit cannot inflate ``total`` between + the count and the delete. ``delete_after`` is the dialect's + ``_delete_messages_after_on_connection`` hook. + """ + total = int( + conn.execute( + sa.select(sa.func.count()) + .select_from(conversations) + .where(conversations.c.ws_id == ws_id) + ).scalar() + or 0 + ) + floor = get_compaction_floor_on_connection(conn, ws_id) + keep_count = max(floor, total - remove_count) + return delete_after(conn, ws_id, keep_count) + + +def delete_messages_after_core( + conn: Any, + ws_id: str, + keep_count: int, + *, + pre_delete: Callable[[Any, str, Any], None] | None = None, +) -> int: + """Cutoff → dialect pre-delete hook → DELETE..RETURNING → ref release. + + Shared by both dialects; the caller owns the parent lock / writer + transaction. ``pre_delete(conn, ws_id, cutoff_id)`` is SQLite's FTS5 + external-content cleanup slot. GC ownership derives from the DELETE's + RETURNING rows, never a prior SELECT, so a row inserted between the two + statements cannot be released without being deleted (PostgreSQL READ + COMMITTED takes a new snapshot per statement; SQLite >= 3.35 RETURNING + is already required by orphan purging). + """ + cutoff_row = conn.execute( + sa.select(conversations.c.id) + .where(conversations.c.ws_id == ws_id) + .order_by(conversations.c.id) + .limit(1) + .offset(keep_count) + ).fetchone() + if cutoff_row is None: + return 0 + cutoff_id = cutoff_row[0] + if pre_delete is not None: + pre_delete(conn, ws_id, cutoff_id) + deleted = conn.execute( + sa.delete(conversations) + .where( + sa.and_( + conversations.c.ws_id == ws_id, + conversations.c.id >= cutoff_id, + ) + ) + .returning(conversations.c.attachments) + ).fetchall() + doomed_ids: list[str] = [] + for (refs,) in deleted: + doomed_ids.extend(parse_attachment_refs(refs)) + release_attachment_refs(conn, doomed_ids) + return len(deleted) + + def reconstruct_turns_checkpointed( rows: list[Any], ws_id: str, @@ -1527,18 +2245,32 @@ def _fork_turn_insert_row( fork_preview = preview meta_envelope: dict[str, Any] = {} - if turn.role is Role.TOOL: + if turn.role is Role.ASSISTANT: + source_meta = turn.meta.extra.get("source_meta") + if isinstance(source_meta, dict) and source_meta: + meta_envelope.update(source_meta) + provenance = TurnProvenance.from_meta(turn.meta.extra.get(PROVENANCE_META_KEY)) + if provenance is not None: + meta_envelope[PROVENANCE_META_KEY] = provenance.to_meta() + elif turn.role is Role.TOOL: if turn.effect_status is not None: meta_envelope["effect_status"] = turn.effect_status.value if fork_preview is not None: meta_envelope["preview"] = fork_preview + acting_principal = turn.meta.extra.get("acting_principal") + if isinstance(acting_principal, str) and acting_principal: + meta_envelope["acting_principal"] = acting_principal else: source_meta = turn.meta.extra.get("source_meta") sender = turn.meta.extra.get("sender") if isinstance(source_meta, dict) and source_meta: meta_envelope = source_meta - elif turn.role is Role.USER and isinstance(sender, str) and sender: - meta_envelope = {"sender": sender} + elif turn.role is Role.USER: + if isinstance(sender, str) and sender: + meta_envelope["sender"] = sender + client_send_ids = turn.meta.extra.get("client_send_ids") + if isinstance(client_send_ids, list) and client_send_ids: + meta_envelope["client_send_ids"] = list(client_send_ids) meta_json = json.dumps(meta_envelope) if meta_envelope else None source = msg.get("_source") diff --git a/turnstone/core/storage/migrations/versions/071_conversations_commit_key.py b/turnstone/core/storage/migrations/versions/071_conversations_commit_key.py new file mode 100644 index 00000000..06ae6d59 --- /dev/null +++ b/turnstone/core/storage/migrations/versions/071_conversations_commit_key.py @@ -0,0 +1,76 @@ +"""Add idempotency keys for conversation commits. + +``commit_key`` identifies one admitted live conversation row independently of +its content. A retry after an ambiguous database acknowledgement uses the +same key and resolves to the already-committed row instead of appending a +duplicate. The column is nullable so legacy, bulk, and offline writers retain +their append-only semantics; SQLite and PostgreSQL both allow multiple NULLs +under the partial composite unique index. PostgreSQL builds the index +concurrently so upgrading a large live conversation table does not block +writes; SQLite uses its ordinary partial-index DDL. + +Revision ID: 071 +Revises: 070 +Create Date: 2026-08-09 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "071" +down_revision = "070" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + if op.get_bind().dialect.name == "postgresql": + # ``autocommit_block`` commits every preceding operation, so the column + # add must itself be restart-safe: a crash during the concurrent index + # build leaves revision 070 stamped but the column already durable. + # PostgreSQL can also retain an INVALID index after an interrupted + # CREATE INDEX CONCURRENTLY; remove only that unusable residue before + # the idempotent rebuild. The migration runner's session advisory lock + # still serializes competing schema upgrades. + with op.get_context().autocommit_block(): + op.execute("ALTER TABLE conversations ADD COLUMN IF NOT EXISTS commit_key TEXT") + invalid_index = ( + op.get_bind() + .execute( + sa.text( + "SELECT NOT i.indisvalid " + "FROM pg_class AS c " + "JOIN pg_index AS i ON i.indexrelid = c.oid " + "JOIN pg_namespace AS n ON n.oid = c.relnamespace " + "WHERE c.relname = 'uq_conversations_ws_commit_key' " + "AND n.nspname = current_schema()" + ) + ) + .scalar_one_or_none() + ) + if invalid_index: + op.execute("DROP INDEX CONCURRENTLY IF EXISTS uq_conversations_ws_commit_key") + op.execute( + "CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS " + "uq_conversations_ws_commit_key " + "ON conversations (ws_id, commit_key) WHERE commit_key IS NOT NULL" + ) + else: + op.add_column("conversations", sa.Column("commit_key", sa.Text(), nullable=True)) + op.create_index( + "uq_conversations_ws_commit_key", + "conversations", + ["ws_id", "commit_key"], + unique=True, + sqlite_where=sa.text("commit_key IS NOT NULL"), + ) + + +def downgrade() -> None: + if op.get_bind().dialect.name == "postgresql": + with op.get_context().autocommit_block(): + op.execute("DROP INDEX CONCURRENTLY IF EXISTS uq_conversations_ws_commit_key") + else: + op.drop_index("uq_conversations_ws_commit_key", table_name="conversations") + with op.batch_alter_table("conversations") as batch_op: + batch_op.drop_column("commit_key") diff --git a/turnstone/core/trajectory.py b/turnstone/core/trajectory.py index 8f01d69c..3e0c0c27 100644 --- a/turnstone/core/trajectory.py +++ b/turnstone/core/trajectory.py @@ -83,6 +83,70 @@ class AttachmentRef: ContentBlock = TextBlock | AttachmentRef +PROVENANCE_META_KEY = "provenance" + + +@dataclass(frozen=True, slots=True) +class TurnProvenance: + """Immutable identity of the model attempt that produced a turn. + + The four fields are request facts, captured from one resolved serving lane + and the caller's already-pinned principal. They deliberately exclude the + endpoint URL and credential: both can contain secrets, while neither is + needed to distinguish a registry binding for audit or accounting. + + The serialized form rides ``TurnMeta.extra[PROVENANCE_META_KEY]``. Empty + strings and generation zero are explicit "not registry/auth scoped" + values for direct CLI, eval, and test lanes; they are not omitted so every + accepted model turn has one stable four-axis shape. + """ + + model_alias: str = "" + backend_model_id: str = "" + registry_generation: int = 0 + acting_principal_id: str = "" + + def to_meta(self) -> dict[str, str | int]: + """Return the JSON-safe well-known metadata object.""" + return { + "model_alias": self.model_alias, + "backend_model_id": self.backend_model_id, + "registry_generation": self.registry_generation, + "acting_principal_id": self.acting_principal_id, + } + + @classmethod + def from_meta(cls, raw: Any) -> TurnProvenance | None: + """Decode a stored provenance object, rejecting torn/corrupt shapes. + + A partially trustworthy identity is worse than no identity: consumers + could join it to the wrong registry generation or principal. Require + every axis with its exact scalar type and tolerate future sibling keys + by projecting only the four fields owned here. + """ + if not isinstance(raw, dict): + return None + model_alias = raw.get("model_alias") + backend_model_id = raw.get("backend_model_id") + registry_generation = raw.get("registry_generation") + acting_principal_id = raw.get("acting_principal_id") + if ( + not isinstance(model_alias, str) + or not isinstance(backend_model_id, str) + or not isinstance(registry_generation, int) + or isinstance(registry_generation, bool) + or registry_generation < 0 + or not isinstance(acting_principal_id, str) + ): + return None + return cls( + model_alias=model_alias, + backend_model_id=backend_model_id, + registry_generation=registry_generation, + acting_principal_id=acting_principal_id, + ) + + @dataclass(slots=True) class ToolCall: """A client (locally-executed) tool call. Server-side tool calls live in @@ -113,17 +177,25 @@ class ProviderNative: class TurnMeta: """Sidecar metadata: never reaches the wire, never read by the lowering layer. - ``event_id`` is the per-ws SSE ``Last-Event-ID`` resume cursor; ``extra`` holds - open metadata under well-known keys — ``"source_meta"`` (an operator-context + ``event_id`` is the per-ws SSE ``Last-Event-ID`` resume cursor; + ``commit_key`` is the storage idempotency identity used only to reconcile a + live-to-durable history handoff. ``extra`` holds open metadata under + well-known keys — ``"source_meta"`` (an operator-context ``system`` turn's structured per-kind fields, e.g. ``watch_triggered``'s ``watch_name`` / ``command`` / poll counters; persisted in the ``conversations.meta`` column, surfaced to the FE for per-kind rendering) and - ``"attachments_meta"`` (display metadata for by-reference attachments). - Storage-backed canonical loads also carry ``"storage_attachment_ids"``: - the raw ordered row ref-list used only to make fork retention fail-closed; - dict/wire projection deliberately ignores it.""" + ``"attachments_meta"`` (display metadata for by-reference attachments) and + ``"provenance"`` (the immutable model alias / backend id / registry + generation / acting-principal tuple captured by the successful attempt). + Storage-backed canonical loads also carry ``"storage_attachment_ids"`` (the + raw ordered row ref-list used only to make fork retention fail-closed) and, + on TOOL turns, ``"acting_principal"`` (the principal whose turn executed + the effect, for revocation and audit). Both are persistence-only side + channels: dict/wire projection deliberately ignores them, so no public + payload can carry the audit identity.""" event_id: int | None = None + commit_key: str | None = None extra: dict[str, Any] = field(default_factory=dict) @@ -280,6 +352,19 @@ def _content_to_raw(content: tuple[ContentBlock, ...]) -> str | list[dict[str, A return parts +def sanitize_client_send_ids(value: object) -> list[str]: + """Normalize an untrusted ``client_send_ids`` payload to its stable form. + + The single filter for the correlation-id list every boundary reads + (turn reconstruction, /history projection, canonical storage reload) — + a hardening change lands here once, so the same stored row can never + project different correlation sets per path. + """ + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, str) and item] + + def turn_from_dict(msg: dict[str, Any]) -> Turn: """Read an OpenAI-like message dict (with ``_``-side channels) as a ``Turn``.""" role_str = msg.get("role", "") @@ -301,7 +386,11 @@ def turn_from_dict(msg: dict[str, Any]) -> Turn: if pc is not None: native = ProviderNative(producer=msg.get("_producer", ""), blocks=tuple(pc)) - meta = TurnMeta(event_id=msg.get("_event_id")) + raw_commit_key = msg.get("_commit_key") + meta = TurnMeta( + event_id=msg.get("_event_id"), + commit_key=raw_commit_key if isinstance(raw_commit_key, str) and raw_commit_key else None, + ) am = msg.get("_attachments_meta") if am is not None: meta.extra["attachments_meta"] = am @@ -328,6 +417,12 @@ def turn_from_dict(msg: dict[str, Any]) -> Turn: sndr = msg.get("_sender") if sndr: meta.extra["sender"] = sndr + stable_client_send_ids = sanitize_client_send_ids(msg.get("_client_send_ids")) + if stable_client_send_ids: + meta.extra["client_send_ids"] = stable_client_send_ids + provenance = TurnProvenance.from_meta(msg.get("_provenance")) + if provenance is not None: + meta.extra[PROVENANCE_META_KEY] = provenance.to_meta() return Turn( role=role, @@ -365,6 +460,8 @@ def turn_to_dict(turn: Turn) -> dict[str, Any]: msg["_producer"] = turn.native.producer if turn.meta.event_id is not None: msg["_event_id"] = turn.meta.event_id + if turn.meta.commit_key is not None: + msg["_commit_key"] = turn.meta.commit_key am = turn.meta.extra.get("attachments_meta") if am is not None: msg["_attachments_meta"] = am @@ -380,6 +477,12 @@ def turn_to_dict(turn: Turn) -> dict[str, Any]: sndr = turn.meta.extra.get("sender") if sndr: msg["_sender"] = sndr + client_send_ids = turn.meta.extra.get("client_send_ids") + if isinstance(client_send_ids, list) and client_send_ids: + msg["_client_send_ids"] = list(client_send_ids) + provenance = TurnProvenance.from_meta(turn.meta.extra.get(PROVENANCE_META_KEY)) + if provenance is not None: + msg["_provenance"] = provenance.to_meta() return msg diff --git a/turnstone/core/workstream.py b/turnstone/core/workstream.py index 0a25be36..126f9091 100644 --- a/turnstone/core/workstream.py +++ b/turnstone/core/workstream.py @@ -13,7 +13,7 @@ import threading import time import uuid from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, cast, get_args if TYPE_CHECKING: from collections.abc import Callable @@ -26,6 +26,47 @@ if TYPE_CHECKING: # the /send defer keys on exactly these values. WorkerKind = Literal["", "turn", "command"] +# Sanitized operator-facing projection of the accepted-conversation journal. +# Detailed retry bookkeeping stays private to ChatSession; public API rows carry +# only this four-state value so older nodes and unloaded rows can safely default +# to ``healthy`` without exposing commit keys or internal errors. +ConversationPersistenceState = Literal["healthy", "pending", "retrying", "conflict"] +_CONVERSATION_PERSISTENCE_STATES: frozenset[str] = frozenset(get_args(ConversationPersistenceState)) + + +def normalize_conversation_persistence_state(value: Any) -> ConversationPersistenceState: + """Coerce a peer/session status value to the backward-compatible default.""" + if isinstance(value, str) and value in _CONVERSATION_PERSISTENCE_STATES: + return cast("ConversationPersistenceState", value) + return "healthy" + + +def concrete_method(obj: Any, name: str) -> Callable[..., Any] | None: + """Return ``obj``'s real ``name`` method, or ``None`` when it has none. + + Sessions and UIs are consumed through optional hooks that compatibility + shims, older implementations, and test doubles may not carry, so every + caller of such a hook first has to ask whether this object really + implements it. A plain ``getattr`` cannot answer that: ``MagicMock`` + auto-vivifies any missing attribute as a callable child, which would make + every optional hook look present under unit tests. + + A hook counts as concrete when the object's *type* defines it OR the name + is bound in the instance ``__dict__``. The instance term keeps + deliberately installed per-instance hooks working — including an attribute + a test assigned on purpose — while auto-vivified mock children, which live + in ``_mock_children`` and never reach ``__dict__``, stay invisible. An + attribute that resolves to a non-callable reads as "no hook". + """ + instance_fields = getattr(obj, "__dict__", None) + if not ( + callable(getattr(type(obj), name, None)) + or (isinstance(instance_fields, dict) and name in instance_fields) + ): + return None + method = getattr(obj, name, None) + return method if callable(method) else None + # --------------------------------------------------------------------------- # Kind enum — single source of truth for the workstream dispatch classifier @@ -249,8 +290,9 @@ class Workstream: # retry / wake — the default) or "command" (a slash-command worker, # including the minutes-long manual /compact). Written under # ``_lock`` by ``session_worker.send`` in the same acquisition that - # sets ``worker_thread``/``_worker_running``, so readers gating on - # the running flag see a coherent triple. A stale value after the + # sets ``worker_thread``/``_worker_running`` and the active principal, + # so readers gating on the running flag see one coherent slot claim. A + # stale value after the # worker exits is harmless — every reader conjoins # ``_worker_running``. The /send route defers (never queues) while # this reads "command": the mid-turn interjection queue is @@ -258,6 +300,20 @@ class Workstream: # unreachable during command windows — deferred entries live on # ``_pending_sends`` below. worker_kind: WorkerKind = field(default="", repr=False) + # Authenticated principal that owns the active turn worker. Claimed in + # the same ``_lock`` transition as ``_worker_running``/``worker_thread`` + # so queue admission never consults the previous turn's sticky session + # actor while a newly spawned worker is still binding its user context. + # Empty for internal/unauthenticated workers; readers must still conjoin + # ``_worker_running``. + _worker_principal_id: str = field(default="", repr=False) + # Whether operator force-cancel may abandon this exact worker slot and + # admit a concurrent successor. Ordinary sends/compactions are cooperative + # and abandonable; lifecycle mutations that destructively rewrite shared + # history claim ``False`` so force remains a cancellation request but may + # not reopen the slot until the mutation exits. Written and owner-cleared + # atomically with the rest of the worker claim under ``_lock``. + _worker_force_abandonable: bool = field(default=True, repr=False) # Sends deferred while the order barrier holds (full-fidelity # pending entries — see :class:`_PendingSend` above), dispatched in # arrival order by the per-workstream drain thread when the slot @@ -345,3 +401,38 @@ class Workstream: """ drain = self._pending_drain return bool(self._pending_sends) or (drain is not None and drain.is_alive()) + + +def session_persistence_state(session: Any) -> ConversationPersistenceState: + """Return the sanitized conversation-persistence state for a session. + + ``ChatSession.conversation_persistence_status`` is intentionally richer + than the public API. This projection accepts only its documented enum and + fails closed to ``healthy`` for missing sessions, older session + implementations, test doubles, exceptions, and malformed responses. + """ + if session is None: + return "healthy" + status = concrete_method(session, "conversation_persistence_status") + if status is None: + return "healthy" + try: + result = status() + except Exception: + return "healthy" + if not isinstance(result, dict): + return "healthy" + return normalize_conversation_persistence_state(result.get("state")) + + +def workstream_persistence_state(ws: Any) -> ConversationPersistenceState: + """Sanitized conversation-persistence state for a live registry row. + + Registry-resolved callers only (list endpoints walking the manager's + rows). A UI reporting on its *own* session must derive through the + session bound at construction (``SessionUIBase._current_persistence_state``) + instead: a registry lookup by id fails open to ``healthy`` — or to a + replacement workstream after id reuse — exactly while failed-delete + tombstone retention or retirement has the row out of the map. + """ + return session_persistence_state(getattr(ws, "session", None)) diff --git a/turnstone/sdk/__init__.py b/turnstone/sdk/__init__.py index 8c4782c1..6414bea6 100644 --- a/turnstone/sdk/__init__.py +++ b/turnstone/sdk/__init__.py @@ -25,8 +25,10 @@ from turnstone.sdk.events import ( ClusterWsRenameEvent, ConnectedEvent, ContentEvent, + ConversationPersistenceState, ErrorEvent, HistoryEvent, + HistoryResyncEvent, InfoEvent, NodeJoinedEvent, NodeLostEvent, @@ -40,6 +42,7 @@ from turnstone.sdk.events import ( ToolOutputChunkEvent, ToolPendingEvent, ToolResultEvent, + UserTurnEvent, WsActivityEvent, WsClosedEvent, WsRenameEvent, @@ -57,10 +60,13 @@ __all__ = [ "AttachmentUpload", "TurnResult", "TurnstoneAPIError", + "ConversationPersistenceState", # Server events "ServerEvent", "ConnectedEvent", "HistoryEvent", + "HistoryResyncEvent", + "UserTurnEvent", "ThinkingStartEvent", "ThinkingStopEvent", "ReasoningEvent", diff --git a/turnstone/sdk/console.py b/turnstone/sdk/console.py index 4ec35783..b7fb1e93 100644 --- a/turnstone/sdk/console.py +++ b/turnstone/sdk/console.py @@ -363,6 +363,7 @@ class AsyncTurnstoneConsole(_BaseClient): message: str, *, attachment_ids: list[str] | None = None, + client_send_id: str | None = None, ) -> dict[str, Any]: """Send a message to a coordinator workstream. @@ -373,6 +374,8 @@ class AsyncTurnstoneConsole(_BaseClient): body: dict[str, Any] = {"message": message} if attachment_ids is not None: body["attachment_ids"] = attachment_ids + if client_send_id is not None: + body["client_send_id"] = client_send_id return await self._request("POST", f"/v1/api/workstreams/{ws_id}/send", json_body=body) async def coordinator_upload_attachment( @@ -1346,9 +1349,15 @@ class TurnstoneConsole: message: str, *, attachment_ids: list[str] | None = None, + client_send_id: str | None = None, ) -> dict[str, Any]: return self._runner.run( - self._async.coordinator_send(ws_id, message, attachment_ids=attachment_ids) + self._async.coordinator_send( + ws_id, + message, + attachment_ids=attachment_ids, + client_send_id=client_send_id, + ) ) def coordinator_upload_attachment( diff --git a/turnstone/sdk/events.py b/turnstone/sdk/events.py index 724adf46..b6b911cc 100644 --- a/turnstone/sdk/events.py +++ b/turnstone/sdk/events.py @@ -9,6 +9,11 @@ from __future__ import annotations from dataclasses import dataclass, field, fields from typing import Any +# Runtime import keeps the public SDK alias and dataclass annotations resolvable. +from turnstone.core.workstream import ( # noqa: TC001 + ConversationPersistenceState as ConversationPersistenceState, +) + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -80,6 +85,37 @@ class HistoryEvent(ServerEvent): messages: list[dict[str, Any]] = field(default_factory=list) +@dataclass +class HistoryResyncEvent(ServerEvent): + """The rendered REST history no longer matches the live row prefix. + + This is stronger than an event-ring gap. Callers must stop using the + current stream, fetch ``/history`` again, render that response, and open a + new stream with its one-shot handoff token. The SDK intentionally does + not perform that policy automatically. + """ + + type: str = "history_resync" + reason: str = "" + + +@dataclass +class UserTurnEvent(ServerEvent): + """Canonical accepted user row projected to every stream consumer. + + ``client_send_ids`` correlate browser optimistic bubbles only; they are + not idempotency keys. ``_event_id`` is the monotonic row/event identity. + """ + + type: str = "user_turn" + content: str = "" + attachments: list[dict[str, Any]] = field(default_factory=list) + sender: str = "" + source: str = "" + client_send_ids: list[str] = field(default_factory=list) + _event_id: int | None = None + + @dataclass class ThinkingStartEvent(ServerEvent): type: str = "thinking_start" @@ -195,6 +231,10 @@ class ToolResultEvent(ServerEvent): name: str = "" output: str = "" is_error: bool = False + preview: dict[str, Any] | None = None + accepted: bool = False + effect_status: str = "" + _event_id: int | None = None @dataclass @@ -355,6 +395,7 @@ class WsStateEvent(ServerEvent): context_ratio: float = 0.0 activity: str = "" activity_state: str = "" + persistence_state: ConversationPersistenceState = "healthy" content: str = "" # populated on idle transitions only @@ -418,6 +459,7 @@ class ClusterStateEvent(ClusterEvent): context_ratio: float = 0.0 activity: str = "" activity_state: str = "" + persistence_state: ConversationPersistenceState = "healthy" @dataclass @@ -426,6 +468,7 @@ class ClusterWsCreatedEvent(ClusterEvent): ws_id: str = "" node_id: str = "" name: str = "" + persistence_state: ConversationPersistenceState = "healthy" @dataclass @@ -497,6 +540,8 @@ _SERVER_REGISTRY: dict[str, type[ServerEvent]] = { for cls in [ ConnectedEvent, HistoryEvent, + HistoryResyncEvent, + UserTurnEvent, ThinkingStartEvent, ThinkingStopEvent, ReasoningEvent, diff --git a/turnstone/sdk/server.py b/turnstone/sdk/server.py index c72d073e..1d7e7a5c 100644 --- a/turnstone/sdk/server.py +++ b/turnstone/sdk/server.py @@ -38,6 +38,7 @@ from turnstone.api.server_schemas import ( MemoryInfo, SendResponse, UploadAttachmentResponse, + WorkstreamHistoryResponse, ) from turnstone.sdk._base import _BaseClient from turnstone.sdk._sync import _SyncRunner @@ -215,10 +216,13 @@ class AsyncTurnstoneServer(_BaseClient): ws_id: str, *, attachment_ids: list[str] | None = None, + client_send_id: str | None = None, ) -> SendResponse: body: dict[str, Any] = {"message": message} if attachment_ids is not None: body["attachment_ids"] = list(attachment_ids) + if client_send_id is not None: + body["client_send_id"] = client_send_id return await self._request( "POST", f"/v1/api/workstreams/{ws_id}/send", @@ -350,11 +354,56 @@ class AsyncTurnstoneServer(_BaseClient): response_model=StatusResponse, ) + # -- history -------------------------------------------------------------- + + async def get_history( + self, + ws_id: str, + *, + limit: int = 100, + ) -> WorkstreamHistoryResponse: + """Return the requested tail of the authoritative total accepted row prefix. + + A loaded workstream may include a one-shot ``handoff_token``. A caller + that renders this response can pass that token, together with the + optional ``cursor``, to its next :meth:`stream_events` call. A 503 is + raised as :class:`TurnstoneAPIError`; that response is not authoritative + and must not replace a previously rendered transcript. + """ + return await self._request( + "GET", + f"/v1/api/workstreams/{ws_id}/history", + params={"limit": limit}, + response_model=WorkstreamHistoryResponse, + ) + # -- streaming ----------------------------------------------------------- - async def stream_events(self, ws_id: str) -> AsyncIterator[ServerEvent]: - """Iterate over per-workstream SSE events.""" - async for data in self._stream_sse(f"/v1/api/workstreams/{ws_id}/events"): + async def stream_events( + self, + ws_id: str, + *, + last_event_id: int | None = None, + history_token: str | None = None, + ) -> AsyncIterator[ServerEvent]: + """Iterate over per-workstream SSE events. + + ``last_event_id`` and ``history_token`` are initial connection hints. + Pass the cursor and one-shot token returned by the history response only + after rendering that response. If the stream yields + :class:`HistoryResyncEvent`, stop it, fetch and render history again, and + open a new stream with the new hints. This raw iterator does not + reconnect or apply transcript-repair policy automatically. + """ + params: dict[str, Any] = {"user_turn": 1} + if last_event_id is not None: + params["last_event_id"] = last_event_id + if history_token: + params["history_token"] = history_token + async for data in self._stream_sse( + f"/v1/api/workstreams/{ws_id}/events", + params=params, + ): yield ServerEvent.from_dict(data) async def stream_global_events(self) -> AsyncIterator[ServerEvent]: @@ -398,7 +447,10 @@ class AsyncTurnstoneServer(_BaseClient): result = TurnResult(ws_id=ws_id) async def _consume() -> None: - async for data in self._stream_sse(f"/v1/api/workstreams/{ws_id}/events"): + async for data in self._stream_sse( + f"/v1/api/workstreams/{ws_id}/events", + params={"user_turn": 1}, + ): event = ServerEvent.from_dict(data) if on_event: on_event(event) @@ -679,8 +731,16 @@ class TurnstoneServer: ws_id: str, *, attachment_ids: list[str] | None = None, + client_send_id: str | None = None, ) -> SendResponse: - return self._runner.run(self._async.send(message, ws_id, attachment_ids=attachment_ids)) + return self._runner.run( + self._async.send( + message, + ws_id, + attachment_ids=attachment_ids, + client_send_id=client_send_id, + ) + ) # -- attachments --------------------------------------------------------- @@ -738,10 +798,27 @@ class TurnstoneServer: def retry(self, ws_id: str) -> StatusResponse: return self._runner.run(self._async.retry(ws_id)) + # -- history -------------------------------------------------------------- + + def get_history(self, ws_id: str, *, limit: int = 100) -> WorkstreamHistoryResponse: + return self._runner.run(self._async.get_history(ws_id, limit=limit)) + # -- streaming ----------------------------------------------------------- - def stream_events(self, ws_id: str) -> Iterator[ServerEvent]: - return self._runner.run_iter(self._async.stream_events(ws_id)) + def stream_events( + self, + ws_id: str, + *, + last_event_id: int | None = None, + history_token: str | None = None, + ) -> Iterator[ServerEvent]: + return self._runner.run_iter( + self._async.stream_events( + ws_id, + last_event_id=last_event_id, + history_token=history_token, + ) + ) def stream_global_events(self) -> Iterator[ServerEvent]: return self._runner.run_iter(self._async.stream_global_events()) diff --git a/turnstone/server.py b/turnstone/server.py index 1c73a982..cf81fd4d 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -73,6 +73,7 @@ from turnstone.core.model_turn import ( from turnstone.core.ratelimit import resolve_client_ip from turnstone.core.session import ChatSession, GenerationCancelled, SessionUI # noqa: F401 from turnstone.core.session_manager import ( + PERSISTENCE_RECONCILE_INTERVAL_SECONDS, STALE_CREATE_GRACE_SECONDS, STALE_CREATE_SWEEP_INTERVAL_SECONDS, SessionManager, @@ -115,6 +116,8 @@ from turnstone.core.workstream import ( Workstream, WorkstreamKind, WorkstreamState, + concrete_method, + workstream_persistence_state, ) from turnstone.prompts import ClientType @@ -189,6 +192,41 @@ class WebUI(SessionUIBase): """ return self._kind, self._parent_ws_id + # ``_current_persistence_state`` inherited from :class:`SessionUIBase`: + # derives through the session bound at construction, never a registry + # lookup by id (which fails open to "healthy" exactly while tombstone + # retention or retirement has the row out of the map). + + def _publish_global_state_snapshot( + self, + state: str, + payload: dict[str, Any], + *, + include_content: bool, + ) -> None: + """Fan out one rich state snapshot without mutating session state.""" + if WebUI._global_queue is None: + return + kind, parent_ws_id = self._ws_kind_and_parent() + event: dict[str, Any] = { + "type": "ws_state", + "ws_id": self.ws_id, + "state": state, + "tokens": payload["tokens"], + "context_ratio": payload["context_ratio"], + "activity": payload["activity"], + "activity_state": payload["activity_state"], + "kind": kind, + "parent_ws_id": parent_ws_id, + "persistence_state": self._current_persistence_state(), + } + if include_content and state == "idle": + event["content"] = payload["content"] + try: + WebUI._global_queue.put_nowait(event) + except queue.Full: + log.debug("Global SSE queue full, dropping %s event", event.get("type")) + def _broadcast_state(self, state: str) -> None: """Send a state-change event to the global SSE channel. @@ -200,20 +238,6 @@ class WebUI(SessionUIBase): """ if WebUI._global_queue is not None: payload = self.snapshot_and_consume_state_payload(state) - kind, parent_ws_id = self._ws_kind_and_parent() - event: dict[str, Any] = { - "type": "ws_state", - "ws_id": self.ws_id, - "state": state, - "tokens": payload["tokens"], - "context_ratio": payload["context_ratio"], - "activity": payload["activity"], - "activity_state": payload["activity_state"], - "kind": kind, - "parent_ws_id": parent_ws_id, - } - if state == "idle": - event["content"] = payload["content"] # ``pending_approval_detail`` is NO LONGER piggybacked on # state-change events (Stage 3 cleanup). Symmetric event # flow now: initial approval items arrive via bulk fetch @@ -222,10 +246,31 @@ class WebUI(SessionUIBase): # ``intent_verdict`` event class, and resolution via # ``approval_resolved``. Reducer no longer has to dedupe # the piggyback path against the explicit one. - try: - WebUI._global_queue.put_nowait(event) - except queue.Full: - log.debug("Global SSE queue full, dropping %s event", event.get("type")) + self._publish_global_state_snapshot(state, payload, include_content=True) + + def on_persistence_state_changed(self) -> None: + """Refresh the operator row after journal failure or recovery. + + This intentionally uses the global node stream only. Conversation-pane + SSE is a transcript/control channel and must not receive operator-only + storage diagnostics. Unlike a real state transition, this snapshot does + not consume the terminal turn-content accumulator. + """ + # The registry read serves ONLY the row-state field (``ws.state`` + # lives on the manager's row); the persistence field itself derives + # through the bound session inside the snapshot publish. A miss + # here means the row left the roster — there is no operator row to + # refresh, so dropping is correct, and it can no longer launder a + # blocked journal into "healthy" (the pre-fix hazard). + mgr = WebUI._workstream_mgr + if mgr is None: + return + ws = mgr.get(self.ws_id) + if ws is None: + return + payload = self.snapshot_state_payload_non_consuming() + payload["content"] = "" + self._publish_global_state_snapshot(ws.state.value, payload, include_content=False) def _broadcast_activity(self) -> None: """Send an activity-change event to the global SSE channel.""" @@ -751,68 +796,6 @@ def _audit_retry_workstream( ) -def _interactive_dispatch_retry(ws: Workstream, user_msg: str) -> None: - """Re-send ``user_msg`` on an interactive workstream after ``/retry``. - - Passed to :func:`make_retry_handler` as ``dispatch_retry``; called - once :meth:`ChatSession.retry` has truncated the last turn. Drives - the shared :func:`turnstone.core.session_worker.send` dispatcher with - an interactive ``run`` closure (surfaces ``GenerationCancelled`` / - errors through the WebUI hooks) and a hard-reject ``enqueue`` closure - (a retry must not silently queue behind an in-flight turn — preserves - the pre-lift inline behaviour). The shared dispatcher owns the - ``_worker_running`` lifecycle, so the ``run`` closure needs no - ``finally`` flag-clear of its own. - - Deliberately NOT gated on the /send order barrier - (``ws._pending_sends``): a retry is an explicit user action that - rewinds a COMPLETED turn — dispatching it ahead of deferred sends is - an accepted overtake (the user just asked for exactly that turn to - run again), not the silent send-vs-send inversion the barrier exists - to prevent. Deferred entries dispatch after it, order among - themselves preserved. - """ - from turnstone.core import session_worker - - session = ws.session - ui = ws.ui - if session is None or ui is None: - return - - def _run() -> None: - me = threading.current_thread() - try: - session.send(user_msg) - except GenerationCancelled: - if ws.worker_thread is me: - ui.on_stream_end() - ui.on_state_change("idle") - except Exception as exc: - # Deliberately NOT routed through session.ensure_error_recorded: on a - # REUSED session a pre-try raise after a prior errored turn finds - # _has_persisted_error stale-True (it is session-lifetime — cleared - # only by _emit_state idle/running, not per-turn), so the recorder - # would no-op and swallow the fresh error. The DISPLAY string is - # sanitized inline (a credential-bearing base-URL in the exception - # text must not cross into the dashboard SSE, the confidentiality - # floor _record_fatal_error also enforces); the double state emit and - # the pre-try no-persist (a reused-session retry can then have the - # coordinator read a STALE last_error) still need the per-turn - # error-signal redesign and are tracked in #865, matching the /send - # and coord-send sibling closures. - if ws.worker_thread is me: - from turnstone.core.memory import sanitize_error_text - - ui.on_error(f"Error: {sanitize_error_text(str(exc))}") - ui.on_stream_end() - ui.on_state_change("error") - - def _enqueue() -> None: - ui.on_error("Cannot retry: workstream is busy") - - session_worker.send(ws, enqueue=_enqueue, run=_run, thread_name=f"retry-{ws.id[:8]}") - - def _interactive_events_replay( ws: Workstream, ui: Any, request: Request ) -> Iterable[dict[str, Any]]: @@ -832,17 +815,16 @@ def _interactive_events_replay( Pure read — never mutates ``ws`` / ``ui`` / ``session``. """ - session = ws.session - if session is None: + if ws.session is None: # Defensive — the lifted body's UI presence check guarantees # the workstream made it past placeholder state, but the # session can still be detached on the close-then-reopen path. return - # Connected + status preamble — same shape coord replays use; the - # shared helper keeps the two surfaces from drifting on a future - # field add. - yield from session_replay_preamble(session, ui) + # Connected + status preamble — same shape coord replays and the + # lifted reconnect path use; one shared function, no per-kind + # wrapper, so a future field add cannot land on one surface only. + yield from session_replay_preamble(ws.session, ui) # Pending approval re-injection (so a reconnecting tab sees the # prompt) + cached LLM verdicts received since the prompt fired. @@ -1062,6 +1044,7 @@ def _build_node_snapshot(app_state: Any) -> dict[str, Any]: "model_alias": ws.session.model_alias if ws.session else "", "kind": ws.kind, "parent_ws_id": ws.parent_ws_id, + "persistence_state": workstream_persistence_state(ws), "user_id": ws.user_id, "project_id": ws.project_id, "persona": ws.persona, @@ -1441,6 +1424,7 @@ async def dashboard(request: Request) -> JSONResponse: "model_alias": ws.session.model_alias if ws.session else "", "kind": ws.kind, "parent_ws_id": ws.parent_ws_id, + "persistence_state": workstream_persistence_state(ws), "user_id": ws.user_id, "project_id": ws.project_id, "persona": ws.persona, @@ -2128,6 +2112,7 @@ async def command(request: Request) -> JSONResponse: ws, enqueue=_reject_busy, run=run, + expected_session=session, thread_name=thread_name, worker_kind="command", ) @@ -2274,6 +2259,16 @@ async def command(request: Request) -> JSONResponse: updated_name = get_workstream_display_name(session.ws_id) if updated_name: ws.name = updated_name + except GenerationCancelled: + # Defense-in-depth, the sibling of _run_initial's arm (no + # generic command claims a generation today, so this is + # effectively unreachable). ``session_worker._runner`` + # catches only ``Exception`` — without this arm a stray + # cancel would kill the worker thread via + # ``threading.excepthook``; the finally below still + # unblocks the endpoint either way. + if ws.worker_thread is me: + cmd_ui.on_info("Command cancelled.") except Exception as e: # Same guard: a late "Command error:" from an abandoned # worker would land mid-successor-turn. @@ -3037,13 +3032,11 @@ async def _interactive_create_post_install( session = ws.session send_id = uuid.uuid4().hex resolved_atts: list[Any] = [] - staged_ord: list[str] = [] if attachment_ids: - # Resolve (peek) the staged uploads. The buffer DRAIN happens - # after the dispatch below, and only on the spawn path — the - # enqueue path can't deliver attachments, so there they must - # stay staged (see ``_enqueue_init``). - resolved_atts, staged_ord, _drop = _resolve_staged(attachment_ids, ws.id, uid) + # Resolve without draining. Accepted USER journal admission owns + # the atomic transfer; every pre-admission refusal keeps staging + # intact for a retry (including the enqueue path below). + resolved_atts, _staged_ord, _drop = _resolve_staged(attachment_ids, ws.id, uid) def _run_initial() -> None: me = threading.current_thread() @@ -3168,6 +3161,7 @@ async def _interactive_create_post_install( ws, enqueue=_enqueue_init, run=_run_initial, + expected_session=session, thread_name=f"ws-init-{ws.id[:8]}", ) if not init_ok: @@ -3187,20 +3181,6 @@ async def _interactive_create_post_install( ws.id[:8], initial_message_status, ) - if staged_ord and not init_enqueued and init_ok: - # Spawn path took the message: drain the staged copies NOW, - # before this handler returns — the pane's rehydrate can only - # start after it receives this response, so it can never - # observe the consumed uploads as still-pending composer - # chips. (``enqueue`` runs synchronously inside ``send``, so - # ``init_enqueued`` is settled here.) ``_append_user_turn``'s - # own per-id discard then no-ops. - from turnstone.core.attachment_buffer import get_attachment_buffer - - _buf = get_attachment_buffer() - for _aid in staged_ord: - _buf.discard(_aid, ws_id=ws.id, user_id=uid) - out: dict[str, Any] = {} if initial_message_status: # Only present when the initial message was NOT delivered — the @@ -3297,13 +3277,10 @@ async def delete_workstream_endpoint(request: Request) -> JSONResponse: reservation_token, ) mgr = getattr(request.app.state, "workstreams", None) - supports_atomic_delete = mgr is not None and callable( - getattr(type(mgr), "delete_persisted", None) - ) - if supports_atomic_delete: - assert mgr is not None + delete_persisted = concrete_method(mgr, "delete_persisted") + if delete_persisted is not None: deleted = await asyncio.to_thread( - mgr.delete_persisted, + delete_persisted, ws_id, delete_fn=delete_exact, name=name, @@ -3322,7 +3299,7 @@ async def delete_workstream_endpoint(request: Request) -> JSONResponse: # ever-growing tree on the dashboard. Best-effort: an # emit failure must not roll back the storage delete or # 500 the response. - if mgr is not None and not supports_atomic_delete: + if mgr is not None and delete_persisted is None: try: mgr.delete(ws_id, name=name) except Exception: @@ -4889,7 +4866,7 @@ def _idle_cleanup_thread( rate_limiter: Any = None, stop: threading.Event | None = None, ) -> None: - """Run idle eviction plus always-on provisional-create recovery. + """Run persistence repair plus lifecycle and rate-limit maintenance. ``mgr.close_idle`` fires the adapter's ``emit_closed`` for each victim, which pushes ``ws_closed`` onto ``global_queue`` with @@ -4897,44 +4874,58 @@ def _idle_cleanup_thread( is gone — the frontend didn't differentiate "idle" from "closed" anyway and the duplicate event caused spurious UI flicker. - Hidden ``state='creating'`` rows use their own conservative two-hour - grace and are reaped even when ``timeout_sec == 0`` disables ordinary idle - eviction. An initial recovery pass covers restart before the first periodic - wait. ``stop`` (#885) is the lifespan shutdown signal, using the same - ``wait``-as-sleep pattern as :func:`_aggregate_emitter_thread`. + Transient accepted-row persistence retries use a short one-second tick. + Ordinary idle eviction retains its timeout/4 cadence, and hidden + ``state='creating'`` rows retain their independent five-minute sweep and + conservative two-hour grace. Thus ``timeout_sec == 0`` disables only idle + eviction, not either recovery path. ``stop`` (#885) is the lifespan + shutdown signal, using the same ``wait``-as-sleep pattern as + :func:`_aggregate_emitter_thread`. """ del global_queue # adapter handles the emission if stop is None: stop = threading.Event() idle_enabled = timeout_sec > 0 - check_every = ( + lifecycle_check_every = ( min(STALE_CREATE_SWEEP_INTERVAL_SECONDS, timeout_sec / 4) if idle_enabled else float(STALE_CREATE_SWEEP_INTERVAL_SECONDS) ) + check_every = min(PERSISTENCE_RECONCILE_INTERVAL_SECONDS, lifecycle_check_every) + try: + mgr.reconcile_unresolved_persistence() + except Exception: + log.debug("server.persistence_reconcile_initial_failed", exc_info=True) try: mgr.reap_stale_creating_reservations(STALE_CREATE_GRACE_SECONDS) except Exception: log.debug("server.stale_create_cleanup_initial_failed", exc_info=True) last_create_sweep_at = time.monotonic() + last_lifecycle_sweep_at = last_create_sweep_at while not stop.wait(check_every): - if idle_enabled: - try: - mgr.close_idle(timeout_sec) - except Exception: - log.debug("server.idle_cleanup_failed", exc_info=True) + try: + mgr.reconcile_unresolved_persistence() + except Exception: + log.debug("server.persistence_reconcile_failed", exc_info=True) now = time.monotonic() - if now - last_create_sweep_at >= STALE_CREATE_SWEEP_INTERVAL_SECONDS: - # Keep rare hidden-create GC on its own fixed cadence. A short - # idle timeout must not turn the cluster-liveness scan into part - # of the ordinary high-frequency idle sweep. - last_create_sweep_at = now - try: - mgr.reap_stale_creating_reservations(STALE_CREATE_GRACE_SECONDS) - except Exception: - log.debug("server.stale_create_cleanup_failed", exc_info=True) - if rate_limiter is not None: - rate_limiter.cleanup() + if now - last_lifecycle_sweep_at >= lifecycle_check_every: + last_lifecycle_sweep_at = now + if idle_enabled: + try: + mgr.close_idle(timeout_sec) + except Exception: + log.debug("server.idle_cleanup_failed", exc_info=True) + if now - last_create_sweep_at >= STALE_CREATE_SWEEP_INTERVAL_SECONDS: + # Keep rare hidden-create GC on its own fixed cadence. A short + # idle timeout must not turn the cluster-liveness scan into part + # of the ordinary high-frequency persistence sweep. + last_create_sweep_at = now + try: + mgr.reap_stale_creating_reservations(STALE_CREATE_GRACE_SECONDS) + except Exception: + log.debug("server.stale_create_cleanup_failed", exc_info=True) + if rate_limiter is not None: + rate_limiter.cleanup() # Shutdown sentinel for ``_global_fanout_thread`` (#885): the lifespan @@ -5469,7 +5460,6 @@ def create_app( ) retry_handler = make_retry_handler( interactive_endpoint_config, - dispatch_retry=_interactive_dispatch_retry, audit_emit=_audit_retry_workstream, accepted_permissions=("conversation.modify",), ) diff --git a/turnstone/shared_static/base.css b/turnstone/shared_static/base.css index 164a34e9..7a13b431 100644 --- a/turnstone/shared_static/base.css +++ b/turnstone/shared_static/base.css @@ -522,6 +522,26 @@ body { line-height: 1; } +.dash-persistence-badge { + display: inline-flex; + align-items: center; + padding: 1px 5px; + border: 1px solid var(--yellow); + border-radius: 2px; + background: var(--yellow-glow); + color: var(--yellow); + font-size: 0.68rem; + font-weight: 600; + line-height: 1.25; + letter-spacing: 0.01em; + white-space: nowrap; +} +.dash-persistence-badge[data-state="conflict"] { + border-color: var(--red); + background: var(--red-glow); + color: var(--red); +} + /* Inline "create a project" widget (project_creator.js) — a name input + Save + Cancel that replaces the native prompt across every "+ New project…" picker. grid-column lets it span both columns when mounted inside the composer's diff --git a/turnstone/shared_static/composer_queue.js b/turnstone/shared_static/composer_queue.js index 7922cbdf..a5642184 100644 --- a/turnstone/shared_static/composer_queue.js +++ b/turnstone/shared_static/composer_queue.js @@ -194,9 +194,10 @@ export function createQueueController(opts) { }); } - function addQueuedMessage(text, priority) { + function addQueuedMessage(text, priority, clientSendId) { var el = document.createElement("div"); el.className = "msg user msg-queued"; + if (clientSendId) el.dataset.clientSendId = clientSendId; el.setAttribute("role", "status"); var important = priority === "important"; if (important) { @@ -506,11 +507,165 @@ export function parsePriority(text) { return { displayText: text, priority: "notice" }; } +// Mint one opaque browser correlation token. This is deliberately not a +// delivery/idempotency key; the server may accept the same value on multiple +// distinct turns, whose event ids remain authoritative. +export function mintClientSendId() { + if (window.crypto && typeof window.crypto.randomUUID === "function") + return window.crypto.randomUUID(); + var bytes = new Uint8Array(16); + if (window.crypto && typeof window.crypto.getRandomValues === "function") { + window.crypto.getRandomValues(bytes); + } else { + for (var i = 0; i < bytes.length; i++) + bytes[i] = Math.floor(Math.random() * 256); + } + return Array.from(bytes, function (value) { + return value.toString(16).padStart(2, "0"); + }).join(""); +} + +export function sendBubbleWasAccepted(el) { + return !!(el && el.dataset && el.dataset.serverAccepted === "true"); +} + +// Mark at most one optimistic/queued bubble per id occurrence, in DOM order. +// client_send_id is correlation only: callers may legally reuse a token on a +// later distinct send, so a selector that marks every matching element would +// collapse real turns. Removal stays with each pane because queued bubbles +// must go through their queue controller. +export function markAcceptedClientSendBubbles( + candidates, + clientSendIds, + skipAlreadyAccepted = false, +) { + const available = Array.from(candidates || []); + const matched = []; + for (const clientSendId of clientSendIds || []) { + const index = available.findIndex( + (el) => + el.dataset && + el.dataset.clientSendId === clientSendId && + (!skipAlreadyAccepted || el.dataset.serverAccepted !== "true"), + ); + if (index < 0) continue; + const bubble = available.splice(index, 1)[0]; + bubble.dataset.serverAccepted = "true"; + matched.push(bubble); + } + return matched; +} + +// The correlation token is random but not an authorization credential. A +// peer who reuses or guesses it must not settle this viewer's optimistic +// bubble. Older events without sender identity retain their compatibility +// behavior; when both identities are known they must match exactly. +export function clientSendMaySettleForViewer(sender, viewer) { + const eventSender = typeof sender === "string" ? sender : ""; + const viewerId = typeof viewer === "string" ? viewer : ""; + return !eventSender || !viewerId || eventSender === viewerId; +} + +// The viewing operator's opaque user id, retained from /whoami. Storage access +// can be disabled (privacy mode, sandboxed frame); "" then reads as "viewer +// unknown", which is the compatibility case for every consumer above. +export function viewerUserId() { + try { + return sessionStorage.getItem("ts.user_id") || ""; + } catch (_err) { + return ""; + } +} + +// Settle the optimistic bubbles an accepted event names. A queued chip must +// leave through the queue controller (a bare remove() would strand its +// live-set bookkeeping); a plain optimistic bubble just detaches. +export function settleAcceptedClientSends( + messagesEl, + queue, + clientSendIds, + remove, + skipAlreadyAccepted = false, +) { + const candidates = Array.from( + messagesEl.querySelectorAll("[data-client-send-id]"), + ); + const matched = markAcceptedClientSendBubbles( + candidates, + clientSendIds, + skipAlreadyAccepted, + ); + if (remove) { + for (const bubble of matched) { + if (bubble.classList.contains("msg-queued")) queue.remove(bubble); + else bubble.remove(); + } + } + return matched; +} + +// Project one canonical `user_turn` event onto a pane. Everything structural is +// shared — the event-id dedupe that keeps an unpainted turn replayable, the +// viewer gate on settling optimistic bubbles, the settle + removal, the +// attachments handoff, and the system_nudge branch. Only the DOM writes arrive +// via host, so the two panes cannot drift on the projection order. +// +// host: +// renderedEventIds: Set of ids already projected +// messagesEl: container holding this viewer's optimistic bubbles +// queue: queue controller owning the queued chips +// consumeAttachments(ids): composer chip sync for a settled send +// renderNudgeMarker(): the wake-driven empty-turn marker +// renderUserTurn(content, attachments, opts): the ordinary user bubble. +// opts carries {eventId, sender, source, viewer}; the pane owns its own +// attachments default and any viewer-relative labelling. +export function acceptUserTurnEvent(evt, host) { + const eventId = evt._event_id != null ? String(evt._event_id) : null; + if (eventId && host.renderedEventIds.has(eventId)) return; + const viewer = viewerUserId(); + const maySettle = clientSendMaySettleForViewer(evt.sender, viewer); + if (maySettle) { + // Called for the side effect — it settles and removes this viewer's + // optimistic bubbles. Nothing below gates on how many it matched: + // an accepted turn with no local bubble (a create dispatch) still + // owns its attachment consumption. + settleAcceptedClientSends( + host.messagesEl, + host.queue, + evt.client_send_ids, + true, + ); + } + // Chip sync follows the same viewer policy as bubble settling — NOT the + // matched-bubble gate: a create-dispatched first turn (or any accepted + // turn whose optimistic bubble this tab never held) carries consumed + // attachment ids whose staged uploads are spent. A rehydrated pending + // chip left behind re-submits a drained id on the next send (spurious + // dropped-attachment warning) or double-delivers on a racing one. A + // known-DIFFERENT sender still never clears this viewer's chips. + if (maySettle && Array.isArray(evt.attachments)) { + host.consumeAttachments( + evt.attachments.map((item) => item && item.attachment_id).filter(Boolean), + ); + } + if (evt.source === "system_nudge") { + host.renderNudgeMarker(); + } else { + host.renderUserTurn(evt.content || "", evt.attachments, { + eventId: eventId, + sender: evt.sender || "", + source: evt.source || "", + viewer: viewer, + }); + } + if (eventId) host.renderedEventIds.add(eventId); +} + // Settle a parsed /send response against the pane's optimistic state — the // ONE implementation of the status dispatch both panes share (the // applyCompactionEvent hooks pattern: everything pane-specific arrives via -// ctx). The fetch-stage concerns (409 pre-parse, network .catch) stay -// per-pane; this owns everything after a parsed 2xx/handled body. +// ctx). This owns everything after a parsed 2xx/handled body; the fetch stage +// around it is postAndSettleSend below. // // ctx: // queuedEl: the pre-POST queued chip (busy pane) or null @@ -552,13 +707,27 @@ export function settleSendResponse(queue, data, ctx) { // without it the unknown/"ok" fall-through below would deref // data.attached_ids and surface a delivered message as a connection error. data = data || {}; + if ( + sendBubbleWasAccepted(ctx.queuedEl) || + sendBubbleWasAccepted(ctx.optimisticEl) + ) { + // The canonical user_turn event can beat this HTTP response. It proves + // admission even when the response itself was lost; never recreate, + // promote, or report failure for the already-settled optimistic row. + ctx.consumeAttachments(data.attached_ids, data.dropped_attachment_ids); + return; + } var status = data.status; if (status === "queued" && data.msg_id) { var queuedEl = ctx.queuedEl; if (!queuedEl && data.deferred) { if (ctx.optimisticEl && ctx.optimisticEl.isConnected) ctx.optimisticEl.remove(); - queuedEl = queue.addQueuedMessage(ctx.displayText, ctx.priority); + queuedEl = queue.addQueuedMessage( + ctx.displayText, + ctx.priority, + ctx.clientSendId, + ); } if (queuedEl) { queue.bind(queuedEl, data.msg_id, { @@ -646,3 +815,73 @@ export function settleSendResponse(queue, data, ctx) { if (ctx.queuedEl) queue.promote(ctx.queuedEl); ctx.consumeAttachments(data.attached_ids, data.dropped_attachment_ids); } + +// Drive one /send POST from its Response to the pane's optimistic state. The +// request stays pane-owned (base/URL, credentials, abort + timeout, body +// shape); everything from the response onward is identical for all four send +// flows — composer send and edit-and-resend, in each pane — so it lives here: +// the rejected-body normalization, the 409 conversion, settleSendResponse, and +// the transport catch. +// +// sendRequest: the pending fetch Promise for POST /send. +// ctx: exactly settleSendResponse's ctx (documented above). The +// edit-and-resend flows pass queuedEl:null + isBusy:false, which +// reduces the arms below to their single-bubble shape. +export function postAndSettleSend(queue, sendRequest, ctx) { + return sendRequest + .then((response) => { + if (response.ok) return response.json(); + // 409 = the server-side cross-user interjection block (another + // participant's turn is in flight). Convert to a handled status so it + // routes to the clean arm instead of the generic connection-error catch + // — the reactive fallback for the race where the send button wasn't yet + // disabled. + if (response.status === 409) { + return response.json().then( + (body) => ({ + status: "cross_user_interjection", + error: (body && body.error) || "", + }), + () => ({ status: "cross_user_interjection", error: "" }), + ); + } + // Any other rejected send carries {error}, not {status}; without this it + // would fall through to the unknown/"ok" arm and be promote()'d — a + // server-refused message shown as delivered (with a false "already sent" + // notice if it was dismissed). Throwing surfaces the server's {error} + // text ("No session", a rate-limit reason, ...) rather than a bare + // status code. A wedged proxy can answer non-JSON (502/504 HTML); the + // parse-failure arm falls back to the status code so that can't surface + // as an "Unexpected token <" error. + return response.json().then( + (body) => { + throw new Error( + (body && body.error) || "send_http_" + response.status, + ); + }, + () => { + throw new Error("send_http_" + response.status); + }, + ); + }) + .then((data) => settleSendResponse(queue, data, ctx)) + .catch((err) => { + // The canonical user_turn event can beat — or outlive — a failed + // response: an accepted bubble is already settled, so never remove it + // or report failure for it. Otherwise nothing landed; drop the + // optimistic rows and restore the pre-send busy state. + if ( + sendBubbleWasAccepted(ctx.queuedEl) || + sendBubbleWasAccepted(ctx.optimisticEl) + ) { + return; + } + if (ctx.queuedEl) queue.remove(ctx.queuedEl); + if (ctx.optimisticEl && ctx.optimisticEl.isConnected) + ctx.optimisticEl.remove(); + ctx.renderError( + "Connection error: " + (err && err.message ? err.message : err), + ); + if (!ctx.isBusy) ctx.setBusy(false); + }); +} diff --git a/turnstone/shared_static/history_handoff.js b/turnstone/shared_static/history_handoff.js new file mode 100644 index 00000000..2c8675af --- /dev/null +++ b/turnstone/shared_static/history_handoff.js @@ -0,0 +1,292 @@ +/* Shared fail-closed history-handoff repair policy. + * + * DOM, fetch, and EventSource lifecycles stay pane-owned; the safety-critical + * attempt budget and deadline live here so interactive and coordinator cannot + * drift. A manual click is always allowed after automatic recovery parks. + */ + +export const HISTORY_HANDOFF_MAX_ATTEMPTS = 4; +export const HISTORY_HANDOFF_FETCH_TIMEOUT_MS = 15000; + +export function historyHandoffAttemptAllowed( + attempts, + manualOnly, + manualAttempt = false, +) { + return ( + !!manualAttempt || (!manualOnly && attempts < HISTORY_HANDOFF_MAX_ATTEMPTS) + ); +} + +export function createHistoryHandoffDeadline( + onExpire, + timeoutMs = HISTORY_HANDOFF_FETCH_TIMEOUT_MS, +) { + const state = { expired: false, timer: null, settle: null }; + const promise = new Promise((resolve) => { + state.settle = resolve; + state.timer = setTimeout(() => { + state.expired = true; + if (typeof onExpire === "function") onExpire(); + resolve(null); + }, timeoutMs); + }); + return { + state, + promise, + /* The one retirement path, owned here so the panes cannot drift on + * slot order: stop the timer (no late expiry), drop the settle slot, + * and — when cancelling an attempt rather than recording its natural + * settlement — mark it dead (`expire`, the render-inert flag) and + * release the race (`resolve`, so no awaiter or timer closure + * outlives the attempt). Idempotent; `state.expired` stays readable + * but panes never write state slots directly. */ + dispose({ expire = false, resolve = false } = {}) { + if (expire) state.expired = true; + if (state.timer != null) { + clearTimeout(state.timer); + state.timer = null; + } + const settle = state.settle; + state.settle = null; + if (resolve && settle) settle(null); + }, + }; +} + +/* One repair-backoff step: the jittered delay to sleep now plus the doubled + * base for the next step, both capped. Both panes must pace identically. */ +export function nextHistoryHandoffDelay(currentMs, { jitterMs, maxMs }) { + return { + delayMs: Math.min(currentMs + Math.random() * jitterMs, maxMs), + nextBaseMs: Math.min(currentMs * 2, maxMs), + }; +} + +/* The parked-repair prompt. Copy, structure, and class hooks are part of the + * shared contract (tests and both panes key off them); placement, scroll, and + * retry gating stay pane-owned. */ +export function buildHistoryHandoffPrompt({ onRetry, onReload }) { + const prompt = document.createElement("div"); + prompt.className = "msg error history-handoff-repair"; + prompt.setAttribute("role", "alert"); + const copy = document.createElement("div"); + copy.textContent = + "Live updates are paused because conversation history could not be verified."; + prompt.appendChild(copy); + const actions = document.createElement("div"); + actions.className = "msg-actions"; + const retry = document.createElement("button"); + retry.type = "button"; + retry.className = "msg-action-btn history-handoff-retry"; + retry.textContent = "Retry now"; + retry.addEventListener("click", onRetry); + const reload = document.createElement("button"); + reload.type = "button"; + reload.className = "msg-action-btn history-handoff-reload"; + reload.textContent = "Reload page"; + reload.addEventListener( + "click", + onReload || (() => window.location.reload()), + ); + actions.appendChild(retry); + actions.appendChild(reload); + prompt.appendChild(actions); + return prompt; +} + +/* The repair state machine itself, owned once. + * + * Both panes held the same seven slots — pending latch, backoff timer, backoff + * base, in-flight attempt, attempt count, manual-only latch, parked prompt — + * and ran the same clear / park / begin / schedule / settle logic over them. + * Everything that differs arrives through deps: where the prompt goes, how + * history loads, how the transport reopens, what the status line reads, and + * whether the pane is still alive. The pane keeps its own handoff proof token; + * this owns only the recovery policy. + * + * `scope` is the workstream the repair intent belongs to. A pane reassignment + * supersedes the intent rather than repairing the wrong transcript. + * + * deps: + * load(scope, manualAttempt) run one /history attempt for that scope + * connect(scope) reopen the transport once the latch clears + * placePrompt(prompt) attach the parked prompt and scroll to it + * showPaused() pane status display for a parked repair + * setStale(stale) the transcript mutation latch + * deferToShowEdge() record that a hidden tab skipped an attempt + * isAlive() false once the stream lifecycle is released + * baseDelayMs / jitterMs / maxMs backoff pacing (pane-supplied constants) + */ +export function createHistoryHandoffRepair(deps) { + let pending = false; + let scope = null; + let timer = null; + let delayMs = deps.baseDelayMs; + let attempts = 0; + let manualOnly = false; + let promptEl = null; + /* Identity of the attempt currently in flight (null when none), so a settle + * that lost its race against a clear/restart cannot retire a live one. */ + let inFlightId = null; + let nextAttemptId = 0; + let attemptTeardown = null; + + function clear() { + if (timer != null) { + clearTimeout(timer); + timer = null; + } + if (attemptTeardown) { + const teardown = attemptTeardown; + attemptTeardown = null; + teardown(); + } + if (promptEl) { + promptEl.remove(); + promptEl = null; + } + pending = false; + scope = null; + delayMs = deps.baseDelayMs; + attempts = 0; + manualOnly = false; + inFlightId = null; + } + + function showManual() { + manualOnly = true; + deps.showPaused(); + let prompt = promptEl; + if (!prompt || !prompt.isConnected) { + prompt = buildHistoryHandoffPrompt({ + onRetry: () => { + if (pending && inFlightId == null && scope) deps.load(scope, true); + }, + }); + deps.placePrompt(prompt); + promptEl = prompt; + } + const retry = prompt.querySelector(".history-handoff-retry"); + if (retry) retry.disabled = inFlightId != null; + } + + function schedule() { + if (!pending || timer != null || inFlightId != null || !scope) return; + if (!deps.isAlive()) return; + if (!historyHandoffAttemptAllowed(attempts, manualOnly)) { + showManual(); + return; + } + const target = scope; + const { delayMs: delay, nextBaseMs } = nextHistoryHandoffDelay(delayMs, { + jitterMs: deps.jitterMs, + maxMs: deps.maxMs, + }); + delayMs = nextBaseMs; + timer = setTimeout(() => { + timer = null; + if (!pending || scope !== target || !deps.isAlive()) return; + if (document.hidden) { + // There is intentionally no EventSource while this repair is pending, + // so the ordinary hide handler has nothing to close. Mark the deferral + // explicitly; the show edge re-enters connect, whose repair chokepoint + // schedules the next bounded attempt. + deps.deferToShowEdge(); + return; + } + deps.load(target, false); + }, delay); + } + + return { + clear: clear, + showManual: showManual, + schedule: schedule, + + isRepairing(forScope) { + return pending && scope === forScope; + }, + + /* A real pane reassignment supersedes a repair intent: the new workstream + * performs its own history bootstrap. */ + supersede(nextScope) { + if (pending && scope !== nextScope) clear(); + }, + + begin(nextScope) { + if (pending && scope === nextScope) return; + clear(); + pending = true; + scope = nextScope; + // Reuse the transcript mutation latch rather than adding another + // affordance gate. It clears only on a completed render. + deps.setStale(pending); + deps.load(nextScope, false); + }, + + /* Admission for one attempt, before the caller commits to any work. False + * means "do not load": either an attempt is already in flight, or the + * budget is spent and the parked prompt now owns recovery. */ + admitAttempt(manualAttempt) { + if (inFlightId != null) return false; + if (!historyHandoffAttemptAllowed(attempts, manualOnly, manualAttempt)) { + showManual(); + return false; + } + if (timer != null) { + clearTimeout(timer); + timer = null; + } + return true; + }, + + /* Charge the attempt to the budget and park the retry affordance for its + * duration. `teardown` releases whatever fetch-bounding resources the pane + * armed for it, and runs if a clear cancels the attempt mid-flight. */ + startAttempt(manualAttempt, teardown) { + inFlightId = ++nextAttemptId; + attempts += 1; + if (manualAttempt) manualOnly = true; + attemptTeardown = teardown || null; + const retry = promptEl + ? promptEl.querySelector(".history-handoff-retry") + : null; + if (retry) retry.disabled = true; + return inFlightId; + }, + + endAttempt(attemptId) { + if (inFlightId !== attemptId) return; + inFlightId = null; + attemptTeardown = null; + }, + + /* The one repair verdict, for both panes. */ + settle({ outcome, hasToken, manualAttempt }) { + if (hasToken || outcome === "rendered") { + // A token means the full render completed and armed the proof from + // that same response; the reopened transport carries it. A COMPLETED + // render without a token is the server's deliberate tokenless read + // (cold storage-only, or a route with no storage handle): downgrade + // to the tokenless bootstrap, whose cursorless connect gets the + // server's clear_ui convergence instead of a cursor handoff — same + // contract as a pre-handoff server. Either way the repair is over. + const target = scope; + clear(); + deps.connect(target); + return; + } + // Fetch/JSON failure, a render throw, a superseded render, and deadline + // expiry all fail closed. Keep the stale transcript visible, keep its + // mutation latch closed, and retry at a bounded rate without opening an + // unverified stream. + deps.setStale(pending); + if (manualAttempt || attempts >= HISTORY_HANDOFF_MAX_ATTEMPTS) { + showManual(); + } else { + schedule(); + } + }, + }; +} diff --git a/turnstone/shared_static/interactive.js b/turnstone/shared_static/interactive.js index bd736cc0..aad21649 100644 --- a/turnstone/shared_static/interactive.js +++ b/turnstone/shared_static/interactive.js @@ -49,9 +49,14 @@ import { kindIcon, } from "./composer_attachments.js"; import { + acceptUserTurnEvent, + clientSendMaySettleForViewer, createQueueController, + mintClientSendId, parsePriority, - settleSendResponse, + postAndSettleSend, + settleAcceptedClientSends, + viewerUserId, } from "./composer_queue.js"; import { StatusBar } from "./status_bar.js"; import { streamingRender, streamingRenderFinalize } from "./renderer.js"; @@ -62,6 +67,18 @@ import { findMsgActionsBar, } from "./copy_actions.js"; import { makeAnnouncer, operatorSourceLabel } from "./utils.js"; +import { + createHistoryHandoffDeadline, + createHistoryHandoffRepair, + HISTORY_HANDOFF_FETCH_TIMEOUT_MS, +} from "./history_handoff.js"; +import { + acceptedToolEventAlreadyRendered, + enqueueToolOccurrence, + indexLatestToolRow, + recordAcceptedToolEvent, + shiftToolOccurrence, +} from "./tool_projection.js"; import { OVERFLOW_TRIP_COUNT, OVERFLOW_TRIP_WINDOW_MS, @@ -247,6 +264,42 @@ class Pane { this.projectName = ""; this._lastStatusEvt = null; this._historyLoadToken = 0; + // One-shot REST -> SSE bootstrap token. It names the exact live + // conversation revision rendered by the most recent seeded /history + // fetch. connectSSE consumes it into ``?history_token=`` once; later + // manual/native reconnects use only their event cursor. + this._historyHandoffToken = null; + // A history_resync is stronger than an ordinary transport gap: until a + // new /history payload has rendered and supplied a fresh handoff token, + // opening a cursorless/tokenless EventSource would silently accept the + // stale transcript. Keep that repair intent latched across fetch + // failures, hide/show, and unrelated reconnect attempts. One capped + // exponential timer owns retries; terminal pane teardown cancels it. + // The latch, budget, backoff, and parked prompt live in the shared + // controller so this pane and the coordinator cannot drift on them. + this._historyRepair = createHistoryHandoffRepair({ + baseDelayMs: STALE_RETRY_BASE_MS, + jitterMs: STALE_RETRY_JITTER_MS, + maxMs: DEGRADED_COOLDOWN_MAX_MS, + isAlive: () => !!this._visHandler, + load: (wsId, manualAttempt) => + this._loadHistoryThenConnect(wsId, manualAttempt), + connect: (wsId) => this.connectSSE(wsId), + setStale: (stale) => { + this._historyStale = stale; + }, + deferToShowEdge: () => { + this._hiddenDisconnect = true; + }, + showPaused: () => { + this.statusBarEl.classList.add("ws-sb-disconnected"); + this._sbTokens.textContent = "Live updates paused"; + }, + placePrompt: (prompt) => { + this.messagesEl.appendChild(prompt); + this.scrollToBottom(true); + }, + }); // Monotonic STREAM-generation counter (#900), bumped only in // evtSource.onopen. _refetchHistory captures it at dispatch and its // render-time gate requires it unchanged, so a transport that dropped @@ -277,6 +330,15 @@ class Pane { // declaration). Ship a stamp here only if a seedless refetch ever // starts without arming this queue. this._replayQueue = null; + // One microtask-owned stale-history backstop. A replay_ok connection + // prepends its synthetic current-state event before the buffered ring + // slice. If that leading idle starts a refetch synchronously while + // _endReplayQuiesce is still draining, the remainder is diverted into the + // new queue; the refetch paints those rows from history and the diverted + // content then paints them a second time. Deferring to the backlog tail + // preserves FIFO delivery, and this latch collapses multiple idle/error + // edges in the same slice into one guarded attempt. + this._staleBackstopMicrotaskPending = false; // Transcript-staleness latch (#890): TRUE = the visible transcript // no longer matches the server's conversation STRUCTURE. Set at // clear_ui arrival (the server just restructured) and on a ws @@ -329,9 +391,15 @@ class Pane { // so pre-wipe ids can't suppress re-painted rows; this is the // first-load init. this._renderedSystemEventIds = new Set(); + this._renderedUserEventIds = new Set(); + this._renderedToolEventIds = new Set(); this._retryHolderEl = null; this._toolRowIndex = new Map(); this._streamElIndex = new Map(); + // call_id -> { row, nodes }. Only result-owned siblings live here + // (output/media/MCP card/preview chip), so an accepted row can replace a + // provisional receipt without disturbing output-guard warnings. + this._toolResultNodes = new Map(); this._resizeObs = null; // Set when replay_truncated arrives mid-stream (refetching then would // detach the live bubble); consumed on the next idle edge. Cleared by @@ -713,10 +781,15 @@ class Pane { return el; } - addUserMessage(text, attachments) { + addUserMessage(text, attachments, opts) { + opts = opts || {}; this.removeEmptyState(); const el = document.createElement("div"); el.className = "msg user"; + if (opts.clientSendId) el.dataset.clientSendId = opts.clientSendId; + if (opts.eventId != null) el.dataset.eventId = String(opts.eventId); + if (opts.sender) el.dataset.sender = opts.sender; + if (opts.source) el.dataset.source = opts.source; const textEl = document.createElement("div"); textEl.className = "msg-user-text"; textEl.textContent = text; @@ -770,6 +843,28 @@ class Pane { return el; } + _markAcceptedClientSends(clientSendIds, remove, skipAlreadyAccepted = false) { + return settleAcceptedClientSends( + this.messagesEl, + this.queue, + clientSendIds, + remove, + skipAlreadyAccepted, + ); + } + + _acceptUserTurn(evt) { + acceptUserTurnEvent(evt, { + renderedEventIds: this._renderedUserEventIds, + messagesEl: this.messagesEl, + queue: this.queue, + consumeAttachments: (ids) => this.attachments.consume(ids, []), + renderNudgeMarker: () => this.addSystemNudgeMarker(), + renderUserTurn: (content, attachments, opts) => + this.addUserMessage(content, attachments || null, opts), + }); + } + // --- Approval-cycle bookkeeping ----------------------------------------- // The backend registers one ApprovalCycle per human-gated batch; parallel // task agents make several live at once. Cards register here on paint and @@ -1353,6 +1448,7 @@ class Pane { connectSSE(wsId) { this.disconnectSSE(); const wsChanged = this.wsId !== wsId; + if (wsChanged) this._historyRepair.supersede(wsId); this.wsId = wsId; if (wsChanged) { this.attachments.clearChips(); @@ -1400,6 +1496,14 @@ class Pane { if (connectCursor != null) { evtUrl += "?last_event_id=" + encodeURIComponent(connectCursor); } + // Capability is declared on every manual URL so browser-native + // reconnects retain it too. Without this query, the server must project a + // user_turn into a strong-repair frame for older reducers. + evtUrl += (evtUrl.includes("?") ? "&" : "?") + "user_turn=1"; + // Accepted TOOL rows use the same capability discipline. Preliminary + // executor receipts remain backward-compatible ``tool_result`` events; + // only the final guarded row requires reducer upsert semantics. + evtUrl += "&tool_turn=1"; // Close-on-hide / replay-on-show: installed once per pane, removed // by the factory's destroy(). A hidden tab's throttled drain is the // likeliest slow consumer behind server-side queue overflow, and an @@ -1424,7 +1528,26 @@ class Pane { this._hiddenDisconnect = true; return; } + if (this._historyRepair.isRepairing(wsId)) { + // Never fail open from a history mismatch into a cursorless/tokenless + // stream. Only a successfully rendered /history payload clears this + // latch and supplies the token for the next EventSource. + this.statusBarEl.classList.add("ws-sb-disconnected"); + this._sbTokens.textContent = "History out of date — retrying…"; + this._historyRepair.schedule(); + return; + } + if (this._historyHandoffToken != null) { + evtUrl += + (evtUrl.includes("?") ? "&" : "?") + + "history_token=" + + encodeURIComponent(this._historyHandoffToken); + } this.evtSource = new EventSource(evtUrl); + // One successfully-constructed EventSource owns this bootstrap attempt. + // Native reconnects on that source carry Last-Event-ID (which the server + // prioritises); brand-new sources must never reuse the history token. + this._historyHandoffToken = null; this.evtSource.onopen = () => { // Connection generation (#900) — bumped HERE and nowhere else, because @@ -1690,7 +1813,15 @@ class Pane { }, cooldown); } - _loadHistoryThenConnect(wsId) { + _loadHistoryThenConnect(wsId, manualAttempt = false) { + this._historyRepair.supersede(wsId); + const repairingHistoryHandoff = this._historyRepair.isRepairing(wsId); + if ( + repairingHistoryHandoff && + !this._historyRepair.admitAttempt(manualAttempt) + ) { + return; + } // Mirror coord's init() ordering: render history from REST first, // THEN open the live stream. Disconnect any existing stream up // front so stray events from the previously-assigned ws don't paint @@ -1706,6 +1837,7 @@ class Pane { // below gets the new ws's full initial state instead. this._lastEventId = null; this._lastStatusEvt = null; + this._historyHandoffToken = null; // Full-reload cleanup (NOT in disconnectSSE — transport-only reconnects // must preserve these): a stale quiesce queue would wedge the new load's // events behind a flush that never comes, stale agent tracking points at @@ -1765,20 +1897,79 @@ class Pane { // clear_ui re-render still calls _refetchHistory directly with // seedCursor=false — it runs on an already-live stream and must NOT // rewind _lastEventId off the live position. - this._refetchHistory(wsId, token, true).finally(() => { - // Failed-fetch retry rides the connect chokepoint, not this - // callback: a fetch failure leaves _truncatedFromCursor set (only - // replayHistory's full render clears it), so the reconnect below - // presents the truncation-time cursor, draws replay_truncated - // again, and the resync retries — bounded by the churn limiter + - // degraded ladder. The old transcript survives a failed fetch - // (the failure branch never reaches replayHistory's wipe), so the - // retry window shows stale-but-real content, not an empty pane. - if (token === this._historyLoadToken) this.connectSSE(wsId); - }); + let repairAttempt = null; + let repairAttemptId = null; + let repairDeadline = null; + let historyLoad; + if (repairingHistoryHandoff) { + const repairCtrl = + typeof AbortController === "function" ? new AbortController() : null; + const deadlineHandle = createHistoryHandoffDeadline(() => { + if (repairCtrl) repairCtrl.abort(); + }, HISTORY_HANDOFF_FETCH_TIMEOUT_MS); + repairDeadline = deadlineHandle; + repairAttempt = deadlineHandle.state; + repairAttemptId = this._historyRepair.startAttempt(manualAttempt, () => { + if (repairCtrl) repairCtrl.abort(); + // Cancelled mid-flight: dead-not-inert — expire so the late render + // discards, resolve so the race and its timer closure release now. + deadlineHandle.dispose({ expire: true, resolve: true }); + }); + // The logical deadline is load-bearing even when AbortController is + // unavailable and while authFetch is sleeping for Retry-After (that + // sleep is not abort-aware). The late request may still settle, but the + // attempt flag below makes its render inert. + historyLoad = Promise.race([ + this._refetchHistory( + wsId, + token, + true, + repairCtrl ? repairCtrl.signal : undefined, + repairAttempt, + ), + deadlineHandle.promise, + ]); + } else { + historyLoad = this._refetchHistory(wsId, token, true); + } + historyLoad + .catch((err) => { + // Normalize to a fail-closed settle below, but never silently: a + // render throw on the ordinary first-paint path used to surface as + // an unhandled rejection — keep the diagnostic loud. + console.error("history load/render failed", err); + return undefined; + }) + .then((outcome) => { + // Natural settlement: retire the timer and settle slot without + // resolving (the race already settled through whichever arm won). + if (repairDeadline) repairDeadline.dispose(); + this._historyRepair.endAttempt(repairAttemptId); + if (token !== this._historyLoadToken) return; + + if (repairingHistoryHandoff) { + this._historyRepair.settle({ + outcome, + hasToken: this._historyHandoffToken != null, + manualAttempt, + }); + return; + } + + // Ordinary first paint / truncation recovery retains its established + // reconnect behaviour. A truncated failure still re-presents the + // recorded numeric gap; only history_resync needs the stronger proof. + this.connectSSE(wsId); + }); } - async _refetchHistory(wsId, token, seedCursor = false) { + async _refetchHistory( + wsId, + token, + seedCursor = false, + signal, + repairAttempt, + ) { // Fetch conversation history over REST. Used for first paint (before // connecting SSE) and to re-render after a clear_ui signal (rewind / // retry / resume / open). The FETCH is wrapped (network/parse failure @@ -1800,31 +1991,24 @@ class Pane { // fetch, which is the safe direction. const epoch = this._connectEpoch; let data = null; - // Deliberately unbounded and unabortable, deferred not overlooked - // (#900, tracked as #905): coordinator.js carries an AbortController - // set plus a 15s bound so destroy() can cut a slow /history loose. - // Here the load token already makes a post-teardown settle - // render-inert, so the residual is a resource cost, not a correctness - // defect. The honest bound is wider than "one request": authFetch - // retries up to three attempts, sleeping Retry-After on 429 and doing - // a refresh round-trip on 401, so the detached pane's closure stays - // reachable for all of it — and that same unbounded await is what - // makes the slow transport-bounce cases (recover beat ~5s, degraded - // timer 15-120s) reachable by the epoch gate below. REOPEN when a - // /history blocks long enough for the pin to matter (large-session - // resume), or with the shared recovery core, which should own one - // implementation rather than a third hand-port. + // Ordinary history loads retain their established authFetch retry + // policy. Strong handoff repair passes both an AbortSignal and a logical + // attempt flag: the outer Promise.race settles after 15s even if this + // authFetch is inside its non-abort-aware Retry-After sleep, and the flag + // prevents that late request from ever rendering. try { const r = await authFetch( this._base + "/v1/api/workstreams/" + encodeURIComponent(id) + "/history", + signal ? { signal: signal } : undefined, ); if (r && r.ok) data = await r.json(); } catch (err) { data = null; } + if (repairAttempt && repairAttempt.expired) return; // Drop a superseded load: a newer _loadHistoryThenConnect (ws switch) // bumped the token while this fetch was in flight, so rendering now would // paint the wrong ws's history into the pane. @@ -1924,6 +2108,20 @@ class Pane { } finally { this._endReplayQuiesce(token); } + // Arm the handoff only after the full render succeeds. A render throw + // must not open a stream claiming an incomplete transcript matches this + // revision; _loadHistoryThenConnect's rejected promise remains loud. + if (seedCursor) { + this._historyHandoffToken = + typeof data.handoff_token === "string" && data.handoff_token + ? data.handoff_token + : null; + } + // The outcome tells the repair settle whether a TOKENLESS response + // was a completed render (the server's deliberate cold storage-only + // read — downgrade to the tokenless bootstrap) or a failure (fail + // closed and retry). + return "rendered"; } else { // Failed fetch = DOM + ref + repair-intent no-op (#890, the G3 // guard-before-wipe ported from coordinator.js refetchHistory) — @@ -1971,6 +2169,39 @@ class Pane { } } + _deferStaleHistoryBackstop() { + if (this._staleBackstopMicrotaskPending) return; + this._staleBackstopMicrotaskPending = true; + const staleToken = this._historyLoadToken; + const staleWs = this.wsId; + queueMicrotask(() => { + this._staleBackstopMicrotaskPending = false; + // Re-check every owner at the backlog tail. A later event in the same + // flush may have started a turn, re-armed a clear_ui queue, transferred + // repair ownership to replay_truncated, switched workstreams, or torn + // the pane down. None of those may be overtaken by this stale-idle heal. + if ( + !staleWs || + this.wsId !== staleWs || + staleToken !== this._historyLoadToken || + !this._historyStale || + this._replayQueue || + this.busy || + this.currentAssistantEl || + this.currentReasoningEl || + this._pendingTruncatedResync || + this._truncatedFromCursor != null || + this._resyncTimer != null || + !this.el || + !this.el.isConnected + ) { + return; + } + this._beginReplayQuiesce(staleToken); + this._refetchHistory(staleWs, staleToken); + }); + } + _clearAgentTracking() { // Release task-agent bookkeeping ahead of (or after) a full rebuild. // Entries left in _agentCards would pin every replaced card subtree as @@ -1988,6 +2219,7 @@ class Pane { // DOM refs into the subtree being replaced) — drop them together. if (this._toolRowIndex) this._toolRowIndex.clear(); if (this._streamElIndex) this._streamElIndex.clear(); + if (this._toolResultNodes) this._toolResultNodes.clear(); } _toolRow(callId) { @@ -1999,14 +2231,32 @@ class Pane { if (!callId) return null; let row = this._toolRowIndex.get(callId); if (row && row.isConnected && row.dataset.callId === callId) return row; - row = this.messagesEl.querySelector( + const rows = this.messagesEl.querySelectorAll( '.conv-row[data-call-id="' + CSS.escape(callId) + '"]', ); + row = rows.length ? rows[rows.length - 1] : null; if (row) this._toolRowIndex.set(callId, row); else this._toolRowIndex.delete(callId); return row; } + _indexToolRows(root) { + if (!root) return; + root.querySelectorAll(".conv-row[data-call-id]").forEach((row) => { + const callId = row.dataset.callId || ""; + if (!callId) return; + // Reused provider ids are valid across turns. The newest rendered batch + // owns future result events; release only the old tracking reference, + // never the old turn's still-visible DOM. + indexLatestToolRow( + this._toolRowIndex, + this._toolResultNodes, + callId, + row, + ); + }); + } + _streamEl(callId) { // Same cache discipline as _toolRow for the per-tool streaming 
 —
     // resolved on every tool_output_chunk, the chattiest event in an agent
@@ -2194,7 +2444,9 @@ class Pane {
             // Staleness-latch backstop (#890): a clear_ui refetch and
             // its one bounded retry both failed, so the transcript
             // still doesn't match the server and rewind/edit are
-            // latch-closed.  The turn just settled — refetch now.
+            // latch-closed. The turn just settled — defer the refetch to the
+            // current event-backlog tail so an earlier synthetic idle cannot
+            // divert later canonical replay events into the repair queue.
             //
             // TRANSPORT-FREE BY DESIGN (ruled, do not "upgrade" this
             // to _loadHistoryThenConnect): the heal must never touch
@@ -2221,9 +2473,7 @@ class Pane {
             // Fire-and-forget (no .catch) is deliberate: no composer state
             // rides this heal to un-strand (that is the clear_ui caller's
             // .catch), so a render throw stays loud, as in the load path.
-            const staleToken = this._historyLoadToken;
-            this._beginReplayQuiesce(staleToken);
-            this._refetchHistory(this.wsId, staleToken);
+            this._deferStaleHistoryBackstop();
           }
           // Only steal focus if this is the active pane and no approval pending.
           if (this._host.isFocused(this) && !this.pendingApproval) {
@@ -2304,13 +2554,30 @@ class Pane {
         break;
 
       case "tool_result":
-        this.appendToolOutput(
-          evt.call_id || "",
-          evt.name,
-          evt.output,
-          evt.is_error,
-          evt.preview,
-        );
+        if (acceptedToolEventAlreadyRendered(this._renderedToolEventIds, evt)) {
+          break;
+        }
+        // Record the accepted event as rendered ONLY when a paint actually
+        // happened: a false return (no target row — clear_ui wipe with the
+        // refetch in flight, a fresh mid-turn join) must leave the id
+        // unrecorded so the ring's later replay of the same event can paint
+        // it instead of being deduped into a permanently missing output.
+        if (
+          this.appendToolOutput(
+            evt.call_id || "",
+            evt.name,
+            evt.output,
+            evt.is_error,
+            evt.preview,
+            {
+              accepted: evt.accepted === true,
+              eventId: evt._event_id,
+              effectStatus: evt.effect_status,
+            },
+          )
+        ) {
+          recordAcceptedToolEvent(this._renderedToolEventIds, evt);
+        }
         break;
 
       case "status":
@@ -2328,6 +2595,10 @@ class Pane {
         this.addErrorMessage(evt.message);
         break;
 
+      case "user_turn":
+        this._acceptUserTurn(evt);
+        break;
+
       case "system_turn": {
         // First-class operator-context system turn (output-guard finding,
         // user interjection, metacognitive nudge — see
@@ -2345,6 +2616,14 @@ class Pane {
         if (sysEid && this._renderedSystemEventIds.has(sysEid)) {
           break;
         }
+        if (
+          evt.source === "user_interjection" &&
+          evt.meta &&
+          evt.meta.client_send_id &&
+          clientSendMaySettleForViewer(evt.meta.sender, viewerUserId())
+        ) {
+          this._markAcceptedClientSends([evt.meta.client_send_id], true);
+        }
         this.addSystemContext(
           evt.content || "",
           evt.source || "",
@@ -2363,7 +2642,13 @@ class Pane {
 
       case "message_queued":
         // Confirmation from server that a queued message was accepted.
-        // The UI already showed the message optimistically in addQueuedMessage.
+        // Mark the exact optimistic chip as admitted before the HTTP response:
+        // if that ACK is lost, the catch must not erase/report an unsent row.
+        if (
+          evt.client_send_id &&
+          clientSendMaySettleForViewer(evt.sender, viewerUserId())
+        )
+          this._markAcceptedClientSends([evt.client_send_id], false, true);
         break;
 
       case "message_dispatched":
@@ -2569,30 +2854,43 @@ class Pane {
             if (!this._pendingEditSend) return;
             const editText = this._pendingEditSend;
             this._pendingEditSend = null;
-            this.setBusy(true);
-            this.addUserMessage(editText);
-            // Known settle gap (deliberately deferred, pre-branch path):
-            // this POST consumes only the .catch — a queued/deferred/
-            // queue_full body is silently dropped, so a /compact window
-            // opened from another tab in exactly this instant leaves the
-            // resent message parked with no chip and busy stranded
-            // "server". Narrow (rewind just ran; the slot was ours) and
-            // original-strata; route through settleSendResponse when
-            // this flow is next touched.
-            authFetch(
-              this._base +
-                "/v1/api/workstreams/" +
-                encodeURIComponent(this.wsId) +
-                "/send",
-              {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ message: editText }),
-              },
-            ).catch((err) => {
-              this.addErrorMessage("Connection error: " + err.message);
-              this.setBusy(false);
+            const editClientSendId = mintClientSendId();
+            const editPriority = parsePriority(editText);
+            this.setBusy(true, "optimistic");
+            const editEl = this.addUserMessage(editText, null, {
+              clientSendId: editClientSendId,
             });
+            postAndSettleSend(
+              this.queue,
+              authFetch(
+                this._base +
+                  "/v1/api/workstreams/" +
+                  encodeURIComponent(this.wsId) +
+                  "/send",
+                {
+                  method: "POST",
+                  headers: { "Content-Type": "application/json" },
+                  body: JSON.stringify({
+                    message: editText,
+                    client_send_id: editClientSendId,
+                  }),
+                },
+              ),
+              {
+                queuedEl: null,
+                optimisticEl: editEl,
+                isBusy: false,
+                displayText: editPriority.displayText,
+                priority: editPriority.priority,
+                clientSendId: editClientSendId,
+                setBusy: (value) => this.setBusy(value),
+                busyIsOptimistic: () =>
+                  this.busy && this.busySource === "optimistic",
+                paneIsBusy: () => this.busy,
+                renderError: (message) => this.addErrorMessage(message),
+                consumeAttachments: () => {},
+              },
+            );
           })
           .catch((err) => {
             // The render runs outside _refetchHistory's try/catch by design;
@@ -2605,6 +2903,15 @@ class Pane {
         break;
       }
 
+      case "history_resync":
+        // The live conversation revision changed after /history rendered but
+        // before this listener registered. Numeric event replay is not an
+        // authoritative substitute for a committed conversation row. The
+        // explicit repair mode also survives a failed /history request: no
+        // cursorless/tokenless stream may reopen until a fresh proof renders.
+        this._historyRepair.begin(this.wsId);
+        break;
+
       case "replay_truncated":
         // The stream just admitted losing events past recovery — treat
         // the connection as DEAD and run the full fresh-connect flow
@@ -3364,6 +3671,8 @@ class Pane {
     // already painted from /history.  A later SSE replay that redelivers one
     // (resume-cursor overlap) is skipped by the system_turn handler.
     this._renderedSystemEventIds = new Set();
+    this._renderedUserEventIds = new Set();
+    this._renderedToolEventIds = new Set();
     if (!messages.length) {
       this.showEmptyState();
       return;
@@ -3375,12 +3684,12 @@ class Pane {
     // normally.  WCAG 4.1.3 — historical content should not behave like
     // real-time updates.
     this.messagesEl.setAttribute("aria-busy", "true");
-    // pendingAssessments[call_id] = output_assessment dict.  Populated
-    // from the assistant branch, consumed by the role==="tool" branch
-    // (or after the loop, for legacy rows missing tool_call_id).
-    // Replaces a JSON.stringify→dataset→JSON.parse round-trip with an
-    // in-memory map keyed by call_id.
-    const pendingAssessments = {};
+    // Occurrence queues, not one-value call-id maps: providers may reuse an id
+    // in a later turn, and malformed same-batch duplicates deliberately take
+    // the strong /history repair path. The repair renderer must therefore pair
+    // each sequential TOOL row with the matching sequential call occurrence.
+    const pendingToolRows = new Map();
+    const pendingAssessments = new Map();
     // Task-agent recall: call_id -> card .conv-agent wrap, so the tool-result
     // branch can flip the card's done/error state from the task's own result
     // (mirroring the live appendToolOutput), not from sub-step errors.
@@ -3398,7 +3707,13 @@ class Pane {
           lastToolBlock = null;
           continue;
         }
-        this.addUserMessage(msg.content || "", msg.attachments || null);
+        this.addUserMessage(msg.content || "", msg.attachments || null, {
+          eventId: msg.event_id,
+          sender: msg.sender || "",
+          source: msg.source || "",
+        });
+        if (msg.event_id != null)
+          this._renderedUserEventIds.add(String(msg.event_id));
         lastToolBlock = null;
       } else if (msg.role === "assistant") {
         // Reasoning bubble (Phase 1 reasoning persistence) — render
@@ -3478,6 +3793,15 @@ class Pane {
                 );
               }
               block.appendChild(row);
+              if (tc.id) {
+                enqueueToolOccurrence(pendingToolRows, tc.id, row);
+                indexLatestToolRow(
+                  this._toolRowIndex,
+                  this._toolResultNodes,
+                  tc.id,
+                  row,
+                );
+              }
               // Task-agent recall: rebuild the collapsible card under this row
               // from its stashed sub-trajectory (the /history `agent_steps`
               // overlay).  Absent ⇒ flat parent row (cold / not-retained).
@@ -3492,10 +3816,11 @@ class Pane {
                 tc.output_assessment.risk_level &&
                 tc.output_assessment.risk_level !== "none"
               ) {
-                pendingAssessments[tc.id || ""] = {
+                const assessmentKey = tc.id || "";
+                enqueueToolOccurrence(pendingAssessments, assessmentKey, {
                   assessment: tc.output_assessment,
                   toolDiv: row,
-                };
+                });
               }
             });
             block.appendChild(
@@ -3523,9 +3848,17 @@ class Pane {
           // (legacy rows pre-dating the wire-format addition).
           let resultTarget = null;
           if (msg.tool_call_id) {
-            resultTarget = lastToolBlock.querySelector(
-              '.conv-row[data-call-id="' + CSS.escape(msg.tool_call_id) + '"]',
+            resultTarget = shiftToolOccurrence(
+              pendingToolRows,
+              msg.tool_call_id,
             );
+            if (!resultTarget) {
+              resultTarget = lastToolBlock.querySelector(
+                '.conv-row[data-call-id="' +
+                  CSS.escape(msg.tool_call_id) +
+                  '"]',
+              );
+            }
           }
           // Cursor-style append: cursor advances after each insert so
           // the next sibling lands AFTER the previous one.  Fixes the
@@ -3535,7 +3868,8 @@ class Pane {
           // Resulting order with all present:
           //   [tool div][output][output-warning]
           let insertCursor = resultTarget;
-          const insertChained = (node) => {
+          const resultNodes = [];
+          const insertChained = (node, resultOwned = true) => {
             if (insertCursor) {
               insertCursor.after(node);
               insertCursor = node;
@@ -3544,6 +3878,7 @@ class Pane {
               if (bdg) lastToolBlock.insertBefore(node, bdg);
               else lastToolBlock.appendChild(node);
             }
+            if (resultOwned) resultNodes.push(node);
           };
           if (stripped && !isDenied) {
             const media = !isToolError ? tryParseMedia(stripped) : null;
@@ -3563,8 +3898,16 @@ class Pane {
               buildPreviewChip(msg.preview, (d) => this._host.onPreview(d)),
             );
           }
+          // A row naming a call_id this batch does not own is an ORPHAN
+          // (a result for an earlier batch, or a second writer's append)
+          // — it must not stamp THIS batch as failed.  The shared outcome
+          // index skips unmatched occurrences for the same reason.  A row
+          // with no call_id at all is the legacy positional case and does
+          // belong to the preceding batch, so it still stamps.
+          const isOrphanResult = !!msg.tool_call_id && !resultTarget;
           if (
             isToolError &&
+            !isOrphanResult &&
             !lastToolBlock.classList.contains("conv-batch--denied")
           ) {
             lastToolBlock.classList.add("conv-batch--error");
@@ -3575,21 +3918,41 @@ class Pane {
           // assistant branch).  Skip when the tool result was denied —
           // the ✗ denied badge already signals the deny path.
           if (!isDenied && msg.tool_call_id) {
-            const pending = pendingAssessments[msg.tool_call_id];
-            if (pending) {
-              insertChained(_buildOutputWarningEl(pending.assessment));
-              delete pendingAssessments[msg.tool_call_id];
+            const assessment = shiftToolOccurrence(
+              pendingAssessments,
+              msg.tool_call_id,
+            );
+            if (assessment) {
+              insertChained(
+                _buildOutputWarningEl(assessment.assessment),
+                false,
+              );
             }
           }
+          if (msg.tool_call_id && resultTarget) {
+            if (msg.effect_status) {
+              resultTarget.dataset.effectStatus = String(msg.effect_status);
+            } else {
+              delete resultTarget.dataset.effectStatus;
+            }
+            this._toolResultNodes.set(msg.tool_call_id, {
+              row: resultTarget,
+              nodes: resultNodes,
+            });
+          }
           // Task-agent recall: flip its card done/error from the task's OWN
           // result (matching the live appendToolOutput) — NOT from sub-step
           // errors, since a sub-tool can fail and the agent still synthesize.
-          if (msg.tool_call_id && agentCardWraps[msg.tool_call_id]) {
-            agentCardWraps[msg.tool_call_id].dataset.state = msg.is_error
-              ? "error"
-              : "done";
+          const resultAgentWrap =
+            (resultTarget && resultTarget.querySelector(".conv-agent")) ||
+            (msg.tool_call_id && agentCardWraps[msg.tool_call_id]);
+          if (resultAgentWrap) {
+            resultAgentWrap.dataset.state = msg.is_error ? "error" : "done";
           }
         }
+        if (msg.event_id != null) {
+          this._renderedToolEventIds.add(String(msg.event_id));
+        }
       } else if (msg.role === "system") {
         // First-class operator-context turn (output-guard finding, user
         // interjection, metacognitive nudge — see make_system_turn).  These
@@ -3605,7 +3968,13 @@ class Pane {
         if (msg.event_id != null) {
           this._renderedSystemEventIds.add(String(msg.event_id));
         }
-        lastToolBlock = null;
+        // Deliberately does NOT null lastToolBlock: a system row is not a
+        // turn boundary.  The batch's result window ends at the next
+        // assistant/user row (both reset the anchor in their own
+        // branches) — same rule as indexHistoryToolOutcomes.  Nulling
+        // here made every tool result AFTER an interleaved system row
+        // (mid-turn operator context, a second writer's append) silently
+        // vanish from this pane while the coordinator rendered it.
       }
     }
     // Flush any output_assessments left in the map — these correspond
@@ -3613,14 +3982,13 @@ class Pane {
     // tool_call_id (legacy / migrated rows pre-dating the wire-format
     // addition).  Render the warning under the tool div itself rather
     // than dropping the safety information silently.
-    const leftoverIds = Object.keys(pendingAssessments);
-    for (let p = 0; p < leftoverIds.length; p++) {
-      const leftover = pendingAssessments[leftoverIds[p]];
-      if (!leftover) continue;
-      leftover.toolDiv.insertAdjacentElement(
-        "afterend",
-        _buildOutputWarningEl(leftover.assessment),
-      );
+    for (const leftovers of pendingAssessments.values()) {
+      for (const leftover of leftovers) {
+        leftover.toolDiv.insertAdjacentElement(
+          "afterend",
+          _buildOutputWarningEl(leftover.assessment),
+        );
+      }
     }
     this._attachRetryToLastAssistant();
     this.scrollToBottom();
@@ -3755,6 +4123,7 @@ class Pane {
     });
     this.announcedBlocks.set(key, block);
     this.messagesEl.appendChild(block);
+    this._indexToolRows(block);
     this._relinkAgentCards(list);
     this.scrollToBottom(stick);
     toolAnnounce(_toolAnnounceText(list));
@@ -3881,6 +4250,7 @@ class Pane {
     // (not yet in the DOM) would be mistaken for an orphan and immediately
     // pruned if we registered before appending.
     if (!announced) this.messagesEl.appendChild(block);
+    this._indexToolRows(block);
     if (!autoApproved) {
       this._registerApprovalCycle(cycleId, [block], items);
       const fb = block.querySelector(".conv-feedback");
@@ -4173,7 +4543,8 @@ class Pane {
     return card.wrap;
   }
 
-  appendToolOutput(callId, name, output, isError, preview) {
+  appendToolOutput(callId, name, output, isError, preview, opts = {}) {
+    const accepted = opts.accepted === true;
     // Capture pin before the streamEl removal + result insertion change
     // scrollHeight — see announceToolBlock.  The result block is the other
     // tall one-shot append in the tool flow (up to 10 lines before collapse).
@@ -4196,9 +4567,9 @@ class Pane {
       // nested yet must NOT graft its output onto the last top-level batch row
       // — that mislabels a sub-tool's result as a main-harness tool's.  Its row
       // arrives via the orphan flush; skip until then.
-      if (callId && callId.includes("::")) return;
+      if (callId && callId.includes("::")) return false;
       const blocks = this.messagesEl.querySelectorAll(".conv-batch");
-      if (!blocks.length) return;
+      if (!blocks.length) return false;
       const block = blocks[blocks.length - 1];
       const tools = block.querySelectorAll(".conv-row");
       for (let i = tools.length - 1; i >= 0; i--) {
@@ -4209,7 +4580,29 @@ class Pane {
       }
       if (!target && tools.length) target = tools[tools.length - 1];
     }
-    if (!target) return;
+    if (!target) return false;
+
+    if (accepted) {
+      const targetBatch = target.closest(".conv-batch");
+      // Stop can synthesize a final result after tool_pending but before the
+      // authoritative tool_info/approval event consumes the early shell. Once
+      // this exact shell owns an accepted row it is committed transcript DOM,
+      // not replaceable early paint. Retire map ownership without removing it;
+      // a later turn may legitimately reuse the same provider call id.
+      if (targetBatch && this.announcedBlocks) {
+        for (const [key, announced] of this.announcedBlocks.entries()) {
+          if (announced === targetBatch) {
+            this.announcedBlocks.delete(key);
+            break;
+          }
+        }
+      }
+      if (opts.effectStatus) {
+        target.dataset.effectStatus = String(opts.effectStatus);
+      } else {
+        delete target.dataset.effectStatus;
+      }
+    }
 
     // Remove the streaming output element for this tool
     let streamEl = this._streamEl(callId);
@@ -4224,9 +4617,21 @@ class Pane {
       if (callId) this._streamElIndex.delete(callId);
     }
 
-    const stripped = stripAnsi(output || "").trim();
-    if (!stripped) return;
+    // A preliminary receipt and its final accepted row own the same result
+    // slot. Remove only nodes we created for THIS row; warnings and approval
+    // badges are independent siblings and remain intact. A provider may reuse
+    // call ids in a later turn, so never remove nodes tracked against an older
+    // still-visible row.
+    const priorResult = callId ? this._toolResultNodes.get(callId) : null;
+    if (priorResult && priorResult.row === target) {
+      priorResult.nodes.forEach((node) => {
+        if (node && node.isConnected) node.remove();
+      });
+    } else if (callId) {
+      this._toolResultNodes.delete(callId);
+    }
 
+    const stripped = stripAnsi(output || "").trim();
     // Skip rendering for denied/blocked tool results — the ✗ denied
     // badge from resolveApproval already shows the denial reason; the
     // SSE tool_result event would otherwise duplicate the text.  Mirror
@@ -4238,50 +4643,44 @@ class Pane {
       (parentBlock && parentBlock.classList.contains("conv-batch--denied")) ||
       /^Denied by user/.test(stripped) ||
       /^Blocked/.test(stripped);
-    if (isDenied) return;
+    const resultNodes = [];
+    let insertCursor = target;
+    const insertResult = (node) => {
+      insertCursor.after(node);
+      insertCursor = node;
+      resultNodes.push(node);
+    };
 
-    // Detect structured media output and render interactive embed
-    if (!isError) {
-      const media = tryParseMedia(stripped);
-      if (media) {
-        const embed = buildMediaEmbed(media, stripped);
-        target.after(embed);
-        this.scrollToBottom(stick);
-        return;
+    if (!isDenied && stripped) {
+      let resultNode = null;
+      // Detect structured media output and render interactive embed.
+      if (!isError) {
+        const media = tryParseMedia(stripped);
+        if (media) resultNode = buildMediaEmbed(media, stripped);
       }
-    }
 
-    // Detect structured MCP error envelope and render an interactive
-    // consent / re-consent / forbidden / operator card.  The existing
-    // ✗ error badge from appendToolErrorBadge still fires below.
-    if (isError) {
-      const mcpErr = tryParseMcpError(stripped);
-      if (mcpErr) {
-        if (
-          parentBlock &&
-          !parentBlock.classList.contains("conv-batch--denied")
-        ) {
-          parentBlock.classList.add("conv-batch--error");
-          appendToolErrorBadge(parentBlock);
-        }
-        target.after(
-          buildMcpErrorEmbed(mcpErr, stripped, (s) =>
+      // Detect structured MCP error envelopes before the plain renderer. The
+      // shared consent card remains the canonical error presentation.
+      if (!resultNode && isError) {
+        const mcpErr = tryParseMcpError(stripped);
+        if (mcpErr) {
+          resultNode = buildMcpErrorEmbed(mcpErr, stripped, (s) =>
             this._host.onConsentDetected(s),
-          ),
-        );
-        this.scrollToBottom(stick);
-        return;
+          );
+        }
       }
+
+      if (!resultNode) {
+        resultNode = renderCollapsibleOutput(stripped, isError);
+      }
+      insertResult(resultNode);
     }
 
-    // The media / MCP-error dispatch above both early-return, so by here it's
-    // the plain-output path — the shared helper applies (test_app_js pins that
-    // tryParseMcpError precedes this renderer call).
-    const out = renderCollapsibleOutput(stripped, isError);
-
-    // Mark the parent approval block as errored
+    // Mark the parent approval block as errored. Idempotent badge construction
+    // preserves the approval verdict beside the error disposition.
     if (
       isError &&
+      !isDenied &&
       parentBlock &&
       !parentBlock.classList.contains("conv-batch--denied")
     ) {
@@ -4289,17 +4688,22 @@ class Pane {
       appendToolErrorBadge(parentBlock);
     }
 
-    target.after(out);
-    // Preview descriptor (open_preview): chip in the transcript always; the
-    // pane auto-opens only while THIS pane is the user's focus — a
-    // backgrounded session must not commandeer the split, and the chip
-    // remains the deliberate reopen for that case (and for replay).
-    if (preview && !isError) {
+    // Accepted preview rows mirror /history: the reopen chip survives even on
+    // a cancelled/error result, but accepted publication never auto-opens a
+    // pane. The provisional receipt remains the sole focused auto-open edge.
+    if (preview && !isDenied && (!isError || accepted)) {
       const chip = buildPreviewChip(preview, (d) => this._host.onPreview(d));
-      out.after(chip);
-      if (this._host.isFocused(this)) this._host.onPreview(preview);
+      insertResult(chip);
+      if (!accepted && !isError && this._host.isFocused(this)) {
+        this._host.onPreview(preview);
+      }
     }
-    this.scrollToBottom(stick);
+
+    if (callId) {
+      this._toolResultNodes.set(callId, { row: target, nodes: resultNodes });
+    }
+    if (resultNodes.length) this.scrollToBottom(stick);
+    return true;
   }
 
   sendMessage() {
@@ -4397,6 +4801,7 @@ class Pane {
     const isBusy = this.busy;
     let queuedEl = null;
     let optimisticEl = null;
+    const clientSendId = mintClientSendId();
     const snap = this.attachments.snapshot();
 
     // Display-only strip of the !!! prefix (the server re-parses it
@@ -4406,13 +4811,19 @@ class Pane {
 
     if (isBusy) {
       this.removeEmptyState();
-      queuedEl = this.queue.addQueuedMessage(displayText, priority);
+      queuedEl = this.queue.addQueuedMessage(
+        displayText,
+        priority,
+        clientSendId,
+      );
     } else {
       // "optimistic": no server state event asserted this — the settle
       // arms may undo it if the send turns out deferred/refused (see
       // setBusy's busySource contract).
       this.setBusy(true, "optimistic");
-      optimisticEl = this.addUserMessage(text, snap.attachments);
+      optimisticEl = this.addUserMessage(text, snap.attachments, {
+        clientSendId,
+      });
     }
     this.composer.clear();
 
@@ -4428,6 +4839,7 @@ class Pane {
       body: JSON.stringify({
         message: text,
         attachment_ids: snap.attachment_ids,
+        client_send_id: clientSendId,
       }),
     };
     let sendTimer = null;
@@ -4448,67 +4860,24 @@ class Pane {
       sendInit,
     );
     if (sendTimer) sendReq = sendReq.finally(() => clearTimeout(sendTimer));
-    sendReq
-      .then((r) => {
-        // A rejected send (4xx/5xx) carries {error}, not {status}; without
-        // this guard it falls through to the "unknown status" branch and gets
-        // promote()'d — a server-refused message shown as delivered (with a
-        // false "already sent" toast if it was dismissed). Route it to the
-        // .catch (removes the bubble + shows the error) instead, surfacing the
-        // server's {error} text ("No session", a rate-limit reason, etc.)
-        // rather than a bare status code. A wedged proxy can answer non-JSON
-        // (502/504 HTML); the parse-failure arm falls back to the status code
-        // so that can't surface as an "Unexpected token <" error.
-        if (!r.ok) {
-          // 409 = the server-side cross-user interjection block (another
-          // participant's turn is in flight). Convert to a handled status
-          // object so it routes to the clean branch below instead of the
-          // generic "Connection error" catch — this is the reactive fallback
-          // for the race where the button wasn't yet disabled.
-          if (r.status === 409) {
-            return r.json().then(
-              (b) => ({
-                status: "cross_user_interjection",
-                error: (b && b.error) || "",
-              }),
-              () => ({ status: "cross_user_interjection", error: "" }),
-            );
-          }
-          return r.json().then(
-            (b) => {
-              throw new Error((b && b.error) || "send_http_" + r.status);
-            },
-            () => {
-              throw new Error("send_http_" + r.status);
-            },
-          );
-        }
-        return r.json();
-      })
-      .then((data) => {
-        // The full status dispatch (queued/retro-convert, busy,
-        // queue_full, attachments_busy, cross_user, unknown-ok) lives in
-        // the shared helper — ONE settle matrix for both panes; see
-        // settleSendResponse's contract for the arm semantics.
-        settleSendResponse(this.queue, data, {
-          queuedEl,
-          optimisticEl,
-          isBusy,
-          displayText,
-          priority,
-          setBusy: (b) => this.setBusy(b),
-          busyIsOptimistic: () => this.busy && this.busySource === "optimistic",
-          paneIsBusy: () => this.busy,
-          renderError: (msg) => this.addErrorMessage(msg),
-          consumeAttachments: (attached, droppedIds) =>
-            this.attachments.consume(attached, droppedIds),
-        });
-      })
-      .catch((err) => {
-        if (queuedEl) this.queue.remove(queuedEl);
-        this.addErrorMessage("Connection error: " + err.message);
-        if (!isBusy) this.setBusy(false);
-      });
+    // Response normalization, the full status dispatch (queued/retro-convert,
+    // busy, queue_full, attachments_busy, cross_user, unknown-ok) and the
+    // accepted-guarded transport catch all live in the shared helper — ONE
+    // send settle for both panes and both of each pane's send flows.
+    postAndSettleSend(this.queue, sendReq, {
+      queuedEl,
+      optimisticEl,
+      isBusy,
+      displayText,
+      priority,
+      clientSendId,
+      setBusy: (b) => this.setBusy(b),
+      busyIsOptimistic: () => this.busy && this.busySource === "optimistic",
+      paneIsBusy: () => this.busy,
+      renderError: (msg) => this.addErrorMessage(msg),
+      consumeAttachments: (attached, droppedIds) =>
+        this.attachments.consume(attached, droppedIds),
+    });
   }
 
   cancelGeneration() {
@@ -4962,11 +5331,6 @@ function synthToolItem(tc) {
   return { func_name: tc.name, call_id: tc.id || "", header };
 }
 
-function renderVerdictBadge(verdict, judgePending) {
-  // Thin wrapper over the shared builder (returns a fragment [badge, detail]).
-  return buildConvVerdict(verdict, { judgePending });
-}
-
 // Append an "✗ error" pill to an approval block as a sibling of the
 // existing approved/denied/auto-approved pill, so the approval verdict
 // stays visible alongside the execution outcome. Idempotent — re-fires
@@ -5123,6 +5487,7 @@ function createInteractivePane(root, wsId, opts) {
     // Invalidate any in-flight history load: its .finally would otherwise
     // reopen a stream for a session we just declared dead.
     pane._historyLoadToken = (pane._historyLoadToken || 0) + 1;
+    pane._historyRepair.clear();
     // Same dead-not-inert rule destroy() applies (#900): the bump above
     // already fails the retry's fire guard, but a pane whose session is
     // gone must not hold a live timer — /history would 404 anyway, and a
@@ -5297,6 +5662,7 @@ function createInteractivePane(root, wsId, opts) {
       // below (its arm guard is latch + token, and destroy touched
       // neither).  Every future terminal path must bump this too.
       pane._historyLoadToken = (pane._historyLoadToken || 0) + 1;
+      pane._historyRepair.clear();
       // The clear_ui failure retry survives everything EXCEPT a token
       // bump, so the bump above already renders a firing inert — but a
       // timer into a destroyed pane must be DEAD, not merely inert (it
diff --git a/turnstone/shared_static/shell.js b/turnstone/shared_static/shell.js
index 7698ed32..5671c109 100644
--- a/turnstone/shared_static/shell.js
+++ b/turnstone/shared_static/shell.js
@@ -755,7 +755,13 @@ async function mountShell() {
                     // 404 = nothing left to stop (closed under us / node lost
                     // it) — the user's intent is satisfied; drop the tab.
                     if (r.ok || r.status === 404) pm.close(pane.id);
-                    else failToast();
+                    else if (r.status === 409) {
+                      if (typeof window.showToast === "function")
+                        window.showToast(
+                          "Conversation history is still being saved. Try ending the session again shortly.",
+                          "warning",
+                        );
+                    } else failToast();
                   })
                   .catch(failToast);
               };
diff --git a/turnstone/shared_static/tool_projection.js b/turnstone/shared_static/tool_projection.js
new file mode 100644
index 00000000..8a2396e2
--- /dev/null
+++ b/turnstone/shared_static/tool_projection.js
@@ -0,0 +1,103 @@
+// Small state primitives shared by the two browser TOOL reducers.
+//
+// Provider call ids are correlation hints, not globally unique row ids. A
+// provider may reuse one in a later turn, and malformed same-batch duplicates
+// still have to converge after the server requests a strong history repair.
+
+export function enqueueToolOccurrence(queues, callId, value) {
+  const key = String(callId || "");
+  const pending = queues.get(key) || [];
+  pending.push(value);
+  queues.set(key, pending);
+  return pending.length;
+}
+
+export function shiftToolOccurrence(queues, callId) {
+  const key = String(callId || "");
+  const pending = queues.get(key);
+  if (!pending || !pending.length) return undefined;
+  const value = pending.shift();
+  if (!pending.length) queues.delete(key);
+  return value;
+}
+
+export function indexHistoryToolOutcomes(messages) {
+  const historyMessages = Array.isArray(messages) ? messages : [];
+  const batches = new Map();
+  historyMessages.forEach((message, assistantIndex) => {
+    if (
+      (message.role || "") !== "assistant" ||
+      !Array.isArray(message.tool_calls)
+    )
+      return;
+    const outcomes = new Array(message.tool_calls.length);
+    const unmatched = new Map();
+    message.tool_calls.forEach((toolCall, callIndex) => {
+      enqueueToolOccurrence(
+        unmatched,
+        String((toolCall && toolCall.id) || ""),
+        callIndex,
+      );
+    });
+    for (let i = assistantIndex + 1; i < historyMessages.length; i++) {
+      const result = historyMessages[i];
+      const role = result.role || "tool";
+      // The batch's result window ends at the next conversational turn
+      // (assistant or user).  Other interleaved rows — a mid-turn system
+      // message, a second writer's append (cross-node re-home overlap,
+      // legacy NULL-key import order) — are skipped, not terminators:
+      // treating them as terminators left every later result unmatched
+      // and painted a fully-resolved batch as a permanent orphan shell.
+      if (role === "assistant" || role === "user") break;
+      if (role !== "tool") continue;
+      const callIndex = shiftToolOccurrence(
+        unmatched,
+        String(result.tool_call_id || ""),
+      );
+      if (callIndex === undefined) continue;
+      outcomes[callIndex] = result.denied
+        ? "denied"
+        : result.is_error
+          ? "error"
+          : "ok";
+    }
+    batches.set(message, outcomes);
+  });
+  return batches;
+}
+
+export function indexLatestToolRow(rows, resultOwners, callId, row) {
+  const key = String(callId || "");
+  if (!key || !row) return null;
+  const prior = rows.get(key) || null;
+  // Release the tracked result owner only when a prior row is superseded. A
+  // first sighting has nothing to supersede, and the entry it would drop is
+  // the orphan bubble this row's own result has yet to absorb.
+  if (prior && prior !== row) resultOwners.delete(key);
+  rows.set(key, row);
+  return prior;
+}
+
+export function acceptedToolEventAlreadyRendered(renderedIds, event) {
+  return !!(
+    event &&
+    event.accepted === true &&
+    event._event_id != null &&
+    renderedIds.has(String(event._event_id))
+  );
+}
+
+export function recordAcceptedToolEvent(renderedIds, event) {
+  if (event && event.accepted === true && event._event_id != null) {
+    renderedIds.add(String(event._event_id));
+  }
+}
+
+export function shouldRefreshTasksForToolResult(event, hadResult) {
+  return !!(
+    event &&
+    event.name === "tasks" &&
+    !event.is_error &&
+    (event.accepted !== true || !hadResult)
+  );
+}
diff --git a/turnstone/ui/static/app.js b/turnstone/ui/static/app.js
index 817eeb2d..2a4af6bb 100644
--- a/turnstone/ui/static/app.js
+++ b/turnstone/ui/static/app.js
@@ -38,6 +38,57 @@ const STATE_DISPLAY = {
   error: { symbol: "\u2716", label: "err" },
 };
 
+const PERSISTENCE_DISPLAY = {
+  pending: {
+    label: "History save pending",
+    tooltip:
+      "An accepted conversation turn has not reached durable history yet.",
+  },
+  retrying: {
+    label: "History save retrying",
+    tooltip:
+      "An accepted conversation turn has not reached durable history yet. Automatic recovery is in progress.",
+  },
+  conflict: {
+    label: "History save blocked",
+    tooltip:
+      "An accepted conversation turn cannot reach durable history automatically. Operator intervention is required.",
+  },
+};
+
+function appendPersistenceStatus(container, ws) {
+  const display = PERSISTENCE_DISPLAY[ws.persistence_state];
+  if (!display) return;
+  const badge = document.createElement("span");
+  badge.className = "dash-persistence-badge";
+  badge.dataset.state = ws.persistence_state;
+  badge.textContent = display.label;
+  badge.title = display.tooltip;
+  badge.setAttribute("role", "status");
+  badge.setAttribute("aria-label", display.label + ". " + display.tooltip);
+  container.appendChild(badge);
+}
+
+function renderDashboardSubline(container, ws) {
+  container.replaceChildren();
+  container.classList.toggle("sub-attention", ws.activity_state === "approval");
+  appendPersistenceStatus(container, ws);
+  if (ws.activity) {
+    if (container.childNodes.length) container.append(" \u00b7 ");
+    container.append(ws.activity);
+  }
+}
+
+function setPersistenceRowAria(row, persistenceState) {
+  const base =
+    row.dataset.baseAriaLabel || row.getAttribute("aria-label") || "";
+  const display = PERSISTENCE_DISPLAY[persistenceState];
+  row.setAttribute(
+    "aria-label",
+    base + (display ? ", " + display.label.toLowerCase() : ""),
+  );
+}
+
 // ===========================================================================
 //  5. Health polling
 // ===========================================================================
@@ -874,17 +925,40 @@ function closeWorkstream(wsId) {
     body: "{}",
   })
     .then(function (r) {
-      return r.json();
+      // Non-JSON fallback (proxy 502/504 HTML, empty body): keep the HTTP
+      // status so the 409 arm still fires — the sibling postAndSettleSend
+      // handles exactly this proxy case the same way.
+      return r
+        .json()
+        .catch(function () {
+          return {};
+        })
+        .then(function (data) {
+          return { data: data, status: r.status };
+        });
     })
-    .then(function (data) {
+    .then(function (result) {
+      const data = result.data;
       if (data.status === "ok") {
         delete workstreams[wsId];
         closeSessionPane(wsId);
         fireRender();
         if (!Object.keys(workstreams).length) showDashboard();
+      } else if (result.status === 409) {
+        showToast(
+          "Conversation history is still being saved. Try ending the session again shortly.",
+          "warning",
+        );
       } else if (data.error) {
         showToast(data.error, "warning");
+      } else {
+        showToast("Couldn't end the session. Try again shortly.", "warning");
       }
+    })
+    .catch(function () {
+      // Transport failure: the close may not have reached the server at
+      // all — say so instead of leaving the pane open with no feedback.
+      showToast("Couldn't end the session. Try again shortly.", "warning");
     });
 }
 
@@ -983,6 +1057,15 @@ function renderDashboardTable(wsList, agg) {
     return;
   }
   wsList.forEach(function (ws) {
+    // The REST dashboard row is the authoritative initial projection for
+    // operator-only journal state. Keep the Tier-1 roster copy aligned so a
+    // later activity-only delta cannot repaint the sub-line from stale data.
+    if (workstreams[ws.ws_id]) {
+      workstreams[ws.ws_id].persistence_state =
+        ws.persistence_state || "healthy";
+      workstreams[ws.ws_id].activity = ws.activity || "";
+      workstreams[ws.ws_id].activity_state = ws.activity_state || "";
+    }
     const liveState =
       (workstreams[ws.ws_id] && workstreams[ws.ws_id].state) ||
       ws.state ||
@@ -1006,7 +1089,8 @@ function renderDashboardTable(wsList, agg) {
     if (ws.tokens) ariaLabel += ", " + formatTokens(ws.tokens) + " tokens";
     if (ws.context_ratio > 0)
       ariaLabel += ", " + Math.round(ws.context_ratio * 100) + "% context";
-    row.setAttribute("aria-label", ariaLabel);
+    row.dataset.baseAriaLabel = ariaLabel;
+    setPersistenceRowAria(row, ws.persistence_state);
 
     const main = document.createElement("div");
     main.className = "dash-row-main";
@@ -1080,8 +1164,7 @@ function renderDashboardTable(wsList, agg) {
 
     const sub = document.createElement("div");
     sub.className = "dash-row-sub";
-    if (ws.activity_state === "approval") sub.classList.add("sub-attention");
-    sub.textContent = ws.activity || "";
+    renderDashboardSubline(sub, ws);
     row.appendChild(sub);
 
     row.onclick = function () {
@@ -1848,19 +1931,23 @@ function connectGlobalSSE() {
         context_ratio: data.context_ratio,
         activity: data.activity,
         activity_state: data.activity_state,
+        persistence_state: data.persistence_state,
       });
     } else if (data.type === "ws_activity") {
-      const row = document.querySelector(
-        '#dash-ws-table .dash-row[data-ws-id="' + data.ws_id + '"]',
-      );
-      if (row) {
-        const sub = row.querySelector(".dash-row-sub");
-        if (sub) {
-          sub.textContent = data.activity || "";
-          if (data.activity_state === "approval")
-            sub.classList.add("sub-attention");
-          else sub.classList.remove("sub-attention");
-        }
+      // Membership-gated like ws_rename below: a trailing activity event
+      // for a closed workstream (its dashboard row can outlive ws_closed
+      // until the next REST-driven repaint) must not re-insert a skeletal
+      // roster entry — that ghost suppresses the empty-state transition
+      // and paints a nameless rail tab until a full resync.
+      const roster = workstreams[data.ws_id];
+      if (roster) {
+        roster.activity = data.activity || "";
+        roster.activity_state = data.activity_state || "";
+        const row = document.querySelector(
+          '#dash-ws-table .dash-row[data-ws-id="' + data.ws_id + '"]',
+        );
+        const sub = row && row.querySelector(".dash-row-sub");
+        if (sub) renderDashboardSubline(sub, roster);
       }
     } else if (data.type === "ws_rename") {
       if (workstreams[data.ws_id]) workstreams[data.ws_id].name = data.name;
@@ -1875,6 +1962,7 @@ function connectGlobalSSE() {
       // without waiting for a roster refetch (null = unattached).
       workstreams[data.ws_id].project_id = data.project_id || null;
       workstreams[data.ws_id].persona = data.persona || "";
+      workstreams[data.ws_id].persistence_state = "healthy";
       renderTabBar();
     } else if (data.type === "ws_closed") {
       const wsId = data.ws_id;
@@ -2406,6 +2494,8 @@ function applyRosterSnapshot(list, opts) {
     // workstreams, and that authoritative empty must not be masked by a
     // stale in-memory value.
     cur.persona = "persona" in ws ? ws.persona || "" : cur.persona || "";
+    cur.persistence_state =
+      "persistence_state" in ws ? ws.persistence_state || "healthy" : "healthy";
     workstreams[ws.id] = cur;
   });
   if (evict) {
@@ -2680,7 +2770,17 @@ function renderTabBar() {
   fireRender();
 }
 function updateTabIndicator(wsId, state, extra) {
-  if (workstreams[wsId]) workstreams[wsId].state = state;
+  const roster = workstreams[wsId];
+  if (roster) {
+    roster.state = state;
+    if (extra) {
+      if (extra.activity !== undefined) roster.activity = extra.activity || "";
+      if (extra.activity_state !== undefined)
+        roster.activity_state = extra.activity_state || "";
+      if (extra.persistence_state !== undefined)
+        roster.persistence_state = extra.persistence_state || "healthy";
+    }
+  }
   fireRender(); // rail glyph (the only surface fireRender repaints)
   // Patch the Dashboard row in place — fireRender fans out to the rail, NOT the
   // #dash-ws-table cells, so without this a watched row's STATE/TOKENS/CTX go
@@ -2715,15 +2815,12 @@ function updateTabIndicator(wsId, state, extra) {
           : "";
     }
   }
-  if (extra.activity !== undefined) {
+  if (extra.activity !== undefined || extra.persistence_state !== undefined) {
     const sub = row.querySelector(".dash-row-sub");
-    if (sub) {
-      sub.textContent = extra.activity || "";
-      if (extra.activity_state === "approval")
-        sub.classList.add("sub-attention");
-      else sub.classList.remove("sub-attention");
-    }
+    if (sub) renderDashboardSubline(sub, roster || extra);
   }
+  if (extra.persistence_state !== undefined)
+    setPersistenceRowAria(row, (roster || extra).persistence_state);
 }
 
 // Synthesize the one-node clusterState shape the rail consumes
diff --git a/turnstone/ui/static/style.css b/turnstone/ui/static/style.css
index 891ed0e9..d657bda7 100644
--- a/turnstone/ui/static/style.css
+++ b/turnstone/ui/static/style.css
@@ -1426,7 +1426,7 @@ audio.media-player {
    here (not chat.css) because these classes only render on the
    per-node UI surface; coord uses the .approval-dock pattern instead.
 
-   DOM shape rendered by buildToolDiv() + renderVerdictBadge():
+   DOM shape rendered by buildToolDiv() + buildConvVerdict():
      .msg.ts-approval.ts-approval--inline[.approved]  ← outer card
        .ts-approval-tool
          .tool-name                   ← tool function name