From 1dbf7f410c30fe862c4bad420956b7b3700333f5 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Thu, 16 Jul 2026 19:39:25 -0700 Subject: [PATCH] feat(compaction): lifecycle events, web progress card, history re-render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compaction becomes visible: a first-class 'compaction' SSE lifecycle (start/progress/end, compaction_id-correlated, superseded-flagged ends) replaces the loose info lines; both web panes render a progress-bar card that settles into a persistent result card, re-rendered after reload via the /history projection of the compaction marker row. Slash commands echo as command chips instead of fake user turns. The enabling rework: /command dispatches onto the workstream worker slot (the old inline path blocked the node's event loop for whole compactions and let /clear interleave with live turns). Busy refusals answer 409; quick commands are awaited loop-natively with a 60s backstop; /compact is fire-and-forget. Sends during a command window park in the /send route and dispatch full-fidelity afterwards — the interjection queue (length cap, cross-user guard, identity-swap hazards) is unreachable there — with a compaction-aware client abort bound shared by both panes. compact_now() carries send()'s full generation discipline; Stop aborts the in-flight summary HTTP stream via a generation-scoped cancel ref; force-abandoned compactions retire at their next checkpoint and their stragglers are fenced off every surface (panes, pill latch, CLI). Every session retry backoff is cancel-aware via one shared helper. Docs, OpenAPI spec, and both SDKs updated. Verified: 9457-test non-live suite, JS pin suites, headless-Chrome reducer harness; five unprimed multi-agent review rounds (correctness trend 15/6/6/4/4) with plan-level design passes on every fix round. --- CHANGELOG.md | 82 ++ docs/api-reference.md | 88 +- sdk/typescript/openapi-server.json | 22 +- sdk/typescript/src/events.ts | 38 + tests/test_compaction_checkpoint.py | 35 + tests/test_cooperative_compaction.py | 785 +++++++++++++++++- tests/test_interactive_pane_js.py | 32 + tests/test_notify_tool.py | 25 +- tests/test_server_authz.py | 468 ++++++++++- turnstone/api/server_spec.py | 6 +- turnstone/cli.py | 62 ++ turnstone/console/coordinator_adapter.py | 11 + .../console/static/coordinator/coordinator.js | 48 +- turnstone/core/session.py | 581 +++++++++++-- turnstone/core/session_routes.py | 229 +++-- turnstone/core/session_ui_base.py | 157 +++- turnstone/core/session_worker.py | 27 + turnstone/core/storage/_postgresql.py | 11 +- turnstone/core/storage/_protocol.py | 14 +- turnstone/core/storage/_sqlite.py | 11 +- turnstone/core/storage/_utils.py | 22 +- turnstone/core/workstream.py | 12 + turnstone/eval/core.py | 3 + turnstone/sdk/events.py | 42 + turnstone/server.py | 229 ++++- turnstone/shared_static/chat.css | 102 +++ turnstone/shared_static/composer_queue.js | 5 + turnstone/shared_static/conversation.js | 251 ++++++ turnstone/shared_static/interactive.js | 110 ++- 29 files changed, 3262 insertions(+), 246 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45035291..107e5d29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,21 @@ Earlier stable lines (`stable/1.6`, `stable/1.5`) are frozen. ### Added +- **Compaction is visible now: lifecycle events, a progress bar, and a + persistent transcript card.** Context compaction (manual `/compact` and + auto) emits a first-class `compaction` SSE event + (`start` / `progress` / `end` — see the API reference) instead of loose + info lines. The web UI renders an in-transcript card with a real progress + bar (determinate `part k of N` during chunked summarization, indeterminate + for single-call compactions) that settles into a result card — token delta + plus the summary behind a fold — in both the interactive pane and the + coordinator viewer. The result survives reloads: the persisted compaction + marker now projects through `/history` as a `role="system"`, + `source="compaction"` entry (resume/export/search unchanged), stamped with + the end event's id so repaint and SSE replay can't double-render. The + marker's `meta` additionally records `before_tokens` / `after_tokens` / + `trigger`. Python and TypeScript SDKs gain a typed `CompactionEvent`. + - **One provider transport: every model call now streams (#831).** The per-adapter non-streaming entry (`create_completion`) is retired; single-shot lanes — judges, titles, compaction, web-fetch extraction, @@ -160,6 +175,73 @@ Earlier stable lines (`stable/1.6`, `stable/1.5`) are frozen. ### Fixed +- **Manual `/compact` from the web UI: no phantom user turn, no frozen + server, cancellable.** A slash command typed into the web composer no + longer renders as a user chat bubble (it echoes as a distinct command + chip — commands aren't conversation turns and were never persisted as + such). `/compact` itself now dispatches onto the workstream's worker + slot instead of running inline on the server's event loop — previously a + long compaction froze every SSE stream on the node for its whole + duration, which is also why its own progress only ever arrived as one + burst after the fact. The manual path carries `send()`'s full generation + discipline (`compact_now()`): a force-abandoned compaction goes stale + instead of swapping history under a successor turn — and retires at its + next checkpoint instead of running out its remaining summary calls, + with its late lifecycle events fenced off (`compaction_id` on every + event, `superseded` on end events — both in the SDKs) so they can't + animate, tear down, re-title, or falsely narrate a successor's card or + activity pill; a cancel aimed at it is consumed on exit (previously it + bricked every `/compact` retry until the next message); a Stop click on + an idle session can't pre-abort the next compaction; a Stop that lands + in the completion tail — after the last cancel check, or during a retry + backoff (which now aborts immediately instead of sleeping it out) — is + honored rather than silently eaten; and Stop now aborts the in-flight + summary HTTP call itself (the compaction lane registers its stream in + the same abort seam the main loop uses), so cancelling a compaction is + immediate instead of waiting out a model call. Messages sent while any + slash command holds the worker slot **park** and then run as ordinary + full-fidelity sends when the command finishes — they are never routed + through the mid-turn interjection queue, whose semantics are + turn-shaped: previously a send during a manual `/compact` was silently + truncated to 2,000 characters, a second participant in a shared + workstream was locked out with a misleading "another participant's + turn" 409 for the whole compaction, and a message queued across a + `/resume`/`/new` could be answered into the post-swap workstream. + A `/compact` raced against an in-flight turn is refused with an + explicit busy response. Every other slash command runs through the same + worker slot too — mutual exclusion against sends, a running compaction, + and each other, with a busy answer replacing the old silent interleave — + while the endpoint still awaits quick commands' completion off-loop + (without parking an executor thread per request); the post-command pane + refreshes (`clear_ui` after `/clear`/`/new`/`/resume`, the + workstream-name sync) ride the worker itself, so a command that + outlives the endpoint's 60s response backstop still refreshes every + pane on completion (the `/command` response contract — `ok` / `running`, + with busy refusals answering a loud HTTP 409 rather than a silent 200 — + is now documented in the API reference and the OpenAPI spec). Manual compaction + success also refreshes the status line/context pill immediately (parity + with auto-compaction), compaction failures keep feeding the typed + `error` event and the node error counter (while a CLI Ctrl-C reports as + cancelled, not a failure), one Stop prints one notice (a cancelled + auto-compaction no longer stacks "Compaction cancelled." on top of + send's own "[Generation cancelled]"), the workstream activity pill + shows "Compacting context…" for the whole summarize phase, restores + cleanly afterwards, and can no longer be stranded by a force-stopped + compaction (a new turn's generation claim breaks a stale latch). Every + retry backoff on the session (stream retries, task agents, notify + delivery, compaction) now aborts immediately on Stop via one shared + cancel-aware helper instead of sleeping out its exponential delay. The + web composer bounds a parked send at 10 minutes while a compaction card + is visible (15 s wedged-node default otherwise, shared by both panes), + and dismissing a queued bubble aborts the in-flight POST so a dismissed + message can't dispatch minutes later. A compaction failure reports + exactly once (auto-compaction errors defer to the turn's fatal handler + instead of doubling the red row and the error metric), and a manual + `/compact` failure no longer crashes the CLI REPL. Post-command pane + refreshes and error notices are owner-guarded, so a force-cancelled + wedged command that unwedges late can't wipe panes or inject stray + notices into a successor turn. + - **Static MCP servers: a pushed catalog change no longer wedges the shared session (#839).** The static-path `*/list_changed` handler awaited its catalog refresh inline in the SDK's receive loop, but the refresh's own diff --git a/docs/api-reference.md b/docs/api-reference.md index e9e63af4..5e080650 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -467,6 +467,43 @@ Each item in `items` (shared by `tool_info` and `approve_request`): {"type": "info", "message": "Session cleared."} ``` +**`compaction`** -- context-compaction lifecycle (manual `/compact` and +auto-compaction). `phase: "start"` opens the operation (`trigger` is +`"manual"` or `"auto"`; auto adds `where` — e.g. `"mid-turn"` — and, when +the percentage threshold actually fired, `pct`; the context-overflow retry +path compacts without a `pct` since no threshold was evaluated). +`phase: "progress"` reports chunked summarization (`part`/`total`/`depth`, +where depth 0 summarizes transcript batches and deeper levels merge partial +summaries), a transient-error retry wait (`retry_in` seconds + `error`), or +`warning: "summary_truncated"`. `phase: "end"` settles it: `ok: true` +carries `before_tokens`/`after_tokens` and the produced `summary`; +`ok: false` carries a `reason` +(`"not_enough_messages"` / `"irreducible"` / `"empty_summary"` / +`"cancelled"` / `"error"`) and a human-readable `message` — for +`reason: "error"` the same message is also emitted as a paired typed +`error` event (that is the renderable error surface; the end event is +card-teardown). Every end (ok or failed) carries `trigger`, and every +event carries `compaction_id` — an opaque integer correlating the +start/progress/end of one compaction run (a client that force-stopped one +compaction can use it to ignore stragglers from the abandoned run). End +events also carry `superseded`: `true` marks a force-abandoned compaction +retiring after a successor generation took over — skip failure notices for +those (an OK end's result card still stands: the history swap happened). +Superseded start/progress events are never emitted. +Exactly one `start` and one `end` are emitted per attempt, +so clients can key an in-progress affordance (progress bar) on the pair. A +successful end is also persisted: the summary replays from `/history` as a +`role: "system"`, `source: "compaction"` entry whose `meta` carries +`{watermark, before_tokens, after_tokens, trigger}` and whose `event_id` +matches the end event's id (dedup across repaint + replay). + +```json +{"type": "compaction", "phase": "start", "compaction_id": 7, "trigger": "auto", "where": "mid-turn", "pct": 80} +{"type": "compaction", "phase": "progress", "compaction_id": 7, "part": 2, "total": 5, "depth": 0} +{"type": "compaction", "phase": "end", "ok": true, "compaction_id": 7, "trigger": "auto", + "before_tokens": 128400, "after_tokens": 9200, "summary": "## Decisions\n..."} +``` + **`error`** -- an error message. ```json @@ -816,7 +853,41 @@ automatically approved without prompting. ### `POST /v1/api/command` -Executes a slash command in the given workstream. +Executes a slash command in the given workstream. Commands run on the +workstream's worker slot (mutual exclusion against sends, a running +compaction, and each other) — the endpoint is **not** unconditionally +synchronous: + +- **Quick commands** (everything except `/compact`): the endpoint waits for + completion, so `{"status": "ok"}` means the command ran. A command still + running after 60 s answers `{"status": "running"}` — the worker keeps + going, its output reaches the pane via SSE, and the post-command pane + refreshes below still fire when it completes. +- **`/compact`**: dispatched fire-and-forget — `{"status": "ok"}` means the + compaction *started*. A large context can legitimately compact for many + minutes; progress streams as `compaction` SSE events (see the event + reference) and the persisted marker row lands on completion. Do not read + `/history` expecting the compacted transcript immediately after the + response. +- **Busy refusal**: if a turn or another command holds the worker slot, the + command is refused with HTTP **409** `{"status": "busy", "error": ...}` and + did **not** run. Retry after the current turn finishes. (The old inline + endpoint executed commands unconditionally mid-turn; the 409 makes the + refusal loud for callers that only check the HTTP status.) + +While a command holds the slot, `POST .../send` requests **park** server-side +and dispatch as ordinary full-fidelity sends when the command's window closes +— they are never routed through the mid-turn interjection queue (no length +cap, no cross-user rejection). A client that aborts a parked send before the +window closes abandons it: the message is not dispatched. Clients must +therefore bound parked sends generously: the bundled web composer uses a +10-minute abort while a compaction progress card is visible (its usual bound +is ~15 s, sized for wedged-node detection), and dismissing a queued bubble +aborts the in-flight POST so a dismissed message can't dispatch later. +Deployment note: a reverse proxy's read timeout bounds the effective park — +behind a stock 60 s proxy, sends parked longer than that fail at the proxy +(the park sees the disconnect and abandons; nothing is dispatched — resend +after the command completes, or raise the proxy read timeout). **Request body:** @@ -832,7 +903,9 @@ Executes a slash command in the given workstream. If the command is `/clear` or `/new`, the server pushes a `clear_ui` SSE event to instruct the client to reset its message display. If the command is `/resume`, the server pushes `clear_ui` followed by a `history` event -containing the resumed session's messages. +containing the resumed session's messages. These follow-ups are emitted by the +command worker itself, so they fire even when the endpoint already answered +`{"status": "running"}`. **Response:** @@ -840,12 +913,15 @@ containing the resumed session's messages. {"status": "ok"} ``` +or `{"status": "running"}` as above. + **Error responses:** -| Status | Body | Condition | -|--------|------------------------------------|----------------------| -| 400 | `{"error": "Empty command"}` | Command is empty | -| 404 | `{"error": "Unknown workstream"}` | `ws_id` not found | +| Status | Body | Condition | +|--------|------------------------------------|----------------------------------| +| 400 | `{"error": "Empty command"}` | Command is empty | +| 404 | `{"error": "Unknown workstream"}` | `ws_id` not found | +| 409 | `{"status": "busy", "error": ...}` | A turn/command holds the worker | --- diff --git a/sdk/typescript/openapi-server.json b/sdk/typescript/openapi-server.json index c124c02d..321d7eac 100644 --- a/sdk/typescript/openapi-server.json +++ b/sdk/typescript/openapi-server.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Server API", - "version": "1.7.0rc1", + "version": "1.8.0a2", "description": "Single-node workstream management, chat interaction, and real-time streaming." }, "paths": { @@ -228,6 +228,16 @@ } } } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } }, @@ -390,6 +400,16 @@ } } } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } } diff --git a/sdk/typescript/src/events.ts b/sdk/typescript/src/events.ts index 8d33d117..c9715a55 100644 --- a/sdk/typescript/src/events.ts +++ b/sdk/typescript/src/events.ts @@ -157,6 +157,43 @@ export interface CancelledEvent { type: "cancelled"; } +/** + * Context-compaction lifecycle. `start` carries `trigger` ("manual"/"auto"; + * auto adds `where` + `pct`); `progress` carries chunked-summarization + * `part`/`total`/`depth` (or `retry_in`/`error` for a retry wait); `end` + * carries `ok` plus either `before_tokens`/`after_tokens`/`summary` or the + * failure `reason`/`message`. The successful end's summary also replays from + * `/history` as a `role: "system"`, `source: "compaction"` entry. + */ +export interface CompactionEvent { + type: "compaction"; + phase: "start" | "progress" | "end"; + /** Correlates every event of one compaction run (0 from legacy emitters). */ + compaction_id?: number; + /** + * End events only: true marks a force-abandoned compaction retiring + * after a successor generation took over — skip failure notices for + * those (an OK end's result still stands; the history swap happened). + */ + superseded?: boolean; + /** Present on start and on every end (ok or failed). */ + trigger?: "manual" | "auto"; + where?: string; + pct?: number; + part?: number; + total?: number; + depth?: number; + retry_in?: number; + error?: string; + warning?: string; + ok?: boolean; + reason?: string; + message?: string; + before_tokens?: number; + after_tokens?: number; + summary?: string; +} + // Global events export interface WsStateEvent { @@ -212,6 +249,7 @@ export type ServerEvent = | BusyErrorEvent | ClearUiEvent | CancelledEvent + | CompactionEvent | WsStateEvent | WsActivityEvent | WsRenameEvent diff --git a/tests/test_compaction_checkpoint.py b/tests/test_compaction_checkpoint.py index bcd5fea3..c78c2a6e 100644 --- a/tests/test_compaction_checkpoint.py +++ b/tests/test_compaction_checkpoint.py @@ -186,6 +186,41 @@ class TestDisplayPath: assert "SUMMARY" not in contents assert contents == ["q", "a"] # true transcript, no injected summary + def test_include_compaction_projects_marker_as_system_row(self, storage_backend): + """The /history display path (include_compaction=True) surfaces the + marker IN PLACE as a first-class system row — source="compaction", + meta = the marker's stored fields — so the UI re-renders its + compaction card after a reload. Export/search (default False) + stay on the drop path pinned above.""" + st = storage_backend + ws = _register(st) + st.save_message(ws, "user", "q") + st.save_message(ws, "assistant", "a") + wm = st.get_compaction_watermark(ws, 0) + st.save_message( + ws, + "assistant", + "SUMMARY", + source="compaction", + meta=json.dumps( + {"watermark": wm, "before_tokens": 900, "after_tokens": 80, "trigger": "manual"} + ), + ) + st.save_message(ws, "user", "later question") + + msgs = st.load_messages(ws, include_compaction=True) + assert [m.get("content") for m in msgs] == ["q", "a", "SUMMARY", "later question"] + marker = msgs[2] + assert marker["role"] == "system" # display row, not a fake assistant turn + assert marker.get("_source") == "compaction" + meta = marker.get("_source_meta") + assert meta == { + "watermark": wm, + "before_tokens": 900, + "after_tokens": 80, + "trigger": "manual", + } + # --------------------------------------------------------------------------- # End-to-end: compaction writes the marker, resume is bounded diff --git a/tests/test_cooperative_compaction.py b/tests/test_cooperative_compaction.py index f97a9169..54aeac66 100644 --- a/tests/test_cooperative_compaction.py +++ b/tests/test_cooperative_compaction.py @@ -14,6 +14,7 @@ the harness collapses the transcript: from __future__ import annotations +import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -154,21 +155,37 @@ class TestMidturnCompactionPolicy: advise.assert_not_called() def test_do_auto_compact_rounds_percentage(self, session): - """The notice uses round(), not int() — 0.58 must render '58%', not the - float-truncated '57%'.""" + """The start event's pct uses round(), not int() — 0.58 must render 58, + not the float-truncated 57. The auto notice rides the on_compaction + start payload now (where/pct), not an on_info string.""" session.auto_compact_pct = 0.58 with ( - patch.object(session, "_compact_messages") as compact, + patch.object(session, "_compact_messages_impl", return_value=True) as impl, patch.object(session, "_print_status_line"), - patch.object(session.ui, "on_info") as on_info, + patch.object(session.ui, "on_compaction") as on_compaction, ): session._do_auto_compact("mid-turn") - compact.assert_called_once_with( - auto=True, preserve_tail=0, my_generation=0, carry_spill=False - ) - msg = on_info.call_args.args[0] - assert "58%" in msg - assert "mid-turn" in msg + impl.assert_called_once_with(True, 0, 0, False) + start = on_compaction.call_args_list[0].args[0] + assert start["phase"] == "start" + assert start["trigger"] == "auto" + assert start["pct"] == 58 + assert start["where"] == "mid-turn" + + def test_overflow_auto_compact_start_carries_no_pct(self, session): + """The context-overflow retry path compacts with auto=True but never + evaluated the percentage threshold — its start event must not claim + one (the CLI would print a fabricated 'prompt exceeds N%' notice + contradicting the overflow notice above it).""" + with ( + patch.object(session, "_compact_messages_impl", return_value=True), + patch.object(session.ui, "on_compaction") as on_compaction, + ): + session._compact_messages(auto=True, my_generation=3) + start = on_compaction.call_args_list[0].args[0] + assert start["phase"] == "start" + assert start["trigger"] == "auto" + assert "pct" not in start # --------------------------------------------------------------------------- @@ -1216,7 +1233,7 @@ class TestChunkerOverflowSplit: blocks = ["A" * 4000, "B" * 4000, "C" * 4000] bodies: list[int] = [] - def fake_once(_system_prompt, body): + def fake_once(_system_prompt, body, _my_generation=0): bodies.append(len(body)) if len(body) > 6_000: # a multi-block body overflows the token window raise RuntimeError("maximum context length is 524288 tokens") @@ -1243,7 +1260,7 @@ class TestChunkerOverflowSplit: blocks = [f"b{i:02d} " + "z" * 500 for i in range(8)] calls: list[str] = [] - def fake_once(_system_prompt, body): + def fake_once(_system_prompt, body, _my_generation=0): calls.append(body) if body.count("\n\n") >= 4: # a body of 5+ blocks overflows the window raise RuntimeError("maximum context length is 524288 tokens") @@ -1269,7 +1286,7 @@ class TestChunkerOverflowSplit: floor = session._MIN_SUMMARY_BUDGET_CHARS calls: list[int] = [] - def fake_once(_system_prompt, body): + def fake_once(_system_prompt, body, _my_generation=0): calls.append(len(body)) if len(body) > floor: raise RuntimeError("maximum context length is 524288 tokens") @@ -1293,7 +1310,7 @@ class TestChunkerOverflowSplit: floor = session._MIN_SUMMARY_BUDGET_CHARS calls: list[int] = [] - def fake_once(_system_prompt, body): + def fake_once(_system_prompt, body, _my_generation=0): calls.append(len(body)) if len(body) > 9_000: # only bodies well above the floor overflow raise RuntimeError("maximum context length is 524288 tokens") @@ -1318,7 +1335,7 @@ class TestChunkerOverflowSplit: _CompactionIrreducibleError — NOT an unbounded recurse into RecursionError. Regression for the depth-check-only-on-the-multi-batch-path bug.""" - def no_shrink(_system_prompt, body): + def no_shrink(_system_prompt, body, _my_generation=0): if "\n\n" in body: # any multi-block body overflows the window raise RuntimeError("maximum context length is 524288 tokens") return body # a single-block 'summary' is the block itself — no shrink @@ -1338,7 +1355,7 @@ class TestChunkerOverflowSplit: blocks = ["A" * 2000, "B" * 2000, "C" * 2000, "D" * 2000] bodies: list[str] = [] - def fake_once(_system_prompt, body): + def fake_once(_system_prompt, body, _my_generation=0): bodies.append(body) if "CC" in body and "\n\n" in body: # the multi-block batch holding C raise RuntimeError("maximum context length is 524288 tokens") @@ -1579,3 +1596,739 @@ class TestRetryRewindSkipSummary: COMPACTION_SUMMARY_LABEL, "the dense summary", ] + + +# --------------------------------------------------------------------------- +# Compaction lifecycle events (the on_compaction start/progress/end contract) +# --------------------------------------------------------------------------- + + +def _compaction_events(on_compaction): + """The payload list an on_compaction mock saw.""" + return [c.args[0] for c in on_compaction.call_args_list] + + +def _seed_two_messages(session): + """Minimal compactable history — the shared two-turn seed.""" + session.messages = turns_from_dicts( + [ + {"role": "user", "content": "do the thing"}, + {"role": "assistant", "content": "did the thing"}, + ] + ) + session._msg_tokens = [5, 5] + session._system_tokens = 0 + + +class TestCompactionLifecycleEvents: + """Every _compact_messages exit emits exactly one start and one end — + a UI that paints an in-progress card on start must never be left with a + stuck progress bar.""" + + def test_manual_success_emits_start_then_ok_end(self, session): + _seed_two_messages(session) + summary = SimpleNamespace(content="## Decisions\ndense", finish_reason="stop") + with ( + patch.object(session, "_utility_completion", return_value=summary), + patch.object(session.ui, "on_compaction", return_value=41) as oc, + ): + assert session._compact_messages() is True + events = _compaction_events(oc) + assert events[0]["phase"] == "start" + assert events[0]["trigger"] == "manual" + assert "pct" not in events[0] # manual start carries no auto notice + end = events[-1] + assert end["phase"] == "end" and end["ok"] is True + assert end["summary"] == "## Decisions\ndense" + assert end["before_tokens"] > 0 and end["after_tokens"] > 0 + # Exactly one start and one end. + phases = [e["phase"] for e in events] + assert phases.count("start") == 1 and phases.count("end") == 1 + + def test_bail_too_few_messages_emits_failed_end(self, session): + session.messages = turns_from_dicts([{"role": "user", "content": "hi"}]) + session._msg_tokens = [1] + with patch.object(session.ui, "on_compaction") as oc: + assert session._compact_messages() is False + events = _compaction_events(oc) + assert [e["phase"] for e in events] == ["start", "end"] + assert events[1]["ok"] is False + assert events[1]["reason"] == "not_enough_messages" + assert events[1]["message"] == "Not enough messages to compact." + + def test_summary_error_emits_failed_end_and_returns_false(self, session): + _seed_two_messages(session) + with ( + patch.object(session, "_summarize_blocks", side_effect=RuntimeError("boom")), + patch.object(session.ui, "on_compaction") as oc, + ): + assert session._compact_messages() is False + end = _compaction_events(oc)[-1] + assert end["phase"] == "end" and end["ok"] is False + assert end["reason"] == "error" + assert "boom" in end["message"] + + def test_irreducible_emits_failed_end(self, session): + _seed_two_messages(session) + with ( + patch.object(session, "_summarize_blocks", side_effect=_CompactionIrreducibleError()), + patch.object(session.ui, "on_compaction") as oc, + ): + assert session._compact_messages() is False + end = _compaction_events(oc)[-1] + assert end["reason"] == "irreducible" + + def test_empty_summary_emits_failed_end(self, session): + _seed_two_messages(session) + blank = SimpleNamespace(content=" ", finish_reason="stop") + with ( + patch.object(session, "_utility_completion", return_value=blank), + patch.object(session.ui, "on_compaction") as oc, + ): + assert session._compact_messages() is False + end = _compaction_events(oc)[-1] + assert end["reason"] == "empty_summary" + + def test_cancel_mid_summary_emits_cancelled_end_and_reraises(self, session): + """GenerationCancelled must still propagate (history untouched) AND + retire the in-progress card via a cancelled end event.""" + _seed_two_messages(session) + with ( + patch.object(session, "_summarize_blocks", side_effect=GenerationCancelled()), + patch.object(session.ui, "on_compaction") as oc, + pytest.raises(GenerationCancelled), + ): + session._compact_messages() + events = _compaction_events(oc) + assert [e["phase"] for e in events] == ["start", "end"] + assert events[1]["ok"] is False + assert events[1]["reason"] == "cancelled" + assert events[1]["trigger"] == "manual" # failure ends carry trigger + + def test_keyboard_interrupt_ends_cancelled_not_error(self, session): + """Ctrl-C during a CLI compaction is a deliberate abort — the + backstop must retire the card as cancelled, never fire the typed + error channel with an empty-detail 'Compaction failed: ' line + (str(KeyboardInterrupt()) is '').""" + _seed_two_messages(session) + with ( + patch.object(session, "_summarize_blocks", side_effect=KeyboardInterrupt()), + patch.object(session.ui, "on_error") as on_error, + patch.object(session.ui, "on_compaction") as oc, + pytest.raises(KeyboardInterrupt), + ): + session._compact_messages() + on_error.assert_not_called() + end = _compaction_events(oc)[-1] + assert end["phase"] == "end" and end["ok"] is False + assert end["reason"] == "cancelled" + + def test_multi_batch_emits_part_progress(self, session): + """Chunked summarization reports part k/N via progress events (the + web card's determinate bar), replacing the old on_info lines.""" + session.context_window = 5_000 + session.compact_max_tokens = 4_000 + session._system_tokens = 0 + session.messages = turns_from_dicts( + [ + { + "role": "user" if i % 2 == 0 else "assistant", + "content": f"msg-{i:02d} " + "c" * 200, + } + for i in range(30) + ] + ) + session._msg_tokens = [1] * 30 + + def fake_uc(messages, **_kwargs): + return SimpleNamespace(content="PARTIAL", finish_reason="stop") + + with ( + patch.object(session, "_utility_completion", side_effect=fake_uc), + patch.object(session.ui, "on_compaction", return_value=7) as oc, + ): + assert session._compact_messages(auto=True) is True + progress = [e for e in _compaction_events(oc) if e["phase"] == "progress"] + assert progress, "multi-batch compaction must emit part progress" + first = progress[0] + assert first["part"] == 1 + assert first["total"] >= 2 + assert first["depth"] == 0 + + def test_marker_row_carries_token_meta_and_end_event_id(self, session): + """The persisted checkpoint marker's meta gains the display fields the + /history compaction card renders, and the row is stamped with the end + event's id so repaint and replay dedup against each other.""" + _seed_two_messages(session) + session._ws_id = "ws-compact-meta" + summary = SimpleNamespace(content="dense", finish_reason="stop") + saved: dict = {} + + def fake_save(ws_id, role, content, **kwargs): + saved.update({"ws_id": ws_id, "role": role, "content": content, **kwargs}) + return 1 + + with ( + patch.object(session, "_utility_completion", return_value=summary), + patch.object(session.ui, "on_compaction", return_value=99) as oc, + patch("turnstone.core.session.get_compaction_watermark", return_value=17), + patch("turnstone.core.session.save_message", side_effect=fake_save), + ): + assert session._compact_messages() is True + + import json as _json + + assert saved["source"] == COMPACTION_SOURCE + assert saved["event_id"] == 99 # the ok end event's id + meta = _json.loads(saved["meta"]) + assert meta["watermark"] == 17 + assert meta["trigger"] == "manual" + end = _compaction_events(oc)[-1] + assert meta["before_tokens"] == end["before_tokens"] + assert meta["after_tokens"] == end["after_tokens"] + + +# --------------------------------------------------------------------------- +# compact_now — the manual path's generation discipline (review fix round) +# --------------------------------------------------------------------------- + + +class TestCompactNow: + def test_stale_preset_cancel_event_does_not_brick(self, session): + """A Stop click on an idle session leaves _cancel_event set; the next + /compact must install a fresh event (send()'s entry discipline) and + run real work instead of instantly aborting as 'cancelled'.""" + _seed_two_messages(session) + session._cancel_event.set() # idle-cancel residue + summary = SimpleNamespace(content="dense", finish_reason="stop") + with patch.object(session, "_utility_completion", return_value=summary): + assert session.compact_now() is True + + def test_cancel_during_compaction_is_consumed(self, session): + """A cancel aimed at THIS compaction must not leak: the exit clears + the event (while still the active generation), so an immediate retry + does real work — previously every retry insta-aborted until the next + send.""" + _seed_two_messages(session) + + def cancel_mid_summary(*_a, **_kw): + session._cancel_event.set() + raise GenerationCancelled() + + with ( + patch.object(session, "_summarize_blocks", side_effect=cancel_mid_summary), + pytest.raises(GenerationCancelled), + ): + session.compact_now() + assert not session._cancel_event.is_set() + + # Retry succeeds without any external reset. + summary = SimpleNamespace(content="dense", finish_reason="stop") + with patch.object(session, "_utility_completion", return_value=summary): + assert session.compact_now() is True + + def test_superseded_compaction_never_swaps_history(self, session): + """The force-cancel race: a successor turn claims the next generation + while the abandoned compaction is mid-summary — the pre-swap check + must abort the swap (send() prevents the identical race the same + way). my_generation=0 used to skip this guard entirely.""" + _seed_two_messages(session) + before = list(session.messages) + + def supersede(*_a, **_kw): + # A successor send claims the next generation + fresh event + # while this compaction is inside its summarize call. + session._generation += 1 + session._cancel_event = threading.Event() + return "stale summary" + + with ( + patch.object(session, "_summarize_blocks", side_effect=supersede), + pytest.raises(GenerationCancelled), + ): + session.compact_now() + assert session.messages == before # history untouched + + def test_success_refreshes_status_line(self, session): + _seed_two_messages(session) + summary = SimpleNamespace(content="dense", finish_reason="stop") + with ( + patch.object(session, "_utility_completion", return_value=summary), + patch.object(session, "_print_status_line") as status, + ): + assert session.compact_now() is True + status.assert_called_once() + + def test_stop_landing_in_completion_tail_still_raises(self, session): + """A Stop that lands AFTER the impl's last cancel check (the swap / + marker-persist / status-line tail) completes the compaction but must + still be honored: compact_now consumes the event and re-raises, so + the web worker's exit seam flushes queued messages instead of + auto-running an answering turn the user just tried to stop.""" + _seed_two_messages(session) + + def complete_then_cancel(*_a, **_kw): + # The cancel arrives after every check inside the impl. + session._cancel_event.set() + return True + + with ( + patch.object(session, "_compact_messages", side_effect=complete_then_cancel), + patch.object(session, "_print_status_line") as status, + pytest.raises(GenerationCancelled), + ): + session.compact_now() + assert not session._cancel_event.is_set() # consumed, not leaked + # The compaction genuinely happened — the pill must reflect the + # freed window even though the tail Stop is honored: a stale pill + # next to a "context compacted" card claims two contradictory + # states at once. + status.assert_called_once() + + +class TestPreHookUICompat: + def test_ui_without_on_compaction_still_compacts(self, session): + """A duck-typed SessionUI predating the compaction hook gets no + lifecycle events but must not crash — an unguarded emit would + AttributeError inside send()'s auto-compaction and permanently + wedge every long session on such a UI.""" + _seed_two_messages(session) + session.ui = SimpleNamespace( + on_thinking_start=lambda: None, + on_thinking_stop=lambda: None, + on_error=lambda _m: None, + ) + summary = SimpleNamespace(content="dense", finish_reason="stop") + with patch.object(session, "_utility_completion", return_value=summary): + assert session._compact_messages() is True + + +class TestPreSwapQueueFlush: + def test_new_flushes_stranded_queue_into_old_workstream(self, session): + """A message stranded in the queue from BEFORE a /new (a dying send + worker's closing race) must be persisted into the workstream it was + ADDRESSED to — flushed pre-swap, never carried across the identity + change into the fresh workstream's transcript.""" + session._ws_id = "ws-old" + session.queue_message("stranded text") + saved: list[tuple[str, str, str]] = [] + + def fake_save(ws_id, role, content, **_kw): + saved.append((ws_id, role, content)) + return 1 + + with ( + patch("turnstone.core.session.save_message", side_effect=fake_save), + patch("turnstone.core.memory.register_workstream"), + patch.object(session, "_save_config"), + patch.object(session, "_follow_watch_registration"), + ): + session.handle_command("/new") + assert session._ws_id != "ws-old" + assert not session._queued_messages + flushed = [row for row in saved if row[2] == "stranded text"] + assert flushed and flushed[0][0] == "ws-old" # old identity, pre-swap + assert all(row[2] != "stranded text" for row in saved if row[0] != "ws-old") + + +class TestOrphanedCompactionRetirement: + """A force-abandoned compaction must retire at its next checkpoint once + a successor claims the generation — the checkpoint's event arm alone + can't see it (the successor installed a fresh, clear cancel event).""" + + def test_summarize_batch_raises_when_generation_superseded(self, session): + session._generation = 5 + with ( + patch.object(session, "_utility_completion") as uc, + pytest.raises(GenerationCancelled), + ): + session._summarize_blocks(["block-a"], my_generation=4) + uc.assert_not_called() # retired BEFORE spending another model call + + def test_cancel_during_retry_backoff_aborts_immediately(self, session): + """The backoff waits on the cancel event, not time.sleep — a Stop + during the (possibly minutes-long) wait aborts without burning the + delay plus one more model call.""" + session._cancel_event.set() + with ( + patch.object(session, "_utility_completion", side_effect=RuntimeError("transient")), + patch.object(session, "_stop_retrying", return_value=False), + pytest.raises(GenerationCancelled), + ): + session._summarize_once("sys", "body") + + def test_summary_call_registers_abortable_stream(self, session): + """Each summary attempt passes a fresh _CancelRef so cancel() can + close the in-flight summary HTTP stream — Stop during compaction + aborts the blocked read instead of waiting out a model call.""" + from turnstone.core.session import _CancelRef + + seen: list[object] = [] + + def fake_uc(_turns, *, cancel_ref=None, **_kw): + seen.append(cancel_ref) + stream = SimpleNamespace(closed=False, close=lambda: None) + cancel_ref.append(stream) + assert session._cancel_stream is stream # eager registration + return SimpleNamespace(content="dense", finish_reason="stop") + + with patch.object(session, "_utility_completion", side_effect=fake_uc): + assert session._summarize_once("sys", "body") == "dense" + assert len(seen) == 1 + assert isinstance(seen[0], _CancelRef) + assert seen[0] is not session._cancel_ref # scoped, never the shared ref + + def test_cancel_ref_aborted_property_tracks_event(self, session): + """model_turn consults cancel_ref.aborted to suppress drain retries + — a stream our own Stop closed must not be resurrected.""" + from turnstone.core.session import _CancelRef + + ref = _CancelRef(session) + assert ref.aborted is False + session._cancel_event.set() + assert ref.aborted is True + # Close-on-arrival: a stream appended after the Stop is closed + # immediately to unblock the waiting read. + closed: list[bool] = [] + ref.append(SimpleNamespace(close=lambda: closed.append(True))) + assert closed == [True] + + def test_superseded_ref_does_not_hijack_cancel_stream(self, session): + """A zombie summary stream arriving AFTER a successor generation + registered its live stream must neither overwrite _cancel_stream + (Stop would close the zombie and hang on the live read) nor stay + open burning tokens — it is closed on arrival.""" + from turnstone.core.session import _CancelRef + + session._generation = 5 + live = SimpleNamespace(close=lambda: None) + session._cancel_stream = live + ref = _CancelRef(session, my_generation=4) # abandoned compaction's ref + closed: list[bool] = [] + ref.append(SimpleNamespace(close=lambda: closed.append(True))) + assert session._cancel_stream is live # not hijacked + assert closed == [True] # zombie closed on arrival + assert ref.aborted is True # drain retries suppressed + + def test_current_generation_ref_still_registers(self, session): + from turnstone.core.session import _CancelRef + + session._generation = 3 + ref = _CancelRef(session, my_generation=3) + closed: list[bool] = [] + stream = SimpleNamespace(close=lambda: closed.append(True)) + ref.append(stream) + assert session._cancel_stream is stream + assert closed == [] + assert ref.aborted is False + + def test_summarize_once_scopes_ref_to_its_generation(self, session): + """The wiring, not just the mechanism: the ref _summarize_once + constructs must carry the compaction's my_generation, or the + zombie gate above never engages.""" + from turnstone.core.session import _CancelRef + + session._generation = 3 + seen: list[object] = [] + + def fake_uc(_turns, *, cancel_ref=None, **_kw): + seen.append(cancel_ref) + return SimpleNamespace(content="dense", finish_reason="stop") + + with patch.object(session, "_utility_completion", side_effect=fake_uc): + session._summarize_once("sys", "body", my_generation=3) + assert isinstance(seen[0], _CancelRef) + assert seen[0]._my_generation == 3 + + def test_stream_closed_by_cancel_maps_to_cancelled_not_error(self, session): + """A provider error induced by our own stream close (Stop) must end + the compaction as CANCELLED — checked before the retry policy, so + even a final-attempt failure never renders 'Compaction failed'.""" + + def fake_uc(_turns, **_kw): + session._cancel_event.set() # the Stop lands mid-call + raise RuntimeError("connection closed") + + with ( + patch.object(session, "_utility_completion", side_effect=fake_uc), + pytest.raises(GenerationCancelled), + ): + session._summarize_once("sys", "body") + + +# --------------------------------------------------------------------------- +# Error channel + activity pill (review fix round) +# --------------------------------------------------------------------------- + + +class TestCompactionErrorChannel: + def test_handled_error_bail_fires_on_error(self, session): + """reason='error' bails feed the typed error event (red row + + Prometheus counter via WebUI.on_error) — the surface compaction + failures fed before the lifecycle events replaced on_error here.""" + _seed_two_messages(session) + with ( + patch.object(session, "_summarize_blocks", side_effect=RuntimeError("boom")), + patch.object(session.ui, "on_error") as on_error, + patch.object(session.ui, "on_compaction") as oc, + ): + assert session._compact_messages() is False + on_error.assert_called_once() + assert "boom" in on_error.call_args.args[0] + end = [c.args[0] for c in oc.call_args_list][-1] + assert end["reason"] == "error" + + def test_non_error_bail_does_not_fire_on_error(self, session): + session.messages = turns_from_dicts([{"role": "user", "content": "a"}]) + session._msg_tokens = [1] + with ( + patch.object(session.ui, "on_error") as on_error, + patch.object(session.ui, "on_compaction"), + ): + assert session._compact_messages() is False + on_error.assert_not_called() + + def test_auto_raising_exit_defers_error_to_send_handler(self, session): + """An AUTO raising exit propagates into send()'s fatal handler, + which owns the single on_error — the wrapper emitting a second one + doubled every pane's red rows and the node's error metric.""" + _seed_two_messages(session) + with ( + patch.object(session, "_compact_messages_impl", side_effect=RuntimeError("boom")), + patch.object(session.ui, "on_error") as on_error, + pytest.raises(RuntimeError), + ): + session._compact_messages(auto=True, my_generation=1) + on_error.assert_not_called() + + def test_manual_raising_exit_emits_exactly_one_error(self, session): + """MANUAL raising exits have no downstream emitter (the web compact + worker's runner only logs; the CLI suppresses) — the wrapper's red + row is the one notification.""" + _seed_two_messages(session) + with ( + patch.object(session, "_compact_messages_impl", side_effect=RuntimeError("boom")), + patch.object(session.ui, "on_error") as on_error, + pytest.raises(RuntimeError), + ): + session._compact_messages() + on_error.assert_called_once() + + def test_cli_compact_error_does_not_crash_repl(self, session): + """The CLI REPL calls handle_command with no try/except — a raising + manual compaction error must be swallowed there (after the + wrapper's red row), not crash the whole REPL.""" + with patch.object(session, "compact_now", side_effect=RuntimeError("boom")): + assert not session.handle_command("/compact") + + def test_truncated_summary_warns_via_progress_event(self, session): + _seed_two_messages(session) + clipped = SimpleNamespace(content="partial", finish_reason="length") + with ( + patch.object(session, "_utility_completion", return_value=clipped), + patch.object(session.ui, "on_compaction", return_value=5) as oc, + ): + assert session._compact_messages() is True + warnings = [ + c.args[0] + for c in oc.call_args_list + if c.args[0].get("phase") == "progress" and c.args[0].get("warning") + ] + assert len(warnings) == 1 + assert warnings[0]["warning"] == "summary_truncated" + + +class TestCompactionActivityPill: + def test_pill_survives_thinking_start_and_restores_on_end(self, session): + """on_compaction(start) owns the pill for the whole window — the + impl's own on_thinking_start must not clobber it — and the end + restores the pre-compaction pair so a bail can't strand + 'Compacting context…' on an idle workstream.""" + ui = session.ui # NullUI(SessionUIBase) — the real base machinery + ui.on_compaction({"phase": "start", "trigger": "manual"}) + assert ui._ws_current_activity == "Compacting context…" + ui.on_thinking_start() + assert ui._ws_current_activity == "Compacting context…" # not clobbered + ui.on_compaction({"phase": "end", "ok": False, "reason": "not_enough_messages"}) + assert ui._ws_current_activity == "" # idle pair restored + assert ui._ws_activity_state == "" + # Outside a compaction window, thinking writes normally again. + ui.on_thinking_start() + assert ui._ws_current_activity == "Thinking…" + + def test_stale_end_leaves_successor_latch_alone(self, session): + """A force-abandoned compaction's late end must not unlatch — or + restore a stale pill pair over — a successor compaction that has + since taken the latch over (owner = compaction_id).""" + ui = session.ui + ui._ws_current_activity = "Running tool: bash" + ui._ws_activity_state = "tool" + ui.on_compaction({"phase": "start", "trigger": "manual", "compaction_id": 1}) + # Successor compaction takes the latch over; the ORIGINAL saved + # pair must survive (not the orphan's "Compacting context…"). + ui.on_compaction({"phase": "start", "trigger": "auto", "compaction_id": 2}) + # The orphan retires late — superseded, no longer the owner. + ui.on_compaction( + { + "phase": "end", + "ok": False, + "reason": "cancelled", + "message": "Compaction cancelled.", + "trigger": "manual", + "compaction_id": 1, + "superseded": True, + } + ) + assert ui._compaction_activity_live # successor still latched + assert ui._ws_current_activity == "Compacting context…" + # The successor's own end restores the ORIGINAL pre-compaction pair. + ui.on_compaction( + { + "phase": "end", + "ok": False, + "reason": "irreducible", + "message": "x", + "trigger": "auto", + "compaction_id": 2, + } + ) + assert not ui._compaction_activity_live + assert ui._ws_current_activity == "Running tool: bash" + assert ui._ws_activity_state == "tool" + + def test_superseded_end_unlatches_without_restoring(self, session): + """When the orphan's end arrives with no successor compaction, the + latch must clear (so the live turn's thinking writes resume) but its + stale saved pair must NOT overwrite the live turn's pill.""" + ui = session.ui + ui.on_compaction({"phase": "start", "trigger": "manual", "compaction_id": 3}) + ui.on_compaction( + { + "phase": "end", + "ok": False, + "reason": "cancelled", + "message": "Compaction cancelled.", + "trigger": "manual", + "compaction_id": 3, + "superseded": True, + } + ) + assert not ui._compaction_activity_live + assert ui._ws_current_activity == "Compacting context…" # no stale restore + ui.on_thinking_start() # the live turn recovers the pill + assert ui._ws_current_activity == "Thinking…" + + def test_superseded_progress_is_swallowed(self, session): + """An abandoned compaction's progress chatter is dropped at the base + (returns None, nothing enqueued) — panes must not re-create a card + for a dead compaction via their defensive-create — while its END + still flows so the pane can retire the card it painted live.""" + ui = session.ui + live = ui.on_compaction( + {"phase": "progress", "part": 1, "total": 2, "depth": 0, "compaction_id": 4} + ) + assert isinstance(live, int) + stale = ui.on_compaction( + { + "phase": "progress", + "part": 2, + "total": 2, + "depth": 0, + "compaction_id": 4, + "superseded": True, + } + ) + assert stale is None + end = ui.on_compaction( + { + "phase": "end", + "ok": False, + "reason": "cancelled", + "message": "x", + "trigger": "manual", + "compaction_id": 4, + "superseded": True, + } + ) + assert isinstance(end, int) + + def test_superseded_split_on_the_wire(self, session): + """Ends carry ``superseded`` on the wire (typed in both SDKs — a + pane must skip failure notices for a force-abandoned compaction's + late end); start/progress never carry it: live ones are by + definition not superseded and stale ones are swallowed whole.""" + ui = session.ui + with patch.object(ui, "_enqueue", return_value=9) as enq: + ui.on_compaction({"phase": "start", "trigger": "manual", "compaction_id": 5}) + assert "superseded" not in enq.call_args.args[0] + assert enq.call_args.args[0]["compaction_id"] == 5 + with patch.object(ui, "_enqueue", return_value=10) as enq: + ui.on_compaction( + { + "phase": "end", + "ok": True, + "trigger": "manual", + "compaction_id": 5, + "summary": "s", + } + ) + assert enq.call_args.args[0]["superseded"] is False + with patch.object(ui, "_enqueue", return_value=11) as enq: + ui.on_compaction( + { + "phase": "end", + "ok": False, + "reason": "cancelled", + "message": "x", + "trigger": "manual", + "compaction_id": 5, + "superseded": True, + } + ) + assert enq.call_args.args[0]["superseded"] is True + + def test_generation_claim_breaks_stale_latch_and_restores(self, session): + """A new generation claim is the moment any live latch is provably + stale (the worker slot serializes turns) — the claim unlatches AND + restores the saved pill pair, so the successor turn's thinking + writes resume immediately and a follow-up /compact's start saves a + correct pre-pair instead of the orphan's 'Compacting context…'.""" + ui = session.ui + ui._ws_current_activity = "Running tool: bash" + ui._ws_activity_state = "tool" + ui.on_compaction({"phase": "start", "trigger": "manual", "compaction_id": 7}) + assert ui._compaction_activity_live + # Successor claims (send() or compact_now() entry) — the orphan's + # latch breaks and the pre-compaction pair is restored. + session._claim_generation() + assert not ui._compaction_activity_live + assert ui._ws_current_activity == "Running tool: bash" + assert ui._ws_activity_state == "tool" + # And thinking writes work again for the live turn. + ui.on_thinking_start() + assert ui._ws_current_activity == "Thinking…" + # No live latch: a claim is a no-op (no restore of stale pairs). + ui._ws_current_activity = "Custom" + session._claim_generation() + assert ui._ws_current_activity == "Custom" + + def test_force_stop_then_second_compact_restores_pill(self, session): + """End-to-end pill lifecycle across an orphaned compaction: force + stop → a second /compact runs and completes → the pill must settle + on the PRE-ORPHAN pair, not a stranded 'Compacting context…'.""" + _seed_two_messages(session) + ui = session.ui + ui._ws_current_activity = "" + ui._ws_activity_state = "" + # First /compact latches, then is force-abandoned (no end event + # reaches the UI before the successor starts). + ui.on_compaction({"phase": "start", "trigger": "manual", "compaction_id": 1}) + assert ui._ws_current_activity == "Compacting context…" + # Second /compact: compact_now claims (breaking the stale latch, + # restoring the idle pair), then runs a real compaction. + summary = SimpleNamespace(content="dense", finish_reason="stop") + with patch.object(session, "_utility_completion", return_value=summary): + assert session.compact_now() is True + assert not ui._compaction_activity_live + assert ui._ws_current_activity == "" # idle pair, not the orphan's text + assert ui._ws_activity_state == "" diff --git a/tests/test_interactive_pane_js.py b/tests/test_interactive_pane_js.py index a49fb4f2..f6eaab12 100644 --- a/tests/test_interactive_pane_js.py +++ b/tests/test_interactive_pane_js.py @@ -636,3 +636,35 @@ def test_connectsse_defers_open_when_tab_hidden() -> None: ) assert head.index("this.wsId = wsId;") < head.index("if (document.hidden) {") assert head.index('addEventListener("visibilitychange"') < head.index("if (document.hidden) {") + + +def test_send_abort_bound_is_compaction_aware() -> None: + """The send POST's abort bound must be selected via the shared + ``sendAbortMs`` helper in BOTH panes: sends during a slash-command + window park server-side (a manual /compact legitimately runs for + minutes), so a hard-coded ~15s abort silently dropped any send made + >15s into a long compaction. The compaction card is the long-bound + signal; the wedged-node default stays otherwise.""" + interactive = _INTERACTIVE.read_text(encoding="utf-8") + assert "sendAbortMs(this._compaction)" in interactive, ( + "interactive sendMessage must select its abort bound off the compaction holder" + ) + coordinator = (_ROOT / "turnstone/console/static/coordinator/coordinator.js").read_text( + encoding="utf-8" + ) + assert "sendAbortMs(compactionHolder)" in coordinator, ( + "coordinator coordSend must share the same abort-bound policy" + ) + conversation = (_ROOT / "turnstone/shared_static/conversation.js").read_text(encoding="utf-8") + assert "export function sendAbortMs" in conversation + assert "600000 : 15000" in conversation.replace("\n", " "), ( + "long bound while a compaction card is live; 15s wedged-node default otherwise" + ) + # The queued bubble's pre-response x must abort the in-flight (possibly + # parked) POST — otherwise a dismissed message dispatches anyway when + # the command window closes. + assert "queuedEl._sendAbort = () => sendCtrl.abort()" in interactive + composer_queue = (_ROOT / "turnstone/shared_static/composer_queue.js").read_text( + encoding="utf-8" + ) + assert "el._sendAbort()" in composer_queue diff --git a/tests/test_notify_tool.py b/tests/test_notify_tool.py index a39e9ebd..0a8665c1 100644 --- a/tests/test_notify_tool.py +++ b/tests/test_notify_tool.py @@ -222,7 +222,7 @@ class TestExecNotify: with ( patch("turnstone.core.session.get_storage", return_value=storage), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), ): call_id, msg = session._exec_notify(item) @@ -293,7 +293,7 @@ class TestExecNotify: side_effect=ConnectionError("refused"), ), patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), ): # All fail — counter should stay at 0 for _i in range(3): @@ -330,7 +330,7 @@ class TestExecNotify: side_effect=ConnectionError("refused"), ), patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), ): call_id, msg = session._exec_notify(item) @@ -401,7 +401,7 @@ class TestExecNotify: patch("turnstone.core.session.get_storage", return_value=storage), patch("turnstone.core.session.httpx.post") as mock_post, patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), ): call_id, msg = session._exec_notify(item) @@ -456,13 +456,14 @@ class TestExecNotify: patch("turnstone.core.session.get_storage", return_value=mock_storage), patch("turnstone.core.session.httpx.post", return_value=mock_resp), patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep") as mock_sleep, + patch.object(session, "_backoff_or_cancelled") as mock_backoff, ): call_id, msg = session._exec_notify(item) assert "sent successfully" in msg.lower() - # Should have slept twice (retry delays) - assert mock_sleep.call_count == 2 + # Should have backed off twice (retry delays) — via the shared + # cancel-aware helper, not a Stop-blind time.sleep. + assert mock_backoff.call_count == 2 def test_retry_on_all_gateways_failed(self, tmp_path): """Retries when all gateways fail on first attempt but succeed on retry.""" @@ -502,12 +503,12 @@ class TestExecNotify: patch("turnstone.core.session.get_storage", return_value=storage), patch("turnstone.core.session.httpx.post", side_effect=_post), patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep") as mock_sleep, + patch.object(session, "_backoff_or_cancelled") as mock_backoff, ): call_id, msg = session._exec_notify(item) assert "sent successfully" in msg.lower() - assert mock_sleep.call_count == 1 + assert mock_backoff.call_count == 1 def test_no_services_logs_warning(self, tmp_path): """Server-side warning is logged when no services are available.""" @@ -530,7 +531,7 @@ class TestExecNotify: with ( patch("turnstone.core.session.get_storage", return_value=storage), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), patch("turnstone.core.session.log") as mock_log, ): session._exec_notify(item) @@ -569,7 +570,7 @@ class TestExecNotify: side_effect=ConnectionError("refused"), ), patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), patch("turnstone.core.session.log") as mock_log, ): session._exec_notify(item) @@ -610,7 +611,7 @@ class TestExecNotify: patch("turnstone.core.session.get_storage", return_value=storage), patch("turnstone.core.session.httpx.post", return_value=mock_resp), patch.dict("os.environ", {}, clear=False), - patch("turnstone.core.session.time.sleep"), + patch.object(session, "_backoff_or_cancelled"), ): call_id, msg = session._exec_notify(item) diff --git a/tests/test_server_authz.py b/tests/test_server_authz.py index d2908570..62227a5e 100644 --- a/tests/test_server_authz.py +++ b/tests/test_server_authz.py @@ -12,6 +12,7 @@ from __future__ import annotations import json import queue import threading +import time from typing import Any from unittest.mock import MagicMock @@ -102,6 +103,9 @@ class _FakeUI: self.auto_approve = False self.auto_approve_tools: set[str] = set() self._enqueued: list[dict[str, Any]] = [] + self.states: list[str] = [] + self.infos: list[str] = [] + self.errors: list[str] = [] self._listeners: list[queue.Queue[dict[str, Any]]] = [] self._listeners_lock = threading.Lock() self._pending_approval: dict[str, Any] | None = None @@ -196,11 +200,14 @@ class _FakeUI: def on_stream_end(self) -> None: pass - def on_state_change(self, _state: str) -> None: - pass + def on_state_change(self, state: str) -> None: + self.states.append(state) - def on_error(self, _msg: str) -> None: - pass + def on_info(self, msg: str) -> None: + self.infos.append(msg) + + def on_error(self, msg: str) -> None: + self.errors.append(msg) def resolve_approval(self, *_a: Any, **_kw: Any) -> None: self._approval_event.set() @@ -217,10 +224,41 @@ class _FakeSession: self.messages: list[dict[str, Any]] = [] self._last_usage: dict[str, int] | None = None self._pending_retry: str | None = None + # Real sessions always carry one; the /send route's cancel-drain + # poll reads it whenever a worker is live at send time. + self._cancel_event = threading.Event() self.sends: list[tuple[str, Any, Any]] = [] + self.commands: list[str] = [] + self.compacts = 0 + self.compact_raises: BaseException | None = None + self.send_raises: BaseException | None = None + self.exit_commands: set[str] = set() + self.queued_flushes = 0 + self.queued_text = "" + # Gates let a test hold the worker slot open mid-command so a + # concurrent /send provably lands in the PARK path. + self.compact_gate: threading.Event | None = None + self.command_gate: threading.Event | None = None + self.command_raises: BaseException | None = None + # Every queue_message call — the park invariant is that command + # windows NEVER reach the interjection queue. + self.queue_calls: list[str] = [] def send(self, text: str, *, attachments: Any = None, send_id: Any = None) -> None: self.sends.append((text, attachments, send_id)) + if self.send_raises is not None: + raise self.send_raises + + def queue_message( + self, + text: str, + attachment_ids: Any = None, + queue_msg_id: str | None = None, + interjector_user_id: str = "", + ) -> tuple[str, str, str]: + self.queue_calls.append(text) + cleaned = text[:2000] + "..." if len(text) > 2000 else text + return cleaned, "notice", queue_msg_id or "m1" def set_watch_runner(self, *_a: Any, **_kw: Any) -> None: pass @@ -234,8 +272,26 @@ class _FakeSession: def close(self) -> None: pass - def handle_command(self, _cmd: str) -> bool: - return False + def handle_command(self, cmd: str) -> bool: + self.commands.append(cmd) + if self.command_gate is not None: + self.command_gate.wait(timeout=10) + if self.command_raises is not None: + raise self.command_raises + return cmd in self.exit_commands + + def compact_now(self) -> bool: + self.compacts += 1 + if self.compact_gate is not None: + self.compact_gate.wait(timeout=10) + if self.compact_raises is not None: + raise self.compact_raises + return True + + def flush_queued_messages(self) -> bool: + self.queued_flushes += 1 + had, self.queued_text = bool(self.queued_text), "" + return had def request_title_refresh(self, _title: str) -> None: pass @@ -1177,3 +1233,403 @@ class TestInteractiveEventsLifted: headers=_auth("user-1"), ) assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# POST /v1/api/command — /compact worker dispatch (progress-events fix) +# --------------------------------------------------------------------------- + + +class TestCompactCommandDispatch: + """Manual /compact runs on the workstream's worker slot: the event loop + stays free to stream the compaction progress events, and a concurrent + send takes the queue path instead of racing the history swap.""" + + def _create_ws(self, client) -> str: + resp = client.post( + "/v1/api/workstreams/new", + json={"name": "compact-me"}, + headers=_auth("user-1"), + ) + assert resp.status_code == 200 + return resp.json()["ws_id"] + + def test_compact_dispatches_to_worker(self, app_client): + client, mgr = app_client + ws_id = self._create_ws(client) + resp = client.post( + "/v1/api/command", + json={"command": "/compact", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + assert resp.status_code == 200 + assert resp.json() == {"status": "ok"} + + ws = mgr.get(ws_id) + assert ws is not None + # The compaction runs on the spawned worker — join it (the runner + # clears _worker_running before the thread exits, so the join + # subsumes the flag). + ws.worker_thread.join(timeout=5) + assert not ws.worker_thread.is_alive() + assert ws.session.compacts == 1 + assert ws._worker_running is False + # The slot was classified as a command window (what the /send + # route's park keys on; stale after exit is harmless — every + # reader conjoins _worker_running). + assert ws.worker_kind == "command" + # Exit seam: stranded-text backstop flush only — no drain, no + # answering send (sends during the window park in the /send route + # and dispatch as their own workers afterwards). + assert ws.session.queued_flushes == 1 + assert ws.session.sends == [] + # The worker wrapped the run in busy/idle state transitions. + assert ws.ui.states == ["thinking", "idle"] + + def test_send_during_compact_window_parks_then_dispatches_fully(self, app_client): + """A /send during a manual /compact PARKS and then runs as an + ordinary full-fidelity send — it must never enter the interjection + queue, whose 2000-char cap silently truncated pasted logs/code and + whose cross-user guard locked second participants out for the + whole compaction.""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + gate = threading.Event() + ws.session.compact_gate = gate + resp = client.post( + "/v1/api/command", + json={"command": "/compact", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + assert resp.json() == {"status": "ok"} + big = "x" * 5000 # over the interjection cap — must survive intact + send_result: dict = {} + + def _send() -> None: + r = client.post( + f"/v1/api/workstreams/{ws_id}/send", + json={"message": big}, + headers=_auth("user-1"), + ) + send_result["status"] = r.status_code + send_result["body"] = r.json() + + sender = threading.Thread(target=_send, daemon=True) + sender.start() + # The send is parked: give it time to have taken the park path, + # then prove nothing was dispatched or queued yet. + for _ in range(50): + if ws.session.queue_calls or ws.session.sends: + break + time.sleep(0.02) + assert ws.session.sends == [] + assert ws.session.queue_calls == [] # the queue is unreachable + gate.set() # compaction finishes; the park releases + sender.join(timeout=10) + assert not sender.is_alive() + assert send_result["status"] == 200 + assert send_result["body"]["status"] == "ok" + # Full fidelity: the exact 5000-char text, via a normal send. + assert [s[0] for s in ws.session.sends] == [big] + assert ws.session.queue_calls == [] + + def test_cancelled_compact_flushes_queue_without_answering(self, app_client): + """A user-stopped compaction must not auto-run a turn they may no + longer want — queued text lands in the transcript via the flush + drain instead (the cancel-seam precedent).""" + from turnstone.core.session import GenerationCancelled + + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + ws.session.compact_raises = GenerationCancelled() + ws.session.queued_text = "queued mid-compact" + resp = client.post( + "/v1/api/command", + json={"command": "/compact", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + assert resp.status_code == 200 + ws.worker_thread.join(timeout=5) + assert ws.session.compacts == 1 + assert ws.session.queued_flushes == 1 + assert ws.session.sends == [] + assert ws.ui.states == ["thinking", "idle"] + + def test_compact_refused_while_worker_running(self, app_client): + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + with ws._lock: + ws._worker_running = True # a turn is in flight + try: + resp = client.post( + "/v1/api/command", + json={"command": "/compact", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + finally: + with ws._lock: + ws._worker_running = False + assert resp.status_code == 409 # loud refusal for status-code-only callers + body = resp.json() + assert body["status"] == "busy" + assert "busy" in body["error"].lower() + # Never ran inline, never queued a phantom compaction. + assert ws.session.compacts == 0 + assert ws.session.commands == [] + + def test_non_compact_commands_complete_before_response(self, app_client): + """Quick commands dispatch through the same worker slot (mutual + exclusion vs sends / a running compaction / each other) but the + endpoint awaits completion, preserving the synchronous contract: + handle_command has finished by the time the response returns.""" + client, mgr = app_client + ws_id = self._create_ws(client) + resp = client.post( + "/v1/api/command", + json={"command": "/skill", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + assert resp.status_code == 200 + assert resp.json() == {"status": "ok"} + ws = mgr.get(ws_id) + # handle_command completed before the response (the done-Event + # gates it); join before asserting the slot state (the runner + # clears _worker_running before the thread exits, so the join + # subsumes the flag; without it this assert races the worker's + # last steps). + assert ws.session.commands == ["/skill"] + ws.worker_thread.join(timeout=5) + assert not ws.worker_thread.is_alive() + assert ws._worker_running is False + # No exit seam work at all for quick commands: no drain, no flush, + # no follow-up send, and no state chatter (they never left idle). + assert ws.session.queued_flushes == 0 + assert ws.session.sends == [] + assert ws.ui.states == [] + + def test_parked_send_delivers_attachments_after_release(self, app_client, monkeypatch): + """Attachments are peek-resolved BEFORE the park (the bytes ride the + request closure), so a send parked through a long compaction still + delivers them on dispatch — the pre-park refusal + (attachments_busy) must never apply to a command window.""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + + def fake_resolve(requested_ids, _ws_id, _user_id): + assert list(requested_ids) == ["a1"] + return (["fake-attachment-bytes"], ["a1"], []) + + monkeypatch.setattr("turnstone.core.attachments.resolve_staged_attachments", fake_resolve) + gate = threading.Event() + ws.session.compact_gate = gate + client.post( + "/v1/api/command", + json={"command": "/compact", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + send_result: dict = {} + + def _send() -> None: + r = client.post( + f"/v1/api/workstreams/{ws_id}/send", + json={"message": "with attachment", "attachment_ids": ["a1"]}, + headers=_auth("user-1"), + ) + send_result["body"] = r.json() + + sender = threading.Thread(target=_send, daemon=True) + sender.start() + time.sleep(0.3) + assert ws.session.sends == [] # still parked + gate.set() + sender.join(timeout=10) + assert not sender.is_alive() + assert send_result["body"]["status"] == "ok" + assert send_result["body"]["attached_ids"] == ["a1"] + text, attachments, _sid = ws.session.sends[0] + assert text == "with attachment" + assert attachments == ["fake-attachment-bytes"] + assert ws.session.queue_calls == [] + + def test_send_during_quick_command_window_parks(self, app_client): + """The park applies to EVERY command window, not just /compact — a + send racing a quick command dispatches after the window with full + fidelity instead of entering the interjection queue.""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + gate = threading.Event() + ws.session.command_gate = gate + cmd_result: dict = {} + + def _cmd() -> None: + r = client.post( + "/v1/api/command", + json={"command": "/skill", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + cmd_result["body"] = r.json() + + runner = threading.Thread(target=_cmd, daemon=True) + runner.start() + # Wait until the command worker actually holds the slot. + for _ in range(100): + if ws._worker_running: + break + time.sleep(0.02) + assert ws._worker_running + assert ws.worker_kind == "command" + send_result: dict = {} + + def _send() -> None: + r = client.post( + f"/v1/api/workstreams/{ws_id}/send", + json={"message": "mid-command send"}, + headers=_auth("user-1"), + ) + send_result["body"] = r.json() + + sender = threading.Thread(target=_send, daemon=True) + sender.start() + time.sleep(0.3) # long enough for a queue-path regression to show + assert ws.session.queue_calls == [] + assert ws.session.sends == [] + gate.set() + runner.join(timeout=10) + sender.join(timeout=10) + assert not sender.is_alive() + assert cmd_result["body"] == {"status": "ok"} + assert send_result["body"]["status"] == "ok" + assert [s[0] for s in ws.session.sends] == ["mid-command send"] + assert ws.session.queue_calls == [] + + def test_clear_ui_rides_the_worker_not_the_endpoint(self, app_client): + """The clear_ui follow-up runs on the worker after handle_command — + parked after the endpoint's 60s done-wait it was silently skipped + for any command that outlived the backstop, leaving every pane + rendering a transcript the server no longer holds.""" + client, mgr = app_client + ws_id = self._create_ws(client) + resp = client.post( + "/v1/api/command", + json={"command": "/clear", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + assert resp.status_code == 200 + ws = mgr.get(ws_id) + ws.worker_thread.join(timeout=5) + assert {"type": "clear_ui"} in ws.ui._enqueued + + def _run_abandoned_command(self, client, mgr, ws_id, command="/clear", raises=None): + """Dispatch a gated command, force-abandon its worker mid-run, then + release the gate so the abandoned thread finishes late. Returns + (endpoint response body, the abandoned worker thread).""" + ws = mgr.get(ws_id) + gate = threading.Event() + ws.session.command_gate = gate + ws.session.command_raises = raises + result: dict = {} + + def _cmd() -> None: + r = client.post( + "/v1/api/command", + json={"command": command, "ws_id": ws_id}, + headers=_auth("user-1"), + ) + result["body"] = r.json() + + runner = threading.Thread(target=_cmd, daemon=True) + runner.start() + for _ in range(100): + if ws._worker_running: + break + time.sleep(0.02) + assert ws._worker_running + worker = ws.worker_thread + # The cancel handler's force path shape: abandon the worker. + with ws._lock: + ws.worker_thread = None + ws._worker_running = False + gate.set() # the wedged command unwedges LATE + worker.join(timeout=5) + runner.join(timeout=10) + assert not worker.is_alive() and not runner.is_alive() + return result["body"], worker + + def test_abandoned_command_worker_fires_no_followups(self, app_client): + """A force-cancelled wedged command that unwedges minutes later + must not fire clear_ui (every pane would wipe its transcript + mid-successor-turn) — the owner guard every sibling worker closure + applies. done still fires, so the endpoint never hangs.""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + body, _ = self._run_abandoned_command(client, mgr, ws_id, command="/clear") + assert not any(e.get("type") == "clear_ui" for e in ws.ui._enqueued) + # handle_command genuinely completed, so the (unhung) endpoint's + # answer reflects that. + assert body["status"] == "ok" + + def test_abandoned_command_worker_swallows_late_error(self, app_client): + """The except arm carries the same guard: a stray late 'Command + error:' from an abandoned worker must not land mid-successor-turn.""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + self._run_abandoned_command( + client, mgr, ws_id, command="/skill", raises=RuntimeError("late boom") + ) + assert ws.ui.errors == [] + + def test_exit_command_emits_ended_info_and_never_answers(self, app_client): + """should_exit commands shut the session down — the worker emits + the ended notice and never launches an answering turn (sends + during the window park in the /send route and belong to whatever + follows the shutdown, not to this worker).""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + ws.session.exit_commands = {"/exit"} + resp = client.post( + "/v1/api/command", + json={"command": "/exit", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + assert resp.status_code == 200 + ws.worker_thread.join(timeout=5) + assert ws.session.sends == [] + assert any("Session ended" in m for m in ws.ui.infos) + + def test_non_compact_command_refused_while_worker_running(self, app_client): + """The old inline path was serialized by the event loop itself; the + worker-slot dispatch restores that mutual exclusion with an explicit + busy answer — /clear can no longer interleave with a live turn or a + running compaction.""" + client, mgr = app_client + ws_id = self._create_ws(client) + ws = mgr.get(ws_id) + assert ws is not None + with ws._lock: + ws._worker_running = True # a turn / compaction is in flight + try: + resp = client.post( + "/v1/api/command", + json={"command": "/clear", "ws_id": ws_id}, + headers=_auth("user-1"), + ) + finally: + with ws._lock: + ws._worker_running = False + assert resp.status_code == 409 + assert resp.json()["status"] == "busy" + assert ws.session.commands == [] diff --git a/turnstone/api/server_spec.py b/turnstone/api/server_spec.py index 9d507aa8..7d121ddc 100644 --- a/turnstone/api/server_spec.py +++ b/turnstone/api/server_spec.py @@ -102,7 +102,9 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ "Send a user message", request_model=SendRequest, response_model=SendResponse, - error_codes=[400, 404], + # 409: cross_user_interjection (shipped with the multi-user work; + # the spec had drifted behind the implementation). + error_codes=[400, 404, 409], tags=["Chat"], ), EndpointSpec( @@ -134,7 +136,7 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ "Execute a slash command", request_model=CommandRequest, response_model=StatusResponse, - error_codes=[400, 404], + error_codes=[400, 404, 409], tags=["Chat"], ), EndpointSpec( diff --git a/turnstone/cli.py b/turnstone/cli.py index 3ac10d39..682a9a78 100644 --- a/turnstone/cli.py +++ b/turnstone/cli.py @@ -328,6 +328,68 @@ class TerminalUI(SessionUI): sys.stdout.write(f"{YELLOW}[{label}]{RESET} {content}\n") sys.stdout.flush() + def on_compaction(self, payload: dict[str, Any]) -> int | None: + """Render compaction lifecycle events as the terminal's classic text + lines — the notice, ``part k/N`` progress, and the token-delta + + boxed-summary result the CLI printed before these became structured + events (the web UI renders the same payloads as a progress card). + """ + phase = payload.get("phase") + if phase == "start": + # The threshold notice prints only when a threshold actually + # fired (pct present — _do_auto_compact). The overflow-retry + # path compacts with auto=True but prints its own "[Context + # overflow — auto-compacting and retrying]" line; claiming a + # percentage there would fabricate the trigger. Manual starts + # print nothing — the thinking spinner covers it. + pct = payload.get("pct") + if payload.get("trigger") == "auto" and pct is not None: + where = payload.get("where") or "" + qualifier = f" {where}" if where else "" + self.on_info( + f"\n[Auto-compacting{qualifier}: prompt exceeds {pct}% of context window]" + ) + elif phase == "progress": + if payload.get("warning") == "summary_truncated": + self.on_info("[Warning: compaction summary was truncated]") + elif payload.get("retry_in") is not None: + self.on_info( + f"[Compact retrying in {payload['retry_in']:.0f}s: {payload.get('error', '')}]" + ) + else: + self.on_info(f"[compacting part {payload.get('part')}/{payload.get('total')}…]") + elif phase == "end": + if payload.get("ok"): + before = payload.get("before_tokens", 0) + after = payload.get("after_tokens", 0) + self.on_info(f"[compacted: ~{before:,} -> ~{after:,} tokens]") + separator = "─" * 60 + lines = [separator] + for line in str(payload.get("summary") or "").splitlines(): + lines.append(f" {line}") + lines.append(separator) + self.on_info("\n".join(lines)) + elif payload.get("reason") != "error": + # reason="error" already printed through on_error (red) — + # the end event is card-teardown for the web panes, not a + # second line here. A cancelled AUTO compaction is also + # silent: the surrounding send prints "[Generation + # cancelled]" itself, and pre-lifecycle-events one Stop + # printed exactly one line. (A cancelled MANUAL /compact + # keeps the message — it's the only line that path prints.) + # A SUPERSEDED end is a force-abandoned compaction retiring + # late — nobody is waiting on it; narrating it mid-turn + # reads as the LIVE work being cancelled. + # HAND-SYNCED SIBLING: conversation.js applyCompactionEvent + # implements this same three-clause display policy (skip + # error-reason / skip superseded / skip cancelled+auto) for + # the web panes — change both or they drift. + if not payload.get("superseded") and not ( + payload.get("reason") == "cancelled" and payload.get("trigger") == "auto" + ): + self.on_info(str(payload.get("message") or "")) + return None + def on_state_change(self, state: str) -> None: pass # base TerminalUI ignores state changes diff --git a/turnstone/console/coordinator_adapter.py b/turnstone/console/coordinator_adapter.py index 620c1d46..147ddb1e 100644 --- a/turnstone/console/coordinator_adapter.py +++ b/turnstone/console/coordinator_adapter.py @@ -15,6 +15,7 @@ for the storage-seeded children rebuild. from __future__ import annotations +import queue from typing import TYPE_CHECKING, Any from turnstone.core import session_worker @@ -372,6 +373,16 @@ class CoordinatorAdapter: # fresh workstream so the catch is defense-in-depth. Nothing to # release: the staged bytes were peeked, not soft-locked, and a # rejected enqueue never drained them. + if ws.worker_kind == "command": + # A slash-command worker (e.g. a minutes-long /compact aimed + # at this workstream from the interactive UI) holds the + # raced slot: the interjection queue is turn-shaped and must + # stay unreachable during command windows — same rule as the + # /send route's park. Fail the dispatch (returns False, the + # caller's retryable-backpressure surface) rather than queue + # a message that would be capped and could cross a /resume + # identity swap. + raise queue.Full() att_ids = [a.attachment_id for a in _attachments] if _attachments else None session.queue_message( message, diff --git a/turnstone/console/static/coordinator/coordinator.js b/turnstone/console/static/coordinator/coordinator.js index c2bb9e5b..8d9ad242 100644 --- a/turnstone/console/static/coordinator/coordinator.js +++ b/turnstone/console/static/coordinator/coordinator.js @@ -28,6 +28,10 @@ // --------------------------------------------------------------------------- import { buildWatchResultCard, + buildCompactionCard, + applyCompactionEvent, + resetCompactionHolder, + sendAbortMs, buildSystemNudgeMarker, maxSeverityItem, buildConvBatchShell, @@ -511,6 +515,12 @@ function createCoordinatorPane(root, wsId, opts) { // ui/static/app.js's per-pane _renderedSystemEventIds. const renderedSystemEventIds = new Set(); + // Compaction lifecycle holder for the shared reducer + // (conversation.applyCompactionEvent); `card` is the in-progress card + // between start and end, nulled wherever the transcript is wiped — live + // events re-create it defensively. + const compactionHolder = { card: null, cid: null }; + // Cache of judge verdicts keyed by call_id. intent_verdict and // approve_request are async and may arrive in either order; the // cache lets each handler apply data to the other without assuming @@ -600,7 +610,11 @@ function createCoordinatorPane(root, wsId, opts) { rows = [parsed]; } if (rows.length === 0) { - return "
" + esc(redactCredentials(JSON.stringify(parsed, null, 2))) + "
"; + return ( + "
" +
+        esc(redactCredentials(JSON.stringify(parsed, null, 2))) +
+        "
" + ); } const lines = rows.map((row) => { const safeWs = row.ws_id && WS_ID_RE.test(row.ws_id) ? row.ws_id : null; @@ -817,6 +831,14 @@ function createCoordinatorPane(root, wsId, opts) { // labeled operator bubble. function renderSystemTurn(source, content, meta) { const m = meta && typeof meta === "object" ? meta : null; + // /history projection of a persisted compaction marker — same result + // card the live `compaction` end event paints (shared builder). + if (source === "compaction") { + const card = buildCompactionCard(m, content || ""); + messagesEl.appendChild(card); + _scheduleScroll(); + return card; + } if (source === "watch_triggered" && m) return appendWatchResult(m, content || ""); if (source === "output_guard" && m) return appendGuardFinding(m); @@ -1975,7 +1997,10 @@ function createCoordinatorPane(root, wsId, opts) { let sendTimer = null; if (sendCtrl) { sendInit.signal = sendCtrl.signal; - sendTimer = setTimeout(() => sendCtrl.abort(), 15000); + // Same policy as the interactive composer: long bound while this + // pane's compaction card is live (the send parks server-side for + // the command window), wedged-node default otherwise. + sendTimer = setTimeout(() => sendCtrl.abort(), sendAbortMs(compactionHolder)); } let sendReq = authFetch( "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/send", @@ -2745,6 +2770,11 @@ function createCoordinatorPane(root, wsId, opts) { } break; case "stream_end": + // A live in-progress compaction card here means a FORCE stop + // abandoned the compaction worker (the lifecycle wrapper otherwise + // always retires the card with an end event before any stream_end + // can follow) — remove it instead of leaving a frozen bar. + resetCompactionHolder(compactionHolder); finishAssistantStream(); break; case "stream_overflow": @@ -2922,6 +2952,19 @@ function createCoordinatorPane(root, wsId, opts) { if (sysEid) renderedSystemEventIds.add(sysEid); break; } + case "compaction": + // Context-compaction lifecycle — the shared reducer + // (conversation.applyCompactionEvent) is the one state machine for + // this viewer and the interactive pane, so the two can't drift. + // reason="error" ends render via the paired `error` event (red row); + // the reducer emits only the non-error failure notices here. + applyCompactionEvent(compactionHolder, ev, { + container: messagesEl, + renderedIds: renderedSystemEventIds, + onNotice: (msg) => appendText("info", msg, { label: "info" }), + scroll: () => _scheduleScroll(), + }); + break; case "connected": // First yield from _coord_events_replay — populates the // status bar's model cell before any history arrives. Also @@ -5009,6 +5052,7 @@ function createCoordinatorPane(root, wsId, opts) { toolRows.clear(); activeBatch = null; renderedSystemEventIds.clear(); + resetCompactionHolder(compactionHolder); if (!hist) return; // Fresh-connect fast-forward: when the trailing turn is an executing // in-flight tool batch the server can replay, /history returns a diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 675862cd..0a8eb1e5 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -308,24 +308,60 @@ class _CancelRef(list[Any]): immediately. If cancellation was already requested before the stream was created (e.g. cancel during retry backoff), the stream is closed on arrival so the blocked iteration is unblocked. + + ``my_generation`` scopes a ref to one generation (compaction's summary + calls pass theirs; the main loop's long-lived shared ref keeps the + default 0 = unconditional). A superseded ref's late-arriving stream — + an abandoned compaction that passed its boundary check just before a + force-cancel and opened one final zombie call — must neither hijack + ``_cancel_stream`` from the successor generation's live stream nor + keep burning tokens, so the append skips the registration and closes + the stream on arrival. The generation check and the register are two + lockless statements; the residual bytecode-width TOCTOU (successor + claims AND registers between them) is accepted — its harm is one + delayed Stop (closes a dead handle; the event arm still cancels at the + next chunk), not corruption — versus the model-call-width window this + closes. """ - __slots__ = ("_session",) + __slots__ = ("_session", "_my_generation") - def __init__(self, session: ChatSession) -> None: + def __init__(self, session: ChatSession, my_generation: int = 0) -> None: super().__init__() self._session = session + self._my_generation = my_generation + + def _superseded(self) -> bool: + gen = self._my_generation + return bool(gen and self._session._generation != gen) def append(self, stream: Any) -> None: super().append(stream) - self._session._cancel_stream = stream + superseded = self._superseded() + if not superseded: + self._session._cancel_stream = stream # If cancel was requested before the first chunk arrived (the worker # thread is blocked inside the provider generator waiting for the HTTP - # response), close the stream immediately to unblock it. - if self._session._cancel_event.is_set(): + # response), close the stream immediately to unblock it. Same for a + # superseded ref's zombie stream: nobody will consume it. + if superseded or self._session._cancel_event.is_set(): with contextlib.suppress(Exception): stream.close() + @property + def aborted(self) -> bool: + """Whether this ref's stream must not be resurrected. + + ``model_turn`` consults ``cancel_ref.aborted`` before re-issuing a + request after a mid-drain transport failure (the deadline daemon's + :class:`~turnstone.core.deadline.StreamAbortRef` contract): a + stream that died because :meth:`ChatSession.cancel` closed it — or + because it belongs to a superseded generation — must not be + resurrected behind the user's Stop; the failure surfaces and the + caller's own cancel check turns it into ``GenerationCancelled``. + """ + return self._session._cancel_event.is_set() or self._superseded() + # Image extensions handled as vision content (SVG excluded — it's XML text) _IMAGE_EXTENSIONS: frozenset[str] = frozenset( @@ -1139,6 +1175,21 @@ class SessionUI(Protocol): def on_system_turn( self, content: str, source: str, meta: dict[str, Any] | None = None ) -> int | None: ... + def on_compaction(self, payload: dict[str, Any]) -> int | None: + """Called through the compaction lifecycle. + + ``payload["phase"]`` is ``"start"`` (fields: ``trigger`` = + ``"manual"``/``"auto"``, and for auto ``where``/``pct``), + ``"progress"`` (``part``/``total``/``depth``, or ``retry_in``/ + ``error`` for a retry wait), or ``"end"`` (``ok``, and either + ``before_tokens``/``after_tokens``/``summary`` or ``reason``/ + ``message``). Returns the UI event id when the transport + assigns one (see :meth:`on_system_turn`) so the persisted + compaction marker row can be stamped with the matching resume + cursor; ``None`` for UIs without an event stream. + """ + ... + def on_state_change(self, state: str) -> None: ... def on_rename(self, name: str) -> None: ... def on_intent_verdict(self, verdict: dict[str, Any], judge_event: object | None = None) -> None: @@ -3247,29 +3298,27 @@ class ChatSession: my_generation: int = 0, carry_spill: bool = False, ) -> bool: - """Emit the auto-compaction notice, compact, and refresh the status - line. Shared by the mid-turn policy (:meth:`_maybe_compact_midturn`) - and the end-of-turn check so the notice wording, the percentage, and - the post-compaction status refresh stay in lockstep. ``where`` is an - optional qualifier for the notice (e.g. ``"mid-turn"``); ``preserve_tail`` + """Run an auto-compaction and refresh the status line. Shared by the + mid-turn policy (:meth:`_maybe_compact_midturn`) and the end-of-turn + check so the trigger wording, the percentage, and the post-compaction + status refresh stay in lockstep. The auto notice itself rides the + ``on_compaction`` start event (via ``where``/``pct``). ``where`` is an + optional qualifier for that notice (e.g. ``"mid-turn"``); ``preserve_tail`` is forwarded to :meth:`_compact_messages` (e.g. to keep an in-flight tool-call turn during compact-before-truncate); ``my_generation`` is forwarded so the message swap aborts if a newer send supersedes this one - mid-compaction (0 = the manual /compact path, which has no generation); + mid-compaction (the manual path claims its own via :meth:`compact_now`); ``carry_spill`` is forwarded so the end-of-turn site can copy the model's wind-down turn onto the summary verbatim. Returns whether a summary was actually produced (False if compaction bailed) so callers can avoid acting on a compaction that did not happen.""" - qualifier = f" {where}" if where else "" - pct_display = round(self.auto_compact_pct * 100) - self.ui.on_info( - f"\n[Auto-compacting{qualifier}: prompt exceeds {pct_display}% of context window]" - ) compacted = self._compact_messages( auto=True, preserve_tail=preserve_tail, my_generation=my_generation, carry_spill=carry_spill, + where=where, + threshold_pct=round(self.auto_compact_pct * 100), ) self._print_status_line() return compacted @@ -4826,6 +4875,7 @@ class ChatSession: max_tokens: int = 4096, temperature: float | None = None, reasoning_effort: str | None = None, + cancel_ref: list[Any] | None = None, ) -> ModelTurnResult: """Run a lightweight internal completion (title gen, compaction, extraction) through ``model_turn`` on the session's primary lane. @@ -4867,6 +4917,13 @@ class ChatSession: max_tokens=clamped, temperature=self.temperature if temperature is None else temperature, reasoning_effort=reasoning_effort, + # The abort seam (default None): compaction passes a fresh + # per-attempt _CancelRef so a user Stop closes the in-flight + # summary HTTP stream instead of waiting it out. Title-gen + # keeps None (not user-cancellable), and web-fetch extraction + # MUST keep None — it runs on parallel tool threads and a + # registration would clobber the main stream's _cancel_stream. + cancel_ref=cancel_ref, ) # Utility completions (title gen, compaction, web-fetch extraction) # bypass the streaming on_status path — record their usage so the @@ -5264,7 +5321,9 @@ class ChatSession: last_err = e delay = self._RETRY_BASE_DELAY * (2**attempt) self.ui.on_info(f"[Retrying in {delay:.0f}s: {ename}]") - time.sleep(delay) + # Cancel-aware backoff (event arm only — no generation in + # scope here, matching the loop-top _check_cancelled()). + self._backoff_or_cancelled(delay) assert last_err is not None # unreachable, but satisfies type checker raise last_err @@ -5308,6 +5367,65 @@ class ChatSession: if my_generation and my_generation != self._generation: raise GenerationCancelled() + def _claim_generation(self) -> int: + """Claim the next generation and install its fresh cancel event. + + The entry half of the per-generation cancel discipline shared by + :meth:`send` and :meth:`compact_now`. The old event object stays + set for any abandoned thread — ``_exec_bash`` captures a local + reference so subprocesses from old generations are still killed. + + The claim is also the exact moment any prior compaction becomes + provably stale (the worker slot serializes turns), so the UI is + told to break a stale compaction activity latch here — otherwise a + force-abandoned compaction's latch suppresses the whole successor + turn's pill writes, re-broadcasting "Compacting context…" through + a live turn. getattr-guarded like ``on_aux_usage``: minimal UI + stubs predate the hook and must not crash a claim. + """ + self._generation += 1 + self._cancel_event = threading.Event() + release = getattr(self.ui, "on_generation_claimed", None) + if release is not None: + release(self._generation) + return self._generation + + def _consume_cancel(self, my_generation: int) -> bool: + """Clear this generation's cancel signal on exit; report if one landed. + + The exit half of the per-generation discipline: a cancel that + targeted THIS generation must not leak into a later idle operation. + Only acts while still the active generation — a successor owns a + fresh event that must not be cleared from under it. Returns + whether a cancel had been requested (set-but-unraised), so a + caller whose body completed anyway can still honor the stop. + """ + if self._generation != my_generation: + return False + landed = self._cancel_event.is_set() + self._cancel_event.clear() + return landed + + def _backoff_or_cancelled(self, delay: float, my_generation: int = 0) -> None: + """Sleep out a retry backoff, aborting the instant a Stop lands. + + Waits on the CURRENT cancel event rather than ``time.sleep`` so a + cancel during a (possibly minutes-long exponential) backoff aborts + immediately instead of burning the delay plus one more model call. + Two arms, both required: the event object is replaced per + generation, so a wait on a superseded generation's old event can + time out without ever seeing the successor's cancel — the + ``_check_cancelled`` re-check catches that via its generation arm. + ``my_generation=0`` callers keep event-arm-only semantics (same + default as ``_check_cancelled``). Raises + :class:`GenerationCancelled` — the shared shape for EVERY retry + backoff on this class: a hand-rolled ``time.sleep`` backoff is + Stop-blind and burns the full delay plus one more model call. + """ + if self._cancel_event.wait(delay): + raise GenerationCancelled() from None + self._check_cancelled(my_generation) + def _append_user_turn( self, user_input: str, @@ -5734,12 +5852,7 @@ class ChatSession: # would otherwise leave the latch set on the long-lived session and # trip a premature, advisory-skipping compaction on the next send. self._compaction_advised = False - self._generation += 1 - my_generation = self._generation - # Fresh cancel event per generation. The old event object stays - # set for any abandoned thread — _exec_bash captures a local - # reference so subprocesses from old generations are still killed. - self._cancel_event = threading.Event() + my_generation = self._claim_generation() self._cancelled_partial_msg = None # Fresh per-send attachment wire-part memo (see __init__): bounds the # heavy rasterized-page parts to one send and picks up any mid-session @@ -6333,12 +6446,10 @@ class ChatSession: # Consume this generation's cancel signal on exit so a cancel that # targeted THIS send can't later abort an unrelated idle operation # (e.g. a manual /compact between sends would otherwise inherit the - # still-set event). Only when still the active generation — a newer - # send owns a fresh event we must not clear out from under it. This is - # why a manual /compact no longer needs to reset the event itself - # (which would have disarmed a cancel aimed at a concurrent send). - if self._generation == my_generation: - self._cancel_event.clear() + # still-set event). A late-landing cancel needs no extra handling + # here — send's exits never auto-run further work (queued messages + # are flushed, not answered), unlike compact_now's. + self._consume_cancel(my_generation) def _drain_pending_advisories(self) -> None: """Drop the abandoned generation's advisory nudges — not external events. @@ -7438,7 +7549,7 @@ class ChatSession: batches.append(current) return batches - def _summarize_once(self, system_prompt: str, body: str) -> str: + def _summarize_once(self, system_prompt: str, body: str, my_generation: int = 0) -> str: """Run one summary completion over ``body`` and return the cleaned text. Owns the retry loop (transient errors only, exponential backoff) and the @@ -7455,25 +7566,51 @@ class ChatSession: result = self._utility_completion( summary_msgs, max_tokens=self._summary_output_tokens(), + # Fresh per-attempt abort seam: _CancelRef.append + # registers the summary HTTP stream in _cancel_stream + # eagerly (and closes on arrival if a Stop already + # landed), so cancel() aborts the blocked read instead + # of waiting out a whole model call — the force-stop + # orphan window collapses from one summary call to the + # next checkpoint. A fresh instance (never the shared + # self._cancel_ref) keeps the main loop's per-attempt + # clear and [0]-fallback semantics untouched, and the + # boundary checks in _summarize_batch/_backoff guarantee + # a superseded compaction makes no further calls — so + # it can never clobber a successor's registration. + cancel_ref=_CancelRef(self, my_generation), ) break except Exception as e: + # A closed-by-cancel stream surfaces as a provider error — + # map it to the cancel BEFORE the retry policy reads it, so + # a Stop on the final attempt ends the compaction as + # cancelled, never as a red "Compaction failed" (the main + # drain path makes the same closed-stream→GenerationCancelled + # translation). + self._check_cancelled(my_generation) ename = type(e).__name__ if self._stop_retrying(e, attempt, self._provider): # Overflow is deterministic — let _summarize_batch subdivide # instead of retrying an identical oversized call. raise delay = self._RETRY_BASE_DELAY * (2**attempt) - self.ui.on_info(f"[Compact retrying in {delay:.0f}s: {ename}]") - time.sleep(delay) + self._compaction_event( + my_generation, {"phase": "progress", "retry_in": delay, "error": ename} + ) + self._backoff_or_cancelled(delay, my_generation) assert result is not None # Strip any / tags the summarizer may emit summary = self._strip_reasoning(result.content or "") if result.finish_reason == "length": - self.ui.on_info("[Warning: compaction summary was truncated]") + self._compaction_event( + my_generation, {"phase": "progress", "warning": "summary_truncated"} + ) return summary - def _summarize_blocks(self, blocks: list[str], *, depth: int = 0) -> str: + def _summarize_blocks( + self, blocks: list[str], *, depth: int = 0, my_generation: int = 0 + ) -> str: """Summarize ``blocks`` into one dense summary, chunking + recursing so no single model call exceeds the model window. @@ -7502,7 +7639,7 @@ class ChatSession: raise _CompactionIrreducibleError batches = self._pack_blocks(blocks, self._summary_input_budget_chars()) if len(batches) == 1: - return self._summarize_batch(system_prompt, batches[0], depth) + return self._summarize_batch(system_prompt, batches[0], depth, my_generation) # More than one batch: recurse-merge the per-batch summaries. A block-count # guard would be wrong here — _summarize_batch's binary subdivision can @@ -7511,11 +7648,18 @@ class ChatSession: total = len(batches) summaries: list[str] = [] for k, batch in enumerate(batches, start=1): - self.ui.on_info(f"[compacting part {k}/{total}…]") - summaries.append(self._summarize_batch(system_prompt, batch, depth)) - return self._summarize_blocks(summaries, depth=depth + 1) + # depth 0 = summarizing transcript batches; depth > 0 = merging + # partial summaries. The web card renders depth 0 as a + # determinate part-k-of-N bar and deeper levels as a merge note. + self._compaction_event( + my_generation, {"phase": "progress", "part": k, "total": total, "depth": depth} + ) + summaries.append(self._summarize_batch(system_prompt, batch, depth, my_generation)) + return self._summarize_blocks(summaries, depth=depth + 1, my_generation=my_generation) - def _summarize_batch(self, system_prompt: str, batch: list[str], depth: int) -> str: + def _summarize_batch( + self, system_prompt: str, batch: list[str], depth: int, my_generation: int = 0 + ) -> str: """Summarize one packed batch, subdividing on a token-window overflow. The char budget that produced ``batch`` is only an estimate, so the model @@ -7533,10 +7677,13 @@ class ChatSession: # Cooperative cancellation: a cancel mid-compaction aborts here. It raises # GenerationCancelled (a BaseException), so _compact_messages' ``except # Exception`` can't swallow it and the message-swap below never runs — the - # history is left untouched. - self._check_cancelled() + # history is left untouched. my_generation matters for the force-cancel + # orphan: a successor's claim REPLACES the cancel event, so the event arm + # alone would let an abandoned compaction keep issuing summary calls — + # the generation arm is what retires it at the next batch boundary. + self._check_cancelled(my_generation) try: - return self._summarize_once(system_prompt, "\n\n".join(batch)) + return self._summarize_once(system_prompt, "\n\n".join(batch), my_generation) except Exception as e: if not _is_ctx_overflow(e): raise @@ -7547,9 +7694,11 @@ class ChatSession: # as fits — a wide over-window batch costs ~log2(N) calls, not one # model call per block. mid = len(batch) // 2 - left = self._summarize_batch(system_prompt, batch[:mid], depth) - right = self._summarize_batch(system_prompt, batch[mid:], depth) - return self._summarize_blocks([left, right], depth=depth + 1) + left = self._summarize_batch(system_prompt, batch[:mid], depth, my_generation) + right = self._summarize_batch(system_prompt, batch[mid:], depth, my_generation) + return self._summarize_blocks( + [left, right], depth=depth + 1, my_generation=my_generation + ) # A lone block overflows by itself: the char budget over-estimated how # many tokens it holds. Shrink progressively — halve the truncation # budget and retry, keeping as much of the message as the real window @@ -7561,7 +7710,7 @@ class ChatSession: while True: try: return self._summarize_once( - system_prompt, self._truncate_block(batch[0], budget) + system_prompt, self._truncate_block(batch[0], budget), my_generation ) except Exception as e2: if not _is_ctx_overflow(e2): @@ -7576,6 +7725,134 @@ class ChatSession: preserve_tail: int = 0, my_generation: int = 0, carry_spill: bool = False, + where: str = "", + threshold_pct: int | None = None, + ) -> bool: + """Compact conversation history — the lifecycle-event wrapper. + + Owns the cooperative-latch clear and the ``on_compaction`` lifecycle + contract: exactly one ``start`` event, then exactly one ``end`` event + on EVERY exit — the handled bails inside + :meth:`_compact_messages_impl` emit their own failed ``end`` (with a + per-site reason), and this wrapper backstops the raising exits + (``GenerationCancelled`` from a cancel mid-summary, unexpected + errors) so a UI that painted an in-progress card on ``start`` can + never be left with a stuck progress bar. ``where`` is the auto + trigger's qualifier (``"mid-turn"`` …) and ``threshold_pct`` the + auto-compact percentage — both ride the ``start`` event, and only + :meth:`_do_auto_compact` passes the pct: the context-overflow retry + path also compacts with ``auto=True`` but never evaluated the + threshold, so a pct there would fabricate a trigger explanation + contradicting the overflow notice printed a line above it. + """ + # Clear the cooperative latch on every compaction *attempt*, ahead of + # the early-return guards in the impl — a bailed compaction (too few/ + # large messages, summary error) must fall back to the advisory grace + # state next cycle rather than retry-storm on the same over-soft + # estimate. + self._compaction_advised = False + trigger = "auto" if auto else "manual" + start_payload: dict[str, Any] = {"phase": "start", "trigger": trigger} + if auto: + start_payload["where"] = where + if threshold_pct is not None: + start_payload["pct"] = threshold_pct + self._compaction_event(my_generation, start_payload) + try: + return self._compact_messages_impl(auto, preserve_tail, my_generation, carry_spill) + except BaseException as e: + # GenerationCancelled is a BaseException — the except above the + # message swap lets it propagate so history stays untouched; the + # UIs still need the end event to retire the in-progress card. + # Any OTHER non-Exception BaseException (KeyboardInterrupt at + # the CLI, SystemExit) is a deliberate abort too, not a + # compaction failure — report cancelled, never a red error row + # (str(KeyboardInterrupt()) is "" and would render "Compaction + # failed: "). + cancelled = isinstance(e, GenerationCancelled) or not isinstance(e, Exception) + message = "Compaction cancelled." if cancelled else f"Compaction failed: {e}" + # _compaction_bailed is the single failed-end emitter. The + # error notification is trigger-scoped: AUTO raising exits + # propagate into send()'s fatal handler, which fires the one + # on_error — emitting here too doubled the red rows and the + # error metric. MANUAL raising exits have no downstream + # emitter (the web compact worker's runner only logs; the CLI + # REPL suppresses below), so the wrapper's is the one red row. + self._compaction_bailed( + "cancelled" if cancelled else "error", + message, + trigger=trigger, + my_generation=my_generation, + emit_error=(trigger == "manual"), + ) + raise + + def _compaction_event(self, my_generation: int, payload: dict[str, Any]) -> int | None: + """Emit one compaction lifecycle event stamped with its owning id. + + ``compaction_id`` (the owning generation) ties every progress/end + event to the compaction that started it, and ``superseded`` marks + events from a generation that is no longer current (a + force-abandoned worker running out its last summary call). + :meth:`SessionUIBase.on_compaction ` + consumes ``superseded`` (never enqueued): a superseded event must + not drive the activity-pill restore or animate a successor's card. + ``my_generation`` is 0 on direct test invocations — falsy, so such + events are never marked superseded (matching ``_check_cancelled``). + """ + stale = bool(my_generation and my_generation != self._generation) + # getattr-guarded like on_generation_claimed/on_aux_usage: a + # duck-typed SessionUI predating the hook gets no compaction events + # rather than an AttributeError that wedges every long session at + # its first auto-compaction. + emit = getattr(self.ui, "on_compaction", None) + if emit is None: + return None + result = emit({"compaction_id": my_generation, "superseded": stale, **payload}) + # Duck-typed hooks aren't bound to the protocol's return type; the + # marker-stamp consumer needs int-or-None, nothing else. + return result if isinstance(result, int) else None + + def _compaction_bailed( + self, + reason: str, + message: str, + *, + trigger: str, + my_generation: int, + emit_error: bool = True, + ) -> bool: + """Emit a failed-compaction ``end`` event and return ``False``. + + The single failed-end emitter: :meth:`_compact_messages_impl`'s + handled bails AND the wrapper's raising backstop both route here — + each carries a machine ``reason`` (drives the web card's failure + state) and the human ``message``. ``reason="error"`` + additionally fires :meth:`on_error` — the typed error event and the + Prometheus error counter this path fed before the lifecycle events + existed; the panes render the red row from THAT and treat the end + event as card-teardown only, so the text isn't shown twice. + + ``emit_error=False`` is the RAISING backstop's auto-trigger arm: + those exceptions propagate into ``send()``'s fatal handler, whose + ``_record_fatal_error`` fires the single on_error — a second one + here doubled every pane's red rows and the node's error metric. + Handled bails never propagate, so they always emit. + """ + if reason == "error" and emit_error: + self.ui.on_error(message) + self._compaction_event( + my_generation, + {"phase": "end", "ok": False, "reason": reason, "message": message, "trigger": trigger}, + ) + return False + + def _compact_messages_impl( + self, + auto: bool = False, + preserve_tail: int = 0, + my_generation: int = 0, + carry_spill: bool = False, ) -> bool: """Compact conversation history by summarizing it into a summary turn. @@ -7603,14 +7880,16 @@ class ChatSession: the summarizer also reads the spill, but its paraphrase must not be the only survivor. """ - # Clear the cooperative latch on every compaction *attempt*, ahead of - # the early-return guards below — a bailed compaction (too few/large - # messages, summary error) must fall back to the advisory grace state - # next cycle rather than retry-storm on the same over-soft estimate. - self._compaction_advised = False + # Presentation label derived from the one semantic flag — deriving + # locally makes an auto=True/trigger="manual" drift impossible. + trigger = "auto" if auto else "manual" if len(self.messages) < 2: - self.ui.on_info("Not enough messages to compact.") - return False + return self._compaction_bailed( + "not_enough_messages", + "Not enough messages to compact.", + trigger=trigger, + my_generation=my_generation, + ) # Optionally keep the last ``preserve_tail`` messages verbatim — e.g. an # in-flight assistant tool-call whose results are about to be appended, or @@ -7645,24 +7924,37 @@ class ChatSession: to_summarize_dicts = dicts_from_turns(to_summarize) blocks = self._summary_blocks(to_summarize_dicts) if not blocks: - self.ui.on_info("Not enough messages to compact.") - return False + return self._compaction_bailed( + "not_enough_messages", + "Not enough messages to compact.", + trigger=trigger, + my_generation=my_generation, + ) self.ui.on_thinking_start() try: - summary = self._summarize_blocks(blocks) + summary = self._summarize_blocks(blocks, my_generation=my_generation) except _CompactionIrreducibleError: - self.ui.on_info("Messages too large to fit in summary context.") - return False + return self._compaction_bailed( + "irreducible", + "Messages too large to fit in summary context.", + trigger=trigger, + my_generation=my_generation, + ) except Exception as e: - self.ui.on_error(f"Compaction failed: {e}") - return False + return self._compaction_bailed( + "error", f"Compaction failed: {e}", trigger=trigger, my_generation=my_generation + ) finally: self.ui.on_thinking_stop() if not summary.strip(): - self.ui.on_info("Compaction produced an empty summary; keeping history.") - return False + return self._compaction_bailed( + "empty_summary", + "Compaction produced an empty summary; keeping history.", + trigger=trigger, + my_generation=my_generation, + ) # The verbatim carries: the wind-down spill and the continuation-hint # quote of the ask. Both can fire on the SAME compaction (the @@ -7772,14 +8064,20 @@ class ChatSession: "total_tokens": after_tokens, } - self.ui.on_info(f"[compacted: ~{before_tokens:,} -> ~{after_tokens:,} tokens]") - separator = "\u2500" * 60 - lines = [separator] - for line in summary.splitlines(): - lines.append(f" {line}") - lines.append(separator) - self.ui.on_info("\n".join(lines)) - + # The successful end event carries everything a UI needs to paint the + # result card (token delta + the summary text); its id stamps the + # marker row below so /history and the live stream stay aligned. + end_event_id = self._compaction_event( + my_generation, + { + "phase": "end", + "ok": True, + "trigger": trigger, + "before_tokens": before_tokens, + "after_tokens": after_tokens, + "summary": summary, + }, + ) # Persist a compaction checkpoint so a reopen rehydrates [summary]+[tail] # instead of the full transcript — which, on a long session or one switched # to a smaller-context model, can exceed the window and deadlock the first @@ -7792,13 +8090,27 @@ class ChatSession: if self._ws_id: watermark = get_compaction_watermark(self._ws_id, preserve_tail) if watermark is not None: + # ``before_tokens``/``after_tokens``/``trigger`` are display + # additions for the /history compaction card; the resume + # slice reads only ``watermark`` (parse_checkpoint_watermark + # ignores the extra keys). The row is stamped with the end + # event's id so a fresh-connect cursor computed from /history + # sits at-or-past the live event — repaint and replay can't + # both render the card. save_message( self._ws_id, "assistant", summary, source=COMPACTION_SOURCE, - meta=json.dumps({"watermark": watermark}), - event_id=self._ui_event_id(), + meta=json.dumps( + { + "watermark": watermark, + "before_tokens": before_tokens, + "after_tokens": after_tokens, + "trigger": trigger, + } + ), + event_id=end_event_id if end_event_id is not None else self._ui_event_id(), producer=self._provider.provider_name if self._provider else None, ) return True @@ -8724,6 +9036,79 @@ class ChatSession: popped = self._queued_messages.pop(msg_id, None) return popped is not None + def flush_queued_messages(self) -> bool: + """Drain queued messages into a combined user turn (public seam). + + Defensive backstop for workers that can find text stranded in the + queue from BEFORE their window (a dying send worker's closing race) + — the /compact worker's exit seam is the caller. Messages sent + DURING a command window never reach this queue: the /send route + parks them while ``worker_kind == "command"`` and dispatches them + as ordinary sends afterwards. Must only be called by the thread + that owns the worker slot — it mutates ``self.messages``. + """ + return self._flush_queued_messages() + + @staticmethod + def _combine_queued_items(items: list[tuple[str, str]]) -> str: + """Render drained queue items to one text block ([IMPORTANT] framing).""" + from turnstone.core.tool_advisory import PRIORITY_IMPORTANT + + return "\n\n".join( + f"[IMPORTANT] {msg}" if pri == PRIORITY_IMPORTANT else msg for msg, pri in items + ) + + def compact_now(self) -> bool: + """Manual compaction with send()'s full generation discipline. + + The web /compact worker path. Mirrors send()'s entry — claim the + next generation and install a fresh cancel event — so: + + * an abandoned (force-cancelled) compaction can never swap history + under a successor turn: the successor's claim makes this + generation stale and the pre-swap ``_check_cancelled`` raises + (send() prevents the identical race the identical way); + * a cancel that landed while idle (a Stop click between turns) + can't instantly abort the next /compact — the stale set event is + replaced here, exactly as send() replaces it per generation; + * a cancel aimed at THIS compaction is consumed on exit (while + still the active generation), so it can't leak into a later idle + operation — previously nothing cleared it until the next send, + which bricked every /compact retry as instantly \"cancelled\". + + Raises :class:`GenerationCancelled` (after consuming the event) so + the caller can distinguish a user stop from a bail; returns whether + a summary was produced otherwise. That raise ALSO covers a Stop + that lands after the impl's last cancel check (the swap / + marker-persist tail) or during a retry backoff that then bails: + the compaction outcome stands, but an explicit Stop must never be + silently eaten — the caller must not auto-run anything on the + user's behalf after one. The CLI's ``handle_command`` route + delegates here too — the REPL is single-threaded so the discipline + is redundant there, but one path means one behaviour. + """ + my_generation = self._claim_generation() + cancel_landed = False + try: + compacted = self._compact_messages(my_generation=my_generation) + finally: + # Consume this generation's cancel signal (send()'s exit does + # the same). A body raise propagates past this; the landed + # flag matters only on the completed-anyway paths below. + cancel_landed = self._consume_cancel(my_generation) + if compacted: + # Refresh the status line/context pill so the freed window is + # visible immediately — parity with _do_auto_compact. BEFORE + # honoring a tail-landing Stop: the compaction genuinely + # happened (swap + marker + OK card), so the pill must reflect + # it either way — the raise below only suppresses follow-up + # work, it must not leave the pill claiming a full context + # next to a "context compacted" card. + self._print_status_line() + if cancel_landed: + raise GenerationCancelled() + return compacted + def _flush_queued_messages(self, prefix: str = "") -> bool: """Drain queued messages into a single combined user turn. @@ -8743,21 +9128,19 @@ class ChatSession: Returns ``True`` when any user row was appended (prefix or items), ``False`` when both were empty. """ - from turnstone.core.tool_advisory import PRIORITY_IMPORTANT - with self._queued_lock: items = list(self._queued_messages.values()) self._queued_messages.clear() - if not items and not prefix: + queued_text = self._combine_queued_items(items) + if not queued_text and not prefix: return False - parts = [f"[IMPORTANT] {msg}" if pri == PRIORITY_IMPORTANT else msg for msg, pri in items] - if prefix and parts: - content = prefix + "\n\n" + "\n\n".join(parts) + if prefix and queued_text: + content = prefix + "\n\n" + queued_text elif prefix: content = prefix else: - content = "\n\n".join(parts) + content = queued_text self._append_user_turn(content, ()) return True @@ -15212,7 +15595,9 @@ class ChatSession: last_err = e delay = self._RETRY_BASE_DELAY * (2**attempt) self.ui.on_info(f"[{label} retrying in {delay:.0f}s: {ename}]") - time.sleep(delay) + # Cancel-aware backoff: a Stop mid-agent-retry aborts + # the run instead of burning the delay + one more call. + self._backoff_or_cancelled(delay) assert last_err is not None # unreachable raise last_err @@ -16174,7 +16559,10 @@ class ChatSession: max_retries=self._NOTIFY_MAX_RETRIES, retry_delay=delay, ) - time.sleep(delay) + # Cancel-aware: notify runs as an in-turn tool, so a + # Stop aborts pending delivery retries with the turn + # (the batch synthesizes the cancelled tool_result). + self._backoff_or_cancelled(delay) continue log.warning("notify.no_services_exhausted") msg = "Error: no channel gateway services available" @@ -16223,7 +16611,8 @@ class ChatSession: gateway_count=len(services), retry_delay=delay, ) - time.sleep(delay) + # Same cancel-aware backoff as the no-services arm above. + self._backoff_or_cancelled(delay) else: log.warning( "notify.delivery_failed", @@ -16956,6 +17345,12 @@ class ChatSession: elif cmd == "/new": from turnstone.core.memory import register_workstream + # Flush any stranded queued text BEFORE the identity swap so it + # is persisted into the workstream it was ADDRESSED to. Sends + # during the command window itself park in the /send route and + # never queue; this covers only a message stranded by a dying + # send worker's closing race before this command started. + self._flush_queued_messages() self.messages.clear() self._read_files.clear() self._repeat_detector.clear() @@ -17018,6 +17413,10 @@ class ChatSession: elif target_id == self._ws_id: self.ui.on_info("Already in that workstream.") else: + # Same pre-swap flush as /new: stranded queued text is + # persisted into the CURRENT workstream before resume() + # swaps this session's identity to the target. + self._flush_queued_messages() try: resumed: bool | None = self.resume(target_id) except ValueError as exc: @@ -17168,13 +17567,17 @@ class ChatSession: self.ui.on_info(f"Invalid. Choose from: {', '.join(valid)}") elif cmd == "/compact": - try: - self._compact_messages() - except GenerationCancelled: - # Ctrl-C during a manual compaction aborts cleanly — the message - # swap never ran (the cancel-check precedes it), so history is - # intact, exactly like cancelling a send. - self.ui.on_info("Compaction cancelled.") + # Ctrl-C during a manual compaction aborts cleanly — the message + # swap never ran (the cancel-check precedes it), so history is + # intact, exactly like cancelling a send. The lifecycle wrapper + # already emitted the cancelled end event, so every UI has been + # told; nothing more to print here. Unexpected errors are also + # swallowed: the wrapper's manual-trigger backstop already fired + # on_error (the red row), and the CLI REPL calls handle_command + # with no try/except — re-raising would crash the whole REPL on + # a compaction failure (history is untouched on raising exits). + with contextlib.suppress(GenerationCancelled, Exception): + self.compact_now() elif cmd == "/creative": # Recognized but decommissioned: print a live migration pointer diff --git a/turnstone/core/session_routes.py b/turnstone/core/session_routes.py index 36853e94..08c317f8 100644 --- a/turnstone/core/session_routes.py +++ b/turnstone/core/session_routes.py @@ -48,6 +48,7 @@ if TYPE_CHECKING: from starlette.routing import BaseRoute from turnstone.core.attachments import UploadRejection + from turnstone.core.session import ChatSession from turnstone.core.session_manager import SessionManager from turnstone.core.session_ui_base import SessionUIBase from turnstone.core.workstream import Workstream, WorkstreamKind @@ -1365,6 +1366,18 @@ def make_cancel_handler( # ``session_worker.send`` documents this invariant: # "readers gating on either flag see a coherent # (worker_thread, _worker_running) pair." + # + # Documented bet — force-cancelling a wedged QUICK command + # (worker_kind == "command", e.g. /resume stuck in storage + # I/O): clearing the flag releases any parked /send, whose + # fresh worker can then run while the abandoned command + # thread finishes its in-place mutation — quick commands + # have no generation checkpoints to retire them (compact_now + # does). Same blast radius as force-abandoning a send + # worker mid-tool; accepted because force-cancel is the + # operator escape hatch for an already-wedged session, not a + # routine path. Revisit if commands ever gain generation + # discipline. with ws._lock: ws.worker_thread = None ws._worker_running = False @@ -3444,9 +3457,17 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler: cursor: int | None = None if storage is not None: try: - # repair=False — display read; see reconstruct_messages docstring. + # repair=False — display read; include_compaction=True so a + # persisted compaction marker projects as an in-place + # source="compaction" system row and the UI re-renders its + # compaction card after a reload. See the + # reconstruct_messages docstring for both flags. messages = await asyncio.to_thread( - storage.load_messages, ws_id, limit=limit, repair=False + storage.load_messages, + ws_id, + limit=limit, + repair=False, + include_compaction=True, ) except Exception: log.debug("ws.history.load_failed ws=%s", ws_id[:8], exc_info=True) @@ -4072,32 +4093,96 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: if ws.session is None: return JSONResponse({"error": "No session"}, status_code=500) - session = ws.session - # Captured by ``_enqueue`` only when the dispatcher takes the - # live-worker reuse path. Empty after a fresh-spawn dispatch. - queue_outcome: dict[str, Any] = {} + def _dispatch_once(session: ChatSession) -> tuple[bool, dict[str, Any]]: + """One atomic queue-or-spawn attempt bound to ONE session capture. - def _enqueue() -> None: - try: - cleaned, priority, msg_id = session.queue_message( - message, - attachment_ids=list(ordered_taken), - queue_msg_id=send_id or None, - interjector_user_id=acting_uid, - ) - except AttachmentsNotQueueableError: - queue_outcome["rejected"] = "attachments_busy" - return - except CrossUserInterjectionError: - # A different authenticated participant tried to interject into - # someone else's in-flight turn; folding it in would borrow the - # initiator's credentials and misattribute the message. Reject - # so they resend as a fresh turn once the worker idles. - queue_outcome["rejected"] = "cross_user_interjection" - return - queue_outcome["cleaned"] = cleaned - queue_outcome["priority"] = priority - queue_outcome["msg_id"] = msg_id + The park loop below re-captures ``ws.session`` before every + attempt — a /resume or /new that ran while we parked swaps the + session's workstream identity in place, and closures bound to + the pre-swap capture would send the user's message into the + wrong workstream's transcript. Binding happens here, in one + place, per attempt. ``queue_outcome`` is written only when the + dispatcher takes the live-worker reuse path; empty after a + fresh-spawn dispatch. + """ + queue_outcome: dict[str, Any] = {} + + def _enqueue() -> None: + # Runs under ``ws._lock`` (session_worker.send calls it + # inside the same acquisition that reads _worker_running), + # so this worker_kind read cannot race the spawn write. A + # command window must NEVER reach queue_message — its cap + # and cross-user guard are turn semantics — so refuse and + # let the route loop back to the park. + if ws.worker_kind == "command": + queue_outcome["rejected"] = "command_window" + return + try: + cleaned, priority, msg_id = session.queue_message( + message, + attachment_ids=list(ordered_taken), + queue_msg_id=send_id or None, + interjector_user_id=acting_uid, + ) + except AttachmentsNotQueueableError: + queue_outcome["rejected"] = "attachments_busy" + return + except CrossUserInterjectionError: + # A different authenticated participant tried to interject into + # someone else's in-flight turn; folding it in would borrow the + # initiator's credentials and misattribute the message. Reject + # so they resend as a fresh turn once the worker idles. + queue_outcome["rejected"] = "cross_user_interjection" + return + queue_outcome["cleaned"] = cleaned + queue_outcome["priority"] = priority + queue_outcome["msg_id"] = msg_id + + def _run() -> None: + me = threading.current_thread() + try: + kwargs: dict[str, Any] = {} + if resolved_atts: + kwargs["attachments"] = resolved_atts + if send_id: + kwargs["send_id"] = send_id + # Fresh turn: rebind per-user MCP credentials to the + # authenticated sender. Bound here (not via a send() + # kwarg) so per-kind session stubs with explicit send + # signatures keep working; getattr-guarded for the same + # reason. The queue path above never rebinds. + bind = getattr(session, "bind_acting_user", None) + if acting_uid and callable(bind): + bind(acting_uid) + session.send(message, **kwargs) + except GenerationCancelled: + # Safety net — send() normally handles this internally. + # If this thread was force-abandoned, ws.worker_thread + # was set to None — don't emit spurious events. + if ws.worker_thread is me: + _emit_ui("on_stream_end") + _emit_ui("on_state_change", "idle") + except Exception: + # Undrained staged uploads aren't locked (the buffer is a peek, + # not a reservation) — they expire on the buffer TTL — so the + # only cleanup owed here is the UI streaming hook. + if ws.worker_thread is me: + # ``session.send()`` already fired ``on_error`` + # (with sanitized text), persisted ``last_error``, + # and emitted ``state='error'`` via + # :meth:`ChatSession._record_fatal_error` before + # re-raising. The route handler only needs the + # streaming-cleanup hook the worker contract owes + # the UI listeners. + _emit_ui("on_stream_end") + + ok = session_worker.send( + ws, + enqueue=_enqueue, + run=_run, + thread_name=f"send-worker-{ws.id[:8]}", + ) + return ok, queue_outcome def _emit_ui(hook_name: str, *args: Any) -> None: """Best-effort UI hook dispatch. @@ -4105,7 +4190,9 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: Each call is wrapped in try/except so a failure in one hook (e.g. listener-queue full → on_error raises) doesn't suppress the others. Mirrors the pre-P1.5 - coord_adapter.send per-hook defense. + coord_adapter.send per-hook defense. Defined after + ``_dispatch_once`` but resolved late (when a worker runs) — + every dispatch happens strictly below this line. """ if ui is None: return @@ -4122,50 +4209,48 @@ def make_send_handler(cfg: SessionEndpointConfig) -> Handler: exc_info=True, ) - def _run() -> None: - me = threading.current_thread() - try: - kwargs: dict[str, Any] = {} - if resolved_atts: - kwargs["attachments"] = resolved_atts - if send_id: - kwargs["send_id"] = send_id - # Fresh turn: rebind per-user MCP credentials to the - # authenticated sender. Bound here (not via a send() - # kwarg) so per-kind session stubs with explicit send - # signatures keep working; getattr-guarded for the same - # reason. The queue path above never rebinds. - bind = getattr(session, "bind_acting_user", None) - if acting_uid and callable(bind): - bind(acting_uid) - session.send(message, **kwargs) - except GenerationCancelled: - # Safety net — send() normally handles this internally. - # If this thread was force-abandoned, ws.worker_thread - # was set to None — don't emit spurious events. - if ws.worker_thread is me: - _emit_ui("on_stream_end") - _emit_ui("on_state_change", "idle") - except Exception: - # Undrained staged uploads aren't locked (the buffer is a peek, - # not a reservation) — they expire on the buffer TTL — so the - # only cleanup owed here is the UI streaming hook. - if ws.worker_thread is me: - # ``session.send()`` already fired ``on_error`` - # (with sanitized text), persisted ``last_error``, - # and emitted ``state='error'`` via - # :meth:`ChatSession._record_fatal_error` before - # re-raising. The route handler only needs the - # streaming-cleanup hook the worker contract owes - # the UI listeners. - _emit_ui("on_stream_end") - - ok = session_worker.send( - ws, - enqueue=_enqueue, - run=_run, - thread_name=f"send-worker-{ws.id[:8]}", - ) + # Park-then-dispatch. While a slash-command worker holds the slot + # (a manual /compact can hold it for MINUTES), a send must not take + # the interjection-queue path — its 2000-char cap and cross-user + # guard are mid-TURN semantics, and a queued message would cross a + # /resume//new identity swap into the wrong workstream. Parking + # reproduces the pre-worker-dispatch observable behavior (the + # request waited out the then-blocked event loop, then ran as a + # normal full-fidelity send under the sender's own identity) with + # the loop free: SSE keeps streaming the compaction progress card + # while we wait. The inner park is deliberately unbounded — the + # composer bounds its POST at ~15s and aborts, and we poll + # ``is_disconnected`` because starlette does NOT cancel a running + # handler on client disconnect: dispatching after the client gave + # up would surprise the user with a duplicate turn when they + # resend. The outer bound only caps command-window FLAPPING + # (back-to-back commands re-claiming the slot between our park + # exit and dispatch). + ok = False + queue_outcome: dict[str, Any] = {} + for _ in range(20): + while ws._worker_running and ws.worker_kind == "command": + if await request.is_disconnected(): + # Client abandoned the send mid-park; the response is + # discarded — the status is for the access log only. + return JSONResponse({"status": "abandoned"}) + await asyncio.sleep(0.25) + session_now = ws.session + if session_now is None: + return JSONResponse({"error": "No session"}, status_code=500) + ok, queue_outcome = _dispatch_once(session_now) + if queue_outcome.get("rejected") != "command_window": + break + else: + # 20 consecutive command-window collisions — surface the same + # retryable backpressure shape as a saturated queue. + return JSONResponse( + { + "status": "queue_full", + "attached_ids": [], + "dropped_attachment_ids": list(requested_ids), + } + ) if not ok: if ws._closed: # ``send`` refused because the workstream closed between our diff --git a/turnstone/core/session_ui_base.py b/turnstone/core/session_ui_base.py index 9bfc9e5c..3a3868bf 100644 --- a/turnstone/core/session_ui_base.py +++ b/turnstone/core/session_ui_base.py @@ -587,6 +587,19 @@ class SessionUIBase: # Activity tracking for dashboard ("thinking" / "tool" / ""). self._ws_current_activity: str = "" self._ws_activity_state: str = "" + # Compaction pill window: while True, on_thinking_start leaves the + # activity alone (the impl's own thinking wrap would otherwise + # clobber "Compacting context…" with "Thinking…" milliseconds after + # the start event set it, for the whole summarize phase). The end + # event restores the pre-compaction pair, so a bail can't strand a + # stale "Compacting context…" on an idle workstream either. + self._compaction_activity_live: bool = False + self._pre_compaction_activity: tuple[str, str] = ("", "") + # Which compaction owns the latch (its ``compaction_id``): a + # force-abandoned compaction's LATE end event must not restore a + # stale pill pair over — or unlatch — a successor compaction that + # has since taken the latch over. + self._compaction_activity_owner: int = 0 # Turn-content accumulator: assistant tokens piggybacked onto # the ``ws_state:idle`` broadcast so the dashboard renders the # turn without an extra storage round-trip. Cleared on IDLE / @@ -2970,10 +2983,17 @@ class SessionUIBase: self._reset_inflight_buffers_locked() def on_thinking_start(self) -> None: - """Track that the model is thinking; broadcast activity + enqueue.""" + """Track that the model is thinking; broadcast activity + enqueue. + + Inside a compaction window the activity write is skipped — + :meth:`on_compaction` owns the pill there ("Compacting context…" + must survive the summarize phase's own thinking wrap); the + ``thinking_start`` event still flows for the spinner UIs. + """ with self._ws_lock: - self._ws_current_activity = "Thinking…" - self._ws_activity_state = "thinking" + if not self._compaction_activity_live: + self._ws_current_activity = "Thinking…" + self._ws_activity_state = "thinking" self._broadcast_activity() self._enqueue({"type": "thinking_start"}) @@ -3255,6 +3275,137 @@ class SessionUIBase: {"type": "system_turn", "content": content, "source": source, "meta": meta or None} ) + def on_compaction(self, payload: dict[str, Any]) -> int | None: + """Surface one compaction lifecycle event (``phase`` = start/progress/end). + + The transcript affordance for context compaction: ``start`` paints + the in-progress card, ``progress`` drives its bar (chunked + summarization emits ``part``/``total``/``depth``), and ``end`` + replaces it with the result card (or the failure notice). The + successful ``end`` event's id is returned so ``_compact_messages`` + stamps the persisted compaction marker row with the matching + resume cursor — the same live-event/history-row alignment + :meth:`on_system_turn` provides for operator turns, which is what + keeps a ``/history`` repaint and an SSE replay from double-rendering + the result card. + + Inside a task agent the events are dropped (same rule as + :meth:`on_info`): a sub-agent's compaction is progress chatter that + carries no ``call_id``, so it cannot nest under the task card and + must not paint a top-level compaction card on the pane. + + ``superseded`` (stamped by ``ChatSession._compaction_event``) marks + events from a force-abandoned compaction whose generation a + successor has since claimed. Its start/progress events are + swallowed — animating a card for a dead compaction, or re-creating + one via the panes' defensive-create, is pure corruption — but its + END flows WITH the flag on the wire: the pane needs the event to + retire the card the abandoned compaction painted while it was + live, and the flag to know its failure notice would narrate a + compaction nobody is waiting on (a superseded OK end still renders + the result card — the history swap really happened). On the latch + side a superseded end unlatches (so the live turn's thinking + writes resume) without restoring — its saved pair predates the + successor turn and would overwrite the live pill. + """ + if _agent_scope_var.get() > 0: + return None + payload = dict(payload) + superseded = bool(payload.pop("superseded", False)) + cid = int(payload.get("compaction_id") or 0) + phase = payload.get("phase") + if phase == "end": + # Ends carry the flag on the wire (typed in both SDKs); + # start/progress never do — the live ones are by definition + # not superseded and the stale ones are swallowed below. + payload["superseded"] = superseded + if phase == "start" and not superseded: + # The activity pill mirrors on_thinking_start's mechanics but + # names the actual work — the summarize calls can run for a + # while and "Thinking…" undersells what the session is doing. + # Save the prior pair for the end-side restore, and latch the + # window so the impl's own on_thinking_start can't clobber it. + with self._ws_lock: + if not self._compaction_activity_live: + # A stale (abandoned) compaction may still hold the + # latch — keep ITS saved pair as the restore target + # rather than capturing the "Compacting context…" it + # wrote. + self._pre_compaction_activity = ( + self._ws_current_activity, + self._ws_activity_state, + ) + self._ws_current_activity = "Compacting context…" + self._ws_activity_state = "thinking" + self._compaction_activity_live = True + self._compaction_activity_owner = cid + self._broadcast_activity() + elif phase == "end": + # Restore whatever the pill showed before the compaction — a + # mid-turn auto-compaction hands back to the send loop (whose + # next thinking/tool write overwrites anyway); a manual bail + # returns the idle session's blank pair instead of stranding + # "Compacting context…" on an idle workstream. Owner-gated: + # an abandoned compaction's late end must leave a successor's + # latch (and pill) alone. + changed = False + with self._ws_lock: + if self._compaction_activity_live and self._compaction_activity_owner == cid: + # A superseded owner-end only unlatches: its saved pair + # predates the successor turn and must not overwrite the + # live pill (the latch itself isn't in the broadcast + # snapshot, so unlatch-only is also broadcast-free). + changed = self._release_compaction_latch_locked(restore=not superseded) + if changed: + self._broadcast_activity() + if superseded and phase != "end": + return None + return self._enqueue({"type": "compaction", **payload}) + + def _release_compaction_latch_locked(self, *, restore: bool) -> bool: + """Unlatch the compaction pill window; optionally restore the pair. + + The shared release half of the latch invariant — called under + ``_ws_lock`` by the owner-gated ``end`` arm of :meth:`on_compaction` + (restore unless superseded) and by :meth:`on_generation_claimed` + (always restore). Returns whether the broadcast snapshot — the + ``(activity, state)`` pair — actually changed. + """ + self._compaction_activity_live = False + if restore: + ( + self._ws_current_activity, + self._ws_activity_state, + ) = self._pre_compaction_activity + return True + return False + + def on_generation_claimed(self, generation: int) -> None: + """Break a stale compaction activity latch at a new generation claim. + + Called by ``ChatSession._claim_generation`` (getattr-guarded, like + ``on_aux_usage``). The claim is the exact moment any live latch is + provably stale — the worker slot serializes turns, so a latch held + at claim time belongs to a force-abandoned compaction whose late + events may be a full summary call away. Without this, the whole + successor turn's ``on_thinking_start`` writes are suppressed and + the pill re-broadcasts "Compacting context…" over a live turn. + + Unlatch AND restore the saved pre-compaction pair: restoring keeps + ``_pre_compaction_activity`` coherent for a successor compaction's + own start (which saves the CURRENT pair as its restore target — a + leftover "Compacting context…" here would get re-stranded on an + idle workstream after that successor completes). A send claimer's + first thinking/tool write overwrites the restored pair within + milliseconds either way. + """ + changed = False + with self._ws_lock: + if self._compaction_activity_live and self._compaction_activity_owner != generation: + changed = self._release_compaction_latch_locked(restore=True) + if changed: + self._broadcast_activity() + # ------------------------------------------------------------------ # Broadcast hooks — kind-specific transport. # diff --git a/turnstone/core/session_worker.py b/turnstone/core/session_worker.py index 923b4ddd..b8d2e2cc 100644 --- a/turnstone/core/session_worker.py +++ b/turnstone/core/session_worker.py @@ -93,6 +93,7 @@ def send( enqueue: Callable[[], None], run: Callable[[], None], thread_name: str | None = None, + worker_kind: str = "turn", ) -> bool: """Dispatch work onto a workstream's worker thread. @@ -106,6 +107,31 @@ def send( ``ChatSession`` they want to drive, so the worker can't be racing a concurrent ``ws.session`` swap. + ``worker_kind`` classifies what the slot holds — ``"turn"`` (send / + retry / wake / init, the default) or ``"command"`` (slash-command + workers, including the minutes-long manual /compact). Written to + ``ws.worker_kind`` in the same lock acquisition as the + ``(worker_thread, _worker_running)`` pair. This is the ONLY site + that sets ``_worker_running=True``, so the classification cannot be + bypassed by a new dispatch caller. The /send route parks while a + command holds the slot instead of taking the interjection-queue + path (whose length cap / cross-user guard are turn semantics); an + ``enqueue`` callback that can fire during a command window (the + coordinator adapter's, the init race's) must refuse rather than + queue — see the command-window refusals at those closures. + + The refusal is DELIBERATELY not centralized here despite the three + hand-written guards: each surface needs a different refusal channel + (the /send route signals "re-park" via its ``queue_outcome`` flag; + the coordinator adapter and the init race raise ``queue.Full`` into + their existing backpressure statuses), and a central refusal inside + this function can only return ``False`` — indistinguishable from + queue-full/closed for the route's re-park decision and mislabeled by + the init path's status derivation. Making it distinguishable means + a tri-state contract change across every dispatch caller, which is + more surface than three four-line guards. If you add a NEW enqueue + closure that can queue turn work, copy the guard. + Returns: ``True`` on successful enqueue (existing worker accepted) or thread spawn (no live worker). @@ -196,6 +222,7 @@ def send( # lock-window cost is dominated by the spawn branch's identity # write either way. ws._worker_running = True + ws.worker_kind = worker_kind t = threading.Thread(target=_runner, name=name, daemon=True) ws.worker_thread = t # ``t.start()`` may run user code (worker body) before returning; diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index d4da1136..cb4e2f59 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -480,10 +480,17 @@ class PostgreSQLBackend: return msg_rows, (attachments or None) def load_messages( - self, ws_id: str, *, limit: int | None = None, repair: bool = True + self, + ws_id: str, + *, + limit: int | None = None, + repair: bool = True, + include_compaction: bool = False, ) -> list[dict[str, Any]]: msg_rows, attachments = self._conversation_rows(ws_id, limit) - return _reconstruct_messages(msg_rows, ws_id, attachments, repair=repair) + return _reconstruct_messages( + msg_rows, ws_id, attachments, repair=repair, include_compaction=include_compaction + ) def load_message_turns(self, ws_id: str, *, checkpointed: bool = True) -> list[Turn]: """Load the conversation as canonical ``Turn``s (unresolved AttachmentRef) diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 7a0429a5..6c875e64 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -227,7 +227,12 @@ class StorageBackend(Protocol): ... def load_messages( - self, ws_id: str, *, limit: int | None = None, repair: bool = True + self, + ws_id: str, + *, + limit: int | None = None, + repair: bool = True, + include_compaction: bool = False, ) -> list[dict[str, Any]]: """Load messages for a workstream and reconstruct OpenAI message format. @@ -251,6 +256,13 @@ class StorageBackend(Protocol): Attachments are resolved to inline content parts (the materialized bytes a display/export consumer needs); :meth:`load_message_turns` is the unresolved, by-reference counterpart for resume. + + ``include_compaction`` (default False) surfaces persisted compaction + checkpoint markers as in-place ``role="system"`` display rows + (``_source="compaction"``, ``meta`` = watermark/token counts) instead + of dropping them — the ``/history`` display path passes True so the + UI can re-render its compaction card after a reload; export/search + keep the unannotated transcript. """ ... diff --git a/turnstone/core/storage/_sqlite.py b/turnstone/core/storage/_sqlite.py index 04f7e93e..99fe052d 100644 --- a/turnstone/core/storage/_sqlite.py +++ b/turnstone/core/storage/_sqlite.py @@ -538,10 +538,17 @@ class SQLiteBackend: return msg_rows, (attachments or None) def load_messages( - self, ws_id: str, *, limit: int | None = None, repair: bool = True + self, + ws_id: str, + *, + limit: int | None = None, + repair: bool = True, + include_compaction: bool = False, ) -> list[dict[str, Any]]: msg_rows, attachments = self._conversation_rows(ws_id, limit) - return _reconstruct_messages(msg_rows, ws_id, attachments, repair=repair) + return _reconstruct_messages( + msg_rows, ws_id, attachments, repair=repair, include_compaction=include_compaction + ) def load_message_turns(self, ws_id: str, *, checkpointed: bool = True) -> list[Turn]: """Load the conversation as canonical ``Turn``s (unresolved AttachmentRef). diff --git a/turnstone/core/storage/_utils.py b/turnstone/core/storage/_utils.py index 112e0946..62f54378 100644 --- a/turnstone/core/storage/_utils.py +++ b/turnstone/core/storage/_utils.py @@ -909,6 +909,7 @@ def reconstruct_messages( attachments_by_msg: dict[int, list[dict[str, Any]]] | None = None, *, repair: bool = True, + include_compaction: bool = False, ) -> list[dict[str, Any]]: """Reconstruct OpenAI message format from stored conversation rows. @@ -946,12 +947,21 @@ def reconstruct_messages( the user sees the actual partial state — refreshing during tool execution otherwise silently drops the trailing turn from the UI. """ - # Drop compaction checkpoint markers: they are resume-only artifacts (the - # persisted summary that lets a reopened session rehydrate a bounded context, - # see reconstruct_turns_checkpointed), not real conversation turns, so - # /history, export, and search show the true transcript without an injected - # summary. - rows = [r for r in rows if not _is_compaction_marker(r)] + # Compaction checkpoint markers are resume artifacts (the persisted summary + # that lets a reopened session rehydrate a bounded context, see + # reconstruct_turns_checkpointed), not real conversation turns — dropped by + # default so export and search show the true transcript without an injected + # summary. ``include_compaction=True`` (the /history display path) instead + # re-rows each marker as a first-class ``system`` row IN PLACE: assistant + # rows drop ``_source``/``meta`` on reconstruction, but a system row keeps + # both, so the marker flows through the standard operator-context + # projection (``_source="compaction"`` + ``meta`` = watermark/token + # counts) and the frontend renders its compaction card at the point in + # the transcript where the compaction actually happened. + if include_compaction: + rows = [(r[0], "system", *r[2:]) if _is_compaction_marker(r) else r for r in rows] + else: + rows = [r for r in rows if not _is_compaction_marker(r)] turns = reconstruct_turns(rows, ws_id, attachments_by_msg) if repair: turns = recover_trajectory(turns) diff --git a/turnstone/core/workstream.py b/turnstone/core/workstream.py index 2a1859a8..d42134ed 100644 --- a/turnstone/core/workstream.py +++ b/turnstone/core/workstream.py @@ -139,6 +139,18 @@ class Workstream: # racing ``Thread.is_alive()``. Used by both interactive and # coordinator paths since Stage 2 P1. _worker_running: bool = field(default=False, repr=False) + # What KIND of work the current worker slot holds: "turn" (a send / + # retry / wake — the default) or "command" (a slash-command worker, + # including the minutes-long manual /compact). Written under + # ``_lock`` by ``session_worker.send`` in the same acquisition that + # sets ``worker_thread``/``_worker_running``, so readers gating on + # the running flag see a coherent triple. A stale value after the + # worker exits is harmless — every reader conjoins + # ``_worker_running``. The /send route parks (never queues) while + # this reads "command": the mid-turn interjection queue is + # turn-shaped (length cap, cross-user guard) and must be + # unreachable during command windows. + worker_kind: str = field(default="", repr=False) # True once ``SessionManager.commit_create`` (or the non-deferred # path through ``SessionManager.create``) has fired the lifecycle # ``emit_created`` event for this workstream. Used by diff --git a/turnstone/eval/core.py b/turnstone/eval/core.py index 3468c8c5..69943418 100644 --- a/turnstone/eval/core.py +++ b/turnstone/eval/core.py @@ -132,6 +132,9 @@ class NullUI: def on_system_turn(self, content: str, source: str, meta: dict[str, Any] | None = None) -> None: pass + def on_compaction(self, payload: dict[str, Any]) -> int | None: + return None + def on_state_change(self, state: str) -> None: pass diff --git a/turnstone/sdk/events.py b/turnstone/sdk/events.py index 216a65b6..254b4430 100644 --- a/turnstone/sdk/events.py +++ b/turnstone/sdk/events.py @@ -297,6 +297,47 @@ class OutputWarningEvent(ServerEvent): judge_model: str = "" +@dataclass +class CompactionEvent(ServerEvent): + """Context-compaction lifecycle (``phase`` = start / progress / end). + + ``start`` carries ``trigger`` (``"manual"`` / ``"auto"``; auto adds + ``where`` + ``pct``); ``progress`` carries chunked-summarization + ``part``/``total``/``depth`` (or ``retry_in``/``error`` for a retry + wait); ``end`` carries ``ok`` plus either the result + (``before_tokens``/``after_tokens``/``summary``) or the failure + ``reason``/``message`` — failure ends carry ``trigger`` too. The + successful end's summary is also persisted as a compaction marker + row and replays from ``/history`` as a ``role="system"``, + ``source="compaction"`` entry. ``compaction_id`` correlates every + event of one compaction run (0 from internal/legacy emitters). End + events additionally carry ``superseded``: True marks a force-abandoned + compaction retiring after a successor generation took over — clients + should skip failure notices for those (an OK end's result still + stands; the history swap happened). + """ + + type: str = "compaction" + phase: str = "" + compaction_id: int = 0 + superseded: bool = False + trigger: str = "" + where: str = "" + pct: int | None = None + part: int | None = None + total: int | None = None + depth: int | None = None + retry_in: float | None = None + error: str = "" + warning: str = "" + ok: bool | None = None + reason: str = "" + message: str = "" + before_tokens: int | None = None + after_tokens: int | None = None + summary: str = "" + + # --------------------------------------------------------------------------- # Server global events (/v1/api/events/global) # --------------------------------------------------------------------------- @@ -473,6 +514,7 @@ _SERVER_REGISTRY: dict[str, type[ServerEvent]] = { CancelledEvent, IntentVerdictEvent, OutputWarningEvent, + CompactionEvent, WsStateEvent, WsActivityEvent, WsRenameEvent, diff --git a/turnstone/server.py b/turnstone/server.py index 30bd44c9..ad41b5a6 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1715,23 +1715,190 @@ async def command(request: Request) -> JSONResponse: status_code=400, ) - should_exit = ws.session.handle_command(cmd) - if should_exit: - ui.on_info("Session ended. You can close this tab.") - # Handle UI updates for workstream-changing commands - if cmd_word in ("/clear", "/new"): - ui._enqueue({"type": "clear_ui"}) - elif cmd_word == "/resume": - # clear_ui signals the frontend to re-fetch history via REST. - ui._enqueue({"type": "clear_ui"}) - # Sync in-memory workstream name after any command that can change it. - # This ensures /api/workstreams and future page loads see the right name. - if cmd_word in ("/name", "/resume"): - from turnstone.core.memory import get_workstream_display_name + from turnstone.core import session_worker - updated_name = get_workstream_display_name(ws.session.ws_id) if ws.session else None - if updated_name: - ws.name = updated_name + session = ws.session + cmd_ui = ui + busy_hit = {"hit": False} + + def _reject_busy() -> None: + # Worker already running (a turn or another command is in + # flight). Commands aren't queueable work — surface "busy" + # instead. Server-side mirror of the composer's client guard, + # and a strict improvement for API callers: the old inline path + # let /clear & co. mutate the session mid-turn. + busy_hit["hit"] = True + + def _dispatch_command( + run: Callable[[], None], thread_name: str, busy_hint: str + ) -> JSONResponse | None: + """Dispatch a command worker; return the refusal response or None. + + One envelope for both command branches: the unknown-workstream + 404 and the busy refusal differ only in the retry hint. The + worker slot is claimed with ``worker_kind="command"`` — the + /send route PARKS (never queues) while that kind holds the + slot, so the mid-turn interjection queue and its turn-shaped + semantics (length cap, cross-user guard) are unreachable for + the whole command window; messages sent mid-command dispatch + as ordinary full-fidelity sends when the window closes. + """ + dispatched = session_worker.send( + ws, + enqueue=_reject_busy, + run=run, + thread_name=thread_name, + worker_kind="command", + ) + if not dispatched: + return JSONResponse({"error": "Unknown workstream"}, status_code=404) + if busy_hit["hit"]: + # 409, not 200: the refusal is deliberate (mutual exclusion + # replaced the old inline mid-turn interleave) but it must + # be LOUD — a status-code-only SDK caller treats a 200 as + # "command ran" and silently loses the rename/clear/config + # change. Same shape as /send's cross-user 409. + return JSONResponse( + { + "status": "busy", + "error": ( + "Session is busy — wait for the current turn to " + f"finish, then {busy_hint}." + ), + }, + status_code=409, + ) + return None + + if cmd_word == "/compact": + # Manual compaction runs LLM summary calls — seconds to minutes. + # It gets the send path's worker dispatch instead of an inline + # call so (a) the event loop stays free to stream the compaction + # progress events this very command produces (inline, they only + # flushed in one burst after the blocking call returned — the + # "no visible indicator" bug), and (b) a message sent + # mid-compaction takes the existing queue path instead of racing + # the history swap on a second worker. compact_now() carries + # send()'s generation discipline, so a force-abandoned compact + # thread goes stale instead of swapping history under a + # successor, and a cancel aimed at it is consumed on exit. + # Fire-and-forget: the response returns as soon as the worker is + # dispatched and NO completion bound applies — a large context + # can legitimately compact for many minutes; progress streams + # over SSE and Stop cancels it. (The 60s wait below is for the + # quick commands only.) + + def _run_compact() -> None: + me = threading.current_thread() + try: + cmd_ui.on_state_change("thinking") + session.compact_now() + except GenerationCancelled: + # User stopped it — including a Stop that landed in the + # completion tail, which compact_now re-raises after + # consuming. + pass + finally: + # Abandoned-worker guard — mirrors the send/retry + # closures: a force-cancelled compact thread must not + # touch state a successor worker now owns. + if ws.worker_thread is me: + try: + # Backstop only: sends during the command window + # PARK in the /send route (they cannot reach the + # interjection queue — see _dispatch_command), so + # anything found here predates the window (a + # message stranded by a dying send worker's + # closing race). Record it in the transcript + # rather than leaving it invisible. + if session.flush_queued_messages(): + log.warning("ws.compact.stranded_queue_flushed ws=%s", ws.id[:8]) + except Exception: + log.exception("ws.compact.exit_seam_failed ws=%s", ws.id[:8]) + cmd_ui.on_state_change("idle") + + refusal = _dispatch_command( + _run_compact, f"compact-worker-{ws.id[:8]}", "run /compact again" + ) + if refusal is not None: + return refusal + return JSONResponse({"status": "ok"}) + + # Every other command ALSO runs on the workstream's worker slot — + # the inline call this replaced was serialized by the event loop + # itself (nothing else could interleave with it); to_thread alone + # would let /clear & co. race a running /compact worker, a live + # send, or another command. The slot restores that mutual + # exclusion with an explicit busy answer, and the endpoint awaits + # completion off-loop so the response still reflects the outcome. + loop = asyncio.get_running_loop() + done = asyncio.Event() + + def _run_cmd() -> None: + me = threading.current_thread() + try: + should_exit = session.handle_command(cmd) + # Post-command follow-ups run HERE, on the worker, not + # after the endpoint's done-wait: past the 60s backstop the + # endpoint has already answered {"status": "running"}, and + # follow-ups parked there were silently skipped — a >60s + # /resume left every pane rendering the pre-resume + # transcript against a session whose history had changed, + # and the workstream list kept the stale name. + # Abandoned-worker guard, mirroring every sibling closure: + # a force-cancelled wedged command that unwedges minutes + # later must not fire clear_ui into a successor turn's live + # stream (every pane would wipe mid-answer) nor write a + # post-swap name from a stale session read. + if ws.worker_thread is me: + if should_exit: + cmd_ui.on_info("Session ended. You can close this tab.") + if cmd_word in ("/clear", "/new", "/resume"): + # clear_ui signals the frontend to re-fetch history + # via REST. + cmd_ui._enqueue({"type": "clear_ui"}) + if cmd_word in ("/name", "/resume"): + # Sync the in-memory workstream name after any + # command that can change it, so /api/workstreams + # and future page loads see the right name. + from turnstone.core.memory import get_workstream_display_name + + updated_name = get_workstream_display_name(session.ws_id) + if updated_name: + ws.name = updated_name + except Exception as e: + # Same guard: a late "Command error:" from an abandoned + # worker would land mid-successor-turn. + if ws.worker_thread is me: + cmd_ui.on_error(f"Command error: {e}") + finally: + # Unblock the endpoint response. Suppress the loop-closed + # RuntimeError (process shutdown mid-command): nobody is + # waiting anymore. No queue drain here: sends during the + # command window PARK in the /send route (the interjection + # queue is unreachable while worker_kind == "command"), so + # they dispatch as ordinary sends when this worker exits — + # the stranded-message backstop lives on the /compact seam, + # the only command window long enough to matter. + with contextlib.suppress(RuntimeError): + loop.call_soon_threadsafe(done.set) + + refusal = _dispatch_command(_run_cmd, f"command-worker-{ws.id[:8]}", "retry the command") + if refusal is not None: + return refusal + # Commands are quick (the long-runner, /compact, took the branch + # above); the bound is a backstop so a wedged command can't hold + # this request open forever — the worker keeps running, its output + # reaches the pane via SSE, and the post-command follow-ups run on + # the worker itself, so a late completion still refreshes the + # panes. Loop-native wait (call_soon_threadsafe from the worker's + # finally): a thread parked in Event.wait via to_thread would hold + # a shared default-executor slot for up to 60s per wedged command. + try: + async with asyncio.timeout(60): + await done.wait() + except TimeoutError: + return JSONResponse({"status": "running"}) except Exception as e: ui.on_error(f"Command error: {e}") @@ -2275,6 +2442,7 @@ async def _interactive_create_post_install( resolved_atts, staged_ord, _drop = _resolve_staged(attachment_ids, ws.id, uid) def _run_initial() -> None: + me = threading.current_thread() try: session.send( initial_message, @@ -2282,15 +2450,23 @@ async def _interactive_create_post_install( send_id=send_id if resolved_atts else None, ) except (Exception, GenerationCancelled): - if isinstance(ws.ui, WebUI): + # Abandoned-worker guard (sibling of _run_cmd/_run_compact/ + # the send closures): a force-cancelled init that unwedges + # late must not stamp idle over — or emit stream_end into — + # a successor turn. + if ws.worker_thread is me and isinstance(ws.ui, WebUI): ws.ui.on_stream_end() ws.ui.on_state_change("idle") finally: - try: - last_content = _extract_last_assistant_content(session) - _fire_notify_targets(ws, last_content) - except Exception: - log.warning("notify_completion.hook_error", ws_id=ws.id, exc_info=True) + # Owner only — an abandoned init's late notify would + # duplicate the successor's. (Guard, not early-return: a + # return in a finally silences in-flight exceptions.) + if ws.worker_thread is me: + try: + last_content = _extract_last_assistant_content(session) + _fire_notify_targets(ws, last_content) + except Exception: + log.warning("notify_completion.hook_error", ws_id=ws.id, exc_info=True) init_enqueued = False @@ -2311,6 +2487,15 @@ async def _interactive_create_post_install( # message were delivered. nonlocal init_enqueued init_enqueued = True + if ws.worker_kind == "command": + # A command worker (e.g. a minutes-long /compact) holds the + # raced slot: the interjection queue is turn-shaped (length + # cap, and the text would cross a /resume identity swap) and + # must stay unreachable during command windows — same rule + # as the /send route's park. queue.Full is the existing + # backpressure surface: the create reports the first message + # as undelivered (``queue_full``) and the client retries. + raise queue.Full() session.queue_message(initial_message) if resolved_atts: log.warning( diff --git a/turnstone/shared_static/chat.css b/turnstone/shared_static/chat.css index 0a49a76e..51ff22cf 100644 --- a/turnstone/shared_static/chat.css +++ b/turnstone/shared_static/chat.css @@ -994,6 +994,108 @@ margin-top: 6px; } +/* Compaction card — both the in-progress state (``.compaction-running``, with + the progress bar) and the settled result (token delta + summary fold). + Built by ``buildCompactionProgressCard`` / ``buildCompactionCard`` + (shared conversation.js) for the interactive pane AND the coord viewer. + Magenta accent: a memory/infrastructure operation — distinct from the cyan + tool surface, the yellow operator bubbles, and the amber user turns. */ +.msg.compaction-card { + border-left-color: var(--magenta); + background: var(--panel); + padding: 8px 12px; + margin: 6px 0; + width: 100%; +} +.msg.compaction-card .msg-compaction-header { + color: var(--magenta); + font-family: var(--font-mono); + font-size: 11px; + font-weight: 600; + text-transform: lowercase; +} +.msg.compaction-card .msg-compaction-detail { + color: var(--fg-dim); + font-family: var(--font-mono); + font-size: 12px; + margin-top: 4px; +} +.msg.compaction-card .msg-compaction-note { + color: var(--ink-3); + font-size: 11px; + margin-top: 6px; +} +.msg.compaction-card .msg-compaction-bar { + position: relative; + height: 4px; + margin-top: 8px; + border-radius: 2px; + background: var(--magenta-glow); + overflow: hidden; +} +.msg.compaction-card .msg-compaction-bar-fill { + height: 100%; + width: 0; + border-radius: 2px; + background: var(--magenta); + transition: width 300ms ease; +} +/* Indeterminate state — a single-batch summarization emits no part-k/N + progress, so the bar sweeps until the end event settles it. */ +.msg.compaction-card .msg-compaction-bar.indeterminate .msg-compaction-bar-fill { + width: 40%; + animation: compaction-sweep 1.4s ease-in-out infinite; +} +@keyframes compaction-sweep { + 0% { + margin-left: -40%; + } + 100% { + margin-left: 100%; + } +} +@media (prefers-reduced-motion: reduce) { + .msg.compaction-card .msg-compaction-bar.indeterminate .msg-compaction-bar-fill { + animation: none; + width: 100%; + opacity: 0.4; + } + .msg.compaction-card .msg-compaction-bar-fill { + transition: none; + } +} +.msg.compaction-card .msg-compaction-fold { + margin-top: 6px; +} +.msg.compaction-card .msg-compaction-fold > summary { + color: var(--ink-3); + font-size: 11px; + cursor: pointer; + user-select: none; + text-transform: lowercase; +} +.msg.compaction-card .msg-compaction-body { + font-family: var(--font-mono); + font-size: 12px; + margin: 6px 0 0; + white-space: pre-wrap; + word-break: break-word; + color: var(--fg-dim); + max-height: 320px; + overflow-y: auto; +} + +/* Slash-command echo — control-plane input, not a conversational turn. + Prompt-styled mono line (the leading "/" is the glyph), quieter than a + user bubble so command chatter doesn't read as trajectory. */ +.msg.command-echo { + border-left-color: var(--fg-dim); + color: var(--fg-dim); + font-family: var(--font-mono); + font-size: 12px; + padding: 4px 12px; +} + /* Guard-finding card — the ``output_guard`` operator-context system turn. The inner warning element (``.output-warning`` interactive / ``.coord-tool-row- warning`` coord) carries the risk colour + flags + redaction; this wrapper diff --git a/turnstone/shared_static/composer_queue.js b/turnstone/shared_static/composer_queue.js index 0701f768..3093913b 100644 --- a/turnstone/shared_static/composer_queue.js +++ b/turnstone/shared_static/composer_queue.js @@ -297,6 +297,11 @@ export function createQueueController(opts) { if (el.getAttribute("aria-busy") === "true") return; el.dataset.dismissAttempted = "1"; _setDismissing(el, true); + // A send whose POST is still in flight (parked server-side during a + // command window — possibly for minutes) attaches an abort hook: the × + // must kill the request itself, or the dismissed message dispatches + // anyway when the window closes. Post-response the hook is a no-op. + if (typeof el._sendAbort === "function") el._sendAbort(); var msgId = el.dataset.msgId; if (!msgId) { // Pre-bind: bind() confirms once the server returns the msg_id (it diff --git a/turnstone/shared_static/conversation.js b/turnstone/shared_static/conversation.js index 8f583ffa..5591022b 100644 --- a/turnstone/shared_static/conversation.js +++ b/turnstone/shared_static/conversation.js @@ -73,6 +73,257 @@ export function buildWatchResultCard(meta, content) { return el; } +// Compaction result card — the settled state of a context compaction. Renders +// from BOTH sources of the same fact: the live `compaction` end event and the +// `/history` projection of the persisted marker row (role="system", +// source="compaction"), so a reload paints the identical card. `meta` carries +// the structured fields ({before_tokens, after_tokens, trigger}); `summary` is +// the produced summary text, offered behind a
fold rather than +// inline — it exists for provenance, not for re-reading every visit. +export function buildCompactionCard(meta, summary) { + const m = meta && typeof meta === "object" ? meta : {}; + const el = document.createElement("div"); + el.className = "msg compaction-card"; + el.setAttribute("role", "article"); + el.setAttribute("data-ts-role", "compaction"); + el.setAttribute("aria-label", "context compacted"); + const header = document.createElement("div"); + header.className = "msg-compaction-header"; + header.textContent = + "context compacted" + (m.trigger === "auto" ? " · auto" : ""); + el.appendChild(header); + const before = Number(m.before_tokens); + const after = Number(m.after_tokens); + if (Number.isFinite(before) && Number.isFinite(after) && before > 0) { + const detail = document.createElement("div"); + detail.className = "msg-compaction-detail"; + detail.textContent = + "~" + + before.toLocaleString() + + " → ~" + + after.toLocaleString() + + " tokens"; + el.appendChild(detail); + } + const text = String(summary == null ? "" : summary).trim(); + if (text) { + const fold = document.createElement("details"); + fold.className = "msg-compaction-fold"; + const label = document.createElement("summary"); + label.textContent = "summary"; + fold.appendChild(label); + const body = document.createElement("pre"); + body.className = "msg-compaction-body"; + body.textContent = text; + fold.appendChild(body); + el.appendChild(fold); + } + return el; +} + +// In-progress compaction card — the transient affordance between the +// `compaction` start and end events. Starts with an indeterminate bar +// (a single-batch summarization emits no progress events); the first +// part-k-of-N progress event flips it determinate via +// updateCompactionProgress. The end event replaces the card with +// buildCompactionCard (or a failure notice). +export function buildCompactionProgressCard(isAuto) { + const el = document.createElement("div"); + el.className = "msg compaction-card compaction-running"; + el.setAttribute("role", "status"); + el.setAttribute("data-ts-role", "compaction"); + el.setAttribute("aria-label", "compacting context"); + const header = document.createElement("div"); + header.className = "msg-compaction-header"; + header.textContent = "compacting context…" + (isAuto ? " · auto" : ""); + el.appendChild(header); + const bar = document.createElement("div"); + bar.className = "msg-compaction-bar indeterminate"; + const fill = document.createElement("div"); + fill.className = "msg-compaction-bar-fill"; + bar.appendChild(fill); + el.appendChild(bar); + const note = document.createElement("div"); + note.className = "msg-compaction-note"; + note.textContent = "summarizing conversation…"; + el.appendChild(note); + return el; +} + +// Advance an in-progress compaction card from a `compaction` progress event. +// Depth 0 = summarizing transcript batches (determinate part k of N); deeper +// levels merge partial summaries — those update the note ONLY, never the bar: +// an over-window depth-0 batch subdivides mid-loop (emitting depth>0 events +// between depth-0 parts), so any width a merge event wrote would snap +// backwards when the outer loop resumed. The depth-0 width itself is +// monotonic (max with the current fill) for the same reason. A retry wait +// ({retry_in, error}) and the truncated-summary warning annotate the note +// without touching the bar. +export function updateCompactionProgress(el, evt) { + const bar = el.querySelector(".msg-compaction-bar"); + const fill = el.querySelector(".msg-compaction-bar-fill"); + const note = el.querySelector(".msg-compaction-note"); + if (!bar || !fill || !note) return; + if (evt.warning === "summary_truncated") { + note.textContent = "summary was truncated — continuing…"; + return; + } + if (evt.retry_in != null) { + note.textContent = + "retrying in " + + Math.round(Number(evt.retry_in)) + + "s (" + + String(evt.error || "error") + + ")…"; + return; + } + const part = Number(evt.part); + const total = Number(evt.total); + if (!Number.isFinite(part) || !Number.isFinite(total) || total < 1) return; + if (Number(evt.depth) > 0) { + note.textContent = "merging summaries (" + part + " of " + total + ")…"; + return; + } + bar.classList.remove("indeterminate"); + // part is emitted BEFORE its batch summarizes — show the k-1 completed + // fraction so the bar never claims work that hasn't happened yet. + const pct = Math.max(0, Math.min(100, ((part - 1) / total) * 100)); + const current = parseFloat(fill.style.width) || 0; + fill.style.width = Math.max(current, pct) + "%"; + note.textContent = "summarizing part " + part + " of " + total + "…"; +} + +// Shared compaction-lifecycle reducer — ONE state machine for the +// interactive pane and the coordinator viewer (each previously carried a +// hand-synced copy, and they drifted within their first diff). `holder` is +// the pane's mutable `{card}` slot (nulled wherever the transcript DOM is +// wiped); `hooks` supplies the pane-specific seams: +// container — the transcript element to append into +// renderedIds — the pane's rendered-event-id Set (dedups the ok-end +// card against the /history-projected marker row) +// onNotice(msg) — render a non-error failure notice (info styling) +// scroll(force) — the pane's scroll-to-bottom +// reason="error" ends render NO notice here: the backend pairs them with a +// typed `error` event, which each pane's existing error handler styles red +// (and which feeds the node's error metrics) — emitting here too would show +// the message twice. +export function applyCompactionEvent(holder, evt, hooks) { + // Lifecycle ownership: events carry the backend's compaction_id and the + // holder remembers which compaction painted the live card, so a stale + // event — a force-abandoned compaction retiring after a successor + // started — can't animate or tear down the successor's card. The id + // gates only while a live card exists (with no card there is nothing to + // protect), and a missing id on either side matches everything so + // replays from older backends keep working. + const owns = + !holder.card || + holder.cid == null || + evt.compaction_id == null || + String(evt.compaction_id) === holder.cid; + if (evt.phase === "start") { + if (holder.card) holder.card.remove(); + holder.card = buildCompactionProgressCard(evt.trigger === "auto"); + holder.cid = evt.compaction_id != null ? String(evt.compaction_id) : null; + hooks.container.appendChild(holder.card); + hooks.scroll(true); + return; + } + if (evt.phase === "progress") { + if (!owns) return; + // Defensive create: a fresh connect mid-compaction (dead replay + // buffer) can see a progress event with no preceding start. The + // `false` is deliberate: progress events carry no trigger field, so + // this card cannot know it should wear the "· auto" suffix — + // cosmetic, and threading trigger through the summarize stack for it + // is disproportionate. + if (!holder.card) { + holder.card = buildCompactionProgressCard(false); + holder.cid = evt.compaction_id != null ? String(evt.compaction_id) : null; + hooks.container.appendChild(holder.card); + } + updateCompactionProgress(holder.card, evt); + hooks.scroll(false); + return; + } + if (evt.phase === "end") { + if (owns && holder.card) { + holder.card.remove(); + holder.card = null; + holder.cid = null; + } + if (evt.ok) { + // The persisted marker row is stamped with THIS event's id, so + // whichever of /history repaint or live/replayed event renders + // first wins. Rendered even for a non-owning end: a completed + // compaction's result is real regardless of whose card is live. + const eid = evt._event_id != null ? String(evt._event_id) : null; + if (eid && hooks.renderedIds.has(eid)) return; + hooks.container.appendChild( + buildCompactionCard( + { + before_tokens: evt.before_tokens, + after_tokens: evt.after_tokens, + trigger: evt.trigger, + }, + evt.summary || "", + ), + ); + if (eid) hooks.renderedIds.add(eid); + } else if ( + owns && + !evt.superseded && + evt.reason !== "error" && + !(evt.reason === "cancelled" && evt.trigger === "auto") + ) { + // cancelled / not_enough_messages / irreducible / empty_summary — + // informational, not an error state. Three suppressions: a stale + // end's notice would narrate a dead compaction underneath a live + // one; a superseded end (a force-abandoned compaction retiring + // late, flagged by the backend) is one nobody is waiting on — its + // notice mid-turn reads as the LIVE work being cancelled; and an + // auto-compaction's cancel is just part of cancelling the + // surrounding turn — the send loop emits its own "[Generation + // cancelled]" info line, so a second line here stacked two notices + // for one Stop click. (A superseded OK end above still renders + // its result card — the history swap really happened.) + // HAND-SYNCED SIBLING: cli.py TerminalUI.on_compaction implements + // this same three-clause policy for the terminal — change both or + // they drift (two runtimes, no shared code path). + hooks.onNotice(evt.message || "Compaction skipped."); + } + hooks.scroll(true); + } +} + +// Send-POST abort bound for a pane, selected off its compaction holder. +// Sends during a slash-command window PARK server-side and dispatch when +// the window closes — a manual /compact legitimately runs for minutes, so +// while ITS progress card is live the composer must not abort the parked +// POST at the wedged-node default (~15s) and silently drop the message. +// The card is the one cross-tab signal that a long window is in progress +// (SSE-driven via applyCompactionEvent); with no card the short default +// stands — a WEDGED quick command past 15s should fail loudly, not hang +// the composer for 10 minutes. Deployment note: reverse proxies bound +// the effective park at their own read timeout (documented in the API +// reference). Shared by the interactive composer and the coordinator +// viewer so the policy can't drift between panes. +export function sendAbortMs(holder) { + return holder && holder.card ? 600000 : 15000; +} + +// Retire a pane's in-progress compaction card (if any) and clear the +// holder. The teardown half of the lifecycle the reducer above owns — +// exported so the panes' stream_end handlers (force-stop abandons the +// compaction worker without an end event in flight) and transcript-wipe +// sites share ONE implementation instead of four hand-synced copies. +// Element.remove() on an already-detached node (a wipe that +// replaceChildren()'d the transcript) is a harmless no-op. +export function resetCompactionHolder(holder) { + if (holder.card) holder.card.remove(); + holder.card = null; + holder.cid = null; +} + // Thin `.msg.user.system-nudge` marker — the visible-but-subtle anchor a // wake-driven empty user turn renders, so the operator-context `system` turns // that follow it land in the right place. The caller handles empty-state diff --git a/turnstone/shared_static/interactive.js b/turnstone/shared_static/interactive.js index fd65ee51..a3e283f6 100644 --- a/turnstone/shared_static/interactive.js +++ b/turnstone/shared_static/interactive.js @@ -23,6 +23,10 @@ import { stripAnsi, buildWatchResultCard, + buildCompactionCard, + applyCompactionEvent, + resetCompactionHolder, + sendAbortMs, buildSystemNudgeMarker, buildConvBatchShell, buildConvRow, @@ -261,6 +265,10 @@ class Pane { this._scrollPinPending = false; this._scrollPinForce = false; this._thinkingEl = null; + // Compaction lifecycle holder for the shared reducer + // (conversation.applyCompactionEvent); `card` is the in-progress card + // between start and end, nulled wherever the transcript DOM is wiped. + this._compaction = { card: null, cid: null }; this._retryHolderEl = null; this._toolRowIndex = new Map(); this._streamElIndex = new Map(); @@ -458,6 +466,7 @@ class Pane { // Instance ref, not a container query: removeThinkingIndicator runs on // EVERY content/reasoning delta, and a class-selector miss walks the // whole transcript subtree — O(N) per streamed token at 5000 messages. + if (this._compaction.card) return; // the compaction card owns the affordance if (this._thinkingEl) return; const el = document.createElement("div"); el.className = "thinking-indicator"; @@ -492,6 +501,15 @@ class Pane { // carries structured `meta` (watch_name / command / poll counters) → the // richer `.msg.watch-result` card instead of the plain operator bubble. this.removeEmptyState(); + // The /history projection of a persisted compaction marker (an in-place + // source="compaction" system row) — render the same result card the live + // `compaction` end event paints, so a reload reproduces the transcript. + if (source === "compaction") { + const card = buildCompactionCard(meta, content || ""); + this.messagesEl.appendChild(card); + this.scrollToBottom(true); + return card; + } if (source === "watch_triggered" && meta && typeof meta === "object") { const card = buildWatchResultCard(meta, content || ""); this.messagesEl.appendChild(card); @@ -534,6 +552,40 @@ class Pane { return el; } + handleCompactionEvent(evt) { + // Shared reducer (conversation.applyCompactionEvent) — one lifecycle + // state machine for this pane and the coordinator viewer. The dedup + // set is the same one the system_turn path uses: the persisted marker + // row is stamped with the ok-end event's id, so whichever of /history + // repaint or live/replayed event renders first wins. reason="error" + // ends render through the paired `error` event (red row), not here. + this.removeEmptyState(); + if (!this._renderedSystemEventIds) { + this._renderedSystemEventIds = new Set(); + } + applyCompactionEvent(this._compaction, evt, { + container: this.messagesEl, + renderedIds: this._renderedSystemEventIds, + onNotice: (msg) => this.addInfoMessage(msg), + scroll: (force) => this.scrollToBottom(force), + }); + } + + addCommandEcho(text) { + // A slash command is control-plane input, not a conversational turn — + // echo it as a distinct command chip (styled like a prompt line), not a + // user bubble. It is deliberately NOT persisted: commands don't join + // the trajectory, so a bubble that vanished on reload was a lie. + this.removeEmptyState(); + const el = document.createElement("div"); + el.className = "msg command-echo"; + el.setAttribute("aria-label", "command"); + el.textContent = text; + this.messagesEl.appendChild(el); + this.scrollToBottom(true); + return el; + } + addUserMessage(text, attachments) { this.removeEmptyState(); const el = document.createElement("div"); @@ -1643,6 +1695,13 @@ class Pane { clearTimeout(this._forceTimeout); this._forceTimeout = null; } + // A live in-progress compaction card at stream_end means a FORCE + // stop abandoned the compaction worker (the lifecycle wrapper + // otherwise always retires the card with an end event before any + // stream_end can follow) — remove it now instead of leaving a + // frozen bar until the abandoned worker notices at its next + // checkpoint. + resetCompactionHolder(this._compaction); // Reset the segment state BEFORE the finalize render, and guard the // render with a plain-text fallback (mirrors coordinator.js). With // the old order a finalize throw skipped these clears, so every @@ -1879,6 +1938,13 @@ class Pane { break; } + case "compaction": + // Context-compaction lifecycle: start paints the in-progress card, + // progress drives its bar, end swaps it for the result card (or a + // failure notice). See handleCompactionEvent. + this.handleCompactionEvent(evt); + break; + case "message_queued": // Confirmation from server that a queued message was accepted. // The UI already showed the message optimistically in addQueuedMessage. @@ -2657,6 +2723,9 @@ class Pane { this.currentAssistantBodyEl = null; this.currentReasoningEl = null; this.contentBuffer = ""; + // In-progress compaction card: the transcript wipe orphaned it; live + // events re-create it defensively (see handleCompactionEvent). + resetCompactionHolder(this._compaction); // Approval cycles + announce shells point into the wiped subtree too; // the replayed history / detail snapshot re-registers live ones. this.approvalCycles = new Map(); @@ -3606,7 +3675,11 @@ class Pane { if (!text) return; if (text.startsWith("/")) { - if (this.busy) return; // commands not allowed while busy + if (this.busy) { + // Was a silent return — say why nothing happened. + this.addInfoMessage("Session is busy — commands can't run mid-turn."); + return; + } // /rewind and /retry were lifted to path-keyed endpoints (#549); // reroute hand-typed ones so they don't 400 against /command. const parts = text.split(/\s+/); @@ -3633,8 +3706,27 @@ class Pane { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ command: text, ws_id: this.wsId }), - }); - this.addUserMessage(text); + }) + .then((r) => + r.json().then( + (b) => b || {}, + () => ({}), + ), + ) + .then((body) => { + // /compact dispatched onto an already-busy worker reports + // {status: "busy"} — surface it (the optimistic busy guard + // above can lose that race). + if (body.status === "busy") { + this.addInfoMessage( + body.error || "Session is busy — try again shortly.", + ); + } + }) + .catch(() => {}); + // Echo as a command chip, not addUserMessage — a slash command is + // control-plane input, not a conversational user turn. + this.addCommandEcho(text); this.composer.clear(); return; } @@ -3677,7 +3769,17 @@ class Pane { let sendTimer = null; if (sendCtrl) { sendInit.signal = sendCtrl.signal; - sendTimer = setTimeout(() => sendCtrl.abort(), 15000); + // Long bound while a compaction card is live (the send is parked + // server-side for the command window); wedged-node default + // otherwise — see sendAbortMs. + sendTimer = setTimeout( + () => sendCtrl.abort(), + sendAbortMs(this._compaction), + ); + // An × on the queued bubble BEFORE the response arrives (pre-bind) + // must also kill the parked POST — otherwise the dismissed message + // dispatches anyway when the command window closes, minutes later. + if (queuedEl) queuedEl._sendAbort = () => sendCtrl.abort(); } let sendReq = authFetch( this._base +