fix: pre-push review fixes from the canonical-trajectory deep-dive

A deep-dive review of the branch surfaced a budget regression, SDK doc
drift, dead code, and stale docstrings. Each was boundary-spiked before
fixing.

- R1 (regression): by-reference document attachments were invisible to the
  token budget — _msg_text_chars returned 0 doc_chars for a
  {type:document,attachment_id} placeholder, and the comment's claim that
  the budget "lands at calibration" was false (calibration discards
  doc_chars). Thread the doc size through _attachments_meta (size_bytes, at
  both the live-append and reconstruct build sites) and count it in
  _msg_text_chars, guarded against double-counting the inline form.
  Regression test added.
- F1: the history-DTO schema description and the TS HistoryEvent docstring
  still advertised the removed reminders/advisories keys and omitted the
  system role; corrected server_schemas.py + sdk/typescript/src/events.ts to
  match the shipped shape. The committed OpenAPI JSON snapshots were already
  ~679 lines stale on main; their regen is left to its own chore branch.
- D1: removed AttachmentBuffer.take() — dead (no production caller; the
  commit path uses discard()) and scope-weak (ws_id only, unlike its
  siblings) — with its test and the now-orphaned Iterable import.
- O1: 4 docstrings referenced the moved ChatSession._fold_system_turns →
  lowering.fold_system_turns.

sdk/typescript/src/events.ts

@@ -14,17 +14,18 @@ export interface ConnectedEvent {
 export interface HistoryEvent {
   type: "history";
   /**
-   * Per-message dicts the frontend consumes directly. Common optional keys:
-   * - `role`: "user" | "assistant" | "tool"
-   * - `content`: string or list (image/document parts)
+   * Per-message dicts the frontend consumes directly. Notable optional keys:
+   * - `role`: "user" | "assistant" | "tool" | "system"
+   * - `content`: string for text turns, list for image/document parts
    * - `tool_calls`: assistant turns — list of `{id, name, arguments, verdict?, output_assessment?}`
    * - `tool_call_id`: tool turns — id of the originating call
-   * - `reminders`: metacognitive nudge bubbles (user/tool channels)
-   * - `advisories`: extracted `UserInterjection` payloads on tool turns
-   * - `reasoning`: concatenated reasoning text for assistant turns whose
-   *   `provider_data` carried reasoning-bearing blocks (Anthropic
-   *   `thinking`, OpenAI Responses `reasoning`, or synthetic
-   *   `reasoning_text` from path-3 servers). Present only when the
+   * - `source`: the operator-context kind on a `system` turn (`output_guard` /
+   *   `user_interjection` / `tool_error` / ...), or `system_nudge` on a
+   *   wake-driven empty user turn
+   * - `attachments`: per-attachment metadata `{kind, filename, mime_type, size_bytes}`
+   * - `reasoning`: concatenated reasoning text for assistant turns that
+   *   round-tripped a thinking-block lane (Anthropic-with-thinking today;
+   *   OpenAI Responses + Gemini in later phases). Present only when the
    *   active model's `surface_persisted_reasoning` flag is true.
    */
   messages: Array<Record<string, unknown>>;

tests/test_attachment_buffer.py

@@ -66,16 +66,6 @@ def test_discard_is_scope_checked() -> None:
     assert buf.get(entry.attachment_id, ws_id="ws1", user_id="u1") is None
 
 
-def test_take_pops_in_scope_and_skips_others() -> None:
-    buf = AttachmentBuffer()
-    a = _stage(buf, content=b"a")
-    b = _stage(buf, content=b"b", ws="ws2")
-    taken = buf.take([a.attachment_id, b.attachment_id, "missing"], ws_id="ws1")
-    assert [t.attachment_id for t in taken] == [a.attachment_id]  # only ws1's, missing skipped
-    assert buf.get(a.attachment_id, ws_id="ws1", user_id="u1") is None  # popped
-    assert buf.get(b.attachment_id, ws_id="ws2", user_id="u1") is not None  # other ws untouched
-
-
 def test_ttl_eviction_on_access() -> None:
     clock = [0.0]
     buf = AttachmentBuffer(ttl_seconds=10.0, clock=lambda: clock[0])

tests/test_session_attachments.py

@@ -262,8 +262,13 @@ class TestProviderIntegration:
         _run_send(s, "desc", attachments=atts)
         meta = turn_to_dict(s.messages[-1]).get("_attachments_meta")
         assert meta == [
-            {"kind": "image", "filename": "dog.png", "mime_type": "image/png"},
-            {"kind": "text", "filename": "notes.md", "mime_type": "text/markdown"},
+            {
+                "kind": "image",
+                "filename": "dog.png",
+                "mime_type": "image/png",
+                "size_bytes": len(PNG_1x1),
+            },
+            {"kind": "text", "filename": "notes.md", "mime_type": "text/markdown", "size_bytes": 2},
         ]
 
     def test_attachments_meta_stripped_before_openai_wire(self, tmp_db, mock_openai_client):
@@ -362,3 +367,35 @@ class TestTokenAccounting:
         # The ~4000-char doc lands at the resolved boundary (the per-turn
         # placeholder no longer carries the bytes), well above the budget floor.
         assert doc_chars - plain_chars >= 900
+
+    def test_by_reference_doc_counted_without_materialization(self):
+        """R1: a canonical by-reference document turn (NOT yet materialized) must
+        count its size in the char budget via ``_attachments_meta``.  Before the
+        fix the placeholder carried no bytes, so ``doc_chars`` was 0 and a reloaded
+        document conversation under-counted its context (the budget feeds
+        compaction / trim decisions)."""
+        meta = [
+            {"kind": "text", "filename": "big.md", "mime_type": "text/markdown", "size_bytes": 4000}
+        ]
+        by_ref = {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "see doc"},
+                {"type": "document", "attachment_id": "x"},
+            ],
+            "_attachments_meta": meta,
+        }
+        _t, _i, doc_chars = ChatSession._msg_text_chars(by_ref)
+        assert doc_chars == 4000  # was 0 before the fix
+
+        # A materialized inline document that still carries meta must count ONCE,
+        # not twice — the inline_doc guard suppresses the meta term.
+        inline_plus_meta = {
+            "role": "user",
+            "content": [
+                {"type": "document", "document": {"data": "x" * 4000, "name": "", "media_type": ""}}
+            ],
+            "_attachments_meta": meta,
+        }
+        _t2, _i2, doc2 = ChatSession._msg_text_chars(inline_plus_meta)
+        assert doc2 == 4000

tests/test_storage_attachments.py

@@ -52,9 +52,7 @@ class TestContentAddressedWrite:
     def test_origin_tool_recorded(self, backend):
         backend.register_workstream("ws-origin")
         aid = _hash(PNG_1x1)
-        backend.save_attachment(
-            aid, "t.png", "image/png", len(PNG_1x1), "image", PNG_1x1, "tool"
-        )
+        backend.save_attachment(aid, "t.png", "image/png", len(PNG_1x1), "image", PNG_1x1, "tool")
         row = backend.get_attachment(aid)
         assert row is not None
         assert row["origin"] == "tool"
@@ -91,9 +89,7 @@ class TestGetAttachments:
         a1 = _hash(b"one")
         a2 = _hash(PNG_1x1)
         backend.save_attachment(a1, "one.txt", "text/plain", 3, "text", b"one")
-        backend.save_attachment(
-            a2, "img.png", "image/png", len(PNG_1x1), "image", PNG_1x1
-        )
+        backend.save_attachment(a2, "img.png", "image/png", len(PNG_1x1), "image", PNG_1x1)
         by_id = {r["attachment_id"]: r for r in backend.get_attachments([a1, a2])}
         assert by_id[a1]["content"] == b"one"
         assert by_id[a2]["content"] == PNG_1x1
@@ -166,12 +162,8 @@ class TestLoadMessagesReconstructsMultipart:
         msg_id = backend.save_message("ws-multi", "user", "look at these")
         img_id = _hash(PNG_1x1)
         doc_id = _hash(b"# hi\n")
-        backend.save_attachment(
-            img_id, "tiny.png", "image/png", len(PNG_1x1), "image", PNG_1x1
-        )
-        backend.save_attachment(
-            doc_id, "notes.md", "text/markdown", 5, "text", b"# hi\n"
-        )
+        backend.save_attachment(img_id, "tiny.png", "image/png", len(PNG_1x1), "image", PNG_1x1)
+        backend.save_attachment(doc_id, "notes.md", "text/markdown", 5, "text", b"# hi\n")
         backend.set_message_attachments("ws-multi", msg_id, [img_id, doc_id])
 
         msgs = backend.load_messages("ws-multi")
@@ -292,7 +284,12 @@ class TestReconstructMetaSibling:
         backend.set_message_attachments("ws-meta", mid, [aid])
         meta = backend.load_messages("ws-meta")[0].get("_attachments_meta")
         assert isinstance(meta, list) and len(meta) == 1
-        assert meta[0] == {"kind": "text", "filename": "doc.md", "mime_type": "text/markdown"}
+        assert meta[0] == {
+            "kind": "text",
+            "filename": "doc.md",
+            "mime_type": "text/markdown",
+            "size_bytes": 2,
+        }
 
 
 class TestRefcountGC:
@@ -321,14 +318,10 @@ class TestRefcountGC:
         backend.register_workstream("ws-shared")
         shared = _hash(b"shared-bytes")
         m1 = backend.save_message("ws-shared", "user", "first")
-        backend.save_attachment(
-            shared, "s.txt", "text/plain", 12, "text", b"shared-bytes"
-        )
+        backend.save_attachment(shared, "s.txt", "text/plain", 12, "text", b"shared-bytes")
         backend.set_message_attachments("ws-shared", m1, [shared])
         m2 = backend.save_message("ws-shared", "user", "second")
-        backend.save_attachment(
-            shared, "s.txt", "text/plain", 12, "text", b"shared-bytes"
-        )
+        backend.save_attachment(shared, "s.txt", "text/plain", 12, "text", b"shared-bytes")
         backend.set_message_attachments("ws-shared", m2, [shared])
 
         assert backend.get_attachment(shared)["refcount"] == 2
@@ -355,14 +348,10 @@ class TestRefcountGC:
         backend.register_workstream("ws-two")
         shared = _hash(b"cross-ws")
         m1 = backend.save_message("ws-one", "user", "a")
-        backend.save_attachment(
-            shared, "s.txt", "text/plain", 8, "text", b"cross-ws"
-        )
+        backend.save_attachment(shared, "s.txt", "text/plain", 8, "text", b"cross-ws")
         backend.set_message_attachments("ws-one", m1, [shared])
         m2 = backend.save_message("ws-two", "user", "b")
-        backend.save_attachment(
-            shared, "s.txt", "text/plain", 8, "text", b"cross-ws"
-        )
+        backend.save_attachment(shared, "s.txt", "text/plain", 8, "text", b"cross-ws")
         backend.set_message_attachments("ws-two", m2, [shared])
         assert backend.get_attachment(shared)["refcount"] == 2
 
@@ -410,9 +399,7 @@ class TestParametrizedKind:
         payload = PNG_1x1 if kind == "image" else b"x" * 42
         mime = "image/png" if kind == "image" else "text/plain"
         aid = _hash(payload)
-        backend.save_attachment(
-            aid, f"f.{kind}", mime, len(payload), kind, payload
-        )
+        backend.save_attachment(aid, f"f.{kind}", mime, len(payload), kind, payload)
         rows = backend.get_attachments([aid])
         assert len(rows) == 1
         assert rows[0]["content"] == payload

turnstone/api/server_schemas.py

@@ -491,9 +491,10 @@ class WorkstreamHistoryResponse(BaseModel):
         default_factory=list,
         description=(
             "Tail of the workstream's message history, projected to the "
-            "canonical render shape (flat tool_calls with verdict / "
-            "output_assessment, top-level source / reminders / "
-            "attachments, derived denied / is_error / pending). Bounded "
+            "canonical render shape (``role`` may be ``system`` for "
+            "operator-context turns; flat tool_calls with verdict / "
+            "output_assessment; top-level source / attachments / reasoning; "
+            "derived denied / is_error / pending). Bounded "
             "by the ``limit`` query parameter (default 100, max 500)."
         ),
     )

turnstone/core/attachment_buffer.py

@@ -23,7 +23,7 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable
+    from collections.abc import Callable
 
 # OOM-safety backstops (per node), not a product policy — the per-user upload cap was
 # removed.  Generous: the goal is only to bound a pathological flood of unsent uploads.
@@ -113,21 +113,6 @@ class AttachmentBuffer:
             del self._entries[handle]
             return True
 
-    def take(self, handles: Iterable[str], *, ws_id: str) -> list[StagedAttachment]:
-        """Pop the staged entries for *handles* at send-commit (ws-scoped).
-
-        Missing / out-of-scope handles are skipped — the send proceeds with whatever
-        committed (a buffer eviction or crash between upload and send drops the upload).
-        """
-        with self._lock:
-            taken: list[StagedAttachment] = []
-            for handle in handles:
-                entry = self._entries.get(handle)
-                if entry is not None and entry.ws_id == ws_id:
-                    taken.append(entry)
-                    del self._entries[handle]
-            return taken
-
     # -- eviction (caller holds the lock) ------------------------------------
     def _evict_expired_locked(self) -> None:
         if self._ttl <= 0:

turnstone/core/fence.py

@@ -11,7 +11,7 @@ forge or break the boundary.  One mechanism, two trust polarities:
   declares the fence *form* (``<tool_output_NONCE>``) as untrusted data, so a
   fresh per-call nonce is enough.
 
-* **Operator fold** (``ChatSession._fold_system_turns``) wraps TRUSTED operator
+* **Operator fold** (``lowering.fold_system_turns``) wraps TRUSTED operator
   instructions folded into a neighbouring turn for models without native
   mid-conversation system messages.  The nonce stops untrusted host text from
   forging a *fake* trusted block; the system prompt

turnstone/core/providers/_anthropic.py

@@ -368,7 +368,7 @@ class AnthropicProvider:
         messages (the base prompt) still hoist into the top-level
         ``system`` param, but a system message appearing AFTER a
         non-system turn is a mid-conversation operator turn (see
-        ``ChatSession._fold_system_turns``) and is emitted inline as a
+        ``lowering.fold_system_turns``) and is emitted inline as a
         ``{"role": "system"}`` message so it keeps its trajectory
         position.  When False (every other model) all system messages
         hoist, as before — the fold pass has already removed any

turnstone/core/session.py

@@ -3580,6 +3580,9 @@ class ChatSession:
                     "kind": a.kind,
                     "filename": a.filename,
                     "mime_type": a.mime_type,
+                    # Doc-budget proxy mirrored from the reconstruct path so the
+                    # live and reloaded shapes count identically (``_msg_text_chars``).
+                    "size_bytes": len(a.content),
                 }
                 for a in attachments
             ]
@@ -4803,6 +4806,7 @@ class ChatSession:
         n = 0
         images = 0
         doc_chars = 0
+        inline_doc = False
         if isinstance(content, list):
             for p in content:
                 ptype = p.get("type")
@@ -4813,14 +4817,29 @@ class ChatSession:
                     # — both cost one fixed image budget.
                     images += 1
                 elif ptype == "document" and not p.get("attachment_id"):
-                    # Resolved inline document; the by-reference placeholder carries
-                    # no bytes, so its char budget lands at send-time calibration.
+                    # Resolved inline document (the transient materialized form):
+                    # count its data chars directly.
+                    inline_doc = True
                     d = p.get("document", {})
                     doc_chars += len(d.get("data", ""))
                     doc_chars += len(d.get("name", ""))
                     doc_chars += len(d.get("media_type", ""))
         else:
             n += len(content or "")
+        # A by-reference document placeholder (``{type:document, attachment_id}``)
+        # carries no inline bytes, so its budget comes from the sibling
+        # ``_attachments_meta`` (``size_bytes`` per text-kind attachment).  Skip
+        # when an inline document was already counted: canonical messages are
+        # by-reference + meta and the materialized wire form is inline-without-meta,
+        # so the two are mutually exclusive — the guard makes that robust either way.
+        if not inline_doc:
+            meta = msg.get("_attachments_meta")
+            if isinstance(meta, list):
+                doc_chars += sum(
+                    int(e.get("size_bytes") or 0)
+                    for e in meta
+                    if isinstance(e, dict) and e.get("kind") == "text"
+                )
         for tc in msg.get("tool_calls", []):
             n += len(tc.get("id", ""))
             n += len(tc.get("function", {}).get("name", ""))

turnstone/core/storage/_utils.py

@@ -242,6 +242,10 @@ def _reconstruct_attachment_refs(
                 "kind": str(att.get("kind") or ""),
                 "filename": str(att.get("filename") or ""),
                 "mime_type": str(att.get("mime_type") or ""),
+                # Doc-budget proxy for the by-reference placeholder (which carries
+                # no inline bytes): the token estimator reads this so a reloaded
+                # document turn isn't counted as ~free.  See ``_msg_text_chars``.
+                "size_bytes": int(att.get("size_bytes") or 0),
             }
         )
     return refs, meta

turnstone/core/tool_advisory.py

@@ -8,7 +8,7 @@ either kept inline (native mid-conversation system messages — claude-opus-4-8)
 or folded into the preceding turn as a nonce-delimited ``<system-reminder_
 {nonce}>`` fence for every other model.  The fence mechanism (mint / neutralise
 / wrap) lives in :mod:`turnstone.core.fence`, shared with the output-guard judge
-so the two trust boundaries cannot drift; ``ChatSession._fold_system_turns``
+so the two trust boundaries cannot drift; ``lowering.fold_system_turns``
 applies it.
 
 This module also hosts :func:`parse_priority` (the ``!!!`` priority prefix on
@@ -122,7 +122,7 @@ def make_system_turn(source: str, content: str, **meta: Any) -> dict[str, Any]:
     done here.  It belongs to the fallback fold step, which wraps the content
     in a nonce-delimited ``<system-reminder_{nonce}>`` fence via
     :func:`turnstone.core.fence.wrap` (applied in
-    ``ChatSession._fold_system_turns``).  Escaping in this builder would corrupt
+    ``lowering.fold_system_turns``).  Escaping in this builder would corrupt
     the native path, where there is no fence to break out of.
     """
     if source not in SYSTEM_TURN_SOURCES: