diff --git a/sdk/typescript/src/events.ts b/sdk/typescript/src/events.ts index 63984977..2fba228b 100644 --- a/sdk/typescript/src/events.ts +++ b/sdk/typescript/src/events.ts @@ -14,17 +14,18 @@ export interface ConnectedEvent { export interface HistoryEvent { type: "history"; /** - * Per-message dicts the frontend consumes directly. Common optional keys: - * - `role`: "user" | "assistant" | "tool" - * - `content`: string or list (image/document parts) + * Per-message dicts the frontend consumes directly. Notable optional keys: + * - `role`: "user" | "assistant" | "tool" | "system" + * - `content`: string for text turns, list for image/document parts * - `tool_calls`: assistant turns — list of `{id, name, arguments, verdict?, output_assessment?}` * - `tool_call_id`: tool turns — id of the originating call - * - `reminders`: metacognitive nudge bubbles (user/tool channels) - * - `advisories`: extracted `UserInterjection` payloads on tool turns - * - `reasoning`: concatenated reasoning text for assistant turns whose - * `provider_data` carried reasoning-bearing blocks (Anthropic - * `thinking`, OpenAI Responses `reasoning`, or synthetic - * `reasoning_text` from path-3 servers). Present only when the + * - `source`: the operator-context kind on a `system` turn (`output_guard` / + * `user_interjection` / `tool_error` / ...), or `system_nudge` on a + * wake-driven empty user turn + * - `attachments`: per-attachment metadata `{kind, filename, mime_type, size_bytes}` + * - `reasoning`: concatenated reasoning text for assistant turns that + * round-tripped a thinking-block lane (Anthropic-with-thinking today; + * OpenAI Responses + Gemini in later phases). Present only when the * active model's `surface_persisted_reasoning` flag is true. */ messages: Array>; diff --git a/tests/test_attachment_buffer.py b/tests/test_attachment_buffer.py index 49f429d0..9417a406 100644 --- a/tests/test_attachment_buffer.py +++ b/tests/test_attachment_buffer.py @@ -66,16 +66,6 @@ def test_discard_is_scope_checked() -> None: assert buf.get(entry.attachment_id, ws_id="ws1", user_id="u1") is None -def test_take_pops_in_scope_and_skips_others() -> None: - buf = AttachmentBuffer() - a = _stage(buf, content=b"a") - b = _stage(buf, content=b"b", ws="ws2") - taken = buf.take([a.attachment_id, b.attachment_id, "missing"], ws_id="ws1") - assert [t.attachment_id for t in taken] == [a.attachment_id] # only ws1's, missing skipped - assert buf.get(a.attachment_id, ws_id="ws1", user_id="u1") is None # popped - assert buf.get(b.attachment_id, ws_id="ws2", user_id="u1") is not None # other ws untouched - - def test_ttl_eviction_on_access() -> None: clock = [0.0] buf = AttachmentBuffer(ttl_seconds=10.0, clock=lambda: clock[0]) diff --git a/tests/test_session_attachments.py b/tests/test_session_attachments.py index 4f6d889c..046a9c70 100644 --- a/tests/test_session_attachments.py +++ b/tests/test_session_attachments.py @@ -262,8 +262,13 @@ class TestProviderIntegration: _run_send(s, "desc", attachments=atts) meta = turn_to_dict(s.messages[-1]).get("_attachments_meta") assert meta == [ - {"kind": "image", "filename": "dog.png", "mime_type": "image/png"}, - {"kind": "text", "filename": "notes.md", "mime_type": "text/markdown"}, + { + "kind": "image", + "filename": "dog.png", + "mime_type": "image/png", + "size_bytes": len(PNG_1x1), + }, + {"kind": "text", "filename": "notes.md", "mime_type": "text/markdown", "size_bytes": 2}, ] def test_attachments_meta_stripped_before_openai_wire(self, tmp_db, mock_openai_client): @@ -362,3 +367,35 @@ class TestTokenAccounting: # The ~4000-char doc lands at the resolved boundary (the per-turn # placeholder no longer carries the bytes), well above the budget floor. assert doc_chars - plain_chars >= 900 + + def test_by_reference_doc_counted_without_materialization(self): + """R1: a canonical by-reference document turn (NOT yet materialized) must + count its size in the char budget via ``_attachments_meta``. Before the + fix the placeholder carried no bytes, so ``doc_chars`` was 0 and a reloaded + document conversation under-counted its context (the budget feeds + compaction / trim decisions).""" + meta = [ + {"kind": "text", "filename": "big.md", "mime_type": "text/markdown", "size_bytes": 4000} + ] + by_ref = { + "role": "user", + "content": [ + {"type": "text", "text": "see doc"}, + {"type": "document", "attachment_id": "x"}, + ], + "_attachments_meta": meta, + } + _t, _i, doc_chars = ChatSession._msg_text_chars(by_ref) + assert doc_chars == 4000 # was 0 before the fix + + # A materialized inline document that still carries meta must count ONCE, + # not twice — the inline_doc guard suppresses the meta term. + inline_plus_meta = { + "role": "user", + "content": [ + {"type": "document", "document": {"data": "x" * 4000, "name": "", "media_type": ""}} + ], + "_attachments_meta": meta, + } + _t2, _i2, doc2 = ChatSession._msg_text_chars(inline_plus_meta) + assert doc2 == 4000 diff --git a/tests/test_storage_attachments.py b/tests/test_storage_attachments.py index 07f85580..8e7aea3c 100644 --- a/tests/test_storage_attachments.py +++ b/tests/test_storage_attachments.py @@ -52,9 +52,7 @@ class TestContentAddressedWrite: def test_origin_tool_recorded(self, backend): backend.register_workstream("ws-origin") aid = _hash(PNG_1x1) - backend.save_attachment( - aid, "t.png", "image/png", len(PNG_1x1), "image", PNG_1x1, "tool" - ) + backend.save_attachment(aid, "t.png", "image/png", len(PNG_1x1), "image", PNG_1x1, "tool") row = backend.get_attachment(aid) assert row is not None assert row["origin"] == "tool" @@ -91,9 +89,7 @@ class TestGetAttachments: a1 = _hash(b"one") a2 = _hash(PNG_1x1) backend.save_attachment(a1, "one.txt", "text/plain", 3, "text", b"one") - backend.save_attachment( - a2, "img.png", "image/png", len(PNG_1x1), "image", PNG_1x1 - ) + backend.save_attachment(a2, "img.png", "image/png", len(PNG_1x1), "image", PNG_1x1) by_id = {r["attachment_id"]: r for r in backend.get_attachments([a1, a2])} assert by_id[a1]["content"] == b"one" assert by_id[a2]["content"] == PNG_1x1 @@ -166,12 +162,8 @@ class TestLoadMessagesReconstructsMultipart: msg_id = backend.save_message("ws-multi", "user", "look at these") img_id = _hash(PNG_1x1) doc_id = _hash(b"# hi\n") - backend.save_attachment( - img_id, "tiny.png", "image/png", len(PNG_1x1), "image", PNG_1x1 - ) - backend.save_attachment( - doc_id, "notes.md", "text/markdown", 5, "text", b"# hi\n" - ) + backend.save_attachment(img_id, "tiny.png", "image/png", len(PNG_1x1), "image", PNG_1x1) + backend.save_attachment(doc_id, "notes.md", "text/markdown", 5, "text", b"# hi\n") backend.set_message_attachments("ws-multi", msg_id, [img_id, doc_id]) msgs = backend.load_messages("ws-multi") @@ -292,7 +284,12 @@ class TestReconstructMetaSibling: backend.set_message_attachments("ws-meta", mid, [aid]) meta = backend.load_messages("ws-meta")[0].get("_attachments_meta") assert isinstance(meta, list) and len(meta) == 1 - assert meta[0] == {"kind": "text", "filename": "doc.md", "mime_type": "text/markdown"} + assert meta[0] == { + "kind": "text", + "filename": "doc.md", + "mime_type": "text/markdown", + "size_bytes": 2, + } class TestRefcountGC: @@ -321,14 +318,10 @@ class TestRefcountGC: backend.register_workstream("ws-shared") shared = _hash(b"shared-bytes") m1 = backend.save_message("ws-shared", "user", "first") - backend.save_attachment( - shared, "s.txt", "text/plain", 12, "text", b"shared-bytes" - ) + backend.save_attachment(shared, "s.txt", "text/plain", 12, "text", b"shared-bytes") backend.set_message_attachments("ws-shared", m1, [shared]) m2 = backend.save_message("ws-shared", "user", "second") - backend.save_attachment( - shared, "s.txt", "text/plain", 12, "text", b"shared-bytes" - ) + backend.save_attachment(shared, "s.txt", "text/plain", 12, "text", b"shared-bytes") backend.set_message_attachments("ws-shared", m2, [shared]) assert backend.get_attachment(shared)["refcount"] == 2 @@ -355,14 +348,10 @@ class TestRefcountGC: backend.register_workstream("ws-two") shared = _hash(b"cross-ws") m1 = backend.save_message("ws-one", "user", "a") - backend.save_attachment( - shared, "s.txt", "text/plain", 8, "text", b"cross-ws" - ) + backend.save_attachment(shared, "s.txt", "text/plain", 8, "text", b"cross-ws") backend.set_message_attachments("ws-one", m1, [shared]) m2 = backend.save_message("ws-two", "user", "b") - backend.save_attachment( - shared, "s.txt", "text/plain", 8, "text", b"cross-ws" - ) + backend.save_attachment(shared, "s.txt", "text/plain", 8, "text", b"cross-ws") backend.set_message_attachments("ws-two", m2, [shared]) assert backend.get_attachment(shared)["refcount"] == 2 @@ -410,9 +399,7 @@ class TestParametrizedKind: payload = PNG_1x1 if kind == "image" else b"x" * 42 mime = "image/png" if kind == "image" else "text/plain" aid = _hash(payload) - backend.save_attachment( - aid, f"f.{kind}", mime, len(payload), kind, payload - ) + backend.save_attachment(aid, f"f.{kind}", mime, len(payload), kind, payload) rows = backend.get_attachments([aid]) assert len(rows) == 1 assert rows[0]["content"] == payload diff --git a/turnstone/api/server_schemas.py b/turnstone/api/server_schemas.py index 92c56e20..bf50127f 100644 --- a/turnstone/api/server_schemas.py +++ b/turnstone/api/server_schemas.py @@ -491,9 +491,10 @@ class WorkstreamHistoryResponse(BaseModel): default_factory=list, description=( "Tail of the workstream's message history, projected to the " - "canonical render shape (flat tool_calls with verdict / " - "output_assessment, top-level source / reminders / " - "attachments, derived denied / is_error / pending). Bounded " + "canonical render shape (``role`` may be ``system`` for " + "operator-context turns; flat tool_calls with verdict / " + "output_assessment; top-level source / attachments / reasoning; " + "derived denied / is_error / pending). Bounded " "by the ``limit`` query parameter (default 100, max 500)." ), ) diff --git a/turnstone/core/attachment_buffer.py b/turnstone/core/attachment_buffer.py index e2a9a7c1..465512c6 100644 --- a/turnstone/core/attachment_buffer.py +++ b/turnstone/core/attachment_buffer.py @@ -23,7 +23,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Callable, Iterable + from collections.abc import Callable # OOM-safety backstops (per node), not a product policy — the per-user upload cap was # removed. Generous: the goal is only to bound a pathological flood of unsent uploads. @@ -113,21 +113,6 @@ class AttachmentBuffer: del self._entries[handle] return True - def take(self, handles: Iterable[str], *, ws_id: str) -> list[StagedAttachment]: - """Pop the staged entries for *handles* at send-commit (ws-scoped). - - Missing / out-of-scope handles are skipped — the send proceeds with whatever - committed (a buffer eviction or crash between upload and send drops the upload). - """ - with self._lock: - taken: list[StagedAttachment] = [] - for handle in handles: - entry = self._entries.get(handle) - if entry is not None and entry.ws_id == ws_id: - taken.append(entry) - del self._entries[handle] - return taken - # -- eviction (caller holds the lock) ------------------------------------ def _evict_expired_locked(self) -> None: if self._ttl <= 0: diff --git a/turnstone/core/fence.py b/turnstone/core/fence.py index a842e9f4..08a0d009 100644 --- a/turnstone/core/fence.py +++ b/turnstone/core/fence.py @@ -11,7 +11,7 @@ forge or break the boundary. One mechanism, two trust polarities: declares the fence *form* (````) as untrusted data, so a fresh per-call nonce is enough. -* **Operator fold** (``ChatSession._fold_system_turns``) wraps TRUSTED operator +* **Operator fold** (``lowering.fold_system_turns``) wraps TRUSTED operator instructions folded into a neighbouring turn for models without native mid-conversation system messages. The nonce stops untrusted host text from forging a *fake* trusted block; the system prompt diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index 64d91593..14c5590f 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -368,7 +368,7 @@ class AnthropicProvider: messages (the base prompt) still hoist into the top-level ``system`` param, but a system message appearing AFTER a non-system turn is a mid-conversation operator turn (see - ``ChatSession._fold_system_turns``) and is emitted inline as a + ``lowering.fold_system_turns``) and is emitted inline as a ``{"role": "system"}`` message so it keeps its trajectory position. When False (every other model) all system messages hoist, as before — the fold pass has already removed any diff --git a/turnstone/core/session.py b/turnstone/core/session.py index fab5f389..274e94fe 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -3580,6 +3580,9 @@ class ChatSession: "kind": a.kind, "filename": a.filename, "mime_type": a.mime_type, + # Doc-budget proxy mirrored from the reconstruct path so the + # live and reloaded shapes count identically (``_msg_text_chars``). + "size_bytes": len(a.content), } for a in attachments ] @@ -4803,6 +4806,7 @@ class ChatSession: n = 0 images = 0 doc_chars = 0 + inline_doc = False if isinstance(content, list): for p in content: ptype = p.get("type") @@ -4813,14 +4817,29 @@ class ChatSession: # — both cost one fixed image budget. images += 1 elif ptype == "document" and not p.get("attachment_id"): - # Resolved inline document; the by-reference placeholder carries - # no bytes, so its char budget lands at send-time calibration. + # Resolved inline document (the transient materialized form): + # count its data chars directly. + inline_doc = True d = p.get("document", {}) doc_chars += len(d.get("data", "")) doc_chars += len(d.get("name", "")) doc_chars += len(d.get("media_type", "")) else: n += len(content or "") + # A by-reference document placeholder (``{type:document, attachment_id}``) + # carries no inline bytes, so its budget comes from the sibling + # ``_attachments_meta`` (``size_bytes`` per text-kind attachment). Skip + # when an inline document was already counted: canonical messages are + # by-reference + meta and the materialized wire form is inline-without-meta, + # so the two are mutually exclusive — the guard makes that robust either way. + if not inline_doc: + meta = msg.get("_attachments_meta") + if isinstance(meta, list): + doc_chars += sum( + int(e.get("size_bytes") or 0) + for e in meta + if isinstance(e, dict) and e.get("kind") == "text" + ) for tc in msg.get("tool_calls", []): n += len(tc.get("id", "")) n += len(tc.get("function", {}).get("name", "")) diff --git a/turnstone/core/storage/_utils.py b/turnstone/core/storage/_utils.py index bedfbf0d..bec2e99c 100644 --- a/turnstone/core/storage/_utils.py +++ b/turnstone/core/storage/_utils.py @@ -242,6 +242,10 @@ def _reconstruct_attachment_refs( "kind": str(att.get("kind") or ""), "filename": str(att.get("filename") or ""), "mime_type": str(att.get("mime_type") or ""), + # Doc-budget proxy for the by-reference placeholder (which carries + # no inline bytes): the token estimator reads this so a reloaded + # document turn isn't counted as ~free. See ``_msg_text_chars``. + "size_bytes": int(att.get("size_bytes") or 0), } ) return refs, meta diff --git a/turnstone/core/tool_advisory.py b/turnstone/core/tool_advisory.py index b1f1a438..b34cb24f 100644 --- a/turnstone/core/tool_advisory.py +++ b/turnstone/core/tool_advisory.py @@ -8,7 +8,7 @@ either kept inline (native mid-conversation system messages — claude-opus-4-8) or folded into the preceding turn as a nonce-delimited ```` fence for every other model. The fence mechanism (mint / neutralise / wrap) lives in :mod:`turnstone.core.fence`, shared with the output-guard judge -so the two trust boundaries cannot drift; ``ChatSession._fold_system_turns`` +so the two trust boundaries cannot drift; ``lowering.fold_system_turns`` applies it. This module also hosts :func:`parse_priority` (the ``!!!`` priority prefix on @@ -122,7 +122,7 @@ def make_system_turn(source: str, content: str, **meta: Any) -> dict[str, Any]: done here. It belongs to the fallback fold step, which wraps the content in a nonce-delimited ```` fence via :func:`turnstone.core.fence.wrap` (applied in - ``ChatSession._fold_system_turns``). Escaping in this builder would corrupt + ``lowering.fold_system_turns``). Escaping in this builder would corrupt the native path, where there is no fence to break out of. """ if source not in SYSTEM_TURN_SOURCES: