From cd7e3ab787b1b45db7ece1c409dab07a8fcdfd88 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Mon, 15 Jun 2026 18:50:07 -0700 Subject: [PATCH] feat(attachments): universal perception fallback for non-native modalities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `perception.model_alias` model role: when the primary model can't ingest an attachment natively and can't be shown a degraded-but-native form, a configured perception model perceives it and its output is carried as text. Mirrors the STT role — a role alias plus a module-level memo so the extra LLM round-trip runs once per attachment, not once per conversation turn. The call goes through the provider abstraction's create_completion (the path the intent judge uses), so any vision/omni provider works. Bottom-tier, universal ladder — perception only fills the remaining gap: - pdf : native supports_pdf -> rasterize-to-vision-primary -> perception -> extracted text -> placeholder - image: native vision -> perception (non-vision primary) -> native image_url - audio: native supports_audio_input -> STT -> perception (omni) -> placeholder Folds in two review findings the role subsumes: - bug-1: thread the active attempt's capabilities into _resolve_attachments (bound in _try_stream) so a model fallback materializes attachments against the fallback model's caps, not the primary's. - bug-2: charge a by-reference pdf/audio a bounded budget min(size_bytes, 16K) instead of zero, so a large-attachment turn isn't budgeted as ~empty (the exact materialized size isn't known until wire build). --- tests/test_perception.py | 91 ++++++++++++++ tests/test_session_attachments.py | 146 +++++++++++++++++++++ turnstone/core/perception.py | 142 +++++++++++++++++++++ turnstone/core/session.py | 188 +++++++++++++++++++++++----- turnstone/core/settings_registry.py | 15 +++ 5 files changed, 554 insertions(+), 28 deletions(-) create mode 100644 tests/test_perception.py create mode 100644 turnstone/core/perception.py diff --git a/tests/test_perception.py b/tests/test_perception.py new file mode 100644 index 00000000..9d3293af --- /dev/null +++ b/tests/test_perception.py @@ -0,0 +1,91 @@ +"""Unit tests for the perception wire-fallback (turnstone/core/perception.py).""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any + +import pytest + +from turnstone.core import perception + +if TYPE_CHECKING: + from collections.abc import Iterator + + +class _StubProvider: + """Minimal LLMProvider stand-in: counts calls, can fail the first N.""" + + def __init__(self, *, content: str = "a description", fail_times: int = 0) -> None: + self.calls = 0 + self._content = content + self._fail_times = fail_times + self.last_messages: list[dict[str, Any]] | None = None + + def create_completion( + self, *, client: Any, model: str, messages: list[dict[str, Any]], **_: Any + ) -> SimpleNamespace: + self.calls += 1 + self.last_messages = messages + if self.calls <= self._fail_times: + raise RuntimeError("backend down") + return SimpleNamespace(content=self._content) + + +@pytest.fixture(autouse=True) +def _clear_cache() -> Iterator[None]: + perception._clear_perception_cache_for_test() + yield + perception._clear_perception_cache_for_test() + + +def _parts() -> list[dict[str, Any]]: + return [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}] + + +def test_describe_builds_prompt_then_parts() -> None: + prov = _StubProvider(content="desc") + out = perception.describe(provider=prov, client=object(), model="m", parts=_parts()) # type: ignore[arg-type] + assert out == "desc" + assert prov.last_messages is not None + content = prov.last_messages[0]["content"] + assert content[0]["type"] == "text" # prompt leads + assert content[1]["type"] == "image_url" # attachment parts follow + + +def test_describe_empty_parts_skips_backend() -> None: + prov = _StubProvider() + assert perception.describe(provider=prov, client=object(), model="m", parts=[]) == "" # type: ignore[arg-type] + assert prov.calls == 0 + + +def test_describe_cached_memoizes_by_alias_and_hash() -> None: + prov = _StubProvider(content="desc") + kw: dict[str, Any] = { + "provider": prov, + "client": object(), + "model": "m", + "alias": "omni", + "content_hash": "h1", + "parts": _parts(), + } + assert perception.describe_cached(**kw) == "desc" + assert perception.describe_cached(**kw) == "desc" + assert prov.calls == 1 # second served from cache + perception.describe_cached(**{**kw, "content_hash": "h2"}) + assert prov.calls == 2 # distinct hash → fresh perceive + + +def test_describe_cached_does_not_cache_failures() -> None: + prov = _StubProvider(content="recovered", fail_times=1) + kw: dict[str, Any] = { + "provider": prov, + "client": object(), + "model": "m", + "alias": "omni", + "content_hash": "h", + "parts": _parts(), + } + assert perception.describe_cached(**kw) == "" # backend down → "" (uncached) + assert perception.describe_cached(**kw) == "recovered" # retried, succeeds + assert prov.calls == 2 diff --git a/tests/test_session_attachments.py b/tests/test_session_attachments.py index c34365a4..d92c72cb 100644 --- a/tests/test_session_attachments.py +++ b/tests/test_session_attachments.py @@ -2,10 +2,12 @@ from __future__ import annotations +from types import SimpleNamespace from unittest.mock import MagicMock import pytest +from turnstone.core import perception from turnstone.core.attachments import Attachment from turnstone.core.memory import ( get_attachment, @@ -525,3 +527,147 @@ class TestCapabilityGatedFallback: out = materialize_attachments(msgs, resolve) types = [p["type"] for p in out[0]["content"]] assert types == ["text", "image_url", "image_url"] + + +class TestPerceptionFallback: + """Universal perception bottom tier: image/PDF/audio for primaries that + can't ingest them, when a capable perception model is configured.""" + + def _att(self, kind, content=b"x", fn="f", mime="application/octet-stream"): + return { + "attachment_id": "aP", + "filename": fn, + "mime_type": mime, + "kind": kind, + "content": content, + } + + def _with_perception(self, s, *, perc_caps, content="DESCRIPTION"): + """Wire a stub perception backend onto the session; return the provider mock.""" + perception._clear_perception_cache_for_test() + prov = MagicMock() + prov.create_completion.return_value = SimpleNamespace(content=content) + s._config_store = MagicMock() + s._config_store.get = lambda k, *a: "omni" if k == "perception.model_alias" else "" + s._registry = MagicMock() + s._registry.has_alias = lambda a: a == "omni" + s._registry.resolve = lambda a: (object(), "omni-model", object()) + s._registry.get_provider = lambda a: prov + s._resolve_capabilities = lambda *a, **k: perc_caps # type: ignore[method-assign] + return prov + + def test_image_perception_when_primary_blind(self, tmp_db, mock_openai_client): + s = _make_session(mock_openai_client) + prov = self._with_perception(s, perc_caps=ModelCapabilities(supports_vision=True)) + part = s._wire_content_part( + self._att("image", PNG_1x1, "i.png", "image/png"), + ModelCapabilities(), # primary: no vision + ) + assert part["type"] == "text" + assert "DESCRIPTION" in part["text"] + assert "image attachment 'i.png'" in part["text"] + prov.create_completion.assert_called_once() + + def test_image_falls_through_to_native_without_perception(self, tmp_db, mock_openai_client): + # No perception configured (registry/config_store None) → native image_url: + # the pre-existing behavior; perception is purely additive. + s = _make_session(mock_openai_client) + part = s._wire_content_part( + self._att("image", PNG_1x1, "i.png", "image/png"), + ModelCapabilities(), + ) + assert part["type"] == "image_url" + + def test_pdf_perception_renders_pages(self, tmp_db, mock_openai_client, monkeypatch): + s = _make_session(mock_openai_client) + monkeypatch.setattr("turnstone.core.pdf.rasterize_pdf", lambda data: [b"pg1", b"pg2"]) + prov = self._with_perception(s, perc_caps=ModelCapabilities(supports_vision=True)) + part = s._wire_content_part( + self._att("pdf", b"%PDF", "r.pdf", "application/pdf"), + ModelCapabilities(supports_pdf=False), # primary: no pdf, no vision + ) + assert part["type"] == "text" + assert "DESCRIPTION" in part["text"] + # the perception model was handed the rasterized pages, not the raw PDF + sent = prov.create_completion.call_args.kwargs["messages"][0]["content"] + assert [p["type"] for p in sent] == ["text", "image_url", "image_url"] + + def test_audio_perception_when_omni_and_no_stt(self, tmp_db, mock_openai_client): + s = _make_session(mock_openai_client) + self._with_perception(s, perc_caps=ModelCapabilities(supports_audio_input=True)) + part = s._wire_content_part( + self._att("audio", b"RIFFxxxxWAVE", "a.wav", "audio/wav"), + ModelCapabilities(supports_audio_input=False), + ) + assert part["type"] == "text" + assert "DESCRIPTION" in part["text"] + + def test_perception_skipped_when_model_lacks_modality(self, tmp_db, mock_openai_client): + # Perception model has vision but not audio → audio falls through to the + # placeholder rather than calling a model that can't hear. + s = _make_session(mock_openai_client) + prov = self._with_perception(s, perc_caps=ModelCapabilities(supports_vision=True)) + part = s._wire_content_part( + self._att("audio", b"RIFF", "a.wav", "audio/wav"), + ModelCapabilities(supports_audio_input=False), + ) + assert part["type"] == "text" + assert "no transcription backend" in part["text"] + prov.create_completion.assert_not_called() + + +class TestResolveAttachmentsCapsThreading: + """bug-1: the resolver materializes against the caps it is handed (the active + attempt's), not the primary session model's.""" + + def _att(self): + return { + "attachment_id": "aT", + "filename": "r.pdf", + "mime_type": "application/pdf", + "kind": "pdf", + "content": b"%PDF", + } + + def test_resolver_uses_passed_caps(self, tmp_db, mock_openai_client, monkeypatch): + s = _make_session(mock_openai_client) + monkeypatch.setattr("turnstone.core.session.get_attachments", lambda ids: [self._att()]) + monkeypatch.setattr("turnstone.core.pdf.rasterize_pdf", lambda data: [b"pg"]) + # Passed caps (vision, no native PDF) drive rasterize-to-images — not + # whatever the primary 'test-model' happens to support. + out = s._resolve_attachments( + ["aT"], ModelCapabilities(supports_pdf=False, supports_vision=True) + ) + part = out["aT"] + assert isinstance(part, list) + assert all(p["type"] == "image_url" for p in part) + + def test_resolver_native_with_pdf_caps(self, tmp_db, mock_openai_client, monkeypatch): + s = _make_session(mock_openai_client) + monkeypatch.setattr("turnstone.core.session.get_attachments", lambda ids: [self._att()]) + out = s._resolve_attachments(["aT"], ModelCapabilities(supports_pdf=True)) + assert out["aT"]["type"] == "document" + assert out["aT"]["document"]["media_type"] == "application/pdf" + + +class TestByReferenceMediaBudget: + """bug-2: by-reference pdf/audio are charged a bounded budget — not zero + (over-context), not the full multi-MB source blob (over-trim).""" + + def test_pdf_and_audio_charged_capped(self): + msg = { + "role": "user", + "content": [], + "_attachments_meta": [ + {"kind": "pdf", "size_bytes": 32_000_000}, + {"kind": "audio", "size_bytes": 25_000_000}, + {"kind": "text", "size_bytes": 500}, + {"kind": "image", "size_bytes": 99}, + ], + } + _text, images, doc_chars = ChatSession._msg_text_chars(msg) + # pdf + audio each capped at 16_000; text counted in full; image excluded + # (a real by-reference image is charged a fixed image budget in the + # content loop, so counting it here too would double-charge). + assert doc_chars == 16_000 + 16_000 + 500 + assert images == 0 diff --git a/turnstone/core/perception.py b/turnstone/core/perception.py new file mode 100644 index 00000000..13301dbe --- /dev/null +++ b/turnstone/core/perception.py @@ -0,0 +1,142 @@ +"""Universal perception fallback for attachments the active model can't ingest. + +When the primary model lacks native support for an attachment's modality — and +can't be shown a degraded-but-native form either (a non-vision model can't read +rasterized PDF pages) — a separately-configured "perception" model perceives the +attachment and its description/transcript is sent as a text part. This mirrors +the speech-to-text fallback in :mod:`turnstone.core.audio`: a model-role alias +(``perception.model_alias``) plus a module-level memo so the perceive call — an +extra LLM round-trip — runs once per attachment, not once per conversation turn. + +It is a *bottom-tier, universal* safety net: + +* vision: native ``supports_pdf``/``supports_vision`` → rasterize-to-vision-primary + (PDF) → **perception** (if the perception model has vision) → extract-text / placeholder. +* audio: native ``supports_audio_input`` → STT transcription role → **perception** + (if the perception model has audio input) → placeholder. + +A vision-capable primary still receives the real image / rasterized pages, and a +configured STT model still wins for audio — perception only fills the remaining +gap. Point it at an omni model (text+vision+audio) to cover every modality from +one alias; a vision-only model covers image/PDF and is simply skipped for audio. + +The call goes through the provider abstraction's ``create_completion`` (the same +path the intent judge uses for its secondary model), so any provider works; the +parts are OpenAI-shaped (``image_url`` / ``input_audio``) and the provider +translates them to its own wire form. +""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING, Any + +from turnstone.core.log import get_logger + +if TYPE_CHECKING: + from turnstone.core.providers._protocol import LLMProvider + +log = get_logger(__name__) + +# Config key naming the model used for perception fallbacks. +PERCEPTION_SETTING = "perception.model_alias" + +_DESCRIBE_PROMPT = ( + "You are a perception backend for another AI model that cannot perceive this " + "attachment. Convey it in full, faithful detail: transcribe all text and " + "speech verbatim, and describe any figures, tables, diagrams, layout, or " + "non-speech audio. Do not summarize away or omit content — the reader relies " + "entirely on your output to understand the attachment." +) + + +class PerceptionUnavailableError(RuntimeError): + """No usable perception backend is configured/resolvable (maps to a placeholder).""" + + +class PerceptionBackendError(RuntimeError): + """A configured perception backend failed during the perceive call.""" + + +def describe( + *, + provider: LLMProvider, + client: Any, + model: str, + parts: list[dict[str, Any]], + prompt: str = _DESCRIBE_PROMPT, +) -> str: + """Perceive ``parts`` via the perception model, returning the text. + + ``parts`` are OpenAI-shaped content parts — ``image_url`` for image/PDF-page + perception, ``input_audio`` for audio (the provider translates them to its + own wire shape). Raises :class:`PerceptionBackendError` if the backend call + fails. Never caches — see :func:`describe_cached`. + """ + if not parts: + return "" + messages = [{"role": "user", "content": [{"type": "text", "text": prompt}, *parts]}] + try: + result = provider.create_completion( + client=client, + model=model, + messages=messages, + max_tokens=4096, + temperature=0.2, + ) + except Exception as exc: + raise PerceptionBackendError(f"perception backend failed: {exc}") from exc + return (result.content or "").strip() + + +# -- perception memoization (no-native-modality wire fallback) ---------------- +# Mirrors audio.transcribe_cached: the wire resolver re-materializes every +# attachment on every send, so without this memo an attachment perceived early +# in a conversation would be re-perceived (an extra LLM round-trip) on every +# subsequent turn. +_CACHE_MAX = 256 +_cache_lock = threading.Lock() +_cache: dict[str, str] = {} + + +def _clear_perception_cache_for_test() -> None: + with _cache_lock: + _cache.clear() + + +def describe_cached( + *, + provider: LLMProvider, + client: Any, + model: str, + alias: str, + content_hash: str, + parts: list[dict[str, Any]], + prompt: str = _DESCRIBE_PROMPT, +) -> str: + """Memoized, non-raising :func:`describe` for the wire fallback. + + Keyed by ``(alias, content_hash)``. Returns ``""`` on a backend failure (a + placeholder is rendered upstream) and does *not* cache failures, so a + transient outage doesn't poison the memo. + """ + key = f"{alias}:{content_hash}" + with _cache_lock: + if key in _cache: + return _cache[key] + try: + text = describe( + provider=provider, + client=client, + model=model, + parts=parts, + prompt=prompt, + ) + except PerceptionBackendError as exc: + log.warning("perception fallback failed (alias=%s): %s", alias, exc) + return "" + with _cache_lock: + if key not in _cache and len(_cache) >= _CACHE_MAX: + _cache.pop(next(iter(_cache)), None) + _cache[key] = text + return text diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 39c9f374..2ae24a78 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -619,6 +619,13 @@ _SPEC_ARGUMENTS_LITERAL_RE = re.compile(r"\$ARGUMENTS\b(?!\[)") # not its earliest. _WATCH_QUEUE_SOFT_CAP = 50 +# Bounded budget charge for a by-reference pdf/audio attachment. Its source +# blob can be multi-MB, but the form the model actually sees (perception / STT / +# extracted text, or rasterized pages) is far smaller and its exact size isn't +# known until wire build — so the trimming budget charges min(size_bytes, this). +# Sized to the perception describe cap (max_tokens ~4096 -> ~16K chars). +_DOC_BUDGET_CHAR_CAP = 16_000 + _RERANK_TIMEOUT_CAP_S = 15.0 # reranking <=50 short docs is fast; cap so a hung # endpoint falls back to BM25 in seconds, not up to tools.timeout (120s default). # Per-turn memory rerank makes the long timeout a turn-stall hazard. @@ -2925,7 +2932,9 @@ class ChatSession: via :meth:`_resolve_attachments` — resolution lives at the C layer.""" return self.system_messages + dicts_from_turns(self.messages) - def _resolve_attachments(self, ids: list[str]) -> dict[str, Any]: + def _resolve_attachments( + self, ids: list[str], caps: ModelCapabilities | None = None + ) -> dict[str, Any]: """Resolve content-addressed attachment ids to inline wire content parts. The send-time materialization of the by-reference content lane: handed to @@ -2941,7 +2950,11 @@ class ChatSession: fires on a history render.""" if not ids: return {} - caps = self._get_capabilities() + # caps is the ACTIVE attempt's capabilities, threaded from _try_stream so + # a fallback to a model with different media support converts on the + # right caps; default to the primary only when called without one. + if caps is None: + caps = self._get_capabilities() out: dict[str, Any] = {} for att in get_attachments(ids): part = self._wire_content_part(att, caps) @@ -2953,18 +2966,31 @@ class ChatSession: self, att: dict[str, Any], caps: ModelCapabilities ) -> dict[str, Any] | list[dict[str, Any]] | None: """The active model's inline part(s) for one blob: native where - supported, else a client-side fallback for a kind it can't read. PDF → - rasterized page images (vision models) or extracted text; audio → STT - transcript. A PDF rasterized to images returns several parts.""" + supported, else the fallback ladder for a kind it can't read. + + PDF → rasterized page images (vision primary) → perception → extracted + text → placeholder. Image → native image_url, or perception first when + the primary has no vision. Audio → STT transcript → perception → + placeholder. Perception (the ``perception.model_alias`` role) is the + universal bottom tier: it engages only when the primary can't handle the + kind and a capable perception model is configured. A PDF rasterized to + images returns several parts.""" kind = att.get("kind") if kind == "pdf" and not caps.supports_pdf: - # Vision models: rasterize pages to images (better fidelity, esp. for - # scanned PDFs with no text layer); otherwise extract text. + # Vision primary: rasterize pages to images (better fidelity, esp. + # for scanned PDFs with no text layer). Non-vision primary: + # perception, else extracted text / placeholder. if caps.supports_vision: return self._pdf_rasterize_fallback_parts(att) - return self._pdf_text_fallback_part(att) + return self._pdf_nonvision_part(att) + if kind == "image" and not caps.supports_vision: + perceived = self._perception_fallback_part(att, "image") + if perceived is not None: + return perceived + # No perception backend configured: emit the native image_url + # unchanged (the model may ignore it) — pre-existing behavior. if kind == "audio" and not caps.supports_audio_input: - return self._audio_transcript_fallback_part(att) + return self._audio_fallback_part(att) return attachment_to_content_part(att) def _pdf_rasterize_fallback_parts( @@ -3014,23 +3040,42 @@ class ChatSession: }, } - def _audio_transcript_fallback_part(self, att: dict[str, Any]) -> dict[str, Any]: - """Non-omni model: transcribe via the STT role and carry the transcript. + def _pdf_nonvision_part(self, att: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]: + """Non-vision primary + PDF: perception (renders pages for a perception + model that can see) when configured, else extracted text / placeholder.""" + perceived = self._perception_fallback_part(att, "pdf") + if perceived is not None: + return perceived + return self._pdf_text_fallback_part(att) - Only engages when an operator has configured an STT role (which may be a - local backend) — otherwise a placeholder, never a surprise external call.""" + def _audio_fallback_part(self, att: dict[str, Any]) -> dict[str, Any]: + """Non-omni primary + audio: STT transcript (preferred), else perception + (if the perception model can hear), else a placeholder.""" + transcript = self._stt_transcript_part(att) + if transcript is not None: + return transcript + perceived = self._perception_fallback_part(att, "audio") + if perceived is not None: + return perceived + name = str(att.get("filename") or "audio") + return { + "type": "text", + "text": f"[audio attachment '{name}' — no transcription backend configured]", + } + + def _stt_transcript_part(self, att: dict[str, Any]) -> dict[str, Any] | None: + """Transcribe via the STT role, or ``None`` when no STT role is + configured or the transcript is empty (caller falls through to + perception). Only engages a configured backend — never a surprise call.""" from turnstone.core.audio import resolve_role_alias, transcribe_cached - name = str(att.get("filename") or "audio") raw = att.get("content") alias = resolve_role_alias( config_store=self._config_store, registry=self._registry, role="stt" ) if not alias or not isinstance(raw, bytes): - return { - "type": "text", - "text": f"[audio attachment '{name}' — no transcription backend configured]", - } + return None + name = str(att.get("filename") or "audio") transcript = transcribe_cached( registry=self._registry, alias=alias, @@ -3039,15 +3084,92 @@ class ChatSession: filename=name, ) if not transcript: - return { - "type": "text", - "text": f"[audio attachment '{name}' — transcription unavailable]", - } + return None return { "type": "text", "text": f"[Transcript of audio attachment '{name}']\n\n{transcript}", } + def _resolve_perception( + self, + ) -> tuple[LLMProvider, Any, str, str, ModelCapabilities] | None: + """Resolve the perception role → ``(provider, client, model, alias, caps)``. + + ``None`` when no ``perception.model_alias`` is configured / resolvable, so + the caller falls through to the next fallback tier.""" + from turnstone.core.perception import PERCEPTION_SETTING + + if self._config_store is None or self._registry is None: + return None + alias = (self._config_store.get(PERCEPTION_SETTING) or "").strip() + if not alias or not self._registry.has_alias(alias): + return None + try: + client, model, _cfg = self._registry.resolve(alias) + provider = self._registry.get_provider(alias) + caps = self._resolve_capabilities(provider, model, alias) + except Exception as exc: + log.warning("perception alias %r not resolvable: %s", alias, exc) + return None + return provider, client, model, alias, caps + + def _perception_parts(self, att: dict[str, Any], kind: str) -> list[dict[str, Any]]: + """Build the OpenAI-shaped parts handed to the perception model: PDF → + rasterized page images; image / audio → the native content part.""" + raw = att.get("content") + if not isinstance(raw, bytes): + return [] + if kind == "pdf": + import base64 + + from turnstone.core.pdf import rasterize_pdf + + return [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{base64.b64encode(p).decode('ascii')}" + }, + } + for p in rasterize_pdf(raw) + ] + part = attachment_to_content_part(att) # image_url / input_audio, native shape + return [part] if part is not None else [] + + def _perception_fallback_part(self, att: dict[str, Any], kind: str) -> dict[str, Any] | None: + """Universal bottom-tier fallback: have the configured perception model + perceive the attachment and carry its output as text. ``None`` when no + perception backend is configured, it can't handle this modality, or it + produced nothing — the caller falls through.""" + resolved = self._resolve_perception() + if resolved is None: + return None + provider, client, model, alias, caps = resolved + if kind in ("pdf", "image") and not caps.supports_vision: + return None + if kind == "audio" and not caps.supports_audio_input: + return None + parts = self._perception_parts(att, kind) + if not parts: + return None + from turnstone.core.perception import describe_cached + + text = describe_cached( + provider=provider, + client=client, + model=model, + alias=alias, + content_hash=str(att.get("attachment_id")), + parts=parts, + ) + if not text: + return None + name = str(att.get("filename") or kind) + return { + "type": "text", + "text": f"[Perception of {kind} attachment '{name}']\n\n{text}", + } + def _prepare_wire_messages( self, messages: list[dict[str, Any]], @@ -3575,7 +3697,7 @@ class ChatSession: replay_reasoning_to_model=self._resolve_replay_reasoning_to_model( model_alias, caps=resolved_caps ), - resolve_attachments=self._resolve_attachments, + resolve_attachments=lambda ids: self._resolve_attachments(ids, resolved_caps), ) except Exception as e: ename = type(e).__name__ @@ -5004,11 +5126,21 @@ class ChatSession: if not inline_doc: meta = msg.get("_attachments_meta") if isinstance(meta, list): - doc_chars += sum( - int(e.get("size_bytes") or 0) - for e in meta - if isinstance(e, dict) and e.get("kind") == "text" - ) + for e in meta: + if not isinstance(e, dict): + continue + k = e.get("kind") + sz = int(e.get("size_bytes") or 0) + if k == "text": + doc_chars += sz + elif k in ("pdf", "audio"): + # By-reference media materializes to a much smaller form + # whose exact size isn't known here; charge a bounded + # estimate so the turn is neither budgeted as ~zero + # (over-context) nor as the full source blob (over-trim). + doc_chars += min(sz, _DOC_BUDGET_CHAR_CAP) + # image by-reference is already charged a fixed image budget + # in the content loop above. for tc in msg.get("tool_calls", []): n += len(tc.get("id", "")) n += len(tc.get("function", {}).get("name", "")) diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index c438e1d3..fdf47753 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -450,6 +450,21 @@ def _build_registry() -> dict[str, SettingDef]: "Degraded backends are deprioritised in the fallback chain but requests are never " "blocked. The backend recovers automatically when a request succeeds.", ), + # -- perception role ----------------------------------------------- + SettingDef( + "perception.model_alias", + "str", + "", + "Model alias for the perception fallback — image/PDF/audio (empty = disabled)", + "perception", + help="Which registered model perceives attachments a primary model can't ingest " + "natively — describing images/PDFs, and (for an omni model) transcribing audio — and " + "returns the result as text. Last-resort fallback only: a vision-capable primary still " + "gets the actual image / rasterized pages, and a configured speech-to-text model still " + "wins for audio; perception fills the remaining gap. Point it at a vision-capable (or " + "omni) chat model alias. Empty disables the fallback (such attachments then degrade to " + "extracted text or a placeholder).", + ), # -- audio / voice roles ------------------------------------------- # Keys are section-prefixed (audio.*) with distinct leaves so the # Settings tab (which labels by the key's last segment) doesn't render