diff --git a/README.md b/README.md index 8d197f18..a6bc7f65 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct | Tool | Description | Auto-approved | |------|-------------|:---:| | `bash` | Execute shell commands | | -| `read_file` | Read file contents | yes | +| `read_file` | Read file contents (text or images with vision models) | yes | | `write_file` | Write/create files | | | `edit_file` | Fuzzy-match file editing | | | `search` | Search files by name/content | yes | diff --git a/docs/architecture.md b/docs/architecture.md index f83fcac5..ef300eb5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -560,21 +560,23 @@ LLMProvider (protocol) |------|--------| | `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason` | | `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage` | -| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search` | +| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search`, `supports_tool_search`, `supports_vision` | | `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens` | **OpenAIProvider** (`_openai.py`): passes messages through unchanged (they are -already in OpenAI format). Model capability lookup table covers -GPT-5/5.1/5.2, O-series, and search models (`gpt-5-search-api`). +already in OpenAI format), including multi-part content blocks (text + images) +in tool results. Model capability lookup table covers GPT-5/5.1/5.2/5.3/5.4, +O-series, and search models (`gpt-5-search-api`) — all with `supports_vision`. For search models, injects `web_search_options` and removes the `web_search` function tool (the model always searches). Citations from `url_citation` annotations are formatted as footnotes. Unknown models (local servers) get -permissive defaults and use Tavily for web search. +permissive defaults with `supports_vision=False` and use Tavily for web search. **AnthropicProvider** (`_anthropic.py`): converts OpenAI-format messages to Anthropic content blocks, maps `system`/`developer` roles to the `system` parameter, groups consecutive `tool` result messages into user-role content -blocks, and translates tool schemas from OpenAI function-calling format to +blocks (converting `image_url` parts to Anthropic's `image` source format), +and translates tool schemas from OpenAI function-calling format to Anthropic's `input_schema` format. Supports both manual and adaptive thinking modes, with effort parameter support for models like Claude Opus 4.6 and Sonnet 4.6. Replaces the `web_search` function tool with Anthropic's native @@ -620,6 +622,18 @@ agent_model = "claude" Each `[models.*]` entry produces a `ModelConfig` with a `provider` field (default: `"openai"`). Supported values: `"openai"` and `"anthropic"`. +An optional `[models.*.capabilities]` sub-table overrides per-model +`ModelCapabilities` flags (useful for local models whose capabilities +cannot be detected programmatically): + +```toml +[models.qwen-vl] +base_url = "http://localhost:8000/v1" +model = "qwen-3.5-vl" + +[models.qwen-vl.capabilities] +supports_vision = true +``` **Lifecycle:** 1. `load_model_registry()` reads `[models.*]` sections from config.toml and diff --git a/docs/diagrams/03-core-engine-classes.puml b/docs/diagrams/03-core-engine-classes.puml index dfbaa3b4..8937e890 100644 --- a/docs/diagrams/03-core-engine-classes.puml +++ b/docs/diagrams/03-core-engine-classes.puml @@ -109,6 +109,7 @@ class "ModelCapabilities" as ModelCaps <> { + supports_effort: bool + supports_web_search: bool + supports_tool_search: bool + + supports_vision: bool } ' ChatSession diff --git a/docs/diagrams/04-conversation-turn.puml b/docs/diagrams/04-conversation-turn.puml index e4715015..0a543376 100644 --- a/docs/diagrams/04-conversation-turn.puml +++ b/docs/diagrams/04-conversation-turn.puml @@ -112,7 +112,7 @@ group loop [while tool_calls present] note right of TP Parallel execution: bash → Popen + line-by-line streaming - read_file → open().read() + read_file → open().read() or base64 image search → grep subprocess edit_file → string replace task/plan → _run_agent() sub-loop diff --git a/docs/diagrams/05-tool-pipeline.puml b/docs/diagrams/05-tool-pipeline.puml index 32a37e90..e30a8042 100644 --- a/docs/diagrams/05-tool-pipeline.puml +++ b/docs/diagrams/05-tool-pipeline.puml @@ -100,7 +100,7 @@ partition "Phase 3: Execute" #E3F2FD { if item.denied → return denial message else → item["execute"](item) ├─ _exec_bash: subprocess.run(["bash", script.sh]) - ├─ _exec_read_file: open().readlines() + ├─ _exec_read_file: open().readlines() or _exec_read_image (base64) ├─ _exec_write_file: makedirs + write ├─ _exec_edit_file: find_occurrences + replace ├─ _exec_search: grep subprocess diff --git a/docs/diagrams/png/03-core-engine-classes.png b/docs/diagrams/png/03-core-engine-classes.png index 090eb5ba..c102247c 100644 --- a/docs/diagrams/png/03-core-engine-classes.png +++ b/docs/diagrams/png/03-core-engine-classes.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4512be48a51f7cd1136ea8e3344489a8d225c611cb1b961643b869351de76812 -size 549668 +oid sha256:c53ddce800c59f9432d7a016c7d66282a449555d452b7fe9dd393f4282f08c46 +size 554721 diff --git a/docs/diagrams/png/04-conversation-turn.png b/docs/diagrams/png/04-conversation-turn.png index d4dc048e..0e99dfa7 100644 --- a/docs/diagrams/png/04-conversation-turn.png +++ b/docs/diagrams/png/04-conversation-turn.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:24bdc6a83259e4db6aaa24f581bed59d351c7a83e1c282aac123c52b32d9f80d -size 288250 +oid sha256:e3044c738d6d6853aab5c4990e6c67bab0165eba991a4f5bebdfc4d4a0b305ee +size 289165 diff --git a/docs/diagrams/png/05-tool-pipeline.png b/docs/diagrams/png/05-tool-pipeline.png index 47525813..46b4ec29 100644 --- a/docs/diagrams/png/05-tool-pipeline.png +++ b/docs/diagrams/png/05-tool-pipeline.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:027ad99469d69f1d6b2e73ee802b50a75617cd286392375aa69133c13d3683dc -size 256347 +oid sha256:282820fe416961e735d050f86ecdc079e29824d2b3c4d5c8c174d0533d41f211 +size 258045 diff --git a/docs/tools.md b/docs/tools.md index bbe0df86..59253f8d 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -189,15 +189,17 @@ Execute a bash command and return stdout + stderr. ### read_file -Read the contents of a file, returning numbered lines. +Read the contents of a file, returning numbered lines for text files or +base64-encoded image data for supported image formats. | Parameter | Type | Required | Description | |-----------|---------|----------|-------------| | `path` | string | yes | Absolute or relative file path. | -| `offset` | integer | no | Line number to start from (1-based, default: 1). | -| `limit` | integer | no | Maximum number of lines to read. Omit for full file. | +| `offset` | integer | no | Line number to start from (1-based, default: 1). Text files only. | +| `limit` | integer | no | Maximum number of lines to read. Omit for full file. Text files only. | -- **What it does**: Reads the file and returns content with line numbers. Must be called before `edit_file` on the same path (the session tracks which files have been read). +- **What it does**: For text files, reads and returns content with line numbers. For image files (PNG, JPEG, GIF, WebP, BMP, TIFF, ICO), returns image data as multi-part content when the model supports vision, or a text description when it does not. SVG files are read as text. Images larger than 4 MB are rejected. Must be called before `edit_file` on the same path (the session tracks which files have been read). +- **Vision support**: Controlled by `ModelCapabilities.supports_vision`. All commercial OpenAI and Anthropic models have vision enabled. Local models (vLLM, llama.cpp, NIM) default to off — enable via `[models.*.capabilities] supports_vision = true` in config.toml. - **Auto-approve**: Yes. - **Agent availability**: `agent` and `task_agent`. diff --git a/tests/test_providers.py b/tests/test_providers.py index abbd563c..184fb96a 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -2049,3 +2049,138 @@ class TestModelCapabilitiesToolSearch: caps = ModelCapabilities() assert caps.supports_tool_search is False + + +# --------------------------------------------------------------------------- +# Vision support +# --------------------------------------------------------------------------- + + +class TestVisionCapabilities: + """Test supports_vision flag across providers.""" + + def test_default_is_false(self) -> None: + from turnstone.core.providers._protocol import ModelCapabilities + + caps = ModelCapabilities() + assert caps.supports_vision is False + + def test_openai_commercial_supports_vision(self) -> None: + provider = OpenAIProvider() + for model in ("gpt-5", "gpt-5-mini", "gpt-5.4", "o3", "o4-mini"): + caps = provider.get_capabilities(model) + assert caps.supports_vision is True, f"{model} should support vision" + + def test_openai_default_no_vision(self) -> None: + """Unknown models (local servers) default to no vision.""" + provider = OpenAIProvider() + caps = provider.get_capabilities("some-local-model") + assert caps.supports_vision is False + + def test_anthropic_supports_vision(self) -> None: + from turnstone.core.providers._anthropic import AnthropicProvider + + provider = AnthropicProvider() + for model in ("claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5"): + caps = provider.get_capabilities(model) + assert caps.supports_vision is True, f"{model} should support vision" + + def test_anthropic_default_supports_vision(self) -> None: + """Anthropic default (unknown Claude model) supports vision.""" + from turnstone.core.providers._anthropic import AnthropicProvider + + provider = AnthropicProvider() + caps = provider.get_capabilities("claude-unknown-9") + assert caps.supports_vision is True + + +class TestAnthropicVisionConversion: + """Test image content conversion in _convert_messages.""" + + def setup_method(self) -> None: + from turnstone.core.providers._anthropic import AnthropicProvider + + self.provider = AnthropicProvider() + + def test_tool_result_with_image_content(self) -> None: + """Tool result with list content converts image_url to Anthropic image.""" + messages = [ + {"role": "user", "content": "Read this image"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "function": {"name": "read_file", "arguments": '{"path": "img.png"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "text", "text": "Image file: img.png (1024 bytes)"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ], + }, + ] + _, converted = self.provider._convert_messages(messages) + # Tool result should be in a user message + tool_user_msg = converted[2] + assert tool_user_msg["role"] == "user" + tool_result = tool_user_msg["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["tool_use_id"] == "call_1" + # Content should be a list with converted image block + content = tool_result["content"] + assert isinstance(content, list) + assert content[0] == {"type": "text", "text": "Image file: img.png (1024 bytes)"} + assert content[1]["type"] == "image" + assert content[1]["source"]["type"] == "base64" + assert content[1]["source"]["media_type"] == "image/png" + assert content[1]["source"]["data"] == "iVBORw0KGgo=" + + def test_tool_result_with_string_content_unchanged(self) -> None: + """Tool result with plain string content is unchanged.""" + messages = [ + {"role": "user", "content": "Read file"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_2", + "function": {"name": "read_file", "arguments": '{"path": "f.py"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_2", + "content": " 1\tprint('hello')", + }, + ] + _, converted = self.provider._convert_messages(messages) + tool_result = converted[2]["content"][0] + assert tool_result["content"] == " 1\tprint('hello')" + + def test_convert_content_parts_static_method(self) -> None: + """_convert_content_parts handles both image_url and text.""" + from turnstone.core.providers._anthropic import AnthropicProvider + + parts = [ + {"type": "text", "text": "description"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}, + }, + ] + result = AnthropicProvider._convert_content_parts(parts) + assert result[0] == {"type": "text", "text": "description"} + assert result[1]["type"] == "image" + assert result[1]["source"]["media_type"] == "image/jpeg" + assert result[1]["source"]["data"] == "/9j/4AAQ" diff --git a/tests/test_session.py b/tests/test_session.py index d24fe653..89eaff65 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,9 +1,10 @@ """Tests for turnstone.core.session — ChatSession construction.""" +import base64 import json from unittest.mock import MagicMock, patch -from turnstone.core.session import ChatSession +from turnstone.core.session import _IMAGE_EXTENSIONS, _IMAGE_SIZE_CAP, ChatSession class NullUI: @@ -265,3 +266,154 @@ class TestPlanExec: call_id, content, _ = self._run_plan(session, "do stuff", agent_return=agent_output) assert call_id == "test-call-1" assert content == agent_output + + +# --------------------------------------------------------------------------- +# Vision / image support +# --------------------------------------------------------------------------- + + +class TestImageExtensions: + """Test _IMAGE_EXTENSIONS constant and detection logic.""" + + def test_common_image_extensions(self): + for ext in (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".ico"): + assert ext in _IMAGE_EXTENSIONS, f"{ext} should be in _IMAGE_EXTENSIONS" + + def test_svg_excluded(self): + assert ".svg" not in _IMAGE_EXTENSIONS + + def test_text_extensions_excluded(self): + for ext in (".py", ".txt", ".json", ".md", ".rs", ".go"): + assert ext not in _IMAGE_EXTENSIONS + + +class TestExecReadImage: + """Test _exec_read_image method.""" + + def _make_png(self, path: str, size: int = 100) -> None: + """Write a minimal valid-ish PNG header to a file.""" + # 8-byte PNG signature + enough bytes to reach target size + header = b"\x89PNG\r\n\x1a\n" + with open(path, "wb") as f: + f.write(header + b"\x00" * max(0, size - len(header))) + + def test_image_returns_content_parts(self, tmp_db, tmp_path): + """read_file on a PNG with vision support returns content parts.""" + img = tmp_path / "test.png" + self._make_png(str(img)) + + session = _make_session() + # Mock provider to report vision support + mock_caps = MagicMock() + mock_caps.supports_vision = True + session._provider.get_capabilities = MagicMock(return_value=mock_caps) + + item = {"call_id": "c1", "path": str(img), "offset": None, "limit": None} + call_id, output = session._exec_read_file(item) + + assert call_id == "c1" + assert isinstance(output, list) + assert len(output) == 2 + assert output[0]["type"] == "text" + assert "test.png" in output[0]["text"] + assert output[1]["type"] == "image_url" + url = output[1]["image_url"]["url"] + assert url.startswith("data:image/png;base64,") + # Verify base64 round-trip + b64part = url.split(",", 1)[1] + decoded = base64.b64decode(b64part) + assert decoded == img.read_bytes() + + def test_no_vision_returns_text(self, tmp_db, tmp_path): + """read_file on image with non-vision model returns text description.""" + img = tmp_path / "photo.jpg" + self._make_png(str(img), size=2048) + + session = _make_session() + mock_caps = MagicMock() + mock_caps.supports_vision = False + session._provider.get_capabilities = MagicMock(return_value=mock_caps) + + item = {"call_id": "c2", "path": str(img), "offset": None, "limit": None} + call_id, output = session._exec_read_file(item) + + assert call_id == "c2" + assert isinstance(output, str) + assert "does not support vision" in output + assert "photo.jpg" in output + + def test_oversized_image_returns_error(self, tmp_db, tmp_path): + """Images exceeding _IMAGE_SIZE_CAP return an error string.""" + img = tmp_path / "huge.png" + # Write slightly over the cap + with open(img, "wb") as f: + f.write(b"\x89PNG\r\n\x1a\n" + b"\x00" * _IMAGE_SIZE_CAP) + + session = _make_session() + mock_caps = MagicMock() + mock_caps.supports_vision = True + session._provider.get_capabilities = MagicMock(return_value=mock_caps) + + item = {"call_id": "c3", "path": str(img), "offset": None, "limit": None} + call_id, output = session._exec_read_file(item) + + assert call_id == "c3" + assert isinstance(output, str) + assert "exceeds" in output + + def test_missing_image_returns_error(self, tmp_db, tmp_path): + """read_file on non-existent image returns error.""" + session = _make_session() + mock_caps = MagicMock() + mock_caps.supports_vision = True + session._provider.get_capabilities = MagicMock(return_value=mock_caps) + + item = {"call_id": "c4", "path": str(tmp_path / "nope.png"), "offset": None, "limit": None} + call_id, output = session._exec_read_file(item) + assert isinstance(output, str) + assert "not found" in output + + def test_svg_read_as_text(self, tmp_db, tmp_path): + """SVG files are read as text, not as images.""" + svg = tmp_path / "icon.svg" + svg.write_text('') + + session = _make_session() + item = {"call_id": "c5", "path": str(svg), "offset": None, "limit": None} + call_id, output = session._exec_read_file(item) + assert isinstance(output, str) + assert " list[dict[str, Any]]: + """Convert OpenAI-format content parts to Anthropic format. + + Transforms ``image_url`` parts (with ``data:`` URIs) to Anthropic's + ``image`` source blocks. Text parts pass through unchanged. + """ + converted: list[dict[str, Any]] = [] + for part in parts: + if part.get("type") == "image_url": + url = part.get("image_url", {}).get("url", "") + if url.startswith("data:") and "," in url: + # Parse "data:image/png;base64," + header, _, b64data = url.partition(",") + media_type = header.split(":", 1)[1].split(";", 1)[0] + converted.append( + { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": b64data, + }, + } + ) + else: + # URL-based image — pass as Anthropic URL source + converted.append( + { + "type": "image", + "source": {"type": "url", "url": url}, + } + ) + else: + converted.append(part) + return converted + # -- tool conversion ----------------------------------------------------- def convert_tools( diff --git a/turnstone/core/providers/_openai.py b/turnstone/core/providers/_openai.py index b1fdde10..608f839c 100644 --- a/turnstone/core/providers/_openai.py +++ b/turnstone/core/providers/_openai.py @@ -30,6 +30,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_temperature=False, reasoning_effort_values=("minimal", "low", "medium", "high"), default_reasoning_effort="medium", + supports_vision=True, ), "gpt-5-mini": ModelCapabilities( context_window=400000, @@ -37,6 +38,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_temperature=False, reasoning_effort_values=("minimal", "low", "medium", "high"), default_reasoning_effort="medium", + supports_vision=True, ), "gpt-5-nano": ModelCapabilities( context_window=400000, @@ -44,6 +46,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_temperature=False, reasoning_effort_values=("minimal", "low", "medium", "high"), default_reasoning_effort="medium", + supports_vision=True, ), # GPT-5 pro — high reasoning only, extended output "gpt-5-pro": ModelCapabilities( @@ -52,6 +55,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_temperature=False, reasoning_effort_values=("high",), default_reasoning_effort="high", + supports_vision=True, ), # GPT-5.1 — temperature OK when reasoning_effort=none (default) "gpt-5.1": ModelCapabilities( @@ -59,6 +63,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=128000, reasoning_effort_values=("none", "low", "medium", "high"), default_reasoning_effort="none", + supports_vision=True, ), # GPT-5.2 — adds xhigh "gpt-5.2": ModelCapabilities( @@ -66,6 +71,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=128000, reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), default_reasoning_effort="none", + supports_vision=True, ), # GPT-5.2 pro — always-reasoning variant "gpt-5.2-pro": ModelCapabilities( @@ -74,6 +80,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_temperature=False, reasoning_effort_values=("medium", "high", "xhigh"), default_reasoning_effort="medium", + supports_vision=True, ), # GPT-5.3 — same capabilities as 5.2 (matches gpt-5.3-chat-latest, codex) "gpt-5.3": ModelCapabilities( @@ -81,6 +88,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=128000, reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), default_reasoning_effort="none", + supports_vision=True, ), # GPT-5.4 — 1M context window, native tool search "gpt-5.4": ModelCapabilities( @@ -89,6 +97,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), default_reasoning_effort="none", supports_tool_search=True, + supports_vision=True, ), # GPT-5.4 pro — always-reasoning, 1M context, native tool search "gpt-5.4-pro": ModelCapabilities( @@ -98,6 +107,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { reasoning_effort_values=("medium", "high", "xhigh"), default_reasoning_effort="medium", supports_tool_search=True, + supports_vision=True, ), # O-series reasoning models "o1": ModelCapabilities( @@ -105,33 +115,39 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=100000, supports_temperature=False, supports_streaming=False, + supports_vision=True, ), "o1-mini": ModelCapabilities( context_window=128000, max_output_tokens=65536, supports_temperature=False, supports_streaming=False, + supports_vision=True, ), "o3": ModelCapabilities( context_window=200000, max_output_tokens=100000, supports_temperature=False, + supports_vision=True, ), "o3-mini": ModelCapabilities( context_window=200000, max_output_tokens=100000, supports_temperature=False, + supports_vision=True, ), "o3-pro": ModelCapabilities( context_window=200000, max_output_tokens=100000, supports_temperature=False, supports_streaming=False, + supports_vision=True, ), "o4-mini": ModelCapabilities( context_window=200000, max_output_tokens=100000, supports_temperature=False, + supports_vision=True, ), # Search models — always search on every request, no reasoning_effort "gpt-5-search-api": ModelCapabilities( @@ -140,6 +156,7 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_temperature=False, supports_web_search=True, reasoning_effort_values=(), + supports_vision=True, ), } diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index 72ad13f3..537e8451 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -77,6 +77,7 @@ class ModelCapabilities: default_reasoning_effort: str = "medium" supports_web_search: bool = False supports_tool_search: bool = False + supports_vision: bool = False def _lookup_capabilities( diff --git a/turnstone/core/session.py b/turnstone/core/session.py index f4490a4c..2d4d045b 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -8,9 +8,12 @@ to receive events and handle approval prompts. from __future__ import annotations +import base64 import concurrent.futures import contextlib +import dataclasses import json +import mimetypes import os import re import signal @@ -71,8 +74,21 @@ if TYPE_CHECKING: from turnstone.core.healthcheck import BackendHealthMonitor from turnstone.core.mcp_client import MCPClientManager - from turnstone.core.model_registry import ModelRegistry - from turnstone.core.providers import CompletionResult, LLMProvider, StreamChunk + from turnstone.core.model_registry import ModelConfig, ModelRegistry + from turnstone.core.providers import ( + CompletionResult, + LLMProvider, + ModelCapabilities, + StreamChunk, + ) + +# Image extensions handled as vision content (SVG excluded — it's XML text) +_IMAGE_EXTENSIONS: frozenset[str] = frozenset( + {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".ico"} +) + +# 4 MB raw → ~5.3 MB base64, safely under Anthropic's per-block limit +_IMAGE_SIZE_CAP: int = 4 * 1024 * 1024 # --------------------------------------------------------------------------- # SessionUI protocol — the contract every frontend must implement @@ -246,6 +262,18 @@ class ChatSession: def model_alias(self) -> str | None: return self._model_alias + def _get_capabilities(self) -> ModelCapabilities: + """Get model capabilities, applying config.toml overrides if present.""" + caps = self._provider.get_capabilities(self.model) + if self._registry and self._model_alias: + cfg: ModelConfig = self._registry.get_config(self._model_alias) + if cfg.capabilities: + fields = {f.name for f in dataclasses.fields(type(caps))} + overrides = {k: v for k, v in cfg.capabilities.items() if k in fields} + if overrides: + caps = dataclasses.replace(caps, **overrides) + return caps + def _save_config(self) -> None: """Persist LLM-affecting config so resumed workstreams behave identically.""" save_workstream_config( @@ -505,7 +533,7 @@ class ChatSession: ] # Tool search hint (client-side mode only — native mode needs no hint) if self._tool_search: - caps = self._provider.get_capabilities(self.model) + caps = self._get_capabilities() if not caps.supports_tool_search: dev_parts.append( "\n\nAdditional tools are available via tool_search. " @@ -564,7 +592,7 @@ class ChatSession: if not self._tool_search: return self._tools # Check if provider supports native tool search - caps = self._provider.get_capabilities(self.model) + caps = self._get_capabilities() if caps.supports_tool_search: # Provider handles defer_loading — send all tools return self._tools @@ -576,7 +604,7 @@ class ChatSession: """Return names of deferred tools for native provider search, or None.""" if not self._tool_search: return None - caps = self._provider.get_capabilities(self.model) + caps = self._get_capabilities() if not caps.supports_tool_search: return None # Client-side mode — no deferred names for provider deferred = self._tool_search.get_deferred_tools() @@ -746,13 +774,27 @@ class ChatSession: # Map tool_call_id → tool name for logging _tc_names = {c["id"]: c.get("function", {}).get("name", "") for c in tool_calls} for tc_id, output in results: - tool_msg = { + tool_msg: dict[str, Any] = { "role": "tool", "tool_call_id": tc_id, "content": output, } self.messages.append(tool_msg) - self._msg_tokens.append(max(1, int(len(output) / self._chars_per_token))) + + # Token estimation — image content uses a fixed heuristic + if isinstance(output, list): + text_chars = sum( + len(p.get("text", "")) for p in output if p.get("type") == "text" + ) + image_count = sum(1 for p in output if p.get("type") == "image_url") + tok_est = max( + 1, + int(text_chars / self._chars_per_token) + image_count * 1000, + ) + else: + tok_est = max(1, int(len(output) / self._chars_per_token)) + self._msg_tokens.append(tok_est) + # Log tool result (skip memory tools to avoid noise) _tname = _tc_names.get(tc_id, "") if _tname not in ( @@ -760,10 +802,17 @@ class ChatSession: "forget", "recall", ): + # For image content, store text description only + if isinstance(output, list): + store_text = " ".join( + p.get("text", "") for p in output if p.get("type") == "text" + )[:2000] + else: + store_text = output[:2000] save_message( self._ws_id, "tool_result", - output[:2000], + store_text, _tname, tool_call_id=tc_id, ) @@ -1047,6 +1096,16 @@ class ChatSession: tool_calls = m.get("tool_calls") tc_id = m.get("tool_call_id") + # Flatten list content (image tool results) for display + if isinstance(content, list): + parts = [] + for p in content: + if p.get("type") == "text": + parts.append(p.get("text", "")) + elif p.get("type") == "image_url": + parts.append("[image]") + content = " ".join(parts) + # Truncate long content for readability if len(content) > 300: display = content[:200] + f"...({len(content)} chars)..." + content[-50:] @@ -1076,7 +1135,11 @@ class ChatSession: def _msg_char_count(self, msg: dict[str, Any]) -> int: """Count characters in a message, including tool call arguments.""" - n = len(msg.get("content") or "") + content = msg.get("content") + if isinstance(content, list): + n = sum(len(p.get("text", "")) for p in content if p.get("type") == "text") + else: + n = len(content or "") for tc in msg.get("tool_calls", []): n += len(tc.get("function", {}).get("name", "")) n += len(tc.get("function", {}).get("arguments", "")) @@ -1134,6 +1197,16 @@ class ChatSession: role = m["role"].upper() content = m.get("content") or "" + # Flatten list content (image tool results) to text for summary + if isinstance(content, list): + text_parts = [] + for p in content: + if p.get("type") == "text": + text_parts.append(p["text"]) + elif p.get("type") == "image_url": + text_parts.append("[image]") + content = " ".join(text_parts) + if m.get("tool_calls"): calls = [] for tc in m["tool_calls"]: @@ -1321,7 +1394,7 @@ class ChatSession: def _execute_tools( self, tool_calls: list[dict[str, Any]] - ) -> tuple[list[tuple[str, str]], str | None]: + ) -> tuple[list[tuple[str, str | list[dict[str, Any]]]], str | None]: """Execute tool calls with batch preview and approval. Returns (results, user_feedback) where user_feedback is an optional @@ -1343,12 +1416,14 @@ class ChatSession: user_feedback = None # feedback is in the denial_msg # Phase 3: execute - def run_one(item: dict[str, Any]) -> tuple[str, str]: + def run_one( + item: dict[str, Any], + ) -> tuple[str, str | list[dict[str, Any]]]: if item.get("error"): return item["call_id"], item["error"] if item.get("denied"): return item["call_id"], item.get("denial_msg", "Denied by user") - result: tuple[str, str] = item["execute"](item) + result: tuple[str, str | list[dict[str, Any]]] = item["execute"](item) return result if len(items) == 1: @@ -1366,6 +1441,7 @@ class ChatSession: and not self.auto_approve ): cid, output = results[i] + assert isinstance(output, str) # plan always returns text # Let the UI present the plan for review self._emit_state("attention") resp = self.ui.on_plan_review(output) @@ -2210,13 +2286,18 @@ class ChatSession: self.ui.on_error(msg) return call_id, msg - def _exec_read_file(self, item: dict[str, Any]) -> tuple[str, str]: - """Read a file and return numbered lines, optionally sliced.""" + def _exec_read_file(self, item: dict[str, Any]) -> tuple[str, str | list[dict[str, Any]]]: + """Read a file and return numbered lines, or image content parts.""" call_id, path = item["call_id"], item["path"] offset = item.get("offset") # 1-based, or None limit = item.get("limit") # max lines, or None resolved = os.path.realpath(path) + # Image file detection (branch before text open) + ext = os.path.splitext(path)[1].lower() + if ext in _IMAGE_EXTENSIONS: + return self._exec_read_image(call_id, path, resolved) + try: with open(path) as f: all_lines = f.readlines() @@ -2251,6 +2332,61 @@ class ChatSession: return call_id, output if output else "(empty file)" + def _exec_read_image( + self, call_id: str, path: str, resolved: str + ) -> tuple[str, str | list[dict[str, Any]]]: + """Read an image file and return as base64 content parts for vision.""" + caps = self._get_capabilities() + if not caps.supports_vision: + try: + size = os.path.getsize(path) + except OSError as e: + self._read_files.discard(resolved) + return call_id, f"Error: {path}: {e}" + self._read_files.add(resolved) + desc = f"image (no vision, {size:,} bytes)" + self.ui.on_tool_result(call_id, "read_file", desc) + return call_id, ( + f"Binary image file: {path} ({size:,} bytes). " + "Current model does not support vision." + ) + + try: + with open(path, "rb") as f: + raw = f.read() + except FileNotFoundError: + self._read_files.discard(resolved) + return call_id, f"Error: {path} not found" + except Exception as e: + self._read_files.discard(resolved) + return call_id, f"Error reading {path}: {e}" + + if len(raw) > _IMAGE_SIZE_CAP: + self._read_files.discard(resolved) + size_mb = len(raw) / (1024 * 1024) + cap_mb = _IMAGE_SIZE_CAP / (1024 * 1024) + return call_id, ( + f"Error: image {path} is {size_mb:.1f} MB, " + f"exceeds {cap_mb:.0f} MB limit for vision." + ) + + self._read_files.add(resolved) + b64data = base64.b64encode(raw).decode("ascii") + mime, _ = mimetypes.guess_type(path) + if not mime: + mime = "image/png" + + content_parts: list[dict[str, Any]] = [ + {"type": "text", "text": f"Image file: {path} ({len(raw):,} bytes)"}, + { + "type": "image_url", + "image_url": {"url": f"data:{mime};base64,{b64data}"}, + }, + ] + + self.ui.on_tool_result(call_id, "read_file", f"image ({len(raw):,} bytes)") + return call_id, content_parts + def _exec_search(self, item: dict[str, Any]) -> tuple[str, str]: """Search file contents for a regex pattern using grep.""" call_id = item["call_id"] diff --git a/turnstone/eval.py b/turnstone/eval.py index b59cbd7f..9a651082 100644 --- a/turnstone/eval.py +++ b/turnstone/eval.py @@ -267,7 +267,15 @@ class HeadlessSession(ChatSession): with _suppress_stdout(): results, _ = self._execute_tools(assistant_msg["tool_calls"]) - for tc, (tc_id, output) in zip(assistant_msg["tool_calls"], results, strict=False): + for tc, (tc_id, raw_output) in zip(assistant_msg["tool_calls"], results, strict=False): + # Flatten list content (image tool results) to text for logging + if isinstance(raw_output, list): + output = " ".join( + p.get("text", "[image]") if p.get("type") == "text" else "[image]" + for p in raw_output + ) + else: + output = raw_output func_name = tc["function"]["name"] args: dict[str, Any] try: @@ -302,7 +310,7 @@ class HeadlessSession(ChatSession): tool_msg = { "role": "tool", "tool_call_id": tc_id, - "content": output, + "content": raw_output, } self.messages.append(tool_msg) self._msg_tokens.append(max(1, int(len(output) / self._chars_per_token))) diff --git a/turnstone/tools/read_file.json b/turnstone/tools/read_file.json index 4c51f125..9f36f535 100644 --- a/turnstone/tools/read_file.json +++ b/turnstone/tools/read_file.json @@ -1,6 +1,6 @@ { "name": "read_file", - "description": "Read the contents of a file. Returns numbered lines. Must be called before edit_file on the same path.", + "description": "Read the contents of a file. Returns numbered lines for text files. For image files (PNG, JPEG, GIF, WebP, BMP, TIFF, ICO), returns the image content if the model supports vision, or a text description otherwise. The offset and limit parameters apply to text files only.", "parameters": { "type": "object", "properties": {