From f02972c11dfb503eee3d8b936d64b7a822d8a7b4 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Tue, 3 Mar 2026 17:12:49 -0800 Subject: [PATCH] Add provider-native web search with Tavily fallback (#13) * Add provider-native web search with Tavily fallback Replace client-side Tavily web search with provider-native implementations: - Anthropic: inject web_search_20250305 server-side tool, handle server_tool_use / web_search_tool_result streaming blocks, emit info_delta for search status display - OpenAI: inject web_search_options for gpt-5-search-api, format url_citation annotations as footnote sources - Local/vLLM: preserve existing Tavily-based web_search tool as fallback Add supports_web_search to ModelCapabilities and info_delta to StreamChunk. Remove end-of-life GPT-4o model entries from capability tables. Update docs, diagrams, and README. 88 provider tests (32 new). * Fix Copilot PR #13 review: capture streaming url_citation annotations Accumulate url_citation annotations during OpenAI streaming and emit formatted citations as a final info_delta chunk after the stream ends. Previously annotations were only captured in non-streaming mode, so search model users in the interactive path never saw citation sources. --- README.md | 8 +- docs/architecture.md | 25 +- docs/diagrams/03-core-engine-classes.puml | 9 +- docs/diagrams/04-conversation-turn.puml | 4 +- docs/diagrams/05-tool-pipeline.puml | 2 +- docs/diagrams/png/03-core-engine-classes.png | 4 +- docs/diagrams/png/04-conversation-turn.png | 4 +- docs/diagrams/png/05-tool-pipeline.png | 4 +- docs/docker.md | 2 +- docs/tools.md | 7 +- tests/test_providers.py | 615 +++++++++++++++++++ turnstone/core/providers/_anthropic.py | 99 ++- turnstone/core/providers/_openai.py | 84 ++- turnstone/core/providers/_protocol.py | 2 + turnstone/core/session.py | 5 + 15 files changed, 829 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 51e3a535..33c4d9f9 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct | `math` | Sandboxed Python evaluation | | | `man` | Read man pages | yes | | `web_fetch` | Fetch URL content | | -| `web_search` | Search via Tavily API | | +| `web_search` | Web search (provider-native or Tavily) | | | `remember` | Save persistent facts | yes | | `recall` | Search memories and history | yes | | `forget` | Remove a memory | yes | @@ -264,8 +264,8 @@ context_window = 200000 [models.openai] base_url = "https://api.openai.com/v1" api_key = "sk-..." -model = "gpt-4o" -context_window = 128000 +model = "gpt-5" +context_window = 400000 [model] default = "local" # which model to use by default @@ -285,7 +285,7 @@ All entry points read `~/.config/turnstone/config.toml`. CLI flags override conf [api] base_url = "http://localhost:8000/v1" api_key = "" -tavily_key = "" +tavily_key = "" # only needed for local/vLLM models without native search [model] name = "" # empty = auto-detect diff --git a/docs/architecture.md b/docs/architecture.md index 17b42127..b3bf99d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -411,7 +411,7 @@ from each schema and builds: - `edit_file` -- string replacement in an existing file (requires prior `read_file`) - `math` -- execute Python in sandboxed subprocess (via `turnstone.core.sandbox`) - `web_fetch` -- fetch a URL (with SSRF protection via `turnstone.core.web`) -- `web_search` -- search the web via Tavily API +- `web_search` -- search the web (provider-native for Anthropic/OpenAI, Tavily fallback for local models) **Agent (delegated sub-sessions)**: - `task` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`) @@ -514,15 +514,18 @@ LLMProvider (protocol) | Type | Fields | |------|--------| -| `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `usage`, `finish_reason` | +| `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason` | | `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage` | -| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort` | +| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search` | | `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens` | **OpenAIProvider** (`_openai.py`): passes messages through unchanged (they are -already in OpenAI format). Model capability lookup table covers GPT-4o, -GPT-5/5.1/5.2, and O-series models. Unknown models (local servers) get -permissive defaults. +already in OpenAI format). Model capability lookup table covers +GPT-5/5.1/5.2, O-series, and search models (`gpt-5-search-api`). +For search models, injects `web_search_options` and removes the `web_search` +function tool (the model always searches). Citations from `url_citation` +annotations are formatted as footnotes. Unknown models (local servers) get +permissive defaults and use Tavily for web search. **AnthropicProvider** (`_anthropic.py`): converts OpenAI-format messages to Anthropic content blocks, maps `system`/`developer` roles to the `system` @@ -530,7 +533,11 @@ parameter, groups consecutive `tool` result messages into user-role content blocks, and translates tool schemas from OpenAI function-calling format to Anthropic's `input_schema` format. Supports both manual and adaptive thinking modes, with effort parameter support for models like Claude Opus 4.6 and -Sonnet 4.6. The `anthropic` SDK is imported lazily so it remains an optional +Sonnet 4.6. Replaces the `web_search` function tool with Anthropic's native +`web_search_20250305` server-side tool — Claude decides when to search, the +API executes it, and results stream back as `server_tool_use` / +`web_search_tool_result` content blocks (emitted as `info_delta` for UI +display). The `anthropic` SDK is imported lazily so it remains an optional dependency (`pip install turnstone[anthropic]`). **Factory functions** (`__init__.py`): `create_provider(name)` returns a @@ -558,8 +565,8 @@ context_window = 200000 [models.openai] base_url = "https://api.openai.com/v1" api_key = "sk-..." -model = "gpt-4o" -context_window = 128000 +model = "gpt-5" +context_window = 400000 [model] default = "local" diff --git a/docs/diagrams/03-core-engine-classes.puml b/docs/diagrams/03-core-engine-classes.puml index f292a3de..02ccc804 100644 --- a/docs/diagrams/03-core-engine-classes.puml +++ b/docs/diagrams/03-core-engine-classes.puml @@ -79,9 +79,11 @@ interface "LLMProvider" as LLMProvider <> { class "OpenAIProvider" as OpenAIProv { Model capability lookup table - (GPT-4o, GPT-5.x, O-series) + (GPT-5.x, O-series, search) Passthrough: messages already - in OpenAI format + in OpenAI format. + Search models: web_search_options + + url_citation annotations. -- core/providers/_openai.py } @@ -90,6 +92,8 @@ class "AnthropicProvider" as AnthropicProv { Converts OpenAI messages to Anthropic content blocks. Adaptive + manual thinking. + Native web search via + web_search_20250305 server tool. Lazy anthropic SDK import. -- core/providers/_anthropic.py @@ -103,6 +107,7 @@ class "ModelCapabilities" as ModelCaps <> { + token_param: str + thinking_mode: str + supports_effort: bool + + supports_web_search: bool } ' ChatSession diff --git a/docs/diagrams/04-conversation-turn.puml b/docs/diagrams/04-conversation-turn.puml index 13be7f28..0aef6819 100644 --- a/docs/diagrams/04-conversation-turn.puml +++ b/docs/diagrams/04-conversation-turn.puml @@ -52,6 +52,8 @@ group loop [while tool_calls present] CS -> UI : on_content_token(text) else tool_call delta CS -> CS : accumulate in tool_calls_acc + else info_delta present + CS -> UI : on_info(text)\n(e.g. server-side web search status) end end @@ -116,7 +118,7 @@ group loop [while tool_calls present] task/plan → _run_agent() sub-loop math → sandboxed subprocess web_fetch → httpx + LLM summarize - web_search → Tavily API + web_search → provider-native or Tavily fallback remember/recall/forget → SQLite end note diff --git a/docs/diagrams/05-tool-pipeline.puml b/docs/diagrams/05-tool-pipeline.puml index 7c1e175e..32efaac5 100644 --- a/docs/diagrams/05-tool-pipeline.puml +++ b/docs/diagrams/05-tool-pipeline.puml @@ -105,7 +105,7 @@ partition "Phase 3: Execute" #E3F2FD { ├─ _exec_math: sandboxed subprocess ├─ _exec_man: man/info subprocess ├─ _exec_web_fetch: httpx.get + LLM summary - ├─ _exec_web_search: Tavily API POST + ├─ _exec_web_search: Tavily API POST (fallback for local models) ├─ _exec_task: _run_agent(TASK_AGENT_TOOLS) ├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only) ├─ _exec_remember: SQLite INSERT OR REPLACE diff --git a/docs/diagrams/png/03-core-engine-classes.png b/docs/diagrams/png/03-core-engine-classes.png index b11acd2e..152f2f95 100644 --- a/docs/diagrams/png/03-core-engine-classes.png +++ b/docs/diagrams/png/03-core-engine-classes.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3266b92c7f2313aa5aa91dd043f24c7dc236a028be91ec05aa243e2b222f62e -size 378406 +oid sha256:637458e0d78df82752746e519cd7300a830c8ce211f21625694ad0c162ca316d +size 481637 diff --git a/docs/diagrams/png/04-conversation-turn.png b/docs/diagrams/png/04-conversation-turn.png index caf08cc0..e40ec574 100644 --- a/docs/diagrams/png/04-conversation-turn.png +++ b/docs/diagrams/png/04-conversation-turn.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a90dec1546dd0f8343e3e27cf6f235d95f3ffc375928d34d0df5d8f25d63e5ad -size 269901 +oid sha256:dc3b64c9e48153641af62ed43fbc1d89a31d1a8a61e7e71cfc550c805000310d +size 288290 diff --git a/docs/diagrams/png/05-tool-pipeline.png b/docs/diagrams/png/05-tool-pipeline.png index bb89cf25..7b400901 100644 --- a/docs/diagrams/png/05-tool-pipeline.png +++ b/docs/diagrams/png/05-tool-pipeline.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb36b4924394cf54d6aaef454cced317b72e336e580cc6ac25cd4b9d0917bec5 -size 243422 +oid sha256:b842683d238664a3e35d04358fecfc56cefd013f7dca5f13357b0376f881e1b3 +size 245043 diff --git a/docs/docker.md b/docs/docker.md index 25c48092..c96cc5da 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -57,7 +57,7 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl |----------|---------|-------------| | `LLM_BASE_URL` | `http://host.docker.internal:8000/v1` | OpenAI-compatible API URL | | `OPENAI_API_KEY` | `dummy` | API key (`dummy` for local servers) | -| `TAVILY_API_KEY` | — | Web search API key (optional) | +| `TAVILY_API_KEY` | — | Web search API key (only needed for local/vLLM models; Anthropic and OpenAI search models use native search) | ### Redis diff --git a/docs/tools.md b/docs/tools.md index f88e251f..9f86e649 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -302,8 +302,11 @@ Search the web using a text query. | `max_results` | integer | no | Max results to return (default 5, max 20). | | `topic` | string | no | Search topic: `general`, `news`, or `finance` (default `general`). | -- **What it does**: Searches the web via the Tavily API and returns ranked results with titles, URLs, and content snippets. -- **Auto-approve**: No -- requires user confirmation (makes network requests). +- **What it does**: Searches the web and returns ranked results with titles, URLs, and content snippets. Uses provider-native search when available: + - **Anthropic**: Replaced at the API boundary with Anthropic's `web_search_20250305` server-side tool. Claude decides when to search; the API executes it and returns results with citations inline. No Tavily key needed. + - **OpenAI search models** (`gpt-5-search-api`): Replaced with `web_search_options` parameter. The model always searches and returns `url_citation` annotations. + - **Local/vLLM models**: Falls back to the Tavily API. Requires `tavily_key` in `config.toml` or `$TAVILY_API_KEY`. +- **Auto-approve**: Yes (auto-approved for all tool dispatch paths). - **Agent availability**: `agent` and `task_agent`. --- diff --git a/tests/test_providers.py b/tests/test_providers.py index dc44bb07..7debe395 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -110,6 +110,9 @@ def _anthropic_event( stop_delta.stop_reason = kwargs.get("stop_reason") event.delta = stop_delta + elif event_type == "content_block_stop": + event.index = kwargs.get("index", 0) + elif event_type == "message_start": msg = MagicMock() if "usage_input_tokens" in kwargs: @@ -1038,6 +1041,16 @@ class TestDataclasses: assert cr.finish_reason == "stop" assert cr.usage is None + def test_stream_chunk_info_delta_default(self) -> None: + sc = StreamChunk() + assert sc.info_delta == "" + + def test_model_capabilities_web_search_default(self) -> None: + from turnstone.core.providers._protocol import ModelCapabilities + + caps = ModelCapabilities() + assert caps.supports_web_search is False + # =========================================================================== # TestParameterGating — model capability parameter gating @@ -1111,3 +1124,605 @@ class TestAnthropicReasoningNone: result = self.provider._reasoning_params("low", None, max_tokens=4096) assert "thinking" in result assert result["thinking"]["budget_tokens"] == 1024 + + +# =========================================================================== +# TestWebSearch — provider-native web search +# =========================================================================== + + +class TestAnthropicWebSearch: + """Tests for Anthropic native web search tool injection and streaming.""" + + def setup_method(self) -> None: + from turnstone.core.providers._anthropic import AnthropicProvider + + self.provider = AnthropicProvider() + + def test_web_search_capability_flag(self) -> None: + """All Anthropic models should support native web search.""" + caps = self.provider.get_capabilities("claude-opus-4-6") + assert caps.supports_web_search is True + caps = self.provider.get_capabilities("claude-sonnet-4") + assert caps.supports_web_search is True + # Unknown models use default which also has web search + caps = self.provider.get_capabilities("claude-unknown-99") + assert caps.supports_web_search is True + + def test_inject_web_search_replaces_function_tool(self) -> None: + """web_search function tool should be replaced with native server-side tool.""" + caps = self.provider.get_capabilities("claude-opus-4-6") + tools = [ + {"name": "bash", "description": "Run bash", "input_schema": {"type": "object"}}, + {"name": "web_search", "description": "Search web", "input_schema": {"type": "object"}}, + ] + result = self.provider._inject_web_search(tools, caps) + names = [t.get("name") for t in result] + assert "bash" in names + assert "web_search" in names + # The web_search entry should be the native tool, not the function tool + ws_tool = next(t for t in result if t.get("name") == "web_search") + from turnstone.core.providers._anthropic import _WEB_SEARCH_TOOL_TYPE + + assert ws_tool["type"] == _WEB_SEARCH_TOOL_TYPE + assert "input_schema" not in ws_tool + + def test_inject_web_search_no_op_without_tool(self) -> None: + """If no web_search tool in list, no injection happens.""" + caps = self.provider.get_capabilities("claude-opus-4-6") + tools = [ + {"name": "bash", "description": "Run bash", "input_schema": {"type": "object"}}, + ] + result = self.provider._inject_web_search(tools, caps) + assert result is tools # Unchanged + + def test_streaming_server_tool_use_emits_search_info(self) -> None: + """server_tool_use block should emit info_delta with search query.""" + events = [ + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_123", + block_name="web_search", + index=0, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"query": "python web frameworks"}', + index=0, + ), + _anthropic_event("content_block_stop", index=0), + ] + + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 1 + assert "python web frameworks" in info_chunks[0].info_delta + assert "Searching" in info_chunks[0].info_delta + + def test_streaming_web_search_result_emits_count(self) -> None: + """web_search_tool_result block should emit result count info.""" + # Build mock search results + result1 = MagicMock() + result1.type = "web_search_result" + result2 = MagicMock() + result2.type = "web_search_result" + + events = [ + _anthropic_event( + "content_block_start", + block_type="web_search_tool_result", + index=1, + ), + ] + # Set up the content attribute with search results + events[0].content_block.content = [result1, result2] + + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 1 + assert "Found 2 results" in info_chunks[0].info_delta + + def test_streaming_web_search_error_emits_info(self) -> None: + """web_search_tool_result with error should emit error info.""" + error_content = MagicMock() + error_content.type = "web_search_tool_result_error" + error_content.error_code = "too_many_requests" + + events = [ + _anthropic_event( + "content_block_start", + block_type="web_search_tool_result", + index=1, + ), + ] + events[0].content_block.content = error_content + + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 1 + assert "too_many_requests" in info_chunks[0].info_delta + + def test_streaming_server_tool_use_not_emitted_as_tool_call(self) -> None: + """server_tool_use should NOT produce tool_call_deltas (it's server-side).""" + events = [ + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_123", + block_name="web_search", + index=0, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"query": "test"}', + index=0, + ), + ] + chunks = list(self.provider._iter_anthropic_stream(events)) + tool_chunks = [c for c in chunks if c.tool_call_deltas] + assert len(tool_chunks) == 0 + + def test_streaming_mixed_text_and_search(self) -> None: + """Full sequence: text + server search + results + more text.""" + events = [ + # Initial text + _anthropic_event( + "content_block_start", + block_type="text", + index=0, + ), + _anthropic_event( + "content_block_delta", + delta_type="text_delta", + text="Let me search.", + index=0, + ), + # Server tool use + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_1", + block_name="web_search", + index=1, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"query": "test query"}', + index=1, + ), + _anthropic_event("content_block_stop", index=1), + # Response text + _anthropic_event( + "content_block_start", + block_type="text", + index=3, + ), + _anthropic_event( + "content_block_delta", + delta_type="text_delta", + text="Based on the results...", + index=3, + ), + # Finish + _anthropic_event("message_delta", stop_reason="end_turn"), + ] + + chunks = list(self.provider._iter_anthropic_stream(events)) + text_chunks = [c for c in chunks if c.content_delta] + info_chunks = [c for c in chunks if c.info_delta] + assert len(text_chunks) == 2 + assert text_chunks[0].content_delta == "Let me search." + assert text_chunks[1].content_delta == "Based on the results..." + assert len(info_chunks) == 1 + assert "test query" in info_chunks[0].info_delta + + def test_pause_turn_normalized_to_stop(self) -> None: + """pause_turn stop reason should normalize to 'stop'.""" + from turnstone.core.providers._anthropic import _normalize_finish_reason + + assert _normalize_finish_reason("pause_turn") == "stop" + + def test_completion_skips_server_blocks(self) -> None: + """create_completion should skip server_tool_use and web_search_tool_result.""" + # Build mock response with mixed block types + text_block = MagicMock() + text_block.type = "text" + text_block.text = "Here are the results." + + server_tu_block = MagicMock() + server_tu_block.type = "server_tool_use" + + search_result_block = MagicMock() + search_result_block.type = "web_search_tool_result" + + response = MagicMock() + response.content = [server_tu_block, search_result_block, text_block] + response.stop_reason = "end_turn" + response.usage.input_tokens = 100 + response.usage.output_tokens = 50 + + client = MagicMock() + client.messages.create.return_value = response + + with patch("turnstone.core.providers._anthropic._ensure_anthropic"): + result = self.provider.create_completion( + client=client, + model="claude-opus-4-6", + messages=[{"role": "user", "content": "search test"}], + ) + assert result.content == "Here are the results." + assert result.tool_calls is None + + def test_streaming_multiple_searches(self) -> None: + """Multiple server_tool_use blocks in one response should each emit info.""" + events = [ + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_1", + block_name="web_search", + index=0, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"query": "first search"}', + index=0, + ), + _anthropic_event("content_block_stop", index=0), + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_2", + block_name="web_search", + index=2, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"query": "second search"}', + index=2, + ), + _anthropic_event("content_block_stop", index=2), + ] + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 2 + assert "first search" in info_chunks[0].info_delta + assert "second search" in info_chunks[1].info_delta + + def test_streaming_interleaved_tool_use_and_server_tool_use(self) -> None: + """Regular tool_use and server_tool_use at different indices.""" + events = [ + # Regular tool call at index 0 + _anthropic_event( + "content_block_start", + block_type="tool_use", + block_id="toolu_1", + block_name="bash", + index=0, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"command": "ls"}', + index=0, + ), + # Server tool at index 1 + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_1", + block_name="web_search", + index=1, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json='{"query": "test"}', + index=1, + ), + _anthropic_event("content_block_stop", index=1), + ] + chunks = list(self.provider._iter_anthropic_stream(events)) + tool_chunks = [c for c in chunks if c.tool_call_deltas] + info_chunks = [c for c in chunks if c.info_delta] + # Regular tool_use should produce tool_call_deltas + assert len(tool_chunks) == 2 # start + delta + assert tool_chunks[0].tool_call_deltas[0].name == "bash" + # Server tool_use should produce info_delta only + assert len(info_chunks) == 1 + assert "test" in info_chunks[0].info_delta + + def test_streaming_malformed_server_tool_json(self) -> None: + """Malformed JSON in server tool input should emit fallback info.""" + events = [ + _anthropic_event( + "content_block_start", + block_type="server_tool_use", + block_id="srvtoolu_1", + block_name="web_search", + index=0, + ), + _anthropic_event( + "content_block_delta", + delta_type="input_json_delta", + partial_json="{bad json", + index=0, + ), + _anthropic_event("content_block_stop", index=0), + ] + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 1 + assert info_chunks[0].info_delta == "[Searching...]" + + def test_web_search_result_empty_list(self) -> None: + """Empty search results list should report 0 results.""" + events = [ + _anthropic_event( + "content_block_start", + block_type="web_search_tool_result", + index=0, + ), + ] + events[0].content_block.content = [] + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 1 + assert "Found 0 results" in info_chunks[0].info_delta + + def test_content_block_stop_for_text_block_no_spurious_info(self) -> None: + """content_block_stop for a text block should not emit info_delta.""" + events = [ + _anthropic_event("content_block_start", block_type="text", index=0), + _anthropic_event( + "content_block_delta", + delta_type="text_delta", + text="hello", + index=0, + ), + _anthropic_event("content_block_stop", index=0), + ] + chunks = list(self.provider._iter_anthropic_stream(events)) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 0 + + +class TestOpenAIWebSearch: + """Tests for OpenAI native web search with search models.""" + + def setup_method(self) -> None: + self.provider = OpenAIProvider() + + def test_search_model_capability(self) -> None: + """Search models should have supports_web_search=True.""" + caps = self.provider.get_capabilities("gpt-5-search-api") + assert caps.supports_web_search is True + + def test_non_search_model_no_web_search(self) -> None: + """Regular models should not have supports_web_search.""" + caps = self.provider.get_capabilities("gpt-5") + assert caps.supports_web_search is False + caps = self.provider.get_capabilities("gpt-5.2") + assert caps.supports_web_search is False + + def test_apply_web_search_injects_options(self) -> None: + """For search models, web_search_options should be added to kwargs.""" + caps = self.provider.get_capabilities("gpt-5-search-api") + kwargs: dict[str, Any] = {"model": "gpt-5-search-api"} + tools: list[dict[str, Any]] = [ + {"type": "function", "function": {"name": "bash", "description": "Run bash"}}, + {"type": "function", "function": {"name": "web_search", "description": "Search"}}, + ] + result = self.provider._apply_web_search(kwargs, caps, tools) + # web_search_options should be in kwargs + assert "web_search_options" in kwargs + # web_search tool should be removed + assert result is not None + names = [t["function"]["name"] for t in result] + assert "web_search" not in names + assert "bash" in names + + def test_apply_web_search_no_op_for_regular_models(self) -> None: + """For non-search models, no web_search_options, tools unchanged.""" + caps = self.provider.get_capabilities("gpt-5") + kwargs: dict[str, Any] = {"model": "gpt-5"} + tools: list[dict[str, Any]] = [ + {"type": "function", "function": {"name": "web_search", "description": "Search"}}, + ] + result = self.provider._apply_web_search(kwargs, caps, tools) + assert "web_search_options" not in kwargs + assert result is tools # Unchanged + + def test_apply_web_search_returns_none_when_only_web_search(self) -> None: + """If web_search was the only tool, return None after removing it.""" + caps = self.provider.get_capabilities("gpt-5-search-api") + kwargs: dict[str, Any] = {} + tools: list[dict[str, Any]] = [ + {"type": "function", "function": {"name": "web_search", "description": "Search"}}, + ] + result = self.provider._apply_web_search(kwargs, caps, tools) + assert result is None + + def test_format_citations_appends_sources(self) -> None: + """url_citation annotations should be formatted as footnote sources.""" + ann = MagicMock() + ann.type = "url_citation" + citation = MagicMock() + citation.title = "Example Page" + citation.url = "https://example.com" + ann.url_citation = citation + + content = "Some search result text." + result = OpenAIProvider._format_citations(content, [ann]) + assert "Sources:" in result + assert "[Example Page](https://example.com)" in result + + def test_format_citations_deduplicates(self) -> None: + """Duplicate URLs should not appear twice in sources.""" + ann1 = MagicMock() + ann1.type = "url_citation" + ann1.url_citation = MagicMock(title="Page", url="https://example.com") + + ann2 = MagicMock() + ann2.type = "url_citation" + ann2.url_citation = MagicMock(title="Page Again", url="https://example.com") + + content = "Text." + result = OpenAIProvider._format_citations(content, [ann1, ann2]) + assert result.count("example.com") == 1 + + def test_format_citations_skips_non_url_citation(self) -> None: + """Non-url_citation annotations should be ignored.""" + ann = MagicMock() + ann.type = "something_else" + + content = "Text." + result = OpenAIProvider._format_citations(content, [ann]) + assert "Sources:" not in result + + def test_format_citations_empty_title(self) -> None: + """Citation with empty title should show plain URL.""" + ann = MagicMock() + ann.type = "url_citation" + ann.url_citation = MagicMock(title="", url="https://example.com") + + result = OpenAIProvider._format_citations("Text.", [ann]) + assert "https://example.com" in result + # Should not have markdown link format when title is empty + assert "[](https://example.com)" not in result + + def test_format_citations_none_citation(self) -> None: + """Citation with None url_citation should be skipped.""" + ann = MagicMock() + ann.type = "url_citation" + ann.url_citation = None + + result = OpenAIProvider._format_citations("Text.", [ann]) + assert "Sources:" not in result + + def test_apply_web_search_with_no_tools(self) -> None: + """Search model with tools=None should still inject web_search_options.""" + caps = self.provider.get_capabilities("gpt-5-search-api") + kwargs: dict[str, Any] = {} + result = self.provider._apply_web_search(kwargs, caps, None) + assert "web_search_options" in kwargs + assert result is None + + def test_streaming_creates_with_web_search_options(self) -> None: + """Streaming with a search model should pass web_search_options.""" + client = MagicMock() + client.chat.completions.create.return_value = iter( + [ + _openai_stream_chunk(content="Result text"), + ] + ) + list( + self.provider.create_streaming( + client=client, + model="gpt-5-search-api", + messages=[{"role": "user", "content": "search something"}], + tools=[ + { + "type": "function", + "function": {"name": "web_search", "description": "Search"}, + }, + ], + ) + ) + call_kwargs = client.chat.completions.create.call_args[1] + assert "web_search_options" in call_kwargs + # web_search tool should not be in the tools + assert "tools" not in call_kwargs or not any( + t.get("function", {}).get("name") == "web_search" for t in call_kwargs.get("tools", []) + ) + + def test_completion_with_annotations(self) -> None: + """Non-streaming completion with search model should format citations.""" + ann = MagicMock() + ann.type = "url_citation" + ann.url_citation = MagicMock(title="Test", url="https://test.com") + + msg = MagicMock() + msg.content = "Found information." + msg.annotations = [ann] + msg.tool_calls = None + + choice = MagicMock() + choice.message = msg + choice.finish_reason = "stop" + + response = MagicMock() + response.choices = [choice] + response.usage.prompt_tokens = 50 + response.usage.completion_tokens = 20 + response.usage.total_tokens = 70 + + client = MagicMock() + client.chat.completions.create.return_value = response + + result = self.provider.create_completion( + client=client, + model="gpt-5-search-api", + messages=[{"role": "user", "content": "search test"}], + ) + assert "Found information." in result.content + assert "Sources:" in result.content + assert "[Test](https://test.com)" in result.content + + def test_streaming_emits_citations_as_info_delta(self) -> None: + """Streaming with search model should emit citations as final info_delta.""" + ann = MagicMock() + ann.type = "url_citation" + ann.url_citation = MagicMock(title="Result", url="https://example.com") + + # Content chunk, then a chunk with annotation, then finish + content_chunk = _openai_stream_chunk(content="Search result text.") + content_chunk.choices[0].delta.annotations = None + + ann_chunk = _openai_stream_chunk(content=None) + ann_chunk.choices[0].delta.annotations = [ann] + + finish_chunk = _openai_stream_chunk(finish_reason="stop") + finish_chunk.choices[0].delta.annotations = None + + client = MagicMock() + client.chat.completions.create.return_value = iter([content_chunk, ann_chunk, finish_chunk]) + chunks = list( + self.provider.create_streaming( + client=client, + model="gpt-5-search-api", + messages=[{"role": "user", "content": "search test"}], + ) + ) + info_chunks = [c for c in chunks if c.info_delta] + assert len(info_chunks) == 1 + assert "Sources:" in info_chunks[0].info_delta + assert "[Result](https://example.com)" in info_chunks[0].info_delta + + +class TestTavilyFallback: + """Tests for Tavily fallback when providers don't support native search.""" + + def test_local_model_no_web_search(self) -> None: + """Local/vLLM models should not have supports_web_search.""" + provider = OpenAIProvider() + caps = provider.get_capabilities("my-local-model") + assert caps.supports_web_search is False + + def test_web_search_tool_preserved_for_local_models(self) -> None: + """For local models, web_search function tool stays in the tools list.""" + provider = OpenAIProvider() + caps = provider.get_capabilities("llama-3-70b") + kwargs: dict[str, Any] = {} + tools = [ + {"type": "function", "function": {"name": "web_search", "description": "Search"}}, + ] + result = provider._apply_web_search(kwargs, caps, tools) + assert result is tools + assert "web_search_options" not in kwargs diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index 3e4f1bc5..270f2c01 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -61,6 +61,9 @@ def _merge_consecutive(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: return merged +# Tool version for Anthropic's server-side web search (update when new version ships) +_WEB_SEARCH_TOOL_TYPE = "web_search_20250305" + # -- model capabilities ------------------------------------------------------- _ANTHROPIC_DEFAULT = ModelCapabilities( @@ -68,6 +71,7 @@ _ANTHROPIC_DEFAULT = ModelCapabilities( max_output_tokens=64000, token_param="max_tokens", thinking_mode="manual", + supports_web_search=True, ) _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { @@ -78,6 +82,7 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { thinking_mode="adaptive", supports_effort=True, effort_levels=("low", "medium", "high", "max"), + supports_web_search=True, ), "claude-sonnet-4-6": ModelCapabilities( context_window=200000, @@ -86,18 +91,21 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { thinking_mode="adaptive", supports_effort=True, effort_levels=("low", "medium", "high"), + supports_web_search=True, ), "claude-haiku-4-5": ModelCapabilities( context_window=200000, max_output_tokens=64000, token_param="max_tokens", thinking_mode="manual", + supports_web_search=True, ), "claude-sonnet-4-5": ModelCapabilities( context_window=200000, max_output_tokens=64000, token_param="max_tokens", thinking_mode="manual", + supports_web_search=True, ), "claude-opus-4-5": ModelCapabilities( context_window=200000, @@ -106,18 +114,21 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { thinking_mode="manual", supports_effort=True, effort_levels=("low", "medium", "high"), + supports_web_search=True, ), "claude-opus-4": ModelCapabilities( context_window=200000, max_output_tokens=32000, token_param="max_tokens", thinking_mode="manual", + supports_web_search=True, ), "claude-sonnet-4": ModelCapabilities( context_window=200000, max_output_tokens=64000, token_param="max_tokens", thinking_mode="manual", + supports_web_search=True, ), } @@ -147,6 +158,30 @@ class AnthropicProvider: def get_capabilities(self, model: str) -> ModelCapabilities: return _lookup_capabilities(model, _ANTHROPIC_CAPABILITIES, _ANTHROPIC_DEFAULT) + # -- web search tool injection ------------------------------------------- + + def _inject_web_search( + self, + tools: list[dict[str, Any]], + caps: ModelCapabilities, + ) -> list[dict[str, Any]]: + """Replace ``web_search`` function tool with native server-side tool. + + If the tools list contains a ``web_search`` function tool and the model + supports native web search, replace it with Anthropic's + ``web_search_20250305`` server-side tool. The server executes the search + autonomously — no client-side tool execution loop needed. + """ + if not caps.supports_web_search: + return tools + has_web_search = any(t.get("name") == "web_search" for t in tools) + if not has_web_search: + return tools + # Remove the function-based web_search and add the native tool + filtered = [t for t in tools if t.get("name") != "web_search"] + filtered.append({"type": _WEB_SEARCH_TOOL_TYPE, "name": "web_search"}) + return filtered + # -- shared param logic -------------------------------------------------- def _build_thinking_and_kwargs( @@ -180,7 +215,9 @@ class AnthropicProvider: if system_prompt: kwargs["system"] = system_prompt if tools: - kwargs["tools"] = self.convert_tools(tools) + anthropic_tools = self.convert_tools(tools) + anthropic_tools = self._inject_web_search(anthropic_tools, caps) + kwargs["tools"] = anthropic_tools kwargs.update(thinking_params) # Effort param for models that support it (Opus 4.6, Sonnet 4.6, Opus 4.5) @@ -351,6 +388,8 @@ class AnthropicProvider: # Map content block index → tool call index for our accumulator tool_block_to_index: dict[int, int] = {} next_tool_index = 0 + # Track server-side tool blocks (web search) — accumulate query input + server_tool_blocks: dict[int, dict[str, str]] = {} for event in stream: sc = StreamChunk() @@ -365,6 +404,26 @@ class AnthropicProvider: sc.tool_call_deltas.append( ToolCallDelta(index=idx, id=block.id, name=block.name) ) + elif block.type == "server_tool_use": + # Server-side tool (web search) — track for query accumulation + server_tool_blocks[event.index] = { + "name": getattr(block, "name", ""), + "input_json": "", + } + elif block.type == "web_search_tool_result": + # Search results arrived — count results for info display + content = getattr(block, "content", None) + if isinstance(content, list): + n = sum( + 1 for r in content if getattr(r, "type", None) == "web_search_result" + ) + sc.info_delta = f"[Found {n} result{'s' if n != 1 else ''}]" + elif ( + content is not None + and getattr(content, "type", None) == "web_search_tool_result_error" + ): + code = getattr(content, "error_code", "unknown") + sc.info_delta = f"[Web search error: {code}]" elif event_type == "content_block_delta": delta = event.delta @@ -373,13 +432,29 @@ class AnthropicProvider: elif delta.type == "thinking_delta": sc.reasoning_delta = delta.thinking elif delta.type == "input_json_delta": - tool_idx = tool_block_to_index.get(event.index, event.index) - sc.tool_call_deltas.append( - ToolCallDelta( - index=tool_idx, - arguments_delta=delta.partial_json, + if event.index in server_tool_blocks: + # Accumulate server tool input (search query) + server_tool_blocks[event.index]["input_json"] += delta.partial_json + else: + tool_idx = tool_block_to_index.get(event.index, event.index) + sc.tool_call_deltas.append( + ToolCallDelta( + index=tool_idx, + arguments_delta=delta.partial_json, + ) ) - ) + + elif event_type == "content_block_stop": + # When a server tool block completes, emit search query info + if event.index in server_tool_blocks: + info = server_tool_blocks.pop(event.index) + query = "" + try: + parsed = json.loads(info["input_json"]) + query = parsed.get("query", "") + except (json.JSONDecodeError, TypeError): + pass + sc.info_delta = f"[Searching: {query}]" if query else "[Searching...]" elif event_type == "message_delta": if hasattr(event, "usage") and event.usage: @@ -408,7 +483,7 @@ class AnthropicProvider: sc.is_first = True first = False - if has_content or sc.finish_reason or sc.usage: + if has_content or sc.finish_reason or sc.usage or sc.info_delta: yield sc # -- non-streaming ------------------------------------------------------- @@ -442,7 +517,9 @@ class AnthropicProvider: response = client.messages.create(**kwargs) - # Extract content and tool_calls from content blocks + # Extract content and tool_calls from content blocks. + # Skip server-side blocks (server_tool_use, web_search_tool_result) + # which are handled server-side and don't require client execution. content_parts: list[str] = [] tool_calls: list[dict[str, Any]] = [] for block in response.content: @@ -459,6 +536,7 @@ class AnthropicProvider: }, } ) + # server_tool_use, web_search_tool_result — skip (server-handled) finish_reason = _normalize_finish_reason(response.stop_reason or "end_turn") @@ -502,4 +580,7 @@ def _normalize_finish_reason(reason: str) -> str: return "tool_calls" if reason == "max_tokens": return "length" + if reason == "pause_turn": + # Server-side tool (web search) paused a long turn; treat as stop + return "stop" return reason diff --git a/turnstone/core/providers/_openai.py b/turnstone/core/providers/_openai.py index 2eaed666..4fb36548 100644 --- a/turnstone/core/providers/_openai.py +++ b/turnstone/core/providers/_openai.py @@ -23,15 +23,6 @@ from turnstone.core.providers._protocol import ( # -- model capabilities ------------------------------------------------------- _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { - # GPT-4o family - "gpt-4o": ModelCapabilities( - context_window=128000, - max_output_tokens=16384, - ), - "gpt-4o-mini": ModelCapabilities( - context_window=128000, - max_output_tokens=16384, - ), # GPT-5 base — NO temperature support "gpt-5": ModelCapabilities( context_window=400000, @@ -102,6 +93,14 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=100000, supports_temperature=False, ), + # Search models — always search on every request, no reasoning_effort + "gpt-5-search-api": ModelCapabilities( + context_window=400000, + max_output_tokens=128000, + supports_temperature=False, + supports_web_search=True, + reasoning_effort_values=(), + ), } # Default for unknown models (local servers: vLLM, llama.cpp, etc.) @@ -148,6 +147,32 @@ class OpenAIProvider: if caps.reasoning_effort_values and reasoning_effort and reasoning_effort != "none": kwargs["reasoning_effort"] = reasoning_effort + # -- web search ---------------------------------------------------------- + + def _apply_web_search( + self, + kwargs: dict[str, Any], + caps: ModelCapabilities, + tools: list[dict[str, Any]] | None, + ) -> list[dict[str, Any]] | None: + """Inject ``web_search_options`` for search models. + + For models with ``supports_web_search``, the web search function tool + is removed (the model searches automatically) and ``web_search_options`` + is added to the request kwargs. + + Returns the (possibly filtered) tools list. + """ + if not caps.supports_web_search: + return tools + # Remove web_search function tool — model has built-in search + if tools: + tools = [t for t in tools if t.get("function", {}).get("name") != "web_search"] + if not tools: + tools = None + kwargs["web_search_options"] = {} + return tools + # -- streaming ----------------------------------------------------------- def create_streaming( @@ -171,6 +196,7 @@ class OpenAIProvider: "stream_options": {"include_usage": True}, } self._apply_model_params(kwargs, caps, temperature, reasoning_effort) + tools = self._apply_web_search(kwargs, caps, tools) if tools: kwargs["tools"] = tools if extra_params: @@ -182,6 +208,7 @@ class OpenAIProvider: def _iter_stream(self, stream: Any) -> Iterator[StreamChunk]: """Convert OpenAI stream chunks to normalized StreamChunks.""" first = True + annotations: list[Any] = [] for chunk in stream: sc = StreamChunk() @@ -231,6 +258,11 @@ class OpenAIProvider: tcd.arguments_delta = tc_delta.function.arguments sc.tool_call_deltas.append(tcd) + # Accumulate url_citation annotations from search models + delta_anns = getattr(delta, "annotations", None) + if delta_anns: + annotations.extend(delta_anns) + has_content = sc.content_delta or sc.reasoning_delta or sc.tool_call_deltas if has_content and first: sc.is_first = True @@ -239,6 +271,12 @@ class OpenAIProvider: if has_content or sc.finish_reason or sc.usage: yield sc + # Emit accumulated citations as a final info chunk + if annotations: + citation_text = self._format_citations("", annotations).strip() + if citation_text: + yield StreamChunk(info_delta=citation_text) + # -- non-streaming ------------------------------------------------------- def create_completion( @@ -261,6 +299,7 @@ class OpenAIProvider: "stream": False, } self._apply_model_params(kwargs, caps, temperature, reasoning_effort) + tools = self._apply_web_search(kwargs, caps, tools) if tools: kwargs["tools"] = tools if extra_params: @@ -284,6 +323,12 @@ class OpenAIProvider: for tc in msg.tool_calls ] + # Extract url_citation annotations from web search models + content = msg.content or "" + annotations = getattr(msg, "annotations", None) + if annotations: + content = self._format_citations(content, annotations) + usage = None if hasattr(response, "usage") and response.usage: u = response.usage @@ -295,12 +340,31 @@ class OpenAIProvider: ) return CompletionResult( - content=msg.content or "", + content=content, tool_calls=tool_calls, finish_reason=choice.finish_reason or "stop", usage=usage, ) + @staticmethod + def _format_citations(content: str, annotations: list[Any]) -> str: + """Append url_citation sources as footnotes at the end of the content.""" + seen_urls: set[str] = set() + sources: list[str] = [] + for ann in annotations: + ann_type = getattr(ann, "type", None) + if ann_type == "url_citation": + citation = getattr(ann, "url_citation", None) + if citation: + title = getattr(citation, "title", "") + url = getattr(citation, "url", "") + if url and url not in seen_urls: + seen_urls.add(url) + sources.append(f"[{title}]({url})" if title else url) + if sources: + content += "\n\nSources:\n" + "\n".join(f"- {s}" for s in sources) + return content + # -- tool conversion ----------------------------------------------------- def convert_tools( diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index 2ee77e54..425e5b86 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -43,6 +43,7 @@ class StreamChunk: usage: UsageInfo | None = None finish_reason: str | None = None is_first: bool = False + info_delta: str = "" @dataclass @@ -72,6 +73,7 @@ class ModelCapabilities: effort_levels: tuple[str, ...] = () reasoning_effort_values: tuple[str, ...] = () default_reasoning_effort: str = "medium" + supports_web_search: bool = False def _lookup_capabilities( diff --git a/turnstone/core/session.py b/turnstone/core/session.py index deb87e80..4ee479d3 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -770,6 +770,11 @@ class ChatSession: if tcd.arguments_delta: tc["function"]["arguments"] += tcd.arguments_delta + # Informational messages (e.g. server-side web search status) + if chunk.info_delta: + _stop_spinner_once() + self.ui.on_info(f"{GRAY}{chunk.info_delta}{RESET}") + # Flush any remaining buffered text if pending: _flush_text(pending, in_think)