"""Tests for turnstone.core.providers — protocol, OpenAI provider, Anthropic provider.""" from __future__ import annotations import json from typing import Any from unittest.mock import MagicMock, PropertyMock, patch import pytest from turnstone.core.lowering import repair_wire_messages from turnstone.core.providers._openai import OpenAIProvider from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider from turnstone.core.providers._openai_common import ( OPENAI_COMPAT_DEFAULT, apply_cache_retention, apply_temperature_and_effort, apply_tool_search, format_citations, lookup_openai_capabilities, sanitize_messages, ) from turnstone.core.providers._protocol import ( CompletionResult, LLMProvider, ModelCapabilities, StreamChunk, ToolCallDelta, UsageInfo, ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _openai_stream_chunk( *, content: str | None = None, reasoning: str | None = None, reasoning_content: str | None = None, tool_calls: list[MagicMock] | None = None, finish_reason: str | None = None, usage: MagicMock | None = None, empty_choices: bool = False, ) -> MagicMock: """Build a mock OpenAI streaming chunk.""" chunk = MagicMock() if empty_choices: chunk.choices = [] chunk.usage = usage return chunk delta = MagicMock() delta.content = content delta.tool_calls = tool_calls # Reasoning attributes accessed via getattr type(delta).reasoning = PropertyMock(return_value=reasoning) type(delta).reasoning_content = PropertyMock(return_value=reasoning_content) choice = MagicMock() choice.delta = delta choice.finish_reason = finish_reason chunk.choices = [choice] chunk.usage = usage return chunk def _openai_tool_call_delta( *, index: int = 0, tc_id: str | None = None, name: str | None = None, arguments: str | None = None, ) -> MagicMock: """Build a mock OpenAI tool call delta within a streaming chunk.""" tcd = MagicMock() tcd.index = index tcd.id = tc_id tcd.function = MagicMock() tcd.function.name = name tcd.function.arguments = arguments return tcd def _anthropic_event( event_type: str, **kwargs: Any, ) -> MagicMock: """Build a mock Anthropic streaming event.""" event = MagicMock() event.type = event_type if event_type == "content_block_start": block = MagicMock() block.type = kwargs.get("block_type", "text") block.id = kwargs.get("block_id", "") block.name = kwargs.get("block_name", "") event.content_block = block event.index = kwargs.get("index", 0) elif event_type == "content_block_delta": delta = MagicMock() delta.type = kwargs.get("delta_type", "text_delta") delta.text = kwargs.get("text", "") delta.thinking = kwargs.get("thinking", "") delta.signature = kwargs.get("signature", "") delta.partial_json = kwargs.get("partial_json", "") event.delta = delta event.index = kwargs.get("index", 0) elif event_type == "message_delta": if "usage_output_tokens" in kwargs: usage = MagicMock() usage.input_tokens = kwargs.get("usage_input_tokens", 0) usage.output_tokens = kwargs.get("usage_output_tokens", 0) event.usage = usage else: event.usage = None stop_delta = MagicMock() stop_delta.stop_reason = kwargs.get("stop_reason") event.delta = stop_delta elif event_type == "content_block_stop": event.index = kwargs.get("index", 0) elif event_type == "message_start": msg = MagicMock() if "usage_input_tokens" in kwargs: msg_usage = MagicMock() msg_usage.input_tokens = kwargs.get("usage_input_tokens", 0) msg_usage.cache_creation_input_tokens = 0 msg_usage.cache_read_input_tokens = 0 msg.usage = msg_usage else: msg.usage = None event.message = msg return event # =========================================================================== # TestOpenAIProvider # =========================================================================== class TestOpenAIProvider: """Tests for the OpenAI Chat Completions provider adapter.""" def setup_method(self) -> None: self.provider = OpenAIProvider() def test_provider_name(self) -> None: assert self.provider.provider_name == "openai-compatible" # -- reasoning template kwargs (_finalize_extra_body) --------------------- def test_thinking_mode_none_does_nothing(self) -> None: """No toggle injected when thinking_mode is 'none'; operator keys pass.""" caps = ModelCapabilities(thinking_mode="none") extra_params = {"chat_template_kwargs": {"reasoning_effort": "medium"}} eb = self.provider._finalize_extra_body(extra_params, caps, "medium") assert eb is not None assert "enable_thinking" not in eb["chat_template_kwargs"] assert eb["chat_template_kwargs"]["reasoning_effort"] == "medium" def test_thinking_mode_manual_injects_param(self) -> None: """Manual thinking mode injects enable_thinking into chat_template_kwargs.""" caps = ModelCapabilities(thinking_mode="manual") extra_params = {"chat_template_kwargs": {"reasoning_effort": "medium"}} eb = self.provider._finalize_extra_body(extra_params, caps, "medium") assert eb is not None assert eb["chat_template_kwargs"]["enable_thinking"] is True assert eb["chat_template_kwargs"]["reasoning_effort"] == "medium" def test_thinking_mode_manual_knob_none_disables(self) -> None: """Effort knob "none" turns the template toggle off, not just quiet.""" caps = ModelCapabilities(thinking_mode="manual") eb = self.provider._finalize_extra_body(None, caps, "none") assert eb == {"chat_template_kwargs": {"enable_thinking": False}} def test_thinking_mode_custom_param(self) -> None: """Custom thinking_param (e.g. Granite's 'thinking') is used.""" caps = ModelCapabilities(thinking_mode="manual", thinking_param="thinking") eb = self.provider._finalize_extra_body(None, caps, "medium") assert eb == {"chat_template_kwargs": {"thinking": True}} def test_thinking_mode_does_not_override_explicit(self) -> None: """If operator explicitly set the param to False, provider respects it.""" caps = ModelCapabilities(thinking_mode="manual") extra_params = {"chat_template_kwargs": {"enable_thinking": False}} eb = self.provider._finalize_extra_body(extra_params, caps, "medium") assert eb is not None assert eb["chat_template_kwargs"]["enable_thinking"] is False def test_thinking_mode_adaptive_never_knob_disables(self) -> None: """Adaptive = model self-regulates; knob "none" must not force false.""" caps = ModelCapabilities(thinking_mode="adaptive") for knob in ("high", "none", ""): eb = self.provider._finalize_extra_body(None, caps, knob) assert eb == {"chat_template_kwargs": {"enable_thinking": True}} def test_effort_param_suppresses_flat_reasoning_effort(self) -> None: """Declaring the ctk effort channel must not double-send the flat param.""" from turnstone.core.providers._openai_common import apply_temperature_and_effort caps = ModelCapabilities( effort_param="reasoning_effort", reasoning_effort_values=("low", "medium", "high"), ) kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, 0.5, "medium") assert "reasoning_effort" not in kwargs # Without effort_param the flat param still flows (commercial path). flat_caps = ModelCapabilities(reasoning_effort_values=("low", "medium", "high")) kwargs = {} apply_temperature_and_effort(kwargs, flat_caps, 0.5, "medium") assert kwargs["reasoning_effort"] == "medium" def test_effort_param_injects_knob_value(self) -> None: """effort_param carries the knob into chat_template_kwargs (gpt-oss); a knob above the declared ceiling rides the ceiling, not the default.""" caps = ModelCapabilities( thinking_mode="none", effort_param="reasoning_effort", reasoning_effort_values=("low", "medium", "high"), default_reasoning_effort="medium", ) eb = self.provider._finalize_extra_body(None, caps, "xhigh") assert eb == {"chat_template_kwargs": {"reasoning_effort": "high"}} assert self.provider._finalize_extra_body(None, caps, "none") is None def test_caller_extra_params_not_mutated(self) -> None: """The session dict and its ctk sub-dict survive injection untouched.""" caps = ModelCapabilities(thinking_mode="manual") extra_params = {"chat_template_kwargs": {"foo": 1}} self.provider._finalize_extra_body(extra_params, caps, "medium") assert extra_params == {"chat_template_kwargs": {"foo": 1}} # -- _sanitize_messages --------------------------------------------------- def test_sanitize_messages_none_content_no_tool_calls(self) -> None: msgs = [{"role": "assistant", "content": None}] assert sanitize_messages(msgs) == [{"role": "assistant", "content": ""}] def test_sanitize_messages_none_content_with_tool_calls(self) -> None: msgs = [{"role": "assistant", "content": None, "tool_calls": [{"id": "1"}]}] result = sanitize_messages(msgs) assert result[0]["content"] is None assert result[0]["tool_calls"] == [{"id": "1"}] def test_sanitize_messages_empty_string_passthrough(self) -> None: msgs = [{"role": "assistant", "content": ""}] assert sanitize_messages(msgs) == msgs def test_sanitize_messages_non_assistant_unchanged(self) -> None: msgs = [{"role": "user", "content": None}] result = sanitize_messages(msgs) assert result[0]["content"] is None def test_sanitize_messages_does_not_mutate_original(self) -> None: original = {"role": "assistant", "content": None} sanitize_messages([original]) assert original["content"] is None def test_sanitize_messages_strips_underscore_sibling_keys(self) -> None: """Internal sibling metadata (``_reminders``, ``_reminders_delivered``, ``_attachments_meta``, ``_provider_content``) must be stripped before the wire — the OpenAI-compat APIs reject unknown fields.""" msgs = [ { "role": "user", "content": "hi", "_reminders": [{"type": "correction", "text": "watch"}], "_reminders_delivered": True, "_attachments_meta": [{"kind": "image"}], } ] result = sanitize_messages(msgs) assert result == [{"role": "user", "content": "hi"}] assert "_reminders" not in result[0] assert "_reminders_delivered" not in result[0] assert "_attachments_meta" not in result[0] # -- sanitize_messages: orphan detection ----------------------------------- def test_sanitize_orphaned_tool_call_synthesized(self) -> None: """Tool_call with no matching tool result gets a synthetic error result.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "bash", "arguments": "{}"}, }, ], }, {"role": "user", "content": "next"}, ] result = sanitize_messages(repair_wire_messages(msgs)) assert len(result) == 3 assert result[1]["role"] == "tool" assert result[1]["tool_call_id"] == "call_1" assert "cancelled" in result[1]["content"] assert result[2]["role"] == "user" def test_sanitize_partial_results(self) -> None: """Only the missing tool_call gets a synthetic result.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "a", "arguments": "{}"}, }, { "id": "call_2", "type": "function", "function": {"name": "b", "arguments": "{}"}, }, ], }, {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, ] result = sanitize_messages(repair_wire_messages(msgs)) assert len(result) == 3 assert result[1]["tool_call_id"] == "call_1" assert result[1]["content"] == "ok" assert result[2]["role"] == "tool" assert result[2]["tool_call_id"] == "call_2" assert "cancelled" in result[2]["content"] def test_sanitize_complete_results_unchanged(self) -> None: """All tool_calls paired → no changes.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "a", "arguments": "{}"}, }, ], }, {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, {"role": "user", "content": "thanks"}, ] result = sanitize_messages(msgs) assert len(result) == 3 assert result[0]["tool_calls"][0]["id"] == "call_1" assert result[1]["content"] == "ok" assert result[2]["role"] == "user" def test_sanitize_trailing_orphan(self) -> None: """Orphaned tool_call at end of conversation (no following messages).""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "a", "arguments": "{}"}, }, ], }, ] result = sanitize_messages(repair_wire_messages(msgs)) assert len(result) == 2 assert result[1]["role"] == "tool" assert result[1]["tool_call_id"] == "call_1" def test_sanitize_orphaned_tool_result_dropped(self) -> None: """Tool result with no matching tool_call in preceding assistant → dropped.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "a", "arguments": "{}"}, }, ], }, {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, {"role": "tool", "tool_call_id": "call_ORPHAN", "content": "stale"}, ] result = sanitize_messages(msgs) assert len(result) == 2 assert result[1]["tool_call_id"] == "call_1" def test_sanitize_empty_tool_call_id_filled(self) -> None: """Empty tool_call IDs get synthetic values; tool results are remapped to match.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ {"id": "", "type": "function", "function": {"name": "a", "arguments": "{}"}}, ], }, {"role": "tool", "tool_call_id": "", "content": "ok"}, ] result = sanitize_messages(msgs) new_id = result[0]["tool_calls"][0]["id"] assert new_id.startswith("call_") assert len(new_id) > 10 # Tool result must have been remapped to match assert result[1]["tool_call_id"] == new_id # No synthetic result needed — the pairing is complete assert len(result) == 2 def test_sanitize_empty_tool_call_id_orphan_synthesized(self) -> None: """An empty-id tool_call with no result: sanitize back-fills the id AND synthesizes its cancellation (the upstream repair can't see an id-less call, so this lane owns it).""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ {"id": "", "type": "function", "function": {"name": "a", "arguments": "{}"}}, ], }, {"role": "user", "content": "never mind"}, ] result = sanitize_messages(msgs) new_id = result[0]["tool_calls"][0]["id"] assert new_id.startswith("call_") tool_msgs = [m for m in result if m.get("role") == "tool"] assert len(tool_msgs) == 1 assert tool_msgs[0]["tool_call_id"] == new_id # paired to the back-filled id assert "cancelled" in tool_msgs[0]["content"].lower() def test_sanitize_stale_result_with_orphan(self) -> None: """Stale tool results are dropped even when orphaned calls are present.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "a", "arguments": "{}"}, }, { "id": "call_2", "type": "function", "function": {"name": "b", "arguments": "{}"}, }, ], }, {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, {"role": "tool", "tool_call_id": "call_STALE", "content": "stale"}, ] result = sanitize_messages(repair_wire_messages(msgs)) result_tc_ids = [m["tool_call_id"] for m in result if m.get("role") == "tool"] assert "call_STALE" not in result_tc_ids assert "call_1" in result_tc_ids assert "call_2" in result_tc_ids # synthesized def test_sanitize_orphan_no_mutation(self) -> None: """Original messages and dicts are not mutated by orphan detection.""" tc = {"id": "", "type": "function", "function": {"name": "a", "arguments": "{}"}} msg = {"role": "assistant", "content": None, "tool_calls": [tc]} sanitize_messages([msg]) assert tc["id"] == "" # original dict untouched assert msg["tool_calls"][0]["id"] == "" def test_sanitize_repeated_ids_across_turns(self) -> None: """Reused tool_call IDs across turns are handled per-turn, not globally.""" msgs = [ # Turn 1: call_1 fully paired {"role": "user", "content": "do A"}, { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "a", "arguments": "{}"}, }, ], }, {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, # Turn 2: reuses call_1 but has no result → must be synthesized {"role": "user", "content": "do B"}, { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "b", "arguments": "{}"}, }, ], }, ] result = sanitize_messages(repair_wire_messages(msgs)) # Turn 2's orphaned call_1 should get a synthetic result tool_msgs = [m for m in result if m.get("role") == "tool"] assert len(tool_msgs) == 2 # one real from turn 1, one synthetic from turn 2 def test_sanitize_drops_is_error_from_tool_messages(self) -> None: """``is_error`` is the neutral error flag (Anthropic renders it); the OpenAI-compatible tool message has no such field, so it is dropped.""" msgs = [ { "role": "assistant", "content": None, "tool_calls": [ {"id": "c1", "type": "function", "function": {"name": "a", "arguments": "{}"}}, ], }, {"role": "tool", "tool_call_id": "c1", "content": "boom", "is_error": True}, ] result = sanitize_messages(msgs) tool_msg = next(m for m in result if m.get("role") == "tool") assert "is_error" not in tool_msg assert tool_msg["content"] == "boom" # payload otherwise intact # -- convert_tools -------------------------------------------------------- def test_convert_tools_passthrough(self) -> None: tools = [ { "type": "function", "function": { "name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"path": {"type": "string"}}}, }, } ] assert self.provider.convert_tools(tools) is tools def test_streaming_content(self) -> None: chunks = [ _openai_stream_chunk(content="Hello"), _openai_stream_chunk(content=" world"), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) ) assert len(results) == 2 assert results[0].content_delta == "Hello" assert results[1].content_delta == " world" def test_streaming_reasoning(self) -> None: chunks = [ _openai_stream_chunk(reasoning_content="thinking..."), _openai_stream_chunk(reasoning_content="more thought"), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="qwen3-32b", messages=[{"role": "user", "content": "hi"}], ) ) assert len(results) == 2 assert results[0].reasoning_delta == "thinking..." assert results[1].reasoning_delta == "more thought" def test_streaming_tool_calls(self) -> None: tc1 = _openai_tool_call_delta(index=0, tc_id="call_1", name="read_file") tc2 = _openai_tool_call_delta(index=0, arguments='{"path":') tc3 = _openai_tool_call_delta(index=0, arguments='"foo.py"}') chunks = [ _openai_stream_chunk(tool_calls=[tc1]), _openai_stream_chunk(tool_calls=[tc2]), _openai_stream_chunk(tool_calls=[tc3]), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-4o", messages=[{"role": "user", "content": "read a file"}], ) ) assert len(results) == 3 assert results[0].tool_call_deltas[0].id == "call_1" assert results[0].tool_call_deltas[0].name == "read_file" assert results[1].tool_call_deltas[0].arguments_delta == '{"path":' assert results[2].tool_call_deltas[0].arguments_delta == '"foo.py"}' def test_streaming_usage(self) -> None: usage = MagicMock() usage.prompt_tokens = 10 usage.completion_tokens = 20 usage.total_tokens = 30 chunks = [ _openai_stream_chunk(content="Hi"), _openai_stream_chunk(empty_choices=True, usage=usage), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) ) # Last yielded chunk should carry usage usage_chunk = [r for r in results if r.usage is not None] assert len(usage_chunk) == 1 assert usage_chunk[0].usage is not None assert usage_chunk[0].usage.prompt_tokens == 10 assert usage_chunk[0].usage.completion_tokens == 20 assert usage_chunk[0].usage.total_tokens == 30 def test_streaming_finish_reason(self) -> None: chunks = [ _openai_stream_chunk(content="done"), _openai_stream_chunk(finish_reason="stop"), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) ) finish_chunks = [r for r in results if r.finish_reason is not None] assert len(finish_chunks) == 1 assert finish_chunks[0].finish_reason == "stop" def test_streaming_finish_reason_tool_calls(self) -> None: tc = _openai_tool_call_delta(index=0, tc_id="call_1", name="fn") chunks = [ _openai_stream_chunk(tool_calls=[tc]), _openai_stream_chunk(finish_reason="tool_calls"), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) ) finish_chunks = [r for r in results if r.finish_reason is not None] assert finish_chunks[0].finish_reason == "tool_calls" def test_streaming_is_first(self) -> None: chunks = [ _openai_stream_chunk(content="A"), _openai_stream_chunk(content="B"), _openai_stream_chunk(content="C"), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) ) assert results[0].is_first is True assert results[1].is_first is False assert results[2].is_first is False def test_completion_basic(self) -> None: response = MagicMock() response.choices = [MagicMock()] response.choices[0].message.content = "Hello world" response.choices[0].message.tool_calls = None response.choices[0].finish_reason = "stop" response.usage.prompt_tokens = 10 response.usage.completion_tokens = 5 response.usage.total_tokens = 15 client = MagicMock() client.chat.completions.create.return_value = response result = self.provider.create_completion( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) assert isinstance(result, CompletionResult) assert result.content == "Hello world" assert result.tool_calls is None assert result.finish_reason == "stop" def test_completion_with_tools(self) -> None: tc = MagicMock() tc.id = "call_abc" tc.function.name = "read_file" tc.function.arguments = '{"path": "foo.py"}' response = MagicMock() response.choices = [MagicMock()] response.choices[0].message.content = None response.choices[0].message.tool_calls = [tc] response.choices[0].finish_reason = "tool_calls" response.usage.prompt_tokens = 8 response.usage.completion_tokens = 12 response.usage.total_tokens = 20 client = MagicMock() client.chat.completions.create.return_value = response result = self.provider.create_completion( client=client, model="gpt-4o", messages=[{"role": "user", "content": "read"}], ) assert result.content == "" assert result.tool_calls is not None assert len(result.tool_calls) == 1 assert result.tool_calls[0]["id"] == "call_abc" assert result.tool_calls[0]["type"] == "function" assert result.tool_calls[0]["function"]["name"] == "read_file" assert result.tool_calls[0]["function"]["arguments"] == '{"path": "foo.py"}' assert result.finish_reason == "tool_calls" def test_completion_usage(self) -> None: response = MagicMock() response.choices = [MagicMock()] response.choices[0].message.content = "ok" response.choices[0].message.tool_calls = None response.choices[0].finish_reason = "stop" response.usage.prompt_tokens = 100 response.usage.completion_tokens = 50 response.usage.total_tokens = 150 client = MagicMock() client.chat.completions.create.return_value = response result = self.provider.create_completion( client=client, model="gpt-4o", messages=[{"role": "user", "content": "hi"}], ) assert result.usage is not None assert result.usage.prompt_tokens == 100 assert result.usage.completion_tokens == 50 assert result.usage.total_tokens == 150 def test_retryable_errors(self) -> None: errors = self.provider.retryable_error_names assert isinstance(errors, frozenset) assert "APIError" in errors assert "APIConnectionError" in errors assert "RateLimitError" in errors assert "Timeout" in errors assert "APITimeoutError" in errors # =========================================================================== # TestAnthropicProvider # =========================================================================== class TestAnthropicProvider: """Tests for the Anthropic native provider adapter.""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_provider_name(self) -> None: assert self.provider.provider_name == "anthropic" def test_convert_tools(self) -> None: openai_tools = [ { "type": "function", "function": { "name": "read_file", "description": "Read a file from disk", "parameters": { "type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"], }, }, }, { "type": "function", "function": { "name": "write_file", "description": "Write a file", "parameters": { "type": "object", "properties": { "path": {"type": "string"}, "content": {"type": "string"}, }, }, }, }, ] result = self.provider.convert_tools(openai_tools) assert len(result) == 2 assert result[0]["name"] == "read_file" assert result[0]["description"] == "Read a file from disk" assert result[0]["input_schema"]["type"] == "object" assert "path" in result[0]["input_schema"]["properties"] # No "type": "function" wrapper assert "function" not in result[0] assert "type" not in result[0] assert result[1]["name"] == "write_file" def test_message_conversion_basic(self) -> None: messages = [ {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there!"}, {"role": "user", "content": "How are you?"}, ] system, converted = self.provider._convert_messages(messages) assert system == "You are helpful." assert len(converted) == 3 assert converted[0]["role"] == "user" assert converted[0]["content"] == "Hello" assert converted[1]["role"] == "assistant" assert converted[1]["content"] == [{"type": "text", "text": "Hi there!"}] assert converted[2]["role"] == "user" assert converted[2]["content"] == "How are you?" def test_message_conversion_tool_calls(self) -> None: messages = [ { "role": "assistant", "content": "Let me check that.", "tool_calls": [ { "id": "call_1", "function": { "name": "read_file", "arguments": '{"path": "foo.py"}', }, } ], }, {"role": "tool", "tool_call_id": "call_1", "content": "file contents"}, ] _, converted = self.provider._convert_messages(messages) assert len(converted) == 2 blocks = converted[0]["content"] assert len(blocks) == 2 assert blocks[0] == {"type": "text", "text": "Let me check that."} assert blocks[1]["type"] == "tool_use" assert blocks[1]["id"] == "call_1" assert blocks[1]["name"] == "read_file" assert blocks[1]["input"] == {"path": "foo.py"} # Tool result in user message assert converted[1]["role"] == "user" def test_message_conversion_tool_results(self) -> None: messages = [ {"role": "tool", "tool_call_id": "call_1", "content": "file contents here"}, {"role": "tool", "tool_call_id": "call_2", "content": "another result"}, ] _, converted = self.provider._convert_messages(messages) assert len(converted) == 1 assert converted[0]["role"] == "user" blocks = converted[0]["content"] assert len(blocks) == 2 assert blocks[0]["type"] == "tool_result" assert blocks[0]["tool_use_id"] == "call_1" assert blocks[0]["content"] == "file contents here" assert blocks[1]["type"] == "tool_result" assert blocks[1]["tool_use_id"] == "call_2" assert blocks[1]["content"] == "another result" def test_message_conversion_alternating_merge(self) -> None: messages = [ {"role": "user", "content": "Hello"}, {"role": "user", "content": "Are you there?"}, {"role": "assistant", "content": "Yes"}, {"role": "assistant", "content": "I am here"}, ] _, converted = self.provider._convert_messages(messages) assert len(converted) == 2 # First merged user message assert converted[0]["role"] == "user" assert converted[0]["content"] == [ {"type": "text", "text": "Hello"}, {"type": "text", "text": "Are you there?"}, ] # Second merged assistant message assert converted[1]["role"] == "assistant" assert converted[1]["content"] == [ {"type": "text", "text": "Yes"}, {"type": "text", "text": "I am here"}, ] def test_message_conversion_developer_role_as_system(self) -> None: messages = [ {"role": "developer", "content": "System prompt via developer role."}, {"role": "user", "content": "Hi"}, ] system, converted = self.provider._convert_messages(messages) assert system == "System prompt via developer role." assert len(converted) == 1 assert converted[0]["role"] == "user" def test_message_conversion_multiple_system(self) -> None: messages = [ {"role": "system", "content": "Part 1."}, {"role": "system", "content": "Part 2."}, {"role": "user", "content": "Go."}, ] system, _ = self.provider._convert_messages(messages) assert system == "Part 1.\n\nPart 2." def test_mid_conversation_system_hoisted_when_not_native(self) -> None: # Default (supports_mid_conversation_system=False): a system message # after a user turn still hoists — non-native models rely on the fold # pass having stripped operator turns before the converter sees them. messages = [ {"role": "user", "content": "hi"}, {"role": "system", "content": "operator note"}, ] system, converted = self.provider._convert_messages(messages) assert "operator note" in system assert all(m["role"] != "system" for m in converted) def test_leading_system_hoists_even_when_native(self) -> None: messages = [ {"role": "system", "content": "base prompt"}, {"role": "user", "content": "hi"}, ] system, converted = self.provider._convert_messages( messages, supports_mid_conversation_system=True ) assert system == "base prompt" assert [m["role"] for m in converted] == ["user"] def test_mid_conversation_system_inline_when_native(self) -> None: messages = [ {"role": "user", "content": "review this"}, {"role": "assistant", "content": "done"}, {"role": "system", "content": "from now on, add type hints"}, ] system, converted = self.provider._convert_messages( messages, supports_mid_conversation_system=True ) assert system == "" # nothing leading to hoist assert [m["role"] for m in converted] == ["user", "assistant", "system"] assert converted[-1]["content"] == "from now on, add type hints" def test_leading_empty_assistant_then_system_hoists_when_native(self) -> None: # An assistant turn that converts to nothing (empty content, no # tool_calls, no provider_content) still flips seen_non_system, but it # appended nothing — so a following operator system turn must NOT become # messages[0] (the API requires messages[0]=user). It hoists into the # system param instead. Guards the bug where ``seen_non_system`` alone # gated inline emission. messages = [ {"role": "assistant", "content": ""}, {"role": "system", "content": "operator note"}, {"role": "user", "content": "hi"}, ] system, converted = self.provider._convert_messages( messages, supports_mid_conversation_system=True ) assert "operator note" in system assert converted[0]["role"] == "user" assert not any(m["role"] == "system" for m in converted) def test_consecutive_mid_conversation_system_coalesced_when_native(self) -> None: messages = [ {"role": "user", "content": "go"}, {"role": "system", "content": "first"}, {"role": "system", "content": "second"}, ] _, converted = self.provider._convert_messages( messages, supports_mid_conversation_system=True ) # _merge_consecutive coalesces the two system turns into one message # (the API forbids consecutive system messages). assert [m["role"] for m in converted] == ["user", "system"] body = converted[1]["content"] flat = ( body if isinstance(body, str) else " ".join(p.get("text", "") for p in body if isinstance(p, dict)) ) assert "first" in flat and "second" in flat def test_mid_conversation_system_after_tool_result_when_native(self) -> None: messages = [ {"role": "user", "content": "run it"}, { "role": "assistant", "content": "", "tool_calls": [ { "id": "c1", "type": "function", "function": {"name": "run", "arguments": "{}"}, } ], }, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, {"role": "system", "content": "user said: also update changelog"}, ] _, converted = self.provider._convert_messages( messages, supports_mid_conversation_system=True ) # The operator turn lands after the tool_result user turn — a valid slot. roles = [m["role"] for m in converted] assert roles[-1] == "system" assert roles[-2] == "user" # the packed tool_result turn assert converted[-1]["content"] == "user said: also update changelog" def test_reasoning_params_mapping(self) -> None: assert self.provider._reasoning_params("low", None, max_tokens=32768) == { "thinking": {"type": "enabled", "budget_tokens": 1024} } assert self.provider._reasoning_params("medium", None, max_tokens=32768) == { "thinking": {"type": "enabled", "budget_tokens": 4096} } assert self.provider._reasoning_params("high", None, max_tokens=32768) == { "thinking": {"type": "enabled", "budget_tokens": 16384} } def test_reasoning_params_override(self) -> None: result = self.provider._reasoning_params( "low", {"thinking_budget_tokens": 8192}, max_tokens=32768 ) assert result == {"thinking": {"type": "enabled", "budget_tokens": 8192}} def test_reasoning_params_unknown_effort(self) -> None: # Unknown effort falls back to 4096 result = self.provider._reasoning_params("turbo", None, max_tokens=32768) assert result == {"thinking": {"type": "enabled", "budget_tokens": 4096}} def test_reasoning_params_budget_clamped(self) -> None: # Budget >= max_tokens gets clamped to leave room for response result = self.provider._reasoning_params("high", None, max_tokens=4096) assert result == {"thinking": {"type": "enabled", "budget_tokens": 3072}} def test_finish_reason_normalization(self) -> None: from turnstone.core.providers._anthropic import _normalize_finish_reason assert _normalize_finish_reason("end_turn") == "stop" assert _normalize_finish_reason("tool_use") == "tool_calls" assert _normalize_finish_reason("max_tokens") == "length" assert _normalize_finish_reason("other_reason") == "other_reason" @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_completion_basic(self, mock_ensure: MagicMock) -> None: text_block = MagicMock() text_block.type = "text" text_block.text = "Hello world" response = MagicMock() response.content = [text_block] response.stop_reason = "end_turn" response.usage = MagicMock() response.usage.input_tokens = 10 response.usage.output_tokens = 5 response.usage.cache_creation_input_tokens = 0 response.usage.cache_read_input_tokens = 0 client = MagicMock() stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=stream_ctx) stream_ctx.__exit__ = MagicMock(return_value=False) stream_ctx.get_final_message.return_value = response client.messages.stream.return_value = stream_ctx result = self.provider.create_completion( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "hi"}], ) assert isinstance(result, CompletionResult) assert result.content == "Hello world" assert result.tool_calls is None assert result.finish_reason == "stop" @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_completion_with_tool_use(self, mock_ensure: MagicMock) -> None: text_block = MagicMock() text_block.type = "text" text_block.text = "Let me read that." tool_block = MagicMock() tool_block.type = "tool_use" tool_block.id = "toolu_abc" tool_block.name = "read_file" tool_block.input = {"path": "foo.py"} response = MagicMock() response.content = [text_block, tool_block] response.stop_reason = "tool_use" response.usage = MagicMock() response.usage.input_tokens = 15 response.usage.output_tokens = 20 response.usage.cache_creation_input_tokens = 0 response.usage.cache_read_input_tokens = 0 client = MagicMock() stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=stream_ctx) stream_ctx.__exit__ = MagicMock(return_value=False) stream_ctx.get_final_message.return_value = response client.messages.stream.return_value = stream_ctx result = self.provider.create_completion( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "read foo.py"}], ) assert result.content == "Let me read that." assert result.finish_reason == "tool_calls" assert result.tool_calls is not None assert len(result.tool_calls) == 1 tc = result.tool_calls[0] assert tc["id"] == "toolu_abc" assert tc["type"] == "function" assert tc["function"]["name"] == "read_file" assert json.loads(tc["function"]["arguments"]) == {"path": "foo.py"} @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_completion_usage(self, mock_ensure: MagicMock) -> None: text_block = MagicMock() text_block.type = "text" text_block.text = "ok" response = MagicMock() response.content = [text_block] response.stop_reason = "end_turn" response.usage = MagicMock() response.usage.input_tokens = 100 response.usage.output_tokens = 50 response.usage.cache_creation_input_tokens = 0 response.usage.cache_read_input_tokens = 0 client = MagicMock() stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=stream_ctx) stream_ctx.__exit__ = MagicMock(return_value=False) stream_ctx.get_final_message.return_value = response client.messages.stream.return_value = stream_ctx result = self.provider.create_completion( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "hi"}], ) assert result.usage is not None assert result.usage.prompt_tokens == 100 assert result.usage.completion_tokens == 50 assert result.usage.total_tokens == 150 @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_text_delta(self, mock_ensure: MagicMock) -> None: events = [ _anthropic_event("content_block_delta", delta_type="text_delta", text="Hello"), _anthropic_event("content_block_delta", delta_type="text_delta", text=" world"), ] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "hi"}], ) ) assert len(results) == 2 assert results[0].content_delta == "Hello" assert results[0].is_first is True assert results[1].content_delta == " world" assert results[1].is_first is False @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_thinking_delta(self, mock_ensure: MagicMock) -> None: events = [ _anthropic_event( "content_block_delta", delta_type="thinking_delta", thinking="reasoning step 1", ), _anthropic_event( "content_block_delta", delta_type="thinking_delta", thinking="reasoning step 2", ), ] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "think"}], ) ) assert len(results) == 2 assert results[0].reasoning_delta == "reasoning step 1" assert results[1].reasoning_delta == "reasoning step 2" @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_tool_use(self, mock_ensure: MagicMock) -> None: events = [ _anthropic_event( "content_block_start", block_type="tool_use", block_id="toolu_123", block_name="read_file", index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"path":', index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='"foo.py"}', index=0, ), ] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "read a file"}], ) ) assert len(results) == 3 # First chunk: content_block_start with tool id and name assert results[0].tool_call_deltas[0].id == "toolu_123" assert results[0].tool_call_deltas[0].name == "read_file" assert results[0].tool_call_deltas[0].index == 0 # Subsequent chunks: argument fragments assert results[1].tool_call_deltas[0].arguments_delta == '{"path":' assert results[2].tool_call_deltas[0].arguments_delta == '"foo.py"}' @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_message_delta_usage(self, mock_ensure: MagicMock) -> None: events = [ _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi"), _anthropic_event( "message_delta", stop_reason="end_turn", usage_input_tokens=0, usage_output_tokens=12, ), ] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "hi"}], ) ) # The message_delta event should carry usage and finish_reason delta_chunks = [r for r in results if r.finish_reason is not None] assert len(delta_chunks) == 1 assert delta_chunks[0].finish_reason == "stop" assert delta_chunks[0].usage is not None assert delta_chunks[0].usage.completion_tokens == 12 @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_message_start_usage(self, mock_ensure: MagicMock) -> None: events = [ _anthropic_event("message_start", usage_input_tokens=42), _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi"), ] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-20250514", messages=[{"role": "user", "content": "hi"}], ) ) # message_start with usage should be yielded start_chunks = [r for r in results if r.usage is not None and r.usage.prompt_tokens == 42] assert len(start_chunks) == 1 assert start_chunks[0].usage is not None assert start_chunks[0].usage.prompt_tokens == 42 def test_retryable_errors(self) -> None: errors = self.provider.retryable_error_names assert isinstance(errors, frozenset) assert "RateLimitError" in errors assert "APITimeoutError" in errors assert "APIConnectionError" in errors assert "InternalServerError" in errors assert "APIError" in errors assert "OverloadedError" in errors # =========================================================================== # TestAnthropicHelpers # =========================================================================== class TestAnthropicHelpers: """Tests for Anthropic module-level helper functions.""" def test_merge_consecutive(self) -> None: from turnstone.core.providers._anthropic import _merge_consecutive messages = [ {"role": "user", "content": "A"}, {"role": "user", "content": "B"}, {"role": "assistant", "content": "C"}, {"role": "user", "content": "D"}, ] merged = _merge_consecutive(messages) assert len(merged) == 3 assert merged[0]["role"] == "user" assert merged[0]["content"] == [ {"type": "text", "text": "A"}, {"type": "text", "text": "B"}, ] assert merged[1]["role"] == "assistant" assert merged[2]["role"] == "user" def test_merge_consecutive_empty(self) -> None: from turnstone.core.providers._anthropic import _merge_consecutive assert _merge_consecutive([]) == [] def test_merge_consecutive_no_duplicates(self) -> None: from turnstone.core.providers._anthropic import _merge_consecutive messages = [ {"role": "user", "content": "A"}, {"role": "assistant", "content": "B"}, {"role": "user", "content": "C"}, ] merged = _merge_consecutive(messages) assert len(merged) == 3 def test_to_blocks_string(self) -> None: from turnstone.core.providers._anthropic import _to_blocks result = _to_blocks("hello") assert result == [{"type": "text", "text": "hello"}] def test_to_blocks_list(self) -> None: from turnstone.core.providers._anthropic import _to_blocks blocks = [{"type": "text", "text": "already a block"}] result = _to_blocks(blocks) assert result == blocks def test_to_blocks_other(self) -> None: from turnstone.core.providers._anthropic import _to_blocks result = _to_blocks(42) assert result == [{"type": "text", "text": "42"}] def test_capabilities_lookup_exact(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-opus-4-6") assert caps.context_window == 1000000 assert caps.max_output_tokens == 128000 assert caps.thinking_mode == "adaptive" assert caps.supports_effort is True def test_capabilities_lookup_prefix(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() # Prefix match: "claude-sonnet-4-6" matches dated variants caps = provider.get_capabilities("claude-sonnet-4-6-20260101") assert caps.context_window == 1000000 assert caps.token_param == "max_tokens" assert caps.thinking_mode == "adaptive" def test_capabilities_fable_5(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-fable-5") assert caps.context_window == 1000000 assert caps.max_output_tokens == 128000 assert caps.thinking_mode == "adaptive" assert caps.supports_effort is True assert "xhigh" in caps.effort_levels assert "max" in caps.effort_levels assert caps.supports_temperature is False assert caps.thinking_display == "summarized" assert caps.supports_web_search is True assert caps.supports_tool_search is True assert caps.supports_vision is True assert caps.supports_reasoning_replay is True assert caps.supports_mid_conversation_system is True def test_capabilities_fable_5_dated(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-fable-5-20260815") assert caps.context_window == 1000000 assert caps.supports_temperature is False assert caps.thinking_display == "summarized" assert caps.supports_mid_conversation_system is True def test_capabilities_opus_4_8(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-opus-4-8") assert caps.context_window == 1000000 assert caps.max_output_tokens == 128000 assert caps.thinking_mode == "adaptive" assert caps.supports_effort is True assert "xhigh" in caps.effort_levels assert "max" in caps.effort_levels assert caps.supports_temperature is False assert caps.thinking_display == "summarized" assert caps.supports_web_search is True assert caps.supports_tool_search is True assert caps.supports_vision is True assert caps.supports_reasoning_replay is True assert caps.supports_mid_conversation_system is True def test_capabilities_opus_4_8_dated(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-opus-4-8-20260601") assert caps.context_window == 1000000 assert caps.supports_temperature is False assert caps.thinking_display == "summarized" def test_capabilities_opus_4_7(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-opus-4-7") assert caps.context_window == 1000000 assert caps.max_output_tokens == 128000 assert caps.thinking_mode == "adaptive" assert caps.supports_effort is True assert "xhigh" in caps.effort_levels assert caps.supports_temperature is False assert caps.thinking_display == "summarized" assert caps.supports_web_search is True assert caps.supports_tool_search is True assert caps.supports_vision is True def test_capabilities_opus_4_7_dated(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-opus-4-7-20260416") assert caps.context_window == 1000000 assert caps.supports_temperature is False assert caps.thinking_display == "summarized" def test_capabilities_lookup_unknown(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("unknown-model-xyz") # Falls back to default assert caps.context_window == 200000 assert caps.thinking_mode == "manual" assert caps.token_param == "max_tokens" # =========================================================================== # TestProviderFactory # =========================================================================== class TestProviderFactory: """Tests for create_provider and create_client factory functions.""" def test_create_provider_openai(self) -> None: from turnstone.core.providers import OpenAIResponsesProvider, create_provider provider = create_provider("openai") assert isinstance(provider, OpenAIResponsesProvider) assert provider.provider_name == "openai" def test_create_provider_anthropic(self) -> None: from turnstone.core.providers import create_provider provider = create_provider("anthropic") assert provider.provider_name == "anthropic" def test_create_provider_unknown(self) -> None: from turnstone.core.providers import create_provider with pytest.raises(ValueError, match="Unknown provider"): create_provider("gemini") @patch("openai.OpenAI") def test_create_client_openai(self, mock_openai_cls: MagicMock) -> None: from turnstone.core.providers import create_client mock_openai_cls.return_value = MagicMock() client = create_client("openai", base_url="http://localhost:8000/v1", api_key="test-key") mock_openai_cls.assert_called_once_with( base_url="http://localhost:8000/v1", api_key="test-key" ) assert client is mock_openai_cls.return_value @patch("openai.OpenAI") def test_create_client_empty_api_key_passes_none(self, mock_openai_cls: MagicMock) -> None: from turnstone.core.providers import create_client mock_openai_cls.return_value = MagicMock() create_client("openai", base_url="http://localhost:8000/v1", api_key="") mock_openai_cls.assert_called_once_with(base_url="http://localhost:8000/v1", api_key=None) @patch("openai.OpenAI") def test_create_client_empty_api_key_no_base_url(self, mock_openai_cls: MagicMock) -> None: from turnstone.core.providers import create_client mock_openai_cls.return_value = MagicMock() create_client("openai", base_url="", api_key="") mock_openai_cls.assert_called_once_with(api_key=None) @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_create_client_anthropic_empty_api_key_omits_kwarg( self, mock_ensure: MagicMock ) -> None: from turnstone.core.providers import create_client mock_anthropic_cls = MagicMock() mock_mod = MagicMock() mock_mod.Anthropic = mock_anthropic_cls mock_ensure.return_value = mock_mod create_client("anthropic", base_url="", api_key="") mock_anthropic_cls.assert_called_once_with() @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_create_client_anthropic_nonempty_api_key_passes_kwarg( self, mock_ensure: MagicMock ) -> None: from turnstone.core.providers import create_client mock_anthropic_cls = MagicMock() mock_mod = MagicMock() mock_mod.Anthropic = mock_anthropic_cls mock_ensure.return_value = mock_mod create_client("anthropic", base_url="", api_key="sk-ant-test") mock_anthropic_cls.assert_called_once_with(api_key="sk-ant-test") @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_create_client_anthropic_empty_api_key_with_custom_base_url( self, mock_ensure: MagicMock ) -> None: from turnstone.core.providers import create_client mock_anthropic_cls = MagicMock() mock_mod = MagicMock() mock_mod.Anthropic = mock_anthropic_cls mock_ensure.return_value = mock_mod create_client("anthropic", base_url="http://my-proxy:8000", api_key="") mock_anthropic_cls.assert_called_once_with(base_url="http://my-proxy:8000") def test_create_client_unknown(self) -> None: from turnstone.core.providers import create_client with pytest.raises(ValueError, match="Unknown provider"): create_client("gemini", base_url="http://x", api_key="k") def test_is_llm_provider(self) -> None: """Verify runtime_checkable protocol works with isinstance.""" provider = OpenAIProvider() assert isinstance(provider, LLMProvider) def test_non_provider_not_instance(self) -> None: """A plain object should not satisfy LLMProvider protocol check.""" class NotAProvider: pass assert not isinstance(NotAProvider(), LLMProvider) def test_create_provider_openai_compatible(self) -> None: from turnstone.core.providers import create_provider provider = create_provider("openai-compatible") assert isinstance(provider, OpenAIChatCompletionsProvider) assert provider.provider_name == "openai-compatible" def test_create_provider_openai_vs_compatible_distinct(self) -> None: from turnstone.core.providers import OpenAIResponsesProvider, create_provider openai_prov = create_provider("openai") compat = create_provider("openai-compatible") assert openai_prov is not compat assert isinstance(openai_prov, OpenAIResponsesProvider) assert isinstance(compat, OpenAIChatCompletionsProvider) assert openai_prov.provider_name == "openai" assert compat.provider_name == "openai-compatible" def test_openai_compatible_never_consults_commercial_registry(self) -> None: """Local-lane model ids are operator-chosen strings — a prefix collision with a cloud model id must not inherit that model's sampling/effort contract, on either API surface. Cloud lookups are unaffected.""" from turnstone.core.providers import create_provider compat = create_provider("openai-compatible") compat_responses = create_provider("openai-compatible", api_surface="responses") for name in ("gpt-5.5-my-finetune", "o3-distill", "deepseek-v4-flash", ""): assert compat.get_capabilities(name) is OPENAI_COMPAT_DEFAULT assert compat_responses.get_capabilities(name) is OPENAI_COMPAT_DEFAULT # The commercial lane keeps resolving its registry rows — through # the factory AND through the non-compat class default. cloud = create_provider("openai").get_capabilities("gpt-5.5") assert cloud.default_reasoning_effort == "medium" assert "xhigh" in cloud.reasoning_effort_values assert create_provider("openai") is not compat_responses def test_create_provider_returns_singleton(self) -> None: from turnstone.core.providers import create_provider p1 = create_provider("openai") p2 = create_provider("openai") assert p1 is p2 def test_create_provider_compat_responses_surface(self) -> None: """openai-compatible + api_surface=responses returns the Responses provider.""" from turnstone.core.providers import OpenAIResponsesProvider, create_provider provider = create_provider("openai-compatible", api_surface="responses") assert isinstance(provider, OpenAIResponsesProvider) def test_create_provider_compat_chat_surface_default(self) -> None: """openai-compatible defaults to Chat Completions.""" from turnstone.core.providers import create_provider for surface in (None, "", "chat"): provider = create_provider("openai-compatible", api_surface=surface) assert isinstance(provider, OpenAIChatCompletionsProvider) def test_create_provider_invalid_api_surface(self) -> None: from turnstone.core.providers import create_provider with pytest.raises(ValueError, match="Unknown api_surface"): create_provider("openai-compatible", api_surface="bogus") def test_create_provider_openai_ignores_api_surface(self) -> None: """Cloud OpenAI is always Responses regardless of api_surface.""" from turnstone.core.providers import OpenAIResponsesProvider, create_provider provider = create_provider("openai", api_surface="chat") assert isinstance(provider, OpenAIResponsesProvider) # -- Google provider ------------------------------------------------------- def test_create_provider_google(self) -> None: from turnstone.core.providers import create_provider from turnstone.core.providers._google import GoogleProvider provider = create_provider("google") assert isinstance(provider, GoogleProvider) assert provider.provider_name == "google" def test_create_provider_google_singleton(self) -> None: from turnstone.core.providers import create_provider p1 = create_provider("google") p2 = create_provider("google") assert p1 is p2 @patch("openai.OpenAI") def test_create_client_google_default_base_url(self, mock_openai_cls: MagicMock) -> None: from turnstone.core.providers import create_client from turnstone.core.providers._google import GOOGLE_DEFAULT_BASE_URL mock_openai_cls.return_value = MagicMock() create_client("google", base_url="", api_key="test-key") mock_openai_cls.assert_called_once_with( base_url=GOOGLE_DEFAULT_BASE_URL, api_key="test-key" ) @patch("openai.OpenAI") def test_create_client_google_custom_base_url(self, mock_openai_cls: MagicMock) -> None: from turnstone.core.providers import create_client mock_openai_cls.return_value = MagicMock() create_client("google", base_url="http://custom:8080/v1", api_key="k") mock_openai_cls.assert_called_once_with(base_url="http://custom:8080/v1", api_key="k") def test_google_capabilities_defaults(self) -> None: from turnstone.core.providers import create_provider provider = create_provider("google") caps = provider.get_capabilities("gemini-2.5-pro") assert caps.context_window == 2_000_000 assert caps.max_output_tokens == 65_536 assert caps.token_param == "max_tokens" assert caps.supports_temperature is True assert caps.supports_vision is True def test_google_capabilities_same_for_all_models(self) -> None: from turnstone.core.providers import create_provider provider = create_provider("google") c1 = provider.get_capabilities("gemini-2.5-pro") c2 = provider.get_capabilities("gemini-2.0-flash") c3 = provider.get_capabilities("") assert c1 is c2 is c3 def test_list_known_models_google_empty(self) -> None: from turnstone.core.providers import list_known_models assert list_known_models("google") == [] def test_lookup_model_capabilities_google_returns_none(self) -> None: from turnstone.core.providers import lookup_model_capabilities assert lookup_model_capabilities("google", "gemini-2.5-pro") is None def test_resolve_openai_provider_googleapis(self) -> None: from turnstone.core.model_registry import _resolve_openai_provider assert ( _resolve_openai_provider( "openai", "https://generativelanguage.googleapis.com/v1beta/openai/", ) == "google" ) def test_resolve_openai_provider_not_spoofable(self) -> None: from turnstone.core.model_registry import _resolve_openai_provider # evil-googleapis.com must NOT match — requires the dot prefix assert ( _resolve_openai_provider("openai", "https://evil-googleapis.com/v1") == "openai-compatible" ) def test_resolve_openai_provider_api_openai_unchanged(self) -> None: from turnstone.core.model_registry import _resolve_openai_provider assert _resolve_openai_provider("openai", "https://api.openai.com/v1") == "openai" # =========================================================================== # Google provider fidelity # =========================================================================== class TestGoogleEffortKnob: """The session effort knob reaches Gemini as a flat reasoning_effort.""" def _create_kwargs(self, reasoning_effort: str) -> dict[str, Any]: from turnstone.core.providers._google import GoogleProvider prov = GoogleProvider() client = MagicMock() client.chat.completions.create.return_value = iter([]) list( prov.create_streaming( client=client, model="gemini-3-flash", messages=[{"role": "user", "content": "hi"}], reasoning_effort=reasoning_effort, ) ) return client.chat.completions.create.call_args[1] def test_knob_values_forward_verbatim(self) -> None: for knob in ("minimal", "low", "medium", "high"): assert self._create_kwargs(knob)["reasoning_effort"] == knob def test_off_list_knob_snaps_to_high(self) -> None: """xhigh/max are not in Gemini's vocabulary — snap down to high.""" for knob in ("xhigh", "max"): assert self._create_kwargs(knob)["reasoning_effort"] == "high" def test_none_omits_the_param(self) -> None: """Knob none never sends "none" — 2.5 Pro / 3.x reject disabling.""" assert "reasoning_effort" not in self._create_kwargs("none") class TestGoogleProviderFidelity: """Tests for thought_signature round-trip via provider_blocks.""" def test_prepare_messages_strips_provider_content(self) -> None: from turnstone.core.providers._google import GoogleProvider prov = GoogleProvider() msgs = [ { "role": "assistant", "content": "", "tool_calls": [ {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}}, ], "_provider_content": [ { "id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}, "thought_signature": "sig123", }, ], }, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ] cleaned = prov._prepare_messages(msgs) # _provider_content must be stripped for m in cleaned: assert "_provider_content" not in m # tool_calls must be reconstructed with thought_signature tc = cleaned[0]["tool_calls"][0] assert tc["thought_signature"] == "sig123" def test_prepare_messages_passthrough_without_provider_content(self) -> None: from turnstone.core.providers._google import GoogleProvider prov = GoogleProvider() msgs = [ {"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi"}, ] cleaned = prov._prepare_messages(msgs) assert len(cleaned) == 2 assert cleaned[0]["content"] == "hello" def test_non_streaming_captures_provider_blocks(self) -> None: from turnstone.core.providers._google import GoogleProvider prov = GoogleProvider() # Build a mock response with thought_signature in __pydantic_extra__ mock_tc = MagicMock() mock_tc.id = "c1" mock_tc.function.name = "write_file" mock_tc.function.arguments = '{"path":"test.txt"}' mock_tc.model_dump.return_value = { "id": "c1", "type": "function", "function": {"name": "write_file", "arguments": '{"path":"test.txt"}'}, "thought_signature": "sig_abc", } mock_msg = MagicMock() mock_msg.tool_calls = [mock_tc] mock_msg.content = "" mock_msg.annotations = None mock_choice = MagicMock() mock_choice.message = mock_msg mock_choice.finish_reason = "tool_calls" mock_response = MagicMock() mock_response.choices = [mock_choice] mock_response.usage = None mock_client = MagicMock() mock_client.chat.completions.create.return_value = mock_response result = prov.create_completion( client=mock_client, model="gemini-2.5-pro", messages=[{"role": "user", "content": "test"}], ) # Normalised tool_calls should NOT have thought_signature assert result.tool_calls is not None assert "thought_signature" not in result.tool_calls[0] # provider_blocks should have the raw dict WITH thought_signature assert len(result.provider_blocks) == 1 assert result.provider_blocks[0]["thought_signature"] == "sig_abc" def test_prepare_messages_base_class_unchanged(self) -> None: """Base class _prepare_messages just calls sanitize_messages.""" from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider prov = OpenAIChatCompletionsProvider() msgs = [ {"role": "assistant", "content": None}, # should get content="" {"role": "user", "content": "hi"}, ] cleaned = prov._prepare_messages(msgs) assert cleaned[0]["content"] == "" def test_streaming_captures_thought_signature(self) -> None: """Streaming _iter_stream taps raw deltas and emits provider_blocks.""" from turnstone.core.providers._google import GoogleProvider prov = GoogleProvider() # Build a minimal mock stream with 2 chunks: # chunk 1: tool call header with thought_signature # chunk 2: finish reason mock_fn = MagicMock() mock_fn.name = "write_file" mock_fn.arguments = '{"path":"test.txt"}' mock_tc_delta = MagicMock() mock_tc_delta.index = 0 mock_tc_delta.id = "call_abc" mock_tc_delta.function = mock_fn mock_tc_delta.__pydantic_extra__ = {"thought_signature": "sig_stream"} mock_delta1 = MagicMock() mock_delta1.content = None mock_delta1.tool_calls = [mock_tc_delta] mock_delta1.annotations = None # reasoning fields mock_delta1.reasoning = None mock_delta1.reasoning_content = None mock_choice1 = MagicMock() mock_choice1.finish_reason = None mock_choice1.delta = mock_delta1 mock_chunk1 = MagicMock() mock_chunk1.choices = [mock_choice1] mock_chunk1.usage = None # Finish chunk mock_delta2 = MagicMock() mock_delta2.content = None mock_delta2.tool_calls = None mock_delta2.annotations = None mock_delta2.reasoning = None mock_delta2.reasoning_content = None mock_choice2 = MagicMock() mock_choice2.finish_reason = "tool_calls" mock_choice2.delta = mock_delta2 mock_chunk2 = MagicMock() mock_chunk2.choices = [mock_choice2] mock_chunk2.usage = None chunks = list(prov._iter_stream([mock_chunk1, mock_chunk2])) # Find the chunk with finish_reason finish_chunks = [c for c in chunks if c.finish_reason] assert len(finish_chunks) == 1 fc = finish_chunks[0] assert len(fc.provider_blocks) == 1 assert fc.provider_blocks[0]["thought_signature"] == "sig_stream" assert fc.provider_blocks[0]["id"] == "call_abc" assert fc.provider_blocks[0]["function"]["name"] == "write_file" def test_base_extract_tool_calls_returns_empty_provider_blocks(self) -> None: """Base class _extract_tool_calls returns empty provider_blocks.""" from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider prov = OpenAIChatCompletionsProvider() mock_tc = MagicMock() mock_tc.id = "c1" mock_tc.function.name = "test" mock_tc.function.arguments = "{}" tool_calls, provider_blocks = prov._extract_tool_calls([mock_tc]) assert len(tool_calls) == 1 assert provider_blocks == [] # =========================================================================== # TestDataclasses # =========================================================================== class TestDataclasses: """Tests for protocol dataclass construction and defaults.""" def test_stream_chunk_defaults(self) -> None: sc = StreamChunk() assert sc.content_delta == "" assert sc.reasoning_delta == "" assert sc.tool_call_deltas == [] assert sc.usage is None assert sc.finish_reason is None assert sc.is_first is False def test_tool_call_delta_defaults(self) -> None: tcd = ToolCallDelta(index=0) assert tcd.index == 0 assert tcd.id == "" assert tcd.name == "" assert tcd.arguments_delta == "" def test_usage_info(self) -> None: u = UsageInfo(prompt_tokens=10, completion_tokens=5, total_tokens=15) assert u.prompt_tokens == 10 assert u.completion_tokens == 5 assert u.total_tokens == 15 def test_completion_result_defaults(self) -> None: cr = CompletionResult(content="hello") assert cr.content == "hello" assert cr.tool_calls is None assert cr.finish_reason == "stop" assert cr.usage is None def test_stream_chunk_info_delta_default(self) -> None: sc = StreamChunk() assert sc.info_delta == "" def test_model_capabilities_web_search_default(self) -> None: from turnstone.core.providers._protocol import ModelCapabilities caps = ModelCapabilities() assert caps.supports_web_search is False # =========================================================================== # TestParameterGating — model capability parameter gating # =========================================================================== class TestOpenAIParameterGating: """Verify _apply_model_params gates temperature and reasoning_effort correctly.""" def setup_method(self) -> None: self.provider = OpenAIProvider() def test_local_model_effort_forwarded_verbatim(self) -> None: """Local-lane models receive the session knob verbatim on the flat param (effort_passthrough) — the user's effort setting always reaches the wire; "none" stays omitted (nothing to disable beyond the template toggle).""" caps = self.provider.get_capabilities("my-local-model") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert kwargs["reasoning_effort"] == "medium" assert kwargs["temperature"] == 0.7 kwargs = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert "reasoning_effort" not in kwargs def test_gpt5_no_temperature_has_reasoning_effort(self) -> None: """GPT-5 base: no temperature, reasoning_effort sent.""" caps = lookup_openai_capabilities("gpt-5") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "high" def test_gpt51_temperature_when_effort_none(self) -> None: """GPT-5.1: temperature only when reasoning_effort='none'; the declared "none" level is forwarded explicitly (knob = off).""" caps = lookup_openai_capabilities("gpt-5.1") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 assert kwargs["reasoning_effort"] == "none" def test_gpt51_no_temperature_when_reasoning_active(self) -> None: """GPT-5.1: no temperature when reasoning is active.""" caps = lookup_openai_capabilities("gpt-5.1") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "high" def test_o_series_no_temperature_but_effort_forwarded(self) -> None: """O-series: no temperature; low/medium/high ARE valid effort values (all o-series except o1-mini) and the knob reaches them.""" caps = lookup_openai_capabilities("o3") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "medium" def test_o1_mini_has_no_effort_control(self) -> None: """o1-mini is the one o-series model without reasoning_effort.""" caps = lookup_openai_capabilities("o1-mini") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert "reasoning_effort" not in kwargs def test_gpt5_pro_unsupported_effort_falls_back(self) -> None: """GPT-5 pro only supports 'high'; unsupported values fall back to default.""" caps = lookup_openai_capabilities("gpt-5-pro") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "high" # fell back to default def test_gpt5_pro_supported_effort_passes_through(self) -> None: """GPT-5 pro accepts 'high' directly.""" caps = lookup_openai_capabilities("gpt-5-pro") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high") assert kwargs["reasoning_effort"] == "high" def test_gpt54_1m_context_and_effort(self) -> None: """GPT-5.4: 1M context, temperature when effort=none, xhigh supported.""" caps = lookup_openai_capabilities("gpt-5.4") assert caps.context_window == 1050000 kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 assert kwargs["reasoning_effort"] == "none" # declared level, forwarded kwargs2: dict[str, Any] = {} apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh") assert "temperature" not in kwargs2 assert kwargs2["reasoning_effort"] == "xhigh" def test_gpt54_pro_no_temperature_always_reasoning(self) -> None: """GPT-5.4 pro: no temperature, medium/high/xhigh only.""" caps = lookup_openai_capabilities("gpt-5.4-pro") assert caps.context_window == 1050000 kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "medium" # fell back from unsupported "low" def test_gpt55_1m_context_and_effort(self) -> None: """GPT-5.5: 1M context, temperature when effort=none, xhigh supported.""" caps = lookup_openai_capabilities("gpt-5.5") assert caps.context_window == 1050000 assert caps.supports_tool_search is True assert caps.supports_vision is True kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 assert kwargs["reasoning_effort"] == "none" # declared level, forwarded kwargs2: dict[str, Any] = {} apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh") assert "temperature" not in kwargs2 assert kwargs2["reasoning_effort"] == "xhigh" def test_gpt55_pro_no_temperature_always_reasoning(self) -> None: """GPT-5.5 pro: no temperature, medium/high/xhigh only.""" caps = lookup_openai_capabilities("gpt-5.5-pro") assert caps.context_window == 1050000 assert caps.supports_tool_search is True kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "medium" # fell back from unsupported "low" class TestAnthropicOrphanedToolUse: """Verify _convert_messages synthesizes tool_results for orphaned tool_use.""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_orphaned_tool_use_gets_synthetic_result(self) -> None: """Assistant has tool_calls but next message is user (no tool results).""" messages = [ {"role": "user", "content": "do something"}, { "role": "assistant", "content": "I'll run that.", "tool_calls": [ { "id": "call_abc", "function": {"name": "bash", "arguments": '{"command": "ls"}'}, } ], }, {"role": "user", "content": "never mind, do something else"}, ] _, converted = self.provider._convert_messages(repair_wire_messages(messages)) # Should have: user, assistant(tool_use), user(synthetic tool_result), user # After _merge_consecutive, the two user messages may merge. # Find the synthetic tool_result tool_results = [] for msg in converted: if msg["role"] == "user" and isinstance(msg["content"], list): for block in msg["content"]: if isinstance(block, dict) and block.get("type") == "tool_result": tool_results.append(block) assert len(tool_results) == 1 assert tool_results[0]["tool_use_id"] == "call_abc" assert tool_results[0]["is_error"] is True assert "cancelled" in tool_results[0]["content"].lower() def test_multiple_orphaned_tool_calls(self) -> None: """Assistant has 3 tool_calls, none have results.""" messages = [ {"role": "user", "content": "do three things"}, { "role": "assistant", "content": "", "tool_calls": [ {"id": "c1", "function": {"name": "bash", "arguments": "{}"}}, {"id": "c2", "function": {"name": "read_file", "arguments": "{}"}}, {"id": "c3", "function": {"name": "write_file", "arguments": "{}"}}, ], }, {"role": "user", "content": "skip all that"}, ] _, converted = self.provider._convert_messages(repair_wire_messages(messages)) tool_results = [] for msg in converted: if msg["role"] == "user" and isinstance(msg["content"], list): for block in msg["content"]: if isinstance(block, dict) and block.get("type") == "tool_result": tool_results.append(block) assert len(tool_results) == 3 result_ids = {r["tool_use_id"] for r in tool_results} assert result_ids == {"c1", "c2", "c3"} def test_partial_results_only_orphans_synthesized(self) -> None: """2 tool_calls, only 1 has a result — synthesize for the missing one.""" messages = [ {"role": "user", "content": "do two things"}, { "role": "assistant", "content": "", "tool_calls": [ {"id": "c1", "function": {"name": "bash", "arguments": "{}"}}, {"id": "c2", "function": {"name": "write_file", "arguments": "{}"}}, ], }, {"role": "tool", "tool_call_id": "c1", "content": "file1.txt"}, {"role": "user", "content": "skip the write"}, ] _, converted = self.provider._convert_messages(repair_wire_messages(messages)) # c1 should have a real result, c2 should have a synthetic one tool_results = [] for msg in converted: if msg["role"] == "user" and isinstance(msg["content"], list): for block in msg["content"]: if isinstance(block, dict) and block.get("type") == "tool_result": tool_results.append(block) # Real result should come before synthetic (ordering matters for Anthropic) assert len(tool_results) == 2 assert tool_results[0]["tool_use_id"] == "c1" assert tool_results[0]["content"] == "file1.txt" # real result assert tool_results[0].get("is_error") is not True assert tool_results[1]["tool_use_id"] == "c2" assert tool_results[1]["is_error"] is True # synthetic def test_complete_results_no_synthesis(self) -> None: """All tool_calls have results — no synthesis needed.""" messages = [ {"role": "user", "content": "do it"}, { "role": "assistant", "content": "", "tool_calls": [ {"id": "c1", "function": {"name": "bash", "arguments": "{}"}}, ], }, {"role": "tool", "tool_call_id": "c1", "content": "done"}, {"role": "user", "content": "thanks"}, ] _, converted = self.provider._convert_messages(messages) # No synthetic results — only the real one (no is_error flag) tool_results = [] for msg in converted: if msg["role"] == "user" and isinstance(msg["content"], list): for block in msg["content"]: if isinstance(block, dict) and block.get("type") == "tool_result": tool_results.append(block) assert len(tool_results) == 1 assert tool_results[0]["tool_use_id"] == "c1" assert tool_results[0].get("is_error") is not True def test_trailing_orphan(self) -> None: """Orphaned tool_use at end of conversation (no following messages).""" messages = [ {"role": "user", "content": "do it"}, { "role": "assistant", "content": "Running...", "tool_calls": [ {"id": "c1", "function": {"name": "bash", "arguments": "{}"}}, ], }, ] _, converted = self.provider._convert_messages(repair_wire_messages(messages)) tool_results = [] for msg in converted: if msg["role"] == "user" and isinstance(msg["content"], list): for block in msg["content"]: if isinstance(block, dict) and block.get("type") == "tool_result": tool_results.append(block) assert len(tool_results) == 1 assert tool_results[0]["tool_use_id"] == "c1" assert tool_results[0]["is_error"] is True def test_provider_content_orphan(self) -> None: """Orphaned tool_use inside _provider_content (Anthropic raw blocks).""" messages = [ {"role": "user", "content": "run something"}, { "role": "assistant", "content": "Running...", "_provider_content": [ {"type": "text", "text": "Running..."}, { "type": "tool_use", "id": "toolu_abc", "name": "bash", "input": {"command": "sleep 30"}, }, ], "tool_calls": [ { "id": "toolu_abc", "function": {"name": "bash", "arguments": '{"command": "sleep 30"}'}, }, ], }, {"role": "user", "content": "never mind"}, ] _, converted = self.provider._convert_messages(repair_wire_messages(messages)) # Should synthesize a tool_result for the orphaned tool_use in provider_content tool_results = [] for msg in converted: if msg["role"] == "user" and isinstance(msg["content"], list): for block in msg["content"]: if isinstance(block, dict) and block.get("type") == "tool_result": tool_results.append(block) assert len(tool_results) == 1 assert tool_results[0]["tool_use_id"] == "toolu_abc" assert tool_results[0]["is_error"] is True class TestAnthropicReasoningNone: """Verify 'none' effort disables thinking for manual-thinking models.""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_none_effort_disables_thinking(self) -> None: result = self.provider._reasoning_params("none", None, max_tokens=4096) assert result == {} def test_empty_effort_disables_thinking(self) -> None: result = self.provider._reasoning_params("", None, max_tokens=4096) assert result == {} def test_low_effort_enables_thinking(self) -> None: result = self.provider._reasoning_params("low", None, max_tokens=4096) assert "thinking" in result assert result["thinking"]["budget_tokens"] == 1024 def test_map_xhigh_effort(self) -> None: from turnstone.core.providers._anthropic import _map_reasoning_to_effort result = _map_reasoning_to_effort("xhigh", ("low", "medium", "high", "xhigh", "max")) assert result == "xhigh" def test_map_xhigh_snaps_up_through_gap_to_max(self) -> None: """Levels with a hole (no xhigh) round the knob UP to the next declared level rather than dropping output_config entirely.""" from turnstone.core.providers._anthropic import _map_reasoning_to_effort result = _map_reasoning_to_effort("xhigh", ("low", "medium", "high", "max")) assert result == "max" def test_map_above_ceiling_rides_ceiling(self) -> None: from turnstone.core.providers._anthropic import _map_reasoning_to_effort assert _map_reasoning_to_effort("max", ("low", "medium", "high")) == "high" assert _map_reasoning_to_effort("minimal", ("low", "medium", "high")) == "low" assert _map_reasoning_to_effort("none", ("low", "medium", "high")) is None # =========================================================================== # TestWebSearch — provider-native web search # =========================================================================== class TestAnthropicWebSearch: """Tests for Anthropic native web search tool injection and streaming.""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_web_search_capability_flag(self) -> None: """All Anthropic models should support native web search.""" caps = self.provider.get_capabilities("claude-opus-4-6") assert caps.supports_web_search is True caps = self.provider.get_capabilities("claude-sonnet-4-6") assert caps.supports_web_search is True # Unknown models use default which also has web search caps = self.provider.get_capabilities("claude-unknown-99") assert caps.supports_web_search is True def test_inject_web_search_replaces_function_tool(self) -> None: """web_search function tool should be replaced with native server-side tool.""" caps = self.provider.get_capabilities("claude-opus-4-6") tools = [ {"name": "bash", "description": "Run bash", "input_schema": {"type": "object"}}, {"name": "web_search", "description": "Search web", "input_schema": {"type": "object"}}, ] result = self.provider._inject_web_search(tools, caps) names = [t.get("name") for t in result] assert "bash" in names assert "web_search" in names # The web_search entry should be the native tool, not the function tool ws_tool = next(t for t in result if t.get("name") == "web_search") from turnstone.core.providers._anthropic import _WEB_SEARCH_TOOL_TYPE assert ws_tool["type"] == _WEB_SEARCH_TOOL_TYPE assert "input_schema" not in ws_tool def test_inject_web_search_no_op_without_tool(self) -> None: """If no web_search tool in list, no injection happens.""" caps = self.provider.get_capabilities("claude-opus-4-6") tools = [ {"name": "bash", "description": "Run bash", "input_schema": {"type": "object"}}, ] result = self.provider._inject_web_search(tools, caps) assert result is tools # Unchanged def test_streaming_server_tool_use_emits_search_info(self) -> None: """server_tool_use block should emit info_delta with search query.""" events = [ _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_123", block_name="web_search", index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"query": "python web frameworks"}', index=0, ), _anthropic_event("content_block_stop", index=0), ] chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 1 assert "python web frameworks" in info_chunks[0].info_delta assert "Searching" in info_chunks[0].info_delta def test_streaming_web_search_result_emits_count(self) -> None: """web_search_tool_result block should emit result count info.""" # Build mock search results result1 = MagicMock() result1.type = "web_search_result" result2 = MagicMock() result2.type = "web_search_result" events = [ _anthropic_event( "content_block_start", block_type="web_search_tool_result", index=1, ), ] # Set up the content attribute with search results events[0].content_block.content = [result1, result2] chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 1 assert "Found 2 results" in info_chunks[0].info_delta def test_streaming_web_search_error_emits_info(self) -> None: """web_search_tool_result with error should emit error info.""" error_content = MagicMock() error_content.type = "web_search_tool_result_error" error_content.error_code = "too_many_requests" events = [ _anthropic_event( "content_block_start", block_type="web_search_tool_result", index=1, ), ] events[0].content_block.content = error_content chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 1 assert "too_many_requests" in info_chunks[0].info_delta def test_streaming_server_tool_use_not_emitted_as_tool_call(self) -> None: """server_tool_use should NOT produce tool_call_deltas (it's server-side).""" events = [ _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_123", block_name="web_search", index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"query": "test"}', index=0, ), ] chunks = list(self.provider._iter_anthropic_stream(events)) tool_chunks = [c for c in chunks if c.tool_call_deltas] assert len(tool_chunks) == 0 def test_streaming_mixed_text_and_search(self) -> None: """Full sequence: text + server search + results + more text.""" events = [ # Initial text _anthropic_event( "content_block_start", block_type="text", index=0, ), _anthropic_event( "content_block_delta", delta_type="text_delta", text="Let me search.", index=0, ), # Server tool use _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_1", block_name="web_search", index=1, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"query": "test query"}', index=1, ), _anthropic_event("content_block_stop", index=1), # Response text _anthropic_event( "content_block_start", block_type="text", index=3, ), _anthropic_event( "content_block_delta", delta_type="text_delta", text="Based on the results...", index=3, ), # Finish _anthropic_event("message_delta", stop_reason="end_turn"), ] chunks = list(self.provider._iter_anthropic_stream(events)) text_chunks = [c for c in chunks if c.content_delta] info_chunks = [c for c in chunks if c.info_delta] assert len(text_chunks) == 2 assert text_chunks[0].content_delta == "Let me search." assert text_chunks[1].content_delta == "Based on the results..." assert len(info_chunks) == 1 assert "test query" in info_chunks[0].info_delta def test_pause_turn_normalized_to_stop(self) -> None: """pause_turn stop reason should normalize to 'stop'.""" from turnstone.core.providers._anthropic import _normalize_finish_reason assert _normalize_finish_reason("pause_turn") == "stop" def test_completion_skips_server_blocks(self) -> None: """create_completion should skip server_tool_use and web_search_tool_result.""" # Build mock response with mixed block types text_block = MagicMock() text_block.type = "text" text_block.text = "Here are the results." server_tu_block = MagicMock() server_tu_block.type = "server_tool_use" search_result_block = MagicMock() search_result_block.type = "web_search_tool_result" response = MagicMock() response.content = [server_tu_block, search_result_block, text_block] response.stop_reason = "end_turn" response.usage.input_tokens = 100 response.usage.output_tokens = 50 response.usage.cache_creation_input_tokens = 0 response.usage.cache_read_input_tokens = 0 client = MagicMock() stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=stream_ctx) stream_ctx.__exit__ = MagicMock(return_value=False) stream_ctx.get_final_message.return_value = response client.messages.stream.return_value = stream_ctx with patch("turnstone.core.providers._anthropic._ensure_anthropic"): result = self.provider.create_completion( client=client, model="claude-opus-4-6", messages=[{"role": "user", "content": "search test"}], ) assert result.content == "Here are the results." assert result.tool_calls is None def test_streaming_multiple_searches(self) -> None: """Multiple server_tool_use blocks in one response should each emit info.""" events = [ _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_1", block_name="web_search", index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"query": "first search"}', index=0, ), _anthropic_event("content_block_stop", index=0), _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_2", block_name="web_search", index=2, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"query": "second search"}', index=2, ), _anthropic_event("content_block_stop", index=2), ] chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 2 assert "first search" in info_chunks[0].info_delta assert "second search" in info_chunks[1].info_delta def test_streaming_interleaved_tool_use_and_server_tool_use(self) -> None: """Regular tool_use and server_tool_use at different indices.""" events = [ # Regular tool call at index 0 _anthropic_event( "content_block_start", block_type="tool_use", block_id="toolu_1", block_name="bash", index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"command": "ls"}', index=0, ), # Server tool at index 1 _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_1", block_name="web_search", index=1, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json='{"query": "test"}', index=1, ), _anthropic_event("content_block_stop", index=1), ] chunks = list(self.provider._iter_anthropic_stream(events)) tool_chunks = [c for c in chunks if c.tool_call_deltas] info_chunks = [c for c in chunks if c.info_delta] # Regular tool_use should produce tool_call_deltas assert len(tool_chunks) == 2 # start + delta assert tool_chunks[0].tool_call_deltas[0].name == "bash" # Server tool_use should produce info_delta only assert len(info_chunks) == 1 assert "test" in info_chunks[0].info_delta def test_streaming_malformed_server_tool_json(self) -> None: """Malformed JSON in server tool input should emit fallback info.""" events = [ _anthropic_event( "content_block_start", block_type="server_tool_use", block_id="srvtoolu_1", block_name="web_search", index=0, ), _anthropic_event( "content_block_delta", delta_type="input_json_delta", partial_json="{bad json", index=0, ), _anthropic_event("content_block_stop", index=0), ] chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 1 assert info_chunks[0].info_delta == "[Searching...]" def test_web_search_result_empty_list(self) -> None: """Empty search results list should report 0 results.""" events = [ _anthropic_event( "content_block_start", block_type="web_search_tool_result", index=0, ), ] events[0].content_block.content = [] chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 1 assert "Found 0 results" in info_chunks[0].info_delta def test_content_block_stop_for_text_block_no_spurious_info(self) -> None: """content_block_stop for a text block should not emit info_delta.""" events = [ _anthropic_event("content_block_start", block_type="text", index=0), _anthropic_event( "content_block_delta", delta_type="text_delta", text="hello", index=0, ), _anthropic_event("content_block_stop", index=0), ] chunks = list(self.provider._iter_anthropic_stream(events)) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 0 class TestOpenAIWebSearch: """Tests for OpenAI native web search with search models.""" def setup_method(self) -> None: self.provider = OpenAIProvider() def test_search_model_capability(self) -> None: """Search models should have supports_web_search=True.""" caps = lookup_openai_capabilities("gpt-5-search-api") assert caps.supports_web_search is True def test_non_search_model_no_web_search(self) -> None: """Regular models should not have supports_web_search.""" caps = lookup_openai_capabilities("gpt-5") assert caps.supports_web_search is False caps = lookup_openai_capabilities("gpt-5.2") assert caps.supports_web_search is False def test_apply_web_search_injects_options(self) -> None: """For search models, web_search_options should be added to kwargs.""" caps = lookup_openai_capabilities("gpt-5-search-api") kwargs: dict[str, Any] = {"model": "gpt-5-search-api"} tools: list[dict[str, Any]] = [ {"type": "function", "function": {"name": "bash", "description": "Run bash"}}, {"type": "function", "function": {"name": "web_search", "description": "Search"}}, ] result = self.provider._apply_web_search(kwargs, caps, tools) # web_search_options should be in kwargs assert "web_search_options" in kwargs # web_search tool should be removed assert result is not None names = [t["function"]["name"] for t in result] assert "web_search" not in names assert "bash" in names def test_apply_web_search_no_op_for_regular_models(self) -> None: """For non-search models, no web_search_options, tools unchanged.""" caps = lookup_openai_capabilities("gpt-5") kwargs: dict[str, Any] = {"model": "gpt-5"} tools: list[dict[str, Any]] = [ {"type": "function", "function": {"name": "web_search", "description": "Search"}}, ] result = self.provider._apply_web_search(kwargs, caps, tools) assert "web_search_options" not in kwargs assert result is tools # Unchanged def test_apply_web_search_returns_none_when_only_web_search(self) -> None: """If web_search was the only tool, return None after removing it.""" caps = lookup_openai_capabilities("gpt-5-search-api") kwargs: dict[str, Any] = {} tools: list[dict[str, Any]] = [ {"type": "function", "function": {"name": "web_search", "description": "Search"}}, ] result = self.provider._apply_web_search(kwargs, caps, tools) assert result is None def test_apply_web_search_no_op_when_client_def_absent(self) -> None: """Replace-only: a search model with a NON-EMPTY toolset that never advertised web_search (a persona visibility set or coordinator toolset) must NOT gain native search — the option stays off and the tools pass through untouched. Contrast test_apply_web_search_with_ no_tools, which covers the tool-less utility-call case.""" caps = lookup_openai_capabilities("gpt-5-search-api") assert caps.supports_web_search is True kwargs: dict[str, Any] = {"model": "gpt-5-search-api"} tools: list[dict[str, Any]] = [ {"type": "function", "function": {"name": "bash", "description": "Run bash"}}, {"type": "function", "function": {"name": "read_file", "description": "Read"}}, ] result = self.provider._apply_web_search(kwargs, caps, tools) assert "web_search_options" not in kwargs assert result is tools # unchanged, not filtered or replaced def test_format_citations_appends_sources(self) -> None: """url_citation annotations should be formatted as footnote sources.""" ann = MagicMock() ann.type = "url_citation" citation = MagicMock() citation.title = "Example Page" citation.url = "https://example.com" ann.url_citation = citation content = "Some search result text." result = format_citations(content, [ann]) assert "Sources:" in result assert "[Example Page](https://example.com)" in result def test_format_citations_deduplicates(self) -> None: """Duplicate URLs should not appear twice in sources.""" ann1 = MagicMock() ann1.type = "url_citation" ann1.url_citation = MagicMock(title="Page", url="https://example.com") ann2 = MagicMock() ann2.type = "url_citation" ann2.url_citation = MagicMock(title="Page Again", url="https://example.com") content = "Text." result = format_citations(content, [ann1, ann2]) assert result.count("example.com") == 1 def test_format_citations_skips_non_url_citation(self) -> None: """Non-url_citation annotations should be ignored.""" ann = MagicMock() ann.type = "something_else" content = "Text." result = format_citations(content, [ann]) assert "Sources:" not in result def test_format_citations_empty_title(self) -> None: """Citation with empty title should show plain URL.""" ann = MagicMock() ann.type = "url_citation" ann.url_citation = MagicMock(title="", url="https://example.com") result = format_citations("Text.", [ann]) assert "https://example.com" in result # Should not have markdown link format when title is empty assert "[](https://example.com)" not in result def test_format_citations_none_citation(self) -> None: """Citation with None url_citation should be skipped.""" ann = MagicMock() ann.type = "url_citation" ann.url_citation = None result = format_citations("Text.", [ann]) assert "Sources:" not in result def test_apply_web_search_with_no_tools(self) -> None: """No client web_search def ⇒ no injection (replace-only semantics). A request that never advertised the web_search tool — persona visibility set, coordinator toolset, or a tool-less utility call — must not gain native search at the provider layer. """ caps = lookup_openai_capabilities("gpt-5-search-api") kwargs: dict[str, Any] = {} result = self.provider._apply_web_search(kwargs, caps, None) assert "web_search_options" not in kwargs assert result is None def test_apply_web_search_replaces_client_def(self) -> None: """With the client def present, it is filtered and the option set.""" caps = lookup_openai_capabilities("gpt-5-search-api") kwargs: dict[str, Any] = {} tools = [{"type": "function", "function": {"name": "web_search"}}] result = self.provider._apply_web_search(kwargs, caps, tools) assert "web_search_options" in kwargs assert result is None # the lone def was filtered away def test_streaming_creates_with_web_search_options(self) -> None: """Streaming with a search model should pass web_search_options.""" client = MagicMock() client.chat.completions.create.return_value = iter( [ _openai_stream_chunk(content="Result text"), ] ) list( self.provider.create_streaming( client=client, model="gpt-5-search-api", messages=[{"role": "user", "content": "search something"}], tools=[ { "type": "function", "function": {"name": "web_search", "description": "Search"}, }, ], # The local lane resolves no commercial rows — the search # model's capabilities ride in explicitly, as the session # layer would pass them. capabilities=lookup_openai_capabilities("gpt-5-search-api"), ) ) call_kwargs = client.chat.completions.create.call_args[1] assert "web_search_options" in call_kwargs # web_search tool should not be in the tools assert "tools" not in call_kwargs or not any( t.get("function", {}).get("name") == "web_search" for t in call_kwargs.get("tools", []) ) def test_completion_with_annotations(self) -> None: """Non-streaming completion with search model should format citations.""" ann = MagicMock() ann.type = "url_citation" ann.url_citation = MagicMock(title="Test", url="https://test.com") msg = MagicMock() msg.content = "Found information." msg.annotations = [ann] msg.tool_calls = None choice = MagicMock() choice.message = msg choice.finish_reason = "stop" response = MagicMock() response.choices = [choice] response.usage.prompt_tokens = 50 response.usage.completion_tokens = 20 response.usage.total_tokens = 70 client = MagicMock() client.chat.completions.create.return_value = response result = self.provider.create_completion( client=client, model="gpt-5-search-api", messages=[{"role": "user", "content": "search test"}], ) assert "Found information." in result.content assert "Sources:" in result.content assert "[Test](https://test.com)" in result.content def test_streaming_emits_citations_as_info_delta(self) -> None: """Streaming with search model should emit citations as final info_delta.""" ann = MagicMock() ann.type = "url_citation" ann.url_citation = MagicMock(title="Result", url="https://example.com") # Content chunk, then a chunk with annotation, then finish content_chunk = _openai_stream_chunk(content="Search result text.") content_chunk.choices[0].delta.annotations = None ann_chunk = _openai_stream_chunk(content=None) ann_chunk.choices[0].delta.annotations = [ann] finish_chunk = _openai_stream_chunk(finish_reason="stop") finish_chunk.choices[0].delta.annotations = None client = MagicMock() client.chat.completions.create.return_value = iter([content_chunk, ann_chunk, finish_chunk]) chunks = list( self.provider.create_streaming( client=client, model="gpt-5-search-api", messages=[{"role": "user", "content": "search test"}], ) ) info_chunks = [c for c in chunks if c.info_delta] assert len(info_chunks) == 1 assert "Sources:" in info_chunks[0].info_delta assert "[Result](https://example.com)" in info_chunks[0].info_delta class TestClientSearchFallback: """Tests for the client-side web_search fallback when providers lack native search.""" def test_local_model_no_web_search(self) -> None: """Local/vLLM models should not have supports_web_search.""" provider = OpenAIProvider() caps = provider.get_capabilities("my-local-model") assert caps.supports_web_search is False def test_web_search_tool_preserved_for_local_models(self) -> None: """For local models, web_search function tool stays in the tools list.""" provider = OpenAIProvider() caps = provider.get_capabilities("llama-3-70b") kwargs: dict[str, Any] = {} tools = [ {"type": "function", "function": {"name": "web_search", "description": "Search"}}, ] result = provider._apply_web_search(kwargs, caps, tools) assert result is tools assert "web_search_options" not in kwargs # =========================================================================== # Anthropic provider_blocks / _provider_content round-trip tests # =========================================================================== class TestAnthropicProviderBlocks: """Tests for multi-turn web search content preservation.""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_convert_messages_uses_provider_content(self) -> None: """Assistant message with _provider_content passes through verbatim.""" provider_content = [ {"type": "text", "text": "Here is what I found."}, { "type": "server_tool_use", "id": "stu_123", "name": "web_search", "input": {"query": "turnstone bird"}, }, { "type": "web_search_tool_result", "tool_use_id": "stu_123", "content": [{"type": "web_search_result", "url": "https://example.com"}], "encrypted_content": "abc123encrypted", "encrypted_index": "idx456encrypted", }, ] messages = [ {"role": "user", "content": "Search for turnstone bird"}, { "role": "assistant", "content": "Here is what I found.", "_provider_content": provider_content, }, {"role": "user", "content": "Tell me more"}, ] _, converted = self.provider._convert_messages(messages) # The assistant message should use provider_content verbatim assistant_msg = converted[1] assert assistant_msg["role"] == "assistant" assert assistant_msg["content"] is provider_content assert assistant_msg["content"][2]["encrypted_content"] == "abc123encrypted" def test_convert_messages_without_provider_content_unchanged(self) -> None: """Assistant message without _provider_content uses normal reconstruction.""" messages = [ {"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there"}, ] _, converted = self.provider._convert_messages(messages) assistant_msg = converted[1] assert assistant_msg["role"] == "assistant" assert assistant_msg["content"] == [{"type": "text", "text": "Hi there"}] def test_block_to_dict_with_model_dump(self) -> None: """_block_to_dict uses model_dump(exclude_none=True) when available.""" from turnstone.core.providers._anthropic import _block_to_dict class FakeBlock: def model_dump(self, **kwargs: Any) -> dict[str, Any]: d = {"type": "text", "text": "hello", "extra": True, "nullable": None} if kwargs.get("exclude_none"): return {k: v for k, v in d.items() if v is not None} return d result = _block_to_dict(FakeBlock()) assert result == {"type": "text", "text": "hello", "extra": True} assert "nullable" not in result def test_block_to_dict_fallback(self) -> None: """_block_to_dict extracts known attributes as fallback.""" from turnstone.core.providers._anthropic import _block_to_dict class FakeBlock: type = "web_search_tool_result" content = [{"type": "web_search_result"}] encrypted_content = "enc123" encrypted_index = "idx456" result = _block_to_dict(FakeBlock()) assert result["type"] == "web_search_tool_result" assert result["encrypted_content"] == "enc123" assert result["encrypted_index"] == "idx456" def test_streaming_captures_provider_blocks(self) -> None: """Streaming events produce provider_blocks on the final chunk.""" from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() # Build mock stream events events = [] # Text block text_block = MagicMock() text_block.type = "text" text_block.text = "" text_block.model_dump.return_value = {"type": "text", "text": ""} events.append(MagicMock(type="content_block_start", index=0, content_block=text_block)) events.append( MagicMock( type="content_block_delta", index=0, delta=MagicMock(type="text_delta", text="Hello"), ) ) events.append(MagicMock(type="content_block_stop", index=0)) # Server tool use block stu_block = MagicMock() stu_block.type = "server_tool_use" stu_block.name = "web_search" stu_block.model_dump.return_value = { "type": "server_tool_use", "id": "stu_1", "name": "web_search", "input": {}, } events.append(MagicMock(type="content_block_start", index=1, content_block=stu_block)) events.append( MagicMock( type="content_block_delta", index=1, delta=MagicMock(type="input_json_delta", partial_json='{"query":"test"}'), ) ) events.append(MagicMock(type="content_block_stop", index=1)) # Web search tool result block wsr_block = MagicMock() wsr_block.type = "web_search_tool_result" wsr_block.model_dump.return_value = { "type": "web_search_tool_result", "tool_use_id": "stu_1", "content": [{"type": "web_search_result", "url": "https://example.com"}], "encrypted_content": "enc_data", "encrypted_index": "idx_data", } # Make content iterable for count fake_result = MagicMock() fake_result.type = "web_search_result" wsr_block.content = [fake_result] events.append(MagicMock(type="content_block_start", index=2, content_block=wsr_block)) events.append(MagicMock(type="content_block_stop", index=2)) # Message delta with stop msg_delta = MagicMock(type="message_delta") msg_delta.delta = MagicMock(stop_reason="end_turn") msg_delta.usage = MagicMock(input_tokens=100, output_tokens=50) events.append(msg_delta) chunks = list(provider._iter_anthropic_stream(iter(events))) # Find the final chunk with provider_blocks final_chunks = [c for c in chunks if c.provider_blocks] assert len(final_chunks) == 1 blocks = final_chunks[0].provider_blocks assert len(blocks) == 3 assert blocks[0]["type"] == "text" assert blocks[1]["type"] == "server_tool_use" assert blocks[1]["input"] == {"query": "test"} # parsed from accumulated JSON assert blocks[2]["type"] == "web_search_tool_result" assert blocks[2]["encrypted_content"] == "enc_data" def test_streaming_thinking_block_captures_signature(self) -> None: """Streaming thinking block accumulates signature from signature_delta events.""" thinking_block = MagicMock() thinking_block.type = "thinking" thinking_block.model_dump.return_value = { "type": "thinking", "thinking": "", "signature": "", } text_block = MagicMock() text_block.type = "text" text_block.model_dump.return_value = {"type": "text", "text": ""} events = [ MagicMock(type="content_block_start", index=0, content_block=thinking_block), _anthropic_event( "content_block_delta", delta_type="thinking_delta", thinking="step 1", index=0 ), _anthropic_event( "content_block_delta", delta_type="thinking_delta", thinking=" step 2", index=0 ), _anthropic_event( "content_block_delta", delta_type="signature_delta", signature="sig_part1", index=0, ), _anthropic_event( "content_block_delta", delta_type="signature_delta", signature="sig_part2", index=0, ), _anthropic_event("content_block_stop", index=0), MagicMock(type="content_block_start", index=1, content_block=text_block), _anthropic_event("content_block_delta", delta_type="text_delta", text="Hello", index=1), _anthropic_event("content_block_stop", index=1), _anthropic_event("message_delta", stop_reason="end_turn", usage_output_tokens=50), ] chunks = list(self.provider._iter_anthropic_stream(iter(events))) final_chunks = [c for c in chunks if c.provider_blocks] assert len(final_chunks) == 1 blocks = final_chunks[0].provider_blocks assert blocks[0]["type"] == "thinking" assert blocks[0]["thinking"] == "step 1 step 2" assert blocks[0]["signature"] == "sig_part1sig_part2" def test_thinking_block_multiturn_roundtrip(self) -> None: """Thinking block with signature survives _convert_messages round-trip.""" provider_content = [ { "type": "thinking", "thinking": "Let me reason...", "signature": "ErUBCkYIAxgCIkD_valid_sig", }, {"type": "text", "text": "Here is my answer."}, ] messages = [ {"role": "user", "content": "Question"}, { "role": "assistant", "content": "Here is my answer.", "_provider_content": provider_content, }, {"role": "user", "content": "Follow up"}, ] _, converted = self.provider._convert_messages(messages) assistant_msg = converted[1] assert assistant_msg["content"] is provider_content assert assistant_msg["content"][0]["signature"] == "ErUBCkYIAxgCIkD_valid_sig" assert assistant_msg["content"][0]["type"] == "thinking" def test_block_to_dict_preserves_thinking_signature(self) -> None: """_block_to_dict preserves signature on thinking blocks.""" from turnstone.core.providers._anthropic import _block_to_dict class FakeThinkingBlock: def model_dump(self, **kwargs: Any) -> dict[str, Any]: return { "type": "thinking", "thinking": "reasoning...", "signature": "abc123sig", } result = _block_to_dict(FakeThinkingBlock()) assert result["signature"] == "abc123sig" # Also test fallback path (no model_dump) class FallbackBlock: type = "thinking" thinking = "reasoning..." signature = "abc123sig" result2 = _block_to_dict(FallbackBlock()) assert result2["signature"] == "abc123sig" # --------------------------------------------------------------------------- # Tool search tests # --------------------------------------------------------------------------- class TestAnthropicToolSearch: """Test Anthropic provider tool search injection.""" @pytest.fixture() def provider(self): from turnstone.core.providers._anthropic import AnthropicProvider return AnthropicProvider() def test_tool_search_capability_flag(self, provider): caps = provider.get_capabilities("claude-opus-4-6-20260101") assert caps.supports_tool_search is True def test_tool_search_not_supported_on_haiku(self, provider): caps = provider.get_capabilities("claude-haiku-4-5-20251001") assert caps.supports_tool_search is False def test_inject_tool_search_marks_deferred(self, provider): caps = provider.get_capabilities("claude-opus-4-6-20260101") tools = [ {"name": "bash", "description": "Run commands", "input_schema": {}}, { "name": "mcp__github__create_issue", "description": "Create issue", "input_schema": {}, }, ] deferred = frozenset(["mcp__github__create_issue"]) result = provider._inject_tool_search(tools, caps, deferred) # bash should not be deferred assert result[0].get("defer_loading") is None or result[0].get("defer_loading") is False # MCP tool should be deferred assert result[1]["defer_loading"] is True # Search tool should be appended assert result[-1]["type"] == "tool_search_tool_bm25" assert result[-1]["name"] == "tool_search_tool_bm25" def test_inject_tool_search_no_op_without_deferred(self, provider): caps = provider.get_capabilities("claude-opus-4-6-20260101") tools = [{"name": "bash", "description": "Run commands", "input_schema": {}}] result = provider._inject_tool_search(tools, caps, None) assert result == tools def test_inject_tool_search_no_op_on_unsupported_model(self, provider): caps = provider.get_capabilities("claude-haiku-4-5-20251001") tools = [{"name": "bash", "description": "Run commands", "input_schema": {}}] deferred = frozenset(["some_tool"]) result = provider._inject_tool_search(tools, caps, deferred) assert result == tools class TestOpenAIToolSearch: """Test OpenAI tool search injection (registry rows + shared helper).""" def test_tool_search_capability_on_gpt54(self): caps = lookup_openai_capabilities("gpt-5.4") assert caps.supports_tool_search is True def test_tool_search_not_supported_on_gpt5(self): caps = lookup_openai_capabilities("gpt-5") assert caps.supports_tool_search is False def test_apply_tool_search_marks_deferred(self): caps = lookup_openai_capabilities("gpt-5.4") tools = [ {"type": "function", "function": {"name": "bash", "description": "Run commands"}}, { "type": "function", "function": {"name": "mcp__slack__send", "description": "Send message"}, }, ] deferred = frozenset(["mcp__slack__send"]) result = apply_tool_search(caps, tools, deferred) assert result is not None # bash not deferred assert result[0].get("defer_loading") is None or result[0].get("defer_loading") is False # slack tool deferred assert result[1]["defer_loading"] is True def test_apply_tool_search_no_op_without_deferred(self): caps = lookup_openai_capabilities("gpt-5.4") tools = [ {"type": "function", "function": {"name": "bash", "description": "Run commands"}}, ] result = apply_tool_search(caps, tools, None) assert result == tools def test_apply_tool_search_no_op_on_unsupported_model(self): caps = lookup_openai_capabilities("gpt-5") tools = [ {"type": "function", "function": {"name": "bash", "description": "Run commands"}}, ] deferred = frozenset(["some_tool"]) result = apply_tool_search(caps, tools, deferred) assert result == tools class TestModelCapabilitiesToolSearch: """Test supports_tool_search defaults and values.""" def test_default_is_false(self): from turnstone.core.providers._protocol import ModelCapabilities caps = ModelCapabilities() assert caps.supports_tool_search is False class TestMidConversationSystemCapability: """supports_mid_conversation_system — NextOpus (claude-opus-4-8) only.""" def test_default_is_false(self) -> None: from turnstone.core.providers._protocol import ModelCapabilities caps = ModelCapabilities() assert caps.supports_mid_conversation_system is False def test_opus_4_8_supports_it(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() for model in ("claude-opus-4-8", "claude-opus-4-8-20260601"): caps = provider.get_capabilities(model) assert caps.supports_mid_conversation_system is True, model def test_other_claude_models_do_not(self) -> None: """Only NextOpus has it; older/other Claude models and the default off.""" from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() for model in ( "claude-opus-4-7", "claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5", "claude-opus-4-5", "claude-unknown-9", # Anthropic default ): caps = provider.get_capabilities(model) assert caps.supports_mid_conversation_system is False, model # --------------------------------------------------------------------------- # Vision support # --------------------------------------------------------------------------- class TestVisionCapabilities: """Test supports_vision flag across providers.""" def test_default_is_false(self) -> None: from turnstone.core.providers._protocol import ModelCapabilities caps = ModelCapabilities() assert caps.supports_vision is False def test_openai_commercial_supports_vision(self) -> None: for model in ("gpt-5", "gpt-5-mini", "gpt-5.4", "o3", "o4-mini"): caps = lookup_openai_capabilities(model) assert caps.supports_vision is True, f"{model} should support vision" def test_openai_default_no_vision(self) -> None: """Local-lane models (any name) default to no vision.""" provider = OpenAIProvider() caps = provider.get_capabilities("some-local-model") assert caps.supports_vision is False def test_anthropic_supports_vision(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() for model in ("claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5"): caps = provider.get_capabilities(model) assert caps.supports_vision is True, f"{model} should support vision" def test_anthropic_default_supports_vision(self) -> None: """Anthropic default (unknown Claude model) supports vision.""" from turnstone.core.providers._anthropic import AnthropicProvider provider = AnthropicProvider() caps = provider.get_capabilities("claude-unknown-9") assert caps.supports_vision is True class TestAnthropicVisionConversion: """Test image content conversion in _convert_messages.""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_tool_result_with_image_content(self) -> None: """Tool result with list content converts image_url to Anthropic image.""" messages = [ {"role": "user", "content": "Read this image"}, { "role": "assistant", "content": "", "tool_calls": [ { "id": "call_1", "function": {"name": "read_file", "arguments": '{"path": "img.png"}'}, } ], }, { "role": "tool", "tool_call_id": "call_1", "content": [ {"type": "text", "text": "Image file: img.png (1024 bytes)"}, { "type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, }, ], }, ] _, converted = self.provider._convert_messages(messages) # Tool result should be in a user message tool_user_msg = converted[2] assert tool_user_msg["role"] == "user" tool_result = tool_user_msg["content"][0] assert tool_result["type"] == "tool_result" assert tool_result["tool_use_id"] == "call_1" # Content should be a list with converted image block content = tool_result["content"] assert isinstance(content, list) assert content[0] == {"type": "text", "text": "Image file: img.png (1024 bytes)"} assert content[1]["type"] == "image" assert content[1]["source"]["type"] == "base64" assert content[1]["source"]["media_type"] == "image/png" assert content[1]["source"]["data"] == "iVBORw0KGgo=" def test_tool_result_with_string_content_unchanged(self) -> None: """Tool result with plain string content is unchanged.""" messages = [ {"role": "user", "content": "Read file"}, { "role": "assistant", "content": "", "tool_calls": [ { "id": "call_2", "function": {"name": "read_file", "arguments": '{"path": "f.py"}'}, } ], }, { "role": "tool", "tool_call_id": "call_2", "content": " 1\tprint('hello')", }, ] _, converted = self.provider._convert_messages(messages) tool_result = converted[2]["content"][0] assert tool_result["content"] == " 1\tprint('hello')" def test_convert_content_parts_static_method(self) -> None: """_convert_content_parts handles both image_url and text.""" from turnstone.core.providers._anthropic import AnthropicProvider parts = [ {"type": "text", "text": "description"}, { "type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}, }, ] result = AnthropicProvider._convert_content_parts(parts) assert result[0] == {"type": "text", "text": "description"} assert result[1]["type"] == "image" assert result[1]["source"]["media_type"] == "image/jpeg" assert result[1]["source"]["data"] == "/9j/4AAQ" # =========================================================================== # TestPromptCaching # =========================================================================== class TestAnthropicPromptCaching: """Tests for Anthropic prompt caching (cache_control).""" def setup_method(self) -> None: from turnstone.core.providers._anthropic import AnthropicProvider self.provider = AnthropicProvider() def test_cache_control_set_in_kwargs(self) -> None: """_build_thinking_and_kwargs includes cache_control: ephemeral.""" caps = self.provider.get_capabilities("claude-sonnet-4-6") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="medium", extra_params=None, max_tokens=4096, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="You are helpful.", model="claude-sonnet-4-6", tools=None, ) assert "cache_control" in kwargs assert kwargs["cache_control"] == {"type": "ephemeral"} def test_opus_4_7_no_temperature_in_kwargs(self) -> None: """Opus 4.7 rejects temperature — must not appear in kwargs.""" caps = self.provider.get_capabilities("claude-opus-4-7") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="high", extra_params=None, max_tokens=8192, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="", model="claude-opus-4-7", tools=None, ) assert "temperature" not in kwargs def test_opus_4_6_still_has_temperature(self) -> None: """Opus 4.6 must still send temperature (regression guard).""" caps = self.provider.get_capabilities("claude-opus-4-6") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="high", extra_params=None, max_tokens=8192, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="", model="claude-opus-4-6", tools=None, ) assert "temperature" in kwargs assert kwargs["temperature"] == 1.0 # forced for adaptive thinking def test_opus_4_7_thinking_display_summarized(self) -> None: """Opus 4.7 must opt in to thinking display with 'summarized'.""" caps = self.provider.get_capabilities("claude-opus-4-7") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="high", extra_params=None, max_tokens=8192, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="", model="claude-opus-4-7", tools=None, ) assert kwargs["thinking"] == {"type": "adaptive", "display": "summarized"} def test_opus_4_6_thinking_no_display(self) -> None: """Opus 4.6 adaptive thinking should not include display key.""" caps = self.provider.get_capabilities("claude-opus-4-6") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="high", extra_params=None, max_tokens=8192, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="", model="claude-opus-4-6", tools=None, ) assert kwargs["thinking"] == {"type": "adaptive"} def test_opus_4_7_xhigh_effort(self) -> None: """Opus 4.7 xhigh effort passes through to output_config.""" caps = self.provider.get_capabilities("claude-opus-4-7") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="xhigh", extra_params=None, max_tokens=8192, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="", model="claude-opus-4-7", tools=None, ) assert kwargs["output_config"] == {"effort": "xhigh"} def test_xhigh_effort_snaps_to_max_on_opus_4_6(self) -> None: """Opus 4.6 declares (low, medium, high, max) — a knob of xhigh rounds up to max instead of silently dropping output_config.""" caps = self.provider.get_capabilities("claude-opus-4-6") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, reasoning_effort="xhigh", extra_params=None, max_tokens=8192, temperature=0.5, converted_msgs=[{"role": "user", "content": "hi"}], system_prompt="", model="claude-opus-4-6", tools=None, ) assert kwargs["output_config"] == {"effort": "max"} @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_message_start_cache_metrics(self, mock_ensure: MagicMock) -> None: """Cache metrics from message_start flow into UsageInfo.""" msg_start = MagicMock() msg_start.type = "message_start" msg_usage = MagicMock() msg_usage.input_tokens = 100 msg_usage.cache_creation_input_tokens = 80 msg_usage.cache_read_input_tokens = 0 msg_start.message = MagicMock() msg_start.message.usage = msg_usage text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi") events = [msg_start, text_event] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-6", messages=[{"role": "user", "content": "hi"}], ) ) # prompt_tokens = input_tokens (100) + cache_creation (80) + cache_read (0) = 180 start_chunks = [r for r in results if r.usage is not None and r.usage.prompt_tokens == 180] assert len(start_chunks) == 1 assert start_chunks[0].usage is not None assert start_chunks[0].usage.cache_creation_tokens == 80 assert start_chunks[0].usage.cache_read_tokens == 0 @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_message_delta_cache_metrics(self, mock_ensure: MagicMock) -> None: """Cache metrics from message_delta flow into UsageInfo.""" text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi") delta_event = MagicMock() delta_event.type = "message_delta" delta_usage = MagicMock() delta_usage.input_tokens = 0 delta_usage.output_tokens = 50 delta_usage.cache_creation_input_tokens = 0 delta_usage.cache_read_input_tokens = 120 delta_event.usage = delta_usage delta_event.delta = MagicMock() delta_event.delta.stop_reason = "end_turn" events = [text_event, delta_event] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-6", messages=[{"role": "user", "content": "hi"}], ) ) delta_chunks = [r for r in results if r.finish_reason is not None] assert len(delta_chunks) == 1 u = delta_chunks[0].usage assert u is not None assert u.cache_read_tokens == 120 assert u.cache_creation_tokens == 0 @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_completion_cache_metrics(self, mock_ensure: MagicMock) -> None: """Non-streaming completion extracts cache metrics.""" response = MagicMock() text_block = MagicMock() text_block.type = "text" text_block.text = "Hello" response.content = [text_block] response.stop_reason = "end_turn" usage = MagicMock() usage.input_tokens = 200 usage.output_tokens = 30 usage.cache_creation_input_tokens = 150 usage.cache_read_input_tokens = 50 response.usage = usage client = MagicMock() stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=stream_ctx) stream_ctx.__exit__ = MagicMock(return_value=False) stream_ctx.get_final_message.return_value = response client.messages.stream.return_value = stream_ctx result = self.provider.create_completion( client=client, model="claude-sonnet-4-6", messages=[{"role": "user", "content": "hi"}], ) u = result.usage assert u is not None assert u.cache_creation_tokens == 150 assert u.cache_read_tokens == 50 @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_cache_metrics_missing_gracefully(self, mock_ensure: MagicMock) -> None: """When cache attributes are absent, tokens default to 0.""" import types msg_start = MagicMock() msg_start.type = "message_start" # SimpleNamespace with only input_tokens — no cache attributes at all msg_usage = types.SimpleNamespace(input_tokens=50) msg_start.message = MagicMock() msg_start.message.usage = msg_usage text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi") events = [msg_start, text_event] stream_ctx = MagicMock() stream_ctx.__enter__ = MagicMock(return_value=iter(events)) stream_ctx.__exit__ = MagicMock(return_value=False) client = MagicMock() client.messages.stream.return_value = stream_ctx results = list( self.provider.create_streaming( client=client, model="claude-sonnet-4-6", messages=[{"role": "user", "content": "hi"}], ) ) start_chunks = [r for r in results if r.usage is not None] assert len(start_chunks) >= 1 u = start_chunks[0].usage assert u is not None assert u.cache_creation_tokens == 0 assert u.cache_read_tokens == 0 class TestOpenAIPromptCaching: """Tests for OpenAI prompt caching (automatic + extended retention).""" def setup_method(self) -> None: self.provider = OpenAIProvider() def test_cache_retention_set_for_gpt5(self) -> None: """GPT-5.x models get prompt_cache_retention=24h.""" for model in ( "gpt-5", "gpt-5.1", "gpt-5.2", "gpt-5.4", "gpt-5.4-pro", "gpt-5.5", "gpt-5.5-pro", "gpt-5-mini", "gpt-5-pro", ): kwargs: dict[str, Any] = {} apply_cache_retention(kwargs, model) assert kwargs.get("prompt_cache_retention") == "24h", f"Failed for {model}" def test_cache_retention_not_set_for_non_gpt5(self) -> None: """Non-GPT-5 models do not get cache retention.""" for model in ("o3", "o4-mini", "local-model", "gpt-4o"): kwargs: dict[str, Any] = {} apply_cache_retention(kwargs, model) assert "prompt_cache_retention" not in kwargs, f"Unexpected retention for {model}" def test_streaming_cached_tokens_from_usage(self) -> None: """Streaming usage extracts cached_tokens from prompt_tokens_details.""" usage = MagicMock() usage.prompt_tokens = 100 usage.completion_tokens = 20 usage.total_tokens = 120 ptd = MagicMock() ptd.cached_tokens = 80 usage.prompt_tokens_details = ptd chunks = [ _openai_stream_chunk(content="Hi"), _openai_stream_chunk(empty_choices=True, usage=usage), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-5.1", messages=[{"role": "user", "content": "hi"}], ) ) usage_chunks = [r for r in results if r.usage is not None] assert len(usage_chunks) == 1 u = usage_chunks[0].usage assert u is not None assert u.cache_read_tokens == 80 assert u.cache_creation_tokens == 0 def test_completion_cached_tokens(self) -> None: """Non-streaming completion extracts cached_tokens.""" response = MagicMock() msg = MagicMock() msg.content = "Hello" msg.tool_calls = None msg.annotations = None choice = MagicMock() choice.message = msg choice.finish_reason = "stop" response.choices = [choice] usage = MagicMock() usage.prompt_tokens = 200 usage.completion_tokens = 30 usage.total_tokens = 230 ptd = MagicMock() ptd.cached_tokens = 150 usage.prompt_tokens_details = ptd response.usage = usage client = MagicMock() client.chat.completions.create.return_value = response result = self.provider.create_completion( client=client, model="gpt-5.1", messages=[{"role": "user", "content": "hi"}], ) u = result.usage assert u is not None assert u.cache_read_tokens == 150 assert u.cache_creation_tokens == 0 def test_streaming_no_prompt_tokens_details(self) -> None: """When prompt_tokens_details is absent, cache_read_tokens defaults to 0.""" usage = MagicMock() usage.prompt_tokens = 100 usage.completion_tokens = 20 usage.total_tokens = 120 usage.prompt_tokens_details = None chunks = [ _openai_stream_chunk(content="Hi"), _openai_stream_chunk(empty_choices=True, usage=usage), ] client = MagicMock() client.chat.completions.create.return_value = iter(chunks) results = list( self.provider.create_streaming( client=client, model="gpt-5.1", messages=[{"role": "user", "content": "hi"}], ) ) usage_chunks = [r for r in results if r.usage is not None] assert len(usage_chunks) == 1 u = usage_chunks[0].usage assert u is not None assert u.cache_read_tokens == 0 class TestUsageInfoCacheFields: """Tests for cache fields on UsageInfo dataclass.""" def test_default_cache_fields(self) -> None: u = UsageInfo(prompt_tokens=10, completion_tokens=5, total_tokens=15) assert u.cache_creation_tokens == 0 assert u.cache_read_tokens == 0 def test_explicit_cache_fields(self) -> None: u = UsageInfo( prompt_tokens=100, completion_tokens=50, total_tokens=150, cache_creation_tokens=80, cache_read_tokens=20, ) assert u.cache_creation_tokens == 80 assert u.cache_read_tokens == 20 class TestMetricsCacheTokens: """Tests for cache token recording in MetricsCollector.""" def test_record_cache_tokens(self) -> None: from turnstone.core.metrics import MetricsCollector m = MetricsCollector() m.record_cache_tokens(100, 200) m.record_cache_tokens(50, 300) assert m._tokens["cache_creation"] == 150 assert m._tokens["cache_read"] == 500 def test_prometheus_output_includes_cache_tokens(self) -> None: from turnstone.core.metrics import MetricsCollector m = MetricsCollector() m.record_tokens(1000, 500) m.record_cache_tokens(800, 200) text = m.generate_text(workstream_states={}, total_workstreams=0) assert 'turnstone_tokens_total{type="cache_creation"} 800' in text assert 'turnstone_tokens_total{type="cache_read"} 200' in text assert 'turnstone_tokens_total{type="prompt"} 1000' in text # =========================================================================== # TestOpenAIResponsesProvider — Responses API provider # =========================================================================== class TestOpenAIResponsesProvider: """Tests for the OpenAI Responses API provider.""" def setup_method(self) -> None: from turnstone.core.providers._openai_responses import OpenAIResponsesProvider self.provider = OpenAIResponsesProvider() def test_provider_name(self) -> None: assert self.provider.provider_name == "openai" def test_get_capabilities(self) -> None: caps = self.provider.get_capabilities("gpt-5.4") assert caps.context_window == 1050000 assert caps.supports_tool_search is True class TestResponsesMessageConversion: """Tests for _convert_messages — Chat Completions format to Responses API.""" def setup_method(self) -> None: from turnstone.core.providers._openai_responses import OpenAIResponsesProvider self.provider = OpenAIResponsesProvider() def test_system_message_to_instructions(self) -> None: messages = [ {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}, ] instructions, items = self.provider._convert_messages(messages) assert instructions == "You are helpful." assert len(items) == 1 assert items[0]["role"] == "user" assert items[0]["content"] == "Hello" def test_multiple_system_messages_concatenated(self) -> None: messages = [ {"role": "system", "content": "Rule 1"}, {"role": "developer", "content": "Rule 2"}, {"role": "user", "content": "Hi"}, ] instructions, items = self.provider._convert_messages(messages) assert instructions == "Rule 1\n\nRule 2" assert len(items) == 1 def test_assistant_text_message(self) -> None: messages = [ {"role": "assistant", "content": "Hello back"}, ] _, items = self.provider._convert_messages(messages) assert len(items) == 1 assert items[0]["type"] == "message" assert items[0]["role"] == "assistant" assert items[0]["content"] == "Hello back" def test_assistant_tool_calls(self) -> None: messages = [ { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "function": {"name": "read_file", "arguments": '{"path": "/tmp"}'}, } ], }, ] _, items = self.provider._convert_messages(repair_wire_messages(messages)) # repair_wire_messages synthesizes the missing tool result; the translator renders it assert len(items) == 2 assert items[0]["type"] == "function_call" assert items[0]["call_id"] == "call_1" assert items[0]["name"] == "read_file" assert items[0]["arguments"] == '{"path": "/tmp"}' assert items[1]["type"] == "function_call_output" assert items[1]["call_id"] == "call_1" def test_tool_result(self) -> None: messages = [ {"role": "tool", "tool_call_id": "call_1", "content": "file contents"}, ] _, items = self.provider._convert_messages(messages) assert len(items) == 1 assert items[0]["type"] == "function_call_output" assert items[0]["call_id"] == "call_1" assert items[0]["output"] == "file contents" def test_provider_content_ignored_with_store_false(self) -> None: """With store=False, provider_content is ignored — rebuild from content.""" provider_items = [ { "type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}], }, {"type": "function_call", "call_id": "c1", "name": "f", "arguments": "{}"}, ] messages = [ {"role": "assistant", "content": "Hi", "_provider_content": provider_items}, ] _, items = self.provider._convert_messages(messages) # Should rebuild from content, not passthrough provider_content assert len(items) == 1 assert items[0]["type"] == "message" assert items[0]["content"] == "Hi" def test_no_system_returns_none_instructions(self) -> None: messages = [{"role": "user", "content": "Hello"}] instructions, _ = self.provider._convert_messages(messages) assert instructions is None def test_assistant_with_content_and_tool_calls(self) -> None: """Assistant message with both text and tool calls emits separate items.""" messages = [ { "role": "assistant", "content": "I'll read that file", "tool_calls": [ { "id": "call_1", "function": {"name": "read_file", "arguments": '{"path": "/tmp"}'}, } ], }, ] _, items = self.provider._convert_messages(repair_wire_messages(messages)) # repair_wire_messages synthesizes the missing tool result; the translator renders it assert len(items) == 3 assert items[0]["type"] == "message" assert items[0]["content"] == "I'll read that file" assert items[1]["type"] == "function_call" assert items[1]["name"] == "read_file" assert items[2]["type"] == "function_call_output" assert items[2]["call_id"] == "call_1" class TestResponsesToolConversion: """Tests for _convert_tools — Chat Completions tool format to Responses API.""" def setup_method(self) -> None: from turnstone.core.providers._openai_responses import OpenAIResponsesProvider self.provider = OpenAIResponsesProvider() def test_function_tool_conversion(self) -> None: tools = [ { "type": "function", "function": { "name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"path": {"type": "string"}}}, }, } ] caps = ModelCapabilities() result = self.provider._convert_tools(tools, caps) assert result is not None assert len(result) == 1 assert result[0]["type"] == "function" assert result[0]["name"] == "read_file" assert result[0]["description"] == "Read a file" assert result[0]["strict"] is False def test_web_search_replaced_with_native(self) -> None: tools = [ {"type": "function", "function": {"name": "web_search", "description": "Search"}}, {"type": "function", "function": {"name": "read_file", "description": "Read"}}, ] caps = ModelCapabilities(supports_web_search=True) result = self.provider._convert_tools(tools, caps) assert result is not None names = [t.get("name", t.get("type")) for t in result] assert "web_search" in names # native web_search tool assert "read_file" in names def test_none_tools_returns_none(self) -> None: caps = ModelCapabilities() assert self.provider._convert_tools(None, caps) is None def test_defer_loading_preserved(self) -> None: tools = [ {"type": "function", "function": {"name": "f"}, "defer_loading": True}, ] caps = ModelCapabilities() result = self.provider._convert_tools(tools, caps) assert result is not None assert result[0].get("defer_loading") is True class TestResponsesParamBuilding: """Tests for _build_kwargs — parameter construction for Responses API.""" def setup_method(self) -> None: from turnstone.core.providers._openai_responses import OpenAIResponsesProvider self.provider = OpenAIResponsesProvider() def test_reasoning_effort_as_dict(self) -> None: kwargs = self.provider._build_kwargs( model="gpt-5.4", messages=[{"role": "user", "content": "Hi"}], tools=None, max_tokens=4096, temperature=0.5, reasoning_effort="high", deferred_names=None, ) assert kwargs["reasoning"] == {"effort": "high"} assert "reasoning_effort" not in kwargs def test_none_effort_sends_declared_none_level(self) -> None: """gpt-5.4 declares an explicit "none" level — the knob's off position forwards it rather than omitting (omission would leave the server default in charge on models like gpt-5.5).""" kwargs = self.provider._build_kwargs( model="gpt-5.4", messages=[{"role": "user", "content": "Hi"}], tools=None, max_tokens=4096, temperature=0.5, reasoning_effort="none", deferred_names=None, ) assert kwargs["reasoning"] == {"effort": "none"} def test_store_is_false(self) -> None: kwargs = self.provider._build_kwargs( model="gpt-5.4", messages=[{"role": "user", "content": "Hi"}], tools=None, max_tokens=4096, temperature=0.5, reasoning_effort="medium", deferred_names=None, ) assert kwargs["store"] is False def _kwargs_with(self, tools: list[dict[str, Any]], caps: ModelCapabilities) -> dict[str, Any]: return self.provider._build_kwargs( model="gpt-5.4", messages=[{"role": "user", "content": "Hi"}], tools=tools, max_tokens=4096, temperature=0.5, reasoning_effort="medium", deferred_names=None, capabilities=caps, ) def test_server_side_web_search_needs_surviving_client_def(self) -> None: caps = ModelCapabilities(supports_web_search=True) # Client def present (unrestricted / allowlisted) → native injected. with_def = self._kwargs_with( [{"type": "function", "function": {"name": "web_search"}}], caps ) assert {"type": "web_search"} in (with_def.get("tools") or []) # Client def hidden by the persona/coordinator envelope → suppressed. without_def = self._kwargs_with( [{"type": "function", "function": {"name": "read_file"}}], caps ) assert {"type": "web_search"} not in (without_def.get("tools") or []) def test_server_side_injection_generalizes_beyond_web_search(self) -> None: # The replace-only rule applies to EVERY server-side tool: a provider- # specific one injects only with a same-named client def, so a restricted # persona that never allowlisted it can't get it injected past the wire. caps = ModelCapabilities(server_side_tools=("code_exec",)) without_def = self._kwargs_with( [{"type": "function", "function": {"name": "read_file"}}], caps ) assert {"type": "code_exec"} not in (without_def.get("tools") or []) with_def = self._kwargs_with( [{"type": "function", "function": {"name": "code_exec"}}], caps ) assert {"type": "code_exec"} in (with_def.get("tools") or []) def test_cache_retention_for_gpt5(self) -> None: kwargs = self.provider._build_kwargs( model="gpt-5.4", messages=[{"role": "user", "content": "Hi"}], tools=None, max_tokens=4096, temperature=0.5, reasoning_effort="medium", deferred_names=None, ) assert kwargs["prompt_cache_retention"] == "24h" def test_instructions_from_system_messages(self) -> None: kwargs = self.provider._build_kwargs( model="gpt-5.4", messages=[ {"role": "system", "content": "Be helpful"}, {"role": "user", "content": "Hi"}, ], tools=None, max_tokens=4096, temperature=0.5, reasoning_effort="none", deferred_names=None, ) assert kwargs["instructions"] == "Be helpful" def test_web_search_not_injected_with_no_tools(self) -> None: """No client web_search def ⇒ no server-side web_search entry. Replace-only semantics: a request whose envelope hides web_search (persona visibility set, coordinator toolset, tool-less utility call) must not gain native search at the provider layer. """ kwargs = self.provider._build_kwargs( model="gpt-5-search-api", messages=[{"role": "user", "content": "Hi"}], tools=None, max_tokens=4096, temperature=0.5, reasoning_effort="none", deferred_names=None, ) tool_types = [t.get("type") for t in kwargs.get("tools") or []] assert "web_search" not in tool_types def test_web_search_injected_with_client_def(self) -> None: """The server-side entry stands in for a surviving client def.""" kwargs = self.provider._build_kwargs( model="gpt-5-search-api", messages=[{"role": "user", "content": "Hi"}], tools=[{"type": "function", "function": {"name": "web_search"}}], max_tokens=4096, temperature=0.5, reasoning_effort="none", deferred_names=None, ) assert "tools" in kwargs tool_types = [t.get("type") for t in kwargs["tools"]] assert "web_search" in tool_types def test_web_search_not_injected_for_nonempty_toolset_without_def(self) -> None: """A non-empty toolset lacking web_search gains no native search. Guards the _convert_tools lane: capability alone must not inject — a persona visibility set or the coordinator toolset that hides web_search stays search-free on search-capable models. """ kwargs = self.provider._build_kwargs( model="gpt-5-search-api", messages=[{"role": "user", "content": "Hi"}], tools=[{"type": "function", "function": {"name": "read_file"}}], max_tokens=4096, temperature=0.5, reasoning_effort="none", deferred_names=None, ) tool_types = [t.get("type") for t in kwargs.get("tools") or []] assert "web_search" not in tool_types class TestResponsesCitationFormat: """Test format_citations handles Responses API flat annotation format.""" def test_responses_api_flat_annotation(self) -> None: """Responses API annotations have title/url directly on the object.""" class FlatAnnotation: type = "url_citation" url_citation = None # Not present in Responses API title = "Example" url = "https://example.com" result = format_citations("Text.", [FlatAnnotation()]) assert "Sources:" in result assert "[Example](https://example.com)" in result class TestResponsesStreaming: """Tests for Responses API streaming event handling.""" def setup_method(self) -> None: from turnstone.core.providers._openai_responses import OpenAIResponsesProvider self.provider = OpenAIResponsesProvider() def _make_event(self, event_type: str, **attrs: Any) -> MagicMock: event = MagicMock() event.type = event_type for k, v in attrs.items(): setattr(event, k, v) return event def test_text_delta(self) -> None: events = [ self._make_event("response.output_text.delta", delta="Hello"), self._make_event("response.output_text.delta", delta=" world"), self._make_event( "response.completed", response=MagicMock( status="completed", usage=None, ), ), ] chunks = list(self.provider._iter_stream(iter(events))) text_chunks = [c for c in chunks if c.content_delta] assert len(text_chunks) == 2 assert text_chunks[0].content_delta == "Hello" assert text_chunks[0].is_first is True assert text_chunks[1].content_delta == " world" def test_reasoning_delta(self) -> None: events = [ self._make_event("response.reasoning_text.delta", delta="thinking..."), self._make_event( "response.completed", response=MagicMock( status="completed", usage=None, ), ), ] chunks = list(self.provider._iter_stream(iter(events))) reasoning = [c for c in chunks if c.reasoning_delta] assert len(reasoning) == 1 assert reasoning[0].reasoning_delta == "thinking..." assert reasoning[0].is_first is True def test_tool_call_streaming(self) -> None: item = MagicMock() item.type = "function_call" item.id = "fc_abc123" item.call_id = "call_1" item.name = "read_file" events = [ self._make_event("response.output_item.added", item=item), self._make_event( "response.function_call_arguments.delta", item_id="fc_abc123", delta='{"path":', ), self._make_event( "response.function_call_arguments.delta", item_id="fc_abc123", delta='"/tmp"}', ), self._make_event( "response.completed", response=MagicMock( status="completed", usage=None, ), ), ] chunks = list(self.provider._iter_stream(iter(events))) tc_chunks = [c for c in chunks if c.tool_call_deltas] assert len(tc_chunks) == 3 # First chunk: tool call added with name assert tc_chunks[0].tool_call_deltas[0].name == "read_file" assert tc_chunks[0].tool_call_deltas[0].id == "call_1" # Argument deltas assert tc_chunks[1].tool_call_deltas[0].arguments_delta == '{"path":' assert tc_chunks[2].tool_call_deltas[0].arguments_delta == '"/tmp"}' def test_completed_event_with_usage(self) -> None: usage = MagicMock() usage.input_tokens = 100 usage.output_tokens = 50 usage.total_tokens = 150 usage.input_tokens_details = MagicMock(cached_tokens=80) # Ensure Chat Completions attributes are not present del usage.prompt_tokens del usage.completion_tokens del usage.prompt_tokens_details events = [ self._make_event( "response.completed", response=MagicMock( status="completed", usage=usage, ), ), ] chunks = list(self.provider._iter_stream(iter(events))) final = [c for c in chunks if c.finish_reason] assert len(final) == 1 assert final[0].finish_reason == "stop" assert final[0].usage is not None assert final[0].usage.prompt_tokens == 100 assert final[0].usage.completion_tokens == 50 assert final[0].usage.cache_read_tokens == 80 def test_web_search_events(self) -> None: events = [ self._make_event("response.web_search_call.searching"), self._make_event("response.web_search_call.completed"), self._make_event( "response.completed", response=MagicMock( status="completed", usage=None, ), ), ] chunks = list(self.provider._iter_stream(iter(events))) info = [c for c in chunks if c.info_delta] assert len(info) == 2 assert "Searching" in info[0].info_delta assert "complete" in info[1].info_delta class TestResponsesCompletion: """Tests for non-streaming Responses API completion.""" def setup_method(self) -> None: from turnstone.core.providers._openai_responses import OpenAIResponsesProvider self.provider = OpenAIResponsesProvider() def _make_response( self, text: str = "Hello", tool_calls: list[dict[str, Any]] | None = None, status: str = "completed", ) -> MagicMock: resp = MagicMock() resp.status = status resp.usage = MagicMock() resp.usage.input_tokens = 10 resp.usage.output_tokens = 5 resp.usage.total_tokens = 15 resp.usage.input_tokens_details = MagicMock(cached_tokens=0) # Remove Chat Completions attributes del resp.usage.prompt_tokens del resp.usage.completion_tokens del resp.usage.prompt_tokens_details output: list[Any] = [] if text: msg = MagicMock() msg.type = "message" text_part = MagicMock() text_part.type = "output_text" text_part.text = text text_part.annotations = [] msg.content = [text_part] msg.model_dump.return_value = { "type": "message", "content": [{"type": "output_text", "text": text}], } output.append(msg) if tool_calls: for tc in tool_calls: item = MagicMock() item.type = "function_call" item.call_id = tc["id"] item.name = tc["name"] item.arguments = tc["arguments"] item.model_dump.return_value = { "type": "function_call", "call_id": tc["id"], "name": tc["name"], "arguments": tc["arguments"], } output.append(item) resp.output = output return resp def test_basic_text_completion(self) -> None: resp = self._make_response(text="Hello world") result = self.provider._parse_response(resp) assert result.content == "Hello world" assert result.tool_calls is None assert result.finish_reason == "stop" def test_completion_with_tool_calls(self) -> None: resp = self._make_response( text="", tool_calls=[{"id": "call_1", "name": "read_file", "arguments": '{"path": "/tmp"}'}], ) result = self.provider._parse_response(resp) assert result.tool_calls is not None assert len(result.tool_calls) == 1 assert result.tool_calls[0]["id"] == "call_1" assert result.tool_calls[0]["function"]["name"] == "read_file" def test_provider_blocks_captured(self) -> None: resp = self._make_response(text="Hello") result = self.provider._parse_response(resp) assert len(result.provider_blocks) > 0 assert result.provider_blocks[0]["type"] == "message" def test_incomplete_status_maps_to_length(self) -> None: resp = self._make_response(text="Partial", status="incomplete") result = self.provider._parse_response(resp) assert result.finish_reason == "length" def test_usage_extraction(self) -> None: resp = self._make_response(text="Hi") result = self.provider._parse_response(resp) assert result.usage is not None assert result.usage.prompt_tokens == 10 assert result.usage.completion_tokens == 5