ewilliams28/turnstone
1
0
Fork 0
mirror of https://github.com/turnstonelabs/turnstone.git synced 2026-08-12 23:12:23 -06:00
Code Issues Packages Projects Releases Wiki Activity
Files
4baf6f81c329c3da2f4d0598dcdb4b1deb9ef3c4
turnstone/tests/test_providers.py
T
Patrick Buckley 114ada791b feat(providers): add Claude Fable 5 to the Anthropic provider 
- claude-fable-5 capability entry: 1M context / 128K output, adaptive
  thinking (summarized display), effort low..max incl. xhigh, no
  sampling params, web + tool search, vision, reasoning replay, native
  mid-conversation system messages
- document the Fable 5 wire quirk at the capability table: an explicit
  thinking={"type": "disabled"} is a 400 on this model; the adaptive
  branch never emits "disabled", so adaptive-or-omitted is preserved
- widen the native mid-conversation-system comments from opus-4-8-only
  to opus-4-8 + fable-5 (protocol, provider, tool_advisory, prompts,
  session)
- raise the anthropic SDK floor 0.39 -> 0.108: 0.39 predates every
  named kwarg the provider sends (output_config 0.77, top-level
  cache_control 0.83, mid-conversation system blocks 0.105); 0.108
  adds claude-fable-5
- tests: capability assertions for claude-fable-5 + dated-variant
  prefix match
2026-06-09 10:50:58 -07:00

4457 lines
174 KiB
Python
Raw Blame History

"""Tests for turnstone.core.providers — protocol, OpenAI provider, Anthropic provider."""
from __future__ import annotations
import json
from typing import Any
from unittest.mock import MagicMock, PropertyMock, patch
import pytest
from turnstone.core.lowering import repair_wire_messages
from turnstone.core.providers._openai import OpenAIProvider
from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider
from turnstone.core.providers._openai_common import (
apply_cache_retention,
apply_temperature_and_effort,
apply_tool_search,
format_citations,
sanitize_messages,
)
from turnstone.core.providers._protocol import (
CompletionResult,
LLMProvider,
ModelCapabilities,
StreamChunk,
ToolCallDelta,
UsageInfo,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _openai_stream_chunk(
*,
content: str | None = None,
reasoning: str | None = None,
reasoning_content: str | None = None,
tool_calls: list[MagicMock] | None = None,
finish_reason: str | None = None,
usage: MagicMock | None = None,
empty_choices: bool = False,
) -> MagicMock:
"""Build a mock OpenAI streaming chunk."""
chunk = MagicMock()
if empty_choices:
chunk.choices = []
chunk.usage = usage
return chunk
delta = MagicMock()
delta.content = content
delta.tool_calls = tool_calls
# Reasoning attributes accessed via getattr
type(delta).reasoning = PropertyMock(return_value=reasoning)
type(delta).reasoning_content = PropertyMock(return_value=reasoning_content)
choice = MagicMock()
choice.delta = delta
choice.finish_reason = finish_reason
chunk.choices = [choice]
chunk.usage = usage
return chunk
def _openai_tool_call_delta(
*,
index: int = 0,
tc_id: str | None = None,
name: str | None = None,
arguments: str | None = None,
) -> MagicMock:
"""Build a mock OpenAI tool call delta within a streaming chunk."""
tcd = MagicMock()
tcd.index = index
tcd.id = tc_id
tcd.function = MagicMock()
tcd.function.name = name
tcd.function.arguments = arguments
return tcd
def _anthropic_event(
event_type: str,
**kwargs: Any,
) -> MagicMock:
"""Build a mock Anthropic streaming event."""
event = MagicMock()
event.type = event_type
if event_type == "content_block_start":
block = MagicMock()
block.type = kwargs.get("block_type", "text")
block.id = kwargs.get("block_id", "")
block.name = kwargs.get("block_name", "")
event.content_block = block
event.index = kwargs.get("index", 0)
elif event_type == "content_block_delta":
delta = MagicMock()
delta.type = kwargs.get("delta_type", "text_delta")
delta.text = kwargs.get("text", "")
delta.thinking = kwargs.get("thinking", "")
delta.signature = kwargs.get("signature", "")
delta.partial_json = kwargs.get("partial_json", "")
event.delta = delta
event.index = kwargs.get("index", 0)
elif event_type == "message_delta":
if "usage_output_tokens" in kwargs:
usage = MagicMock()
usage.input_tokens = kwargs.get("usage_input_tokens", 0)
usage.output_tokens = kwargs.get("usage_output_tokens", 0)
event.usage = usage
else:
event.usage = None
stop_delta = MagicMock()
stop_delta.stop_reason = kwargs.get("stop_reason")
event.delta = stop_delta
elif event_type == "content_block_stop":
event.index = kwargs.get("index", 0)
elif event_type == "message_start":
msg = MagicMock()
if "usage_input_tokens" in kwargs:
msg_usage = MagicMock()
msg_usage.input_tokens = kwargs.get("usage_input_tokens", 0)
msg_usage.cache_creation_input_tokens = 0
msg_usage.cache_read_input_tokens = 0
msg.usage = msg_usage
else:
msg.usage = None
event.message = msg
return event
# ===========================================================================
# TestOpenAIProvider
# ===========================================================================
class TestOpenAIProvider:
"""Tests for the OpenAI Chat Completions provider adapter."""
def setup_method(self) -> None:
self.provider = OpenAIProvider()
def test_provider_name(self) -> None:
assert self.provider.provider_name == "openai-compatible"
# -- _apply_thinking_mode -------------------------------------------------
def test_thinking_mode_none_does_nothing(self) -> None:
"""No thinking params injected when thinking_mode is 'none'."""
caps = ModelCapabilities(thinking_mode="none")
extra_body: dict[str, Any] = {"chat_template_kwargs": {"reasoning_effort": "medium"}}
OpenAIProvider._apply_thinking_mode(extra_body, caps)
assert "enable_thinking" not in extra_body["chat_template_kwargs"]
def test_thinking_mode_manual_injects_param(self) -> None:
"""Manual thinking mode injects enable_thinking into chat_template_kwargs."""
caps = ModelCapabilities(thinking_mode="manual")
extra_body: dict[str, Any] = {"chat_template_kwargs": {"reasoning_effort": "medium"}}
OpenAIProvider._apply_thinking_mode(extra_body, caps)
assert extra_body["chat_template_kwargs"]["enable_thinking"] is True
assert extra_body["chat_template_kwargs"]["reasoning_effort"] == "medium"
def test_thinking_mode_custom_param(self) -> None:
"""Custom thinking_param (e.g. Granite's 'thinking') is used."""
caps = ModelCapabilities(thinking_mode="manual", thinking_param="thinking")
extra_body: dict[str, Any] = {"chat_template_kwargs": {}}
OpenAIProvider._apply_thinking_mode(extra_body, caps)
assert extra_body["chat_template_kwargs"]["thinking"] is True
assert "enable_thinking" not in extra_body["chat_template_kwargs"]
def test_thinking_mode_does_not_override_explicit(self) -> None:
"""If operator explicitly set the param to False, provider respects it."""
caps = ModelCapabilities(thinking_mode="manual")
extra_body: dict[str, Any] = {"chat_template_kwargs": {"enable_thinking": False}}
OpenAIProvider._apply_thinking_mode(extra_body, caps)
assert extra_body["chat_template_kwargs"]["enable_thinking"] is False
def test_thinking_mode_creates_ctk_if_missing(self) -> None:
"""Creates chat_template_kwargs dict if not present in extra_body."""
caps = ModelCapabilities(thinking_mode="manual")
extra_body: dict[str, Any] = {}
OpenAIProvider._apply_thinking_mode(extra_body, caps)
assert extra_body["chat_template_kwargs"]["enable_thinking"] is True
def test_thinking_mode_adaptive(self) -> None:
"""Adaptive thinking mode also injects the param."""
caps = ModelCapabilities(thinking_mode="adaptive")
extra_body: dict[str, Any] = {"chat_template_kwargs": {}}
OpenAIProvider._apply_thinking_mode(extra_body, caps)
assert extra_body["chat_template_kwargs"]["enable_thinking"] is True
# -- _sanitize_messages ---------------------------------------------------
def test_sanitize_messages_none_content_no_tool_calls(self) -> None:
msgs = [{"role": "assistant", "content": None}]
assert sanitize_messages(msgs) == [{"role": "assistant", "content": ""}]
def test_sanitize_messages_none_content_with_tool_calls(self) -> None:
msgs = [{"role": "assistant", "content": None, "tool_calls": [{"id": "1"}]}]
result = sanitize_messages(msgs)
assert result[0]["content"] is None
assert result[0]["tool_calls"] == [{"id": "1"}]
def test_sanitize_messages_empty_string_passthrough(self) -> None:
msgs = [{"role": "assistant", "content": ""}]
assert sanitize_messages(msgs) == msgs
def test_sanitize_messages_non_assistant_unchanged(self) -> None:
msgs = [{"role": "user", "content": None}]
result = sanitize_messages(msgs)
assert result[0]["content"] is None
def test_sanitize_messages_does_not_mutate_original(self) -> None:
original = {"role": "assistant", "content": None}
sanitize_messages([original])
assert original["content"] is None
def test_sanitize_messages_strips_underscore_sibling_keys(self) -> None:
"""Internal sibling metadata (``_reminders``, ``_reminders_delivered``,
``_attachments_meta``, ``_provider_content``) must be stripped
before the wire — the OpenAI-compat APIs reject unknown fields."""
msgs = [
{
"role": "user",
"content": "hi",
"_reminders": [{"type": "correction", "text": "watch"}],
"_reminders_delivered": True,
"_attachments_meta": [{"kind": "image"}],
}
]
result = sanitize_messages(msgs)
assert result == [{"role": "user", "content": "hi"}]
assert "_reminders" not in result[0]
assert "_reminders_delivered" not in result[0]
assert "_attachments_meta" not in result[0]
# -- sanitize_messages: orphan detection -----------------------------------
def test_sanitize_orphaned_tool_call_synthesized(self) -> None:
"""Tool_call with no matching tool result gets a synthetic error result."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "bash", "arguments": "{}"},
},
],
},
{"role": "user", "content": "next"},
]
result = sanitize_messages(repair_wire_messages(msgs))
assert len(result) == 3
assert result[1]["role"] == "tool"
assert result[1]["tool_call_id"] == "call_1"
assert "cancelled" in result[1]["content"]
assert result[2]["role"] == "user"
def test_sanitize_partial_results(self) -> None:
"""Only the missing tool_call gets a synthetic result."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "b", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
]
result = sanitize_messages(repair_wire_messages(msgs))
assert len(result) == 3
assert result[1]["tool_call_id"] == "call_1"
assert result[1]["content"] == "ok"
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_2"
assert "cancelled" in result[2]["content"]
def test_sanitize_complete_results_unchanged(self) -> None:
"""All tool_calls paired → no changes."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
{"role": "user", "content": "thanks"},
]
result = sanitize_messages(msgs)
assert len(result) == 3
assert result[0]["tool_calls"][0]["id"] == "call_1"
assert result[1]["content"] == "ok"
assert result[2]["role"] == "user"
def test_sanitize_trailing_orphan(self) -> None:
"""Orphaned tool_call at end of conversation (no following messages)."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
],
},
]
result = sanitize_messages(repair_wire_messages(msgs))
assert len(result) == 2
assert result[1]["role"] == "tool"
assert result[1]["tool_call_id"] == "call_1"
def test_sanitize_orphaned_tool_result_dropped(self) -> None:
"""Tool result with no matching tool_call in preceding assistant → dropped."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
{"role": "tool", "tool_call_id": "call_ORPHAN", "content": "stale"},
]
result = sanitize_messages(msgs)
assert len(result) == 2
assert result[1]["tool_call_id"] == "call_1"
def test_sanitize_empty_tool_call_id_filled(self) -> None:
"""Empty tool_call IDs get synthetic values; tool results are remapped to match."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "", "type": "function", "function": {"name": "a", "arguments": "{}"}},
],
},
{"role": "tool", "tool_call_id": "", "content": "ok"},
]
result = sanitize_messages(msgs)
new_id = result[0]["tool_calls"][0]["id"]
assert new_id.startswith("call_")
assert len(new_id) > 10
# Tool result must have been remapped to match
assert result[1]["tool_call_id"] == new_id
# No synthetic result needed — the pairing is complete
assert len(result) == 2
def test_sanitize_empty_tool_call_id_orphan_synthesized(self) -> None:
"""An empty-id tool_call with no result: sanitize back-fills the id AND
synthesizes its cancellation (the upstream repair can't see an id-less
call, so this lane owns it)."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "", "type": "function", "function": {"name": "a", "arguments": "{}"}},
],
},
{"role": "user", "content": "never mind"},
]
result = sanitize_messages(msgs)
new_id = result[0]["tool_calls"][0]["id"]
assert new_id.startswith("call_")
tool_msgs = [m for m in result if m.get("role") == "tool"]
assert len(tool_msgs) == 1
assert tool_msgs[0]["tool_call_id"] == new_id # paired to the back-filled id
assert "cancelled" in tool_msgs[0]["content"].lower()
def test_sanitize_stale_result_with_orphan(self) -> None:
"""Stale tool results are dropped even when orphaned calls are present."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "b", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
{"role": "tool", "tool_call_id": "call_STALE", "content": "stale"},
]
result = sanitize_messages(repair_wire_messages(msgs))
result_tc_ids = [m["tool_call_id"] for m in result if m.get("role") == "tool"]
assert "call_STALE" not in result_tc_ids
assert "call_1" in result_tc_ids
assert "call_2" in result_tc_ids # synthesized
def test_sanitize_orphan_no_mutation(self) -> None:
"""Original messages and dicts are not mutated by orphan detection."""
tc = {"id": "", "type": "function", "function": {"name": "a", "arguments": "{}"}}
msg = {"role": "assistant", "content": None, "tool_calls": [tc]}
sanitize_messages([msg])
assert tc["id"] == "" # original dict untouched
assert msg["tool_calls"][0]["id"] == ""
def test_sanitize_repeated_ids_across_turns(self) -> None:
"""Reused tool_call IDs across turns are handled per-turn, not globally."""
msgs = [
# Turn 1: call_1 fully paired
{"role": "user", "content": "do A"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
# Turn 2: reuses call_1 but has no result → must be synthesized
{"role": "user", "content": "do B"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "b", "arguments": "{}"},
},
],
},
]
result = sanitize_messages(repair_wire_messages(msgs))
# Turn 2's orphaned call_1 should get a synthetic result
tool_msgs = [m for m in result if m.get("role") == "tool"]
assert len(tool_msgs) == 2 # one real from turn 1, one synthetic from turn 2
def test_sanitize_drops_is_error_from_tool_messages(self) -> None:
"""``is_error`` is the neutral error flag (Anthropic renders it); the
OpenAI-compatible tool message has no such field, so it is dropped."""
msgs = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "c1", "type": "function", "function": {"name": "a", "arguments": "{}"}},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "boom", "is_error": True},
]
result = sanitize_messages(msgs)
tool_msg = next(m for m in result if m.get("role") == "tool")
assert "is_error" not in tool_msg
assert tool_msg["content"] == "boom" # payload otherwise intact
# -- convert_tools --------------------------------------------------------
def test_convert_tools_passthrough(self) -> None:
tools = [
{
"type": "function",
"function": {
"name": "read_file",
"description": "Read a file",
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}},
},
}
]
assert self.provider.convert_tools(tools) is tools
def test_streaming_content(self) -> None:
chunks = [
_openai_stream_chunk(content="Hello"),
_openai_stream_chunk(content=" world"),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
)
assert len(results) == 2
assert results[0].content_delta == "Hello"
assert results[1].content_delta == " world"
def test_streaming_reasoning(self) -> None:
chunks = [
_openai_stream_chunk(reasoning_content="thinking..."),
_openai_stream_chunk(reasoning_content="more thought"),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="qwen3-32b",
messages=[{"role": "user", "content": "hi"}],
)
)
assert len(results) == 2
assert results[0].reasoning_delta == "thinking..."
assert results[1].reasoning_delta == "more thought"
def test_streaming_tool_calls(self) -> None:
tc1 = _openai_tool_call_delta(index=0, tc_id="call_1", name="read_file")
tc2 = _openai_tool_call_delta(index=0, arguments='{"path":')
tc3 = _openai_tool_call_delta(index=0, arguments='"foo.py"}')
chunks = [
_openai_stream_chunk(tool_calls=[tc1]),
_openai_stream_chunk(tool_calls=[tc2]),
_openai_stream_chunk(tool_calls=[tc3]),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "read a file"}],
)
)
assert len(results) == 3
assert results[0].tool_call_deltas[0].id == "call_1"
assert results[0].tool_call_deltas[0].name == "read_file"
assert results[1].tool_call_deltas[0].arguments_delta == '{"path":'
assert results[2].tool_call_deltas[0].arguments_delta == '"foo.py"}'
def test_streaming_usage(self) -> None:
usage = MagicMock()
usage.prompt_tokens = 10
usage.completion_tokens = 20
usage.total_tokens = 30
chunks = [
_openai_stream_chunk(content="Hi"),
_openai_stream_chunk(empty_choices=True, usage=usage),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
)
# Last yielded chunk should carry usage
usage_chunk = [r for r in results if r.usage is not None]
assert len(usage_chunk) == 1
assert usage_chunk[0].usage is not None
assert usage_chunk[0].usage.prompt_tokens == 10
assert usage_chunk[0].usage.completion_tokens == 20
assert usage_chunk[0].usage.total_tokens == 30
def test_streaming_finish_reason(self) -> None:
chunks = [
_openai_stream_chunk(content="done"),
_openai_stream_chunk(finish_reason="stop"),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
)
finish_chunks = [r for r in results if r.finish_reason is not None]
assert len(finish_chunks) == 1
assert finish_chunks[0].finish_reason == "stop"
def test_streaming_finish_reason_tool_calls(self) -> None:
tc = _openai_tool_call_delta(index=0, tc_id="call_1", name="fn")
chunks = [
_openai_stream_chunk(tool_calls=[tc]),
_openai_stream_chunk(finish_reason="tool_calls"),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
)
finish_chunks = [r for r in results if r.finish_reason is not None]
assert finish_chunks[0].finish_reason == "tool_calls"
def test_streaming_is_first(self) -> None:
chunks = [
_openai_stream_chunk(content="A"),
_openai_stream_chunk(content="B"),
_openai_stream_chunk(content="C"),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
)
assert results[0].is_first is True
assert results[1].is_first is False
assert results[2].is_first is False
def test_completion_basic(self) -> None:
response = MagicMock()
response.choices = [MagicMock()]
response.choices[0].message.content = "Hello world"
response.choices[0].message.tool_calls = None
response.choices[0].finish_reason = "stop"
response.usage.prompt_tokens = 10
response.usage.completion_tokens = 5
response.usage.total_tokens = 15
client = MagicMock()
client.chat.completions.create.return_value = response
result = self.provider.create_completion(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
assert isinstance(result, CompletionResult)
assert result.content == "Hello world"
assert result.tool_calls is None
assert result.finish_reason == "stop"
def test_completion_with_tools(self) -> None:
tc = MagicMock()
tc.id = "call_abc"
tc.function.name = "read_file"
tc.function.arguments = '{"path": "foo.py"}'
response = MagicMock()
response.choices = [MagicMock()]
response.choices[0].message.content = None
response.choices[0].message.tool_calls = [tc]
response.choices[0].finish_reason = "tool_calls"
response.usage.prompt_tokens = 8
response.usage.completion_tokens = 12
response.usage.total_tokens = 20
client = MagicMock()
client.chat.completions.create.return_value = response
result = self.provider.create_completion(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "read"}],
)
assert result.content == ""
assert result.tool_calls is not None
assert len(result.tool_calls) == 1
assert result.tool_calls[0]["id"] == "call_abc"
assert result.tool_calls[0]["type"] == "function"
assert result.tool_calls[0]["function"]["name"] == "read_file"
assert result.tool_calls[0]["function"]["arguments"] == '{"path": "foo.py"}'
assert result.finish_reason == "tool_calls"
def test_completion_usage(self) -> None:
response = MagicMock()
response.choices = [MagicMock()]
response.choices[0].message.content = "ok"
response.choices[0].message.tool_calls = None
response.choices[0].finish_reason = "stop"
response.usage.prompt_tokens = 100
response.usage.completion_tokens = 50
response.usage.total_tokens = 150
client = MagicMock()
client.chat.completions.create.return_value = response
result = self.provider.create_completion(
client=client,
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
)
assert result.usage is not None
assert result.usage.prompt_tokens == 100
assert result.usage.completion_tokens == 50
assert result.usage.total_tokens == 150
def test_retryable_errors(self) -> None:
errors = self.provider.retryable_error_names
assert isinstance(errors, frozenset)
assert "APIError" in errors
assert "APIConnectionError" in errors
assert "RateLimitError" in errors
assert "Timeout" in errors
assert "APITimeoutError" in errors
# ===========================================================================
# TestAnthropicProvider
# ===========================================================================
class TestAnthropicProvider:
"""Tests for the Anthropic native provider adapter."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_provider_name(self) -> None:
assert self.provider.provider_name == "anthropic"
def test_convert_tools(self) -> None:
openai_tools = [
{
"type": "function",
"function": {
"name": "read_file",
"description": "Read a file from disk",
"parameters": {
"type": "object",
"properties": {"path": {"type": "string"}},
"required": ["path"],
},
},
},
{
"type": "function",
"function": {
"name": "write_file",
"description": "Write a file",
"parameters": {
"type": "object",
"properties": {
"path": {"type": "string"},
"content": {"type": "string"},
},
},
},
},
]
result = self.provider.convert_tools(openai_tools)
assert len(result) == 2
assert result[0]["name"] == "read_file"
assert result[0]["description"] == "Read a file from disk"
assert result[0]["input_schema"]["type"] == "object"
assert "path" in result[0]["input_schema"]["properties"]
# No "type": "function" wrapper
assert "function" not in result[0]
assert "type" not in result[0]
assert result[1]["name"] == "write_file"
def test_message_conversion_basic(self) -> None:
messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
{"role": "user", "content": "How are you?"},
]
system, converted = self.provider._convert_messages(messages)
assert system == "You are helpful."
assert len(converted) == 3
assert converted[0]["role"] == "user"
assert converted[0]["content"] == "Hello"
assert converted[1]["role"] == "assistant"
assert converted[1]["content"] == [{"type": "text", "text": "Hi there!"}]
assert converted[2]["role"] == "user"
assert converted[2]["content"] == "How are you?"
def test_message_conversion_tool_calls(self) -> None:
messages = [
{
"role": "assistant",
"content": "Let me check that.",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "read_file",
"arguments": '{"path": "foo.py"}',
},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "file contents"},
]
_, converted = self.provider._convert_messages(messages)
assert len(converted) == 2
blocks = converted[0]["content"]
assert len(blocks) == 2
assert blocks[0] == {"type": "text", "text": "Let me check that."}
assert blocks[1]["type"] == "tool_use"
assert blocks[1]["id"] == "call_1"
assert blocks[1]["name"] == "read_file"
assert blocks[1]["input"] == {"path": "foo.py"}
# Tool result in user message
assert converted[1]["role"] == "user"
def test_message_conversion_tool_results(self) -> None:
messages = [
{"role": "tool", "tool_call_id": "call_1", "content": "file contents here"},
{"role": "tool", "tool_call_id": "call_2", "content": "another result"},
]
_, converted = self.provider._convert_messages(messages)
assert len(converted) == 1
assert converted[0]["role"] == "user"
blocks = converted[0]["content"]
assert len(blocks) == 2
assert blocks[0]["type"] == "tool_result"
assert blocks[0]["tool_use_id"] == "call_1"
assert blocks[0]["content"] == "file contents here"
assert blocks[1]["type"] == "tool_result"
assert blocks[1]["tool_use_id"] == "call_2"
assert blocks[1]["content"] == "another result"
def test_message_conversion_alternating_merge(self) -> None:
messages = [
{"role": "user", "content": "Hello"},
{"role": "user", "content": "Are you there?"},
{"role": "assistant", "content": "Yes"},
{"role": "assistant", "content": "I am here"},
]
_, converted = self.provider._convert_messages(messages)
assert len(converted) == 2
# First merged user message
assert converted[0]["role"] == "user"
assert converted[0]["content"] == [
{"type": "text", "text": "Hello"},
{"type": "text", "text": "Are you there?"},
]
# Second merged assistant message
assert converted[1]["role"] == "assistant"
assert converted[1]["content"] == [
{"type": "text", "text": "Yes"},
{"type": "text", "text": "I am here"},
]
def test_message_conversion_developer_role_as_system(self) -> None:
messages = [
{"role": "developer", "content": "System prompt via developer role."},
{"role": "user", "content": "Hi"},
]
system, converted = self.provider._convert_messages(messages)
assert system == "System prompt via developer role."
assert len(converted) == 1
assert converted[0]["role"] == "user"
def test_message_conversion_multiple_system(self) -> None:
messages = [
{"role": "system", "content": "Part 1."},
{"role": "system", "content": "Part 2."},
{"role": "user", "content": "Go."},
]
system, _ = self.provider._convert_messages(messages)
assert system == "Part 1.\n\nPart 2."
def test_mid_conversation_system_hoisted_when_not_native(self) -> None:
# Default (supports_mid_conversation_system=False): a system message
# after a user turn still hoists — non-native models rely on the fold
# pass having stripped operator turns before the converter sees them.
messages = [
{"role": "user", "content": "hi"},
{"role": "system", "content": "operator note"},
]
system, converted = self.provider._convert_messages(messages)
assert "operator note" in system
assert all(m["role"] != "system" for m in converted)
def test_leading_system_hoists_even_when_native(self) -> None:
messages = [
{"role": "system", "content": "base prompt"},
{"role": "user", "content": "hi"},
]
system, converted = self.provider._convert_messages(
messages, supports_mid_conversation_system=True
)
assert system == "base prompt"
assert [m["role"] for m in converted] == ["user"]
def test_mid_conversation_system_inline_when_native(self) -> None:
messages = [
{"role": "user", "content": "review this"},
{"role": "assistant", "content": "done"},
{"role": "system", "content": "from now on, add type hints"},
]
system, converted = self.provider._convert_messages(
messages, supports_mid_conversation_system=True
)
assert system == "" # nothing leading to hoist
assert [m["role"] for m in converted] == ["user", "assistant", "system"]
assert converted[-1]["content"] == "from now on, add type hints"
def test_leading_empty_assistant_then_system_hoists_when_native(self) -> None:
# An assistant turn that converts to nothing (empty content, no
# tool_calls, no provider_content) still flips seen_non_system, but it
# appended nothing — so a following operator system turn must NOT become
# messages[0] (the API requires messages[0]=user). It hoists into the
# system param instead. Guards the bug where ``seen_non_system`` alone
# gated inline emission.
messages = [
{"role": "assistant", "content": ""},
{"role": "system", "content": "operator note"},
{"role": "user", "content": "hi"},
]
system, converted = self.provider._convert_messages(
messages, supports_mid_conversation_system=True
)
assert "operator note" in system
assert converted[0]["role"] == "user"
assert not any(m["role"] == "system" for m in converted)
def test_consecutive_mid_conversation_system_coalesced_when_native(self) -> None:
messages = [
{"role": "user", "content": "go"},
{"role": "system", "content": "first"},
{"role": "system", "content": "second"},
]
_, converted = self.provider._convert_messages(
messages, supports_mid_conversation_system=True
)
# _merge_consecutive coalesces the two system turns into one message
# (the API forbids consecutive system messages).
assert [m["role"] for m in converted] == ["user", "system"]
body = converted[1]["content"]
flat = (
body
if isinstance(body, str)
else " ".join(p.get("text", "") for p in body if isinstance(p, dict))
)
assert "first" in flat and "second" in flat
def test_mid_conversation_system_after_tool_result_when_native(self) -> None:
messages = [
{"role": "user", "content": "run it"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "run", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
{"role": "system", "content": "user said: also update changelog"},
]
_, converted = self.provider._convert_messages(
messages, supports_mid_conversation_system=True
)
# The operator turn lands after the tool_result user turn — a valid slot.
roles = [m["role"] for m in converted]
assert roles[-1] == "system"
assert roles[-2] == "user" # the packed tool_result turn
assert converted[-1]["content"] == "user said: also update changelog"
def test_reasoning_params_mapping(self) -> None:
assert self.provider._reasoning_params("low", None, max_tokens=32768) == {
"thinking": {"type": "enabled", "budget_tokens": 1024}
}
assert self.provider._reasoning_params("medium", None, max_tokens=32768) == {
"thinking": {"type": "enabled", "budget_tokens": 4096}
}
assert self.provider._reasoning_params("high", None, max_tokens=32768) == {
"thinking": {"type": "enabled", "budget_tokens": 16384}
}
def test_reasoning_params_override(self) -> None:
result = self.provider._reasoning_params(
"low", {"thinking_budget_tokens": 8192}, max_tokens=32768
)
assert result == {"thinking": {"type": "enabled", "budget_tokens": 8192}}
def test_reasoning_params_unknown_effort(self) -> None:
# Unknown effort falls back to 4096
result = self.provider._reasoning_params("turbo", None, max_tokens=32768)
assert result == {"thinking": {"type": "enabled", "budget_tokens": 4096}}
def test_reasoning_params_budget_clamped(self) -> None:
# Budget >= max_tokens gets clamped to leave room for response
result = self.provider._reasoning_params("high", None, max_tokens=4096)
assert result == {"thinking": {"type": "enabled", "budget_tokens": 3072}}
def test_finish_reason_normalization(self) -> None:
from turnstone.core.providers._anthropic import _normalize_finish_reason
assert _normalize_finish_reason("end_turn") == "stop"
assert _normalize_finish_reason("tool_use") == "tool_calls"
assert _normalize_finish_reason("max_tokens") == "length"
assert _normalize_finish_reason("other_reason") == "other_reason"
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_completion_basic(self, mock_ensure: MagicMock) -> None:
text_block = MagicMock()
text_block.type = "text"
text_block.text = "Hello world"
response = MagicMock()
response.content = [text_block]
response.stop_reason = "end_turn"
response.usage = MagicMock()
response.usage.input_tokens = 10
response.usage.output_tokens = 5
response.usage.cache_creation_input_tokens = 0
response.usage.cache_read_input_tokens = 0
client = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.get_final_message.return_value = response
client.messages.stream.return_value = stream_ctx
result = self.provider.create_completion(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "hi"}],
)
assert isinstance(result, CompletionResult)
assert result.content == "Hello world"
assert result.tool_calls is None
assert result.finish_reason == "stop"
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_completion_with_tool_use(self, mock_ensure: MagicMock) -> None:
text_block = MagicMock()
text_block.type = "text"
text_block.text = "Let me read that."
tool_block = MagicMock()
tool_block.type = "tool_use"
tool_block.id = "toolu_abc"
tool_block.name = "read_file"
tool_block.input = {"path": "foo.py"}
response = MagicMock()
response.content = [text_block, tool_block]
response.stop_reason = "tool_use"
response.usage = MagicMock()
response.usage.input_tokens = 15
response.usage.output_tokens = 20
response.usage.cache_creation_input_tokens = 0
response.usage.cache_read_input_tokens = 0
client = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.get_final_message.return_value = response
client.messages.stream.return_value = stream_ctx
result = self.provider.create_completion(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "read foo.py"}],
)
assert result.content == "Let me read that."
assert result.finish_reason == "tool_calls"
assert result.tool_calls is not None
assert len(result.tool_calls) == 1
tc = result.tool_calls[0]
assert tc["id"] == "toolu_abc"
assert tc["type"] == "function"
assert tc["function"]["name"] == "read_file"
assert json.loads(tc["function"]["arguments"]) == {"path": "foo.py"}
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_completion_usage(self, mock_ensure: MagicMock) -> None:
text_block = MagicMock()
text_block.type = "text"
text_block.text = "ok"
response = MagicMock()
response.content = [text_block]
response.stop_reason = "end_turn"
response.usage = MagicMock()
response.usage.input_tokens = 100
response.usage.output_tokens = 50
response.usage.cache_creation_input_tokens = 0
response.usage.cache_read_input_tokens = 0
client = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.get_final_message.return_value = response
client.messages.stream.return_value = stream_ctx
result = self.provider.create_completion(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "hi"}],
)
assert result.usage is not None
assert result.usage.prompt_tokens == 100
assert result.usage.completion_tokens == 50
assert result.usage.total_tokens == 150
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_text_delta(self, mock_ensure: MagicMock) -> None:
events = [
_anthropic_event("content_block_delta", delta_type="text_delta", text="Hello"),
_anthropic_event("content_block_delta", delta_type="text_delta", text=" world"),
]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "hi"}],
)
)
assert len(results) == 2
assert results[0].content_delta == "Hello"
assert results[0].is_first is True
assert results[1].content_delta == " world"
assert results[1].is_first is False
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_thinking_delta(self, mock_ensure: MagicMock) -> None:
events = [
_anthropic_event(
"content_block_delta",
delta_type="thinking_delta",
thinking="reasoning step 1",
),
_anthropic_event(
"content_block_delta",
delta_type="thinking_delta",
thinking="reasoning step 2",
),
]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "think"}],
)
)
assert len(results) == 2
assert results[0].reasoning_delta == "reasoning step 1"
assert results[1].reasoning_delta == "reasoning step 2"
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_tool_use(self, mock_ensure: MagicMock) -> None:
events = [
_anthropic_event(
"content_block_start",
block_type="tool_use",
block_id="toolu_123",
block_name="read_file",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"path":',
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='"foo.py"}',
index=0,
),
]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "read a file"}],
)
)
assert len(results) == 3
# First chunk: content_block_start with tool id and name
assert results[0].tool_call_deltas[0].id == "toolu_123"
assert results[0].tool_call_deltas[0].name == "read_file"
assert results[0].tool_call_deltas[0].index == 0
# Subsequent chunks: argument fragments
assert results[1].tool_call_deltas[0].arguments_delta == '{"path":'
assert results[2].tool_call_deltas[0].arguments_delta == '"foo.py"}'
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_message_delta_usage(self, mock_ensure: MagicMock) -> None:
events = [
_anthropic_event("content_block_delta", delta_type="text_delta", text="Hi"),
_anthropic_event(
"message_delta",
stop_reason="end_turn",
usage_input_tokens=0,
usage_output_tokens=12,
),
]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "hi"}],
)
)
# The message_delta event should carry usage and finish_reason
delta_chunks = [r for r in results if r.finish_reason is not None]
assert len(delta_chunks) == 1
assert delta_chunks[0].finish_reason == "stop"
assert delta_chunks[0].usage is not None
assert delta_chunks[0].usage.completion_tokens == 12
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_message_start_usage(self, mock_ensure: MagicMock) -> None:
events = [
_anthropic_event("message_start", usage_input_tokens=42),
_anthropic_event("content_block_delta", delta_type="text_delta", text="Hi"),
]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "hi"}],
)
)
# message_start with usage should be yielded
start_chunks = [r for r in results if r.usage is not None and r.usage.prompt_tokens == 42]
assert len(start_chunks) == 1
assert start_chunks[0].usage is not None
assert start_chunks[0].usage.prompt_tokens == 42
def test_retryable_errors(self) -> None:
errors = self.provider.retryable_error_names
assert isinstance(errors, frozenset)
assert "RateLimitError" in errors
assert "APITimeoutError" in errors
assert "APIConnectionError" in errors
assert "InternalServerError" in errors
assert "APIError" in errors
assert "OverloadedError" in errors
# ===========================================================================
# TestAnthropicHelpers
# ===========================================================================
class TestAnthropicHelpers:
"""Tests for Anthropic module-level helper functions."""
def test_merge_consecutive(self) -> None:
from turnstone.core.providers._anthropic import _merge_consecutive
messages = [
{"role": "user", "content": "A"},
{"role": "user", "content": "B"},
{"role": "assistant", "content": "C"},
{"role": "user", "content": "D"},
]
merged = _merge_consecutive(messages)
assert len(merged) == 3
assert merged[0]["role"] == "user"
assert merged[0]["content"] == [
{"type": "text", "text": "A"},
{"type": "text", "text": "B"},
]
assert merged[1]["role"] == "assistant"
assert merged[2]["role"] == "user"
def test_merge_consecutive_empty(self) -> None:
from turnstone.core.providers._anthropic import _merge_consecutive
assert _merge_consecutive([]) == []
def test_merge_consecutive_no_duplicates(self) -> None:
from turnstone.core.providers._anthropic import _merge_consecutive
messages = [
{"role": "user", "content": "A"},
{"role": "assistant", "content": "B"},
{"role": "user", "content": "C"},
]
merged = _merge_consecutive(messages)
assert len(merged) == 3
def test_to_blocks_string(self) -> None:
from turnstone.core.providers._anthropic import _to_blocks
result = _to_blocks("hello")
assert result == [{"type": "text", "text": "hello"}]
def test_to_blocks_list(self) -> None:
from turnstone.core.providers._anthropic import _to_blocks
blocks = [{"type": "text", "text": "already a block"}]
result = _to_blocks(blocks)
assert result == blocks
def test_to_blocks_other(self) -> None:
from turnstone.core.providers._anthropic import _to_blocks
result = _to_blocks(42)
assert result == [{"type": "text", "text": "42"}]
def test_capabilities_lookup_exact(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-opus-4-6")
assert caps.context_window == 1000000
assert caps.max_output_tokens == 128000
assert caps.thinking_mode == "adaptive"
assert caps.supports_effort is True
def test_capabilities_lookup_prefix(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
# Prefix match: "claude-sonnet-4-6" matches dated variants
caps = provider.get_capabilities("claude-sonnet-4-6-20260101")
assert caps.context_window == 1000000
assert caps.token_param == "max_tokens"
assert caps.thinking_mode == "adaptive"
def test_capabilities_fable_5(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-fable-5")
assert caps.context_window == 1000000
assert caps.max_output_tokens == 128000
assert caps.thinking_mode == "adaptive"
assert caps.supports_effort is True
assert "xhigh" in caps.effort_levels
assert "max" in caps.effort_levels
assert caps.supports_temperature is False
assert caps.thinking_display == "summarized"
assert caps.supports_web_search is True
assert caps.supports_tool_search is True
assert caps.supports_vision is True
assert caps.supports_reasoning_replay is True
assert caps.supports_mid_conversation_system is True
def test_capabilities_fable_5_dated(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-fable-5-20260815")
assert caps.context_window == 1000000
assert caps.supports_temperature is False
assert caps.thinking_display == "summarized"
assert caps.supports_mid_conversation_system is True
def test_capabilities_opus_4_8(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-opus-4-8")
assert caps.context_window == 1000000
assert caps.max_output_tokens == 128000
assert caps.thinking_mode == "adaptive"
assert caps.supports_effort is True
assert "xhigh" in caps.effort_levels
assert "max" in caps.effort_levels
assert caps.supports_temperature is False
assert caps.thinking_display == "summarized"
assert caps.supports_web_search is True
assert caps.supports_tool_search is True
assert caps.supports_vision is True
assert caps.supports_reasoning_replay is True
assert caps.supports_mid_conversation_system is True
def test_capabilities_opus_4_8_dated(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-opus-4-8-20260601")
assert caps.context_window == 1000000
assert caps.supports_temperature is False
assert caps.thinking_display == "summarized"
def test_capabilities_opus_4_7(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-opus-4-7")
assert caps.context_window == 1000000
assert caps.max_output_tokens == 128000
assert caps.thinking_mode == "adaptive"
assert caps.supports_effort is True
assert "xhigh" in caps.effort_levels
assert caps.supports_temperature is False
assert caps.thinking_display == "summarized"
assert caps.supports_web_search is True
assert caps.supports_tool_search is True
assert caps.supports_vision is True
def test_capabilities_opus_4_7_dated(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-opus-4-7-20260416")
assert caps.context_window == 1000000
assert caps.supports_temperature is False
assert caps.thinking_display == "summarized"
def test_capabilities_lookup_unknown(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("unknown-model-xyz")
# Falls back to default
assert caps.context_window == 200000
assert caps.thinking_mode == "manual"
assert caps.token_param == "max_tokens"
# ===========================================================================
# TestProviderFactory
# ===========================================================================
class TestProviderFactory:
"""Tests for create_provider and create_client factory functions."""
def test_create_provider_openai(self) -> None:
from turnstone.core.providers import OpenAIResponsesProvider, create_provider
provider = create_provider("openai")
assert isinstance(provider, OpenAIResponsesProvider)
assert provider.provider_name == "openai"
def test_create_provider_anthropic(self) -> None:
from turnstone.core.providers import create_provider
provider = create_provider("anthropic")
assert provider.provider_name == "anthropic"
def test_create_provider_unknown(self) -> None:
from turnstone.core.providers import create_provider
with pytest.raises(ValueError, match="Unknown provider"):
create_provider("gemini")
@patch("openai.OpenAI")
def test_create_client_openai(self, mock_openai_cls: MagicMock) -> None:
from turnstone.core.providers import create_client
mock_openai_cls.return_value = MagicMock()
client = create_client("openai", base_url="http://localhost:8000/v1", api_key="test-key")
mock_openai_cls.assert_called_once_with(
base_url="http://localhost:8000/v1", api_key="test-key"
)
assert client is mock_openai_cls.return_value
@patch("openai.OpenAI")
def test_create_client_empty_api_key_passes_none(self, mock_openai_cls: MagicMock) -> None:
from turnstone.core.providers import create_client
mock_openai_cls.return_value = MagicMock()
create_client("openai", base_url="http://localhost:8000/v1", api_key="")
mock_openai_cls.assert_called_once_with(base_url="http://localhost:8000/v1", api_key=None)
@patch("openai.OpenAI")
def test_create_client_empty_api_key_no_base_url(self, mock_openai_cls: MagicMock) -> None:
from turnstone.core.providers import create_client
mock_openai_cls.return_value = MagicMock()
create_client("openai", base_url="", api_key="")
mock_openai_cls.assert_called_once_with(api_key=None)
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_create_client_anthropic_empty_api_key_omits_kwarg(
self, mock_ensure: MagicMock
) -> None:
from turnstone.core.providers import create_client
mock_anthropic_cls = MagicMock()
mock_mod = MagicMock()
mock_mod.Anthropic = mock_anthropic_cls
mock_ensure.return_value = mock_mod
create_client("anthropic", base_url="", api_key="")
mock_anthropic_cls.assert_called_once_with()
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_create_client_anthropic_nonempty_api_key_passes_kwarg(
self, mock_ensure: MagicMock
) -> None:
from turnstone.core.providers import create_client
mock_anthropic_cls = MagicMock()
mock_mod = MagicMock()
mock_mod.Anthropic = mock_anthropic_cls
mock_ensure.return_value = mock_mod
create_client("anthropic", base_url="", api_key="sk-ant-test")
mock_anthropic_cls.assert_called_once_with(api_key="sk-ant-test")
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_create_client_anthropic_empty_api_key_with_custom_base_url(
self, mock_ensure: MagicMock
) -> None:
from turnstone.core.providers import create_client
mock_anthropic_cls = MagicMock()
mock_mod = MagicMock()
mock_mod.Anthropic = mock_anthropic_cls
mock_ensure.return_value = mock_mod
create_client("anthropic", base_url="http://my-proxy:8000", api_key="")
mock_anthropic_cls.assert_called_once_with(base_url="http://my-proxy:8000")
def test_create_client_unknown(self) -> None:
from turnstone.core.providers import create_client
with pytest.raises(ValueError, match="Unknown provider"):
create_client("gemini", base_url="http://x", api_key="k")
def test_is_llm_provider(self) -> None:
"""Verify runtime_checkable protocol works with isinstance."""
provider = OpenAIProvider()
assert isinstance(provider, LLMProvider)
def test_non_provider_not_instance(self) -> None:
"""A plain object should not satisfy LLMProvider protocol check."""
class NotAProvider:
pass
assert not isinstance(NotAProvider(), LLMProvider)
def test_create_provider_openai_compatible(self) -> None:
from turnstone.core.providers import create_provider
provider = create_provider("openai-compatible")
assert isinstance(provider, OpenAIChatCompletionsProvider)
assert provider.provider_name == "openai-compatible"
def test_create_provider_openai_vs_compatible_distinct(self) -> None:
from turnstone.core.providers import OpenAIResponsesProvider, create_provider
openai_prov = create_provider("openai")
compat = create_provider("openai-compatible")
assert openai_prov is not compat
assert isinstance(openai_prov, OpenAIResponsesProvider)
assert isinstance(compat, OpenAIChatCompletionsProvider)
assert openai_prov.provider_name == "openai"
assert compat.provider_name == "openai-compatible"
def test_create_provider_returns_singleton(self) -> None:
from turnstone.core.providers import create_provider
p1 = create_provider("openai")
p2 = create_provider("openai")
assert p1 is p2
def test_create_provider_compat_responses_surface(self) -> None:
"""openai-compatible + api_surface=responses returns the Responses provider."""
from turnstone.core.providers import OpenAIResponsesProvider, create_provider
provider = create_provider("openai-compatible", api_surface="responses")
assert isinstance(provider, OpenAIResponsesProvider)
def test_create_provider_compat_chat_surface_default(self) -> None:
"""openai-compatible defaults to Chat Completions."""
from turnstone.core.providers import create_provider
for surface in (None, "", "chat"):
provider = create_provider("openai-compatible", api_surface=surface)
assert isinstance(provider, OpenAIChatCompletionsProvider)
def test_create_provider_invalid_api_surface(self) -> None:
from turnstone.core.providers import create_provider
with pytest.raises(ValueError, match="Unknown api_surface"):
create_provider("openai-compatible", api_surface="bogus")
def test_create_provider_openai_ignores_api_surface(self) -> None:
"""Cloud OpenAI is always Responses regardless of api_surface."""
from turnstone.core.providers import OpenAIResponsesProvider, create_provider
provider = create_provider("openai", api_surface="chat")
assert isinstance(provider, OpenAIResponsesProvider)
# -- Google provider -------------------------------------------------------
def test_create_provider_google(self) -> None:
from turnstone.core.providers import create_provider
from turnstone.core.providers._google import GoogleProvider
provider = create_provider("google")
assert isinstance(provider, GoogleProvider)
assert provider.provider_name == "google"
def test_create_provider_google_singleton(self) -> None:
from turnstone.core.providers import create_provider
p1 = create_provider("google")
p2 = create_provider("google")
assert p1 is p2
@patch("openai.OpenAI")
def test_create_client_google_default_base_url(self, mock_openai_cls: MagicMock) -> None:
from turnstone.core.providers import create_client
from turnstone.core.providers._google import GOOGLE_DEFAULT_BASE_URL
mock_openai_cls.return_value = MagicMock()
create_client("google", base_url="", api_key="test-key")
mock_openai_cls.assert_called_once_with(
base_url=GOOGLE_DEFAULT_BASE_URL, api_key="test-key"
)
@patch("openai.OpenAI")
def test_create_client_google_custom_base_url(self, mock_openai_cls: MagicMock) -> None:
from turnstone.core.providers import create_client
mock_openai_cls.return_value = MagicMock()
create_client("google", base_url="http://custom:8080/v1", api_key="k")
mock_openai_cls.assert_called_once_with(base_url="http://custom:8080/v1", api_key="k")
def test_google_capabilities_defaults(self) -> None:
from turnstone.core.providers import create_provider
provider = create_provider("google")
caps = provider.get_capabilities("gemini-2.5-pro")
assert caps.context_window == 2_000_000
assert caps.max_output_tokens == 65_536
assert caps.token_param == "max_tokens"
assert caps.supports_temperature is True
assert caps.supports_vision is True
def test_google_capabilities_same_for_all_models(self) -> None:
from turnstone.core.providers import create_provider
provider = create_provider("google")
c1 = provider.get_capabilities("gemini-2.5-pro")
c2 = provider.get_capabilities("gemini-2.0-flash")
c3 = provider.get_capabilities("")
assert c1 is c2 is c3
def test_list_known_models_google_empty(self) -> None:
from turnstone.core.providers import list_known_models
assert list_known_models("google") == []
def test_lookup_model_capabilities_google_returns_none(self) -> None:
from turnstone.core.providers import lookup_model_capabilities
assert lookup_model_capabilities("google", "gemini-2.5-pro") is None
def test_resolve_openai_provider_googleapis(self) -> None:
from turnstone.core.model_registry import _resolve_openai_provider
assert (
_resolve_openai_provider(
"openai",
"https://generativelanguage.googleapis.com/v1beta/openai/",
)
== "google"
)
def test_resolve_openai_provider_not_spoofable(self) -> None:
from turnstone.core.model_registry import _resolve_openai_provider
# evil-googleapis.com must NOT match — requires the dot prefix
assert (
_resolve_openai_provider("openai", "https://evil-googleapis.com/v1")
== "openai-compatible"
)
def test_resolve_openai_provider_api_openai_unchanged(self) -> None:
from turnstone.core.model_registry import _resolve_openai_provider
assert _resolve_openai_provider("openai", "https://api.openai.com/v1") == "openai"
# ===========================================================================
# Google provider fidelity
# ===========================================================================
class TestGoogleProviderFidelity:
"""Tests for thought_signature round-trip via provider_blocks."""
def test_prepare_messages_strips_provider_content(self) -> None:
from turnstone.core.providers._google import GoogleProvider
prov = GoogleProvider()
msgs = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}},
],
"_provider_content": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
"thought_signature": "sig123",
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
]
cleaned = prov._prepare_messages(msgs)
# _provider_content must be stripped
for m in cleaned:
assert "_provider_content" not in m
# tool_calls must be reconstructed with thought_signature
tc = cleaned[0]["tool_calls"][0]
assert tc["thought_signature"] == "sig123"
def test_prepare_messages_passthrough_without_provider_content(self) -> None:
from turnstone.core.providers._google import GoogleProvider
prov = GoogleProvider()
msgs = [
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi"},
]
cleaned = prov._prepare_messages(msgs)
assert len(cleaned) == 2
assert cleaned[0]["content"] == "hello"
def test_non_streaming_captures_provider_blocks(self) -> None:
from turnstone.core.providers._google import GoogleProvider
prov = GoogleProvider()
# Build a mock response with thought_signature in __pydantic_extra__
mock_tc = MagicMock()
mock_tc.id = "c1"
mock_tc.function.name = "write_file"
mock_tc.function.arguments = '{"path":"test.txt"}'
mock_tc.model_dump.return_value = {
"id": "c1",
"type": "function",
"function": {"name": "write_file", "arguments": '{"path":"test.txt"}'},
"thought_signature": "sig_abc",
}
mock_msg = MagicMock()
mock_msg.tool_calls = [mock_tc]
mock_msg.content = ""
mock_msg.annotations = None
mock_choice = MagicMock()
mock_choice.message = mock_msg
mock_choice.finish_reason = "tool_calls"
mock_response = MagicMock()
mock_response.choices = [mock_choice]
mock_response.usage = None
mock_client = MagicMock()
mock_client.chat.completions.create.return_value = mock_response
result = prov.create_completion(
client=mock_client,
model="gemini-2.5-pro",
messages=[{"role": "user", "content": "test"}],
)
# Normalised tool_calls should NOT have thought_signature
assert result.tool_calls is not None
assert "thought_signature" not in result.tool_calls[0]
# provider_blocks should have the raw dict WITH thought_signature
assert len(result.provider_blocks) == 1
assert result.provider_blocks[0]["thought_signature"] == "sig_abc"
def test_prepare_messages_base_class_unchanged(self) -> None:
"""Base class _prepare_messages just calls sanitize_messages."""
from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider
prov = OpenAIChatCompletionsProvider()
msgs = [
{"role": "assistant", "content": None}, # should get content=""
{"role": "user", "content": "hi"},
]
cleaned = prov._prepare_messages(msgs)
assert cleaned[0]["content"] == ""
def test_streaming_captures_thought_signature(self) -> None:
"""Streaming _iter_stream taps raw deltas and emits provider_blocks."""
from turnstone.core.providers._google import GoogleProvider
prov = GoogleProvider()
# Build a minimal mock stream with 2 chunks:
# chunk 1: tool call header with thought_signature
# chunk 2: finish reason
mock_fn = MagicMock()
mock_fn.name = "write_file"
mock_fn.arguments = '{"path":"test.txt"}'
mock_tc_delta = MagicMock()
mock_tc_delta.index = 0
mock_tc_delta.id = "call_abc"
mock_tc_delta.function = mock_fn
mock_tc_delta.__pydantic_extra__ = {"thought_signature": "sig_stream"}
mock_delta1 = MagicMock()
mock_delta1.content = None
mock_delta1.tool_calls = [mock_tc_delta]
mock_delta1.annotations = None
# reasoning fields
mock_delta1.reasoning = None
mock_delta1.reasoning_content = None
mock_choice1 = MagicMock()
mock_choice1.finish_reason = None
mock_choice1.delta = mock_delta1
mock_chunk1 = MagicMock()
mock_chunk1.choices = [mock_choice1]
mock_chunk1.usage = None
# Finish chunk
mock_delta2 = MagicMock()
mock_delta2.content = None
mock_delta2.tool_calls = None
mock_delta2.annotations = None
mock_delta2.reasoning = None
mock_delta2.reasoning_content = None
mock_choice2 = MagicMock()
mock_choice2.finish_reason = "tool_calls"
mock_choice2.delta = mock_delta2
mock_chunk2 = MagicMock()
mock_chunk2.choices = [mock_choice2]
mock_chunk2.usage = None
chunks = list(prov._iter_stream([mock_chunk1, mock_chunk2]))
# Find the chunk with finish_reason
finish_chunks = [c for c in chunks if c.finish_reason]
assert len(finish_chunks) == 1
fc = finish_chunks[0]
assert len(fc.provider_blocks) == 1
assert fc.provider_blocks[0]["thought_signature"] == "sig_stream"
assert fc.provider_blocks[0]["id"] == "call_abc"
assert fc.provider_blocks[0]["function"]["name"] == "write_file"
def test_base_extract_tool_calls_returns_empty_provider_blocks(self) -> None:
"""Base class _extract_tool_calls returns empty provider_blocks."""
from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider
prov = OpenAIChatCompletionsProvider()
mock_tc = MagicMock()
mock_tc.id = "c1"
mock_tc.function.name = "test"
mock_tc.function.arguments = "{}"
tool_calls, provider_blocks = prov._extract_tool_calls([mock_tc])
assert len(tool_calls) == 1
assert provider_blocks == []
# ===========================================================================
# TestDataclasses
# ===========================================================================
class TestDataclasses:
"""Tests for protocol dataclass construction and defaults."""
def test_stream_chunk_defaults(self) -> None:
sc = StreamChunk()
assert sc.content_delta == ""
assert sc.reasoning_delta == ""
assert sc.tool_call_deltas == []
assert sc.usage is None
assert sc.finish_reason is None
assert sc.is_first is False
def test_tool_call_delta_defaults(self) -> None:
tcd = ToolCallDelta(index=0)
assert tcd.index == 0
assert tcd.id == ""
assert tcd.name == ""
assert tcd.arguments_delta == ""
def test_usage_info(self) -> None:
u = UsageInfo(prompt_tokens=10, completion_tokens=5, total_tokens=15)
assert u.prompt_tokens == 10
assert u.completion_tokens == 5
assert u.total_tokens == 15
def test_completion_result_defaults(self) -> None:
cr = CompletionResult(content="hello")
assert cr.content == "hello"
assert cr.tool_calls is None
assert cr.finish_reason == "stop"
assert cr.usage is None
def test_stream_chunk_info_delta_default(self) -> None:
sc = StreamChunk()
assert sc.info_delta == ""
def test_model_capabilities_web_search_default(self) -> None:
from turnstone.core.providers._protocol import ModelCapabilities
caps = ModelCapabilities()
assert caps.supports_web_search is False
# ===========================================================================
# TestParameterGating — model capability parameter gating
# ===========================================================================
class TestOpenAIParameterGating:
"""Verify _apply_model_params gates temperature and reasoning_effort correctly."""
def setup_method(self) -> None:
self.provider = OpenAIProvider()
def test_unknown_model_no_reasoning_effort(self) -> None:
"""Unknown/local models should NOT receive top-level reasoning_effort."""
caps = self.provider.get_capabilities("my-local-model")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium")
assert "reasoning_effort" not in kwargs
assert kwargs["temperature"] == 0.7
def test_gpt5_no_temperature_has_reasoning_effort(self) -> None:
"""GPT-5 base: no temperature, reasoning_effort sent."""
caps = self.provider.get_capabilities("gpt-5")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "high"
def test_gpt51_temperature_when_effort_none(self) -> None:
"""GPT-5.1: temperature only when reasoning_effort='none'."""
caps = self.provider.get_capabilities("gpt-5.1")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none")
assert kwargs["temperature"] == 0.7
assert "reasoning_effort" not in kwargs # "none" is skipped
def test_gpt51_no_temperature_when_reasoning_active(self) -> None:
"""GPT-5.1: no temperature when reasoning is active."""
caps = self.provider.get_capabilities("gpt-5.1")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "high"
def test_o_series_no_temperature_no_reasoning_effort(self) -> None:
"""O-series: no temperature, no reasoning_effort."""
caps = self.provider.get_capabilities("o3")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium")
assert "temperature" not in kwargs
assert "reasoning_effort" not in kwargs
def test_gpt5_pro_unsupported_effort_falls_back(self) -> None:
"""GPT-5 pro only supports 'high'; unsupported values fall back to default."""
caps = self.provider.get_capabilities("gpt-5-pro")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "high" # fell back to default
def test_gpt5_pro_supported_effort_passes_through(self) -> None:
"""GPT-5 pro accepts 'high' directly."""
caps = self.provider.get_capabilities("gpt-5-pro")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high")
assert kwargs["reasoning_effort"] == "high"
def test_gpt54_1m_context_and_effort(self) -> None:
"""GPT-5.4: 1M context, temperature when effort=none, xhigh supported."""
caps = self.provider.get_capabilities("gpt-5.4")
assert caps.context_window == 1050000
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none")
assert kwargs["temperature"] == 0.7
assert "reasoning_effort" not in kwargs
kwargs2: dict[str, Any] = {}
apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh")
assert "temperature" not in kwargs2
assert kwargs2["reasoning_effort"] == "xhigh"
def test_gpt54_pro_no_temperature_always_reasoning(self) -> None:
"""GPT-5.4 pro: no temperature, medium/high/xhigh only."""
caps = self.provider.get_capabilities("gpt-5.4-pro")
assert caps.context_window == 1050000
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "medium" # fell back from unsupported "low"
def test_gpt55_1m_context_and_effort(self) -> None:
"""GPT-5.5: 1M context, temperature when effort=none, xhigh supported."""
caps = self.provider.get_capabilities("gpt-5.5")
assert caps.context_window == 1050000
assert caps.supports_tool_search is True
assert caps.supports_vision is True
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none")
assert kwargs["temperature"] == 0.7
assert "reasoning_effort" not in kwargs
kwargs2: dict[str, Any] = {}
apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh")
assert "temperature" not in kwargs2
assert kwargs2["reasoning_effort"] == "xhigh"
def test_gpt55_pro_no_temperature_always_reasoning(self) -> None:
"""GPT-5.5 pro: no temperature, medium/high/xhigh only."""
caps = self.provider.get_capabilities("gpt-5.5-pro")
assert caps.context_window == 1050000
assert caps.supports_tool_search is True
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "medium" # fell back from unsupported "low"
class TestAnthropicOrphanedToolUse:
"""Verify _convert_messages synthesizes tool_results for orphaned tool_use."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_orphaned_tool_use_gets_synthetic_result(self) -> None:
"""Assistant has tool_calls but next message is user (no tool results)."""
messages = [
{"role": "user", "content": "do something"},
{
"role": "assistant",
"content": "I'll run that.",
"tool_calls": [
{
"id": "call_abc",
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
}
],
},
{"role": "user", "content": "never mind, do something else"},
]
_, converted = self.provider._convert_messages(repair_wire_messages(messages))
# Should have: user, assistant(tool_use), user(synthetic tool_result), user
# After _merge_consecutive, the two user messages may merge.
# Find the synthetic tool_result
tool_results = []
for msg in converted:
if msg["role"] == "user" and isinstance(msg["content"], list):
for block in msg["content"]:
if isinstance(block, dict) and block.get("type") == "tool_result":
tool_results.append(block)
assert len(tool_results) == 1
assert tool_results[0]["tool_use_id"] == "call_abc"
assert tool_results[0]["is_error"] is True
assert "cancelled" in tool_results[0]["content"].lower()
def test_multiple_orphaned_tool_calls(self) -> None:
"""Assistant has 3 tool_calls, none have results."""
messages = [
{"role": "user", "content": "do three things"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "c1", "function": {"name": "bash", "arguments": "{}"}},
{"id": "c2", "function": {"name": "read_file", "arguments": "{}"}},
{"id": "c3", "function": {"name": "write_file", "arguments": "{}"}},
],
},
{"role": "user", "content": "skip all that"},
]
_, converted = self.provider._convert_messages(repair_wire_messages(messages))
tool_results = []
for msg in converted:
if msg["role"] == "user" and isinstance(msg["content"], list):
for block in msg["content"]:
if isinstance(block, dict) and block.get("type") == "tool_result":
tool_results.append(block)
assert len(tool_results) == 3
result_ids = {r["tool_use_id"] for r in tool_results}
assert result_ids == {"c1", "c2", "c3"}
def test_partial_results_only_orphans_synthesized(self) -> None:
"""2 tool_calls, only 1 has a result — synthesize for the missing one."""
messages = [
{"role": "user", "content": "do two things"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "c1", "function": {"name": "bash", "arguments": "{}"}},
{"id": "c2", "function": {"name": "write_file", "arguments": "{}"}},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "file1.txt"},
{"role": "user", "content": "skip the write"},
]
_, converted = self.provider._convert_messages(repair_wire_messages(messages))
# c1 should have a real result, c2 should have a synthetic one
tool_results = []
for msg in converted:
if msg["role"] == "user" and isinstance(msg["content"], list):
for block in msg["content"]:
if isinstance(block, dict) and block.get("type") == "tool_result":
tool_results.append(block)
# Real result should come before synthetic (ordering matters for Anthropic)
assert len(tool_results) == 2
assert tool_results[0]["tool_use_id"] == "c1"
assert tool_results[0]["content"] == "file1.txt" # real result
assert tool_results[0].get("is_error") is not True
assert tool_results[1]["tool_use_id"] == "c2"
assert tool_results[1]["is_error"] is True # synthetic
def test_complete_results_no_synthesis(self) -> None:
"""All tool_calls have results — no synthesis needed."""
messages = [
{"role": "user", "content": "do it"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "c1", "function": {"name": "bash", "arguments": "{}"}},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "done"},
{"role": "user", "content": "thanks"},
]
_, converted = self.provider._convert_messages(messages)
# No synthetic results — only the real one (no is_error flag)
tool_results = []
for msg in converted:
if msg["role"] == "user" and isinstance(msg["content"], list):
for block in msg["content"]:
if isinstance(block, dict) and block.get("type") == "tool_result":
tool_results.append(block)
assert len(tool_results) == 1
assert tool_results[0]["tool_use_id"] == "c1"
assert tool_results[0].get("is_error") is not True
def test_trailing_orphan(self) -> None:
"""Orphaned tool_use at end of conversation (no following messages)."""
messages = [
{"role": "user", "content": "do it"},
{
"role": "assistant",
"content": "Running...",
"tool_calls": [
{"id": "c1", "function": {"name": "bash", "arguments": "{}"}},
],
},
]
_, converted = self.provider._convert_messages(repair_wire_messages(messages))
tool_results = []
for msg in converted:
if msg["role"] == "user" and isinstance(msg["content"], list):
for block in msg["content"]:
if isinstance(block, dict) and block.get("type") == "tool_result":
tool_results.append(block)
assert len(tool_results) == 1
assert tool_results[0]["tool_use_id"] == "c1"
assert tool_results[0]["is_error"] is True
def test_provider_content_orphan(self) -> None:
"""Orphaned tool_use inside _provider_content (Anthropic raw blocks)."""
messages = [
{"role": "user", "content": "run something"},
{
"role": "assistant",
"content": "Running...",
"_provider_content": [
{"type": "text", "text": "Running..."},
{
"type": "tool_use",
"id": "toolu_abc",
"name": "bash",
"input": {"command": "sleep 30"},
},
],
"tool_calls": [
{
"id": "toolu_abc",
"function": {"name": "bash", "arguments": '{"command": "sleep 30"}'},
},
],
},
{"role": "user", "content": "never mind"},
]
_, converted = self.provider._convert_messages(repair_wire_messages(messages))
# Should synthesize a tool_result for the orphaned tool_use in provider_content
tool_results = []
for msg in converted:
if msg["role"] == "user" and isinstance(msg["content"], list):
for block in msg["content"]:
if isinstance(block, dict) and block.get("type") == "tool_result":
tool_results.append(block)
assert len(tool_results) == 1
assert tool_results[0]["tool_use_id"] == "toolu_abc"
assert tool_results[0]["is_error"] is True
class TestAnthropicReasoningNone:
"""Verify 'none' effort disables thinking for manual-thinking models."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_none_effort_disables_thinking(self) -> None:
result = self.provider._reasoning_params("none", None, max_tokens=4096)
assert result == {}
def test_empty_effort_disables_thinking(self) -> None:
result = self.provider._reasoning_params("", None, max_tokens=4096)
assert result == {}
def test_low_effort_enables_thinking(self) -> None:
result = self.provider._reasoning_params("low", None, max_tokens=4096)
assert "thinking" in result
assert result["thinking"]["budget_tokens"] == 1024
def test_map_xhigh_effort(self) -> None:
from turnstone.core.providers._anthropic import _map_reasoning_to_effort
result = _map_reasoning_to_effort("xhigh", ("low", "medium", "high", "xhigh", "max"))
assert result == "xhigh"
def test_map_xhigh_rejected_by_model_without_it(self) -> None:
from turnstone.core.providers._anthropic import _map_reasoning_to_effort
result = _map_reasoning_to_effort("xhigh", ("low", "medium", "high", "max"))
assert result is None
# ===========================================================================
# TestWebSearch — provider-native web search
# ===========================================================================
class TestAnthropicWebSearch:
"""Tests for Anthropic native web search tool injection and streaming."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_web_search_capability_flag(self) -> None:
"""All Anthropic models should support native web search."""
caps = self.provider.get_capabilities("claude-opus-4-6")
assert caps.supports_web_search is True
caps = self.provider.get_capabilities("claude-sonnet-4-6")
assert caps.supports_web_search is True
# Unknown models use default which also has web search
caps = self.provider.get_capabilities("claude-unknown-99")
assert caps.supports_web_search is True
def test_inject_web_search_replaces_function_tool(self) -> None:
"""web_search function tool should be replaced with native server-side tool."""
caps = self.provider.get_capabilities("claude-opus-4-6")
tools = [
{"name": "bash", "description": "Run bash", "input_schema": {"type": "object"}},
{"name": "web_search", "description": "Search web", "input_schema": {"type": "object"}},
]
result = self.provider._inject_web_search(tools, caps)
names = [t.get("name") for t in result]
assert "bash" in names
assert "web_search" in names
# The web_search entry should be the native tool, not the function tool
ws_tool = next(t for t in result if t.get("name") == "web_search")
from turnstone.core.providers._anthropic import _WEB_SEARCH_TOOL_TYPE
assert ws_tool["type"] == _WEB_SEARCH_TOOL_TYPE
assert "input_schema" not in ws_tool
def test_inject_web_search_no_op_without_tool(self) -> None:
"""If no web_search tool in list, no injection happens."""
caps = self.provider.get_capabilities("claude-opus-4-6")
tools = [
{"name": "bash", "description": "Run bash", "input_schema": {"type": "object"}},
]
result = self.provider._inject_web_search(tools, caps)
assert result is tools # Unchanged
def test_streaming_server_tool_use_emits_search_info(self) -> None:
"""server_tool_use block should emit info_delta with search query."""
events = [
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_123",
block_name="web_search",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"query": "python web frameworks"}',
index=0,
),
_anthropic_event("content_block_stop", index=0),
]
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 1
assert "python web frameworks" in info_chunks[0].info_delta
assert "Searching" in info_chunks[0].info_delta
def test_streaming_web_search_result_emits_count(self) -> None:
"""web_search_tool_result block should emit result count info."""
# Build mock search results
result1 = MagicMock()
result1.type = "web_search_result"
result2 = MagicMock()
result2.type = "web_search_result"
events = [
_anthropic_event(
"content_block_start",
block_type="web_search_tool_result",
index=1,
),
]
# Set up the content attribute with search results
events[0].content_block.content = [result1, result2]
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 1
assert "Found 2 results" in info_chunks[0].info_delta
def test_streaming_web_search_error_emits_info(self) -> None:
"""web_search_tool_result with error should emit error info."""
error_content = MagicMock()
error_content.type = "web_search_tool_result_error"
error_content.error_code = "too_many_requests"
events = [
_anthropic_event(
"content_block_start",
block_type="web_search_tool_result",
index=1,
),
]
events[0].content_block.content = error_content
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 1
assert "too_many_requests" in info_chunks[0].info_delta
def test_streaming_server_tool_use_not_emitted_as_tool_call(self) -> None:
"""server_tool_use should NOT produce tool_call_deltas (it's server-side)."""
events = [
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_123",
block_name="web_search",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"query": "test"}',
index=0,
),
]
chunks = list(self.provider._iter_anthropic_stream(events))
tool_chunks = [c for c in chunks if c.tool_call_deltas]
assert len(tool_chunks) == 0
def test_streaming_mixed_text_and_search(self) -> None:
"""Full sequence: text + server search + results + more text."""
events = [
# Initial text
_anthropic_event(
"content_block_start",
block_type="text",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="text_delta",
text="Let me search.",
index=0,
),
# Server tool use
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_1",
block_name="web_search",
index=1,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"query": "test query"}',
index=1,
),
_anthropic_event("content_block_stop", index=1),
# Response text
_anthropic_event(
"content_block_start",
block_type="text",
index=3,
),
_anthropic_event(
"content_block_delta",
delta_type="text_delta",
text="Based on the results...",
index=3,
),
# Finish
_anthropic_event("message_delta", stop_reason="end_turn"),
]
chunks = list(self.provider._iter_anthropic_stream(events))
text_chunks = [c for c in chunks if c.content_delta]
info_chunks = [c for c in chunks if c.info_delta]
assert len(text_chunks) == 2
assert text_chunks[0].content_delta == "Let me search."
assert text_chunks[1].content_delta == "Based on the results..."
assert len(info_chunks) == 1
assert "test query" in info_chunks[0].info_delta
def test_pause_turn_normalized_to_stop(self) -> None:
"""pause_turn stop reason should normalize to 'stop'."""
from turnstone.core.providers._anthropic import _normalize_finish_reason
assert _normalize_finish_reason("pause_turn") == "stop"
def test_completion_skips_server_blocks(self) -> None:
"""create_completion should skip server_tool_use and web_search_tool_result."""
# Build mock response with mixed block types
text_block = MagicMock()
text_block.type = "text"
text_block.text = "Here are the results."
server_tu_block = MagicMock()
server_tu_block.type = "server_tool_use"
search_result_block = MagicMock()
search_result_block.type = "web_search_tool_result"
response = MagicMock()
response.content = [server_tu_block, search_result_block, text_block]
response.stop_reason = "end_turn"
response.usage.input_tokens = 100
response.usage.output_tokens = 50
response.usage.cache_creation_input_tokens = 0
response.usage.cache_read_input_tokens = 0
client = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.get_final_message.return_value = response
client.messages.stream.return_value = stream_ctx
with patch("turnstone.core.providers._anthropic._ensure_anthropic"):
result = self.provider.create_completion(
client=client,
model="claude-opus-4-6",
messages=[{"role": "user", "content": "search test"}],
)
assert result.content == "Here are the results."
assert result.tool_calls is None
def test_streaming_multiple_searches(self) -> None:
"""Multiple server_tool_use blocks in one response should each emit info."""
events = [
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_1",
block_name="web_search",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"query": "first search"}',
index=0,
),
_anthropic_event("content_block_stop", index=0),
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_2",
block_name="web_search",
index=2,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"query": "second search"}',
index=2,
),
_anthropic_event("content_block_stop", index=2),
]
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 2
assert "first search" in info_chunks[0].info_delta
assert "second search" in info_chunks[1].info_delta
def test_streaming_interleaved_tool_use_and_server_tool_use(self) -> None:
"""Regular tool_use and server_tool_use at different indices."""
events = [
# Regular tool call at index 0
_anthropic_event(
"content_block_start",
block_type="tool_use",
block_id="toolu_1",
block_name="bash",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"command": "ls"}',
index=0,
),
# Server tool at index 1
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_1",
block_name="web_search",
index=1,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json='{"query": "test"}',
index=1,
),
_anthropic_event("content_block_stop", index=1),
]
chunks = list(self.provider._iter_anthropic_stream(events))
tool_chunks = [c for c in chunks if c.tool_call_deltas]
info_chunks = [c for c in chunks if c.info_delta]
# Regular tool_use should produce tool_call_deltas
assert len(tool_chunks) == 2 # start + delta
assert tool_chunks[0].tool_call_deltas[0].name == "bash"
# Server tool_use should produce info_delta only
assert len(info_chunks) == 1
assert "test" in info_chunks[0].info_delta
def test_streaming_malformed_server_tool_json(self) -> None:
"""Malformed JSON in server tool input should emit fallback info."""
events = [
_anthropic_event(
"content_block_start",
block_type="server_tool_use",
block_id="srvtoolu_1",
block_name="web_search",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="input_json_delta",
partial_json="{bad json",
index=0,
),
_anthropic_event("content_block_stop", index=0),
]
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 1
assert info_chunks[0].info_delta == "[Searching...]"
def test_web_search_result_empty_list(self) -> None:
"""Empty search results list should report 0 results."""
events = [
_anthropic_event(
"content_block_start",
block_type="web_search_tool_result",
index=0,
),
]
events[0].content_block.content = []
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 1
assert "Found 0 results" in info_chunks[0].info_delta
def test_content_block_stop_for_text_block_no_spurious_info(self) -> None:
"""content_block_stop for a text block should not emit info_delta."""
events = [
_anthropic_event("content_block_start", block_type="text", index=0),
_anthropic_event(
"content_block_delta",
delta_type="text_delta",
text="hello",
index=0,
),
_anthropic_event("content_block_stop", index=0),
]
chunks = list(self.provider._iter_anthropic_stream(events))
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 0
class TestOpenAIWebSearch:
"""Tests for OpenAI native web search with search models."""
def setup_method(self) -> None:
self.provider = OpenAIProvider()
def test_search_model_capability(self) -> None:
"""Search models should have supports_web_search=True."""
caps = self.provider.get_capabilities("gpt-5-search-api")
assert caps.supports_web_search is True
def test_non_search_model_no_web_search(self) -> None:
"""Regular models should not have supports_web_search."""
caps = self.provider.get_capabilities("gpt-5")
assert caps.supports_web_search is False
caps = self.provider.get_capabilities("gpt-5.2")
assert caps.supports_web_search is False
def test_apply_web_search_injects_options(self) -> None:
"""For search models, web_search_options should be added to kwargs."""
caps = self.provider.get_capabilities("gpt-5-search-api")
kwargs: dict[str, Any] = {"model": "gpt-5-search-api"}
tools: list[dict[str, Any]] = [
{"type": "function", "function": {"name": "bash", "description": "Run bash"}},
{"type": "function", "function": {"name": "web_search", "description": "Search"}},
]
result = self.provider._apply_web_search(kwargs, caps, tools)
# web_search_options should be in kwargs
assert "web_search_options" in kwargs
# web_search tool should be removed
assert result is not None
names = [t["function"]["name"] for t in result]
assert "web_search" not in names
assert "bash" in names
def test_apply_web_search_no_op_for_regular_models(self) -> None:
"""For non-search models, no web_search_options, tools unchanged."""
caps = self.provider.get_capabilities("gpt-5")
kwargs: dict[str, Any] = {"model": "gpt-5"}
tools: list[dict[str, Any]] = [
{"type": "function", "function": {"name": "web_search", "description": "Search"}},
]
result = self.provider._apply_web_search(kwargs, caps, tools)
assert "web_search_options" not in kwargs
assert result is tools # Unchanged
def test_apply_web_search_returns_none_when_only_web_search(self) -> None:
"""If web_search was the only tool, return None after removing it."""
caps = self.provider.get_capabilities("gpt-5-search-api")
kwargs: dict[str, Any] = {}
tools: list[dict[str, Any]] = [
{"type": "function", "function": {"name": "web_search", "description": "Search"}},
]
result = self.provider._apply_web_search(kwargs, caps, tools)
assert result is None
def test_format_citations_appends_sources(self) -> None:
"""url_citation annotations should be formatted as footnote sources."""
ann = MagicMock()
ann.type = "url_citation"
citation = MagicMock()
citation.title = "Example Page"
citation.url = "https://example.com"
ann.url_citation = citation
content = "Some search result text."
result = format_citations(content, [ann])
assert "Sources:" in result
assert "[Example Page](https://example.com)" in result
def test_format_citations_deduplicates(self) -> None:
"""Duplicate URLs should not appear twice in sources."""
ann1 = MagicMock()
ann1.type = "url_citation"
ann1.url_citation = MagicMock(title="Page", url="https://example.com")
ann2 = MagicMock()
ann2.type = "url_citation"
ann2.url_citation = MagicMock(title="Page Again", url="https://example.com")
content = "Text."
result = format_citations(content, [ann1, ann2])
assert result.count("example.com") == 1
def test_format_citations_skips_non_url_citation(self) -> None:
"""Non-url_citation annotations should be ignored."""
ann = MagicMock()
ann.type = "something_else"
content = "Text."
result = format_citations(content, [ann])
assert "Sources:" not in result
def test_format_citations_empty_title(self) -> None:
"""Citation with empty title should show plain URL."""
ann = MagicMock()
ann.type = "url_citation"
ann.url_citation = MagicMock(title="", url="https://example.com")
result = format_citations("Text.", [ann])
assert "https://example.com" in result
# Should not have markdown link format when title is empty
assert "[](https://example.com)" not in result
def test_format_citations_none_citation(self) -> None:
"""Citation with None url_citation should be skipped."""
ann = MagicMock()
ann.type = "url_citation"
ann.url_citation = None
result = format_citations("Text.", [ann])
assert "Sources:" not in result
def test_apply_web_search_with_no_tools(self) -> None:
"""Search model with tools=None should still inject web_search_options."""
caps = self.provider.get_capabilities("gpt-5-search-api")
kwargs: dict[str, Any] = {}
result = self.provider._apply_web_search(kwargs, caps, None)
assert "web_search_options" in kwargs
assert result is None
def test_streaming_creates_with_web_search_options(self) -> None:
"""Streaming with a search model should pass web_search_options."""
client = MagicMock()
client.chat.completions.create.return_value = iter(
[
_openai_stream_chunk(content="Result text"),
]
)
list(
self.provider.create_streaming(
client=client,
model="gpt-5-search-api",
messages=[{"role": "user", "content": "search something"}],
tools=[
{
"type": "function",
"function": {"name": "web_search", "description": "Search"},
},
],
)
)
call_kwargs = client.chat.completions.create.call_args[1]
assert "web_search_options" in call_kwargs
# web_search tool should not be in the tools
assert "tools" not in call_kwargs or not any(
t.get("function", {}).get("name") == "web_search" for t in call_kwargs.get("tools", [])
)
def test_completion_with_annotations(self) -> None:
"""Non-streaming completion with search model should format citations."""
ann = MagicMock()
ann.type = "url_citation"
ann.url_citation = MagicMock(title="Test", url="https://test.com")
msg = MagicMock()
msg.content = "Found information."
msg.annotations = [ann]
msg.tool_calls = None
choice = MagicMock()
choice.message = msg
choice.finish_reason = "stop"
response = MagicMock()
response.choices = [choice]
response.usage.prompt_tokens = 50
response.usage.completion_tokens = 20
response.usage.total_tokens = 70
client = MagicMock()
client.chat.completions.create.return_value = response
result = self.provider.create_completion(
client=client,
model="gpt-5-search-api",
messages=[{"role": "user", "content": "search test"}],
)
assert "Found information." in result.content
assert "Sources:" in result.content
assert "[Test](https://test.com)" in result.content
def test_streaming_emits_citations_as_info_delta(self) -> None:
"""Streaming with search model should emit citations as final info_delta."""
ann = MagicMock()
ann.type = "url_citation"
ann.url_citation = MagicMock(title="Result", url="https://example.com")
# Content chunk, then a chunk with annotation, then finish
content_chunk = _openai_stream_chunk(content="Search result text.")
content_chunk.choices[0].delta.annotations = None
ann_chunk = _openai_stream_chunk(content=None)
ann_chunk.choices[0].delta.annotations = [ann]
finish_chunk = _openai_stream_chunk(finish_reason="stop")
finish_chunk.choices[0].delta.annotations = None
client = MagicMock()
client.chat.completions.create.return_value = iter([content_chunk, ann_chunk, finish_chunk])
chunks = list(
self.provider.create_streaming(
client=client,
model="gpt-5-search-api",
messages=[{"role": "user", "content": "search test"}],
)
)
info_chunks = [c for c in chunks if c.info_delta]
assert len(info_chunks) == 1
assert "Sources:" in info_chunks[0].info_delta
assert "[Result](https://example.com)" in info_chunks[0].info_delta
class TestClientSearchFallback:
"""Tests for the client-side web_search fallback when providers lack native search."""
def test_local_model_no_web_search(self) -> None:
"""Local/vLLM models should not have supports_web_search."""
provider = OpenAIProvider()
caps = provider.get_capabilities("my-local-model")
assert caps.supports_web_search is False
def test_web_search_tool_preserved_for_local_models(self) -> None:
"""For local models, web_search function tool stays in the tools list."""
provider = OpenAIProvider()
caps = provider.get_capabilities("llama-3-70b")
kwargs: dict[str, Any] = {}
tools = [
{"type": "function", "function": {"name": "web_search", "description": "Search"}},
]
result = provider._apply_web_search(kwargs, caps, tools)
assert result is tools
assert "web_search_options" not in kwargs
# ===========================================================================
# Anthropic provider_blocks / _provider_content round-trip tests
# ===========================================================================
class TestAnthropicProviderBlocks:
"""Tests for multi-turn web search content preservation."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_convert_messages_uses_provider_content(self) -> None:
"""Assistant message with _provider_content passes through verbatim."""
provider_content = [
{"type": "text", "text": "Here is what I found."},
{
"type": "server_tool_use",
"id": "stu_123",
"name": "web_search",
"input": {"query": "turnstone bird"},
},
{
"type": "web_search_tool_result",
"tool_use_id": "stu_123",
"content": [{"type": "web_search_result", "url": "https://example.com"}],
"encrypted_content": "abc123encrypted",
"encrypted_index": "idx456encrypted",
},
]
messages = [
{"role": "user", "content": "Search for turnstone bird"},
{
"role": "assistant",
"content": "Here is what I found.",
"_provider_content": provider_content,
},
{"role": "user", "content": "Tell me more"},
]
_, converted = self.provider._convert_messages(messages)
# The assistant message should use provider_content verbatim
assistant_msg = converted[1]
assert assistant_msg["role"] == "assistant"
assert assistant_msg["content"] is provider_content
assert assistant_msg["content"][2]["encrypted_content"] == "abc123encrypted"
def test_convert_messages_without_provider_content_unchanged(self) -> None:
"""Assistant message without _provider_content uses normal reconstruction."""
messages = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there"},
]
_, converted = self.provider._convert_messages(messages)
assistant_msg = converted[1]
assert assistant_msg["role"] == "assistant"
assert assistant_msg["content"] == [{"type": "text", "text": "Hi there"}]
def test_block_to_dict_with_model_dump(self) -> None:
"""_block_to_dict uses model_dump(exclude_none=True) when available."""
from turnstone.core.providers._anthropic import _block_to_dict
class FakeBlock:
def model_dump(self, **kwargs: Any) -> dict[str, Any]:
d = {"type": "text", "text": "hello", "extra": True, "nullable": None}
if kwargs.get("exclude_none"):
return {k: v for k, v in d.items() if v is not None}
return d
result = _block_to_dict(FakeBlock())
assert result == {"type": "text", "text": "hello", "extra": True}
assert "nullable" not in result
def test_block_to_dict_fallback(self) -> None:
"""_block_to_dict extracts known attributes as fallback."""
from turnstone.core.providers._anthropic import _block_to_dict
class FakeBlock:
type = "web_search_tool_result"
content = [{"type": "web_search_result"}]
encrypted_content = "enc123"
encrypted_index = "idx456"
result = _block_to_dict(FakeBlock())
assert result["type"] == "web_search_tool_result"
assert result["encrypted_content"] == "enc123"
assert result["encrypted_index"] == "idx456"
def test_streaming_captures_provider_blocks(self) -> None:
"""Streaming events produce provider_blocks on the final chunk."""
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
# Build mock stream events
events = []
# Text block
text_block = MagicMock()
text_block.type = "text"
text_block.text = ""
text_block.model_dump.return_value = {"type": "text", "text": ""}
events.append(MagicMock(type="content_block_start", index=0, content_block=text_block))
events.append(
MagicMock(
type="content_block_delta",
index=0,
delta=MagicMock(type="text_delta", text="Hello"),
)
)
events.append(MagicMock(type="content_block_stop", index=0))
# Server tool use block
stu_block = MagicMock()
stu_block.type = "server_tool_use"
stu_block.name = "web_search"
stu_block.model_dump.return_value = {
"type": "server_tool_use",
"id": "stu_1",
"name": "web_search",
"input": {},
}
events.append(MagicMock(type="content_block_start", index=1, content_block=stu_block))
events.append(
MagicMock(
type="content_block_delta",
index=1,
delta=MagicMock(type="input_json_delta", partial_json='{"query":"test"}'),
)
)
events.append(MagicMock(type="content_block_stop", index=1))
# Web search tool result block
wsr_block = MagicMock()
wsr_block.type = "web_search_tool_result"
wsr_block.model_dump.return_value = {
"type": "web_search_tool_result",
"tool_use_id": "stu_1",
"content": [{"type": "web_search_result", "url": "https://example.com"}],
"encrypted_content": "enc_data",
"encrypted_index": "idx_data",
}
# Make content iterable for count
fake_result = MagicMock()
fake_result.type = "web_search_result"
wsr_block.content = [fake_result]
events.append(MagicMock(type="content_block_start", index=2, content_block=wsr_block))
events.append(MagicMock(type="content_block_stop", index=2))
# Message delta with stop
msg_delta = MagicMock(type="message_delta")
msg_delta.delta = MagicMock(stop_reason="end_turn")
msg_delta.usage = MagicMock(input_tokens=100, output_tokens=50)
events.append(msg_delta)
chunks = list(provider._iter_anthropic_stream(iter(events)))
# Find the final chunk with provider_blocks
final_chunks = [c for c in chunks if c.provider_blocks]
assert len(final_chunks) == 1
blocks = final_chunks[0].provider_blocks
assert len(blocks) == 3
assert blocks[0]["type"] == "text"
assert blocks[1]["type"] == "server_tool_use"
assert blocks[1]["input"] == {"query": "test"} # parsed from accumulated JSON
assert blocks[2]["type"] == "web_search_tool_result"
assert blocks[2]["encrypted_content"] == "enc_data"
def test_streaming_thinking_block_captures_signature(self) -> None:
"""Streaming thinking block accumulates signature from signature_delta events."""
thinking_block = MagicMock()
thinking_block.type = "thinking"
thinking_block.model_dump.return_value = {
"type": "thinking",
"thinking": "",
"signature": "",
}
text_block = MagicMock()
text_block.type = "text"
text_block.model_dump.return_value = {"type": "text", "text": ""}
events = [
MagicMock(type="content_block_start", index=0, content_block=thinking_block),
_anthropic_event(
"content_block_delta", delta_type="thinking_delta", thinking="step 1", index=0
),
_anthropic_event(
"content_block_delta", delta_type="thinking_delta", thinking=" step 2", index=0
),
_anthropic_event(
"content_block_delta",
delta_type="signature_delta",
signature="sig_part1",
index=0,
),
_anthropic_event(
"content_block_delta",
delta_type="signature_delta",
signature="sig_part2",
index=0,
),
_anthropic_event("content_block_stop", index=0),
MagicMock(type="content_block_start", index=1, content_block=text_block),
_anthropic_event("content_block_delta", delta_type="text_delta", text="Hello", index=1),
_anthropic_event("content_block_stop", index=1),
_anthropic_event("message_delta", stop_reason="end_turn", usage_output_tokens=50),
]
chunks = list(self.provider._iter_anthropic_stream(iter(events)))
final_chunks = [c for c in chunks if c.provider_blocks]
assert len(final_chunks) == 1
blocks = final_chunks[0].provider_blocks
assert blocks[0]["type"] == "thinking"
assert blocks[0]["thinking"] == "step 1 step 2"
assert blocks[0]["signature"] == "sig_part1sig_part2"
def test_thinking_block_multiturn_roundtrip(self) -> None:
"""Thinking block with signature survives _convert_messages round-trip."""
provider_content = [
{
"type": "thinking",
"thinking": "Let me reason...",
"signature": "ErUBCkYIAxgCIkD_valid_sig",
},
{"type": "text", "text": "Here is my answer."},
]
messages = [
{"role": "user", "content": "Question"},
{
"role": "assistant",
"content": "Here is my answer.",
"_provider_content": provider_content,
},
{"role": "user", "content": "Follow up"},
]
_, converted = self.provider._convert_messages(messages)
assistant_msg = converted[1]
assert assistant_msg["content"] is provider_content
assert assistant_msg["content"][0]["signature"] == "ErUBCkYIAxgCIkD_valid_sig"
assert assistant_msg["content"][0]["type"] == "thinking"
def test_block_to_dict_preserves_thinking_signature(self) -> None:
"""_block_to_dict preserves signature on thinking blocks."""
from turnstone.core.providers._anthropic import _block_to_dict
class FakeThinkingBlock:
def model_dump(self, **kwargs: Any) -> dict[str, Any]:
return {
"type": "thinking",
"thinking": "reasoning...",
"signature": "abc123sig",
}
result = _block_to_dict(FakeThinkingBlock())
assert result["signature"] == "abc123sig"
# Also test fallback path (no model_dump)
class FallbackBlock:
type = "thinking"
thinking = "reasoning..."
signature = "abc123sig"
result2 = _block_to_dict(FallbackBlock())
assert result2["signature"] == "abc123sig"
# ---------------------------------------------------------------------------
# Tool search tests
# ---------------------------------------------------------------------------
class TestAnthropicToolSearch:
"""Test Anthropic provider tool search injection."""
@pytest.fixture()
def provider(self):
from turnstone.core.providers._anthropic import AnthropicProvider
return AnthropicProvider()
def test_tool_search_capability_flag(self, provider):
caps = provider.get_capabilities("claude-opus-4-6-20260101")
assert caps.supports_tool_search is True
def test_tool_search_not_supported_on_haiku(self, provider):
caps = provider.get_capabilities("claude-haiku-4-5-20251001")
assert caps.supports_tool_search is False
def test_inject_tool_search_marks_deferred(self, provider):
caps = provider.get_capabilities("claude-opus-4-6-20260101")
tools = [
{"name": "bash", "description": "Run commands", "input_schema": {}},
{
"name": "mcp__github__create_issue",
"description": "Create issue",
"input_schema": {},
},
]
deferred = frozenset(["mcp__github__create_issue"])
result = provider._inject_tool_search(tools, caps, deferred)
# bash should not be deferred
assert result[0].get("defer_loading") is None or result[0].get("defer_loading") is False
# MCP tool should be deferred
assert result[1]["defer_loading"] is True
# Search tool should be appended
assert result[-1]["type"] == "tool_search_tool_bm25"
assert result[-1]["name"] == "tool_search_tool_bm25"
def test_inject_tool_search_no_op_without_deferred(self, provider):
caps = provider.get_capabilities("claude-opus-4-6-20260101")
tools = [{"name": "bash", "description": "Run commands", "input_schema": {}}]
result = provider._inject_tool_search(tools, caps, None)
assert result == tools
def test_inject_tool_search_no_op_on_unsupported_model(self, provider):
caps = provider.get_capabilities("claude-haiku-4-5-20251001")
tools = [{"name": "bash", "description": "Run commands", "input_schema": {}}]
deferred = frozenset(["some_tool"])
result = provider._inject_tool_search(tools, caps, deferred)
assert result == tools
class TestOpenAIToolSearch:
"""Test OpenAI provider tool search injection."""
@pytest.fixture()
def provider(self):
return OpenAIProvider()
def test_tool_search_capability_on_gpt54(self, provider):
caps = provider.get_capabilities("gpt-5.4")
assert caps.supports_tool_search is True
def test_tool_search_not_supported_on_gpt5(self, provider):
caps = provider.get_capabilities("gpt-5")
assert caps.supports_tool_search is False
def test_apply_tool_search_marks_deferred(self, provider):
caps = provider.get_capabilities("gpt-5.4")
tools = [
{"type": "function", "function": {"name": "bash", "description": "Run commands"}},
{
"type": "function",
"function": {"name": "mcp__slack__send", "description": "Send message"},
},
]
deferred = frozenset(["mcp__slack__send"])
result = apply_tool_search(caps, tools, deferred)
assert result is not None
# bash not deferred
assert result[0].get("defer_loading") is None or result[0].get("defer_loading") is False
# slack tool deferred
assert result[1]["defer_loading"] is True
def test_apply_tool_search_no_op_without_deferred(self, provider):
caps = provider.get_capabilities("gpt-5.4")
tools = [
{"type": "function", "function": {"name": "bash", "description": "Run commands"}},
]
result = apply_tool_search(caps, tools, None)
assert result == tools
def test_apply_tool_search_no_op_on_unsupported_model(self, provider):
caps = provider.get_capabilities("gpt-5")
tools = [
{"type": "function", "function": {"name": "bash", "description": "Run commands"}},
]
deferred = frozenset(["some_tool"])
result = apply_tool_search(caps, tools, deferred)
assert result == tools
class TestModelCapabilitiesToolSearch:
"""Test supports_tool_search defaults and values."""
def test_default_is_false(self):
from turnstone.core.providers._protocol import ModelCapabilities
caps = ModelCapabilities()
assert caps.supports_tool_search is False
class TestMidConversationSystemCapability:
"""supports_mid_conversation_system — NextOpus (claude-opus-4-8) only."""
def test_default_is_false(self) -> None:
from turnstone.core.providers._protocol import ModelCapabilities
caps = ModelCapabilities()
assert caps.supports_mid_conversation_system is False
def test_opus_4_8_supports_it(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
for model in ("claude-opus-4-8", "claude-opus-4-8-20260601"):
caps = provider.get_capabilities(model)
assert caps.supports_mid_conversation_system is True, model
def test_other_claude_models_do_not(self) -> None:
"""Only NextOpus has it; older/other Claude models and the default off."""
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
for model in (
"claude-opus-4-7",
"claude-opus-4-6",
"claude-sonnet-4-6",
"claude-haiku-4-5",
"claude-opus-4-5",
"claude-unknown-9", # Anthropic default
):
caps = provider.get_capabilities(model)
assert caps.supports_mid_conversation_system is False, model
# ---------------------------------------------------------------------------
# Vision support
# ---------------------------------------------------------------------------
class TestVisionCapabilities:
"""Test supports_vision flag across providers."""
def test_default_is_false(self) -> None:
from turnstone.core.providers._protocol import ModelCapabilities
caps = ModelCapabilities()
assert caps.supports_vision is False
def test_openai_commercial_supports_vision(self) -> None:
provider = OpenAIProvider()
for model in ("gpt-5", "gpt-5-mini", "gpt-5.4", "o3", "o4-mini"):
caps = provider.get_capabilities(model)
assert caps.supports_vision is True, f"{model} should support vision"
def test_openai_default_no_vision(self) -> None:
"""Unknown models (local servers) default to no vision."""
provider = OpenAIProvider()
caps = provider.get_capabilities("some-local-model")
assert caps.supports_vision is False
def test_anthropic_supports_vision(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
for model in ("claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5"):
caps = provider.get_capabilities(model)
assert caps.supports_vision is True, f"{model} should support vision"
def test_anthropic_default_supports_vision(self) -> None:
"""Anthropic default (unknown Claude model) supports vision."""
from turnstone.core.providers._anthropic import AnthropicProvider
provider = AnthropicProvider()
caps = provider.get_capabilities("claude-unknown-9")
assert caps.supports_vision is True
class TestAnthropicVisionConversion:
"""Test image content conversion in _convert_messages."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_tool_result_with_image_content(self) -> None:
"""Tool result with list content converts image_url to Anthropic image."""
messages = [
{"role": "user", "content": "Read this image"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_1",
"function": {"name": "read_file", "arguments": '{"path": "img.png"}'},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [
{"type": "text", "text": "Image file: img.png (1024 bytes)"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo="},
},
],
},
]
_, converted = self.provider._convert_messages(messages)
# Tool result should be in a user message
tool_user_msg = converted[2]
assert tool_user_msg["role"] == "user"
tool_result = tool_user_msg["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["tool_use_id"] == "call_1"
# Content should be a list with converted image block
content = tool_result["content"]
assert isinstance(content, list)
assert content[0] == {"type": "text", "text": "Image file: img.png (1024 bytes)"}
assert content[1]["type"] == "image"
assert content[1]["source"]["type"] == "base64"
assert content[1]["source"]["media_type"] == "image/png"
assert content[1]["source"]["data"] == "iVBORw0KGgo="
def test_tool_result_with_string_content_unchanged(self) -> None:
"""Tool result with plain string content is unchanged."""
messages = [
{"role": "user", "content": "Read file"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_2",
"function": {"name": "read_file", "arguments": '{"path": "f.py"}'},
}
],
},
{
"role": "tool",
"tool_call_id": "call_2",
"content": " 1\tprint('hello')",
},
]
_, converted = self.provider._convert_messages(messages)
tool_result = converted[2]["content"][0]
assert tool_result["content"] == " 1\tprint('hello')"
def test_convert_content_parts_static_method(self) -> None:
"""_convert_content_parts handles both image_url and text."""
from turnstone.core.providers._anthropic import AnthropicProvider
parts = [
{"type": "text", "text": "description"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"},
},
]
result = AnthropicProvider._convert_content_parts(parts)
assert result[0] == {"type": "text", "text": "description"}
assert result[1]["type"] == "image"
assert result[1]["source"]["media_type"] == "image/jpeg"
assert result[1]["source"]["data"] == "/9j/4AAQ"
# ===========================================================================
# TestPromptCaching
# ===========================================================================
class TestAnthropicPromptCaching:
"""Tests for Anthropic prompt caching (cache_control)."""
def setup_method(self) -> None:
from turnstone.core.providers._anthropic import AnthropicProvider
self.provider = AnthropicProvider()
def test_cache_control_set_in_kwargs(self) -> None:
"""_build_thinking_and_kwargs includes cache_control: ephemeral."""
caps = self.provider.get_capabilities("claude-sonnet-4-6")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="medium",
extra_params=None,
max_tokens=4096,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="You are helpful.",
model="claude-sonnet-4-6",
tools=None,
)
assert "cache_control" in kwargs
assert kwargs["cache_control"] == {"type": "ephemeral"}
def test_opus_4_7_no_temperature_in_kwargs(self) -> None:
"""Opus 4.7 rejects temperature — must not appear in kwargs."""
caps = self.provider.get_capabilities("claude-opus-4-7")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="high",
extra_params=None,
max_tokens=8192,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="",
model="claude-opus-4-7",
tools=None,
)
assert "temperature" not in kwargs
def test_opus_4_6_still_has_temperature(self) -> None:
"""Opus 4.6 must still send temperature (regression guard)."""
caps = self.provider.get_capabilities("claude-opus-4-6")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="high",
extra_params=None,
max_tokens=8192,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="",
model="claude-opus-4-6",
tools=None,
)
assert "temperature" in kwargs
assert kwargs["temperature"] == 1.0 # forced for adaptive thinking
def test_opus_4_7_thinking_display_summarized(self) -> None:
"""Opus 4.7 must opt in to thinking display with 'summarized'."""
caps = self.provider.get_capabilities("claude-opus-4-7")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="high",
extra_params=None,
max_tokens=8192,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="",
model="claude-opus-4-7",
tools=None,
)
assert kwargs["thinking"] == {"type": "adaptive", "display": "summarized"}
def test_opus_4_6_thinking_no_display(self) -> None:
"""Opus 4.6 adaptive thinking should not include display key."""
caps = self.provider.get_capabilities("claude-opus-4-6")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="high",
extra_params=None,
max_tokens=8192,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="",
model="claude-opus-4-6",
tools=None,
)
assert kwargs["thinking"] == {"type": "adaptive"}
def test_opus_4_7_xhigh_effort(self) -> None:
"""Opus 4.7 xhigh effort passes through to output_config."""
caps = self.provider.get_capabilities("claude-opus-4-7")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="xhigh",
extra_params=None,
max_tokens=8192,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="",
model="claude-opus-4-7",
tools=None,
)
assert kwargs["output_config"] == {"effort": "xhigh"}
def test_xhigh_effort_not_applied_to_opus_4_6(self) -> None:
"""xhigh is not a valid effort level for Opus 4.6 — should be ignored."""
caps = self.provider.get_capabilities("claude-opus-4-6")
kwargs = self.provider._build_thinking_and_kwargs(
caps=caps,
reasoning_effort="xhigh",
extra_params=None,
max_tokens=8192,
temperature=0.5,
converted_msgs=[{"role": "user", "content": "hi"}],
system_prompt="",
model="claude-opus-4-6",
tools=None,
)
assert "output_config" not in kwargs
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_message_start_cache_metrics(self, mock_ensure: MagicMock) -> None:
"""Cache metrics from message_start flow into UsageInfo."""
msg_start = MagicMock()
msg_start.type = "message_start"
msg_usage = MagicMock()
msg_usage.input_tokens = 100
msg_usage.cache_creation_input_tokens = 80
msg_usage.cache_read_input_tokens = 0
msg_start.message = MagicMock()
msg_start.message.usage = msg_usage
text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi")
events = [msg_start, text_event]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "hi"}],
)
)
# prompt_tokens = input_tokens (100) + cache_creation (80) + cache_read (0) = 180
start_chunks = [r for r in results if r.usage is not None and r.usage.prompt_tokens == 180]
assert len(start_chunks) == 1
assert start_chunks[0].usage is not None
assert start_chunks[0].usage.cache_creation_tokens == 80
assert start_chunks[0].usage.cache_read_tokens == 0
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_message_delta_cache_metrics(self, mock_ensure: MagicMock) -> None:
"""Cache metrics from message_delta flow into UsageInfo."""
text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi")
delta_event = MagicMock()
delta_event.type = "message_delta"
delta_usage = MagicMock()
delta_usage.input_tokens = 0
delta_usage.output_tokens = 50
delta_usage.cache_creation_input_tokens = 0
delta_usage.cache_read_input_tokens = 120
delta_event.usage = delta_usage
delta_event.delta = MagicMock()
delta_event.delta.stop_reason = "end_turn"
events = [text_event, delta_event]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "hi"}],
)
)
delta_chunks = [r for r in results if r.finish_reason is not None]
assert len(delta_chunks) == 1
u = delta_chunks[0].usage
assert u is not None
assert u.cache_read_tokens == 120
assert u.cache_creation_tokens == 0
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_completion_cache_metrics(self, mock_ensure: MagicMock) -> None:
"""Non-streaming completion extracts cache metrics."""
response = MagicMock()
text_block = MagicMock()
text_block.type = "text"
text_block.text = "Hello"
response.content = [text_block]
response.stop_reason = "end_turn"
usage = MagicMock()
usage.input_tokens = 200
usage.output_tokens = 30
usage.cache_creation_input_tokens = 150
usage.cache_read_input_tokens = 50
response.usage = usage
client = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.get_final_message.return_value = response
client.messages.stream.return_value = stream_ctx
result = self.provider.create_completion(
client=client,
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "hi"}],
)
u = result.usage
assert u is not None
assert u.cache_creation_tokens == 150
assert u.cache_read_tokens == 50
@patch("turnstone.core.providers._anthropic._ensure_anthropic")
def test_streaming_cache_metrics_missing_gracefully(self, mock_ensure: MagicMock) -> None:
"""When cache attributes are absent, tokens default to 0."""
import types
msg_start = MagicMock()
msg_start.type = "message_start"
# SimpleNamespace with only input_tokens — no cache attributes at all
msg_usage = types.SimpleNamespace(input_tokens=50)
msg_start.message = MagicMock()
msg_start.message.usage = msg_usage
text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi")
events = [msg_start, text_event]
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=iter(events))
stream_ctx.__exit__ = MagicMock(return_value=False)
client = MagicMock()
client.messages.stream.return_value = stream_ctx
results = list(
self.provider.create_streaming(
client=client,
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "hi"}],
)
)
start_chunks = [r for r in results if r.usage is not None]
assert len(start_chunks) >= 1
u = start_chunks[0].usage
assert u is not None
assert u.cache_creation_tokens == 0
assert u.cache_read_tokens == 0
class TestOpenAIPromptCaching:
"""Tests for OpenAI prompt caching (automatic + extended retention)."""
def setup_method(self) -> None:
self.provider = OpenAIProvider()
def test_cache_retention_set_for_gpt5(self) -> None:
"""GPT-5.x models get prompt_cache_retention=24h."""
for model in (
"gpt-5",
"gpt-5.1",
"gpt-5.2",
"gpt-5.4",
"gpt-5.4-pro",
"gpt-5.5",
"gpt-5.5-pro",
"gpt-5-mini",
"gpt-5-pro",
):
kwargs: dict[str, Any] = {}
apply_cache_retention(kwargs, model)
assert kwargs.get("prompt_cache_retention") == "24h", f"Failed for {model}"
def test_cache_retention_not_set_for_non_gpt5(self) -> None:
"""Non-GPT-5 models do not get cache retention."""
for model in ("o3", "o4-mini", "local-model", "gpt-4o"):
kwargs: dict[str, Any] = {}
apply_cache_retention(kwargs, model)
assert "prompt_cache_retention" not in kwargs, f"Unexpected retention for {model}"
def test_streaming_cached_tokens_from_usage(self) -> None:
"""Streaming usage extracts cached_tokens from prompt_tokens_details."""
usage = MagicMock()
usage.prompt_tokens = 100
usage.completion_tokens = 20
usage.total_tokens = 120
ptd = MagicMock()
ptd.cached_tokens = 80
usage.prompt_tokens_details = ptd
chunks = [
_openai_stream_chunk(content="Hi"),
_openai_stream_chunk(empty_choices=True, usage=usage),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-5.1",
messages=[{"role": "user", "content": "hi"}],
)
)
usage_chunks = [r for r in results if r.usage is not None]
assert len(usage_chunks) == 1
u = usage_chunks[0].usage
assert u is not None
assert u.cache_read_tokens == 80
assert u.cache_creation_tokens == 0
def test_completion_cached_tokens(self) -> None:
"""Non-streaming completion extracts cached_tokens."""
response = MagicMock()
msg = MagicMock()
msg.content = "Hello"
msg.tool_calls = None
msg.annotations = None
choice = MagicMock()
choice.message = msg
choice.finish_reason = "stop"
response.choices = [choice]
usage = MagicMock()
usage.prompt_tokens = 200
usage.completion_tokens = 30
usage.total_tokens = 230
ptd = MagicMock()
ptd.cached_tokens = 150
usage.prompt_tokens_details = ptd
response.usage = usage
client = MagicMock()
client.chat.completions.create.return_value = response
result = self.provider.create_completion(
client=client,
model="gpt-5.1",
messages=[{"role": "user", "content": "hi"}],
)
u = result.usage
assert u is not None
assert u.cache_read_tokens == 150
assert u.cache_creation_tokens == 0
def test_streaming_no_prompt_tokens_details(self) -> None:
"""When prompt_tokens_details is absent, cache_read_tokens defaults to 0."""
usage = MagicMock()
usage.prompt_tokens = 100
usage.completion_tokens = 20
usage.total_tokens = 120
usage.prompt_tokens_details = None
chunks = [
_openai_stream_chunk(content="Hi"),
_openai_stream_chunk(empty_choices=True, usage=usage),
]
client = MagicMock()
client.chat.completions.create.return_value = iter(chunks)
results = list(
self.provider.create_streaming(
client=client,
model="gpt-5.1",
messages=[{"role": "user", "content": "hi"}],
)
)
usage_chunks = [r for r in results if r.usage is not None]
assert len(usage_chunks) == 1
u = usage_chunks[0].usage
assert u is not None
assert u.cache_read_tokens == 0
class TestUsageInfoCacheFields:
"""Tests for cache fields on UsageInfo dataclass."""
def test_default_cache_fields(self) -> None:
u = UsageInfo(prompt_tokens=10, completion_tokens=5, total_tokens=15)
assert u.cache_creation_tokens == 0
assert u.cache_read_tokens == 0
def test_explicit_cache_fields(self) -> None:
u = UsageInfo(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
cache_creation_tokens=80,
cache_read_tokens=20,
)
assert u.cache_creation_tokens == 80
assert u.cache_read_tokens == 20
class TestMetricsCacheTokens:
"""Tests for cache token recording in MetricsCollector."""
def test_record_cache_tokens(self) -> None:
from turnstone.core.metrics import MetricsCollector
m = MetricsCollector()
m.record_cache_tokens(100, 200)
m.record_cache_tokens(50, 300)
assert m._tokens["cache_creation"] == 150
assert m._tokens["cache_read"] == 500
def test_prometheus_output_includes_cache_tokens(self) -> None:
from turnstone.core.metrics import MetricsCollector
m = MetricsCollector()
m.record_tokens(1000, 500)
m.record_cache_tokens(800, 200)
text = m.generate_text(workstream_states={}, total_workstreams=0)
assert 'turnstone_tokens_total{type="cache_creation"} 800' in text
assert 'turnstone_tokens_total{type="cache_read"} 200' in text
assert 'turnstone_tokens_total{type="prompt"} 1000' in text
# ===========================================================================
# TestOpenAIResponsesProvider — Responses API provider
# ===========================================================================
class TestOpenAIResponsesProvider:
"""Tests for the OpenAI Responses API provider."""
def setup_method(self) -> None:
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
self.provider = OpenAIResponsesProvider()
def test_provider_name(self) -> None:
assert self.provider.provider_name == "openai"
def test_get_capabilities(self) -> None:
caps = self.provider.get_capabilities("gpt-5.4")
assert caps.context_window == 1050000
assert caps.supports_tool_search is True
class TestResponsesMessageConversion:
"""Tests for _convert_messages — Chat Completions format to Responses API."""
def setup_method(self) -> None:
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
self.provider = OpenAIResponsesProvider()
def test_system_message_to_instructions(self) -> None:
messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Hello"},
]
instructions, items = self.provider._convert_messages(messages)
assert instructions == "You are helpful."
assert len(items) == 1
assert items[0]["role"] == "user"
assert items[0]["content"] == "Hello"
def test_multiple_system_messages_concatenated(self) -> None:
messages = [
{"role": "system", "content": "Rule 1"},
{"role": "developer", "content": "Rule 2"},
{"role": "user", "content": "Hi"},
]
instructions, items = self.provider._convert_messages(messages)
assert instructions == "Rule 1\n\nRule 2"
assert len(items) == 1
def test_assistant_text_message(self) -> None:
messages = [
{"role": "assistant", "content": "Hello back"},
]
_, items = self.provider._convert_messages(messages)
assert len(items) == 1
assert items[0]["type"] == "message"
assert items[0]["role"] == "assistant"
assert items[0]["content"] == "Hello back"
def test_assistant_tool_calls(self) -> None:
messages = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"function": {"name": "read_file", "arguments": '{"path": "/tmp"}'},
}
],
},
]
_, items = self.provider._convert_messages(repair_wire_messages(messages))
# repair_wire_messages synthesizes the missing tool result; the translator renders it
assert len(items) == 2
assert items[0]["type"] == "function_call"
assert items[0]["call_id"] == "call_1"
assert items[0]["name"] == "read_file"
assert items[0]["arguments"] == '{"path": "/tmp"}'
assert items[1]["type"] == "function_call_output"
assert items[1]["call_id"] == "call_1"
def test_tool_result(self) -> None:
messages = [
{"role": "tool", "tool_call_id": "call_1", "content": "file contents"},
]
_, items = self.provider._convert_messages(messages)
assert len(items) == 1
assert items[0]["type"] == "function_call_output"
assert items[0]["call_id"] == "call_1"
assert items[0]["output"] == "file contents"
def test_provider_content_ignored_with_store_false(self) -> None:
"""With store=False, provider_content is ignored — rebuild from content."""
provider_items = [
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Hi"}],
},
{"type": "function_call", "call_id": "c1", "name": "f", "arguments": "{}"},
]
messages = [
{"role": "assistant", "content": "Hi", "_provider_content": provider_items},
]
_, items = self.provider._convert_messages(messages)
# Should rebuild from content, not passthrough provider_content
assert len(items) == 1
assert items[0]["type"] == "message"
assert items[0]["content"] == "Hi"
def test_no_system_returns_none_instructions(self) -> None:
messages = [{"role": "user", "content": "Hello"}]
instructions, _ = self.provider._convert_messages(messages)
assert instructions is None
def test_assistant_with_content_and_tool_calls(self) -> None:
"""Assistant message with both text and tool calls emits separate items."""
messages = [
{
"role": "assistant",
"content": "I'll read that file",
"tool_calls": [
{
"id": "call_1",
"function": {"name": "read_file", "arguments": '{"path": "/tmp"}'},
}
],
},
]
_, items = self.provider._convert_messages(repair_wire_messages(messages))
# repair_wire_messages synthesizes the missing tool result; the translator renders it
assert len(items) == 3
assert items[0]["type"] == "message"
assert items[0]["content"] == "I'll read that file"
assert items[1]["type"] == "function_call"
assert items[1]["name"] == "read_file"
assert items[2]["type"] == "function_call_output"
assert items[2]["call_id"] == "call_1"
class TestResponsesToolConversion:
"""Tests for _convert_tools — Chat Completions tool format to Responses API."""
def setup_method(self) -> None:
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
self.provider = OpenAIResponsesProvider()
def test_function_tool_conversion(self) -> None:
tools = [
{
"type": "function",
"function": {
"name": "read_file",
"description": "Read a file",
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}},
},
}
]
caps = ModelCapabilities()
result = self.provider._convert_tools(tools, caps)
assert result is not None
assert len(result) == 1
assert result[0]["type"] == "function"
assert result[0]["name"] == "read_file"
assert result[0]["description"] == "Read a file"
assert result[0]["strict"] is False
def test_web_search_replaced_with_native(self) -> None:
tools = [
{"type": "function", "function": {"name": "web_search", "description": "Search"}},
{"type": "function", "function": {"name": "read_file", "description": "Read"}},
]
caps = ModelCapabilities(supports_web_search=True)
result = self.provider._convert_tools(tools, caps)
assert result is not None
names = [t.get("name", t.get("type")) for t in result]
assert "web_search" in names # native web_search tool
assert "read_file" in names
def test_none_tools_returns_none(self) -> None:
caps = ModelCapabilities()
assert self.provider._convert_tools(None, caps) is None
def test_defer_loading_preserved(self) -> None:
tools = [
{"type": "function", "function": {"name": "f"}, "defer_loading": True},
]
caps = ModelCapabilities()
result = self.provider._convert_tools(tools, caps)
assert result is not None
assert result[0].get("defer_loading") is True
class TestResponsesParamBuilding:
"""Tests for _build_kwargs — parameter construction for Responses API."""
def setup_method(self) -> None:
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
self.provider = OpenAIResponsesProvider()
def test_reasoning_effort_as_dict(self) -> None:
kwargs = self.provider._build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=None,
max_tokens=4096,
temperature=0.5,
reasoning_effort="high",
deferred_names=None,
)
assert kwargs["reasoning"] == {"effort": "high"}
assert "reasoning_effort" not in kwargs
def test_no_reasoning_when_none_effort(self) -> None:
kwargs = self.provider._build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=None,
max_tokens=4096,
temperature=0.5,
reasoning_effort="none",
deferred_names=None,
)
assert "reasoning" not in kwargs
def test_store_is_false(self) -> None:
kwargs = self.provider._build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=None,
max_tokens=4096,
temperature=0.5,
reasoning_effort="medium",
deferred_names=None,
)
assert kwargs["store"] is False
def test_cache_retention_for_gpt5(self) -> None:
kwargs = self.provider._build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=None,
max_tokens=4096,
temperature=0.5,
reasoning_effort="medium",
deferred_names=None,
)
assert kwargs["prompt_cache_retention"] == "24h"
def test_instructions_from_system_messages(self) -> None:
kwargs = self.provider._build_kwargs(
model="gpt-5.4",
messages=[
{"role": "system", "content": "Be helpful"},
{"role": "user", "content": "Hi"},
],
tools=None,
max_tokens=4096,
temperature=0.5,
reasoning_effort="none",
deferred_names=None,
)
assert kwargs["instructions"] == "Be helpful"
def test_web_search_injected_with_no_tools(self) -> None:
"""Search-capable models get web_search tool even when tools=None."""
kwargs = self.provider._build_kwargs(
model="gpt-5-search-api",
messages=[{"role": "user", "content": "Hi"}],
tools=None,
max_tokens=4096,
temperature=0.5,
reasoning_effort="none",
deferred_names=None,
)
assert "tools" in kwargs
tool_types = [t.get("type") for t in kwargs["tools"]]
assert "web_search" in tool_types
class TestResponsesCitationFormat:
"""Test format_citations handles Responses API flat annotation format."""
def test_responses_api_flat_annotation(self) -> None:
"""Responses API annotations have title/url directly on the object."""
class FlatAnnotation:
type = "url_citation"
url_citation = None # Not present in Responses API
title = "Example"
url = "https://example.com"
result = format_citations("Text.", [FlatAnnotation()])
assert "Sources:" in result
assert "[Example](https://example.com)" in result
class TestResponsesStreaming:
"""Tests for Responses API streaming event handling."""
def setup_method(self) -> None:
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
self.provider = OpenAIResponsesProvider()
def _make_event(self, event_type: str, **attrs: Any) -> MagicMock:
event = MagicMock()
event.type = event_type
for k, v in attrs.items():
setattr(event, k, v)
return event
def test_text_delta(self) -> None:
events = [
self._make_event("response.output_text.delta", delta="Hello"),
self._make_event("response.output_text.delta", delta=" world"),
self._make_event(
"response.completed",
response=MagicMock(
status="completed",
usage=None,
),
),
]
chunks = list(self.provider._iter_stream(iter(events)))
text_chunks = [c for c in chunks if c.content_delta]
assert len(text_chunks) == 2
assert text_chunks[0].content_delta == "Hello"
assert text_chunks[0].is_first is True
assert text_chunks[1].content_delta == " world"
def test_reasoning_delta(self) -> None:
events = [
self._make_event("response.reasoning_text.delta", delta="thinking..."),
self._make_event(
"response.completed",
response=MagicMock(
status="completed",
usage=None,
),
),
]
chunks = list(self.provider._iter_stream(iter(events)))
reasoning = [c for c in chunks if c.reasoning_delta]
assert len(reasoning) == 1
assert reasoning[0].reasoning_delta == "thinking..."
assert reasoning[0].is_first is True
def test_tool_call_streaming(self) -> None:
item = MagicMock()
item.type = "function_call"
item.id = "fc_abc123"
item.call_id = "call_1"
item.name = "read_file"
events = [
self._make_event("response.output_item.added", item=item),
self._make_event(
"response.function_call_arguments.delta",
item_id="fc_abc123",
delta='{"path":',
),
self._make_event(
"response.function_call_arguments.delta",
item_id="fc_abc123",
delta='"/tmp"}',
),
self._make_event(
"response.completed",
response=MagicMock(
status="completed",
usage=None,
),
),
]
chunks = list(self.provider._iter_stream(iter(events)))
tc_chunks = [c for c in chunks if c.tool_call_deltas]
assert len(tc_chunks) == 3
# First chunk: tool call added with name
assert tc_chunks[0].tool_call_deltas[0].name == "read_file"
assert tc_chunks[0].tool_call_deltas[0].id == "call_1"
# Argument deltas
assert tc_chunks[1].tool_call_deltas[0].arguments_delta == '{"path":'
assert tc_chunks[2].tool_call_deltas[0].arguments_delta == '"/tmp"}'
def test_completed_event_with_usage(self) -> None:
usage = MagicMock()
usage.input_tokens = 100
usage.output_tokens = 50
usage.total_tokens = 150
usage.input_tokens_details = MagicMock(cached_tokens=80)
# Ensure Chat Completions attributes are not present
del usage.prompt_tokens
del usage.completion_tokens
del usage.prompt_tokens_details
events = [
self._make_event(
"response.completed",
response=MagicMock(
status="completed",
usage=usage,
),
),
]
chunks = list(self.provider._iter_stream(iter(events)))
final = [c for c in chunks if c.finish_reason]
assert len(final) == 1
assert final[0].finish_reason == "stop"
assert final[0].usage is not None
assert final[0].usage.prompt_tokens == 100
assert final[0].usage.completion_tokens == 50
assert final[0].usage.cache_read_tokens == 80
def test_web_search_events(self) -> None:
events = [
self._make_event("response.web_search_call.searching"),
self._make_event("response.web_search_call.completed"),
self._make_event(
"response.completed",
response=MagicMock(
status="completed",
usage=None,
),
),
]
chunks = list(self.provider._iter_stream(iter(events)))
info = [c for c in chunks if c.info_delta]
assert len(info) == 2
assert "Searching" in info[0].info_delta
assert "complete" in info[1].info_delta
class TestResponsesCompletion:
"""Tests for non-streaming Responses API completion."""
def setup_method(self) -> None:
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
self.provider = OpenAIResponsesProvider()
def _make_response(
self,
text: str = "Hello",
tool_calls: list[dict[str, Any]] | None = None,
status: str = "completed",
) -> MagicMock:
resp = MagicMock()
resp.status = status
resp.usage = MagicMock()
resp.usage.input_tokens = 10
resp.usage.output_tokens = 5
resp.usage.total_tokens = 15
resp.usage.input_tokens_details = MagicMock(cached_tokens=0)
# Remove Chat Completions attributes
del resp.usage.prompt_tokens
del resp.usage.completion_tokens
del resp.usage.prompt_tokens_details
output: list[Any] = []
if text:
msg = MagicMock()
msg.type = "message"
text_part = MagicMock()
text_part.type = "output_text"
text_part.text = text
text_part.annotations = []
msg.content = [text_part]
msg.model_dump.return_value = {
"type": "message",
"content": [{"type": "output_text", "text": text}],
}
output.append(msg)
if tool_calls:
for tc in tool_calls:
item = MagicMock()
item.type = "function_call"
item.call_id = tc["id"]
item.name = tc["name"]
item.arguments = tc["arguments"]
item.model_dump.return_value = {
"type": "function_call",
"call_id": tc["id"],
"name": tc["name"],
"arguments": tc["arguments"],
}
output.append(item)
resp.output = output
return resp
def test_basic_text_completion(self) -> None:
resp = self._make_response(text="Hello world")
result = self.provider._parse_response(resp)
assert result.content == "Hello world"
assert result.tool_calls is None
assert result.finish_reason == "stop"
def test_completion_with_tool_calls(self) -> None:
resp = self._make_response(
text="",
tool_calls=[{"id": "call_1", "name": "read_file", "arguments": '{"path": "/tmp"}'}],
)
result = self.provider._parse_response(resp)
assert result.tool_calls is not None
assert len(result.tool_calls) == 1
assert result.tool_calls[0]["id"] == "call_1"
assert result.tool_calls[0]["function"]["name"] == "read_file"
def test_provider_blocks_captured(self) -> None:
resp = self._make_response(text="Hello")
result = self.provider._parse_response(resp)
assert len(result.provider_blocks) > 0
assert result.provider_blocks[0]["type"] == "message"
def test_incomplete_status_maps_to_length(self) -> None:
resp = self._make_response(text="Partial", status="incomplete")
result = self.provider._parse_response(resp)
assert result.finish_reason == "length"
def test_usage_extraction(self) -> None:
resp = self._make_response(text="Hi")
result = self.provider._parse_response(resp)
assert result.usage is not None
assert result.usage.prompt_tokens == 10
assert result.usage.completion_tokens == 5
Reference in New Issue View Git Blame Copy Permalink