diff --git a/tests/data/wire_payloads/anthropic_compat__mid_orphan.json b/tests/data/wire_payloads/anthropic_compat__mid_orphan.json new file mode 100644 index 00000000..77e27e21 --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__mid_orphan.json @@ -0,0 +1,78 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Weather in Paris and London?", + "role": "user" + }, + { + "content": [ + { + "id": "call_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + }, + { + "id": "call_2", + "input": { + "city": "London" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear.", + "tool_use_id": "call_1", + "type": "tool_result" + }, + { + "content": "Tool execution was cancelled. Outcome UNKNOWN — this call may have begun executing before the generation was stopped; do not assume it did not run, and reconcile before re-issuing it.", + "is_error": true, + "tool_use_id": "call_2", + "type": "tool_result" + }, + { + "text": "Actually, never mind London.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5, + "tools": [ + { + "description": "Look up the weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/data/wire_payloads/anthropic_compat__multipart.json b/tests/data/wire_payloads/anthropic_compat__multipart.json new file mode 100644 index 00000000..11fe8521 --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__multipart.json @@ -0,0 +1,33 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": [ + { + "text": "What's in this image?", + "type": "text" + }, + { + "source": { + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5 +} diff --git a/tests/data/wire_payloads/anthropic_compat__native_orphan.json b/tests/data/wire_payloads/anthropic_compat__native_orphan.json new file mode 100644 index 00000000..228664ae --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__native_orphan.json @@ -0,0 +1,70 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Think about the weather.", + "role": "user" + }, + { + "content": [ + { + "signature": "sig-abc", + "thinking": "The user wants weather.", + "type": "thinking" + }, + { + "text": "Let me check.", + "type": "text" + }, + { + "id": "call_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "Tool execution was cancelled. Outcome UNKNOWN — this call may have begun executing before the generation was stopped; do not assume it did not run, and reconcile before re-issuing it.", + "is_error": true, + "tool_use_id": "call_1", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5, + "tools": [ + { + "description": "Look up the weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/data/wire_payloads/anthropic_compat__native_reasoning.json b/tests/data/wire_payloads/anthropic_compat__native_reasoning.json new file mode 100644 index 00000000..a8c18dad --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__native_reasoning.json @@ -0,0 +1,69 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Think about the weather.", + "role": "user" + }, + { + "content": [ + { + "signature": "sig-abc", + "thinking": "The user wants weather.", + "type": "thinking" + }, + { + "text": "Let me check.", + "type": "text" + }, + { + "id": "call_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear.", + "tool_use_id": "call_1", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5, + "tools": [ + { + "description": "Look up the weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/data/wire_payloads/anthropic_compat__operator_system.json b/tests/data/wire_payloads/anthropic_compat__operator_system.json new file mode 100644 index 00000000..39158ae2 --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__operator_system.json @@ -0,0 +1,63 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Run the deploy.", + "role": "user" + }, + { + "content": [ + { + "id": "call_1", + "input": {}, + "name": "deploy", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "deployed", + "tool_use_id": "call_1", + "type": "tool_result" + }, + { + "text": "Great, what's next?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "qwen3.6-27b", + "system": "Output-guard: deploy output looked clean.", + "temperature": 0.5, + "tools": [ + { + "description": "Look up the weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/data/wire_payloads/anthropic_compat__text.json b/tests/data/wire_payloads/anthropic_compat__text.json new file mode 100644 index 00000000..0d42ec60 --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__text.json @@ -0,0 +1,33 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Hi there.", + "role": "user" + }, + { + "content": [ + { + "text": "Hello! How can I help?", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "What's the weather in Paris?", + "role": "user" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5 +} diff --git a/tests/data/wire_payloads/anthropic_compat__toolcall_complete.json b/tests/data/wire_payloads/anthropic_compat__toolcall_complete.json new file mode 100644 index 00000000..b76e650a --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__toolcall_complete.json @@ -0,0 +1,69 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + }, + { + "content": [ + { + "id": "call_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear.", + "tool_use_id": "call_1", + "type": "tool_result" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "It's 18C and clear in Paris.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5, + "tools": [ + { + "description": "Look up the weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/data/wire_payloads/anthropic_compat__trailing_orphan.json b/tests/data/wire_payloads/anthropic_compat__trailing_orphan.json new file mode 100644 index 00000000..40d1ad17 --- /dev/null +++ b/tests/data/wire_payloads/anthropic_compat__trailing_orphan.json @@ -0,0 +1,61 @@ +{ + "cache_control": { + "type": "ephemeral" + }, + "extra_body": { + "chat_template_kwargs": { + "enable_thinking": true, + "reasoning_effort": "high" + } + }, + "max_tokens": 4096, + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + }, + { + "content": [ + { + "id": "call_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "Tool execution was cancelled. Outcome UNKNOWN — this call may have begun executing before the generation was stopped; do not assume it did not run, and reconcile before re-issuing it.", + "is_error": true, + "tool_use_id": "call_1", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "qwen3.6-27b", + "temperature": 0.5, + "tools": [ + { + "description": "Look up the weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/test_wire_payload_golden.py b/tests/test_wire_payload_golden.py index 160b9508..3c6cd920 100644 --- a/tests/test_wire_payload_golden.py +++ b/tests/test_wire_payload_golden.py @@ -36,6 +36,7 @@ from turnstone.core.providers._anthropic import AnthropicProvider from turnstone.core.providers._google import GoogleProvider from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider from turnstone.core.providers._openai_responses import OpenAIResponsesProvider +from turnstone.core.providers._protocol import ModelCapabilities if TYPE_CHECKING: from turnstone.core.providers._protocol import LLMProvider @@ -45,11 +46,21 @@ _UPDATE = os.environ.get("UPDATE_WIRE_GOLDENS") == "1" def _capture( - provider: LLMProvider, *, model: str, messages: list[dict[str, Any]], **opts: Any + provider: LLMProvider, + *, + model: str, + messages: list[dict[str, Any]], + caps: ModelCapabilities | None = None, + **opts: Any, ) -> dict[str, Any]: - """Drive ``create_streaming`` against a recording client; return the SDK kwargs.""" + """Drive ``create_streaming`` against a recording client; return the SDK kwargs. + + *caps* overrides the provider's own capability lookup — required for the + anthropic-compatible lane, which has no static table (an operator-run model + definition supplies its capabilities). + """ client = RecordingClient() - caps = provider.get_capabilities(model) + caps = caps or provider.get_capabilities(model) # Mirror the session's wire prep: orphan repair runs once on the canonical # Turns (``ChatSession._prepare_wire_messages``), then the result is lowered # to the dict projection the translator consumes. Fixtures arrive @@ -264,3 +275,36 @@ def test_wire_payload( provider = factory() payload = _capture(provider, model=model, messages=[dict(m) for m in messages], **opts) _assert_golden(f"{provider_id}__{fixture_id}", payload) + + +# The anthropic-compatible lane (vLLM /v1/messages) has no static capability +# table and carries reasoning control in ``extra_body.chat_template_kwargs`` +# rather than the native ``thinking`` param — a wire shape the matrix above +# never exercises (both AnthropicProvider rows are the native lane). Freeze it +# with the capabilities a manual-mode model definition supplies and a real +# effort level, so the graded ``reasoning_effort`` key is pinned in the golden +# (never the native ``thinking`` param, and no forced ``temperature=1.0``). +_COMPAT_CAPS = ModelCapabilities( + context_window=262144, + max_output_tokens=64000, + token_param="max_tokens", + thinking_mode="manual", + thinking_param="enable_thinking", + supports_reasoning_replay=True, +) + + +@pytest.mark.parametrize("fixture_id", sorted(_FIXTURES)) +def test_wire_payload_anthropic_compat(fixture_id: str) -> None: + messages, opts = _FIXTURES[fixture_id] + provider = AnthropicProvider(compat=True) + payload = _capture( + provider, + model="qwen3.6-27b", + messages=[dict(m) for m in messages], + caps=_COMPAT_CAPS, + reasoning_effort="high", + **opts, + ) + assert "thinking" not in payload, "compat lane must never send the native thinking param" + _assert_golden(f"anthropic_compat__{fixture_id}", payload)