mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
030bf2ead9
The Anthropic call sites in session.py passed the operator-side `replay_reasoning_to_model` flag through without checking the model's static `supports_reasoning_replay` capability. The OpenAI Responses path AND-gated both flags in `_build_kwargs` so a model without a reasoning lane (gpt-4o, etc.) silently skipped replay even when the operator flag was set. The Anthropic path had no such gate. For all current Claude entries this was a no-op asymmetry - every `_ANTHROPIC_CAPABILITIES` row sets `supports_reasoning_replay=True`, so `True AND op == op`. But: - The capability flag was dead code on the Anthropic path - A future Claude entry (or any Anthropic-shaped surface) shipping with the cap left at its False default would have replay fire anyway, against the cap declaration - The asymmetry made `supports_reasoning_replay` an unreliable signal - readers couldn't tell if it gated anything per-provider Move the AND-gate into `_resolve_replay_reasoning_to_model` via a new optional `caps=` kwarg. When caps is provided, the resolver returns `operator_on AND caps.supports_reasoning_replay`; when omitted (back-compat for any caller not yet updated), it returns the operator flag unchanged. Thread caps through the three call sites: `_utility_completion` (non-streaming), `_try_stream` (streaming, hoisted resolution out of the retry loop since caps are attempt-invariant), and the agent `_api_call` closure in `_run_agent`. With the AND-gate now living at the session resolver, the redundant in-provider gate in `OpenAIResponsesProvider._build_kwargs` is removed. The provider now trusts the resolved bool it receives, matching the AnthropicProvider shape and giving the cap a single source of truth across providers. The two provider-level tests that pinned the in-provider gate (`test_include_omitted_when_capability_false`, `test_include_omitted_by_default`) drop out; the session-level boundary test `TestSessionToOpenAIResponsesBoundaryIntegration::test_capability_false_omits_include_even_when_flag_true` already covers the same end-to-end invariant. Tests added: - 4 resolver-level tests pinning the AND-gate semantics + back-compat when caps is omitted - 1 wire-boundary integration test mirroring the OpenAI Responses `test_capability_false_omits_include_even_when_flag_true` - drives session._try_stream through the real AnthropicProvider with operator flag True + capability False and asserts the thinking block does NOT reach the SDK boundary Existing `TestUtilityCompletionPassesFlag` test had its caps mock upgraded from `SimpleNamespace` to a real `ModelCapabilities` instance to satisfy the new attribute read and stay robust to future capability fields.
321 lines
12 KiB
Python
321 lines
12 KiB
Python
"""Tests for OpenAI Responses reasoning capture + replay (Phase 3 path 2).
|
|
|
|
Phase 3 wires:
|
|
1. ``include=["reasoning.encrypted_content"]`` on the request when
|
|
the operator flag AND the model capability both allow.
|
|
2. ``_convert_messages`` round-tripping stored reasoning items as
|
|
``ResponseReasoningItemParam`` input items on subsequent turns.
|
|
3. ``OpenAIResponsesProvider.extract_reasoning_text`` walking
|
|
reasoning items and returning concatenated summary + content text.
|
|
|
|
All tests drive through the real ``OpenAIResponsesProvider`` — no
|
|
mocks of the converter/build_kwargs themselves; only the SDK boundary
|
|
is mocked where relevant.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from turnstone.core.providers._openai_responses import (
|
|
OpenAIResponsesProvider,
|
|
_reasoning_item_for_input,
|
|
)
|
|
from turnstone.core.providers._protocol import (
|
|
MAX_REASONING_DISPLAY_CHARS as _MAX_REASONING_DISPLAY_CHARS,
|
|
)
|
|
from turnstone.core.providers._protocol import ModelCapabilities
|
|
|
|
|
|
@pytest.fixture
|
|
def provider() -> OpenAIResponsesProvider:
|
|
return OpenAIResponsesProvider()
|
|
|
|
|
|
def _capable_caps() -> ModelCapabilities:
|
|
"""Capability fixture for a reasoning-replay-capable model."""
|
|
return ModelCapabilities(
|
|
context_window=400000,
|
|
max_output_tokens=128000,
|
|
supports_temperature=False,
|
|
reasoning_effort_values=("low", "medium", "high"),
|
|
default_reasoning_effort="medium",
|
|
supports_reasoning_replay=True,
|
|
)
|
|
|
|
|
|
class TestExtractReasoningText:
|
|
def test_none_returns_empty(self, provider: OpenAIResponsesProvider) -> None:
|
|
assert provider.extract_reasoning_text(None) == ""
|
|
|
|
def test_empty_list_returns_empty(self, provider: OpenAIResponsesProvider) -> None:
|
|
assert provider.extract_reasoning_text([]) == ""
|
|
|
|
def test_no_reasoning_items_returns_empty(self, provider: OpenAIResponsesProvider) -> None:
|
|
blocks = [
|
|
{"type": "message", "role": "assistant", "content": "hi"},
|
|
{"type": "function_call", "call_id": "c1", "name": "x", "arguments": "{}"},
|
|
]
|
|
assert provider.extract_reasoning_text(blocks) == ""
|
|
|
|
def test_summary_text_extracted(self, provider: OpenAIResponsesProvider) -> None:
|
|
# Per ResponseReasoningItem (response_reasoning_item.py:31-62):
|
|
# summary is always present; content is optional.
|
|
blocks = [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [
|
|
{"type": "summary_text", "text": "I considered X"},
|
|
{"type": "summary_text", "text": "then Y"},
|
|
],
|
|
}
|
|
]
|
|
assert provider.extract_reasoning_text(blocks) == "I considered X\nthen Y"
|
|
|
|
def test_content_text_extracted_alongside_summary(
|
|
self, provider: OpenAIResponsesProvider
|
|
) -> None:
|
|
blocks = [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "summary line"}],
|
|
"content": [{"type": "reasoning_text", "text": "raw reasoning"}],
|
|
}
|
|
]
|
|
# Order: summary first, then content (matches the order the SDK
|
|
# surfaces them via streaming events).
|
|
result = provider.extract_reasoning_text(blocks)
|
|
assert "summary line" in result
|
|
assert "raw reasoning" in result
|
|
|
|
def test_truncation_at_64kib_cap(self, provider: OpenAIResponsesProvider) -> None:
|
|
long_text = "x" * (_MAX_REASONING_DISPLAY_CHARS + 1024)
|
|
blocks = [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": long_text}],
|
|
}
|
|
]
|
|
result = provider.extract_reasoning_text(blocks)
|
|
assert len(result) == _MAX_REASONING_DISPLAY_CHARS
|
|
|
|
def test_malformed_summary_entry_skipped(self, provider: OpenAIResponsesProvider) -> None:
|
|
blocks = [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [
|
|
"not a dict",
|
|
{"type": "summary_text"}, # missing text
|
|
{"type": "summary_text", "text": ""}, # empty text
|
|
{"type": "summary_text", "text": "good"},
|
|
],
|
|
}
|
|
]
|
|
assert provider.extract_reasoning_text(blocks) == "good"
|
|
|
|
def test_non_list_input_returns_empty(self, provider: OpenAIResponsesProvider) -> None:
|
|
assert provider.extract_reasoning_text("not a list") == "" # type: ignore[arg-type]
|
|
|
|
def test_other_block_types_skipped_in_walk(self, provider: OpenAIResponsesProvider) -> None:
|
|
# Mixed payload: only the reasoning block contributes.
|
|
blocks = [
|
|
{"type": "message", "role": "assistant", "content": "hi"},
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "thought"}],
|
|
},
|
|
{"type": "function_call", "call_id": "c1", "name": "x", "arguments": "{}"},
|
|
]
|
|
assert provider.extract_reasoning_text(blocks) == "thought"
|
|
|
|
|
|
class TestReasoningItemForInput:
|
|
"""``_reasoning_item_for_input`` projects a stored ``ResponseReasoningItem``
|
|
dict into ``ResponseReasoningItemParam`` shape (drops server-only
|
|
``status``)."""
|
|
|
|
def test_minimal_item_round_trip(self) -> None:
|
|
stored = {
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "x"}],
|
|
"status": "completed",
|
|
}
|
|
result = _reasoning_item_for_input(stored)
|
|
assert result["type"] == "reasoning"
|
|
assert result["id"] == "r_1"
|
|
assert result["summary"] == [{"type": "summary_text", "text": "x"}]
|
|
# status NOT round-tripped (server-only field per
|
|
# ResponseReasoningItemParam at response_reasoning_item_param.py).
|
|
assert "status" not in result
|
|
|
|
def test_encrypted_content_round_trips_when_present(self) -> None:
|
|
stored = {
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "x"}],
|
|
"encrypted_content": "opaque-blob",
|
|
}
|
|
result = _reasoning_item_for_input(stored)
|
|
assert result["encrypted_content"] == "opaque-blob"
|
|
|
|
def test_encrypted_content_omitted_when_absent(self) -> None:
|
|
stored = {
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "x"}],
|
|
}
|
|
result = _reasoning_item_for_input(stored)
|
|
assert "encrypted_content" not in result
|
|
|
|
def test_content_round_trips_when_present(self) -> None:
|
|
stored = {
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "s"}],
|
|
"content": [{"type": "reasoning_text", "text": "raw"}],
|
|
}
|
|
result = _reasoning_item_for_input(stored)
|
|
assert result["content"] == [{"type": "reasoning_text", "text": "raw"}]
|
|
|
|
|
|
class TestBuildKwargsInclude:
|
|
"""``_build_kwargs`` adds ``include=["reasoning.encrypted_content"]``
|
|
when the resolved operator flag is True. The capability AND-gate
|
|
lives upstream in ``ChatSession._resolve_replay_reasoning_to_model``
|
|
(single source of truth across providers); the provider trusts the
|
|
bool it receives. See
|
|
``test_session_replay_reasoning.py::TestSessionToOpenAIResponsesBoundaryIntegration``
|
|
for the end-to-end gate test."""
|
|
|
|
def test_include_added_when_flag_true(self, provider: OpenAIResponsesProvider) -> None:
|
|
kwargs = provider._build_kwargs(
|
|
model="gpt-5",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
tools=None,
|
|
max_tokens=1024,
|
|
temperature=0.5,
|
|
reasoning_effort="medium",
|
|
deferred_names=None,
|
|
capabilities=_capable_caps(),
|
|
replay_reasoning_to_model=True,
|
|
)
|
|
assert kwargs.get("include") == ["reasoning.encrypted_content"]
|
|
|
|
def test_include_omitted_when_flag_false(self, provider: OpenAIResponsesProvider) -> None:
|
|
kwargs = provider._build_kwargs(
|
|
model="gpt-5",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
tools=None,
|
|
max_tokens=1024,
|
|
temperature=0.5,
|
|
reasoning_effort="medium",
|
|
deferred_names=None,
|
|
capabilities=_capable_caps(),
|
|
replay_reasoning_to_model=False,
|
|
)
|
|
assert "include" not in kwargs
|
|
|
|
|
|
class TestConvertMessagesReasoningReplay:
|
|
"""``_convert_messages`` round-trips stored reasoning items as input."""
|
|
|
|
def test_reasoning_item_emitted_before_assistant_when_replay_true(
|
|
self, provider: OpenAIResponsesProvider
|
|
) -> None:
|
|
messages = [
|
|
{"role": "user", "content": "explain"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "Final answer.",
|
|
"_provider_content": [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "I thought"}],
|
|
"encrypted_content": "abc",
|
|
}
|
|
],
|
|
},
|
|
{"role": "user", "content": "follow up"},
|
|
]
|
|
_, items = provider._convert_messages(messages, replay_reasoning_to_model=True)
|
|
# Find the reasoning input item.
|
|
types = [it.get("type") for it in items]
|
|
# Expected: user, reasoning, message (assistant), user.
|
|
assert types == ["message", "reasoning", "message", "message"]
|
|
reasoning_idx = types.index("reasoning")
|
|
r_item = items[reasoning_idx]
|
|
assert r_item["id"] == "r_1"
|
|
assert r_item["encrypted_content"] == "abc"
|
|
# And the reasoning item appears immediately BEFORE the
|
|
# assistant message it belongs to.
|
|
assert items[reasoning_idx + 1]["role"] == "assistant"
|
|
|
|
def test_reasoning_item_dropped_when_replay_false(
|
|
self, provider: OpenAIResponsesProvider
|
|
) -> None:
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": "Answer.",
|
|
"_provider_content": [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "thought"}],
|
|
}
|
|
],
|
|
},
|
|
]
|
|
_, items = provider._convert_messages(messages, replay_reasoning_to_model=False)
|
|
types = [it.get("type") for it in items]
|
|
assert "reasoning" not in types
|
|
|
|
def test_no_reasoning_items_when_provider_content_lacks_reasoning(
|
|
self, provider: OpenAIResponsesProvider
|
|
) -> None:
|
|
# Anthropic-shaped _provider_content reaching OpenAI Responses
|
|
# (cross-provider — operator switch from Anthropic to GPT-5):
|
|
# no type=="reasoning" items, so nothing emitted.
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": "x",
|
|
"_provider_content": [
|
|
{"type": "thinking", "thinking": "anth", "signature": "s"},
|
|
],
|
|
},
|
|
]
|
|
_, items = provider._convert_messages(messages, replay_reasoning_to_model=True)
|
|
types = [it.get("type") for it in items]
|
|
assert "reasoning" not in types
|
|
|
|
def test_default_replay_reasoning_false_omits_reasoning(
|
|
self, provider: OpenAIResponsesProvider
|
|
) -> None:
|
|
# Pre-Phase-3 callers (no kwarg) get the back-compat behaviour:
|
|
# reasoning items are silently dropped (sanitize_messages was
|
|
# already stripping _provider_content anyway).
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": "x",
|
|
"_provider_content": [
|
|
{
|
|
"type": "reasoning",
|
|
"id": "r_1",
|
|
"summary": [{"type": "summary_text", "text": "x"}],
|
|
}
|
|
],
|
|
},
|
|
]
|
|
_, items = provider._convert_messages(messages) # no kwarg
|
|
types = [it.get("type") for it in items]
|
|
assert "reasoning" not in types
|