Files
turnstone/tests/test_model_turn.py
T
Patrick Buckley 70165807c7 fix(reasoning): close the unmarked chain-of-thought leak, gate the tag scan by backend (#940) (#978)
Some serving setups emit model reasoning inline with no think tags and no
reasoning_content at all — nothing any parser can segregate (measured live
on the dev vLLM: 20/20 sampled completions, streamed and not, proxied and
direct). The drain seam correctly passes unmarked prose through, so it
became the artifact on every bounded-artifact lane: workstream titles
("Thinking Process:"), compaction summaries that were ~90% chain-of-
thought, and the web-fetch tool results #940 reports — which then ride
every following turn as context.

Three coordinated changes:

* Utility lanes ask for no reasoning. _utility_completion (title,
  compaction, web-fetch extraction) pins the alias's declared thinking
  toggle off and withholds every reasoning-effort channel — the relayed
  session knob, the lane rung, the definition default, and the graded
  template key — via lane_without_thinking / lane_thinking_suppressed,
  the same suppression omni transcription already used (now shared as
  thinking_off_template_kwargs). Measured end-to-end: the extraction
  that returned 3.7k chars of reasoning returns a 258-char answer.

* server_parses_reasoning capability. A backend that segregates
  reasoning into its own channel declares it, and the inline tag scan
  turns off on every lane: the drain seam, the interactive splitter
  (which now reads the ACTIVE stream's capabilities via the creation-
  time handoff register, never the primary alias's), and the title
  lane's cosmetic peel — so prose that merely quotes a tag can no
  longer be misrouted, and the utility suppression stands down where
  reasoning costs the artifact nothing. The built-in commercial
  capability tables declare it wholesale (known models and table-miss
  defaults); local compat lanes keep the passthrough default the scan
  exists for. Bool-typed capability overrides coerce string spellings
  instead of truthiness-flipping on hand-edited JSON.

* Title selection follows the prompt's contract, not line position:
  the last line within the word cap that ends in a word character —
  rejecting explanation sentences, sign-offs, parentheticals, and
  reasoning headings in any script (terminal punctuation carries
  unspaced scripts where whitespace word counts are meaningless) —
  else the last non-empty line. 20/20 captured live responses title
  correctly (9/20 before, unchanged since well before the seam
  unification: the old and new pipelines scored identically on every
  sample, so the regression source was the backend's output shape,
  not #965).

Also folded in from the review round: a think tag split across a
reasoning-delta boundary reassembles in the drain (partial-tag tail
carry; tool boundaries still flush), Turn.text joins text blocks with a
newline so multi-block answers stop fusing words in notification bodies
and every flattened read, the notify hook reads final_assistant_text
directly instead of through a one-line shim, web-fetch extraction uses
the shared _non_blank_or fallback, and the judge/output-guard suites use
real ModelCapabilities instead of truthy mock attributes.

Closes #940.
2026-08-05 12:58:55 -07:00

894 lines
36 KiB
Python

"""Unit tests for the ``model_turn`` plant-call primitive (#827).
The agent-path tests in ``test_session.py`` exercise ``model_turn`` through
``_run_agent`` (native-lane replay, blank-id gate, minted-id nesting); these
pin the module's own contract directly so the judges (phase 1b) and the
single-shot lanes (phase 2) can build on it without re-deriving semantics.
"""
from __future__ import annotations
import logging
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
import turnstone.core.model_turn as model_turn_mod
from tests._session_helpers import as_stream
from turnstone.core.model_turn import (
ModelLane,
finalize_provider_blocks,
maybe_attach_vllm_chat_reasoning,
model_turn,
resolve_lane,
synth_reasoning_block,
)
from turnstone.core.providers._protocol import (
CompletionResult,
IncompleteStreamError,
ModelCapabilities,
StreamChunk,
UsageInfo,
)
from turnstone.core.trajectory import Role, ToolCall, Turn
class _FakeProvider:
"""Records every ``create_streaming`` call; replays scripted results
as single-chunk streams via the shared ``as_stream`` adapter
(multi-chunk accumulation is pinned by the dedicated ``drain_stream``
unit tests)."""
provider_name = "openai-compatible"
def __init__(self, results: list[CompletionResult]) -> None:
self.results = list(results)
self.calls: list[dict[str, Any]] = []
def get_capabilities(self, model: str) -> ModelCapabilities:
return ModelCapabilities()
def create_streaming(self, **kwargs: Any) -> list[StreamChunk]:
self.calls.append(kwargs)
return as_stream(self.results.pop(0))
def _fake_registry(
*,
capabilities: dict[str, Any] | None = None,
server_compat: dict[str, Any] | None = None,
replay: bool = False,
temperature: float | None = None,
) -> MagicMock:
cfg = SimpleNamespace(
capabilities=capabilities or {},
server_compat=server_compat or {},
replay_reasoning_to_model=replay,
temperature=temperature,
)
reg = MagicMock()
reg.get_config.return_value = cfg
return reg
def _lane(provider: _FakeProvider, **kw: Any) -> ModelLane:
return ModelLane(provider=provider, client=object(), model="m", **kw)
def test_backend_auth_token_binds_sdk_credential_once() -> None:
"""Dynamic credentials use SDK with_options, not an override header."""
provider = _FakeProvider([CompletionResult(content="ok")])
base_client = MagicMock()
bound_client = object()
base_client.with_options.return_value = bound_client
lane = ModelLane(provider=provider, client=base_client, model="m", alias="gateway")
result = model_turn(
lane,
[Turn.user("hello")],
backend_auth_token="minted-token",
)
assert result.content == "ok"
base_client.with_options.assert_called_once_with(api_key="minted-token")
assert provider.calls[0]["client"] is bound_client
assert "extra_headers" not in provider.calls[0]
def test_entra_app_lane_resolver_never_issues_placeholder_client() -> None:
"""A resolver-carrying lane binds its app token before the provider call."""
provider = _FakeProvider([CompletionResult(content="ok")])
placeholder_client = MagicMock(name="backend-auth-placeholder-unused")
bound_client = object()
placeholder_client.with_options.return_value = bound_client
resolver = MagicMock(return_value="app-token")
lane = ModelLane(
provider=provider,
client=placeholder_client,
model="m",
alias="app-gateway",
backend_auth_resolver=resolver,
)
model_turn(lane, [Turn.user("hello")])
resolver.assert_called_once_with("app-gateway")
placeholder_client.with_options.assert_called_once_with(api_key="app-token")
assert provider.calls[0]["client"] is bound_client
class _FlakyProvider:
"""Scripted drain-time deaths: each script entry is either a
``CompletionResult`` (streamed normally) or an exception instance
(raised mid-iteration — AFTER ``create_streaming`` returned, exactly
where a real mid-body wire death surfaces)."""
provider_name = "openai-compatible"
retryable_error_names: frozenset[str] = frozenset({"IncompleteStreamError"})
def __init__(self, script: list[Any]) -> None:
self.script = list(script)
self.calls: list[dict[str, Any]] = []
def get_capabilities(self, model: str) -> ModelCapabilities:
return ModelCapabilities()
def create_streaming(self, **kwargs: Any) -> Any:
self.calls.append(kwargs)
item = self.script.pop(0)
def _iter() -> Any:
if isinstance(item, BaseException):
raise item
yield from as_stream(item)
return _iter()
def test_model_turn_retries_transient_mid_stream_death(monkeypatch: pytest.MonkeyPatch) -> None:
# The retired non-streaming transport read the whole body inside the
# SDK's retried request, so single-shot lanes never saw a mid-body wire
# blip — the drain-scoped loop is that retry's new home.
monkeypatch.setattr("turnstone.core.model_turn._DRAIN_RETRY_BASE_DELAY", 0.0)
provider = _FlakyProvider(
[
IncompleteStreamError("stream died mid-response"),
CompletionResult(content="second try"),
]
)
lane = ModelLane(provider=provider, client=object(), model="m")
result = model_turn(lane, [Turn.user("x")])
assert result.content == "second try"
assert len(provider.calls) == 2
def test_model_turn_gives_up_after_retry_budget(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("turnstone.core.model_turn._DRAIN_RETRY_BASE_DELAY", 0.0)
provider = _FlakyProvider([IncompleteStreamError(f"death {i}") for i in range(5)])
lane = ModelLane(provider=provider, client=object(), model="m")
with pytest.raises(IncompleteStreamError):
model_turn(lane, [Turn.user("x")])
# One initial issue + _DRAIN_RETRIES re-issues, then it propagates.
assert len(provider.calls) == 3
def test_model_turn_retry_backs_off_between_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
# Instant re-issues are guaranteed to re-hit a still-active rate
# limit/overload — the loop paces like the SDK request retry it
# replaces: 0.5s base, doubling, ±50% jitter.
sleeps: list[float] = []
monkeypatch.setattr(model_turn_mod, "time", SimpleNamespace(sleep=sleeps.append))
provider = _FlakyProvider(
[
IncompleteStreamError("death 1"),
IncompleteStreamError("death 2"),
CompletionResult(content="ok"),
]
)
lane = ModelLane(provider=provider, client=object(), model="m")
result = model_turn(lane, [Turn.user("x")])
assert result.content == "ok"
assert len(sleeps) == 2
assert 0.25 <= sleeps[0] <= 0.75 # 0.5 * jitter[0.5, 1.5)
assert 0.5 <= sleeps[1] <= 1.5 # 1.0 * jitter[0.5, 1.5)
def test_model_turn_does_not_retry_unrecognized_errors() -> None:
provider = _FlakyProvider([RuntimeError("schema violation")])
lane = ModelLane(provider=provider, client=object(), model="m")
with pytest.raises(RuntimeError, match="schema violation"):
model_turn(lane, [Turn.user("x")])
assert len(provider.calls) == 1
def test_model_turn_abort_during_backoff_suppresses_reissue(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# The deadline can abandon the worker while it sleeps between
# attempts — the wake-up must die with the original failure, not
# issue one more full request from an abandoned thread.
from turnstone.core.deadline import StreamAbortRef
ref = StreamAbortRef()
monkeypatch.setattr(model_turn_mod, "time", SimpleNamespace(sleep=lambda _delay: ref.abort()))
provider = _FlakyProvider(
[IncompleteStreamError("transient death"), CompletionResult(content="never")]
)
lane = ModelLane(provider=provider, client=object(), model="m")
with pytest.raises(IncompleteStreamError, match="transient death"):
model_turn(lane, [Turn.user("x")], cancel_ref=ref)
assert len(provider.calls) == 1
def test_model_turn_does_not_retry_after_abort(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
# A call the deadline abandoned must not have its request resurrected
# behind its back: in the field the abort closes the stream and the
# drain dies with an error that LOOKS retryable. The fixture models
# only the shape of that — abort landing after dispatch, drain raising
# IncompleteStreamError — because the aborted ref is what gates the
# re-issue regardless of which of the two produced the error. This is
# the RE-ISSUE gate; of the tests below, two cover the pre-dispatch reads
# and the third pins the raised message.
from turnstone.core.deadline import StreamAbortRef
provider = _FlakyProvider(
[IncompleteStreamError("closed by abort"), CompletionResult(content="never")]
)
lane = ModelLane(provider=provider, client=object(), model="m")
ref = StreamAbortRef()
dispatch = provider.create_streaming
def _abort_after_dispatch(**kwargs: Any) -> Any:
stream = dispatch(**kwargs)
ref.abort() # the deadline daemon fires; the request is already out
return stream
monkeypatch.setattr(provider, "create_streaming", _abort_after_dispatch)
with (
caplog.at_level(logging.WARNING, logger="turnstone.core.model_turn"),
pytest.raises(IncompleteStreamError),
):
model_turn(lane, [Turn.user("x")], cancel_ref=ref)
assert len(provider.calls) == 1
# Isolates THIS gate from the post-backoff one its sibling covers. The
# abort is read where the failure surfaces, so the loop never announces a
# re-issue it will not make; delete that gate and the backoff arm still
# ends at one dispatch, but it logs on the way — which is what makes this
# assertion, and not the call count, the discriminating one.
assert "model_turn.drain_retry" not in caplog.text
def test_abort_landing_during_the_backend_auth_mint_still_never_dispatches() -> None:
# The window an entry-only check cannot see: on a dynamic-auth alias
# the resolve can block for seconds on a cache miss, so an abort can
# land after the entry read and before the request. The read
# immediately before create_streaming is what covers it.
from turnstone.core.deadline import DeadlineCancelledError, StreamAbortRef
provider = _FakeProvider([CompletionResult(content="never")])
ref = StreamAbortRef()
client = MagicMock()
def _abort_during_mint(alias: str) -> str:
ref.abort() # the user hits Stop while the mint is blocked
return "minted-token"
lane = ModelLane(
provider=provider,
client=client,
model="m",
alias="obo-gateway",
backend_auth_resolver=_abort_during_mint,
)
with pytest.raises(DeadlineCancelledError):
model_turn(lane, [Turn.user("x")], cancel_ref=ref)
assert provider.calls == []
def test_pre_dispatch_abort_precedes_the_backend_auth_mint() -> None:
# Placement of the FIRST read: an already-abandoned call skips the
# resolve entirely. On a cache miss that resolve is a network mint
# under a cluster-wide lock, so this is work worth not doing — but the
# invariant itself rides the read before create_streaming, not this one.
from turnstone.core.deadline import DeadlineCancelledError, StreamAbortRef
provider = _FakeProvider([CompletionResult(content="never")])
resolver = MagicMock(return_value="minted-token")
client = MagicMock()
lane = ModelLane(
provider=provider,
client=client,
model="m",
alias="obo-gateway",
backend_auth_resolver=resolver,
)
ref = StreamAbortRef()
ref.abort()
with pytest.raises(DeadlineCancelledError):
model_turn(lane, [Turn.user("x")], cancel_ref=ref)
resolver.assert_not_called()
client.with_options.assert_not_called()
assert provider.calls == []
def test_pre_dispatch_abort_does_not_read_as_a_context_overflow() -> None:
# A latent coupling, pinned deliberately rather than a live path: today
# compaction's ``except`` arm re-checks the session first and raises
# GenerationCancelled, and ``_stop_retrying`` short-circuits on the class
# gate, so this message never reaches ``_is_ctx_overflow``. It would the
# moment either shortcut moves — and ``_is_ctx_overflow`` classifies an
# unrecognized class by TEXT, so an overflow reading would send the
# compaction lane subdividing. The message is a wire contract; pin it.
from turnstone.core.deadline import DeadlineCancelledError, StreamAbortRef
from turnstone.core.session import _is_ctx_overflow
provider = _FakeProvider([CompletionResult(content="never")])
lane = ModelLane(provider=provider, client=object(), model="m")
ref = StreamAbortRef()
ref.abort()
with pytest.raises(DeadlineCancelledError) as excinfo:
model_turn(lane, [Turn.user("x")], cancel_ref=ref)
assert not _is_ctx_overflow(excinfo.value)
def _real_semantics_store(**stored: Any) -> SimpleNamespace:
"""A ConfigStore fake with the REAL ``get()`` semantics.
A stored key returns its value; a never-stored key returns the
SETTINGS registry default — which for the sampling keys IS the unset
sentinel (``None`` / ``""``). The old fakes returned ``None`` on any
miss, which masked the default-on-miss collision the round-2 review
caught: never fake a store rung more forgiving than the real one.
"""
from turnstone.core.settings_registry import SETTINGS
def _get(key: str, default: Any = ...) -> Any:
if key in stored:
return stored[key]
if default is not ...:
return default
defn = SETTINGS.get(key)
return defn.default if defn else None
return SimpleNamespace(get=_get)
def test_model_turn_lowers_turns_and_threads_lane_config() -> None:
caps = ModelCapabilities(max_output_tokens=1234)
extra = {"chat_template_kwargs": {"enable_thinking": True}}
provider = _FakeProvider([CompletionResult(content="hi")])
lane = _lane(provider, capabilities=caps, extra_params=extra)
result = model_turn(
lane,
[Turn.user("x")],
tools=[{"type": "function", "function": {"name": "f", "parameters": {}}}],
max_tokens=99,
temperature=0.1,
reasoning_effort="low",
)
(call,) = provider.calls
assert call["messages"][0]["role"] == "user"
assert call["messages"][0]["content"] == "x"
assert call["capabilities"] is caps
assert call["extra_params"] is extra
assert call["max_tokens"] == 99
assert call["temperature"] == 0.1
assert call["reasoning_effort"] == "low"
# No registry on the lane → the operator replay flag resolves False.
assert call["replay_reasoning_to_model"] is False
assert result.turn.role is Role.ASSISTANT
assert result.content == "hi"
assert result.finish_reason == "stop"
def test_model_turn_returns_usage_verbatim() -> None:
usage = UsageInfo(prompt_tokens=9, completion_tokens=1, total_tokens=10)
provider = _FakeProvider([CompletionResult(content="", usage=usage)])
result = model_turn(_lane(provider), [Turn.user("x")])
# Value equality, not identity: ``drain_stream`` max-merges usage across
# chunks into its own instance so it never mutates the provider's object.
assert result.usage == usage
def test_mint_rewrites_mirror_records_map_and_native_keeps_original() -> None:
provider = _FakeProvider(
[
CompletionResult(
content="",
tool_calls=[
{
"id": "call_0",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
provider_blocks=[{"type": "tool_use", "id": "call_0", "name": "f"}],
)
]
)
wire_id_map: dict[str, str] = {}
result = model_turn(
_lane(provider),
[Turn.user("x")],
mint=lambda original: f"parent::r1s1::{original}",
wire_id_map=wire_id_map,
)
# The mirror (execution view) and the Turn both carry the minted id …
assert result.tool_calls[0]["id"] == "parent::r1s1::call_0"
assert result.turn.tool_calls[0].id == "parent::r1s1::call_0"
# … the map records the recovery path …
assert wire_id_map == {"parent::r1s1::call_0": "call_0"}
# … and the native block keeps the provider-original id verbatim (it may
# sit under a reasoning signature and is never rewritten).
assert result.turn.native is not None
assert result.turn.native.blocks[0]["id"] == "call_0"
assert result.turn.native.producer == "openai-compatible"
def test_restore_maps_minted_ids_back_on_the_wire() -> None:
minted = "parent::r1s1::call_0"
provider = _FakeProvider([CompletionResult(content="done")])
turns = [
Turn.user("go"),
Turn.assistant("", tool_calls=(ToolCall(id=minted, name="f", arguments="{}"),)),
Turn.tool(minted, "result"),
]
model_turn(_lane(provider), turns, wire_id_map={minted: "call_0"})
(call,) = provider.calls
assistant = next(m for m in call["messages"] if m["role"] == "assistant")
tool = next(m for m in call["messages"] if m["role"] == "tool")
assert assistant["tool_calls"][0]["id"] == "call_0"
assert tool["tool_call_id"] == "call_0"
def test_blank_ids_repair_native_lane_pairwise() -> None:
# Google-compat shape: blank id on BOTH the mirror and the raw fidelity
# block. The manufactured uuid lands in both (positional pairing), so
# the thought_signature-bearing block SURVIVES instead of the turn
# degrading to loose reasoning text — the unblock for the Gemini judge
# evidence loop on blank-id compat responses.
provider = _FakeProvider(
[
CompletionResult(
content="",
tool_calls=[
{"id": "", "type": "function", "function": {"name": "f", "arguments": "{}"}}
],
provider_blocks=[
{
"id": "",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
"thought_signature": "sig123",
}
],
reasoning="thought",
)
]
)
result = model_turn(_lane(provider), [Turn.user("x")])
manufactured = result.tool_calls[0]["id"]
assert manufactured.startswith("call_")
assert result.turn.native is not None
blocks = list(result.turn.native.blocks)
# The fidelity block survives, id-agreeing with the mirror, signature
# untouched; the loose reasoning still synthesizes alongside it.
assert blocks[0]["id"] == manufactured
assert blocks[0]["thought_signature"] == "sig123"
assert blocks[-1]["type"] == "reasoning_text"
def test_blank_id_repair_never_rewrites_nonblank_ids() -> None:
provider = _FakeProvider(
[
CompletionResult(
content="",
tool_calls=[
{"id": "call_7", "type": "function", "function": {"name": "a"}},
{"id": "", "type": "function", "function": {"name": "b"}},
],
provider_blocks=[
{"id": "call_7", "type": "function", "function": {"name": "a"}},
{"id": "", "type": "function", "function": {"name": "b"}},
],
)
]
)
result = model_turn(_lane(provider), [Turn.user("x")])
assert result.turn.native is not None
blocks = list(result.turn.native.blocks)
# Provider-assigned id untouched (it may sit under a signature) …
assert blocks[0]["id"] == "call_7"
# … only the blank one was manufactured, agreeing with its mirror twin.
assert blocks[1]["id"] == result.tool_calls[1]["id"]
assert blocks[1]["id"].startswith("call_")
def test_blank_id_pairing_mismatch_falls_back_to_reasoning_text_drop() -> None:
# Two mirror calls but only one client block: no trustworthy pairing —
# the total drop rule (the #825-converged fallback) keeps only the
# loose-text reasoning synth.
provider = _FakeProvider(
[
CompletionResult(
content="",
tool_calls=[
{"id": "", "type": "function", "function": {"name": "a"}},
{"id": "", "type": "function", "function": {"name": "b"}},
],
provider_blocks=[{"type": "function", "id": "", "function": {"name": "a"}}],
reasoning="thought",
)
]
)
result = model_turn(_lane(provider), [Turn.user("x")])
assert result.tool_calls[0]["id"].startswith("call_")
assert result.turn.native is not None
assert [b["type"] for b in result.turn.native.blocks] == ["reasoning_text"]
assert result.turn.native.blocks[0]["text"] == "thought"
def test_orphan_client_tool_blocks_stripped_when_no_tool_calls() -> None:
provider = _FakeProvider(
[
CompletionResult(
content="truncated",
tool_calls=None,
provider_blocks=[{"type": "tool_use", "id": "x", "name": "f"}],
)
]
)
result = model_turn(_lane(provider), [Turn.user("x")])
# A tool_use with no mirrored call would replay with no matching
# tool_result — the finalize gate strips it, leaving no lane at all.
assert result.turn.native is None
def test_live_operator_flags_reresolve_per_call() -> None:
registry = _fake_registry(replay=False)
provider = _FakeProvider([CompletionResult(content="a"), CompletionResult(content="b")])
lane = _lane(provider, alias="ali", registry=registry)
model_turn(lane, [Turn.user("x")])
# Operator flips the toggle mid-session (admin write → registry reload).
registry.get_config.return_value.replay_reasoning_to_model = True
model_turn(lane, [Turn.user("x")])
first, second = provider.calls
assert first["replay_reasoning_to_model"] is False
assert second["replay_reasoning_to_model"] is True
def test_resolve_lane_respects_preresolved_values() -> None:
provider = _FakeProvider([])
caps = ModelCapabilities(max_output_tokens=7)
lane = resolve_lane(provider, object(), "m", capabilities=caps, extra_params={"k": "v"})
assert lane.capabilities is caps
assert lane.extra_params == {"k": "v"}
# Explicit None is a valid resolved value, distinct from "resolve for me".
lane_none = resolve_lane(provider, object(), "m", capabilities=caps, extra_params=None)
assert lane_none.extra_params is None
def test_resolve_lane_merges_registry_capability_overrides() -> None:
provider = _FakeProvider([])
registry = _fake_registry(capabilities={"max_output_tokens": 42, "not_a_field": 1})
lane = resolve_lane(provider, object(), "m", alias="ali", registry=registry)
assert lane.capabilities is not None
assert lane.capabilities.max_output_tokens == 42
def test_vllm_attach_gates() -> None:
from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider
msgs = [
{"role": "user", "content": "q"},
{
"role": "assistant",
"content": "a",
"_provider_content": [{"type": "reasoning_text", "text": "cot"}],
},
]
# Non-Chat-Completions provider: untouched (identity).
assert maybe_attach_vllm_chat_reasoning(msgs, _FakeProvider([]), None, "ali") is msgs # type: ignore[arg-type]
chat = OpenAIChatCompletionsProvider()
# All three gates open → reasoning field attached.
on = _fake_registry(server_compat={"server_type": "vllm"}, replay=True)
out = maybe_attach_vllm_chat_reasoning(msgs, chat, on, "ali")
assert out[1]["reasoning"] == "cot"
# Operator flag off → untouched.
off = _fake_registry(server_compat={"server_type": "vllm"}, replay=False)
assert maybe_attach_vllm_chat_reasoning(msgs, chat, off, "ali") is msgs
# Wrong server type → untouched.
sglang = _fake_registry(server_compat={"server_type": "sglang"}, replay=True)
assert maybe_attach_vllm_chat_reasoning(msgs, chat, sglang, "ali") is msgs
def test_synth_reasoning_block_appends_with_source_and_skips_native() -> None:
registry = _fake_registry(server_compat={"server_type": "vllm"})
fidelity = [{"type": "tool_calls", "raw": True}]
out = synth_reasoning_block(fidelity, ["thought"], registry=registry, alias="ali")
# Appends (Google fidelity blocks survive) and tags the source server.
assert out[0] is fidelity[0]
assert out[1] == {"type": "reasoning_text", "text": "thought", "source": "vllm"}
# A native reasoning-bearing block suppresses synthesis (identity return).
native = [{"type": "thinking", "thinking": "t", "signature": "s"}]
assert synth_reasoning_block(native, ["thought"]) is native
def test_finalize_keeps_full_lane_with_tool_calls_and_clean_ids() -> None:
blocks = [
{"type": "thinking", "thinking": "t", "signature": "s"},
{"type": "tool_use", "id": "toolu_1", "name": "f"},
]
out = finalize_provider_blocks(blocks, [""], has_tool_calls=True)
assert out == blocks
def test_mint_without_wire_id_map_raises() -> None:
provider = _FakeProvider([])
with pytest.raises(ValueError, match="wire_id_map"):
model_turn(_lane(provider), [Turn.user("x")], mint=lambda o: f"p::{o}")
# Nothing reached the provider — the guard fires before lowering.
assert provider.calls == []
def test_temperature_inherits_lane_value_when_caller_omits() -> None:
provider = _FakeProvider([CompletionResult(content="")])
lane = _lane(provider, temperature=1.3)
model_turn(lane, [Turn.user("x")])
assert provider.calls[0]["temperature"] == 1.3
def test_temperature_caller_value_wins_over_lane() -> None:
provider = _FakeProvider([CompletionResult(content="")])
lane = _lane(provider, temperature=1.3)
model_turn(lane, [Turn.user("x")], temperature=0.9)
assert provider.calls[0]["temperature"] == 0.9
def test_temperature_unresolved_passes_none_and_wire_omits_it() -> None:
# No caller value, no lane value → model_turn passes temperature=None,
# and the PROVIDER layer omits the field from the wire so the server
# default applies (house rule: code never pins one). Both halves are
# pinned: a Python-signature default of 0.5 anywhere on this path is a
# hidden universal pin — the exact bug the second xhigh review caught.
provider = _FakeProvider([CompletionResult(content="")])
model_turn(_lane(provider), [Turn.user("x")])
assert provider.calls[0]["temperature"] is None
from turnstone.core.providers._openai_common import apply_temperature
kwargs: dict[str, Any] = {}
apply_temperature(kwargs, ModelCapabilities(), None, "medium")
assert "temperature" not in kwargs # None never reaches the wire
apply_temperature(kwargs, ModelCapabilities(), 1.0, "medium")
assert kwargs["temperature"] == 1.0 # a real value still does
def test_resolve_lane_global_config_store_rung() -> None:
# The global rung fires only when the operator actually STORED a
# value; the registry default is the unset sentinel (None), so an
# untouched install resolves None → the wire omits the field.
provider = _FakeProvider([])
registry = _fake_registry(temperature=None)
store = _real_semantics_store(**{"model.temperature": 1.0})
lane = resolve_lane(provider, object(), "m", alias="ali", registry=registry, config_store=store)
assert lane.temperature == 1.0
# The per-model value wins over the global rung.
registry2 = _fake_registry(temperature=0.3)
lane2 = resolve_lane(
provider, object(), "m", alias="ali", registry=registry2, config_store=store
)
assert lane2.temperature == 0.3
# Never-stored global → None (the round-2 headline: ConfigStore.get
# must NOT manufacture a wire value on a miss).
lane3 = resolve_lane(
provider,
object(),
"m",
alias="ali",
registry=_fake_registry(temperature=None),
config_store=_real_semantics_store(),
)
assert lane3.temperature is None
def test_resolve_lane_reasoning_effort_operator_rungs() -> None:
# The lane carries the OPERATOR rungs only: per-model config → stored
# global setting → None. The in-code model definition (caps default)
# applies at the model_turn call, so an operator-silent lane stays
# None — the assignment scheme's "if not set, we don't send it".
provider = _FakeProvider([])
# Per-model config wins.
reg = _fake_registry()
reg.get_config.return_value.reasoning_effort = "high"
lane = resolve_lane(provider, object(), "m", alias="ali", registry=reg)
assert lane.reasoning_effort == "high"
# Stored global setting rung.
reg2 = _fake_registry()
reg2.get_config.return_value.reasoning_effort = None
store = _real_semantics_store(**{"model.reasoning_effort": "low"})
lane2 = resolve_lane(provider, object(), "m", alias="ali", registry=reg2, config_store=store)
assert lane2.reasoning_effort == "low"
# Empty string at any rung is the unset sentinel (a valid settings
# choice meaning fall through) — with a never-stored global (registry
# default IS "") the lane stays operator-silent.
reg3 = _fake_registry()
reg3.get_config.return_value.reasoning_effort = ""
lane3 = resolve_lane(
provider, object(), "m", alias="ali", registry=reg3, config_store=_real_semantics_store()
)
assert lane3.reasoning_effort is None
# Bare lane (no registry, no store): None.
assert resolve_lane(provider, object(), "m").reasoning_effort is None
def test_model_turn_effort_lower_rungs() -> None:
# Below the lane's operator rungs, model_turn applies exactly one more
# rung — the in-code model definition (caps) — then None (wire
# omission). There is deliberately NO caller-default rung: a
# code-chosen effort is an unvetted token on local vocabularies
# (effort_passthrough forwards verbatim) and flips template thinking
# toggles the operator never engaged.
# Bare hand-built lane: nothing anywhere → the provider receives None.
provider = _FakeProvider([CompletionResult(content="")])
model_turn(_lane(provider), [Turn.user("x")])
assert provider.calls[0]["reasoning_effort"] is None
# In-code model definition rung: a declared caps default applies…
caps = ModelCapabilities(default_reasoning_effort="high")
provider2 = _FakeProvider([CompletionResult(content="")])
model_turn(_lane(provider2, capabilities=caps), [Turn.user("x")])
assert provider2.calls[0]["reasoning_effort"] == "high"
# …loses to an operator value on the lane…
provider3 = _FakeProvider([CompletionResult(content="")])
model_turn(
_lane(provider3, capabilities=caps, reasoning_effort="xhigh"),
[Turn.user("x")],
)
assert provider3.calls[0]["reasoning_effort"] == "xhigh"
# …and to an explicit relay (the "none" knob stays distinct from unset).
provider4 = _FakeProvider([CompletionResult(content="")])
model_turn(
_lane(provider4, capabilities=caps),
[Turn.user("x")],
reasoning_effort="none",
)
assert provider4.calls[0]["reasoning_effort"] == "none"
def test_model_turn_fetches_config_once_per_call() -> None:
# ONE get_config per plant call feeds both live flags (replay + vLLM
# attach) — a hot-reload between them cannot mix config generations
# within a single request.
registry = _fake_registry(replay=True)
provider = _FakeProvider([CompletionResult(content="")])
lane = _lane(provider, alias="ali", registry=registry)
model_turn(lane, [Turn.user("x")])
assert registry.get_config.call_count == 1
def test_resolve_lane_inherits_config_temperature() -> None:
provider = _FakeProvider([])
registry = _fake_registry(temperature=0.7)
lane = resolve_lane(provider, object(), "m", alias="ali", registry=registry)
assert lane.temperature == 0.7
# Exactly ONE config fetch feeds caps + extra_params + temperature —
# no cross-generation mixing on a registry hot-reload.
assert registry.get_config.call_count == 1
def test_resolve_lane_survives_get_config_raise() -> None:
provider = _FakeProvider([])
registry = MagicMock()
registry.get_config.side_effect = ValueError("Unknown model alias")
lane = resolve_lane(provider, object(), "m", alias="gone", registry=registry)
# Every facet degrades to its miss behavior instead of raising into a
# caller's constructor (the judge alias-resolution abort case).
assert lane.capabilities is not None
assert lane.extra_params is None
assert lane.temperature is None
def test_resolve_capabilities_survives_get_config_raise() -> None:
from turnstone.core.model_turn import resolve_capabilities
provider = _FakeProvider([])
registry = MagicMock()
registry.get_config.side_effect = KeyError("gone")
caps = resolve_capabilities(provider, "m", "gone", registry)
assert caps == ModelCapabilities()
def test_inline_tags_segregate_to_native_reasoning_text_and_clean_content() -> None:
# A passthrough server's tagged content, drained through the real seam:
# the turn's text is IR-clean and the extracted reasoning lands in the
# native lane as the path-3 synth block (so it survives reload and the
# operator-gated replay), never in any consumer-visible content.
provider = _FakeProvider([CompletionResult(content="<think>plan</think>answer")])
result = model_turn(_lane(provider), [Turn.user("q")])
assert result.content == "answer"
assert result.turn.text == "answer"
assert result.turn.native is not None
synth = [b for b in result.turn.native.blocks if b.get("type") == "reasoning_text"]
assert len(synth) == 1
assert synth[0]["text"] == "plan"
def test_synth_bail_is_silent_and_leaks_nothing(
caplog: pytest.LogCaptureFixture,
) -> None:
# Bailing on an existing native reasoning block is the ROUTINE no-op on
# Anthropic/Responses lanes (reasoning_delta mirrors the block): no log
# event here, and reasoning text never reaches a log payload. The
# genuinely anomalous shape (inline-EXTRACTED text beside a native
# block) is logged at the drain, where it is distinguishable.
import logging
from turnstone.core.model_turn import synth_reasoning_block
secret_reasoning = "the plan nobody logs"
with caplog.at_level(logging.DEBUG):
blocks = synth_reasoning_block(
[{"type": "thinking", "thinking": "native"}], [secret_reasoning]
)
assert blocks == [{"type": "thinking", "thinking": "native"}]
assert secret_reasoning not in caplog.text
def test_capability_bool_overrides_coerced() -> None:
"""The capabilities dict is hand-edited JSON: a string "false" is
truthy, and left raw it would flip every downstream truthiness read
(a ``server_parses_reasoning: "false"`` typo silently turning the
inline tag scan off is #940 reopened by punctuation). Recognized
spellings coerce, ints pass through ``bool()``, and an unrecognized
value drops the key so the field keeps its default."""
from turnstone.core.model_turn import apply_capability_overrides
base = ModelCapabilities()
off = apply_capability_overrides(base, {"server_parses_reasoning": "false"})
assert off.server_parses_reasoning is False
on = apply_capability_overrides(base, {"server_parses_reasoning": "true"})
assert on.server_parses_reasoning is True
coerced = apply_capability_overrides(base, {"supports_vision": 1, "supports_tools": 0})
assert coerced.supports_vision is True
assert coerced.supports_tools is False
# Unrecognized string: key dropped, default kept; non-bool fields untouched.
kept = apply_capability_overrides(
base, {"server_parses_reasoning": "maybe", "thinking_mode": "manual"}
)
assert kept.server_parses_reasoning is False
assert kept.thinking_mode == "manual"