ewilliams28/turnstone
1
0
Fork 0
mirror of https://github.com/turnstonelabs/turnstone.git synced 2026-08-12 23:12:23 -06:00
Code Issues Packages Projects Releases Wiki Activity
Files
e103af94e79a7ff87c1a9cbe19d5b95b9120119b
turnstone/tests/test_session_chat_reasoning_replay.py
T
Patrick Buckley e103af94e7 test(832): port the seam-coupled suites to the folded architecture 
Seventeen files, ~1,300 tests, re-pointed or redesigned per the triage
ledger's recipes: wholesale turn-scripting moves to ModelTurnResult
fakes; streaming-behavior suites drive the REAL wrapper+consumer+drain
path through armed provider fakes (tests/_parity_832.arm_session — the
eager cancel_ref append every real adapter performs, exception elements
for creation-phase failures, sequential per-turn scripts, and the title
lane quieted: a provider-level fake otherwise loses its one-shot script
to best-effort title generation, which is why the old tests patched at
the session level); kwarg-capture suites assert through model_turn's
create_streaming call with system-prepend-aware index math; delegate
wrappers retired by the fold re-aim at their model_turn module twins.
Old-architecture pins are replaced by their new-world equivalents rather
than deleted: no shared cancel ref exists (pinned), the handle slot and
per-attempt refs carry the cancel surface, the retry gate reads the
serving lane's provider, and a superseded generation's death exits send
silently as cancelled — a named delta: no arbitrary exception class
escapes an orphaned thread anymore.

Full suite: 10651 passed, 10 skipped. The wire-payload goldens pass
untouched — the fold's lowering composition is byte-equivalent on every
provider's request path, as designed.
2026-08-06 01:04:32 -07:00

418 lines
18 KiB
Python
Raw Blame History

"""Session-level integration tests for Phase 5 (Chat Completions
``reasoning`` field replay against vLLM).
Phase 5 is the only reasoning-replay path that does NOT use the static
``supports_reasoning_replay`` capability gate. It's a parallel path to
Paths 1+2, gated on three conditions and nothing else:
1. Provider is ``OpenAIChatCompletionsProvider``.
2. ``server_compat.server_type == "vllm"``.
3. Operator-set ``ModelConfig.replay_reasoning_to_model`` is True.
These tests drive through ``model_turn.maybe_attach_vllm_chat_reasoning``
to pin each gate independently, then one round-trip test through the real
OpenAI Python SDK + httpx MockTransport confirms the ``reasoning`` field
actually reaches the wire bytes (the SDK-boundary guarantee that the
attach approach hinges on).
The gate ran behind a ``ChatSession`` wrapper until #832 folded the main
loop onto ``model_turn``; the attach is now one of the seam's own lowering
passes, reading the lane's registry + alias. Same three gates, one
indirection down — and both session funnels (the streaming turn and
``_utility_completion``) reach it through that one seam.
"""
from __future__ import annotations
import json
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
import httpx
import pytest
from tests._parity_832 import ArmedHandle
from tests._session_helpers import as_stream, mock_completion_result, think_tag_stream
from tests._session_helpers import make_session as _make_session
from turnstone.core.model_turn import maybe_attach_vllm_chat_reasoning
from turnstone.core.providers._anthropic import AnthropicProvider
from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider
from turnstone.core.providers._openai_responses import OpenAIResponsesProvider
from turnstone.core.trajectory import turns_from_dicts
def _vllm_registry(*, replay: bool = True, alias: str = "qwen3") -> Any:
"""Stub registry with a vLLM-typed server_compat profile and the
Phase 5 operator flag toggleable.
Mirrors production ModelConfig shape: ``server_compat`` lives at
the top-level dataclass field, NOT inside ``capabilities``. Both
model_registry loader paths (DB row at line 401, config.toml at
line 485) ``caps.pop("server_compat", {})`` and hoist it up, so a
stub that populates ``capabilities["server_compat"]`` would mask
the same bug Phase 5 stepped on initially.
"""
cfg = SimpleNamespace(
replay_reasoning_to_model=replay,
capabilities={},
server_compat={"server_type": "vllm"},
auth_mode="static",
obo_audience="",
)
return SimpleNamespace(
has_alias=lambda a: a == alias,
get_config=lambda a: cfg if a == alias else (_ for _ in ()).throw(KeyError(a)),
)
def _registry_with_server_type(server_type: str, *, replay: bool = True) -> Any:
cfg = SimpleNamespace(
replay_reasoning_to_model=replay,
capabilities={},
server_compat={"server_type": server_type},
auth_mode="static",
obo_audience="",
)
return SimpleNamespace(
has_alias=lambda _alias: True,
get_config=lambda _alias: cfg,
)
def _assistant_msg_with_thinking(text: str = "let me think") -> dict[str, Any]:
"""Anthropic-shape persisted reasoning — the cross-provider case
where workstream started on Anthropic and operator flipped to
vLLM-served Qwen3. Helper must extract the text and discard the
Anthropic signature."""
return {
"role": "assistant",
"content": "Final answer.",
"_provider_content": [
{"type": "thinking", "thinking": text, "signature": "sig"},
{"type": "text", "text": "Final answer."},
],
}
# ---------------------------------------------------------------------------
# Gate tests via ``maybe_attach_vllm_chat_reasoning`` directly
# ---------------------------------------------------------------------------
class TestMaybeAttachVllmChatReasoningGates:
"""The seam pass that combines all three Phase 5 gates.
Reads the registry + alias the LANE carries — what the session
wrapper used to hand it, resolved per call inside ``model_turn`` so a
mid-session admin toggle keeps applying.
"""
def test_all_gates_pass_attaches_reasoning(self) -> None:
provider = OpenAIChatCompletionsProvider()
msgs = [{"role": "user", "content": "q"}, _assistant_msg_with_thinking("CoT")]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, _vllm_registry(replay=True), "qwen3")
assert out[1]["reasoning"] == "CoT"
def test_non_chat_completions_provider_is_no_op(self) -> None:
# Provider isinstance gate: Anthropic / Responses / Google all
# have their own reasoning-replay paths (Paths 1 / 2) — Phase 5
# must not double-attach.
provider = AnthropicProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, _vllm_registry(replay=True), "qwen3")
assert "reasoning" not in out[0]
# Same reference — no copy made.
assert out[0] is msgs[0]
def test_openai_responses_provider_is_no_op(self) -> None:
# OpenAIResponsesProvider is a top-level class (not a subclass of
# OpenAIChatCompletionsProvider) — the isinstance gate rejects
# it cleanly. This is the load-bearing distinction; an
# accidental inheritance refactor would break the gate.
provider = OpenAIResponsesProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, _vllm_registry(replay=True), "qwen3")
assert "reasoning" not in out[0]
@pytest.mark.parametrize("server_type", ["", "llama.cpp", "sglang", "openai", "unknown"])
def test_non_vllm_server_type_is_no_op(self, server_type: str) -> None:
# Server-type pin bounds blast radius — canonical OpenAI Chat
# Completions, llama.cpp, sglang, and any unrecognised server
# never receive the non-standard ``reasoning`` field.
registry = _registry_with_server_type(server_type, replay=True)
provider = OpenAIChatCompletionsProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, registry, "some-model")
assert "reasoning" not in out[0]
def test_operator_flag_off_is_no_op(self) -> None:
registry = _vllm_registry(replay=False) # operator flag OFF
provider = OpenAIChatCompletionsProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, registry, "qwen3")
assert "reasoning" not in out[0]
def test_missing_registry_is_no_op(self) -> None:
provider = OpenAIChatCompletionsProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, None, "qwen3")
assert "reasoning" not in out[0]
def test_missing_alias_is_no_op(self) -> None:
# A lane outside the registry carries ``alias=""``.
provider = OpenAIChatCompletionsProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, _vllm_registry(replay=True), "")
assert "reasoning" not in out[0]
def test_registry_exception_is_no_op(self) -> None:
# Defensive: registry lookup raising must degrade to no-attach,
# not break the call. Conservative default — operator can
# always re-flip the flag once the registry is healthy.
def boom(_alias: str) -> Any:
raise KeyError("missing")
registry = SimpleNamespace(get_config=boom)
provider = OpenAIChatCompletionsProvider()
msgs = [_assistant_msg_with_thinking()]
out = maybe_attach_vllm_chat_reasoning(msgs, provider, registry, "qwen3")
assert "reasoning" not in out[0]
def test_alias_selects_its_own_config(self) -> None:
# The gate reads the config of the alias the LANE resolved — a
# fallback lane's alias, not the session's primary. Mirrors the
# per-alias behaviour pinned for resolve_replay_reasoning_to_model.
def per_alias(alias: str) -> Any:
return SimpleNamespace(
replay_reasoning_to_model=(alias == "wants-replay"),
capabilities={},
server_compat={"server_type": "vllm"},
)
registry = SimpleNamespace(get_config=per_alias)
provider = OpenAIChatCompletionsProvider()
msgs = [_assistant_msg_with_thinking()]
# Flag off for this alias → no attach.
out_primary = maybe_attach_vllm_chat_reasoning(msgs, provider, registry, "primary")
assert "reasoning" not in out_primary[0]
# Flag on for this one → attached.
out_replay = maybe_attach_vllm_chat_reasoning(msgs, provider, registry, "wants-replay")
assert out_replay[0]["reasoning"] == "let me think"
# ---------------------------------------------------------------------------
# End-to-end: SDK passthrough is the load-bearing assumption. Verify it
# with a real OpenAI client wired against an httpx MockTransport that
# inspects the body (per feedback_mock_transport_body_inspection).
# ---------------------------------------------------------------------------
class TestReasoningFieldReachesWireBytes:
"""One round-trip test through the real OpenAI Python SDK confirms
the ``reasoning`` field on an assistant message dict survives the
sanitize_messages strip (only ``_``-prefixed keys are dropped) AND
the SDK's TypedDict input shape (no runtime field filtering)."""
def _capture_client(self) -> tuple[Any, list[dict[str, Any]]]:
from openai import OpenAI
captured: list[dict[str, Any]] = []
def handler(request: httpx.Request) -> httpx.Response:
body = request.content.decode("utf-8") if request.content else ""
captured.append({"url": str(request.url), "body": body})
return httpx.Response(
200,
json={
"id": "chatcmpl-vllm-spike",
"object": "chat.completion",
"created": 0,
"model": "qwen3-test",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
client = OpenAI(
api_key="sk-test",
base_url="http://mock.local/v1",
http_client=httpx.Client(transport=httpx.MockTransport(handler)),
)
return client, captured
def test_reasoning_field_present_in_wire_body_when_attached(self) -> None:
# Send messages that have the Phase 5 ``reasoning`` field
# attached. Drive a real provider call through the real OpenAI
# SDK + mock httpx and verify the field is in the captured POST
# body — the SDK passthrough assumption that the entire
# session-level approach hinges on.
client, captured = self._capture_client()
provider = OpenAIChatCompletionsProvider()
# Mimic the post-attach message shape that
# ``maybe_attach_vllm_chat_reasoning`` produces, then sanitize.
# ``sanitize_messages`` runs inside provider._prepare_messages
# and must preserve the non-``_``-prefixed ``reasoning`` field.
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": "Final answer.",
"reasoning": "vLLM-shaped CoT text",
"_provider_content": [{"type": "reasoning_text", "text": "vLLM-shaped CoT text"}],
},
{"role": "user", "content": "follow-up"},
]
provider.create_streaming(
client=client,
model="qwen3-test",
messages=messages,
max_tokens=10,
temperature=0.5,
reasoning_effort="medium",
extra_params=None,
capabilities=provider.get_capabilities("qwen3-test"),
)
assert captured, "no request captured"
body = json.loads(captured[0]["body"])
assistant_msg = next(m for m in body["messages"] if m["role"] == "assistant")
# Wire-format guarantee: field survives sanitize_messages + SDK.
assert assistant_msg.get("reasoning") == "vLLM-shaped CoT text"
# And the ``_``-prefixed sibling is stripped by sanitize_messages.
assert "_provider_content" not in assistant_msg
def test_reasoning_field_absent_when_not_attached(self) -> None:
# Negative case: when the session-level gate decided NOT to
# attach (any of the 3 gates failed), the SDK round-trip carries
# no ``reasoning`` field — the operator's opt-out / non-vLLM
# destination is honoured all the way to the wire.
client, captured = self._capture_client()
provider = OpenAIChatCompletionsProvider()
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": "Final answer.",
# No ``reasoning`` field — pre-attach shape, gate said no.
"_provider_content": [{"type": "reasoning_text", "text": "would-have-replayed"}],
},
{"role": "user", "content": "follow-up"},
]
provider.create_streaming(
client=client,
model="gpt-4o", # canonical OpenAI, not vLLM
messages=messages,
max_tokens=10,
temperature=0.5,
reasoning_effort="medium",
extra_params=None,
capabilities=provider.get_capabilities("gpt-4o"),
)
body = json.loads(captured[0]["body"])
assistant_msg = next(m for m in body["messages"] if m["role"] == "assistant")
assert "reasoning" not in assistant_msg
assert "_provider_content" not in assistant_msg
# ---------------------------------------------------------------------------
# Call-site integration: confirm the streaming turn and _utility_completion
# both reach the attach. Post-#832 both funnel through ``model_turn``,
# which runs the pass itself — so these pin that each funnel still goes
# through that seam (a call site that grew a private wire path would skip
# it). The sub-agent loop rides the same seam via ``_run_agent``'s
# ``_api_call``; its assistant turns carry no ``_provider_content`` in
# practice, so the pass no-ops there rather than being wired around.
# ---------------------------------------------------------------------------
class TestCallSitesInvokeMaybeAttach:
"""The pass does nothing for a call site that doesn't reach it.
Verify the wiring at each — without this, a refactor that gives one
funnel its own wire build would silently regress Phase 5 there."""
def test_streaming_call_site_attaches(self) -> None:
session = _make_session()
session._registry = _vllm_registry(replay=True)
session._model_alias = "qwen3"
session.model = "qwen3"
captured: dict[str, Any] = {}
def capture_streaming(**kwargs: Any) -> Any:
captured.update(kwargs)
# The armed/creation classifier reads the cancel_ref, so the
# fake arms it eagerly like every real adapter.
ref = kwargs.get("cancel_ref")
if ref is not None:
ref.append(ArmedHandle())
return think_tag_stream("ok")
provider = OpenAIChatCompletionsProvider()
# Patch only the network-facing method so we don't actually call
# an LLM, but keep the real provider instance (so the isinstance
# gate sees the right type).
provider.create_streaming = capture_streaming # type: ignore[method-assign]
session._provider = provider
session.messages = turns_from_dicts([_assistant_msg_with_thinking("from the main loop")])
session._stream_response(0)
# The messages handed to the provider include the attached
# reasoning field — proves the streaming turn reached the attach.
# The wire list carries the session's system messages now, so the
# assistant turn is found by role, not by index.
msgs_sent = captured["messages"]
assistant = next(m for m in msgs_sent if m["role"] == "assistant")
assert assistant["reasoning"] == "from the main loop"
def test_utility_completion_call_site_attaches(self) -> None:
session = _make_session()
session._registry = _vllm_registry(replay=True)
session._model_alias = "qwen3"
captured: dict[str, Any] = {}
def capture_streaming(**kwargs: Any) -> Any:
captured.update(kwargs)
return as_stream(mock_completion_result(""))
provider = OpenAIChatCompletionsProvider()
provider.create_streaming = capture_streaming # type: ignore[method-assign]
session._provider = provider
# No extra_params patch: _utility_completion resolves them inside
# resolve_lane (a module seam reading the registry config), which
# a session-attribute patch cannot intercept.
with patch.object(
session, "_get_capabilities", return_value=provider.get_capabilities("qwen3")
):
session._utility_completion(
turns_from_dicts([_assistant_msg_with_thinking("from utility")]),
)
msgs_sent = captured["messages"]
assert msgs_sent[0]["reasoning"] == "from utility"
Reference in New Issue View Git Blame Copy Permalink