Files
turnstone/tests/_session_helpers.py
T
Patrick Buckley 3ffa8b9057 fix(providers): review round 2 — complete-or-error drain, code-gated retries, truncation-safe blocks
Correctness (3 confirmed + 2 plausible, all fixed):

- drain_stream now raises typed, retryable IncompleteStreamError when a
  stream exhausts without any finish reason — every adapter emits one on
  a healthy stream, so its absence means the generation died
  mid-response behind a cleanly-closing proxy.  This restores the
  retired transport's complete-or-error contract (a half-generated
  compaction summary was previously returned as finish=stop and stored,
  silently replacing real history) and DELETES round 1's suffix-info
  fold: with no finish-less success path there is nothing to classify,
  so a trailing status ping can never be stored as content either.
- Index-degenerate parallel tool calls get distinct slots: a delta whose
  id differs from its slot's opens a new call (id-less fragments still
  follow their index's current call), so historical compat servers that
  emit every parallel call at index 0 no longer fuse distinct calls
  into concatenated garbage arguments.  Result order stays index-sorted
  (stable) like the retired array parse.
- response.failed retryability is code-gated: only transient codes
  (server_error, rate_limit_exceeded) raise the retryable typed error;
  deterministic rejections (invalid prompt, image fetch, policy) raise
  plain RuntimeError and stop retry loops on attempt zero instead of
  running the full backoff ladder against a doomed request.
- Terminal Responses events rebuild provider_blocks from
  response.output when present: the item being generated at
  max_output_tokens truncation never receives output_item.done, and
  storing a reasoning item without its required following item made the
  next turn's replay a 400.
- merge_usage's base case uses dataclasses.replace so a future UsageInfo
  field can't be silently zeroed on drained lanes.

Cleanup: run_abortable_with_deadline bundles the three-point abort
wiring (ref + cancel_ref + on_abandon) so it cannot be half-wired —
both judges converted; scripted_chat_client hoists the 14 chat-lane
fake_create closures (call scripts + .calls recording replace per-test
counter cells); fake_chat_stream gains reasoning=, collapsing the
reasoning-capture suite's hand-rolled chunk shape; FakeAnthropicBlock
hoists the duplicated _Block test class; the class and judge PlantUML
diagrams drop the retired create_completion flow.

Also converts test_model_registry's agent-model fakes, which returned
legacy response objects that iterated as EMPTY streams — they only
passed through the old drain's silent finish=stop default, exactly the
hazard the new gate exists to catch.
2026-07-13 22:39:19 -07:00

331 lines
12 KiB
Python

"""Shared session-test helpers.
Two reasoning-test modules (``test_session_replay_reasoning.py`` and
``test_session_synth_reasoning_block.py``) need the same minimal
``ChatSession`` factory + a ``SessionUIBase`` no-op subclass. Hoisting
keeps a future third caller from drifting on the defaults — the third
existing ``_make_session`` (``test_model_registry.py``) deliberately
takes a different signature (registry / model_alias / reasoning_effort
+ ``_FakeUI``) and is NOT a candidate for sharing this helper.
Module is named with a leading underscore so pytest doesn't try to
collect it as a test file — it's an importable utility, not a test.
"""
from __future__ import annotations
import json
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
from turnstone.core.providers import StreamChunk, ToolCallDelta
from turnstone.core.session import ChatSession
from turnstone.core.session_ui_base import SessionUIBase
class NullUI(SessionUIBase):
"""Bare-bones UI satisfying the SessionUIBase contract for tests
that don't care about UI side effects."""
def __init__(self) -> None:
super().__init__()
def make_session(**kwargs: Any) -> ChatSession:
"""Build a ChatSession with minimal defaults; tests override
individual fields via kwargs."""
defaults: dict[str, Any] = {
"client": MagicMock(),
"model": "test-model",
"ui": NullUI(),
"instructions": None,
"temperature": 0.5,
"max_tokens": 4096,
"tool_timeout": 30,
}
defaults.update(kwargs)
return ChatSession(**defaults)
def mock_completion_result(
content: str = "",
tool_calls: list[dict[str, Any]] | None = None,
) -> MagicMock:
"""A provider result shaped like ``CompletionResult``.
Callers that route through ``model_turn`` (judges, task agents, and
every lane #827 migrates) hit its re-ingest, which iterates
``tool_calls``/``provider_blocks`` and joins ``reasoning`` — a bare
MagicMock attribute would TypeError deep inside the seam, so every
field the re-ingest reads is pinned to a real value here. ONE shared
definition: when the re-ingest starts reading a new CompletionResult
field, add it here and every suite moves together.
"""
result = MagicMock()
result.content = content
result.tool_calls = tool_calls
result.finish_reason = "stop"
result.usage = None
result.provider_blocks = []
result.reasoning = ""
return result
def fake_chat_stream(
*,
content: str | None = None,
tool_calls: list[dict[str, str]] | None = None,
finish_reason: str = "stop",
prompt_tokens: int = 10,
completion_tokens: int = 5,
reasoning_content: str | None = None,
reasoning: str | None = None,
) -> list[Any]:
"""Fake OpenAI Chat Completions SSE chunks for driving the REAL
``OpenAIChatCompletionsProvider`` through a fake SDK client::
client.chat.completions.create = lambda **kw: fake_chat_stream(...)
Exercises the adapter's ``_iter_stream`` plus ``drain_stream`` end to
end (the highest-fidelity fake lane), unlike ``as_stream`` which fakes
at the provider boundary. ``tool_calls`` entries are
``{"id", "name", "arguments"}`` dicts. ``SimpleNamespace`` (not
``MagicMock``) so absent SDK fields read as real ``None`` — an
auto-created mock attribute would leak into ``len()``/string paths.
Emits the realistic three-phase shape: data chunk(s), a finish-reason
chunk, then the ``stream_options.include_usage`` usage-only chunk with
empty ``choices``.
"""
def _delta(
content_val: str | None = None,
tcs: list[Any] | None = None,
rc: str | None = None,
rsn: str | None = None,
) -> SimpleNamespace:
return SimpleNamespace(
content=content_val,
tool_calls=tcs,
reasoning=rsn,
reasoning_content=rc,
annotations=None,
)
chunks: list[Any] = []
if reasoning_content is not None or reasoning is not None:
chunks.append(
SimpleNamespace(
choices=[
SimpleNamespace(
finish_reason=None, delta=_delta(rc=reasoning_content, rsn=reasoning)
)
],
usage=None,
)
)
if content is not None:
chunks.append(
SimpleNamespace(
choices=[SimpleNamespace(finish_reason=None, delta=_delta(content))],
usage=None,
)
)
if tool_calls:
tcs = [
SimpleNamespace(
index=i,
id=tc.get("id", ""),
function=SimpleNamespace(
name=tc.get("name", ""), arguments=tc.get("arguments", "")
),
)
for i, tc in enumerate(tool_calls)
]
chunks.append(
SimpleNamespace(
choices=[SimpleNamespace(finish_reason=None, delta=_delta(None, tcs))],
usage=None,
)
)
chunks.append(
SimpleNamespace(
choices=[SimpleNamespace(finish_reason=finish_reason, delta=_delta())],
usage=None,
)
)
chunks.append(
SimpleNamespace(
choices=[],
usage=SimpleNamespace(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=None,
input_tokens_details=None,
),
)
)
return chunks
def scripted_chat_client(*scripts: Any) -> Any:
"""A fake ``client.chat.completions.create`` that follows a script.
Call N returns the stream described by ``scripts[N]``; the last script
repeats for any further calls. Each script is a dict of
:func:`fake_chat_stream` kwargs or a pre-built chunk list. The
returned callable records every call's kwargs on ``.calls`` — read
``len(fn.calls)`` where a test previously kept its own counter cell,
and ``fn.calls[i]["messages"]`` where it captured request bodies.
"""
def _create(**kwargs: Any) -> Any:
_create.calls.append(kwargs) # type: ignore[attr-defined]
i = min(len(_create.calls) - 1, len(scripts) - 1) # type: ignore[attr-defined]
script = scripts[i]
return fake_chat_stream(**script) if isinstance(script, dict) else script
_create.calls = [] # type: ignore[attr-defined]
return _create
class FakeAnthropicBlock:
"""A full-content Anthropic content-block fake for
:func:`fake_anthropic_stream` — plain attributes plus the
``model_dump()`` the provider's block capture reads."""
def __init__(self, **fields: Any) -> None:
self._fields = fields
for key, value in fields.items():
setattr(self, key, value)
def model_dump(self, **_kw: Any) -> dict[str, Any]:
return dict(self._fields)
def fake_anthropic_stream(
blocks: list[Any],
*,
stop_reason: str = "end_turn",
usage: Any = None,
) -> Any:
"""Fake Anthropic SDK stream context manager for tests that drive the
REAL ``AnthropicProvider`` through a fake client::
client.messages.stream = lambda **kw: fake_anthropic_stream(...)
Accepts the same full-content block fakes the pre-#831
``get_final_message`` fixtures used (objects with ``.type`` + fields
and ``model_dump()``) and synthesizes the real event grammar the
streaming iterator consumes: ``content_block_start`` carries the block
with its text/thinking/signature EMPTIED and ``input`` as ``{}`` (the
SDK start shape), deltas carry the content, ``content_block_stop``
finalizes tool input, and the closing ``message_delta`` carries
``stop_reason`` (+ optional usage object). Without the stripping, the
provider's raw-block accumulator would double every text/thinking
field (start capture + delta append).
"""
events: list[Any] = []
for idx, block in enumerate(blocks):
d = dict(block.model_dump()) if hasattr(block, "model_dump") else dict(vars(block))
btype = d.get("type", "")
start = dict(d)
if btype == "text":
start["text"] = ""
elif btype == "thinking":
start["thinking"] = ""
start["signature"] = ""
elif btype == "tool_use":
start["input"] = {}
events.append(
SimpleNamespace(
type="content_block_start", index=idx, content_block=SimpleNamespace(**start)
)
)
if btype == "text" and d.get("text"):
events.append(
SimpleNamespace(
type="content_block_delta",
index=idx,
delta=SimpleNamespace(type="text_delta", text=d["text"]),
)
)
elif btype == "thinking":
if d.get("thinking"):
events.append(
SimpleNamespace(
type="content_block_delta",
index=idx,
delta=SimpleNamespace(type="thinking_delta", thinking=d["thinking"]),
)
)
if d.get("signature"):
events.append(
SimpleNamespace(
type="content_block_delta",
index=idx,
delta=SimpleNamespace(type="signature_delta", signature=d["signature"]),
)
)
elif btype == "tool_use":
events.append(
SimpleNamespace(
type="content_block_delta",
index=idx,
delta=SimpleNamespace(
type="input_json_delta",
partial_json=json.dumps(d.get("input", {})),
),
)
)
events.append(SimpleNamespace(type="content_block_stop", index=idx))
events.append(
SimpleNamespace(
type="message_delta", usage=usage, delta=SimpleNamespace(stop_reason=stop_reason)
)
)
mgr = MagicMock()
mgr.__enter__ = MagicMock(return_value=events)
mgr.__exit__ = MagicMock(return_value=False)
return mgr
def as_stream(result: Any) -> list[StreamChunk]:
"""Adapt a ``CompletionResult``-shaped fake to a ``create_streaming``
return value (single terminal chunk).
The #831 transport collapse routes every single-shot lane through
``drain_stream(provider.create_streaming(...))``, so provider fakes
return chunk iterables now. Tests keep building result-shaped fakes
(``mock_completion_result`` or hand-rolled) and wrap them at
assignment: ``provider.create_streaming.return_value =
as_stream(result)``. A list re-iterates on every call, so one
``return_value`` serves repeated-call tests; convert AFTER mutating
the fake's fields — the chunk snapshots them.
Multi-chunk accumulation semantics are exercised by the dedicated
``drain_stream`` unit tests, not through this helper.
"""
deltas = [
ToolCallDelta(
index=i,
id=tc.get("id", ""),
name=tc.get("function", {}).get("name", ""),
arguments_delta=tc.get("function", {}).get("arguments", ""),
)
for i, tc in enumerate(result.tool_calls or [])
]
return [
StreamChunk(
content_delta=result.content or "",
reasoning_delta=getattr(result, "reasoning", "") or "",
tool_call_deltas=deltas,
usage=result.usage,
finish_reason=result.finish_reason or "stop",
provider_blocks=list(result.provider_blocks or []),
)
]