turnstone/tests/test_session_backend_error_format.py

"""Tests for :meth:`ChatSession._format_backend_error`.

The helper turns bare backend-boundary exceptions (HTTPX/HTTPX2 ``ReadTimeout``,
OpenAI SDK ``APITimeoutError`` / ``APIConnectionError`` /
``NotFoundError`` / ``RateLimitError`` / ``AuthenticationError``) into
operator-actionable messages that include the provider, base URL, and
model. We bind the method to lightweight stubs carrying one coherent
``ModelLane`` rather than constructing a full :class:`ChatSession`.
"""

from __future__ import annotations

import dataclasses
import threading
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch

import pytest

from tests._session_helpers import RecordingUI, make_session, provider_shell
from turnstone.core.model_turn import ModelLane
from turnstone.core.providers import ModelCapabilities
from turnstone.core.session import ChatSession


def _stub(
    *,
    base_url: str = "http://192.168.0.5:8000/v1",
    provider_name: str = "openai-compatible",
    model: str = "flatspark",
    model_alias: str | None = "flatspark",
    client_attr: str = "base_url",
) -> Any:
    """Build a minimal session-like stub for ``_format_backend_error``.

    ``client_attr`` selects which attribute on the client carries the
    URL — both ``base_url`` (OpenAI / Anthropic SDK public surface) and
    ``_base_url`` (httpx fallback) are exercised by the helper.
    """
    client_kwargs: dict[str, Any] = {client_attr: base_url}
    lane = ModelLane(
        client=SimpleNamespace(**client_kwargs),
        provider=SimpleNamespace(provider_name=provider_name),
        model=model,
        alias=model_alias or "",
    )
    stub = SimpleNamespace(
        model=model,
        _model_alias=model_alias,
        # Dead-binding latches, clear: the formatter checks them first and
        # short-circuits when both are unset, like a healthy session.
        _registry_alias_removed=None,
        _rebind_failed_key=None,
    )
    stub._lane = lane
    stub._primary_lane = lambda: stub._lane
    return stub


def _format(stub: Any, exc: BaseException) -> str | None:
    """Invoke the method as if on a real session — ``__func__`` skips
    the descriptor protocol so we can pass any object as ``self``."""
    return ChatSession._format_backend_error(stub, exc)  # type: ignore[arg-type]


# ---------------------------------------------------------------------------
# Synthetic exception classes — class name is what the helper matches on,
# so we don't need real httpx / openai imports here.
# ---------------------------------------------------------------------------


# N818 (Error suffix on Exception names) is intentionally suppressed
# for the four classes below — they exist to impersonate httpx /
# Anthropic SDK exception class names verbatim, since the formatter
# matches by class name.  Renaming them defeats the test.


class ReadTimeout(Exception):  # noqa: N818
    pass


class WriteTimeout(Exception):  # noqa: N818
    pass


class APITimeoutError(Exception):
    pass


class ConnectError(Exception):  # noqa: N818
    pass


class ConnectTimeout(Exception):  # noqa: N818
    pass


class APIConnectionError(Exception):
    pass


class NotFoundError(Exception):
    pass


class AuthenticationError(Exception):
    pass


class PermissionDeniedError(Exception):
    pass


class RateLimitError(Exception):
    pass


class ReadError(Exception):  # noqa: N818
    pass


class RemoteProtocolError(Exception):  # noqa: N818
    pass


class SomeUnrelatedError(Exception):
    """Outside the recognised set — should fall through to ``None``."""


# ---------------------------------------------------------------------------
# Known categories — each branch produces an operator-actionable message
# ---------------------------------------------------------------------------


@pytest.mark.parametrize("exc_cls", [ReadTimeout, WriteTimeout, APITimeoutError])
def test_timeout_message_names_backend_and_model(exc_cls):
    msg = _format(_stub(), exc_cls())
    assert msg is not None
    assert "Backend timeout" in msg
    assert exc_cls.__name__ in msg
    assert "openai-compatible" in msg
    assert "http://192.168.0.5:8000/v1" in msg
    assert "model=flatspark" in msg
    assert "wedged" in msg


@pytest.mark.parametrize("exc_cls", [ConnectError, ConnectTimeout, APIConnectionError])
def test_connect_message_says_unreachable(exc_cls):
    msg = _format(_stub(), exc_cls("dial tcp: i/o timeout"))
    assert msg is not None
    assert "Backend unreachable" in msg
    assert exc_cls.__name__ in msg
    assert "http://192.168.0.5:8000/v1" in msg
    # Raw exception text is preserved as a tail for grep-correlation.
    assert "dial tcp: i/o timeout" in msg


def test_not_found_points_at_model_name_mismatch():
    msg = _format(_stub(model="flatspark"), NotFoundError("model flatspark not found"))
    assert msg is not None
    assert "Backend reports model not loaded" in msg
    assert "no model named 'flatspark'" in msg
    assert "/v1/models" in msg  # operator hint


# ---------------------------------------------------------------------------
# Model label — lead with the alias the model references, annotate the id
# ---------------------------------------------------------------------------


def test_error_leads_with_alias_and_annotates_backend_id():
    """When the display alias differs from the backend model id, the enriched
    error leads with the ALIAS (the identifier the model references everywhere —
    list_nodes, spawn) and annotates the backend id for the operator, so a
    coordinator correlates the failure with those surfaces without a lookup."""
    msg = _format(
        _stub(model="deepseek-v4-flash", model_alias="DeepSeek-V4-Flash"),
        APIConnectionError("cannot reach"),
    )
    assert msg is not None
    assert "model=DeepSeek-V4-Flash (id=deepseek-v4-flash)" in msg


def test_error_model_label_collapses_when_alias_equals_id():
    """No redundant (id=...) annotation when the alias and backend id coincide."""
    msg = _format(_stub(model="flatspark", model_alias="flatspark"), APIConnectionError("x"))
    assert msg is not None
    assert "model=flatspark" in msg
    assert "(id=" not in msg


@pytest.mark.parametrize("exc_cls", [AuthenticationError, PermissionDeniedError])
def test_auth_message_mentions_api_key(exc_cls):
    msg = _format(_stub(), exc_cls("invalid api key"))
    assert msg is not None
    assert "Backend rejected credentials" in msg
    assert "API key" in msg


def test_rate_limit_message():
    msg = _format(_stub(), RateLimitError("limit exceeded"))
    assert msg is not None
    assert "Backend rate-limited" in msg
    assert "limit exceeded" in msg


def test_rate_limit_with_overflow_phrasing_is_not_mislabeled_overflow():
    """A recognized RateLimitError whose quota text happens to contain a
    context-overflow phrase must still render as rate-limited — the text-based
    overflow branch is gated on 'not a known class', so it can't hijack a
    recognized error and mark a transient 429 as a hard 'Context window exceeded'."""
    msg = _format(
        _stub(), RateLimitError("exceeds the maximum number of tokens allowed per minute")
    )
    assert msg is not None
    assert "Backend rate-limited" in msg
    assert "Context window exceeded" not in msg


# ---------------------------------------------------------------------------
# Stream-death branch — the mid-response wire-failure wording (#937)
# ---------------------------------------------------------------------------


def _stream_death_exemplars() -> list[BaseException]:
    """One realistic instance per name in ``_BACKEND_STREAM_EXC_NAMES``:
    the normalized shape the guarded iterators raise, plus the raw HTTPX-family
    names for any future unguarded path."""
    from turnstone.core.providers import IncompleteStreamError

    return [
        IncompleteStreamError(
            "stream transport failed mid-response "
            "(ReadError: [SSL] record layer failure (_ssl.c:2590))"
        ),
        ReadError("[SSL] record layer failure (_ssl.c:2590)"),
        RemoteProtocolError("peer closed connection without sending complete message body"),
    ]


@pytest.mark.parametrize("exc", _stream_death_exemplars(), ids=lambda e: type(e).__name__)
def test_stream_death_names_backend_and_model(exc):
    msg = _format(_stub(), exc)
    assert msg is not None
    assert "Backend stream died mid-response" in msg
    assert type(exc).__name__ in msg
    assert "openai-compatible" in msg
    assert "http://192.168.0.5:8000/v1" in msg
    assert "model=flatspark" in msg
    assert "retries did not recover it" in msg
    # Raw exception text is preserved as a tail for grep-correlation.
    assert str(exc) in msg


def test_stream_death_first_sentence_survives_discord_cut():
    """Discord truncates ``on_error`` text to 500 chars — the identity-bearing
    first sentence must fit even with realistic-length alias/URL inputs."""
    msg = _format(
        _stub(
            base_url="https://inference-gateway.internal.example-corp.net:8443/serving/v1",
            provider_name="openai-compatible",
            model="deepseek-r2-awq-128k-instruct-20260115",
            model_alias="prod-reasoning-primary",
        ),
        ReadError("[SSL] record layer failure (_ssl.c:2590)"),
    )
    assert msg is not None
    first_sentence = msg[: msg.index(". ") + 1]
    assert "Backend stream died mid-response" in first_sentence
    assert len(first_sentence) < 500


def test_registry_diagnosed_binding_outranks_stream_death():
    """An alias the per-send refresh diagnosed dead outranks the raw stream
    symptom: the rebind wording points at the admin action, the transport
    wording at network health — the former is the actionable one."""
    from turnstone.core.providers import IncompleteStreamError

    stub = _stub()
    stub._registry_alias_removed = "flatspark"
    stub._registry = None
    stub._kind = None
    msg = _format(stub, IncompleteStreamError("stream transport failed mid-response"))
    assert msg is not None
    assert "has been removed from the registry" in msg
    assert "Backend stream died" not in msg


def test_stream_death_with_overflow_phrasing_stays_stream_death():
    """Joining the stream names into ``_BACKEND_KNOWN_EXC_NAMES`` removes them
    from ``_is_ctx_overflow``'s text-detection eligibility (its class
    self-gate) — deliberate: transport/SSL texts never carry real overflow
    phrases, and a stream death must never be misfiled as a deterministic
    overflow (which callers route to a non-retryable compaction path)."""
    msg = _format(_stub(), ReadError("proxy said: maximum context length hint in banner"))
    assert msg is not None
    assert "Backend stream died mid-response" in msg
    assert "Context window exceeded" not in msg


def test_terminal_fallback_stream_death_reports_actual_lane(tmp_db):
    """Fatal formatting consumes the fallback lane that armed the stream.

    The original exception object survives the retry wrapper, while its
    side-table context contains no client, credential, principal, or query.
    """
    from turnstone.core.providers import IncompleteStreamError

    ui = RecordingUI()  # type: ignore[no-untyped-call]
    session = make_session(model_alias="primary", ui=ui)
    provider = provider_shell("fallback-provider")
    fallback_lane = ModelLane(
        provider=provider,
        client=SimpleNamespace(base_url="https://fallback.example/v1?api_key=terminal-secret"),
        model="fallback-kernel",
        alias="fallback-alias",
        registry_generation=19,
        capabilities=ModelCapabilities(),
    )
    death = IncompleteStreamError("peer closed the response")

    def _fail_on_fallback(consumer, *_args, **_kwargs):
        consumer.begin_attempt(SimpleNamespace(armed=True), None, fallback_lane)
        raise death

    session._MID_STREAM_RETRIES = 0
    with (
        patch.object(session, "_model_turn_with_fallback", side_effect=_fail_on_fallback),
        pytest.raises(IncompleteStreamError) as raised,
    ):
        session._stream_response()

    assert raised.value is death
    session._record_fatal_error(raised.value)
    message = ui.of("error")[-1]
    assert "fallback-provider" in message
    assert "https://fallback.example/v1" in message
    assert "model=fallback-alias (id=fallback-kernel)" in message
    assert "primary" not in message
    assert "terminal-secret" not in message


# ---------------------------------------------------------------------------
# Fall-through + degradation behaviour
# ---------------------------------------------------------------------------


def test_unknown_exception_returns_none():
    assert _format(_stub(), SomeUnrelatedError("anything")) is None


def test_unknown_exception_value_error_returns_none():
    assert _format(_stub(), ValueError("not a backend error")) is None


def test_trailing_slash_and_query_string_stripped():
    msg = _format(
        _stub(base_url="http://node-a:8000/v1/?api_key=secret&foo=1"),
        ReadTimeout(),
    )
    assert msg is not None
    assert "http://node-a:8000/v1" in msg
    # Query string (which may carry credentials) is stripped before the
    # message is built — sanitize_error_text is a second line of defence
    # but the helper itself must not embed query params verbatim.
    assert "api_key" not in msg
    assert "secret" not in msg


def test_missing_provider_degrades_to_placeholder():
    stub = _stub()
    stub._lane = dataclasses.replace(stub._lane, provider=None)
    msg = _format(stub, ReadTimeout())
    assert msg is not None
    # No exception, no NoneType formatting leaking through.
    assert "Backend timeout" in msg
    assert "from ?" in msg or "openai-compatible" not in msg


def test_client_base_url_raises_degrades_gracefully():
    class _BadClient:
        @property
        def base_url(self) -> str:
            raise RuntimeError("boom")

    stub = _stub()
    stub._lane = dataclasses.replace(stub._lane, client=_BadClient())
    msg = _format(stub, ReadTimeout())
    assert msg is not None
    assert "Backend timeout" in msg
    # base_url accessor blew up — message still renders with placeholder.
    assert "at ?" in msg


def test_httpx_underscore_base_url_fallback():
    # httpx client carries ``_base_url`` on some versions instead of
    # ``base_url`` — the helper checks both.
    stub = _stub(base_url="http://alt-host:9000", client_attr="_base_url")
    # SimpleNamespace exposes the attr; remove the public one so the
    # fallback path is exercised.
    client = stub._lane.client
    delattr(client, "base_url") if hasattr(client, "base_url") else None
    msg = _format(stub, ReadTimeout())
    assert msg is not None
    assert "http://alt-host:9000" in msg


# ---------------------------------------------------------------------------
# Integration with _record_fatal_error — original bare-class string is
# replaced by the enriched message when the exception type is recognised.
# ---------------------------------------------------------------------------


def _record_fatal_stub(ui: Any, captured: dict[str, str]) -> Any:
    """Build a stub for the ``_record_fatal_error`` integration tests.

    ``_record_fatal_error`` calls ``self._format_backend_error(...)``
    internally, so the stub binds the unbound method to itself rather
    than relying on Python's descriptor protocol (which only kicks in
    when ``self`` is a real instance of the class)."""
    stub = _stub()
    stub._ws_id = "ws-test"
    stub._generation_lock = threading.RLock()
    stub._has_persisted_error = False
    stub.ui = ui
    stub._emit_state = lambda state, **_kwargs: captured.setdefault("state", state)
    stub._format_backend_error = lambda exc: ChatSession._format_backend_error(stub, exc)
    return stub


def test_record_fatal_uses_enriched_message_for_known(monkeypatch):
    """End-to-end: a recognised exception flows through
    ``_record_fatal_error`` and the enriched text reaches both the UI
    and the persist hook."""

    captured: dict[str, str] = {}

    def fake_persist(ws_id: str, msg: str) -> None:
        captured["persist"] = msg

    def fake_sanitize(text: str, *, max_len: int = 1024) -> str:
        # Skip the credential-redaction module (and its module-level
        # regex compile) by returning the input verbatim — the helper
        # under test produces no credentials.
        return text

    import turnstone.core.memory as memory_mod

    monkeypatch.setattr(memory_mod, "persist_last_error", fake_persist)
    monkeypatch.setattr(memory_mod, "sanitize_error_text", fake_sanitize)

    class _UI:
        def __init__(self) -> None:
            self.errors: list[str] = []

        def on_error(self, msg: str) -> None:
            self.errors.append(msg)

    ui = _UI()
    stub = _record_fatal_stub(ui, captured)

    ChatSession._record_fatal_error(stub, ReadTimeout())  # type: ignore[arg-type]

    assert ui.errors, "UI never received error"
    assert "Backend timeout" in ui.errors[0]
    assert "ReadTimeout" in ui.errors[0]
    assert captured["persist"] == ui.errors[0]
    assert captured["state"] == "error"
    assert stub._has_persisted_error is True


def test_record_fatal_falls_back_for_unknown(monkeypatch):
    """An unrecognised exception keeps the legacy
    ``f"{type(exc).__name__}: {exc}"`` shape so we don't regress
    existing call sites that grep on it."""

    captured: dict[str, str] = {}

    def fake_persist(ws_id: str, msg: str) -> None:
        captured["persist"] = msg

    def fake_sanitize(text: str, *, max_len: int = 1024) -> str:
        return text

    import turnstone.core.memory as memory_mod

    monkeypatch.setattr(memory_mod, "persist_last_error", fake_persist)
    monkeypatch.setattr(memory_mod, "sanitize_error_text", fake_sanitize)

    class _UI:
        def __init__(self) -> None:
            self.errors: list[str] = []

        def on_error(self, msg: str) -> None:
            self.errors.append(msg)

    ui = _UI()
    stub = _record_fatal_stub(ui, captured)

    ChatSession._record_fatal_error(stub, ValueError("plain old error"))  # type: ignore[arg-type]

    assert ui.errors == ["ValueError: plain old error"]
    assert captured["persist"] == "ValueError: plain old error"


@pytest.mark.parametrize(
    ("exc", "expect_error_level", "message_substring"),
    [
        pytest.param(ReadTimeout("timed out"), True, "ReadTimeout", id="fault-at-error"),
        pytest.param(KeyboardInterrupt(), False, None, id="ctrl-c-at-info"),
    ],
)
def test_record_fatal_log_level_contract(
    monkeypatch, caplog, exc, expect_error_level, message_substring
):
    """The one journal trace of a fatal turn emits ``session.fatal.recorded``
    with the sanitized text — at ERROR for genuine faults; a Ctrl-C routes
    through the same chokepoint but is a user action, not a fault, and must
    not add an ERROR-level line per CLI interrupt."""
    import logging

    import turnstone.core.memory as memory_mod

    monkeypatch.setattr(memory_mod, "persist_last_error", lambda ws_id, msg: None)
    monkeypatch.setattr(memory_mod, "sanitize_error_text", lambda text, **kw: text)

    class _UI:
        def on_error(self, msg: str) -> None:
            pass

    stub = _record_fatal_stub(_UI(), {})
    with caplog.at_level(logging.INFO, logger="turnstone.core.session"):
        ChatSession._record_fatal_error(stub, exc)  # type: ignore[arg-type]

    recorded = [r for r in caplog.records if "session.fatal.recorded" in r.message]
    assert recorded
    if expect_error_level:
        assert any(r.levelno == logging.ERROR for r in recorded)
    else:
        assert all(r.levelno < logging.ERROR for r in recorded)
        assert any(r.levelno == logging.INFO for r in recorded)
    if message_substring:
        assert any(message_substring in r.message for r in recorded)


def test_backend_auth_unavailable_names_the_mint_not_the_key():
    """The prefix here IS the exception text, so no ``raw_tail`` is appended,
    and the hint points at the mint configuration, not the static key."""
    from turnstone.core.session import BackendAuthUnavailableError

    exc = BackendAuthUnavailableError(
        "Delegated backend authentication unavailable for model alias 'gw'"
    )
    msg = _format(_stub(), exc)
    assert msg is not None
    assert "model alias 'gw'" in msg
    assert "check its auth mode and gateway audience" in msg
    assert "NOT the alias's static API key" in msg
    assert "raw=" not in msg
    assert msg.count("unavailable for model alias 'gw'") == 1