turnstone/tests/test_session_worker.py

"""Unit tests for ``turnstone.core.session_worker``.

The shared worker dispatch is load-bearing for both the interactive
``/v1/api/workstreams/{ws_id}/send`` HTTP handler and the coordinator
``CoordinatorAdapter.send`` path. Tests cover the five invariants the
module must hold:

* live worker → enqueue, no thread spawn
* queue.Full → ``False`` (caller surfaces 429)
* concurrent ``send`` calls produce exactly one worker thread
  (Stage 1 bug-1 — the racy ``Thread.is_alive()`` gate stays caught)
* ``_worker_running`` cleared in ``finally`` even on uncaught exception
* ownership-clear wake backstop: a worker exiting with USER_DRAIN
  nudges queued on an IDLE workstream spawns the wake send that the
  IDLE fan-out (which ran on this worker's own thread) had to drop

Callers pass no-arg closures. For a real ChatSession, dispatch captures its
slot-time cancellation witness through a narrow optional method; session
stubs retain the historical callback-only path. The exit backstop only PEEKS
session nudge state defensively (``getattr`` for ``_nudge_queue``, bail on
stubs) — watch-style dispatchers can still drive a session that isn't
installed on ``ws``.
"""

from __future__ import annotations

import collections
import queue
import threading
from typing import Any

import pytest

from tests._helpers import wait_until as _wait_until
from tests._session_helpers import make_session
from turnstone.core import session_worker
from turnstone.core.idle_nudge_watcher import wake_workstream_if_pending
from turnstone.core.nudge_queue import USER_DRAIN, NudgeQueue
from turnstone.core.session import ChatSession
from turnstone.core.session_routes import SessionEndpointConfig, _make_dispatch_attempt
from turnstone.core.workstream import Workstream, WorkstreamState


class _SendSession:
    """ChatSession-shaped stub recording send / queue_message calls."""

    def __init__(
        self,
        *,
        queue_full: bool = False,
        queue_raises: BaseException | None = None,
        send_gate: threading.Event | None = None,
        send_raises: BaseException | None = None,
    ) -> None:
        self.send_calls: list[str] = []
        self.queue_calls: list[str] = []
        self._queue_full = queue_full
        self._queue_raises = queue_raises
        # Lets a test pin a worker inside ``run`` while a second thread
        # races through ``send`` — proves the lock gate (not
        # Thread.is_alive) is what serialises them.
        self._send_gate = send_gate
        self._send_raises = send_raises

    def send(self, message: str) -> None:
        if self._send_gate is not None:
            self._send_gate.wait(timeout=2.0)
        if self._send_raises is not None:
            raise self._send_raises
        self.send_calls.append(message)

    def queue_message(self, message: str) -> None:
        if self._queue_full:
            raise queue.Full
        if self._queue_raises is not None:
            raise self._queue_raises
        self.queue_calls.append(message)


class _ClaimingSendSession(_SendSession):
    """Dispatch stub exposing the production worker-claim protocol."""

    def __init__(self) -> None:
        super().__init__()
        self._cancel_event = threading.Event()

    def _capture_worker_claim(self, principal_id: str = "") -> session_worker.WorkerClaim:
        return session_worker.WorkerClaim(
            session=self,
            principal_id=principal_id,
            cancel_epoch=0,
            cancel_event=self._cancel_event,
            cancel_event_was_set=False,
        )

    def _worker_claim_is_current(self, claim: session_worker.WorkerClaim) -> bool:
        return claim.session is self and not claim.cancel_event.is_set()

    def queue_message(self, message: str, **_kwargs: Any) -> tuple[str, str, str]:
        super().queue_message(message)
        return message, "notice", "queued-message"


def _make_ws(session: Any = None) -> Workstream:
    ws = Workstream(id="ws-aaaaaaaa", name="ws-aaaa")
    ws.session = session  # type: ignore[assignment]
    return ws


def _send_message(ws: Workstream, session: _SendSession, msg: str) -> bool:
    """Convenience wrapper mirroring the canonical caller shape."""
    return session_worker.send(
        ws,
        enqueue=lambda: session.queue_message(msg),
        run=lambda: session.send(msg),
        thread_name=f"test-worker-{ws.id[:8]}",
    )


def _make_blocked_principal_session(
    *, previous_actor: str
) -> tuple[ChatSession, threading.Event, threading.Event, list[tuple[str, str | None]]]:
    """Build the smallest real ``queue_message`` surface for claim-race tests."""
    session = ChatSession.__new__(ChatSession)
    session._acting_user_id = previous_actor
    session._mcp_user_id = "owner"
    session._queued_lock = threading.Lock()
    session._queued_messages = collections.OrderedDict()
    session._retracted_while_popped = set()
    session._popped_in_flight = set()
    # The worker-slot claim these tests are about is captured from real
    # lifecycle state, so seed the fields ``__init__`` would: an open session
    # with no cancel edge, no close in flight, and no owed TOOL receipts.
    session._generation_lock = threading.RLock()
    session._cancel_event = threading.Event()
    session._approval_cancel_epoch = 0
    session._publication_shutdown = False
    session._soft_close_preparing = False
    session._tool_structural_debt = None

    bind_entered = threading.Event()
    release_bind = threading.Event()
    ran_as: list[tuple[str, str | None]] = []

    def _delayed_bind(user_id: str) -> None:
        bind_entered.set()
        assert release_bind.wait(5), "test did not release acting-user bind"
        session._acting_user_id = user_id

    def _record_send(message: str, **_kwargs: Any) -> None:
        ran_as.append((message, session._mcp_effective_user_id))

    session.bind_acting_user = _delayed_bind  # type: ignore[method-assign]
    session.send = _record_send  # type: ignore[method-assign]
    return session, bind_entered, release_bind, ran_as


def _principal_dispatch_attempt(
    ws: Workstream,
    session: ChatSession,
    *,
    message: str,
    acting_user_id: str,
) -> tuple[bool, dict[str, Any]]:
    """Dispatch through the production HTTP send worker/queue decision."""
    cfg = SessionEndpointConfig(
        permission_gate=None,
        manager_lookup=lambda _request: (None, None),
        tenant_check=None,
        not_found_label="missing",
        audit_action_prefix="workstream",
    )
    attempt = _make_dispatch_attempt(
        ws,
        cfg,
        ws.ui,
        message=message,
        resolved_atts=[],
        ordered_taken=[],
        send_id=f"send-{acting_user_id}",
        acting_uid=acting_user_id,
    )
    return attempt(session)


# ---------------------------------------------------------------------------
# Happy paths
# ---------------------------------------------------------------------------


def test_spawn_worker_runs_target_and_clears_flag() -> None:
    session = _SendSession()
    ws = _make_ws(session)
    ok = _send_message(ws, session, "hello")
    assert ok is True
    assert ws.worker_thread is not None
    ws.worker_thread.join(timeout=2.0)
    assert session.send_calls == ["hello"]
    assert ws._worker_running is False


def test_reuse_path_when_worker_running_takes_enqueue() -> None:
    session = _SendSession()
    ws = _make_ws(session)
    ws._worker_running = True  # simulate a live worker

    ok = _send_message(ws, session, "queued")
    assert ok is True
    # No thread spawned on the reuse path.
    assert ws.worker_thread is None
    assert session.send_calls == []
    assert session.queue_calls == ["queued"]
    # Flag stays True — the caller didn't claim ownership.
    assert ws._worker_running is True


# ---------------------------------------------------------------------------
# Queue.Full / enqueue failure
# ---------------------------------------------------------------------------


def test_enqueue_queue_full_returns_false_no_spawn() -> None:
    session = _SendSession(queue_full=True)
    ws = _make_ws(session)
    ws._worker_running = True

    ok = _send_message(ws, session, "hello")
    assert ok is False
    assert session.send_calls == []
    assert session.queue_calls == []
    assert ws.worker_thread is None
    # _worker_running unchanged — the live worker still owns it.
    assert ws._worker_running is True


def test_enqueue_unexpected_exception_returns_false_logged() -> None:
    session = _SendSession(queue_raises=RuntimeError("boom"))
    ws = _make_ws(session)
    ws._worker_running = True

    ok = _send_message(ws, session, "hello")
    assert ok is False
    assert session.send_calls == []
    assert ws.worker_thread is None
    assert ws._worker_running is True


def test_send_barrier_active_truth_table() -> None:
    """The two-term order barrier (Workstream.send_barrier_active): list
    non-empty OR drain alive.  The drain-alive term covers the CLAIMED-
    entry window (entry popped, dispatch in flight) — the _PendingSend
    invariant "drain not alive ⇒ nothing claimed" is what makes the pair
    exhaustive; a one-term copy at any consumer re-opens the wake-jumps-
    an-acked-send hole."""
    from types import SimpleNamespace

    ws = _make_ws()
    assert ws.send_barrier_active() is False  # empty list, no drain
    ws._pending_sends.append(object())  # type: ignore[arg-type]
    assert ws.send_barrier_active() is True  # list term
    ws._pending_drain = SimpleNamespace(is_alive=lambda: True)  # type: ignore[assignment]
    assert ws.send_barrier_active() is True  # both terms
    ws._pending_sends.clear()
    assert ws.send_barrier_active() is True  # drain-alive term alone (claimed window)
    ws._pending_drain = SimpleNamespace(is_alive=lambda: False)  # type: ignore[assignment]
    assert ws.send_barrier_active() is False  # dead drain, empty list
    ws._pending_drain = None
    assert ws.send_barrier_active() is False


def test_spawn_failure_releases_slot_and_reraises(monkeypatch) -> None:
    """``Thread.start`` raising (thread exhaustion, MemoryError) must not
    wedge the slot: the claim ``(worker_thread, _worker_running)`` taken
    under the lock is rolled back and the exception propagates.  Without
    the rollback the flag's only clearer is a finally on a thread that
    never started — the workstream looks idle forever (no state change
    ever fired) while every dispatch takes the reuse path into a queue
    no worker will drain, until an operator force-cancel."""
    import pytest

    session = _SendSession()
    ws = _make_ws(session)

    class _ExhaustedThread(threading.Thread):
        def start(self) -> None:
            raise RuntimeError("can't start new thread")

    class _ThreadNS:
        # Shim only what session_worker.send touches; patching the real
        # threading module's Thread attribute would break every other
        # test's spawns.
        Thread = _ExhaustedThread
        current_thread = staticmethod(threading.current_thread)

    monkeypatch.setattr(session_worker, "threading", _ThreadNS)
    with pytest.raises(RuntimeError, match="can't start new thread"):
        _send_message(ws, session, "doomed")
    assert ws._worker_running is False
    assert ws.worker_thread is None
    assert session.send_calls == []

    # The slot is genuinely reusable once resources recover.
    monkeypatch.setattr(session_worker, "threading", threading)
    assert _send_message(ws, session, "recovered") is True
    ws.worker_thread.join(timeout=2.0)
    assert session.send_calls == ["recovered"]
    assert ws._worker_running is False


def test_closed_workstream_refused_no_spawn() -> None:
    """Authoritative closed-check: ``close()`` sets ``_closed`` under
    ``ws._lock``, so a wake (or send) racing it must be refused HERE —
    the wake gate's lockless peek can go stale, and a spawn past this
    point would run a full unattended turn (inference, tool calls,
    storage writes) on a workstream whose ``ws_closed`` already fired.
    """
    session = _SendSession()
    ws = _make_ws(session)
    ws._closed = True

    ok = _send_message(ws, session, "hello")

    assert ok is False
    assert session.send_calls == []
    assert session.queue_calls == []
    assert ws.worker_thread is None
    assert ws._worker_running is False


def test_closed_workstream_refused_on_reuse_path_too() -> None:
    """The refusal precedes the enqueue branch: no interjection is queued
    onto a session whose workstream is already closed."""
    session = _SendSession()
    ws = _make_ws(session)
    ws._worker_running = True
    ws._closed = True

    ok = _send_message(ws, session, "hello")

    assert ok is False
    assert session.queue_calls == []
    assert ws._worker_running is True  # untouched — not ours to clear


# ---------------------------------------------------------------------------
# _worker_running lifecycle
# ---------------------------------------------------------------------------


def test_worker_finally_clears_running_flag_on_exception() -> None:
    session = _SendSession(send_raises=RuntimeError("worker-failed"))
    ws = _make_ws(session)

    ok = _send_message(ws, session, "hello")
    assert ok is True
    assert ws.worker_thread is not None
    ws.worker_thread.join(timeout=2.0)
    # Defense-in-depth: even though run() raised, _worker_running is False.
    assert ws._worker_running is False


def test_worker_finally_clears_flag_when_run_swallows() -> None:
    """Mirrors the call-site contract: run() catches its own exceptions
    for UI surfacing; we still clear the flag in finally."""
    session = _SendSession()
    ws = _make_ws(session)

    captured: list[BaseException] = []

    def run() -> None:
        try:
            session.send("hello")
            raise RuntimeError("after-send")
        except Exception as exc:
            captured.append(exc)

    ok = session_worker.send(
        ws,
        enqueue=lambda: session.queue_message("hello"),
        run=run,
    )
    assert ok is True
    assert ws.worker_thread is not None
    ws.worker_thread.join(timeout=2.0)
    assert isinstance(captured[0], RuntimeError)
    assert ws._worker_running is False


# ---------------------------------------------------------------------------
# Concurrency — Stage 1 bug-1 regression
# ---------------------------------------------------------------------------


def test_worker_claim_rejects_previous_actor_before_new_actor_bind_completes() -> None:
    """Queue admission follows the worker claim, never the stale session bind.

    A fresh send claims ``_worker_running`` before its worker thread can finish
    ``bind_acting_user``.  If the queue guard reads the prior mutable session
    actor in that window, the prior actor can inject text into the new actor's
    turn and have it run under the new actor's credentials.
    """
    session, bind_entered, release_bind, ran_as = _make_blocked_principal_session(
        previous_actor="alice"
    )
    ws = _make_ws(session)
    ws.ui = object()  # type: ignore[assignment]

    try:
        first_ok, first_queue = _principal_dispatch_attempt(
            ws,
            session,
            message="bob starts",
            acting_user_id="bob",
        )
        assert first_ok is True
        assert first_queue == {}
        assert bind_entered.wait(5), "new worker never entered acting-user bind"

        second_ok, second_queue = _principal_dispatch_attempt(
            ws,
            session,
            message="alice must not enter bob's turn",
            acting_user_id="alice",
        )

        assert second_ok is True
        assert second_queue == {"rejected": "cross_user_interjection"}
        assert session._queued_messages == {}
    finally:
        release_bind.set()
        worker = ws.worker_thread
        if worker is not None:
            worker.join(timeout=5)

    assert ran_as == [("bob starts", "bob")]


def test_worker_claim_accepts_new_actor_followup_before_session_bind_completes() -> None:
    """The claimed actor may queue a follow-up during its own bind window.

    Fixing the cross-user hole by merely rejecting every pre-bind enqueue would
    turn two rapid sends from the same browser into a false 409.  The immutable
    worker claim already knows who owns the admitted turn, so that identity is
    authoritative for both the allow and deny decisions.
    """
    session, bind_entered, release_bind, ran_as = _make_blocked_principal_session(
        previous_actor="alice"
    )
    ws = _make_ws(session)
    ws.ui = object()  # type: ignore[assignment]

    try:
        first_ok, first_queue = _principal_dispatch_attempt(
            ws,
            session,
            message="bob starts",
            acting_user_id="bob",
        )
        assert first_ok is True
        assert first_queue == {}
        assert bind_entered.wait(5), "new worker never entered acting-user bind"

        second_ok, second_queue = _principal_dispatch_attempt(
            ws,
            session,
            message="bob follows up",
            acting_user_id="bob",
        )

        assert second_ok is True
        assert second_queue == {
            "cleaned": "bob follows up",
            "priority": "notice",
            "msg_id": "send-bob",
        }
        assert list(session._queued_messages) == ["send-bob"]
    finally:
        release_bind.set()
        worker = ws.worker_thread
        if worker is not None:
            worker.join(timeout=5)

    assert ran_as == [("bob starts", "bob")]


def test_dispatch_refuses_session_swapped_after_caller_capture() -> None:
    """A closure-bound session must be the one the worker claim fences.

    The HTTP pending-send drain captures ``ws.session`` before entering the
    shared dispatcher.  A resume-style object swap can land in that gap.  The
    dispatcher must refuse both queue and spawn arms instead of capturing a
    valid claim for the replacement while its callbacks still mutate the
    detached predecessor.
    """
    cfg = SessionEndpointConfig(
        permission_gate=None,
        manager_lookup=lambda _request: (None, None),
        tenant_check=None,
        not_found_label="missing",
        audit_action_prefix="workstream",
    )
    observed: list[tuple[bool, list[str], list[str]]] = []

    for worker_running in (False, True):
        captured_session = _ClaimingSendSession()
        replacement_session = _ClaimingSendSession()
        ws = _make_ws(captured_session)
        ws._worker_running = worker_running
        attempt = _make_dispatch_attempt(
            ws,
            cfg,
            None,
            message="must follow the replacement",
            resolved_atts=[],
            ordered_taken=[],
            send_id="",
            acting_uid="",
        )

        # Exact caller-capture -> dispatcher-capture crossing.
        ws.session = replacement_session  # type: ignore[assignment]
        accepted, _outcome = attempt(captured_session)  # type: ignore[arg-type]
        owner = ws.worker_thread
        if owner is not None:
            owner.join(timeout=2)
        observed.append(
            (
                accepted,
                list(captured_session.send_calls),
                list(captured_session.queue_calls),
            )
        )

    assert observed == [(False, [], []), (False, [], [])]


def test_abandoned_worker_does_not_clear_successor_running_flag() -> None:
    """A force-cancel abandons the worker (``ws.worker_thread`` is cleared /
    reassigned to a successor).  When the abandoned thread finishes late, its
    ``finally`` must NOT clear ``_worker_running`` out from under the live
    successor — otherwise a third send sees ``_worker_running=False`` and
    spawns a second concurrent worker on the same session."""
    send_gate = threading.Event()
    session = _SendSession(send_gate=send_gate)
    ws = _make_ws(session)

    ok = _send_message(ws, session, "hello")
    assert ok is True
    abandoned = ws.worker_thread
    assert abandoned is not None

    # Simulate force-abandon + a successor send claiming ownership while the
    # original worker is still pinned inside run().
    sentinel = threading.Thread(target=lambda: None, name="successor")
    with ws._lock:
        ws.worker_thread = sentinel
        ws._worker_running = True

    # Release the abandoned worker; it runs its finally.
    send_gate.set()
    abandoned.join(timeout=3.0)
    assert not abandoned.is_alive()

    # The successor's ownership is intact — the abandoned worker did not
    # clobber the flag or the thread handle.
    assert ws._worker_running is True
    assert ws.worker_thread is sentinel


def test_abandoned_worker_does_not_clear_successor_principal_claim() -> None:
    """A stale predecessor's finally cannot erase its successor's actor.

    Force cancellation permits the successor to claim the slot before the old
    thread returns.  Principal cleanup therefore needs the same worker-identity
    guard as ``_worker_running`` cleanup; a blind clear reopens both false-allow
    and false-reject queue decisions during the successor turn.
    """
    send_gate = threading.Event()
    session = _SendSession(send_gate=send_gate)
    ws = _make_ws(session)

    assert (
        session_worker.send(
            ws,
            enqueue=lambda: session.queue_message("first"),
            run=lambda: session.send("first"),
            principal_id="alice",
        )
        is True
    )
    abandoned = ws.worker_thread
    assert abandoned is not None
    assert ws._worker_principal_id == "alice"

    sentinel = threading.Thread(target=lambda: None, name="successor")
    with ws._lock:
        ws.worker_thread = sentinel
        ws._worker_running = True
        ws._worker_principal_id = "bob"

    send_gate.set()
    abandoned.join(timeout=3)
    assert not abandoned.is_alive()
    assert ws.worker_thread is sentinel
    assert ws._worker_running is True
    assert ws._worker_principal_id == "bob"


def test_force_during_nonabandonable_mutation_keeps_slot_until_owner_exits() -> None:
    """A destructive mutation cannot overlap a force-admitted successor."""

    mutation_entered = threading.Event()
    release_mutation = threading.Event()
    queued = threading.Event()
    ws = _make_ws(_SendSession())

    def _mutation() -> None:
        mutation_entered.set()
        assert release_mutation.wait(5), "test did not release mutation"

    assert session_worker.send(
        ws,
        enqueue=queued.set,
        run=_mutation,
        worker_kind="command",
        force_abandonable=False,
        thread_name="nonabandonable-mutation",
    )
    owner = ws.worker_thread
    assert owner is not None and mutation_entered.wait(5)

    # Exact ownership decision the force-cancel route performs under ws._lock.
    with ws._lock:
        if ws._worker_force_abandonable:
            ws.worker_thread = None
            ws._worker_running = False
            ws._worker_principal_id = ""

    assert ws.worker_thread is owner
    assert ws._worker_running is True
    assert ws._worker_force_abandonable is False

    # A would-be successor takes the existing-slot path; no concurrent worker
    # can start while the destructive mutation remains inside its transaction.
    assert session_worker.send(ws, enqueue=queued.set, run=lambda: None)
    assert queued.is_set()
    assert ws.worker_thread is owner

    release_mutation.set()
    owner.join(5)
    assert not owner.is_alive()
    assert ws._worker_running is False
    assert ws._worker_force_abandonable is True


def test_claim_capture_never_holds_workstream_lock_behind_generation_lock() -> None:
    """Worker admission cannot invert generation -> UI -> workstream order."""

    generation_lock = threading.Lock()
    generation_held = threading.Event()
    capture_attempted = threading.Event()
    worker_ran = threading.Event()
    commit_acquired_ws: list[bool] = []
    dispatch_results: list[bool] = []

    class _ClaimSession:
        def _capture_worker_claim(self, _principal_id: str = "") -> None:
            capture_attempted.set()
            with generation_lock:
                return None

    ws = _make_ws(_ClaimSession())

    def _generation_commit() -> None:
        with generation_lock:
            generation_held.set()
            assert capture_attempted.wait(5), "dispatcher never attempted claim capture"
            acquired = ws._lock.acquire(timeout=1)
            commit_acquired_ws.append(acquired)
            if acquired:
                ws._lock.release()

    def _dispatch() -> None:
        dispatch_results.append(
            session_worker.send(
                ws,
                enqueue=lambda: None,
                run=worker_ran.set,
                thread_name="lock-order-worker",
            )
        )

    commit = threading.Thread(target=_generation_commit, daemon=True)
    commit.start()
    assert generation_held.wait(5)

    dispatcher = threading.Thread(target=_dispatch, daemon=True)
    dispatcher.start()
    commit.join(5)
    dispatcher.join(5)

    assert not commit.is_alive() and not dispatcher.is_alive()
    assert commit_acquired_ws == [True]
    assert dispatch_results == [True]
    owner = ws.worker_thread
    assert owner is not None
    owner.join(5)
    assert worker_ran.is_set()


def test_concurrent_send_produces_exactly_one_worker_thread() -> None:
    """Two simultaneous send() calls must land as exactly one worker
    spawn and one queued message — not two parallel workers on the
    same ChatSession.

    The send_gate pins the worker inside session.send while the second
    caller races through; the only way the second caller can succeed
    is via the enqueue path. If the lock gate were keyed on
    Thread.is_alive instead of _worker_running, the loser could spawn
    a second worker before the winner reaches session.send.
    """
    send_gate = threading.Event()
    session = _SendSession(send_gate=send_gate)
    ws = _make_ws(session)

    results: list[bool] = []
    results_lock = threading.Lock()
    start_barrier = threading.Barrier(2)

    def _caller(msg: str) -> None:
        start_barrier.wait(timeout=1.0)
        ok = _send_message(ws, session, msg)
        with results_lock:
            results.append(ok)

    t1 = threading.Thread(target=_caller, args=("first",))
    t2 = threading.Thread(target=_caller, args=("second",))
    t1.start()
    t2.start()
    t1.join(timeout=3.0)
    t2.join(timeout=3.0)
    assert not t1.is_alive() and not t2.is_alive()

    # At this point session.send is still pinned on send_gate; the
    # second caller MUST have taken the enqueue path.
    assert len(session.queue_calls) == 1, (
        f"expected exactly one queued message; got {session.queue_calls}"
    )

    # Release the worker, verify final state.
    send_gate.set()
    assert ws.worker_thread is not None
    ws.worker_thread.join(timeout=3.0)

    assert results == [True, True]
    assert len(session.send_calls) == 1
    assert set(session.send_calls + session.queue_calls) == {"first", "second"}
    assert ws._worker_running is False


def test_thread_name_default_uses_ws_prefix() -> None:
    session = _SendSession()
    ws = _make_ws(session)
    ok = session_worker.send(
        ws,
        enqueue=lambda: session.queue_message("hello"),
        run=lambda: session.send("hello"),
    )
    assert ok is True
    assert ws.worker_thread is not None
    assert ws.worker_thread.name.startswith("session-worker-")
    ws.worker_thread.join(timeout=2.0)


def test_thread_name_explicit_override() -> None:
    session = _SendSession()
    ws = _make_ws(session)
    ok = session_worker.send(
        ws,
        enqueue=lambda: session.queue_message("hello"),
        run=lambda: session.send("hello"),
        thread_name="custom-name",
    )
    assert ok is True
    assert ws.worker_thread is not None
    assert ws.worker_thread.name == "custom-name"
    ws.worker_thread.join(timeout=2.0)


class _WakeCapableSession(_SendSession):
    """Adds the ChatSession surface the exit backstop peeks at."""

    def __init__(self, **kwargs: Any) -> None:
        super().__init__(**kwargs)
        self._nudge_queue = NudgeQueue()
        self.deliver_calls = 0
        self.deliver_thread_names: list[str] = []
        self.delivered = threading.Event()

    def deliver_wake_nudge_from_queue(self) -> None:
        # Mirror the real contract: the wake drains its own queue, so
        # the wake worker's OWN exit backstop sees nothing pending and
        # the chain converges instead of spawning wakes forever.
        self.deliver_calls += 1
        self.deliver_thread_names.append(threading.current_thread().name)
        self._nudge_queue.drain(USER_DRAIN)
        self.delivered.set()


class TestWorkerExitWakeBackstop:
    """A worker exiting while its (idle) workstream has USER_DRAIN
    nudges queued spawns the wake send the IDLE fan-out had to drop.

    Production shape being modelled: ``set_state(IDLE)`` fires its
    subscribers on the worker thread from inside ``run()`` —
    ``CoordinatorIdleObserver`` enqueues ``idle_children``, then
    ``IdleNudgeWatcher``'s wake dispatch lands on the reuse path
    (this very worker still owns the flag) and no-ops.  The enqueue
    inside ``run`` below stands in for that observer enqueue.
    """

    def test_worker_exit_delivers_pending_wake(self) -> None:
        session = _WakeCapableSession()
        ws = _make_ws(session)
        assert ws.state is WorkstreamState.IDLE  # dataclass default

        def run() -> None:
            # What the IDLE fan-out's observer does, on this thread.
            session._nudge_queue.enqueue("idle_children", "kids waiting", "any")

        ok = session_worker.send(ws, enqueue=lambda: None, run=run)
        assert ok is True

        # The wake is delivered on a fresh wake-named worker thread…
        assert session.delivered.wait(timeout=2.0), (
            "exit backstop did not deliver the pending nudge"
        )
        assert session.deliver_thread_names[0].startswith("wake-nudge-")
        # …after which the wake worker's own exit backstop sees an empty
        # queue and the chain converges: flag at rest, exactly one deliver.
        _wait_until(lambda: ws._worker_running is False)
        assert session.deliver_calls == 1
        assert len(session._nudge_queue) == 0

    def test_worker_exit_no_wake_when_queue_empty(self) -> None:
        session = _WakeCapableSession()
        ws = _make_ws(session)

        ok = session_worker.send(ws, enqueue=lambda: None, run=lambda: None)
        assert ok is True
        original = ws.worker_thread
        assert original is not None
        original.join(timeout=2.0)

        assert ws.worker_thread is original  # no wake spawned
        assert session.deliver_calls == 0
        assert ws._worker_running is False

    def test_worker_exit_no_wake_for_stub_session_without_queue(self) -> None:
        """The narrow-contract escape hatch: a session without a
        ``_nudge_queue`` (watch-style stubs) is skipped by the shared
        wake gate's own defensive peek — no AttributeError, no wake."""
        session = _SendSession()
        ws = _make_ws(session)

        ok = _send_message(ws, session, "hello")
        assert ok is True
        original = ws.worker_thread
        assert original is not None
        original.join(timeout=2.0)

        assert ws.worker_thread is original
        assert ws._worker_running is False

    def test_worker_exit_no_wake_when_state_not_idle(self) -> None:
        """An ERROR exit stays parked for the operator — pending nudges
        wait for the next real interaction rather than burning
        unattended inference on a failed session."""
        session = _WakeCapableSession()
        ws = _make_ws(session)

        def run() -> None:
            session._nudge_queue.enqueue("idle_children", "kids waiting", "any")
            ws.state = WorkstreamState.ERROR

        ok = session_worker.send(ws, enqueue=lambda: None, run=run)
        assert ok is True
        original = ws.worker_thread
        assert original is not None
        original.join(timeout=2.0)

        assert ws.worker_thread is original
        assert session.deliver_calls == 0
        assert len(session._nudge_queue) == 1  # still queued for later seams

    def test_abandoned_worker_does_not_run_wake_backstop(self) -> None:
        """Only the owner retries: an abandoned worker (successor claimed
        the flag) finishing late must not spawn a wake — the successor's
        own exit runs the backstop."""
        send_gate = threading.Event()
        session = _WakeCapableSession(send_gate=send_gate)
        ws = _make_ws(session)

        ok = _send_message(ws, session, "hello")
        assert ok is True
        abandoned = ws.worker_thread
        assert abandoned is not None

        session._nudge_queue.enqueue("idle_children", "kids waiting", "any")
        sentinel = threading.Thread(target=lambda: None, name="successor")
        with ws._lock:
            ws.worker_thread = sentinel
            ws._worker_running = True

        send_gate.set()
        abandoned.join(timeout=3.0)
        assert not abandoned.is_alive()

        # No wake spawned by the abandoned thread; ownership intact.
        assert ws.worker_thread is sentinel
        assert session.deliver_calls == 0
        assert ws._worker_running is True


class TestWorkerExitInterjectionBackstop:
    """The ownership-clear seam cannot strand a just-accepted interjection."""

    @staticmethod
    def _session() -> ChatSession:
        session = make_session(user_id="alice")
        session._acting_user_id = "alice"
        return session

    def test_enqueue_after_last_flush_hands_off_same_principal_and_correlation(
        self,
    ) -> None:
        """A reuse enqueue at run's tail is delivered by a successor wake."""
        session = self._session()
        ws = _make_ws(session)
        delivered = threading.Event()
        calls: list[tuple[str, tuple[str, ...], str]] = []

        def _record_send(message: str, *, client_send_ids: tuple[str, ...] = ()) -> None:
            claim = session_worker.current_worker_claim(session)
            assert claim is not None
            calls.append((message, client_send_ids, claim.principal_id))
            delivered.set()

        session.send = _record_send  # type: ignore[method-assign]

        def _outgoing_run() -> None:
            # This nested dispatch is the deterministic equivalent of another
            # request winning ws._lock after ChatSession's final queue flush
            # but before this runner enters its ownership-clear finally.
            assert session_worker.send(
                ws,
                enqueue=lambda: session.queue_message(
                    "late follow-up",
                    queue_msg_id="late-message",
                    interjector_user_id="alice",
                    turn_principal_id="alice",
                    client_send_id="client-late",
                ),
                run=lambda: pytest.fail("reuse dispatch spawned a second worker"),
                expected_session=session,
                principal_id="alice",
            )

        assert len(session._nudge_queue) == 0
        assert session_worker.send(
            ws,
            enqueue=lambda: None,
            run=_outgoing_run,
            expected_session=session,
            principal_id="alice",
        )
        assert delivered.wait(5), "worker-exit handoff did not deliver the queued row"
        _wait_until(lambda: ws._worker_running is False)

        assert calls == [("late follow-up", ("client-late",), "alice")]
        assert session._queued_messages == {}
        assert len(session._nudge_queue) == 0

    def test_non_exit_gate_remains_nudge_only(self) -> None:
        session = self._session()
        ws = _make_ws(session)
        session.queue_message(
            "wait for owner clear",
            queue_msg_id="nudge-only-control",
            interjector_user_id="alice",
            turn_principal_id="alice",
        )

        assert wake_workstream_if_pending(ws, trigger="idle-transition") is False
        assert list(session._queued_messages) == ["nudge-only-control"]
        assert ws._worker_running is False

    def test_retraction_before_wake_pop_converges_without_send(self) -> None:
        session = self._session()
        ws = _make_ws(session)
        session.queue_message(
            "retract me",
            queue_msg_id="retracted-message",
            interjector_user_id="alice",
            turn_principal_id="alice",
        )
        pop_entered = threading.Event()
        release_pop = threading.Event()
        real_pop = session._pop_queued_messages
        send_calls: list[str] = []

        def _blocked_pop(**kwargs: Any) -> Any:
            # kwargs passthrough so signature growth never narrows the shim.
            pop_entered.set()
            assert release_pop.wait(5), "test did not release queue pop"
            return real_pop(**kwargs)

        session._pop_queued_messages = _blocked_pop  # type: ignore[method-assign]
        session.send = lambda message, **_kwargs: send_calls.append(message)  # type: ignore[method-assign]

        assert wake_workstream_if_pending(
            ws,
            trigger="test-worker-exit",
            include_interjections=True,
        )
        assert pop_entered.wait(5), "wake worker never reached the queue pop"
        assert session.dequeue_message("retracted-message") is True
        release_pop.set()
        _wait_until(lambda: ws._worker_running is False)

        assert send_calls == []
        assert session._queued_messages == {}
        assert (
            wake_workstream_if_pending(
                ws,
                trigger="test-worker-exit",
                include_interjections=True,
            )
            is False
        )

    def test_empty_interjection_is_discarded_without_a_wake_loop(self) -> None:
        session = self._session()
        ws = _make_ws(session)
        cleaned, _priority, _msg_id = session.queue_message(
            "!!!",
            queue_msg_id="empty-message",
            interjector_user_id="alice",
            turn_principal_id="alice",
        )
        send_calls: list[str] = []
        session.send = lambda message, **_kwargs: send_calls.append(message)  # type: ignore[method-assign]

        assert cleaned == ""
        assert wake_workstream_if_pending(
            ws,
            trigger="test-worker-exit",
            include_interjections=True,
        )
        _wait_until(lambda: ws._worker_running is False)

        assert send_calls == []
        assert session._queued_messages == {}

    def test_restored_failed_wake_is_retained_without_retry_loop(self) -> None:
        session = self._session()
        ws = _make_ws(session)
        session.queue_message(
            "keep after failure",
            queue_msg_id="restored-message",
            interjector_user_id="alice",
            turn_principal_id="alice",
            client_send_id="client-restored",
        )
        attempts: list[str] = []

        def _fail_before_append(message: str, **_kwargs: Any) -> None:
            attempts.append(message)
            raise RuntimeError("injected preamble failure")

        session.send = _fail_before_append  # type: ignore[method-assign]

        assert wake_workstream_if_pending(
            ws,
            trigger="test-worker-exit",
            include_interjections=True,
        )
        wake_worker = ws.worker_thread
        assert wake_worker is not None
        _wait_until(lambda: ws._worker_running is False)

        assert attempts == ["keep after failure"]
        assert list(session._queued_messages) == ["restored-message"]
        # The failed queue-only wake's own exit carried the exact snapshot
        # token and suppressed a retry. No successor worker was spawned.
        assert ws.worker_thread is wake_worker
        assert attempts == ["keep after failure"]

    def test_deferred_wake_token_does_not_suppress_competing_worker_exit(self) -> None:
        """Only a spawned wake owns the snapshot exclusion token.

        A same-principal worker can claim the slot after the queue predicate
        but before the wake dispatch. The wake then takes the reuse/no-op arm;
        if that pre-spawn snapshot were sticky, the competing worker's exit
        could not recover the older queued row after failing before its drain.
        """
        session = self._session()
        ws = _make_ws(session)
        session.queue_message(
            "older stranded row",
            queue_msg_id="older-row",
            interjector_user_id="alice",
            turn_principal_id="alice",
            client_send_id="client-older",
        )
        competing_started = threading.Event()
        release_competing = threading.Event()
        delivered = threading.Event()
        calls: list[tuple[str, tuple[str, ...], str]] = []

        def _record_send(message: str, *, client_send_ids: tuple[str, ...] = ()) -> None:
            claim = session_worker.current_worker_claim(session)
            assert claim is not None
            calls.append((message, client_send_ids, claim.principal_id))
            delivered.set()

        session.send = _record_send  # type: ignore[method-assign]
        real_claim = session.claim_pending_interjection_wake
        injected_competitor = False

        def _claim_then_compete(*, exclude_signature: object | None = None) -> Any:
            nonlocal injected_competitor
            signature = real_claim(exclude_signature=exclude_signature)
            if signature is not None and not injected_competitor:
                injected_competitor = True

                def _competing_run() -> None:
                    competing_started.set()
                    assert release_competing.wait(5), "test did not release competing worker"
                    # Return before touching the retained queue: this models a
                    # fresh worker failing in its pre-drain setup.

                assert session_worker.send(
                    ws,
                    enqueue=lambda: pytest.fail("competitor unexpectedly reused a worker"),
                    run=_competing_run,
                    expected_session=session,
                    principal_id="alice",
                )
                assert competing_started.wait(5), "competing worker did not claim the slot"
            return signature

        session.claim_pending_interjection_wake = _claim_then_compete  # type: ignore[method-assign]

        # The queue-only wake loses the slot race and takes its no-op enqueue
        # arm. It must not leave suppression behind for the real owner.
        assert wake_workstream_if_pending(
            ws,
            trigger="test-worker-exit",
            include_interjections=True,
        )
        assert calls == []
        assert list(session._queued_messages) == ["older-row"]

        release_competing.set()
        assert delivered.wait(5), "competing worker exit did not retry the older row"
        _wait_until(lambda: ws._worker_running is False)

        assert calls == [("older stranded row", ("client-older",), "alice")]
        assert session._queued_messages == {}

    @pytest.mark.parametrize(
        "blocker",
        ["budget", "abandoned", "unresolved", "foreign_principal", "gone"],
    )
    def test_queue_only_wake_refuses_unattended_retry_blockers(self, blocker: str) -> None:
        session = self._session()
        ws = _make_ws(session)
        owner = "bob" if blocker == "foreign_principal" else "alice"
        session.queue_message(
            "wait for an explicit seam",
            queue_msg_id=f"blocked-{blocker}",
            interjector_user_id=owner,
            turn_principal_id=owner,
        )
        if blocker == "budget":
            session._budget_exhausted = True
        elif blocker == "abandoned":
            session._generation_abandoned = True
        elif blocker == "unresolved":
            session.has_unresolved_conversation_persistence = lambda: True  # type: ignore[method-assign]
        elif blocker == "gone":
            # The terminal hard-delete latch: the gone discovery clears the
            # pending-commit journal, so the unresolved-persistence blocker
            # reads False exactly when no turn can ever land — the latch
            # needs its own refusal arm or the wake spawns doomed.
            session._workstream_gone_ws = session._ws_id

        assert (
            wake_workstream_if_pending(
                ws,
                trigger="test-worker-exit",
                include_interjections=True,
            )
            is False
        )
        assert list(session._queued_messages) == [f"blocked-{blocker}"]
        assert ws._worker_running is False

    def test_interjection_claim_itself_refuses_a_gone_workstream(self) -> None:
        """The claim gate carries its own gone arm — it must refuse for ANY
        caller, not only behind the watcher's spawn gate (a future caller
        that consults the claim directly gets the same fail-closed answer)."""
        session = self._session()
        session.queue_message(
            "never wake for this",
            queue_msg_id="gone-q",
            interjector_user_id="alice",
            turn_principal_id="alice",
        )
        session._workstream_gone_ws = session._ws_id
        assert session.claim_pending_interjection_wake() is None
        assert list(session._queued_messages) == ["gone-q"]


def test_does_not_deadlock_when_run_briefly_grabs_ws_lock() -> None:
    """Sanity check: ``run`` is invoked OUTSIDE ``ws._lock``. A worker
    body that briefly takes the lock (e.g. to update worker state)
    must not deadlock with the dispatch path."""
    session = _SendSession()
    ws = _make_ws(session)

    def run() -> None:
        with ws._lock:
            pass  # would deadlock if dispatch held the lock here
        session.send("hello")

    ok = session_worker.send(
        ws,
        enqueue=lambda: session.queue_message("hello"),
        run=run,
    )
    assert ok is True
    assert ws.worker_thread is not None
    ws.worker_thread.join(timeout=2.0)
    assert session.send_calls == ["hello"]
    assert ws._worker_running is False