turnstone/tests/test_notify_dispatcher.py

"""Tests for the console-side ``NotifyDispatcher``.

Exercises the dispatcher against the SQLite synthetic-sweep path so the
suite runs without a Postgres dependency.  The PG path is shaped the
same way (same handler invocation semantics) — the only difference is
the underlying stream's wake-up source, which is covered separately in
``test_storage_notify.py::TestPostgresNotify``.
"""

from __future__ import annotations

import threading
import time

import pytest


@pytest.fixture
def dispatcher_factory(storage):
    """Yield a factory that constructs + tracks dispatchers for teardown."""
    from turnstone.console.notify_dispatcher import NotifyDispatcher

    created: list[NotifyDispatcher] = []

    def _make(*, channels: list[str]) -> NotifyDispatcher:
        d = NotifyDispatcher(storage, channels=channels)
        created.append(d)
        return d

    yield _make

    for d in created:
        d.stop(timeout=2.0)


def _wait_for(predicate, deadline_sec: float = 3.0) -> bool:
    """Poll ``predicate`` until True or timeout.  Returns bool."""
    deadline = time.monotonic() + deadline_sec
    while time.monotonic() < deadline:
        if predicate():
            return True
        time.sleep(0.02)
    return False


def _start_ready(d, *, timeout: float = 5.0) -> None:
    """``d.start()`` + assert the listener is actually listening.

    Closes the start-vs-notify race for backends where ``storage.listen``
    blocks on the network (Postgres ``LISTEN`` over a fresh psycopg
    connection): without the sync, a same-thread ``storage.notify`` can
    fire before the LISTEN registers and the notification is lost.
    """
    d.start()
    if not d.wait_until_ready(timeout=timeout):
        msg = f"dispatcher listener did not open within {timeout}s"
        raise AssertionError(msg)


class TestSubscribe:
    def test_subscribe_registers_handler(self, dispatcher_factory, storage):
        d = dispatcher_factory(channels=["alpha"])
        seen: list = []
        d.subscribe("alpha", lambda n: seen.append(n))
        _start_ready(d)
        # Fire a notify via the storage layer — dispatcher delivers to handler.
        storage.notify("alpha", "hello")
        assert _wait_for(lambda: any(n.payload == "hello" for n in seen))

    def test_subscribe_undeclared_channel_raises(self, dispatcher_factory):
        d = dispatcher_factory(channels=["alpha"])
        with pytest.raises(ValueError, match="not declared"):
            d.subscribe("beta", lambda n: None)

    def test_subscribe_returns_unsubscribe_callable(self, dispatcher_factory, storage):
        d = dispatcher_factory(channels=["alpha"])
        seen: list = []
        unsub = d.subscribe("alpha", lambda n: seen.append(n))
        _start_ready(d)
        storage.notify("alpha", "first")
        assert _wait_for(lambda: any(n.payload == "first" for n in seen))
        unsub()
        # After unsubscribe, the handler no longer fires.  Drain old hits
        # so the next notify-vs-handler-count check is unambiguous.
        seen.clear()
        storage.notify("alpha", "second")
        # Give the dispatcher a beat to deliver if it were going to.
        time.sleep(0.2)
        assert not any(n.payload == "second" for n in seen)

    def test_construction_requires_at_least_one_channel(self, storage):
        from turnstone.console.notify_dispatcher import NotifyDispatcher

        with pytest.raises(ValueError, match="at least one"):
            NotifyDispatcher(storage, channels=[])

    def test_duplicate_channels_deduplicated(self, dispatcher_factory):
        d = dispatcher_factory(channels=["alpha", "alpha", "beta"])
        assert d.channels == ["alpha", "beta"]


class TestDispatch:
    def test_multiple_handlers_each_invoked(self, dispatcher_factory, storage):
        d = dispatcher_factory(channels=["alpha"])
        seen_a: list = []
        seen_b: list = []
        d.subscribe("alpha", lambda n: seen_a.append(n))
        d.subscribe("alpha", lambda n: seen_b.append(n))
        _start_ready(d)
        storage.notify("alpha", "shared")
        assert _wait_for(lambda: seen_a and seen_b)
        assert seen_a[0].payload == "shared"
        assert seen_b[0].payload == "shared"

    def test_handler_exception_does_not_break_dispatch(self, dispatcher_factory, storage):
        d = dispatcher_factory(channels=["alpha"])
        survived: list = []

        def _broken(_n):
            msg = "boom"
            raise RuntimeError(msg)

        d.subscribe("alpha", _broken)
        d.subscribe("alpha", lambda n: survived.append(n))
        _start_ready(d)
        storage.notify("alpha", "after_broken")
        # The second handler runs even though the first raised.
        assert _wait_for(lambda: any(n.payload == "after_broken" for n in survived))

    def test_dispatch_filters_by_channel(self, dispatcher_factory, storage):
        d = dispatcher_factory(channels=["alpha", "beta"])
        seen_a: list = []
        seen_b: list = []
        d.subscribe("alpha", lambda n: seen_a.append(n))
        d.subscribe("beta", lambda n: seen_b.append(n))
        _start_ready(d)
        storage.notify("alpha", "for_a")
        storage.notify("beta", "for_b")
        assert _wait_for(lambda: seen_a and seen_b)
        assert all(n.payload == "for_a" for n in seen_a)
        assert all(n.payload == "for_b" for n in seen_b)


class TestReconnect:
    """Reconnect + synthetic ``reconcile`` notify on stream-open success.

    Uses a stub storage that owns its own listen stream so the test can
    drive a controlled stream-error sequence — the SQLite path can't
    raise :class:`NotifyConnectionError`, and the PG path requires a
    real database outage to exercise this code, neither of which fits a
    unit test.  The dispatcher's threading and reconcile-pending logic
    are storage-agnostic — the dispatcher sees the same
    :class:`NotifyStream` Protocol regardless of backend.
    """

    def test_reconcile_fires_after_reopen_not_before(self):
        from turnstone.console.notify_dispatcher import NotifyDispatcher
        from turnstone.core.storage._notify import Notify, NotifyConnectionError

        # State machine: open -> first poll raises NotifyConnectionError
        # -> dispatcher waits backoff then reopens -> second open's first
        # poll blocks forever (test stops the dispatcher before then).
        # The fix: synthetic reconcile fires AFTER the second open
        # succeeds, not after the first open fails.
        sequence: list[str] = []
        reopen_event = threading.Event()

        class _StubStream:
            def __init__(self, fail_first_poll: bool):
                self._fail = fail_first_poll
                self._closed = False

            def poll(self, _timeout):
                if self._closed:
                    return []
                if self._fail:
                    self._fail = False
                    sequence.append("poll_raises")
                    msg = "fake-disconnect"
                    raise NotifyConnectionError(msg)
                sequence.append("poll_returns")
                # Block until close to simulate a quiet steady-state.
                time.sleep(0.5)
                return []

            def close(self):
                self._closed = True

        class _StubStorage:
            def __init__(self):
                self._open_count = 0

            def listen(self, _channels):
                import contextlib as _contextlib

                @_contextlib.contextmanager
                def _cm():
                    self._open_count += 1
                    sequence.append(f"open_{self._open_count}")
                    if self._open_count == 2:
                        reopen_event.set()
                    stream = _StubStream(fail_first_poll=(self._open_count == 1))
                    try:
                        yield stream
                    finally:
                        stream.close()

                return _cm()

        # Speed up backoff so the reopen happens promptly in the test.
        import turnstone.console.notify_dispatcher as nd_mod

        original_backoff = nd_mod._RECONNECT_BACKOFF_INITIAL
        nd_mod._RECONNECT_BACKOFF_INITIAL = 0.05
        try:
            d = NotifyDispatcher(_StubStorage(), channels=["alpha"])
            got: list[Notify] = []
            d.subscribe("alpha", lambda n: got.append(n))
            d.start()
            try:
                # Wait for the second open (post-reconnect).
                assert reopen_event.wait(3.0), "dispatcher did not reopen after disconnect"
                # Reconcile should be delivered shortly after the reopen.
                deadline = time.monotonic() + 2.0
                while time.monotonic() < deadline:
                    if any(n.payload == "reconcile" for n in got):
                        break
                    time.sleep(0.02)
                assert any(n.payload == "reconcile" for n in got), (
                    f"no reconcile delivered; sequence={sequence}, got={got}"
                )
                # The reconcile must NOT fire before the second open —
                # if it did, the index of 'open_2' in sequence would
                # come after any reconcile-emitting work.  Check ordering:
                # 'open_1' < 'poll_raises' < 'open_2' (synthesize happens
                # inside the with-block of the SECOND open).
                ix_open_1 = sequence.index("open_1")
                ix_raises = sequence.index("poll_raises")
                ix_open_2 = sequence.index("open_2")
                assert ix_open_1 < ix_raises < ix_open_2
            finally:
                d.stop(timeout=2.0)
        finally:
            nd_mod._RECONNECT_BACKOFF_INITIAL = original_backoff

    def test_generic_exception_path_also_synthesizes_reconcile(self):
        """Exceptions thrown during ``listen()`` (not via stream.poll) still trigger reconcile.

        Models the ``psycopg.connect()`` / initial ``LISTEN`` failure
        shape, which doesn't go through the stream's exception
        translator and would hit the generic ``except Exception``
        branch.  Pre-fix, that branch emitted no reconcile.
        """
        from turnstone.console.notify_dispatcher import NotifyDispatcher

        reopen_event = threading.Event()

        class _StubStream:
            def __init__(self):
                self._closed = False

            def poll(self, _timeout):
                if self._closed:
                    return []
                time.sleep(0.5)
                return []

            def close(self):
                self._closed = True

        class _StubStorage:
            def __init__(self):
                self._open_count = 0

            def listen(self, _channels):
                import contextlib as _contextlib

                self._open_count += 1
                if self._open_count == 1:
                    # First open raises a generic exception (e.g.
                    # ``psycopg.OperationalError`` from a failed connect)
                    # — landing in the dispatcher's generic except branch.
                    msg = "fake-connect-failure"
                    raise RuntimeError(msg)

                @_contextlib.contextmanager
                def _cm():
                    reopen_event.set()
                    stream = _StubStream()
                    try:
                        yield stream
                    finally:
                        stream.close()

                return _cm()

        import turnstone.console.notify_dispatcher as nd_mod

        original_backoff = nd_mod._RECONNECT_BACKOFF_INITIAL
        nd_mod._RECONNECT_BACKOFF_INITIAL = 0.05
        try:
            d = NotifyDispatcher(_StubStorage(), channels=["alpha"])
            got: list = []
            d.subscribe("alpha", lambda n: got.append(n))
            d.start()
            try:
                assert reopen_event.wait(3.0), "dispatcher did not reopen after generic exception"
                deadline = time.monotonic() + 2.0
                while time.monotonic() < deadline:
                    if any(n.payload == "reconcile" for n in got):
                        break
                    time.sleep(0.02)
                assert any(n.payload == "reconcile" for n in got), (
                    "no reconcile delivered after generic-exception recovery"
                )
            finally:
                d.stop(timeout=2.0)
        finally:
            nd_mod._RECONNECT_BACKOFF_INITIAL = original_backoff


class TestCoalescing:
    """Same-channel burst collapses to one handler invocation per batch."""

    def test_burst_coalesces_to_one_handler_call_per_channel(self, dispatcher_factory, storage):
        d = dispatcher_factory(channels=["alpha"])
        invocations: list = []
        # Slow handler to ensure all bursts queue up before the first
        # call returns — gives the dispatch loop time to coalesce.
        coalesce_gate = threading.Event()

        def _slow_handler(n):
            invocations.append(n)
            coalesce_gate.wait(0.05)

        d.subscribe("alpha", _slow_handler)
        _start_ready(d)
        # Burst of 10 notifies on the same channel — should coalesce
        # down to many fewer handler invocations.
        for i in range(10):
            storage.notify("alpha", str(i))
        # Wait until the dispatch settles (handler is called at least once
        # and the queue empties).
        deadline = time.monotonic() + 2.0
        while time.monotonic() < deadline:
            if invocations and d._dispatch_queue.empty():
                time.sleep(0.1)  # allow any final coalesced call to land
                break
            time.sleep(0.02)
        coalesce_gate.set()
        # At least one handler call; well fewer than 10 (coalescing
        # collapsed the burst).  Exact count depends on timing — typical
        # is 1-2 invocations per burst on a fast machine.
        assert invocations, "handler never fired"
        assert len(invocations) < 10, (
            f"expected coalescing to collapse burst of 10; got {len(invocations)} invocations"
        )


class TestLifecycle:
    def test_start_is_idempotent(self, dispatcher_factory):
        d = dispatcher_factory(channels=["alpha"])
        d.start()
        d.start()  # No-op, no thread doubling
        # Single listener + single dispatch thread are spawned regardless.
        # Inspect by name so we don't depend on the exact thread count of
        # the test runner.
        listener_threads = [
            t for t in threading.enumerate() if t.name == "notify-dispatcher-listener"
        ]
        dispatch_threads = [
            t for t in threading.enumerate() if t.name == "notify-dispatcher-dispatch"
        ]
        assert len(listener_threads) == 1
        assert len(dispatch_threads) == 1

    def test_stop_is_idempotent(self, dispatcher_factory):
        d = dispatcher_factory(channels=["alpha"])
        d.start()
        d.stop(timeout=2.0)
        d.stop(timeout=2.0)  # No-op, no error

    def test_stop_without_start_is_noop(self, dispatcher_factory):
        d = dispatcher_factory(channels=["alpha"])
        d.stop(timeout=1.0)  # No-op, no thread to join

    def test_stop_joins_threads(self, dispatcher_factory):
        d = dispatcher_factory(channels=["alpha"])
        d.start()
        # Capture thread references then stop and assert they exited.
        threads_before = [
            t
            for t in threading.enumerate()
            if t.name in {"notify-dispatcher-listener", "notify-dispatcher-dispatch"}
        ]
        assert threads_before
        d.stop(timeout=3.0)
        time.sleep(0.05)
        for t in threads_before:
            assert not t.is_alive(), f"{t.name} still alive after stop"