turnstone/tests/test_create_lifecycle_sequencing.py

"""Race regressions for the deferred-create publication boundary."""

from __future__ import annotations

import asyncio
import json
import queue
import threading
import time
from typing import Any

import httpx
import pytest
from starlette.applications import Starlette
from starlette.routing import Route

from tests.test_server_authz import (
    _auth,
    _FakeSession,
)
from tests.test_server_authz import app_client as app_client
from tests.test_session_manager import FakeAdapter, _make_manager
from turnstone.core.session_routes import (
    SessionEndpointConfig,
    make_create_handler,
)


class _BlockingCreateEmitter(FakeAdapter):
    """Pause inside ``emit_created`` after commit admission."""

    def __init__(self) -> None:
        super().__init__()
        self.create_emit_entered = threading.Event()
        self.release_create_emit = threading.Event()

    def emit_created(self, ws: Any) -> None:
        self.create_emit_entered.set()
        assert self.release_create_emit.wait(timeout=10), "test did not release create emit"
        super().emit_created(ws)


@pytest.mark.parametrize("terminal", ["close", "delete"])
def test_terminal_after_commit_admission_observes_created_first(terminal: str) -> None:
    """A close/delete admitted during create fan-out cannot overtake it."""
    adapter = _BlockingCreateEmitter()
    mgr, _, _ = _make_manager(adapter=adapter)
    ws = mgr.create(user_id="u1", name="ordered", defer_emit_created=True)
    commit_result: list[bool] = []
    terminal_result: list[bool] = []
    terminal_started = threading.Event()
    terminal_done = threading.Event()

    def _commit() -> None:
        commit_result.append(mgr.commit_create(ws))

    def _retire() -> None:
        terminal_started.set()
        if terminal == "close":
            terminal_result.append(mgr.close(ws.id))
        else:
            terminal_result.append(mgr.delete(ws.id))
        terminal_done.set()

    commit_thread = threading.Thread(target=_commit, daemon=True)
    terminal_thread = threading.Thread(target=_retire, daemon=True)
    commit_thread.start()
    assert adapter.create_emit_entered.wait(timeout=5), "commit never entered emit_created"
    terminal_thread.start()
    assert terminal_started.wait(timeout=5)
    try:
        assert not terminal_done.wait(timeout=0.1), "terminal transition overtook create emit"
    finally:
        adapter.release_create_emit.set()
    commit_thread.join(timeout=5)
    terminal_thread.join(timeout=5)

    assert not commit_thread.is_alive()
    assert not terminal_thread.is_alive()
    assert commit_result == [True]
    assert terminal_result == [True]
    assert [(event.kind, event.reason) for event in adapter.events] == [
        ("created", None),
        ("closed", "closed" if terminal == "close" else "deleted"),
    ]


def test_pending_idle_deferred_create_is_not_capacity_evicted() -> None:
    """A not-yet-published IDLE reservation remains an in-flight transaction."""
    mgr, adapter, _ = _make_manager(max_active=1)
    pending = mgr.create(
        user_id="u1",
        name="pending",
        defer_emit_created=True,
    )

    with pytest.raises(RuntimeError, match="All 1 slots are active"):
        mgr.create(user_id="u2", name="challenger")

    # Pending reservations are deliberately hidden from public lookup/list
    # surfaces. Prove the exact object survived capacity pressure by committing
    # it successfully, after which it becomes visible as the sole occupant.
    assert mgr.count == 1
    assert adapter.events == []
    assert mgr.commit_create(pending) is True
    assert mgr.get(pending.id) is pending
    assert mgr.list_all() == [pending]
    assert [event.kind for event in adapter.events] == ["created"]
    assert adapter.cleaned_up == []
    assert mgr.eviction_count == 0


def _drain_global_events(app_client: Any) -> list[dict[str, Any]]:
    events: list[dict[str, Any]] = []
    global_queue = app_client.app.state.global_queue
    while True:
        try:
            events.append(global_queue.get_nowait())
        except queue.Empty:
            return events


def _created_audits(storage: Any, ws_id: str) -> list[dict[str, Any]]:
    return [
        event
        for event in storage.list_audit_events(action="workstream.created")
        if event["resource_id"] == ws_id
    ]


async def _wait_for_thread_event(event: threading.Event, timeout: float) -> bool:
    """Poll a thread seam without occupying the loop's default executor."""
    loop = asyncio.get_running_loop()
    deadline = loop.time() + timeout
    while not event.is_set():
        if loop.time() >= deadline:
            return False
        await asyncio.sleep(0.01)
    return True


@pytest.mark.anyio
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
async def test_cancellation_during_session_build_removes_exact_hidden_create(
    app_client: Any,
    monkeypatch: pytest.MonkeyPatch,
    anyio_backend: str,
) -> None:
    """A cancelled request drains the admitted build before exact rollback."""
    from turnstone.core.attachment_buffer import get_attachment_buffer

    assert anyio_backend == "asyncio"
    sync_client, mgr = app_client
    storage = sync_client.app.state.auth_storage
    assert storage is not None
    ws_id = "1" * 32
    build_entered = threading.Event()
    release_build = threading.Event()
    build_finished = threading.Event()
    original_build = mgr._adapter.build_session
    buffer = get_attachment_buffer()
    buffer.clear()

    def _blocked_build(ws: Any, **kwargs: Any) -> Any:
        build_entered.set()
        assert release_build.wait(timeout=10), "test did not release session build"
        try:
            return original_build(ws, **kwargs)
        finally:
            build_finished.set()

    monkeypatch.setattr(mgr._adapter, "build_session", _blocked_build)
    transport = httpx.ASGITransport(app=sync_client.app)
    try:
        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
            request_task = asyncio.create_task(
                client.post(
                    "/v1/api/workstreams/new",
                    json={"ws_id": ws_id, "name": "cancel-during-build"},
                    headers=_auth("user-1"),
                )
            )
            assert await _wait_for_thread_event(build_entered, 5), "create never entered build"

            # A caller-known id can receive a concurrent staged upload while
            # its hidden durable reservation is still being constructed.
            buffer.stage(
                ws_id=ws_id,
                user_id="user-1",
                filename="pending.md",
                mime_type="text/markdown",
                kind="text",
                content=b"pending create upload",
            )
            with mgr._lock:
                pending = mgr._workstreams.get(ws_id)
                assert pending is not None
                assert mgr._pending_creates.get(ws_id) is pending
            assert storage.get_workstream(ws_id) is not None

            request_task.cancel()
            await asyncio.sleep(0.05)
            assert not request_task.done()
            release_build.set()
            with pytest.raises(asyncio.CancelledError):
                await request_task
    finally:
        release_build.set()

    assert build_finished.is_set()
    with mgr._lock:
        assert ws_id not in mgr._workstreams
        assert ws_id not in mgr._pending_creates
    assert storage.get_workstream(ws_id) is None
    assert buffer.list_for(ws_id=ws_id, user_id="user-1") == []
    assert _created_audits(storage, ws_id) == []
    assert not [event for event in _drain_global_events(sync_client) if event.get("ws_id") == ws_id]
    buffer.clear()


@pytest.mark.anyio
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
async def test_second_cancellation_cannot_interrupt_create_rollback(
    app_client: Any,
    monkeypatch: pytest.MonkeyPatch,
    anyio_backend: str,
) -> None:
    """Repeated cancellation is deferred until discard and delete settle."""
    from turnstone.core.attachment_buffer import get_attachment_buffer

    assert anyio_backend == "asyncio"
    sync_client, mgr = app_client
    storage = sync_client.app.state.auth_storage
    assert storage is not None
    ws_id = "2" * 32
    build_entered = threading.Event()
    release_build = threading.Event()
    rollback_entered = threading.Event()
    release_rollback = threading.Event()
    original_build = mgr._adapter.build_session
    original_discard = mgr.discard
    buffer = get_attachment_buffer()
    buffer.clear()

    def _blocked_build(ws: Any, **kwargs: Any) -> Any:
        build_entered.set()
        assert release_build.wait(timeout=10), "test did not release session build"
        return original_build(ws, **kwargs)

    def _blocked_discard(*args: Any, **kwargs: Any) -> bool:
        rollback_entered.set()
        assert release_rollback.wait(timeout=10), "test did not release rollback"
        return original_discard(*args, **kwargs)

    monkeypatch.setattr(mgr._adapter, "build_session", _blocked_build)
    monkeypatch.setattr(mgr, "discard", _blocked_discard)
    transport = httpx.ASGITransport(app=sync_client.app)
    try:
        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
            request_task = asyncio.create_task(
                client.post(
                    "/v1/api/workstreams/new",
                    json={"ws_id": ws_id, "name": "cancel-rollback-twice"},
                    headers=_auth("user-1"),
                )
            )
            assert await _wait_for_thread_event(build_entered, 5), "create never entered build"
            buffer.stage(
                ws_id=ws_id,
                user_id="user-1",
                filename="pending.md",
                mime_type="text/markdown",
                kind="text",
                content=b"survives until rollback",
            )
            request_task.cancel()
            release_build.set()
            assert await _wait_for_thread_event(rollback_entered, 5), "rollback never started"

            # The first cancellation has already transferred ownership to the
            # cleanup bracket. A second one must not strand either half of the
            # in-memory/durable rollback transaction.
            assert request_task.cancel() is True
            await asyncio.sleep(0.05)
            assert not request_task.done()
            assert storage.get_workstream(ws_id) is not None
            assert buffer.list_for(ws_id=ws_id, user_id="user-1")
            release_rollback.set()
            with pytest.raises(asyncio.CancelledError):
                await request_task
    finally:
        release_build.set()
        release_rollback.set()

    with mgr._lock:
        assert ws_id not in mgr._workstreams
        assert ws_id not in mgr._pending_creates
    assert storage.get_workstream(ws_id) is None
    assert buffer.list_for(ws_id=ws_id, user_id="user-1") == []
    assert _created_audits(storage, ws_id) == []
    assert not [event for event in _drain_global_events(sync_client) if event.get("ws_id") == ws_id]
    buffer.clear()


def test_partial_multipart_failure_drops_staged_refs_before_same_id_successor(
    app_client: Any,
) -> None:
    """A partially staged failed request cannot lend uploads to its successor."""
    from turnstone.core.attachment_buffer import get_attachment_buffer

    client, mgr = app_client
    storage = client.app.state.auth_storage
    assert storage is not None
    ws_id = "3" * 32
    buffer = get_attachment_buffer()
    buffer.clear()
    try:
        failed = client.post(
            "/v1/api/workstreams/new",
            data={"meta": json.dumps({"ws_id": ws_id, "name": "partial"})},
            files=[
                ("file", ("valid.md", b"first file stages", "text/markdown")),
                ("file", ("invalid.bin", b"\x00\x01\x02", "application/octet-stream")),
            ],
            headers=_auth("user-1"),
        )

        assert failed.status_code == 400, failed.text
        assert storage.get_workstream(ws_id) is None
        assert buffer.list_for(ws_id=ws_id, user_id="user-1") == []
        with mgr._lock:
            assert ws_id not in mgr._workstreams
            assert ws_id not in mgr._pending_creates
        assert _created_audits(storage, ws_id) == []
        assert not [event for event in _drain_global_events(client) if event.get("ws_id") == ws_id]

        successor = client.post(
            "/v1/api/workstreams/new",
            json={"ws_id": ws_id, "name": "successor"},
            headers=_auth("user-1"),
        )
        assert successor.status_code == 200, successor.text
        assert successor.json()["attachment_ids"] == []
        assert storage.get_workstream(ws_id) is not None
        assert buffer.list_for(ws_id=ws_id, user_id="user-1") == []
    finally:
        buffer.clear()


def test_close_idle_zero_never_retires_pending_create() -> None:
    """The idle sweeper treats a hidden reservation as an in-flight create."""
    mgr, adapter, storage = _make_manager()
    pending = mgr.create(
        user_id="u1",
        name="pending-idle",
        defer_emit_created=True,
    )
    pending.last_active = time.monotonic() - 100

    assert mgr.close_idle(max_age_seconds=0) == []
    assert adapter.cleaned_up == []
    assert adapter.events == []
    assert storage.rows[pending.id].state == "creating"
    with mgr._lock:
        assert mgr._workstreams.get(pending.id) is pending
        assert mgr._pending_creates.get(pending.id) is pending

    assert mgr.commit_create(pending) is True
    assert mgr.get(pending.id) is pending
    assert [event.kind for event in adapter.events] == ["created"]


def test_delete_endpoint_waits_for_admitted_create_publication(
    app_client: Any,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Durable deletion linearizes after a create that already owns admission."""
    client, mgr = app_client
    storage = client.app.state.auth_storage
    assert storage is not None
    ws_id = "4" * 32
    pending = mgr.create(
        ws_id=ws_id,
        user_id="user-1",
        name="commit-before-delete",
        defer_emit_created=True,
    )
    emit_entered = threading.Event()
    release_emit = threading.Event()
    delete_admission_entered = threading.Event()
    durable_delete_entered = threading.Event()
    delete_started = threading.Event()
    original_emit = mgr._event_emitter.emit_created
    original_delete = storage.delete_workstream_if_fork_reserved
    original_delete_persisted = mgr.delete_persisted
    commit_results: list[bool] = []
    delete_responses: list[Any] = []

    def _blocked_emit(ws: Any) -> None:
        emit_entered.set()
        assert release_emit.wait(timeout=10), "test did not release create publication"
        original_emit(ws)

    def _tracked_delete(candidate_id: str, reservation_token: str) -> bool:
        durable_delete_entered.set()
        return original_delete(candidate_id, reservation_token)

    def _tracked_delete_persisted(*args: Any, **kwargs: Any) -> bool:
        delete_admission_entered.set()
        return original_delete_persisted(*args, **kwargs)

    def _commit() -> None:
        commit_results.append(mgr.commit_create(pending))

    def _delete() -> None:
        delete_started.set()
        delete_responses.append(
            client.post(
                f"/v1/api/workstreams/{ws_id}/delete",
                headers=_auth("user-1"),
            )
        )

    monkeypatch.setattr(mgr._event_emitter, "emit_created", _blocked_emit)
    monkeypatch.setattr(storage, "delete_workstream_if_fork_reserved", _tracked_delete)
    monkeypatch.setattr(mgr, "delete_persisted", _tracked_delete_persisted)
    commit_thread = threading.Thread(target=_commit, daemon=True)
    delete_thread = threading.Thread(target=_delete, daemon=True)
    commit_thread.start()
    assert emit_entered.wait(timeout=5), "commit never entered publication"
    delete_thread.start()
    assert delete_started.wait(timeout=5)
    try:
        assert delete_admission_entered.wait(timeout=5), "delete never reached manager admission"
        assert delete_thread.is_alive(), "delete overtook admitted create publication"
        assert not durable_delete_entered.is_set()
        assert storage.get_workstream(ws_id) is not None
    finally:
        release_emit.set()
    commit_thread.join(timeout=10)
    delete_thread.join(timeout=10)

    assert not commit_thread.is_alive()
    assert not delete_thread.is_alive()
    assert commit_results == [True]
    assert len(delete_responses) == 1
    assert delete_responses[0].status_code == 200, delete_responses[0].text
    assert durable_delete_entered.is_set()
    assert storage.get_workstream(ws_id) is None
    assert mgr.get(ws_id) is None
    lifecycle = [
        event["type"]
        for event in _drain_global_events(client)
        if event.get("ws_id") == ws_id and event.get("type") in {"ws_created", "ws_closed"}
    ]
    assert lifecycle == ["ws_created", "ws_closed"]


@pytest.mark.parametrize("terminal", ["close", "delete"])
def test_interactive_post_install_has_no_late_publication_after_terminal(
    app_client: Any,
    monkeypatch: pytest.MonkeyPatch,
    terminal: str,
) -> None:
    """The post-commit tail cannot publish or install onto a retired object."""
    from turnstone.core.audit import record_audit as original_record_audit
    from turnstone.core.storage import get_storage

    client, mgr = app_client
    storage = get_storage()
    assert storage is not None
    source_id = "a" * 32
    destination_id = "b" * 32
    storage.register_workstream(
        source_id,
        node_id="node-test",
        name="source",
        user_id="user-1",
    )
    audit_entered = threading.Event()
    release_audit = threading.Event()
    watch_registrations: list[str] = []

    def _record_audit(*args: Any, **kwargs: Any) -> Any:
        action = args[2] if len(args) > 2 else kwargs.get("action")
        if action == "workstream.created":
            audit_entered.set()
            assert release_audit.wait(timeout=10), "test did not release create audit"
        return original_record_audit(*args, **kwargs)

    def _set_watch_runner(session: _FakeSession, *_args: Any, **_kwargs: Any) -> None:
        watch_registrations.append(session.ws_id)

    monkeypatch.setattr("turnstone.core.audit.record_audit", _record_audit)
    monkeypatch.setattr(_FakeSession, "set_watch_runner", _set_watch_runner)
    client.app.state.watch_runner = object()
    responses: list[Any] = []
    request_errors: list[BaseException] = []

    def _create() -> None:
        try:
            responses.append(
                client.post(
                    "/v1/api/workstreams/new",
                    json={
                        "ws_id": destination_id,
                        "name": "named fork",
                        "resume_ws": source_id,
                    },
                    headers=_auth("user-1"),
                )
            )
        except BaseException as exc:  # pragma: no cover - diagnostic capture
            request_errors.append(exc)

    request_thread = threading.Thread(target=_create, daemon=True)
    request_thread.start()
    assert audit_entered.wait(timeout=5), "create never reached the post-commit audit"
    destination = mgr.get(destination_id)
    assert destination is not None
    try:
        if terminal == "close":
            assert mgr.close(destination_id) is True
        else:
            assert storage.delete_workstream(destination_id) is True
            assert mgr.delete(destination_id) is True

        events_before_release = _drain_global_events(client)
        lifecycle_before_release = [
            event["type"]
            for event in events_before_release
            if event.get("type") in {"ws_created", "ws_rename", "ws_closed"}
        ]
        assert lifecycle_before_release == ["ws_created", "ws_rename", "ws_closed"]
        assert watch_registrations == [destination_id]
    finally:
        release_audit.set()
    request_thread.join(timeout=10)

    assert not request_thread.is_alive()
    assert request_errors == []
    assert len(responses) == 1
    assert responses[0].status_code == 200, responses[0].text
    assert mgr.get(destination_id) is None
    events_after_release = _drain_global_events(client)
    assert not [
        event for event in events_after_release if event.get("type") in {"ws_created", "ws_rename"}
    ]
    assert watch_registrations == [destination_id]


@pytest.mark.anyio
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
async def test_cancellation_after_commit_waits_post_install_and_keeps_one_create(
    anyio_backend: str,
) -> None:
    """Cancellation preserves the admitted create and drains its shielded tail."""
    assert anyio_backend == "asyncio"
    mgr, adapter, _ = _make_manager()
    post_install_entered = asyncio.Event()
    release_post_install = asyncio.Event()
    post_install_completed = False

    def _manager_lookup(_request: Any) -> tuple[Any, None]:
        return mgr, None

    def _build_kwargs(
        _request: Any,
        body: dict[str, Any],
        uid: str,
        _skill_data: dict[str, Any] | None,
        _skill_id: str,
        _skill_version: int,
    ) -> dict[str, Any]:
        return {"user_id": uid or "u1", "name": str(body.get("name") or "")}

    async def _post_install(
        _request: Any,
        _ws: Any,
        _body: dict[str, Any],
        _uid: str,
        _skill_data: dict[str, Any] | None,
        _skill_version: int,
        _attachment_ids: list[str],
    ) -> dict[str, Any]:
        nonlocal post_install_completed
        post_install_entered.set()
        await release_post_install.wait()
        post_install_completed = True
        return {}

    cfg = SessionEndpointConfig(
        permission_gate=None,
        manager_lookup=_manager_lookup,
        tenant_check=None,
        not_found_label="Workstream not found",
        audit_action_prefix="workstream",
        create_build_kwargs=_build_kwargs,
        create_post_install=_post_install,
    )
    app = Starlette(routes=[Route("/new", make_create_handler(cfg), methods=["POST"])])
    transport = httpx.ASGITransport(app=app)
    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
        request_task = asyncio.create_task(client.post("/new", json={"name": "cancelled"}))
        await asyncio.wait_for(post_install_entered.wait(), timeout=5)
        request_task.cancel()
        await asyncio.sleep(0.05)
        assert not request_task.done()
        assert post_install_completed is False
        release_post_install.set()
        with pytest.raises(asyncio.CancelledError):
            await request_task

    assert post_install_completed is True
    assert mgr.count == 1
    created = adapter.events_of("created")
    assert len(created) == 1
    assert created[0].ws_id == mgr.list_all()[0].id
    assert adapter.events_of("closed") == []