"""Tests for turnstone ChatSession and server endpoints. Mock-based tests verify streaming, tool calling, multi-turn conversation, and session configuration WITHOUT a running LLM backend. The mocks replace only the OpenAI streaming layer -- tool execution (bash, read_file) still runs real subprocesses. The TestBackendConnectivity class is marked @pytest.mark.live and requires a running llama-server (or compatible OpenAI API) on localhost:8000. The TestServerHealthMetrics class spins up an in-process HTTP server and needs no LLM backend at all. Run all non-live tests: pytest tests/test_server_live.py -v -m "not live" Run everything (needs backend): pytest tests/test_server_live.py -v --timeout=120 """ import json import os import queue import tempfile import threading from types import SimpleNamespace from unittest.mock import MagicMock import httpx import pytest from openai import OpenAI from tests._session_helpers import replace_session_lane from turnstone.core.session import ChatSession from turnstone.core.storage import init_storage, reset_storage # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- BASE_URL = os.environ.get("TURNSTONE_TEST_BASE_URL", "http://localhost:8000/v1") @pytest.fixture(scope="module") def live_client(): """Create an OpenAI client pointed at the local backend (live tests only).""" return OpenAI( base_url=BASE_URL, api_key=os.environ.get("TURNSTONE_TEST_API_KEY", "not-needed"), ) @pytest.fixture(scope="module") def live_model_id(live_client): """Auto-detect the model name from the backend (live tests only).""" models = live_client.models.list() ids = [m.id for m in models.data] assert len(ids) > 0, "No models found on the backend" return ids[0] class RecordingUI: """Minimal SessionUI that captures events for assertions.""" def __init__(self): self.events: list[tuple[str, ...]] = [] self.content_tokens: list[str] = [] self.reasoning_tokens: list[str] = [] self.tool_results: list[tuple[str, str, str]] = [] self.tool_chunks: list[tuple[str, str]] = [] self.errors: list[str] = [] self.infos: list[str] = [] def on_turn_start(self): self.events.append(("turn_start",)) def on_turn_committed(self): self.events.append(("turn_committed",)) def on_stream_discarded(self): # A live-wire blip entering the retry arm must not crash the fake. self.events.append(("stream_discarded",)) def on_thinking_start(self): self.events.append(("thinking_start",)) def on_thinking_stop(self): self.events.append(("thinking_stop",)) def on_reasoning_token(self, text): self.reasoning_tokens.append(text) def on_content_token(self, text): self.content_tokens.append(text) def on_stream_end(self): self.events.append(("stream_end",)) def approve_tools(self, items): return True, None # auto-approve everything def on_tool_result(self, call_id, name, output, **kwargs): self.tool_results.append((call_id, name, output)) def on_tool_output_chunk(self, call_id, chunk): self.tool_chunks.append((call_id, chunk)) def on_status(self, usage, context_window, effort): self.events.append(("status",)) def on_info(self, message): self.infos.append(message) def on_error(self, message): self.errors.append(message) def on_state_change(self, state): self.events.append(("state_change", state)) def on_rename(self, name: str): self.events.append(("rename", name)) def on_output_warning(self, call_id, assessment): pass def record_output_assessment( self, call_id, assessment, *, tier="heuristic", reasoning="", judge_model="", latency_ms=0, confidence=0.0, ): pass @property def full_content(self) -> str: return "".join(self.content_tokens) @property def full_reasoning(self) -> str: return "".join(self.reasoning_tokens) @pytest.fixture def tmp_db(): """Temp DB to avoid polluting real conversation history.""" with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: path = f.name reset_storage() init_storage("sqlite", path=path, run_migrations=False) yield path reset_storage() os.unlink(path) def _make_session(client, model_id, tmp_db, **kwargs) -> tuple[ChatSession, RecordingUI]: """Create a ChatSession with RecordingUI and sensible test defaults.""" from turnstone.core.memory import register_workstream from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider ui = RecordingUI() defaults = dict( client=client, model=model_id, ui=ui, instructions=None, temperature=0.3, max_tokens=2048, tool_timeout=30, reasoning_effort="low", ) defaults.update(kwargs) session = ChatSession(**defaults) # Production creates the parent workstream before admitting any keyed # conversation row. These direct-session tests mirror that ordering so # the storage orphan-write fence remains exercised rather than bypassed. register_workstream(session.ws_id, user_id=kwargs.get("user_id")) # Mock-based tests use Chat Completions format (client.chat.completions) replace_session_lane(session, provider=OpenAIChatCompletionsProvider()) session.auto_approve = True return session, ui # --------------------------------------------------------------------------- # Mock streaming helpers # --------------------------------------------------------------------------- def _make_chunk( *, content=None, reasoning_content=None, tool_calls=None, finish_reason=None, usage=None, ): """Build a single mock streaming chunk matching the OpenAI format. The chunk structure mirrors openai.types.chat.ChatCompletionChunk: chunk.choices[0].delta.content chunk.choices[0].delta.reasoning_content chunk.choices[0].delta.tool_calls chunk.choices[0].finish_reason chunk.usage """ delta = SimpleNamespace( content=content, reasoning_content=reasoning_content, reasoning=None, tool_calls=tool_calls, role=None, model_extra=None, ) choice = SimpleNamespace(delta=delta, finish_reason=finish_reason) chunk = SimpleNamespace(choices=[choice], usage=usage) return chunk def _make_tool_call_deltas(call_id, name, arguments): """Build a list of tool_call delta objects for a single tool call. Returns a list with one element (single tool call at index 0). """ fn = SimpleNamespace(name=name, arguments=arguments) return [SimpleNamespace(index=0, id=call_id, function=fn)] def _usage(prompt=100, completion=50, total=None): """Build a mock usage object.""" return SimpleNamespace( prompt_tokens=prompt, completion_tokens=completion, total_tokens=total or (prompt + completion), ) def make_mock_stream( content_tokens=None, reasoning_tokens=None, tool_calls=None, finish_reason="stop", usage=None, ): """Create an iterable of mock chunks simulating an OpenAI streaming response. Parameters ---------- content_tokens : list[str] | None Content token strings, each emitted as a separate chunk. reasoning_tokens : list[str] | None Reasoning token strings, emitted before content. tool_calls : list[tuple[str, str, str]] | None Each entry is (call_id, function_name, arguments_json). When provided, finish_reason defaults to "tool_calls". finish_reason : str Finish reason on the last content/tool chunk. usage : SimpleNamespace | None Usage object for the final chunk. Defaults to a sensible value. """ chunks = [] if reasoning_tokens: for token in reasoning_tokens: chunks.append(_make_chunk(reasoning_content=token)) if content_tokens: for i, token in enumerate(content_tokens): is_last = (i == len(content_tokens) - 1) and not tool_calls chunks.append( _make_chunk( content=token, finish_reason=finish_reason if is_last else None, ) ) if tool_calls: for i, (call_id, name, arguments) in enumerate(tool_calls): is_last = i == len(tool_calls) - 1 tc_deltas = _make_tool_call_deltas(call_id, name, arguments) chunks.append( _make_chunk( tool_calls=tc_deltas, finish_reason="tool_calls" if is_last else None, ) ) # Final usage-only chunk (no choices) if usage is None: usage = _usage() chunks.append(SimpleNamespace(choices=[], usage=usage)) return iter(chunks) def _mock_client(): """Create a mock OpenAI client with a patchable chat.completions.create.""" client = MagicMock(spec=OpenAI) client.chat = MagicMock() client.chat.completions = MagicMock() client.chat.completions.create = MagicMock() return client # --------------------------------------------------------------------------- # Tests -- Backend connectivity (live, requires running LLM) # --------------------------------------------------------------------------- @pytest.mark.live class TestBackendConnectivity: """Verify the LLM backend is reachable and returns valid responses.""" def test_models_endpoint(self, live_client): models = live_client.models.list() assert len(models.data) > 0 def test_model_id_detected(self, live_model_id): assert isinstance(live_model_id, str) assert len(live_model_id) > 0 def test_basic_completion(self, live_client, live_model_id): """Raw API call -- no turnstone involved.""" resp = live_client.chat.completions.create( model=live_model_id, messages=[{"role": "user", "content": "Say 'hello'"}], max_completion_tokens=200, temperature=0.0, stream=False, ) message = resp.choices[0].message # OpenAI-compatible servers use either non-standard field for parsed # reasoning (vLLM: ``reasoning``; llama.cpp: ``reasoning_content``). reasoning = getattr(message, "reasoning", None) or getattr( message, "reasoning_content", None ) assert message.content or reasoning assert resp.usage.total_tokens > 0 # --------------------------------------------------------------------------- # Tests -- Streaming session (mocked) # --------------------------------------------------------------------------- class TestStreamingSession: """Test ChatSession.send() with mocked streaming responses.""" def test_simple_response(self, tmp_db): """Mock returns content tokens; verify RecordingUI captures them.""" client = _mock_client() client.chat.completions.create.return_value = make_mock_stream( content_tokens=["Hello", " ", "world"], ) session, ui = _make_session(client, "mock-model", tmp_db) session._title_generated = True # prevent background title generation session.send("Say hello") assert "Hello world" in ui.full_content def test_reasoning_tokens_appear(self, tmp_db): """Mock returns reasoning tokens then content; verify both captured.""" client = _mock_client() client.chat.completions.create.return_value = make_mock_stream( reasoning_tokens=["Let me", " think..."], content_tokens=["The answer", " is 56"], ) session, ui = _make_session(client, "mock-model", tmp_db) session._title_generated = True session.send("What is 7 * 8?") assert len(ui.reasoning_tokens) > 0 assert "think" in ui.full_reasoning.lower() assert "56" in ui.full_content def test_stream_end_event(self, tmp_db): """stream_end event is emitted after response.""" client = _mock_client() client.chat.completions.create.return_value = make_mock_stream( content_tokens=["Hi"], ) session, ui = _make_session(client, "mock-model", tmp_db) session._title_generated = True session.send("Say hi") event_types = [e[0] for e in ui.events] assert "stream_end" in event_types def test_thinking_lifecycle(self, tmp_db): """thinking_start and thinking_stop bracket the response.""" client = _mock_client() client.chat.completions.create.return_value = make_mock_stream( content_tokens=["Hi", " there"], ) session, ui = _make_session(client, "mock-model", tmp_db) session._title_generated = True session.send("Say hi") event_types = [e[0] for e in ui.events] assert "thinking_start" in event_types assert "thinking_stop" in event_types start_idx = event_types.index("thinking_start") stop_idx = event_types.index("thinking_stop") assert start_idx < stop_idx # --------------------------------------------------------------------------- # Tests -- Tool calling (mocked LLM, real tool execution) # --------------------------------------------------------------------------- class TestToolCalling: """Test that mocked tool_calls trigger real tool execution.""" def test_bash_tool(self, tmp_db): """First call returns tool_call for bash, second returns content.""" client = _mock_client() stream1 = make_mock_stream( tool_calls=[("call_bash_1", "bash", json.dumps({"command": "echo hello"}))], ) stream2 = make_mock_stream( content_tokens=["The command printed: ", "hello"], ) client.chat.completions.create.side_effect = [stream1, stream2] session, ui = _make_session(client, "mock-model", tmp_db) session._title_generated = True session.send("Run echo hello") bash_results = [r for r in ui.tool_results if r[1] == "bash"] assert len(bash_results) > 0 assert "hello" in bash_results[0][2] def test_read_file_tool(self, tmp_db): """First call returns tool_call for read_file, second returns content.""" with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: f.write("SECRET_CONTENT_42\n") path = f.name try: client = _mock_client() stream1 = make_mock_stream( tool_calls=[("call_read_1", "read_file", json.dumps({"path": path}))], ) stream2 = make_mock_stream( content_tokens=["The file says: SECRET_CONTENT_42"], ) client.chat.completions.create.side_effect = [stream1, stream2] session, ui = _make_session(client, "mock-model", tmp_db) session._title_generated = True session.send(f"Read {path}") read_results = [r for r in ui.tool_results if r[1] == "read_file"] assert len(read_results) > 0 # Model relays the content assert "SECRET_CONTENT_42" in ui.full_content finally: os.unlink(path) # --------------------------------------------------------------------------- # Tests -- Multi-turn conversation (mocked) # --------------------------------------------------------------------------- class TestMultiTurn: """Test multi-turn conversation state with mocked responses.""" def test_context_retained(self, tmp_db): """Second send references context from the first.""" client = _mock_client() stream1 = make_mock_stream( content_tokens=["I'll remember ", "Zephyr"], ) stream2 = make_mock_stream( content_tokens=["Your name is ", "Zephyr"], ) client.chat.completions.create.side_effect = [stream1, stream2] session, ui = _make_session(client, "mock-model", tmp_db, max_tokens=1024) session._title_generated = True session.send("My name is Zephyr. Remember it.") # Reset UI tracking for second turn ui.content_tokens.clear() ui.reasoning_tokens.clear() session.send("What is my name?") assert "zephyr" in ui.full_content.lower() def test_message_list_grows(self, tmp_db): """Each send adds user + assistant messages.""" client = _mock_client() stream1 = make_mock_stream(content_tokens=["Hello"]) stream2 = make_mock_stream(content_tokens=["World"]) client.chat.completions.create.side_effect = [stream1, stream2] session, ui = _make_session(client, "mock-model", tmp_db, max_tokens=512) session._title_generated = True initial_count = len(session.messages) session.send("Hello") after_first = len(session.messages) assert after_first >= initial_count + 2 session.send("World") after_second = len(session.messages) assert after_second >= after_first + 2 # --------------------------------------------------------------------------- # Tests -- Session configuration (mocked) # --------------------------------------------------------------------------- class TestSessionConfig: """Test session construction and configuration with mocked responses.""" def test_empty_toolset_persona_no_tools_on_wire(self, tmp_db): """Guard: an empty-toolset persona (writer/scribe) sends ZERO tool definitions on the wire — create() is called without a tools kwarg. Replaces the removed /creative fork's equivalent assertion.""" from turnstone.core.personas import PersonaSnapshot client = _mock_client() client.chat.completions.create.return_value = make_mock_stream( content_tokens=["A haiku about code"], ) session, ui = _make_session( client, "mock-model", tmp_db, max_tokens=256, persona_snapshot=PersonaSnapshot( name="writer", prompt="You are a creative writing partner.", tools=frozenset(), mcp=False, memory=True, ), ) session._title_generated = True session.send("Write a haiku about code.") # Verify create() was called without 'tools' in kwargs call_kwargs = client.chat.completions.create.call_args assert "tools" not in call_kwargs.kwargs, ( "tools should not be passed under an empty-toolset persona" ) # Should get content back without tool calls assert len(ui.full_content) > 0 assert len(ui.tool_results) == 0 def test_custom_instructions(self, tmp_db): """Custom instructions appear in system messages.""" client = _mock_client() client.chat.completions.create.return_value = make_mock_stream( content_tokens=["Hello. ENDMARKER"], ) session, ui = _make_session( client, "mock-model", tmp_db, instructions="Always end your response with ENDMARKER.", max_tokens=512, ) session._title_generated = True # Verify custom instructions appear in system messages dev_msg = session.system_messages[0] assert "ENDMARKER" in dev_msg["content"] session.send("Say hello briefly.") assert len(ui.errors) == 0 # --------------------------------------------------------------------------- # Tests -- /health and /metrics endpoints (no live LLM required) # --------------------------------------------------------------------------- _TEST_JWT_SECRET = "test-jwt-secret-minimum-32-chars!" def _server_jwt() -> str: from turnstone.core.auth import JWT_AUD_SERVER, create_jwt return create_jwt( user_id="test-server-live", scopes=frozenset({"read", "write", "approve", "service"}), source="test", secret=_TEST_JWT_SECRET, audience=JWT_AUD_SERVER, ) _SERVER_AUTH_HEADERS = {"Authorization": f"Bearer {_server_jwt()}"} class TestServerHealthMetrics: """Verify /health and /metrics endpoints using a Starlette TestClient. These tests create a Starlette app with a mock SessionManager so no live LLM backend is required. Run them independently with: pytest tests/test_server_live.py::TestServerHealthMetrics -v """ @classmethod def setup_class(cls): from unittest.mock import MagicMock from starlette.testclient import TestClient import turnstone.server as srv_mod from turnstone.core.metrics import MetricsCollector from turnstone.core.workstream import WorkstreamState srv_mod._metrics = MetricsCollector() srv_mod._metrics.model = "test-model" mock_ui = MagicMock() mock_ui._ws_lock = threading.Lock() mock_ui._ws_prompt_tokens = 0 mock_ui._ws_completion_tokens = 0 mock_ui._ws_messages = 0 mock_ui._ws_tool_calls = {} mock_ui._ws_context_ratio = 0.0 mock_session = MagicMock() mock_session.ws_id = "test-session-id" mock_ws = MagicMock() mock_ws.id = "test-ws" mock_ws.name = "test" mock_ws.state = WorkstreamState.IDLE mock_ws.ui = mock_ui mock_ws.session = mock_session mock_mgr = MagicMock() mock_mgr.list_all.return_value = [mock_ws] mock_mgr.max_active = 10 app = srv_mod.create_app( workstreams=mock_mgr, global_queue=queue.Queue(), global_listeners=[], global_listeners_lock=threading.Lock(), skip_permissions=False, jwt_secret=_TEST_JWT_SECRET, ) cls.client = TestClient(app, raise_server_exceptions=False) @classmethod def teardown_class(cls): cls.client.close() def _get(self, path) -> tuple[int, str, str]: """Make a GET request; return (status, content_type, body_str).""" resp = self.client.get(path) ct = resp.headers.get("content-type", "") return resp.status_code, ct, resp.text def test_health_returns_200(self): status, _, _ = self._get("/health") assert status == 200 def test_health_content_type_json(self): _, ct, _ = self._get("/health") assert "application/json" in ct def test_health_response_structure(self): _, _, body = self._get("/health") data = json.loads(body) assert data["status"] == "ok" assert "version" in data assert "uptime_seconds" in data assert "model" in data assert "workstreams" in data def test_health_model_field(self): _, _, body = self._get("/health") data = json.loads(body) assert data["model"] == "test-model" def test_health_workstream_counts(self): _, _, body = self._get("/health") data = json.loads(body) wss = data["workstreams"] assert wss["total"] == 1 assert wss["idle"] == 1 def test_health_uptime_positive(self): _, _, body = self._get("/health") data = json.loads(body) assert data["uptime_seconds"] >= 0 def test_metrics_returns_200(self): status, _, _ = self._get("/metrics") assert status == 200 def test_metrics_content_type_prometheus(self): _, ct, _ = self._get("/metrics") assert "text/plain" in ct assert "version=0.0.4" in ct def test_metrics_contains_uptime(self): _, _, body = self._get("/metrics") assert "turnstone_uptime_seconds" in body def test_metrics_contains_build_info(self): _, _, body = self._get("/metrics") assert "turnstone_build_info" in body assert 'model="test-model"' in body def test_metrics_contains_workstreams(self): _, _, body = self._get("/metrics") assert "turnstone_workstreams_active_total" in body assert "turnstone_workstreams_by_state" in body def test_metrics_contains_token_counters(self): _, _, body = self._get("/metrics") assert "turnstone_tokens_total" in body assert 'type="prompt"' in body assert 'type="completion"' in body def test_metrics_contains_http_requests(self): _, _, body = self._get("/metrics") assert "turnstone_http_requests_total" in body def test_metrics_request_counter_increments(self): """Hitting /health increments the HTTP request counter.""" # Make a known request to /health self._get("/health") _, _, body = self._get("/metrics") # Counter should mention /health endpoint assert 'endpoint="/health"' in body def test_metrics_histogram_present(self): _, _, body = self._get("/metrics") assert "turnstone_http_request_duration_seconds" in body assert 'le="' in body assert 'le="+Inf"' in body def test_unknown_endpoint_returns_404(self): resp = self.client.get("/does-not-exist", headers=_SERVER_AUTH_HEADERS) assert resp.status_code == 404 def test_health_contains_backend_field(self): _, _, body = self._get("/health") data = json.loads(body) assert "backend" in data assert data["backend"]["status"] in ("up", "down") def test_metrics_contains_sse_connections(self): _, _, body = self._get("/metrics") assert "turnstone_sse_connections_active" in body def test_metrics_contains_ratelimit_counter(self): _, _, body = self._get("/metrics") assert "turnstone_ratelimit_rejected_total" in body def test_metrics_contains_backend_up(self): _, _, body = self._get("/metrics") assert "turnstone_backend_up" in body def test_metrics_no_circuit_state(self): """Circuit state metric was removed (passive health tracking only).""" _, _, body = self._get("/metrics") assert "turnstone_circuit_state" not in body def test_metrics_contains_eviction_counter(self): _, _, body = self._get("/metrics") assert "turnstone_workstreams_evicted_total" in body class TestServerRateLimiting: """Verify per-IP rate limiting returns 429 with Retry-After header. Creates a Starlette app with a tight rate limiter (rate=2, burst=3) and verifies that requests beyond the burst are rejected. """ @classmethod def setup_class(cls): from unittest.mock import MagicMock from starlette.testclient import TestClient import turnstone.server as srv_mod from turnstone.core.metrics import MetricsCollector from turnstone.core.ratelimit import RateLimiter from turnstone.core.workstream import WorkstreamState srv_mod._metrics = MetricsCollector() srv_mod._metrics.model = "test-model" mock_ui = MagicMock() mock_ui._ws_lock = threading.Lock() mock_ui._ws_prompt_tokens = 0 mock_ui._ws_completion_tokens = 0 mock_ui._ws_messages = 0 mock_ui._ws_tool_calls = {} mock_ui._ws_context_ratio = 0.0 mock_session = MagicMock() mock_session.ws_id = "test-session-id" mock_ws = MagicMock() mock_ws.id = "test-ws" mock_ws.name = "test" mock_ws.state = WorkstreamState.IDLE mock_ws.ui = mock_ui mock_ws.session = mock_session mock_mgr = MagicMock() mock_mgr.list_all.return_value = [mock_ws] mock_mgr.max_active = 10 app = srv_mod.create_app( workstreams=mock_mgr, global_queue=queue.Queue(), global_listeners=[], global_listeners_lock=threading.Lock(), skip_permissions=False, jwt_secret=_TEST_JWT_SECRET, rate_limiter=RateLimiter(enabled=True, rate=2.0, burst=3), ) cls.client = TestClient(app, raise_server_exceptions=False) @classmethod def teardown_class(cls): cls.client.close() def _get(self, path) -> httpx.Response: return self.client.get(path) def test_burst_requests_succeed(self): """First burst requests should all succeed.""" for _ in range(3): resp = self._get("/health") assert resp.status_code == 200 def test_exceeded_rate_returns_429(self): """After exhausting burst on a non-exempt endpoint, get 429.""" # Exhaust burst on a non-exempt endpoint for _ in range(5): self.client.get("/v1/api/workstreams", headers=_SERVER_AUTH_HEADERS) # At least one should be 429 statuses = [ self.client.get("/v1/api/workstreams", headers=_SERVER_AUTH_HEADERS).status_code for _ in range(3) ] assert 429 in statuses def test_429_includes_retry_after(self): """429 response includes Retry-After header.""" # Burn through burst for _ in range(10): resp = self.client.get("/v1/api/workstreams", headers=_SERVER_AUTH_HEADERS) if resp.status_code == 429: assert "retry-after" in resp.headers data = resp.json() assert "retry_after" in data return pytest.skip("Did not hit rate limit in 10 requests") def test_health_exempt_from_ratelimit(self): """Health endpoint is always accessible regardless of rate limit.""" # Burn through bucket on non-exempt path for _ in range(10): self.client.get("/v1/api/workstreams", headers=_SERVER_AUTH_HEADERS) # Health should still work resp = self._get("/health") assert resp.status_code == 200 def test_metrics_exempt_from_ratelimit(self): """Metrics endpoint is always accessible regardless of rate limit.""" for _ in range(10): self.client.get("/v1/api/workstreams", headers=_SERVER_AUTH_HEADERS) resp = self._get("/metrics") assert resp.status_code == 200