diff --git a/sdk/typescript/src/events.ts b/sdk/typescript/src/events.ts index 5e44a601..b8332435 100644 --- a/sdk/typescript/src/events.ts +++ b/sdk/typescript/src/events.ts @@ -13,6 +13,18 @@ export interface ConnectedEvent { export interface HistoryEvent { type: "history"; + /** + * Per-message dicts the frontend consumes directly. Common optional keys: + * - `role`: "user" | "assistant" | "tool" + * - `content`: string or list (image/document parts) + * - `tool_calls`: assistant turns — list of `{id, name, arguments, verdict?, output_assessment?}` + * - `tool_call_id`: tool turns — id of the originating call + * - `reminders`: metacognitive nudge bubbles (user/tool channels) + * - `advisories`: extracted `UserInterjection` payloads on tool turns + * - `reasoning`: concatenated reasoning text for assistant turns whose + * `provider_data` carried thinking blocks (Anthropic today). Present + * only when the active model's `persist_reasoning` flag is true. + */ messages: Array>; } diff --git a/tests/test_build_history_reminders.py b/tests/test_build_history_reminders.py index 50150cee..832633c0 100644 --- a/tests/test_build_history_reminders.py +++ b/tests/test_build_history_reminders.py @@ -141,3 +141,126 @@ class TestRemindersWidening: } history = _build([msg]) assert history[0]["reminders"] == [{"type": "denial", "text": "ok"}] + + +class _StubRegistry: + """Minimal model registry — only ``get_config`` is read by + ``_build_history``.""" + + def __init__(self, persist_reasoning: bool = True) -> None: + self._cfg = SimpleNamespace(persist_reasoning=persist_reasoning) + + def get_config(self, alias: str) -> Any: + return self._cfg + + +def _build_with_registry( + messages: list[dict[str, Any]], + persist_reasoning: bool = True, +) -> list[dict[str, Any]]: + session = SimpleNamespace( + messages=messages, + _ws_id="ws-test", + _registry=_StubRegistry(persist_reasoning=persist_reasoning), + _model_alias="claude-opus-4-7", + ) + with patch( + "turnstone.server._load_verdict_indexes", + return_value=({}, {}), + ): + return _build_history(session) + + +class TestReasoningSurfacing: + """Phase 1 — surface stored Anthropic thinking blocks on the + history payload so refresh-the-page rehydrates the reasoning bubble. + Drives through the real ``AnthropicProvider`` extractor (no mock-of- + extractor) — only the model registry is stubbed. + """ + + def test_reasoning_surfaces_for_anthropic_thinking_msg(self) -> None: + msg = { + "role": "assistant", + "content": "Final answer.", + "_provider_content": [ + {"type": "thinking", "thinking": "let me think", "signature": "s"}, + {"type": "text", "text": "Final answer."}, + ], + } + history = _build_with_registry([msg], persist_reasoning=True) + assert len(history) == 1 + assert history[0]["reasoning"] == "let me think" + + def test_reasoning_empty_when_persist_flag_false(self) -> None: + msg = { + "role": "assistant", + "content": "Final answer.", + "_provider_content": [ + {"type": "thinking", "thinking": "hidden", "signature": "s"}, + ], + } + history = _build_with_registry([msg], persist_reasoning=False) + assert "reasoning" not in history[0] + + def test_provider_content_never_in_wire_entry(self) -> None: + # The build path does not copy ``_provider_content`` into the + # entry dict regardless of flag — wire payload stays tight. + msg = { + "role": "assistant", + "content": "Final answer.", + "_provider_content": [ + {"type": "thinking", "thinking": "x", "signature": "s"}, + ], + } + history = _build_with_registry([msg], persist_reasoning=True) + assert "_provider_content" not in history[0] + + def test_no_reasoning_field_when_provider_content_missing(self) -> None: + msg = {"role": "assistant", "content": "plain answer"} + history = _build_with_registry([msg], persist_reasoning=True) + assert "reasoning" not in history[0] + + def test_no_reasoning_field_for_non_assistant_messages(self) -> None: + # Defensive — user/tool messages with a stray _provider_content + # do not get the reasoning field stamped. + msgs: list[dict[str, Any]] = [ + {"role": "user", "content": "hi"}, + { + "role": "tool", + "tool_call_id": "c1", + "content": "out", + "_provider_content": [{"type": "thinking", "thinking": "leak", "signature": "s"}], + }, + ] + history = _build_with_registry(msgs, persist_reasoning=True) + assert "reasoning" not in history[0] + assert "reasoning" not in history[1] + + def test_default_true_when_registry_lookup_raises(self) -> None: + # Conservative default — Phase 1 spec mandates rehydration on + # refresh. A registry/alias mismatch must not silently kill the + # bubble. + class BrokenRegistry: + def get_config(self, alias: str) -> Any: + raise KeyError(alias) + + session = SimpleNamespace( + messages=[ + { + "role": "assistant", + "content": "x", + "_provider_content": [ + {"type": "thinking", "thinking": "still works", "signature": "s"} + ], + } + ], + _ws_id="ws-test", + _registry=BrokenRegistry(), + _model_alias="missing-alias", + ) + with patch( + "turnstone.server._load_verdict_indexes", + return_value=({}, {}), + ): + history = _build_history(session) + assert history[0]["reasoning"] == "still works" diff --git a/tests/test_history_decoration.py b/tests/test_history_decoration.py index f737377a..31fe72c8 100644 --- a/tests/test_history_decoration.py +++ b/tests/test_history_decoration.py @@ -453,3 +453,146 @@ class TestDecorateAdvisoryExtraction: assert tool_msg["advisories"] == [ {"type": "user_interjection", "text": "check the logs", "priority": "notice"} ] + + +class TestExtractReasoningForHistory: + """``extract_reasoning_for_history`` — Phase 1 surfaces stored + Anthropic thinking blocks on assistant messages and strips + ``_provider_content`` from the wire payload. + + Drives through the real ``AnthropicProvider.extract_reasoning_text`` + (no mock-of-extractor) — the helper test and the provider unit + test (``tests/test_provider_anthropic_reasoning.py``) together + catch a regression at either layer distinctly. + """ + + def _anthropic_thinking_msg(self, text: str = "let me think") -> dict[str, object]: + return { + "role": "assistant", + "content": "Final answer.", + "_provider_content": [ + {"type": "thinking", "thinking": text, "signature": "sig"}, + {"type": "text", "text": "Final answer."}, + ], + } + + def test_extract_thinking_surfaces_reasoning_field(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [self._anthropic_thinking_msg("let me think")] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert messages[0]["reasoning"] == "let me think" + + def test_strips_provider_content_after_extraction(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [self._anthropic_thinking_msg("anything")] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "_provider_content" not in messages[0] + + def test_strips_provider_content_when_flag_false(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [self._anthropic_thinking_msg("anything")] + extract_reasoning_for_history(messages, persist_reasoning_flag=False) + # Strip is unconditional; reasoning is the conditional bit. + assert "_provider_content" not in messages[0] + assert "reasoning" not in messages[0] + + def test_first_block_thinking_dispatches_to_anthropic(self) -> None: + # Even when text and tool_use blocks follow, the first-block-type + # discriminator routes thinking-prefixed payloads correctly. + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [ + { + "role": "assistant", + "content": "x", + "_provider_content": [ + {"type": "thinking", "thinking": "first", "signature": "s"}, + {"type": "text", "text": "spoken"}, + {"type": "tool_use", "id": "t1", "name": "f", "input": {}}, + ], + } + ] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert messages[0]["reasoning"] == "first" + + def test_first_block_reasoning_dispatches_to_openai_phase3_stub(self) -> None: + # OpenAI Responses extractor returns "" until Phase 3 wires + # ``include=["reasoning.encrypted_content"]``. The dispatcher + # must still route to it (not silently fall through to ""). + # We assert the dispatcher routed by checking the strip happens + # AND no reasoning field is added (because the stub returns ""). + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [ + { + "role": "assistant", + "content": "x", + "_provider_content": [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "s"}]} + ], + } + ] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "reasoning" not in messages[0] + assert "_provider_content" not in messages[0] + + def test_unknown_first_block_type_no_op(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [ + { + "role": "assistant", + "content": "x", + "_provider_content": [{"type": "text", "text": "no reasoning here"}], + } + ] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "reasoning" not in messages[0] + assert "_provider_content" not in messages[0] + + def test_skips_messages_without_provider_content(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [{"role": "assistant", "content": "plain"}] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "reasoning" not in messages[0] + assert messages[0]["content"] == "plain" + + def test_user_and_tool_messages_untouched(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages: list[dict[str, object]] = [ + {"role": "user", "content": "hi"}, + {"role": "tool", "tool_call_id": "c1", "content": "out"}, + self._anthropic_thinking_msg("only this one"), + ] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "reasoning" not in messages[0] + assert "reasoning" not in messages[1] + assert messages[2]["reasoning"] == "only this one" + + def test_empty_provider_content_no_extraction(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages = [{"role": "assistant", "content": "x", "_provider_content": []}] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "reasoning" not in messages[0] + # Empty-list provider_content is still stripped from the wire. + assert "_provider_content" not in messages[0] + + def test_first_block_not_a_dict_skipped(self) -> None: + from turnstone.core.history_decoration import extract_reasoning_for_history + + messages: list[dict[str, object]] = [ + { + "role": "assistant", + "content": "x", + "_provider_content": ["bogus"], + } + ] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert "reasoning" not in messages[0] + assert "_provider_content" not in messages[0] diff --git a/tests/test_model_definition_storage.py b/tests/test_model_definition_storage.py index e7591f13..ea3b2dd5 100644 --- a/tests/test_model_definition_storage.py +++ b/tests/test_model_definition_storage.py @@ -212,3 +212,73 @@ class TestModelDefinitionStorage: m = db.get_model_definition(did) assert m is not None assert m["temperature"] is None + + def test_reasoning_flags_default(self, db: SQLiteBackend) -> None: + """persist_reasoning defaults True; replay_reasoning_to_model defaults False.""" + did = _make_id() + db.create_model_definition(definition_id=did, alias="reason-default", model="gpt-5") + m = db.get_model_definition(did) + assert m is not None + assert m["persist_reasoning"] is True + assert m["replay_reasoning_to_model"] is False + + def test_create_with_explicit_reasoning_flags(self, db: SQLiteBackend) -> None: + did = _make_id() + db.create_model_definition( + definition_id=did, + alias="reason-explicit", + model="claude-opus-4-7", + persist_reasoning=False, + replay_reasoning_to_model=True, + ) + m = db.get_model_definition(did) + assert m is not None + assert m["persist_reasoning"] is False + assert m["replay_reasoning_to_model"] is True + # Same values must round-trip via the alias lookup too. + m_alias = db.get_model_definition_by_alias("reason-explicit") + assert m_alias is not None + assert m_alias["persist_reasoning"] is False + assert m_alias["replay_reasoning_to_model"] is True + + def test_update_persist_reasoning(self, db: SQLiteBackend) -> None: + did = _make_id() + db.create_model_definition(definition_id=did, alias="upd-persist", model="gpt-5") + ok = db.update_model_definition(did, persist_reasoning=False) + assert ok is True + m = db.get_model_definition(did) + assert m is not None + assert m["persist_reasoning"] is False + assert m["replay_reasoning_to_model"] is False # untouched + + def test_update_replay_reasoning_to_model(self, db: SQLiteBackend) -> None: + did = _make_id() + db.create_model_definition(definition_id=did, alias="upd-replay", model="gpt-5") + ok = db.update_model_definition(did, replay_reasoning_to_model=True) + assert ok is True + m = db.get_model_definition(did) + assert m is not None + assert m["persist_reasoning"] is True # untouched + assert m["replay_reasoning_to_model"] is True + + def test_list_returns_reasoning_flags(self, db: SQLiteBackend) -> None: + db.create_model_definition( + definition_id=_make_id(), + alias="list-a", + model="gpt-5", + persist_reasoning=True, + replay_reasoning_to_model=False, + ) + db.create_model_definition( + definition_id=_make_id(), + alias="list-b", + model="claude-opus-4-7", + persist_reasoning=False, + replay_reasoning_to_model=True, + ) + models = db.list_model_definitions() + by_alias = {m["alias"]: m for m in models} + assert by_alias["list-a"]["persist_reasoning"] is True + assert by_alias["list-a"]["replay_reasoning_to_model"] is False + assert by_alias["list-b"]["persist_reasoning"] is False + assert by_alias["list-b"]["replay_reasoning_to_model"] is True diff --git a/tests/test_model_registry.py b/tests/test_model_registry.py index 683eab40..85d53e10 100644 --- a/tests/test_model_registry.py +++ b/tests/test_model_registry.py @@ -76,6 +76,23 @@ class TestModelConfig: assert cfg.temperature == 0.0 assert cfg.temperature is not None + def test_reasoning_flags_default(self) -> None: + cfg = ModelConfig(alias="x", base_url="x", api_key="x", model="x") + assert cfg.persist_reasoning is True + assert cfg.replay_reasoning_to_model is False + + def test_reasoning_flags_set(self) -> None: + cfg = ModelConfig( + alias="x", + base_url="x", + api_key="x", + model="x", + persist_reasoning=False, + replay_reasoning_to_model=True, + ) + assert cfg.persist_reasoning is False + assert cfg.replay_reasoning_to_model is True + # --------------------------------------------------------------------------- # ModelRegistry @@ -686,6 +703,53 @@ class TestLoadModelRegistryWithDB: assert cfg.max_tokens is None assert cfg.reasoning_effort is None + def test_db_reasoning_flags_loaded(self) -> None: + """Per-model reasoning flags from DB are carried in ModelConfig.""" + storage = _MockStorage( + [ + { + "alias": "anth-thinking", + "model": "claude-opus-4-7", + "provider": "anthropic", + "base_url": "", + "api_key": "sk-anth", + "context_window": 200000, + "capabilities": "{}", + "enabled": True, + "persist_reasoning": False, + "replay_reasoning_to_model": True, + } + ] + ) + with patch("turnstone.core.model_registry.load_config", return_value={}): + reg = load_model_registry("http://x/v1", "x", "x", storage=storage) + cfg = reg.get_config("anth-thinking") + assert cfg.persist_reasoning is False + assert cfg.replay_reasoning_to_model is True + + def test_db_reasoning_flags_default_when_absent(self) -> None: + """Pre-052 rows without the columns degrade to dataclass defaults.""" + storage = _MockStorage( + [ + { + "alias": "legacy-row", + "model": "gpt-5", + "provider": "openai", + "base_url": "", + "api_key": "", + "context_window": 32768, + "capabilities": "{}", + "enabled": True, + # persist_reasoning + replay_reasoning_to_model intentionally absent + } + ] + ) + with patch("turnstone.core.model_registry.load_config", return_value={}): + reg = load_model_registry("http://x/v1", "x", "x", storage=storage) + cfg = reg.get_config("legacy-row") + assert cfg.persist_reasoning is True + assert cfg.replay_reasoning_to_model is False + def test_db_default_alias_not_clobbered(self) -> None: """DB model with alias='default' is not overwritten by CLI args.""" storage = _MockStorage( diff --git a/tests/test_provider_anthropic_reasoning.py b/tests/test_provider_anthropic_reasoning.py new file mode 100644 index 00000000..af5972fc --- /dev/null +++ b/tests/test_provider_anthropic_reasoning.py @@ -0,0 +1,123 @@ +"""Tests for ``AnthropicProvider.extract_reasoning_text``. + +Phase 1 of the optional-reasoning-persistence feature: provider-side +extractor that walks stored ``provider_blocks`` and returns the +concatenated thinking text, capped at the operator-friendly UI display +size. + +These tests drive through the real ``AnthropicProvider`` instance — no +mocks of the extractor itself — using fixture-shaped blocks that match +what ``_iter_anthropic_stream`` actually accumulates at +``_anthropic.py:713-724`` (``thinking_delta`` + ``signature_delta`` +combined into ``{"type": "thinking", "thinking": , "signature": +}``). +""" + +from __future__ import annotations + +import pytest + +from turnstone.core.providers._anthropic import ( + _MAX_REASONING_DISPLAY_BYTES, + AnthropicProvider, +) +from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider +from turnstone.core.providers._openai_responses import OpenAIResponsesProvider + + +@pytest.fixture +def anthropic() -> AnthropicProvider: + return AnthropicProvider() + + +class TestExtractReasoningText: + def test_none_input_returns_empty_string(self, anthropic: AnthropicProvider) -> None: + assert anthropic.extract_reasoning_text(None) == "" + + def test_empty_list_returns_empty_string(self, anthropic: AnthropicProvider) -> None: + assert anthropic.extract_reasoning_text([]) == "" + + def test_no_thinking_blocks_returns_empty(self, anthropic: AnthropicProvider) -> None: + blocks: list[dict[str, object]] = [ + {"type": "text", "text": "hello"}, + {"type": "tool_use", "id": "t1", "name": "x", "input": {}}, + ] + assert anthropic.extract_reasoning_text(blocks) == "" + + def test_single_thinking_block_returns_text(self, anthropic: AnthropicProvider) -> None: + blocks = [{"type": "thinking", "thinking": "Let me think about this.", "signature": "abc"}] + assert anthropic.extract_reasoning_text(blocks) == "Let me think about this." + + def test_multiple_thinking_blocks_joined_with_newline( + self, anthropic: AnthropicProvider + ) -> None: + blocks = [ + {"type": "thinking", "thinking": "first thought", "signature": "s1"}, + {"type": "thinking", "thinking": "second thought", "signature": "s2"}, + ] + assert anthropic.extract_reasoning_text(blocks) == "first thought\nsecond thought" + + def test_mixed_blocks_extracts_only_thinking(self, anthropic: AnthropicProvider) -> None: + blocks = [ + {"type": "thinking", "thinking": "reason A", "signature": "s"}, + {"type": "text", "text": "visible answer"}, + {"type": "tool_use", "id": "t1", "name": "x", "input": {}}, + {"type": "thinking", "thinking": "reason B", "signature": "s"}, + ] + assert anthropic.extract_reasoning_text(blocks) == "reason A\nreason B" + + def test_thinking_block_without_thinking_field_skipped( + self, anthropic: AnthropicProvider + ) -> None: + blocks = [{"type": "thinking", "signature": "s"}] + assert anthropic.extract_reasoning_text(blocks) == "" + + def test_thinking_block_with_empty_text_skipped(self, anthropic: AnthropicProvider) -> None: + blocks = [{"type": "thinking", "thinking": "", "signature": "s"}] + assert anthropic.extract_reasoning_text(blocks) == "" + + def test_truncation_at_64kib_cap(self, anthropic: AnthropicProvider) -> None: + long_text = "x" * (_MAX_REASONING_DISPLAY_BYTES + 1024) + blocks = [{"type": "thinking", "thinking": long_text, "signature": "s"}] + result = anthropic.extract_reasoning_text(blocks) + assert len(result) == _MAX_REASONING_DISPLAY_BYTES + + def test_just_under_cap_not_truncated(self, anthropic: AnthropicProvider) -> None: + text = "y" * (_MAX_REASONING_DISPLAY_BYTES - 1) + blocks = [{"type": "thinking", "thinking": text, "signature": "s"}] + assert anthropic.extract_reasoning_text(blocks) == text + + def test_malformed_block_entry_skipped(self, anthropic: AnthropicProvider) -> None: + # A defensive sanity check — we should not crash if some + # entry isn't a dict (e.g. a corrupted JSON payload). + blocks = [ + "not a dict", # type: ignore[list-item] + {"type": "thinking", "thinking": "good one", "signature": "s"}, + ] + assert anthropic.extract_reasoning_text(blocks) == "good one" # type: ignore[arg-type] + + def test_non_list_input_returns_empty(self, anthropic: AnthropicProvider) -> None: + # Defensive against a corrupted provider_data payload. + assert anthropic.extract_reasoning_text("not a list") == "" # type: ignore[arg-type] + assert anthropic.extract_reasoning_text({"type": "thinking"}) == "" # type: ignore[arg-type] + + +class TestOtherProvidersDefault: + """Non-Anthropic providers return "" for the same fixture shapes.""" + + def test_openai_chat_returns_empty(self) -> None: + provider = OpenAIChatCompletionsProvider() + blocks = [{"type": "thinking", "thinking": "would-be-text", "signature": "s"}] + assert provider.extract_reasoning_text(blocks) == "" + + def test_openai_responses_returns_empty(self) -> None: + provider = OpenAIResponsesProvider() + blocks = [{"type": "thinking", "thinking": "would-be-text", "signature": "s"}] + assert provider.extract_reasoning_text(blocks) == "" + + def test_openai_responses_returns_empty_on_reasoning_shaped_blocks(self) -> None: + # Phase 3 stub — reasoning items exist but extractor returns "" + # until ``include=["reasoning.encrypted_content"]`` is wired. + provider = OpenAIResponsesProvider() + blocks = [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "x"}]}] + assert provider.extract_reasoning_text(blocks) == "" diff --git a/tests/test_reasoning_audit_log_discipline.py b/tests/test_reasoning_audit_log_discipline.py new file mode 100644 index 00000000..630682ee --- /dev/null +++ b/tests/test_reasoning_audit_log_discipline.py @@ -0,0 +1,194 @@ +"""Audit-log discipline test for reasoning text. + +Phase 1 of optional reasoning persistence surfaces stored thinking +blocks on the ``/history`` payload (UI rehydration). The bytes ride +through the helper (``extract_reasoning_for_history``), through the +provider extractor (``AnthropicProvider.extract_reasoning_text``), and +through the server build path (``_build_history``). + +This test pins the security-sensitive contract: + + Reasoning text MAY land on ``msg["reasoning"]`` (UI-bound), + but MUST NOT appear in any ``Logger.info`` / ``warning`` / + ``error`` payload at any layer in the pipeline. + +The test mocks the standard-library ``logging.Logger`` info/warning/ +error methods, runs a thinking-bearing turn through the relevant +extractors and history build, then asserts no captured log call's +positional args or kwargs contain the unique marker string. Replaces +the v4 grep-the-output approach (fragile when log strings are +formatted) with a structural mock-and-assert (tests the actual +contract rather than the rendered text). +""" + +from __future__ import annotations + +import logging +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +from turnstone.core.history_decoration import ( + extract_reasoning_for_history, + extract_reasoning_text_from_provider_content, +) +from turnstone.core.providers._anthropic import AnthropicProvider +from turnstone.server import _build_history + +_MARKER = "SECRET_REASONING_MARKER_xyz123_unlikely_collision" + + +def _payload_contains_marker(args: tuple[Any, ...], kwargs: dict[str, Any]) -> bool: + """Walk a captured log call's args + kwargs for the marker string. + + Logger.info-style calls accept a format string + positional substitution + args; the marker could appear in either the format string itself or + the substitution values. Format-time strings (``%`` substitution) are + NOT inspected because they're a stdlib formatting concern, not a + callable our pipeline reaches into. The structural check is "no + user-controlled marker appears in any arg slot we passed". + """ + for a in args: + if isinstance(a, str) and _MARKER in a: + return True + # Defensive — a list/dict/exception arg might carry the marker too. + try: + if _MARKER in repr(a): + return True + except Exception: + continue + for v in kwargs.values(): + if isinstance(v, str) and _MARKER in v: + return True + try: + if _MARKER in repr(v): + return True + except Exception: + continue + return False + + +def _capture_log_calls(): + """Capture every Logger.info / warning / error call into a single list.""" + captured: list[tuple[str, tuple[Any, ...], dict[str, Any]]] = [] + + def make_recorder(level: str): + def _rec(*args: Any, **kwargs: Any) -> None: + captured.append((level, args, kwargs)) + + return _rec + + return captured, [ + patch.object(logging.Logger, "info", side_effect=make_recorder("info"), autospec=True), + patch.object( + logging.Logger, "warning", side_effect=make_recorder("warning"), autospec=True + ), + patch.object(logging.Logger, "error", side_effect=make_recorder("error"), autospec=True), + ] + + +class TestReasoningAuditLogDiscipline: + """Reasoning text never lands at INFO+ severity on any logger.""" + + def _thinking_msg(self, text: str = _MARKER) -> dict[str, Any]: + return { + "role": "assistant", + "content": "Final answer.", + "_provider_content": [ + {"type": "thinking", "thinking": text, "signature": "sig"}, + {"type": "text", "text": "Final answer."}, + ], + } + + def test_anthropic_extractor_does_not_log_reasoning(self) -> None: + captured, patchers = _capture_log_calls() + for p in patchers: + p.start() + try: + provider = AnthropicProvider() + text = provider.extract_reasoning_text( + [{"type": "thinking", "thinking": _MARKER, "signature": "s"}] + ) + assert text == _MARKER # extractor IS allowed to return it + finally: + for p in patchers: + p.stop() + offending = [ + (lvl, args, kwargs) + for lvl, args, kwargs in captured + if _payload_contains_marker(args, kwargs) + ] + assert offending == [], ( + f"AnthropicProvider.extract_reasoning_text leaked reasoning text " + f"into INFO+ logs: {offending}" + ) + + def test_dispatch_helper_does_not_log_reasoning(self) -> None: + captured, patchers = _capture_log_calls() + for p in patchers: + p.start() + try: + text = extract_reasoning_text_from_provider_content( + [{"type": "thinking", "thinking": _MARKER, "signature": "s"}] + ) + assert text == _MARKER + finally: + for p in patchers: + p.stop() + offending = [ + (lvl, args, kwargs) + for lvl, args, kwargs in captured + if _payload_contains_marker(args, kwargs) + ] + assert offending == [], ( + f"extract_reasoning_text_from_provider_content leaked reasoning " + f"text into INFO+ logs: {offending}" + ) + + def test_list_helper_does_not_log_reasoning(self) -> None: + captured, patchers = _capture_log_calls() + for p in patchers: + p.start() + try: + messages = [self._thinking_msg(_MARKER)] + extract_reasoning_for_history(messages, persist_reasoning_flag=True) + assert messages[0]["reasoning"] == _MARKER # UI-bound is allowed + finally: + for p in patchers: + p.stop() + offending = [ + (lvl, args, kwargs) + for lvl, args, kwargs in captured + if _payload_contains_marker(args, kwargs) + ] + assert offending == [], ( + f"extract_reasoning_for_history leaked reasoning text into INFO+ logs: {offending}" + ) + + def test_build_history_does_not_log_reasoning(self) -> None: + registry = SimpleNamespace(get_config=lambda alias: SimpleNamespace(persist_reasoning=True)) + session = SimpleNamespace( + messages=[self._thinking_msg(_MARKER)], + _ws_id="ws-audit", + _registry=registry, + _model_alias="claude-opus-4-7", + ) + captured, patchers = _capture_log_calls() + for p in patchers: + p.start() + try: + with patch( + "turnstone.server._load_verdict_indexes", + return_value=({}, {}), + ): + history = _build_history(session) + assert history[0]["reasoning"] == _MARKER # UI-bound is allowed + finally: + for p in patchers: + p.stop() + offending = [ + (lvl, args, kwargs) + for lvl, args, kwargs in captured + if _payload_contains_marker(args, kwargs) + ] + assert offending == [], f"_build_history leaked reasoning text into INFO+ logs: {offending}" diff --git a/tests/test_workstream_endpoints.py b/tests/test_workstream_endpoints.py index d8161466..d033c374 100644 --- a/tests/test_workstream_endpoints.py +++ b/tests/test_workstream_endpoints.py @@ -2,7 +2,9 @@ from __future__ import annotations +import json import queue +from types import SimpleNamespace from typing import TYPE_CHECKING, Any from unittest.mock import MagicMock, patch @@ -1762,3 +1764,155 @@ class TestTenantCheckOnReadEndpoints: assert cold_check in offloaded, ( f"tenant_check must be invoked through asyncio.to_thread; got {offloaded}" ) + + +class TestHistoryReasoningRehydration: + """The lifted ``GET /v1/api/workstreams/{ws_id}/history`` surfaces + stored Anthropic thinking blocks on assistant messages so a page + refresh re-renders the reasoning bubble. Drives through the real + ``AnthropicProvider.extract_reasoning_text`` and the storage + ``reconstruct_messages`` boundary that JSON-decodes + ``provider_data`` into ``_provider_content``. + """ + + def test_history_handler_surfaces_reasoning_for_anthropic_thinking(self, _inject_storage): + ws_id = "ws-reason-1" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + provider_data = json.dumps( + [ + {"type": "thinking", "thinking": "let me reason", "signature": "s"}, + {"type": "text", "text": "Final answer."}, + ] + ) + _inject_storage.save_message( + ws_id, "assistant", "Final answer.", provider_data=provider_data + ) + # No live session — exercises the storage-only path which + # falls back to default persist_reasoning=True. + mock_mgr = MagicMock() + mock_mgr.get.return_value = None + client = _build_history_app(mock_mgr, _inject_storage) + + r = client.get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + msgs = r.json()["messages"] + assistant = next(m for m in msgs if m.get("role") == "assistant") + assert assistant["reasoning"] == "let me reason" + + def test_history_handler_strips_provider_content(self, _inject_storage): + ws_id = "ws-reason-2" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + provider_data = json.dumps([{"type": "thinking", "thinking": "x", "signature": "s"}]) + _inject_storage.save_message(ws_id, "assistant", "Answer.", provider_data=provider_data) + mock_mgr = MagicMock() + mock_mgr.get.return_value = None + client = _build_history_app(mock_mgr, _inject_storage) + + r = client.get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + for m in r.json()["messages"]: + assert "_provider_content" not in m + + def test_history_handler_with_persist_flag_false_via_live_session(self, _inject_storage): + """Operator-flipped ``persist_reasoning=False`` on the active + model suppresses the reasoning field even when the data is + stored. ``_provider_content`` is still stripped from the wire. + """ + ws_id = "ws-reason-3" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + provider_data = json.dumps([{"type": "thinking", "thinking": "hidden", "signature": "s"}]) + _inject_storage.save_message(ws_id, "assistant", "Answer.", provider_data=provider_data) + live_session = SimpleNamespace( + id=ws_id, + _registry=SimpleNamespace( + get_config=lambda alias: SimpleNamespace(persist_reasoning=False) + ), + _model_alias="claude-opus-4-7", + ) + mock_mgr = MagicMock() + mock_mgr.get.return_value = live_session + client = _build_history_app(mock_mgr, _inject_storage) + + r = client.get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + for m in r.json()["messages"]: + if m.get("role") == "assistant": + assert "reasoning" not in m + assert "_provider_content" not in m + + def test_history_handler_cold_workstream_resolves_via_workstream_config(self, _inject_storage): + """Cold workstream (no live session) — the handler walks + ``workstream_config.model_alias`` (persisted at first send by + the SessionManager rehydrate path) and looks up the active + model's ``persist_reasoning`` flag through the global registry + on ``app.state``. Operator flag-flip is honored uniformly + across live and cold workstreams. + """ + ws_id = "ws-reason-cold" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + # Simulate the model alias persisted by the rehydrate path + # (session_manager.py:628-629 reads it back via the same key). + _inject_storage.save_workstream_config(ws_id, {"model_alias": "claude-opus-4-7"}) + provider_data = json.dumps( + [{"type": "thinking", "thinking": "should not surface", "signature": "s"}] + ) + _inject_storage.save_message(ws_id, "assistant", "Answer.", provider_data=provider_data) + # No live session — handler falls back to workstream_config + registry. + mock_mgr = MagicMock() + mock_mgr.get.return_value = None + + # Build the app with a global registry that reports persist=False + # for the saved alias. + cfg = _interactive_endpoint_cfg(mock_mgr) + handler = make_history_handler(cfg) + app = Starlette( + routes=[ + Mount( + "/v1", + routes=[ + Route( + "/api/workstreams/{ws_id}/history", + handler, + methods=["GET"], + ), + ], + ), + ], + middleware=[Middleware(_InjectAuthMiddleware)], + ) + app.state.workstreams = mock_mgr + app.state.auth_storage = _inject_storage + app.state.registry = SimpleNamespace( + get_config=lambda alias: SimpleNamespace( + persist_reasoning=(alias != "claude-opus-4-7"), + ) + ) + client = TestClient(app) + + r = client.get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + # Flag-flip on the saved alias is honored: reasoning suppressed. + for m in r.json()["messages"]: + if m.get("role") == "assistant": + assert "reasoning" not in m + assert "_provider_content" not in m + + def test_history_handler_cold_workstream_no_alias_defaults_true(self, _inject_storage): + """A workstream that pre-dates the rehydrate-time alias persist + (or one that simply has no workstream_config row) falls through + to the conservative default ``True``. Reasoning surfaces. + """ + ws_id = "ws-reason-cold-no-alias" + _inject_storage.register_workstream(ws_id, kind="interactive", user_id="test-user") + provider_data = json.dumps( + [{"type": "thinking", "thinking": "default-true wins", "signature": "s"}] + ) + _inject_storage.save_message(ws_id, "assistant", "Answer.", provider_data=provider_data) + mock_mgr = MagicMock() + mock_mgr.get.return_value = None + client = _build_history_app(mock_mgr, _inject_storage) + + r = client.get(f"/v1/api/workstreams/{ws_id}/history") + assert r.status_code == 200 + assistant = next(m for m in r.json()["messages"] if m.get("role") == "assistant") + assert assistant["reasoning"] == "default-true wins" diff --git a/turnstone/api/console_schemas.py b/turnstone/api/console_schemas.py index 3c475d9e..96266d33 100644 --- a/turnstone/api/console_schemas.py +++ b/turnstone/api/console_schemas.py @@ -908,6 +908,8 @@ class ModelDefinitionInfo(BaseModel): temperature: float | None = None max_tokens: int | None = None reasoning_effort: str | None = None + persist_reasoning: bool = True + replay_reasoning_to_model: bool = False source: str = "" created_by: str = "" created: str = "" @@ -926,6 +928,8 @@ class CreateModelDefinitionRequest(BaseModel): temperature: float | None = None max_tokens: int | None = None reasoning_effort: str | None = None + persist_reasoning: bool = True + replay_reasoning_to_model: bool = False class UpdateModelDefinitionRequest(BaseModel): @@ -940,6 +944,8 @@ class UpdateModelDefinitionRequest(BaseModel): temperature: float | None = None max_tokens: int | None = None reasoning_effort: str | None = None + persist_reasoning: bool | None = None + replay_reasoning_to_model: bool | None = None class ListModelDefinitionsResponse(BaseModel): diff --git a/turnstone/console/server.py b/turnstone/console/server.py index 46933341..f2c61d6d 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -9904,6 +9904,9 @@ async def admin_create_model_definition(request: Request) -> JSONResponse: if not reasoning_effort: reasoning_effort = None + persist_reasoning = bool(body.get("persist_reasoning", True)) + replay_reasoning_to_model = bool(body.get("replay_reasoning_to_model", False)) + storage.create_model_definition( definition_id=definition_id, alias=alias, @@ -9918,6 +9921,8 @@ async def admin_create_model_definition(request: Request) -> JSONResponse: temperature=temperature, max_tokens=max_tokens, reasoning_effort=reasoning_effort, + persist_reasoning=persist_reasoning, + replay_reasoning_to_model=replay_reasoning_to_model, ) record_audit( @@ -10079,6 +10084,10 @@ async def admin_update_model_definition(request: Request) -> JSONResponse: ) else: updates["reasoning_effort"] = re_val + if "persist_reasoning" in body: + updates["persist_reasoning"] = bool(body["persist_reasoning"]) + if "replay_reasoning_to_model" in body: + updates["replay_reasoning_to_model"] = bool(body["replay_reasoning_to_model"]) if updates: storage.update_model_definition(definition_id, **updates) diff --git a/turnstone/console/static/admin.js b/turnstone/console/static/admin.js index 4b311874..b88e69f1 100644 --- a/turnstone/console/static/admin.js +++ b/turnstone/console/static/admin.js @@ -5008,6 +5008,11 @@ function _renderModels(items) { if (m.max_tokens != null) overrides.push("max_tok=" + m.max_tokens); if (m.reasoning_effort != null) overrides.push("effort=" + m.reasoning_effort); + // Reasoning persistence flags surface only when non-default + // (persist=False is the operator opt-out; replay=True is the + // operator opt-in). Default values are silent. + if (m.persist_reasoning === false) overrides.push("persist=off"); + if (m.replay_reasoning_to_model === true) overrides.push("replay=on"); if (overrides.length) { var ovrSpan = document.createElement("span"); ovrSpan.className = "model-overrides-hint"; @@ -5196,6 +5201,8 @@ function showCreateModelModal() { el.style.borderColor = ""; }); document.getElementById("model-enabled").checked = true; + document.getElementById("model-persist-reasoning").checked = true; + document.getElementById("model-replay-reasoning").checked = false; document.getElementById("model-detect-result").style.display = "none"; document.getElementById("model-detect-btn").disabled = false; document.getElementById("model-detect-btn").textContent = "Detect"; @@ -5278,6 +5285,13 @@ function showEditModelModal(definitionId) { document.getElementById("model-capabilities").value = capsText === "{}" ? "" : capsText; document.getElementById("model-enabled").checked = m.enabled !== false; + // Reasoning persistence flags — defaults match the dataclass + // defaults (persist=true, replay=false) when the API returns + // them as undefined (legacy / pre-052 row). + document.getElementById("model-persist-reasoning").checked = + m.persist_reasoning !== false; + document.getElementById("model-replay-reasoning").checked = + m.replay_reasoning_to_model === true; _applyProviderDefaults(); }) .catch(function () { @@ -5419,6 +5433,16 @@ function submitCreateModel() { form.reasoning_effort = null; } + // Reasoning persistence flags — always serialize so a flip from + // default takes effect on PUT (the server's update path keys off + // "field present in body"). + form.persist_reasoning = document.getElementById( + "model-persist-reasoning", + ).checked; + form.replay_reasoning_to_model = document.getElementById( + "model-replay-reasoning", + ).checked; + var apiKey = document.getElementById("model-api-key").value; if (apiKey) form.api_key = apiKey; diff --git a/turnstone/console/static/coordinator/coordinator.js b/turnstone/console/static/coordinator/coordinator.js index 3fd739e9..ca2d7941 100644 --- a/turnstone/console/static/coordinator/coordinator.js +++ b/turnstone/console/static/coordinator/coordinator.js @@ -4195,6 +4195,20 @@ appendUserMessageWithAttachments(text, [], { label: "user" }); }); } else if (role === "assistant") { + // Reasoning bubble (Phase 1 reasoning persistence) — render + // BEFORE the content card so the visual order matches the + // live SSE flow (reasoning_delta arrives before content_delta + // for thinking-enabled models). Mirrors the live ":1524" / + // snapshot ":2021" call sites — same appendMsg("reasoning") + // helper, just driven from history-render rather than the + // SSE handler. Only present when the active model's + // persist_reasoning flag is true and the message round-tripped + // a thinking lane. + if (typeof m.reasoning === "string" && m.reasoning.length) { + const rEl = appendMsg("reasoning", "", { label: "reasoning" }); + const rBody = rEl && rEl.querySelector(".msg-body"); + if (rBody) rBody.textContent = m.reasoning; + } // Render content BEFORE the tool batch so DOM order matches // chronological order (the model emits text first, then // dispatches tools). Whitespace-only content (e.g. "\n\n" diff --git a/turnstone/console/static/index.html b/turnstone/console/static/index.html index 56703a81..ea5791c2 100644 --- a/turnstone/console/static/index.html +++ b/turnstone/console/static/index.html @@ -4116,7 +4116,7 @@ placeholder='{"supports_vision": true}' style="font-family: var(--font-mono); font-size: 11px" > -
+
+ +