From 54dd4ed50aee9aadf49ca9ea44e418a2282c7693 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Sun, 12 Jul 2026 23:53:48 -0700 Subject: [PATCH] feat(judge): both judges speak Turn IR through model_turn (#827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The intent judge's evidence loop and the output-guard's single shot now build list[Turn] and call model_turn — the hand-built OpenAI-dict message construction is gone, and with it the judges' private interlingua. The assistant turns they append carry the provider-native lane, so the loop keeps reasoning continuity across its own turns. That is what unblocks Gemini: thought_signature rides provider_blocks and is reconstructed by the Google adapter's fidelity swap, so the provider_name == "google" tool-skip is deleted — the Gemini judge runs the same evidence-tool loop as every other provider instead of degrading to a single-shot, tool-blind verdict. judge.py's _resolve_model_capabilities mirror (#826) is deleted; both judges resolve capabilities through the shared lane resolver, and each evaluation builds a ModelLane (fresh client, constructor caps, registry-resolved extra_params + live flags). The shared resolver inherits the mirror's defensive non-dict capabilities check — without it a malformed registry row would silently downgrade a judge to the session model instead of just skipping the overrides. Judge calls now resolve extra_params and replay_reasoning_to_model from the registry like every other lane (previously: never sent, and the protocol's back-compat default respectively). Test mocks grow the CompletionResult fields the model_turn re-ingest reads (provider_blocks, reasoning); alias-registry mocks wire get_config, which the unified resolver uses. --- tests/test_judge.py | 99 ++++++++------ tests/test_output_guard_judge.py | 24 +++- turnstone/core/judge.py | 194 +++++++++++++-------------- turnstone/core/model_turn.py | 9 +- turnstone/core/output_guard_judge.py | 48 ++++--- 5 files changed, 202 insertions(+), 172 deletions(-) diff --git a/tests/test_judge.py b/tests/test_judge.py index b70e6727..ebcdb78d 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -10,12 +10,34 @@ from typing import Any from unittest.mock import MagicMock from turnstone.core.judge import IntentJudge, IntentVerdict, JudgeConfig, evaluate_heuristic +from turnstone.core.trajectory import Role # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- +def _mock_result( + content: str = "", + tool_calls: list[dict[str, Any]] | None = None, +) -> MagicMock: + """A provider result shaped like ``CompletionResult``. + + The judge's calls run through ``model_turn``, whose re-ingest iterates + ``tool_calls``/``provider_blocks`` and joins ``reasoning`` — a bare + MagicMock attribute would TypeError, so every field the seam reads is + pinned to a real value. + """ + result = MagicMock() + result.content = content + result.tool_calls = tool_calls + result.finish_reason = "stop" + result.usage = None + result.provider_blocks = [] + result.reasoning = "" + return result + + def _make_mock_provider( response_content: str = "", tool_calls: list[dict[str, Any]] | None = None, @@ -30,16 +52,10 @@ def _make_mock_provider( caps.max_output_tokens = 4096 provider.get_capabilities.return_value = caps - result = MagicMock() - result.content = response_content - result.tool_calls = tool_calls - result.finish_reason = "stop" - result.usage = None - if side_effect: provider.create_completion.side_effect = side_effect else: - provider.create_completion.return_value = result + provider.create_completion.return_value = _mock_result(response_content, tool_calls) provider.convert_tools.side_effect = lambda tools, **kw: tools @@ -375,22 +391,21 @@ class TestMultiTurnToolUse: provider.convert_tools.side_effect = lambda tools, **kw: tools # Turn 1: tool call - turn1 = MagicMock() - turn1.content = "" - turn1.tool_calls = [ - { - "id": "tc_judge_1", - "function": { - "name": "read_file", - "arguments": json.dumps({"path": "/nonexistent/file.txt"}), - }, - } - ] + turn1 = _mock_result( + "", + [ + { + "id": "tc_judge_1", + "function": { + "name": "read_file", + "arguments": json.dumps({"path": "/nonexistent/file.txt"}), + }, + } + ], + ) # Turn 2: verdict - turn2 = MagicMock() - turn2.content = _good_verdict_json() - turn2.tool_calls = None + turn2 = _mock_result(_good_verdict_json()) provider.create_completion.side_effect = [turn1, turn2] @@ -416,22 +431,21 @@ class TestMultiTurnToolUse: provider.convert_tools.side_effect = lambda tools, **kw: tools # Every turn returns a tool call - tool_result = MagicMock() - tool_result.content = "" - tool_result.tool_calls = [ - { - "id": "tc_loop", - "function": { - "name": "read_file", - "arguments": json.dumps({"path": "/tmp/x"}), - }, - } - ] + tool_result = _mock_result( + "", + [ + { + "id": "tc_loop", + "function": { + "name": "read_file", + "arguments": json.dumps({"path": "/tmp/x"}), + }, + } + ], + ) # Last turn (no tools param) returns text content - final = MagicMock() - final.content = _good_verdict_json() - final.tool_calls = None + final = _mock_result(_good_verdict_json()) # Turns 0-3: tool_call; turn 4 (last, tools=None): final verdict provider.create_completion.side_effect = [ @@ -468,12 +482,12 @@ class TestContextPreparation: result = judge._prepare_context(_make_item(), messages) - # Should have system message + single user message with transcript + # Should have a system Turn + single user Turn with the transcript assert len(result) == 2 - assert result[0]["role"] == "system" - assert result[1]["role"] == "user" - assert "pending human approval" in result[1]["content"] - assert "Conversation context:" in result[1]["content"] + assert result[0].role is Role.SYSTEM + assert result[1].role is Role.USER + assert "pending human approval" in result[1].text + assert "Conversation context:" in result[1].text class TestArgBudget: @@ -541,7 +555,7 @@ class TestArgBudget: ) # Each included history turn renders one "ASSISTANT:" line; the # big-argument call fits strictly fewer of them. - assert big[1]["content"].count("ASSISTANT:") < small[1]["content"].count("ASSISTANT:") + assert big[1].text.count("ASSISTANT:") < small[1].text.count("ASSISTANT:") # --------------------------------------------------------------------------- @@ -901,6 +915,9 @@ class TestModelAliasResolution: cfg.capabilities = capabilities if capabilities is not None else {} registry.has_alias.side_effect = lambda a: a == alias registry.resolve.return_value = (alias_client, underlying_model, cfg) + # The unified lane resolver (model_turn.resolve_capabilities) fetches + # the config itself rather than taking resolve()'s copy. + registry.get_config.return_value = cfg registry.get_provider.return_value = alias_provider return registry diff --git a/tests/test_output_guard_judge.py b/tests/test_output_guard_judge.py index 6452892a..643d4d0e 100644 --- a/tests/test_output_guard_judge.py +++ b/tests/test_output_guard_judge.py @@ -34,14 +34,25 @@ def _make_provider( time.sleep(delay) if raises is not None: raise raises - result = MagicMock() - result.content = content - return result + return _mock_result(content) provider.create_completion = _create_completion return provider +def _mock_result(content: str) -> MagicMock: + """A provider result shaped like ``CompletionResult`` — the guard's call + runs through ``model_turn``, whose re-ingest reads every field below.""" + result = MagicMock() + result.content = content + result.tool_calls = None + result.finish_reason = "stop" + result.usage = None + result.provider_blocks = [] + result.reasoning = "" + return result + + def _make_judge( *, content: str = "", @@ -79,9 +90,7 @@ class TestCapabilityThreading: def _cc(**kwargs: Any) -> Any: captured.update(kwargs) - result = MagicMock() - result.content = '{"risk_level": "none", "flags": []}' - return result + return _mock_result('{"risk_level": "none", "flags": []}') provider = MagicMock() provider.provider_name = "openai" @@ -121,6 +130,9 @@ class TestCapabilityThreading: "local-9b", cfg, ) + # The unified lane resolver (model_turn.resolve_capabilities) fetches + # the config itself rather than taking resolve()'s copy. + registry.get_config.return_value = cfg registry.get_provider.return_value = provider client = MagicMock(base_url="http://s", api_key="k") judge = OutputGuardJudge( diff --git a/turnstone/core/judge.py b/turnstone/core/judge.py index e472be24..23dfd822 100644 --- a/turnstone/core/judge.py +++ b/turnstone/core/judge.py @@ -15,7 +15,7 @@ import re import threading import time import uuid -from dataclasses import dataclass, field, fields, replace +from dataclasses import dataclass, field from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any @@ -26,6 +26,8 @@ from turnstone.core.deadline import ( run_with_deadline, ) from turnstone.core.log import get_logger +from turnstone.core.model_turn import model_turn, resolve_capabilities, resolve_lane +from turnstone.core.trajectory import Turn if TYPE_CHECKING: from collections.abc import Callable @@ -844,33 +846,6 @@ def _positive_window(*candidates: Any, floor: int = _DEFAULT_JUDGE_CONTEXT_WINDO return floor -def _resolve_model_capabilities(provider: LLMProvider, model: str, cfg: Any) -> ModelCapabilities: - """Provider base capabilities with a model definition's ``capabilities`` - overrides applied — the same lowering ``ChatSession._resolve_capabilities`` - performs for the session, utility, and sub-agent completion lanes. - - The judges are the only completion callers that live outside ``ChatSession``, - so they cannot reach ``self._resolve_capabilities``; this mirrors it so a - judge alias honors operator-declared capabilities (effort passthrough, tool - support, temperature, verbosity) exactly like the main loop. ``cfg`` is the - alias's ``ModelConfig``; a missing or non-dict ``capabilities`` is ignored - rather than raised — capability resolution must never crash a judge turn. - - It does NOT fold in ``ModelConfig.context_window``: that is a separate field, - not part of the capabilities JSON, and the caller sizes the judge's window - budget off it directly (the static caps table reports 200000 for local - models, which would silently over-budget them). - """ - caps = provider.get_capabilities(model) - overrides = getattr(cfg, "capabilities", None) - if isinstance(overrides, dict) and overrides: - names = {f.name for f in fields(type(caps))} - applied = {k: v for k, v in overrides.items() if k in names} - if applied: - caps = replace(caps, **applied) - return caps - - def honest_truncate(text: str, budget: int) -> str: """Return *text* untouched when it fits *budget* characters, otherwise the leading ``budget`` characters followed by an explicit note of exactly how @@ -1004,6 +979,9 @@ class IntentJudge: ) -> None: self._config = config self._rule_registry = rule_registry + # Carried into the per-evaluation ModelLane so extra_params and the + # live operator flags resolve from the registry like every other lane. + self._model_registry = model_registry # The caller (ChatSession) resolves the session model's real caps from # _get_capabilities (config/registry-aware) and passes them in; they are # this judge's wire capabilities and window when it inherits the session @@ -1036,8 +1014,15 @@ class IntentJudge: self._provider.provider_name, ) self._model = model_name - self._capabilities = _resolve_model_capabilities( - self._provider, self._model, model_cfg + self._alias = config.model + # The shared lane resolver (model_turn) merges the alias's + # capability overrides; it deliberately does NOT fold in + # ModelConfig.context_window — that is a separate field, + # sized into the judge's window budget right below (the + # static caps table reports 200000 for local models, which + # would silently over-budget them). + self._capabilities = resolve_capabilities( + self._provider, self._model, config.model, model_registry ) # Use the registry's per-model context window, NOT # ``provider.get_capabilities().context_window``: the static @@ -1073,6 +1058,7 @@ class IntentJudge: session_provider.provider_name, ) self._model = session_model + self._alias = "" # Wire caps: the caller's resolved session caps, or the provider's # static table as a last resort for degraded / legacy callers. self._capabilities = ( @@ -1320,19 +1306,36 @@ class IntentJudge: except (TypeError, ValueError): func_args_json = honest_truncate(str(func_args), _VERDICT_ARG_CAP) - # Prepare context - judge_messages = self._prepare_context(item, messages) + # Prepare context (Turn IR — lowered per call inside model_turn) + judge_turns = self._prepare_context(item, messages) # Prepare tools (only if read_only_tools enabled). - # Pass raw OpenAI-format schemas — create_completion handles conversion. - # Google's API requires thought_signature in function call round-trips - # which our normalized tool_calls don't preserve, so skip tools for Google. + # Raw OpenAI-format schemas — the provider adapter converts them. + # These are γ-side instruments, deliberately OUTSIDE the persona + # envelope: middle-rank config must not be able to blind the gate's + # evidence gathering. The old provider_name == "google" skip is gone: + # the judge's trajectory now carries the provider-native lane + # (thought_signature rides ``provider_blocks`` and is reconstructed by + # the Google adapter), so the Gemini judge runs the same evidence loop + # as every other provider. tools: list[dict[str, Any]] | None = None - if self._config.read_only_tools and self._provider.provider_name != "google": + if self._config.read_only_tools: tools = list(_JUDGE_TOOL_SCHEMAS) + # The judge's resolved lane for this evaluation: fresh client per run + # (thread isolation), constructor-resolved capabilities, extra_params + # and live operator flags from the registry like every other lane. + lane = resolve_lane( + self._provider, + client, + self._model, + alias=self._alias, + registry=self._model_registry, + capabilities=self._capabilities, + ) + # Multi-turn judge loop - result = None # will hold the last CompletionResult + result = None # will hold the last ModelTurnResult empty_retries = 0 # track consecutive empty responses for retry turn = 0 @@ -1352,15 +1355,12 @@ class IntentJudge: # On the last turn, strip tools and inject a forcing message # so the model knows it must render a verdict now. if is_last_turn: - judge_messages.append( - { - "role": "user", - "content": ( - "You have gathered enough evidence. " - "You MUST now render your final verdict as JSON. " - "No more tool calls." - ), - } + judge_turns.append( + Turn.user( + "You have gathered enough evidence. " + "You MUST now render your final verdict as JSON. " + "No more tool calls." + ) ) # Per-turn timeout: each turn gets a fresh budget so local @@ -1374,21 +1374,13 @@ class IntentJudge: # poisoned the pool, which is why the restart dance existed. result = run_with_deadline( partial( - self._provider.create_completion, - client=client, - model=self._model, - messages=judge_messages, + model_turn, + lane, + judge_turns, tools=None if is_last_turn else tools, max_tokens=2048, temperature=0.0, reasoning_effort="medium", - # Thread the judge model's operator-declared capabilities - # onto the wire like every other lane — resolved from the - # judge alias's model definition, or the session model on - # fallback. Without this the provider would fall back to - # its static capability table and silently ignore the - # definition's overrides on judge calls alone. - capabilities=self._capabilities, ), timeout=per_call_timeout, cancel_event=cancel_event, @@ -1431,14 +1423,14 @@ class IntentJudge: # Check for tool calls if result.tool_calls: - # Execute read-only tools and append results - judge_messages.append( - { - "role": "assistant", - "content": result.content or None, - "tool_calls": result.tool_calls, - } - ) + # Append the assistant turn — the native lane rides along + # (Gemini thought_signature, Anthropic thinking, Responses + # reasoning items), so the next lowering replays it and the + # evidence loop keeps its reasoning continuity. The judge + # never mints ids: its trajectory is ephemeral and pinned to + # one provider, so provider-original ids stay consistent + # between the native blocks, the mirror, and the results. + judge_turns.append(result.turn) for tc in result.tool_calls: tc_func = tc.get("function", {}) tc_name = tc_func.get("name", "") @@ -1451,13 +1443,7 @@ class IntentJudge: tc_args = {} tool_result = self._exec_read_only_tool(tc_name, tc_args) - judge_messages.append( - { - "role": "tool", - "tool_call_id": tc.get("id", ""), - "content": tool_result, - } - ) + judge_turns.append(Turn.tool(tc.get("id", ""), tool_result)) turn += 1 continue @@ -1486,15 +1472,12 @@ class IntentJudge: ) return None # On earlier turns, inject a nudge and continue - judge_messages.append({"role": "assistant", "content": result.content}) - judge_messages.append( - { - "role": "user", - "content": ( - "Your response was not valid JSON. " - "Please respond ONLY with the JSON verdict object." - ), - } + judge_turns.append(result.turn) + judge_turns.append( + Turn.user( + "Your response was not valid JSON. " + "Please respond ONLY with the JSON verdict object." + ) ) turn += 1 continue @@ -1511,15 +1494,12 @@ class IntentJudge: empty_retries += 1 if empty_retries <= 3: log.info("judge.empty_response.retry", retry=empty_retries, max_retries=3) - judge_messages.append( - { - "role": "user", - "content": ( - "You returned an empty response. " - "Please analyze the tool call and respond with " - "the JSON verdict object." - ), - } + judge_turns.append( + Turn.user( + "You returned an empty response. " + "Please analyze the tool call and respond with " + "the JSON verdict object." + ) ) continue log.info("judge.empty_response.giving_up", retries=empty_retries) @@ -1551,8 +1531,17 @@ class IntentJudge: self, item: dict[str, Any], messages: list[dict[str, Any]], - ) -> list[dict[str, Any]]: - """Build the judge's message list with FIFO-truncated conversation.""" + ) -> list[Turn]: + """Build the judge's opening trajectory with FIFO-truncated conversation. + + *messages* is the session's wire-dict history (read-only input: the + judge observes the session, it does not join it); the output is Turn + IR — the judge's own ephemeral trajectory, lowered per call by + ``model_turn``. The flattened single-user-message transcript is the + judge's deliberate π-projection, not a lowering artifact: the judge + evaluates a projection of the conversation, and strict providers + reject the raw multi-turn role sequence out of context. + """ # Build user message with tool call details func_name = item.get("func_name", item.get("name", "")) func_args = item.get("func_args", {}) @@ -1637,18 +1626,15 @@ class IntentJudge: transcript = "\n\n".join(transcript_lines) return [ - {"role": "system", "content": _JUDGE_SYSTEM_PROMPT}, - { - "role": "user", - "content": ( - f"Conversation context:\n\n{transcript}\n\n" - "---\n\n" - "Please evaluate the following tool call that is " - "pending human approval:\n\n" - f"{tool_detail}\n\n" - "Render your verdict as JSON." - ), - }, + Turn.system(_JUDGE_SYSTEM_PROMPT), + Turn.user( + f"Conversation context:\n\n{transcript}\n\n" + "---\n\n" + "Please evaluate the following tool call that is " + "pending human approval:\n\n" + f"{tool_detail}\n\n" + "Render your verdict as JSON." + ), ] # Paths the judge is never allowed to read (security hardening). diff --git a/turnstone/core/model_turn.py b/turnstone/core/model_turn.py index 6f242400..7bfaa41a 100644 --- a/turnstone/core/model_turn.py +++ b/turnstone/core/model_turn.py @@ -90,9 +90,14 @@ def resolve_capabilities( caps = provider.get_capabilities(model) if registry and alias: cfg = registry.get_config(alias) - if cfg.capabilities: + overrides_raw = getattr(cfg, "capabilities", None) + # Defensive dict-check (inherited from the judges' old mirror): a + # malformed capabilities value must degrade to "no overrides", not + # raise — a raise inside a judge constructor would silently downgrade + # the judge to the session model. + if isinstance(overrides_raw, dict) and overrides_raw: fields = {f.name for f in dataclasses.fields(type(caps))} - overrides = {k: v for k, v in cfg.capabilities.items() if k in fields} + overrides = {k: v for k, v in overrides_raw.items() if k in fields} if overrides: caps = dataclasses.replace(caps, **overrides) return caps diff --git a/turnstone/core/output_guard_judge.py b/turnstone/core/output_guard_judge.py index 48598dee..bef3a84a 100644 --- a/turnstone/core/output_guard_judge.py +++ b/turnstone/core/output_guard_judge.py @@ -51,9 +51,10 @@ from turnstone.core.deadline import ( from turnstone.core.judge import ( _CHARS_PER_TOKEN, _positive_window, - _resolve_model_capabilities, ) from turnstone.core.log import get_logger +from turnstone.core.model_turn import model_turn, resolve_capabilities, resolve_lane +from turnstone.core.trajectory import Turn if TYPE_CHECKING: import threading @@ -270,6 +271,9 @@ class OutputGuardJudge: session_capabilities: ModelCapabilities | None = None, ) -> None: self._config = config + # Carried into the per-evaluation ModelLane so extra_params and the + # live operator flags resolve from the registry like every other lane. + self._model_registry = model_registry # Caller's resolved session-model caps (config/registry-aware): the wire # capabilities + window when this judge inherits the session model, and # the alias path's window fallback. The window comes ONLY from these @@ -303,8 +307,10 @@ class OutputGuardJudge: ) self._model = model_name self._judge_model_alias = config.output_guard_model - self._capabilities = _resolve_model_capabilities( - self._provider, self._model, model_cfg + # Shared lane resolver (model_turn); ModelConfig.context_window + # stays separate and is sized into the guard window below. + self._capabilities = resolve_capabilities( + self._provider, self._model, config.output_guard_model, model_registry ) self._judge_context_window = _positive_window( getattr(model_cfg, "context_window", None), @@ -442,11 +448,10 @@ class OutputGuardJudge: start = time.monotonic() verdict_id = uuid.uuid4().hex timeout = max(self._config.output_guard_llm_timeout, 1.0) - judge_messages = [ - {"role": "system", "content": _SYSTEM_PROMPT}, - { - "role": "user", - "content": self._user_prompt( + judge_turns = [ + Turn.system(_SYSTEM_PROMPT), + Turn.user( + self._user_prompt( output, func_name=func_name, tool_description=tool_description, @@ -454,8 +459,8 @@ class OutputGuardJudge: heuristic_risk=heuristic_risk, heuristic_flags=heuristic_flags, heuristic_annotations=heuristic_annotations, - ), - }, + ) + ), ] # Oversize guard. The heuristic stage has already run and its verdict @@ -466,7 +471,7 @@ class OutputGuardJudge: # warning, and return a LABELLED error verdict so the skip surfaces as a # distinct ``llm_error`` audit row (reason = "output_too_large…") the # operator can see, rather than a silent no-op. - prompt_chars = sum(len(str(m["content"])) for m in judge_messages) + prompt_chars = sum(len(t.text) for t in judge_turns) est_tokens = int(prompt_chars / _CHARS_PER_TOKEN) if est_tokens > self._judge_context_window * _MAX_PROMPT_RATIO: log.warning( @@ -499,20 +504,25 @@ class OutputGuardJudge: # worker is non-daemon, and concurrent.futures joins it from an atexit # hook regardless of shutdown(wait=False) — so a wedged upstream call # would otherwise hang shutdown.) + # Single-shot lane: constructor-resolved capabilities, extra_params + # and live operator flags from the registry like every other lane. + lane = resolve_lane( + self._provider, + client, + self._model, + alias=self._judge_model_alias, + registry=self._model_registry, + capabilities=self._capabilities, + ) try: result = run_with_deadline( - lambda: self._provider.create_completion( - client=client, - model=self._model, - messages=judge_messages, + lambda: model_turn( + lane, + judge_turns, tools=None, max_tokens=512, temperature=0.0, reasoning_effort="low", - # Operator-declared capabilities reach the wire like every - # other lane — from the output_guard alias's definition, or - # the session model on fallback. See IntentJudge for why. - capabilities=self._capabilities, ), timeout=timeout, cancel_event=cancel_event,