ewilliams28/turnstone
1
0
Fork 0
mirror of https://github.com/turnstonelabs/turnstone.git synced 2026-08-12 23:12:23 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(judge): both judges speak Turn IR through model_turn (#827)

Browse Source 
The intent judge's evidence loop and the output-guard's single shot now
build list[Turn] and call model_turn — the hand-built OpenAI-dict
message construction is gone, and with it the judges' private
interlingua. The assistant turns they append carry the provider-native
lane, so the loop keeps reasoning continuity across its own turns.

That is what unblocks Gemini: thought_signature rides provider_blocks
and is reconstructed by the Google adapter's fidelity swap, so the
provider_name == "google" tool-skip is deleted — the Gemini judge runs
the same evidence-tool loop as every other provider instead of
degrading to a single-shot, tool-blind verdict.

judge.py's _resolve_model_capabilities mirror (#826) is deleted; both
judges resolve capabilities through the shared lane resolver, and each
evaluation builds a ModelLane (fresh client, constructor caps,
registry-resolved extra_params + live flags). The shared resolver
inherits the mirror's defensive non-dict capabilities check — without
it a malformed registry row would silently downgrade a judge to the
session model instead of just skipping the overrides.

Judge calls now resolve extra_params and replay_reasoning_to_model
from the registry like every other lane (previously: never sent, and
the protocol's back-compat default respectively).

Test mocks grow the CompletionResult fields the model_turn re-ingest
reads (provider_blocks, reasoning); alias-registry mocks wire
get_config, which the unified resolver uses.
This commit is contained in:
Patrick Buckley
2026-07-12 23:53:48 -07:00
parent ab35eb4215
commit 54dd4ed50a
5 changed files with 202 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_judge.py
+58 -41
View File
@@ -10,12 +10,34 @@ from typing import Any
from unittest.mock import MagicMock
from turnstone.core.judge import IntentJudge, IntentVerdict, JudgeConfig, evaluate_heuristic
from turnstone.core.trajectory import Role
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _mock_result(
content: str = "",
tool_calls: list[dict[str, Any]] | None = None,
) -> MagicMock:
"""A provider result shaped like ``CompletionResult``.
The judge's calls run through ``model_turn``, whose re-ingest iterates
``tool_calls``/``provider_blocks`` and joins ``reasoning`` — a bare
MagicMock attribute would TypeError, so every field the seam reads is
pinned to a real value.
"""
result = MagicMock()
result.content = content
result.tool_calls = tool_calls
result.finish_reason = "stop"
result.usage = None
result.provider_blocks = []
result.reasoning = ""
return result
def _make_mock_provider(
response_content: str = "",
tool_calls: list[dict[str, Any]] | None = None,
@@ -30,16 +52,10 @@ def _make_mock_provider(
caps.max_output_tokens = 4096
provider.get_capabilities.return_value = caps
result = MagicMock()
result.content = response_content
result.tool_calls = tool_calls
result.finish_reason = "stop"
result.usage = None
if side_effect:
provider.create_completion.side_effect = side_effect
else:
provider.create_completion.return_value = result
provider.create_completion.return_value = _mock_result(response_content, tool_calls)
provider.convert_tools.side_effect = lambda tools, **kw: tools
@@ -375,22 +391,21 @@ class TestMultiTurnToolUse:
provider.convert_tools.side_effect = lambda tools, **kw: tools
# Turn 1: tool call
turn1 = MagicMock()
turn1.content = ""
turn1.tool_calls = [
{
"id": "tc_judge_1",
"function": {
"name": "read_file",
"arguments": json.dumps({"path": "/nonexistent/file.txt"}),
},
}
]
turn1 = _mock_result(
"",
[
{
"id": "tc_judge_1",
"function": {
"name": "read_file",
"arguments": json.dumps({"path": "/nonexistent/file.txt"}),
},
}
],
)
# Turn 2: verdict
turn2 = MagicMock()
turn2.content = _good_verdict_json()
turn2.tool_calls = None
turn2 = _mock_result(_good_verdict_json())
provider.create_completion.side_effect = [turn1, turn2]
@@ -416,22 +431,21 @@ class TestMultiTurnToolUse:
provider.convert_tools.side_effect = lambda tools, **kw: tools
# Every turn returns a tool call
tool_result = MagicMock()
tool_result.content = ""
tool_result.tool_calls = [
{
"id": "tc_loop",
"function": {
"name": "read_file",
"arguments": json.dumps({"path": "/tmp/x"}),
},
}
]
tool_result = _mock_result(
"",
[
{
"id": "tc_loop",
"function": {
"name": "read_file",
"arguments": json.dumps({"path": "/tmp/x"}),
},
}
],
)
# Last turn (no tools param) returns text content
final = MagicMock()
final.content = _good_verdict_json()
final.tool_calls = None
final = _mock_result(_good_verdict_json())
# Turns 0-3: tool_call; turn 4 (last, tools=None): final verdict
provider.create_completion.side_effect = [
@@ -468,12 +482,12 @@ class TestContextPreparation:
result = judge._prepare_context(_make_item(), messages)
# Should have system message + single user message with transcript
# Should have a system Turn + single user Turn with the transcript
assert len(result) == 2
assert result[0]["role"] == "system"
assert result[1]["role"] == "user"
assert "pending human approval" in result[1]["content"]
assert "Conversation context:" in result[1]["content"]
assert result[0].role is Role.SYSTEM
assert result[1].role is Role.USER
assert "pending human approval" in result[1].text
assert "Conversation context:" in result[1].text
class TestArgBudget:
@@ -541,7 +555,7 @@ class TestArgBudget:
)
# Each included history turn renders one "ASSISTANT:" line; the
# big-argument call fits strictly fewer of them.
assert big[1]["content"].count("ASSISTANT:") < small[1]["content"].count("ASSISTANT:")
assert big[1].text.count("ASSISTANT:") < small[1].text.count("ASSISTANT:")
# ---------------------------------------------------------------------------
@@ -901,6 +915,9 @@ class TestModelAliasResolution:
cfg.capabilities = capabilities if capabilities is not None else {}
registry.has_alias.side_effect = lambda a: a == alias
registry.resolve.return_value = (alias_client, underlying_model, cfg)
# The unified lane resolver (model_turn.resolve_capabilities) fetches
# the config itself rather than taking resolve()'s copy.
registry.get_config.return_value = cfg
registry.get_provider.return_value = alias_provider
return registry
tests/test_output_guard_judge.py
+18 -6
View File
@@ -34,14 +34,25 @@ def _make_provider(
time.sleep(delay)
if raises is not None:
raise raises
result = MagicMock()
result.content = content
return result
return _mock_result(content)
provider.create_completion = _create_completion
return provider
def _mock_result(content: str) -> MagicMock:
"""A provider result shaped like ``CompletionResult`` — the guard's call
runs through ``model_turn``, whose re-ingest reads every field below."""
result = MagicMock()
result.content = content
result.tool_calls = None
result.finish_reason = "stop"
result.usage = None
result.provider_blocks = []
result.reasoning = ""
return result
def _make_judge(
*,
content: str = "",
@@ -79,9 +90,7 @@ class TestCapabilityThreading:
def _cc(**kwargs: Any) -> Any:
captured.update(kwargs)
result = MagicMock()
result.content = '{"risk_level": "none", "flags": []}'
return result
return _mock_result('{"risk_level": "none", "flags": []}')
provider = MagicMock()
provider.provider_name = "openai"
@@ -121,6 +130,9 @@ class TestCapabilityThreading:
"local-9b",
cfg,
)
# The unified lane resolver (model_turn.resolve_capabilities) fetches
# the config itself rather than taking resolve()'s copy.
registry.get_config.return_value = cfg
registry.get_provider.return_value = provider
client = MagicMock(base_url="http://s", api_key="k")
judge = OutputGuardJudge(
turnstone/core/judge.py
+90 -104
View File
@@ -15,7 +15,7 @@ import re
import threading
import time
import uuid
from dataclasses import dataclass, field, fields, replace
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -26,6 +26,8 @@ from turnstone.core.deadline import (
run_with_deadline,
)
from turnstone.core.log import get_logger
from turnstone.core.model_turn import model_turn, resolve_capabilities, resolve_lane
from turnstone.core.trajectory import Turn
if TYPE_CHECKING:
from collections.abc import Callable
@@ -844,33 +846,6 @@ def _positive_window(*candidates: Any, floor: int = _DEFAULT_JUDGE_CONTEXT_WINDO
return floor
def _resolve_model_capabilities(provider: LLMProvider, model: str, cfg: Any) -> ModelCapabilities:
"""Provider base capabilities with a model definition's ``capabilities``
overrides applied — the same lowering ``ChatSession._resolve_capabilities``
performs for the session, utility, and sub-agent completion lanes.
The judges are the only completion callers that live outside ``ChatSession``,
so they cannot reach ``self._resolve_capabilities``; this mirrors it so a
judge alias honors operator-declared capabilities (effort passthrough, tool
support, temperature, verbosity) exactly like the main loop. ``cfg`` is the
alias's ``ModelConfig``; a missing or non-dict ``capabilities`` is ignored
rather than raised — capability resolution must never crash a judge turn.
It does NOT fold in ``ModelConfig.context_window``: that is a separate field,
not part of the capabilities JSON, and the caller sizes the judge's window
budget off it directly (the static caps table reports 200000 for local
models, which would silently over-budget them).
"""
caps = provider.get_capabilities(model)
overrides = getattr(cfg, "capabilities", None)
if isinstance(overrides, dict) and overrides:
names = {f.name for f in fields(type(caps))}
applied = {k: v for k, v in overrides.items() if k in names}
if applied:
caps = replace(caps, **applied)
return caps
def honest_truncate(text: str, budget: int) -> str:
"""Return *text* untouched when it fits *budget* characters, otherwise the
leading ``budget`` characters followed by an explicit note of exactly how
@@ -1004,6 +979,9 @@ class IntentJudge:
) -> None:
self._config = config
self._rule_registry = rule_registry
# Carried into the per-evaluation ModelLane so extra_params and the
# live operator flags resolve from the registry like every other lane.
self._model_registry = model_registry
# The caller (ChatSession) resolves the session model's real caps from
# _get_capabilities (config/registry-aware) and passes them in; they are
# this judge's wire capabilities and window when it inherits the session
@@ -1036,8 +1014,15 @@ class IntentJudge:
self._provider.provider_name,
)
self._model = model_name
self._capabilities = _resolve_model_capabilities(
self._provider, self._model, model_cfg
self._alias = config.model
# The shared lane resolver (model_turn) merges the alias's
# capability overrides; it deliberately does NOT fold in
# ModelConfig.context_window — that is a separate field,
# sized into the judge's window budget right below (the
# static caps table reports 200000 for local models, which
# would silently over-budget them).
self._capabilities = resolve_capabilities(
self._provider, self._model, config.model, model_registry
)
# Use the registry's per-model context window, NOT
# ``provider.get_capabilities().context_window``: the static
@@ -1073,6 +1058,7 @@ class IntentJudge:
session_provider.provider_name,
)
self._model = session_model
self._alias = ""
# Wire caps: the caller's resolved session caps, or the provider's
# static table as a last resort for degraded / legacy callers.
self._capabilities = (
@@ -1320,19 +1306,36 @@ class IntentJudge:
except (TypeError, ValueError):
func_args_json = honest_truncate(str(func_args), _VERDICT_ARG_CAP)
# Prepare context
judge_messages = self._prepare_context(item, messages)
# Prepare context (Turn IR — lowered per call inside model_turn)
judge_turns = self._prepare_context(item, messages)
# Prepare tools (only if read_only_tools enabled).
# Pass raw OpenAI-format schemas — create_completion handles conversion.
# Google's API requires thought_signature in function call round-trips
# which our normalized tool_calls don't preserve, so skip tools for Google.
# Raw OpenAI-format schemas — the provider adapter converts them.
# These are γ-side instruments, deliberately OUTSIDE the persona
# envelope: middle-rank config must not be able to blind the gate's
# evidence gathering. The old provider_name == "google" skip is gone:
# the judge's trajectory now carries the provider-native lane
# (thought_signature rides ``provider_blocks`` and is reconstructed by
# the Google adapter), so the Gemini judge runs the same evidence loop
# as every other provider.
tools: list[dict[str, Any]] | None = None
if self._config.read_only_tools and self._provider.provider_name != "google":
if self._config.read_only_tools:
tools = list(_JUDGE_TOOL_SCHEMAS)
# The judge's resolved lane for this evaluation: fresh client per run
# (thread isolation), constructor-resolved capabilities, extra_params
# and live operator flags from the registry like every other lane.
lane = resolve_lane(
self._provider,
client,
self._model,
alias=self._alias,
registry=self._model_registry,
capabilities=self._capabilities,
)
# Multi-turn judge loop
result = None # will hold the last CompletionResult
result = None # will hold the last ModelTurnResult
empty_retries = 0 # track consecutive empty responses for retry
turn = 0
@@ -1352,15 +1355,12 @@ class IntentJudge:
# On the last turn, strip tools and inject a forcing message
# so the model knows it must render a verdict now.
if is_last_turn:
judge_messages.append(
{
"role": "user",
"content": (
"You have gathered enough evidence. "
"You MUST now render your final verdict as JSON. "
"No more tool calls."
),
}
judge_turns.append(
Turn.user(
"You have gathered enough evidence. "
"You MUST now render your final verdict as JSON. "
"No more tool calls."
)
)
# Per-turn timeout: each turn gets a fresh budget so local
@@ -1374,21 +1374,13 @@ class IntentJudge:
# poisoned the pool, which is why the restart dance existed.
result = run_with_deadline(
partial(
self._provider.create_completion,
client=client,
model=self._model,
messages=judge_messages,
model_turn,
lane,
judge_turns,
tools=None if is_last_turn else tools,
max_tokens=2048,
temperature=0.0,
reasoning_effort="medium",
# Thread the judge model's operator-declared capabilities
# onto the wire like every other lane — resolved from the
# judge alias's model definition, or the session model on
# fallback. Without this the provider would fall back to
# its static capability table and silently ignore the
# definition's overrides on judge calls alone.
capabilities=self._capabilities,
),
timeout=per_call_timeout,
cancel_event=cancel_event,
@@ -1431,14 +1423,14 @@ class IntentJudge:
# Check for tool calls
if result.tool_calls:
# Execute read-only tools and append results
judge_messages.append(
{
"role": "assistant",
"content": result.content or None,
"tool_calls": result.tool_calls,
}
)
# Append the assistant turn — the native lane rides along
# (Gemini thought_signature, Anthropic thinking, Responses
# reasoning items), so the next lowering replays it and the
# evidence loop keeps its reasoning continuity. The judge
# never mints ids: its trajectory is ephemeral and pinned to
# one provider, so provider-original ids stay consistent
# between the native blocks, the mirror, and the results.
judge_turns.append(result.turn)
for tc in result.tool_calls:
tc_func = tc.get("function", {})
tc_name = tc_func.get("name", "")
@@ -1451,13 +1443,7 @@ class IntentJudge:
tc_args = {}
tool_result = self._exec_read_only_tool(tc_name, tc_args)
judge_messages.append(
{
"role": "tool",
"tool_call_id": tc.get("id", ""),
"content": tool_result,
}
)
judge_turns.append(Turn.tool(tc.get("id", ""), tool_result))
turn += 1
continue
@@ -1486,15 +1472,12 @@ class IntentJudge:
)
return None
# On earlier turns, inject a nudge and continue
judge_messages.append({"role": "assistant", "content": result.content})
judge_messages.append(
{
"role": "user",
"content": (
"Your response was not valid JSON. "
"Please respond ONLY with the JSON verdict object."
),
}
judge_turns.append(result.turn)
judge_turns.append(
Turn.user(
"Your response was not valid JSON. "
"Please respond ONLY with the JSON verdict object."
)
)
turn += 1
continue
@@ -1511,15 +1494,12 @@ class IntentJudge:
empty_retries += 1
if empty_retries <= 3:
log.info("judge.empty_response.retry", retry=empty_retries, max_retries=3)
judge_messages.append(
{
"role": "user",
"content": (
"You returned an empty response. "
"Please analyze the tool call and respond with "
"the JSON verdict object."
),
}
judge_turns.append(
Turn.user(
"You returned an empty response. "
"Please analyze the tool call and respond with "
"the JSON verdict object."
)
)
continue
log.info("judge.empty_response.giving_up", retries=empty_retries)
@@ -1551,8 +1531,17 @@ class IntentJudge:
self,
item: dict[str, Any],
messages: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Build the judge's message list with FIFO-truncated conversation."""
) -> list[Turn]:
"""Build the judge's opening trajectory with FIFO-truncated conversation.
*messages* is the session's wire-dict history (read-only input: the
judge observes the session, it does not join it); the output is Turn
IR — the judge's own ephemeral trajectory, lowered per call by
``model_turn``. The flattened single-user-message transcript is the
judge's deliberate π-projection, not a lowering artifact: the judge
evaluates a projection of the conversation, and strict providers
reject the raw multi-turn role sequence out of context.
"""
# Build user message with tool call details
func_name = item.get("func_name", item.get("name", ""))
func_args = item.get("func_args", {})
@@ -1637,18 +1626,15 @@ class IntentJudge:
transcript = "\n\n".join(transcript_lines)
return [
{"role": "system", "content": _JUDGE_SYSTEM_PROMPT},
{
"role": "user",
"content": (
f"Conversation context:\n\n{transcript}\n\n"
"---\n\n"
"Please evaluate the following tool call that is "
"pending human approval:\n\n"
f"{tool_detail}\n\n"
"Render your verdict as JSON."
),
},
Turn.system(_JUDGE_SYSTEM_PROMPT),
Turn.user(
f"Conversation context:\n\n{transcript}\n\n"
"---\n\n"
"Please evaluate the following tool call that is "
"pending human approval:\n\n"
f"{tool_detail}\n\n"
"Render your verdict as JSON."
),
]
# Paths the judge is never allowed to read (security hardening).
turnstone/core/model_turn.py
+7 -2
View File
@@ -90,9 +90,14 @@ def resolve_capabilities(
caps = provider.get_capabilities(model)
if registry and alias:
cfg = registry.get_config(alias)
if cfg.capabilities:
overrides_raw = getattr(cfg, "capabilities", None)
# Defensive dict-check (inherited from the judges' old mirror): a
# malformed capabilities value must degrade to "no overrides", not
# raise — a raise inside a judge constructor would silently downgrade
# the judge to the session model.
if isinstance(overrides_raw, dict) and overrides_raw:
fields = {f.name for f in dataclasses.fields(type(caps))}
overrides = {k: v for k, v in cfg.capabilities.items() if k in fields}
overrides = {k: v for k, v in overrides_raw.items() if k in fields}
if overrides:
caps = dataclasses.replace(caps, **overrides)
return caps
turnstone/core/output_guard_judge.py
+29 -19
View File
@@ -51,9 +51,10 @@ from turnstone.core.deadline import (
from turnstone.core.judge import (
_CHARS_PER_TOKEN,
_positive_window,
_resolve_model_capabilities,
)
from turnstone.core.log import get_logger
from turnstone.core.model_turn import model_turn, resolve_capabilities, resolve_lane
from turnstone.core.trajectory import Turn
if TYPE_CHECKING:
import threading
@@ -270,6 +271,9 @@ class OutputGuardJudge:
session_capabilities: ModelCapabilities | None = None,
) -> None:
self._config = config
# Carried into the per-evaluation ModelLane so extra_params and the
# live operator flags resolve from the registry like every other lane.
self._model_registry = model_registry
# Caller's resolved session-model caps (config/registry-aware): the wire
# capabilities + window when this judge inherits the session model, and
# the alias path's window fallback. The window comes ONLY from these
@@ -303,8 +307,10 @@ class OutputGuardJudge:
)
self._model = model_name
self._judge_model_alias = config.output_guard_model
self._capabilities = _resolve_model_capabilities(
self._provider, self._model, model_cfg
# Shared lane resolver (model_turn); ModelConfig.context_window
# stays separate and is sized into the guard window below.
self._capabilities = resolve_capabilities(
self._provider, self._model, config.output_guard_model, model_registry
)
self._judge_context_window = _positive_window(
getattr(model_cfg, "context_window", None),
@@ -442,11 +448,10 @@ class OutputGuardJudge:
start = time.monotonic()
verdict_id = uuid.uuid4().hex
timeout = max(self._config.output_guard_llm_timeout, 1.0)
judge_messages = [
{"role": "system", "content": _SYSTEM_PROMPT},
{
"role": "user",
"content": self._user_prompt(
judge_turns = [
Turn.system(_SYSTEM_PROMPT),
Turn.user(
self._user_prompt(
output,
func_name=func_name,
tool_description=tool_description,
@@ -454,8 +459,8 @@ class OutputGuardJudge:
heuristic_risk=heuristic_risk,
heuristic_flags=heuristic_flags,
heuristic_annotations=heuristic_annotations,
),
},
)
),
]
# Oversize guard. The heuristic stage has already run and its verdict
@@ -466,7 +471,7 @@ class OutputGuardJudge:
# warning, and return a LABELLED error verdict so the skip surfaces as a
# distinct ``llm_error`` audit row (reason = "output_too_large…") the
# operator can see, rather than a silent no-op.
prompt_chars = sum(len(str(m["content"])) for m in judge_messages)
prompt_chars = sum(len(t.text) for t in judge_turns)
est_tokens = int(prompt_chars / _CHARS_PER_TOKEN)
if est_tokens > self._judge_context_window * _MAX_PROMPT_RATIO:
log.warning(
@@ -499,20 +504,25 @@ class OutputGuardJudge:
# worker is non-daemon, and concurrent.futures joins it from an atexit
# hook regardless of shutdown(wait=False) — so a wedged upstream call
# would otherwise hang shutdown.)
# Single-shot lane: constructor-resolved capabilities, extra_params
# and live operator flags from the registry like every other lane.
lane = resolve_lane(
self._provider,
client,
self._model,
alias=self._judge_model_alias,
registry=self._model_registry,
capabilities=self._capabilities,
)
try:
result = run_with_deadline(
lambda: self._provider.create_completion(
client=client,
model=self._model,
messages=judge_messages,
lambda: model_turn(
lane,
judge_turns,
tools=None,
max_tokens=512,
temperature=0.0,
reasoning_effort="low",
# Operator-declared capabilities reach the wire like every
# other lane — from the output_guard alias's definition, or
# the session model on fallback. See IntentJudge for why.
capabilities=self._capabilities,
),
timeout=timeout,
cancel_event=cancel_event,