mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
fix(ui): ship risk-none intent verdicts on history replay
The /history decoration layer suppressed intent-verdict rows with risk_level="none" from the wire payload, on the assumption the client filtered them anyway. It never did: buildConvVerdict renders a badge for every verdict it receives, so the live SSE path painted all judge verdicts while rehydration silently dropped the benign majority — a 22-call parallel batch came back from a restart showing only the 3 flagged calls. Ship every stored row and let the client render replay exactly as it rendered the live stream. The output-guard chip pair (showOutputWarning + merge-on-clean) already suppresses consistently on BOTH sides and is unchanged.
This commit is contained in:
@@ -24,12 +24,16 @@ class TestBuildVerdictPayload:
|
||||
"""The wire-shape projection that's the single source of truth for
|
||||
what intent_verdict fields ship to the client."""
|
||||
|
||||
def test_skips_unflagged_baseline(self) -> None:
|
||||
"""``risk_level`` "none" is the unflagged-tool baseline; the
|
||||
client filters those anyway, so projecting None at the wire
|
||||
layer keeps the payload tight on long workstreams."""
|
||||
row = {"risk_level": "none", "recommendation": "approve", "tier": "heuristic"}
|
||||
assert build_verdict_payload(row) is None
|
||||
def test_ships_unflagged_baseline(self) -> None:
|
||||
"""``risk_level`` "none" rows ship like any other — the live
|
||||
path paints a badge for every delivered verdict (the client
|
||||
has no risk filter), so replay must carry the same set or
|
||||
benign verdicts vanish on rehydrate (live/replay parity)."""
|
||||
row = {"risk_level": "none", "recommendation": "approve", "tier": "llm"}
|
||||
out = build_verdict_payload(row)
|
||||
assert out["risk_level"] == "none"
|
||||
assert out["recommendation"] == "approve"
|
||||
assert out["tier"] == "llm"
|
||||
|
||||
def test_drops_call_id_and_func_name(self) -> None:
|
||||
"""The client already has these on ``tc.id`` / ``tc.name``;
|
||||
@@ -243,13 +247,14 @@ class TestDecorateToolCall:
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
def test_skips_unflagged_verdict(self) -> None:
|
||||
"""``build_verdict_payload`` returns None for unflagged rows;
|
||||
decorate_tool_call must not stamp ``verdict`` in that case."""
|
||||
def test_stamps_unflagged_verdict(self) -> None:
|
||||
"""A ``risk_level="none"`` row still stamps ``verdict`` — the
|
||||
operator saw the badge live, so it must survive rehydrate."""
|
||||
tc: dict[str, object] = {"id": "call_1", "name": "bash"}
|
||||
verdicts = {"call_1": {"risk_level": "none", "tier": "heuristic"}}
|
||||
verdicts = {"call_1": {"risk_level": "none", "tier": "llm"}}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
assert "verdict" in tc
|
||||
assert tc["verdict"]["risk_level"] == "none" # type: ignore[index]
|
||||
|
||||
def test_handles_empty_id(self) -> None:
|
||||
"""A tool_call with no id can't be paired against the lookup
|
||||
@@ -311,6 +316,38 @@ class TestDecorateHistoryMessages:
|
||||
assert messages[3]["content"] == "short"
|
||||
assert "advisories" not in messages[3]
|
||||
|
||||
def test_parallel_batch_keeps_every_judged_verdict(self) -> None:
|
||||
"""Regression: a parallel batch where the judge cleared most
|
||||
calls (``risk_level="none"``) must rehydrate with a verdict on
|
||||
EVERY judged call, not just the flagged minority. The old
|
||||
wire-layer ``none`` filter made benign verdicts vanish after a
|
||||
restart while the live stream had shown all of them."""
|
||||
calls = [f"call_{i}" for i in range(8)]
|
||||
verdicts = {
|
||||
cid: {
|
||||
"risk_level": "low" if i < 2 else "none",
|
||||
"recommendation": "approve",
|
||||
"confidence": 0.9,
|
||||
"intent_summary": f"benign op {i}",
|
||||
"tier": "llm",
|
||||
}
|
||||
for i, cid in enumerate(calls)
|
||||
}
|
||||
messages: list[dict[str, object]] = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": cid, "function": {"name": "read_file", "arguments": "{}"}}
|
||||
for cid in calls
|
||||
],
|
||||
},
|
||||
]
|
||||
decorate_history_messages(messages, verdicts, {})
|
||||
tool_calls = messages[0]["tool_calls"] # type: ignore[index]
|
||||
decorated = [tc["verdict"]["risk_level"] for tc in tool_calls]
|
||||
assert decorated == ["low", "low"] + ["none"] * 6
|
||||
|
||||
def test_no_op_on_empty_indexes(self) -> None:
|
||||
"""When neither table has rows for the workstream, the wire
|
||||
shape passes through unchanged — replay must degrade
|
||||
|
||||
@@ -98,13 +98,17 @@ def load_verdict_indexes(
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
|
||||
|
||||
def build_verdict_payload(vrow: dict[str, Any]) -> dict[str, Any] | None:
|
||||
def build_verdict_payload(vrow: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Project a stored ``intent_verdicts`` row into the wire shape.
|
||||
|
||||
Returns ``None`` when the verdict is the unflagged baseline
|
||||
(``risk_level == "none"``) — the client's ``renderVerdictBadge``
|
||||
helper would suppress those anyway, so skipping at the wire layer
|
||||
keeps the payload tight on long workstreams.
|
||||
Ships every row, including the unflagged baseline (``risk_level ==
|
||||
"none"``) — the live path renders a badge for every verdict the
|
||||
judge delivers (``buildConvVerdict`` has no risk filter), so the
|
||||
replay payload must carry the same set or rehydration silently
|
||||
"loses" verdicts the operator watched land live. An earlier
|
||||
revision suppressed ``none`` rows here on the assumption the
|
||||
client filtered them anyway; it never did, and the asymmetry
|
||||
surfaced as benign verdicts vanishing after a restart.
|
||||
|
||||
Drops ``call_id`` and ``func_name`` from the wire payload — they're
|
||||
already carried on the parent ``tc.id`` / ``tc.name`` fields.
|
||||
@@ -115,10 +119,8 @@ def build_verdict_payload(vrow: dict[str, Any]) -> dict[str, Any] | None:
|
||||
badge can render ``⚖ llm:claude-haiku-4`` on history-only batches
|
||||
rather than the bare ``⚖ llm`` label.
|
||||
"""
|
||||
if (vrow.get("risk_level") or "none") == "none":
|
||||
return None
|
||||
payload: dict[str, Any] = {
|
||||
"risk_level": vrow.get("risk_level", "medium"),
|
||||
"risk_level": vrow.get("risk_level") or "none",
|
||||
"recommendation": vrow.get("recommendation", "review"),
|
||||
"confidence": vrow.get("confidence", 0.0),
|
||||
"intent_summary": vrow.get("intent_summary", ""),
|
||||
@@ -190,17 +192,15 @@ def decorate_tool_call(
|
||||
either the OpenAI-nested ``{id, function: {name, arguments}}`` shape —
|
||||
what ``decorate_history_messages`` passes from the REST ``/history``
|
||||
pipeline — or a flattened ``{id, name, arguments}`` shape. No-ops
|
||||
cleanly when the call_id has no matching row (unflagged tools stay
|
||||
clean).
|
||||
cleanly when the call_id has no matching row (tools the judge never
|
||||
evaluated stay clean).
|
||||
"""
|
||||
call_id = tc.get("id", "") or ""
|
||||
if not call_id:
|
||||
return
|
||||
vrow = verdicts_by_call_id.get(call_id)
|
||||
if vrow is not None:
|
||||
verdict = build_verdict_payload(vrow)
|
||||
if verdict is not None:
|
||||
tc["verdict"] = verdict
|
||||
tc["verdict"] = build_verdict_payload(vrow)
|
||||
slot = assessments_by_call_id.get(call_id)
|
||||
if slot is not None:
|
||||
assessment = build_merged_output_assessment_payload(slot)
|
||||
|
||||
Reference in New Issue
Block a user