diff --git a/docs/judge.md b/docs/judge.md index 00be8b4f..d0d6b9c8 100644 --- a/docs/judge.md +++ b/docs/judge.md @@ -372,10 +372,36 @@ redact_secrets = true # auto-redact detected credentials (default) Configurable at runtime via the admin Settings tab. +### Merge semantics (heuristic + LLM judge) + +The chip is a **merge** of the two detectors (issue #560, "show, annotated"), +not a winner-take-all: + +- `risk_level` = **max**(heuristic, llm) and `flags` = **union**. A positive + from either detector surfaces; a negative ("none") or failed/absent LLM + **never lowers** a heuristic positive. The judge reads adversarial tool + output, so it may raise the alarm but must not be able to hide a + deterministic regex finding — defeating the judge can't erase the tripwire. +- Credential **redaction** is a heuristic-only signal the LLM cannot override. +- When the judge returned a verdict, its OWN verdict rides along as + annotation (`judge_risk` / `confidence` / `reasoning` / `judge_model`) so + the operator sees the judge's opinion even when it disagrees with the + displayed (merged) risk. + +The same merge runs live and on reconnect (both call +`output_guard.merge_guard_display_payload`), so the chip can't drift between +the two surfaces. + +The MODEL on the other side of the conversation is shown the merged +`risk_level` + `flags` (via the `GuardAdvisory` spliced into the tool-result +envelope), but is **never** told the judge cleared a finding — a judge fooled +into "none" must not get to talk the model out of caution. The judge's +"benign" verdict is operator-facing only. + ### SSE event: `output_warning` -When the output guard detects risk signals, an `output_warning` SSE event is -emitted to the frontend: +When the merged finding is non-clean (or credentials were redacted), an +`output_warning` SSE event is emitted to the frontend. A regex-only finding: ```json { @@ -386,17 +412,50 @@ emitted to the frontend: "flags": ["credential_leak"], "annotations": ["API key detected (sk-proj-...)"], "output_length": 1024, - "redacted": true + "redacted": true, + "tier": "heuristic" } ``` -The web UI renders this as an inline warning after the tool result. The CLI -shows a colored terminal warning. The server forwards it as an -`OutputWarningEvent` for console subscribers. +When the LLM judge returned a verdict, `tier` is `"llm"` and the event carries +the judge's own verdict as annotation. Here the regex flagged MEDIUM but the +judge assessed the output benign — the finding still surfaces (`risk_level` +stays MEDIUM), annotated with the judge's dissent (`judge_risk: "none"`): -Assessments are persisted to the `output_assessments` table for v2 -calibration. Raw tool output is never stored — only metadata (flags, risk -level, annotations, output length, redaction status). +```json +{ + "type": "output_warning", + "call_id": "call_def456", + "func_name": "web_fetch", + "risk_level": "medium", + "flags": ["camouflaged_injection"], + "annotations": ["Authority-framed directive embedded in the document."], + "output_length": 8192, + "redacted": false, + "tier": "llm", + "judge_risk": "none", + "confidence": 0.92, + "reasoning": "Legitimate analyst commentary; no injection.", + "judge_model": "gpt-5-mini" +} +``` + +`judge_risk` (the judge's OWN risk verdict, which may differ from the merged +`risk_level`), `confidence` (0.0–1.0), `reasoning`, and `judge_model` are +present only on the `"llm"` tier. The identical shape is projected onto +history replay by `build_merged_output_assessment_payload`, so the inline chip +renders the same live and on refresh. + +The web UI renders this as an inline warning after the tool result — the +`"llm"` tier adds a `⚖ LLM · NN%` badge (showing the judge's verdict when it +differs from the displayed risk, e.g. `⚖ LLM: none · 92%`) and the judge's +rationale. The CLI shows a colored terminal warning. The server forwards it as +an `OutputWarningEvent` for console subscribers. + +Assessments are persisted to the `output_assessments` table (one row per +`(call_id, tier)`) for calibration. Raw tool output is never stored — only +metadata: flags, risk level, annotations, output length, redaction status, +and — for the LLM tier — confidence, reasoning, judge model, and latency. ### Session-level skill scan warning diff --git a/tests/test_history_decoration.py b/tests/test_history_decoration.py index a7dad4c0..c3529831 100644 --- a/tests/test_history_decoration.py +++ b/tests/test_history_decoration.py @@ -9,11 +9,14 @@ schema/projection change land in one file. from __future__ import annotations +import json + from turnstone.core.history_decoration import ( - build_output_assessment_payload, + build_merged_output_assessment_payload, build_verdict_payload, decorate_history_messages, decorate_tool_call, + load_verdict_indexes, ) @@ -93,31 +96,125 @@ class TestBuildVerdictPayload: assert "judge_model" not in out -class TestBuildOutputAssessmentPayload: - """Output-guard wire shape — flags decoded from JSON string at - this layer so the client never has to parse twice.""" +class TestBuildMergedOutputAssessmentPayload: + """Replay-side merge of the heuristic + LLM rows into one chip payload. + + Delegates to ``output_guard.merge_guard_display_payload`` — the same + projection the live ``on_output_warning`` path calls — so the inline + finding chip renders identically live and on reconnect. ``slot`` is + ``{"heuristic": row|None, "llm": row|None}`` from ``load_verdict_indexes``. + """ def test_skips_unflagged_baseline(self) -> None: - row = {"risk_level": "none", "flags": "[]"} - assert build_output_assessment_payload(row) is None + slot = {"heuristic": {"risk_level": "none", "flags": "[]"}, "llm": None} + assert build_merged_output_assessment_payload(slot) is None - def test_decodes_flags_from_json(self) -> None: - row = {"risk_level": "high", "flags": '["api_key","email"]', "redacted": 1} - out = build_output_assessment_payload(row) + def test_decodes_heuristic_flags_from_json(self) -> None: + slot = { + "heuristic": {"risk_level": "high", "flags": '["api_key","email"]', "redacted": 1}, + "llm": None, + } + out = build_merged_output_assessment_payload(slot) assert out is not None assert out["flags"] == ["api_key", "email"] assert out["redacted"] is True assert out["risk_level"] == "high" + assert out["tier"] == "heuristic" def test_handles_malformed_flags_json(self) -> None: """Bad JSON in ``flags`` must not block the rest of the assessment from rendering — degrade to empty list.""" - row = {"risk_level": "medium", "flags": "not-json", "redacted": 0} - out = build_output_assessment_payload(row) + slot = { + "heuristic": {"risk_level": "medium", "flags": "not-json", "redacted": 0}, + "llm": None, + } + out = build_merged_output_assessment_payload(slot) assert out is not None assert out["flags"] == [] assert out["redacted"] is False + def test_llm_escalates_over_clean_heuristic(self) -> None: + """LLM positive on a clean heuristic surfaces under tier='llm' with + the judge's own risk/confidence/reasoning/model as annotation.""" + slot = { + "heuristic": {"risk_level": "none", "flags": "[]", "redacted": 0}, + "llm": { + "risk_level": "medium", + "flags": '["camouflaged_injection"]', + "reasoning": "Authority-framed directive embedded in the doc.", + "confidence": 0.82, + "judge_model": "gpt-5-mini", + }, + } + out = build_merged_output_assessment_payload(slot) + assert out is not None + assert out["risk_level"] == "medium" + assert out["flags"] == ["camouflaged_injection"] + assert out["tier"] == "llm" + assert out["judge_risk"] == "medium" + assert out["confidence"] == 0.82 + assert out["reasoning"] == "Authority-framed directive embedded in the doc." + assert out["judge_model"] == "gpt-5-mini" + + def test_llm_none_does_not_lower_heuristic_positive(self) -> None: + """Core merge rule + the vanishing-chip fix: a successful LLM "none" + never lowers a heuristic positive — it surfaces, annotated with the + judge's dissent (judge_risk="none" differs from the displayed risk).""" + slot = { + "heuristic": { + "risk_level": "medium", + "flags": '["camouflaged_injection"]', + "redacted": 0, + }, + "llm": { + "risk_level": "none", + "flags": "[]", + "reasoning": "Benign analyst commentary.", + "confidence": 0.9, + "judge_model": "gpt-5-mini", + }, + } + out = build_merged_output_assessment_payload(slot) + assert out is not None + assert out["risk_level"] == "medium" # heuristic survives + assert out["flags"] == ["camouflaged_injection"] + assert out["tier"] == "llm" + assert out["judge_risk"] == "none" # judge's dissent, drives the badge + assert out["reasoning"] == "Benign analyst commentary." + + def test_flags_are_unioned_and_deduped(self) -> None: + slot = { + "heuristic": { + "risk_level": "high", + "flags": '["prompt_injection","credential_leak"]', + "redacted": 0, + }, + "llm": { + "risk_level": "high", + "flags": '["prompt_injection","data_exfiltration"]', + "reasoning": "x", + "confidence": 0.9, + "judge_model": "m", + }, + } + out = build_merged_output_assessment_payload(slot) + assert out is not None + assert out["flags"] == ["prompt_injection", "credential_leak", "data_exfiltration"] + + def test_heuristic_only_has_no_llm_badge(self) -> None: + """A regex-only finding (no LLM slot) carries no LLM attribution.""" + slot = { + "heuristic": {"risk_level": "high", "flags": '["credential_leak"]', "redacted": 1}, + "llm": None, + } + out = build_merged_output_assessment_payload(slot) + assert out is not None + assert out["tier"] == "heuristic" + assert "judge_risk" not in out + assert "confidence" not in out + assert "reasoning" not in out + assert "judge_model" not in out + class TestDecorateToolCall: """In-place mutation of either OpenAI-format or flattened tool_call @@ -179,7 +276,10 @@ class TestDecorateHistoryMessages: } } assessments = { - "call_a": {"risk_level": "high", "flags": '["secret"]', "redacted": 1}, + "call_a": { + "heuristic": {"risk_level": "high", "flags": '["secret"]', "redacted": 1}, + "llm": None, + }, } messages: list[dict[str, object]] = [ {"role": "user", "content": "hi"}, @@ -850,3 +950,87 @@ class TestAttachVllmChatReasoningField: assert out[3] is plain_assistant assert "reasoning" not in out[3] assert out[4] is msgs[4] + + +class TestLoadVerdictIndexesMerge: + """load_verdict_indexes + the merge, against real storage — pins the + vanishing-chip fix (failed-judge row must not hide a heuristic finding) + and the de-escalation annotate behavior end to end.""" + + def _record(self, storage, **kw) -> None: + base = { + "func_name": "read_file", + "flags": "[]", + "annotations": "[]", + "output_length": 900, + "redacted": False, + } + base.update(kw) + storage.record_output_assessment(**base) + + def test_llm_error_row_does_not_shadow_heuristic(self, storage_backend) -> None: + """A failed-judge row (tier='llm_error', risk='none') is audit-only and + must NOT win the replay merge over a real heuristic finding — the bug + behind the chip that showed live but vanished on reconnect.""" + ws_id, call_id = "ws-merge-err", "call-env" + self._record( + storage_backend, + assessment_id="a-h", + ws_id=ws_id, + call_id=call_id, + flags=json.dumps(["credential_leak", "env_file_leak"]), + risk_level="high", + redacted=True, + tier="heuristic", + ) + self._record( + storage_backend, + assessment_id="a-e", + ws_id=ws_id, + call_id=call_id, + risk_level="none", + tier="llm_error", + reasoning="timeout", + ) + _verdicts, assessments = load_verdict_indexes(ws_id) + # The llm_error row is dropped at load — slot has only the heuristic. + assert assessments[call_id]["llm"] is None + out = build_merged_output_assessment_payload(assessments[call_id]) + assert out is not None + assert out["risk_level"] == "high" + assert "credential_leak" in out["flags"] + assert out["tier"] == "heuristic" # failed judge → no LLM badge + + def test_llm_clear_annotates_heuristic_on_replay(self, storage_backend) -> None: + """A successful LLM "none" on a heuristic positive surfaces the + heuristic finding on reconnect, annotated with the judge's dissent.""" + ws_id, call_id = "ws-merge-clear", "call-doc" + self._record( + storage_backend, + assessment_id="b-h", + ws_id=ws_id, + call_id=call_id, + func_name="web_fetch", + flags=json.dumps(["camouflaged_injection"]), + risk_level="medium", + tier="heuristic", + ) + self._record( + storage_backend, + assessment_id="b-l", + ws_id=ws_id, + call_id=call_id, + func_name="web_fetch", + risk_level="none", + tier="llm", + reasoning="Benign analyst commentary.", + confidence=0.9, + judge_model="gpt-5-mini", + ) + _verdicts, assessments = load_verdict_indexes(ws_id) + out = build_merged_output_assessment_payload(assessments[call_id]) + assert out is not None + assert out["risk_level"] == "medium" # heuristic survives + assert out["tier"] == "llm" + assert out["judge_risk"] == "none" + assert out["reasoning"] == "Benign analyst commentary." diff --git a/tests/test_output_guard.py b/tests/test_output_guard.py index 6accd1df..b267fa33 100644 --- a/tests/test_output_guard.py +++ b/tests/test_output_guard.py @@ -2,7 +2,7 @@ from __future__ import annotations -from turnstone.core.output_guard import evaluate_output +from turnstone.core.output_guard import evaluate_output, merge_guard_display_payload class TestBenignOutput: @@ -401,3 +401,101 @@ class TestBudget: assert r.risk_level in ("none", "low", "medium", "high", "critical") # Confirm the deadline path was actually exercised assert call_count >= 2 + + +class TestMergeGuardDisplayPayload: + """The single chip-payload projection shared by the live and replay + paths (issue #560, "show, annotated"). Rule: risk = max(heuristic, llm), + flags = union; an LLM negative/absent never lowers a heuristic positive.""" + + def test_clean_both_returns_none(self) -> None: + assert ( + merge_guard_display_payload( + heuristic_risk="none", heuristic_flags=[], redacted=False, llm_succeeded=False + ) + is None + ) + + def test_redaction_alone_surfaces_even_at_none_risk(self) -> None: + out = merge_guard_display_payload( + heuristic_risk="none", heuristic_flags=[], redacted=True, llm_succeeded=False + ) + assert out is not None + assert out["redacted"] is True + assert out["tier"] == "heuristic" + + def test_llm_escalates_over_clean_heuristic(self) -> None: + out = merge_guard_display_payload( + heuristic_risk="none", + heuristic_flags=[], + redacted=False, + llm_succeeded=True, + llm_risk="high", + llm_flags=["prompt_injection"], + llm_reasoning="Overt override attempt.", + llm_confidence=0.95, + llm_model="gpt-5-mini", + ) + assert out is not None + assert out["risk_level"] == "high" + assert out["flags"] == ["prompt_injection"] + assert out["tier"] == "llm" + assert out["judge_risk"] == "high" + assert out["confidence"] == 0.95 + + def test_llm_none_never_lowers_heuristic(self) -> None: + """The core fix: an LLM "none" leaves the heuristic positive intact, + annotated with the judge's dissenting verdict.""" + out = merge_guard_display_payload( + heuristic_risk="high", + heuristic_flags=["credential_leak"], + redacted=True, + llm_succeeded=True, + llm_risk="none", + llm_flags=[], + llm_reasoning="No injection detected.", + llm_confidence=0.9, + llm_model="gpt-5-mini", + ) + assert out is not None + assert out["risk_level"] == "high" # heuristic survives + assert out["flags"] == ["credential_leak"] + assert out["tier"] == "llm" + assert out["judge_risk"] == "none" # dissent, for the badge + assert out["redacted"] is True + + def test_failed_or_absent_llm_is_heuristic_only(self) -> None: + out = merge_guard_display_payload( + heuristic_risk="medium", + heuristic_flags=["camouflaged_injection"], + redacted=False, + llm_succeeded=False, + ) + assert out is not None + assert out["risk_level"] == "medium" + assert out["tier"] == "heuristic" + assert "judge_risk" not in out + assert "confidence" not in out + assert "reasoning" not in out + + def test_heuristic_annotations_ride_through(self) -> None: + """The heuristic's human-readable messages surface as `annotations` + (the only prose a regex-only finding carries); omitted when empty.""" + out = merge_guard_display_payload( + heuristic_risk="high", + heuristic_flags=["credential_leak"], + heuristic_annotations=["Output contains a PEM-encoded private key block."], + redacted=True, + llm_succeeded=False, + ) + assert out is not None + assert out["annotations"] == ["Output contains a PEM-encoded private key block."] + # No heuristic annotations → key omitted (SDK defaults to []). + bare = merge_guard_display_payload( + heuristic_risk="high", + heuristic_flags=["credential_leak"], + redacted=True, + llm_succeeded=False, + ) + assert bare is not None + assert "annotations" not in bare diff --git a/tests/test_sdk_events.py b/tests/test_sdk_events.py index 7c06b5b3..ac8e2eb8 100644 --- a/tests/test_sdk_events.py +++ b/tests/test_sdk_events.py @@ -17,6 +17,7 @@ from turnstone.sdk.events import ( InfoEvent, NodeJoinedEvent, NodeLostEvent, + OutputWarningEvent, PlanResolvedEvent, PlanReviewEvent, ReasoningEvent, @@ -119,6 +120,71 @@ def test_tool_output_chunk_event(): assert e.chunk == "line1\n" +def test_output_warning_event_llm_tier(): + """LLM-tier finding carries confidence + reasoning + judge_model so SDK + consumers see the same attribution the UI chip renders.""" + e = ServerEvent.from_dict( + { + "type": "output_warning", + "call_id": "c1", + "func_name": "web_fetch", + "risk_level": "medium", + "flags": ["camouflaged_injection"], + "redacted": False, + "tier": "llm", + "judge_risk": "none", + "confidence": 0.82, + "reasoning": "Authority-framed directive embedded in the doc.", + "judge_model": "gpt-5-mini", + } + ) + assert isinstance(e, OutputWarningEvent) + assert e.tier == "llm" + assert e.judge_risk == "none" # the judge's OWN verdict (may differ from risk_level) + assert e.confidence == 0.82 + assert e.flags == ["camouflaged_injection"] + assert e.reasoning == "Authority-framed directive embedded in the doc." + assert e.judge_model == "gpt-5-mini" + + +def test_output_warning_event_heuristic_defaults(): + """A regex-only finding defaults tier=heuristic with no confidence.""" + e = ServerEvent.from_dict( + {"type": "output_warning", "call_id": "c1", "risk_level": "high", "redacted": True} + ) + assert isinstance(e, OutputWarningEvent) + assert e.tier == "heuristic" + assert e.confidence == 0.0 + assert e.redacted is True + + +def test_output_warning_event_covers_every_merge_payload_key(): + """Drift guard: every key the server-side merge can emit must be a declared + OutputWarningEvent field, else from_dict silently drops it (the bug that let + `annotations` go stale). Builds the maximal payload and checks the field-set.""" + import dataclasses + + from turnstone.core.output_guard import merge_guard_display_payload + + # Maximal payload — every optional field populated. + payload = merge_guard_display_payload( + heuristic_risk="high", + heuristic_flags=["credential_leak"], + heuristic_annotations=["API key detected."], + redacted=True, + llm_succeeded=True, + llm_risk="none", + llm_flags=["camouflaged_injection"], + llm_reasoning="Benign.", + llm_confidence=0.9, + llm_model="gpt-5-mini", + ) + assert payload is not None + declared = {f.name for f in dataclasses.fields(OutputWarningEvent)} + missing = set(payload) - declared + assert not missing, f"OutputWarningEvent is missing merge-payload fields: {missing}" + + def test_status_event(): e = ServerEvent.from_dict( { diff --git a/tests/test_session.py b/tests/test_session.py index f0750bfd..fc445077 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -2023,24 +2023,33 @@ class TestEvaluateOutputLLMStage: assert assessment.risk_level == "medium" assert "camouflaged_injection" in assessment.flags - # Heuristic row persisted with its verdict; llm row persisted with - # the error reason so audit can distinguish failure-from-disabled. + # Heuristic row persisted with its verdict; the FAILURE row rides the + # distinct "llm_error" tier (not "llm") so audit can tell + # failure-from-disabled AND the replay merge treats it as absent — + # a risk="none" failure row must never shadow the heuristic finding. tiers = [r["tier"] for r in records] assert tiers.count("heuristic") == 1 - assert tiers.count("llm") == 1 - llm_row = next(r for r in records if r["tier"] == "llm") - assert llm_row["reasoning"] == "timeout" - assert llm_row["judge_model"] == "gpt-5-mini" - assert llm_row["risk_level"] == "none" - assert llm_row["flags"] == [] + assert tiers.count("llm_error") == 1 + assert "llm" not in tiers # no successful-verdict row was written + err_row = next(r for r in records if r["tier"] == "llm_error") + assert err_row["reasoning"] == "timeout" + assert err_row["judge_model"] == "gpt-5-mini" + assert err_row["risk_level"] == "none" + assert err_row["flags"] == [] - def test_llm_enabled_can_de_escalate_clean(self) -> None: - """LLM saying 'none' on regex-flagged content wins — heuristic was a false positive.""" + def test_llm_clear_annotates_does_not_suppress(self) -> None: + """A successful LLM "none" on a regex-flagged output does NOT suppress + the heuristic finding (issue #560, "show, annotated"): merged risk = + max, so the finding survives and the judge's "benign" verdict rides + along as annotation. An LLM negative never lowers a heuristic + positive — the judge reads adversarial output and may escalate but + must not be able to hide a deterministic regex hit. + """ from turnstone.core.output_guard_judge import OutputJudgeVerdict session, records = self._make_session_with_recording_ui(llm_enabled=True) - # Heuristic would flag this (recommendation + caps action), but it's - # legitimate sell-side analyst commentary. + # Heuristic flags this (recommendation + caps action SELL), but the + # judge assesses it as legitimate sell-side analyst commentary. legit = ( "The recommended action consistent with our research framework " "is SELL based on this quarter's revenue miss." @@ -2059,12 +2068,18 @@ class TestEvaluateOutputLLMStage: with patch.object(session, "_ensure_output_guard_judge", return_value=mock_judge): out, assessment = session._evaluate_output("call-1", legit, "web_fetch") - # Acted = "none" so the call returns None (skips the warning). - assert assessment is None - # But both tier rows are still persisted for audit completeness. + # The heuristic finding SURVIVES (no silent de-escalation) — merged + # risk is the heuristic's medium, not the LLM's "none". + assert assessment is not None + assert assessment.risk_level == "medium" + assert "camouflaged_injection" in assessment.flags + # Both tier rows persisted; the LLM row carries its own "none" verdict. tiers = [r["tier"] for r in records] assert "heuristic" in tiers assert "llm" in tiers + llm_row = next(r for r in records if r["tier"] == "llm") + assert llm_row["risk_level"] == "none" + assert llm_row["reasoning"] == "Legitimate financial commentary, no injection." def test_llm_evaluate_raises_falls_back(self) -> None: """If the judge's evaluate raises, the heuristic still acts (no crash).""" @@ -2148,6 +2163,153 @@ class TestEvaluateOutputLLMStage: llm_rows = [r for r in records if r["tier"] == "llm"] assert llm_rows == [] + def test_llm_judge_runs_on_heuristic_clean_output(self) -> None: + """Issue #560 regression: the LLM judge runs on EVERY output, not + just regex-flagged ones. A heuristic-clean tool result must still + reach ``OutputGuardJudge.evaluate`` so the camouflaged payloads the + regex set misses get a semantic pass. Guards against re-introducing + an 'only judge what the heuristic flagged' gate. + """ + from turnstone.core.output_guard_judge import OutputJudgeVerdict + + session, records = self._make_session_with_recording_ui(llm_enabled=True) + # Plain build output — the regex stage finds nothing here. + clean = "Build succeeded. 42 tests passed in 3.2s." + + mock_judge = MagicMock() + mock_judge.evaluate.return_value = OutputJudgeVerdict( + verdict_id="v1", + call_id="call-1", + risk_level="none", + confidence=0.95, + judge_model="gpt-5-mini", + latency_ms=40, + ) + with patch.object(session, "_ensure_output_guard_judge", return_value=mock_judge): + session._evaluate_output("call-1", clean, "bash") + + # The judge was invoked exactly once despite a clean heuristic verdict. + assert mock_judge.evaluate.call_count == 1 + # An llm-tier row is persisted even though no heuristic row is + # (skip-on-clean): the audit-trail proof that the judge sees every + # output, flagged or not. + assert [r["tier"] for r in records] == ["llm"] + + def _make_session_capturing_warnings( + self, *, llm_enabled: bool + ) -> tuple[ChatSession, list[dict[str, object]]]: + """Build a ChatSession whose UI captures every on_output_warning dict.""" + from turnstone.core.judge import JudgeConfig + + warnings: list[dict[str, object]] = [] + + class _WarnUI(NullUI): + def on_output_warning(self, call_id, assessment): + warnings.append({"call_id": call_id, **assessment}) + + session = _make_session( + judge_config=JudgeConfig(output_guard=True, output_guard_llm=llm_enabled), + ui=_WarnUI(), + ) + return session, warnings + + def test_output_warning_carries_llm_attribution(self) -> None: + """When the LLM judge owns the finding, the live on_output_warning + dict carries tier='llm' + confidence + reasoning + judge_model so the + inline chip can annotate the finding and show how certain the judge + was. Must match build_merged_output_assessment_payload's replay shape. + """ + from turnstone.core.output_guard_judge import OutputJudgeVerdict + + session, warnings = self._make_session_capturing_warnings(llm_enabled=True) + clean_text = "The build completed in 3.2 seconds with no warnings." + + mock_judge = MagicMock() + mock_judge.evaluate.return_value = OutputJudgeVerdict( + verdict_id="v1", + call_id="call-1", + risk_level="medium", + flags=("semantic_injection",), + reasoning="Subtle directive embedded in build output.", + confidence=0.77, + judge_model="gpt-5-mini", + latency_ms=120, + ) + with patch.object(session, "_ensure_output_guard_judge", return_value=mock_judge): + session._evaluate_output("call-1", clean_text, "bash") + + assert len(warnings) == 1 + w = warnings[0] + assert w["tier"] == "llm" + assert w["confidence"] == 0.77 + assert w["reasoning"] == "Subtle directive embedded in build output." + assert w["judge_model"] == "gpt-5-mini" + assert w["risk_level"] == "medium" + assert "semantic_injection" in w["flags"] + + def test_output_warning_heuristic_tier_when_llm_disabled(self) -> None: + """A regex-only finding marks tier='heuristic' and omits the LLM + confidence/reasoning/judge_model fields — the chip stays a bare + regex finding with no fabricated confidence number. + """ + session, warnings = self._make_session_capturing_warnings(llm_enabled=False) + camo = ( + "Given recent volatility, the appropriate recommendation consistent " + "with our risk framework is SELL pending Q4 review." + ) + session._evaluate_output("call-1", camo, "web_fetch") + + assert len(warnings) == 1 + w = warnings[0] + assert w["tier"] == "heuristic" + assert "confidence" not in w + assert "reasoning" not in w + assert "judge_model" not in w + assert w["risk_level"] == "medium" + + def test_output_warning_credential_redaction_keeps_llm_attribution(self) -> None: + """Edge case guarded by the _evaluate_output comment: when the + heuristic redacts a credential (acted=heuristic, regex owns the + flags) but the LLM judge also ran and succeeded, the live warning + dict still marks tier='llm' and carries the model's confidence / + reasoning / judge_model — while flags stay the heuristic's + credential_leak. Pins the attribution semantics so a future + 'make tier follow the flags' source' refactor can't silently + change what the chip shows. + """ + from turnstone.core.output_guard_judge import OutputJudgeVerdict + + session, warnings = self._make_session_capturing_warnings(llm_enabled=True) + with_secret = ( + "Configuration loaded. OPENAI_API_KEY=sk-proj-aaaaaaaaaaaaaaaaaaaa123456 now in use." + ) + + mock_judge = MagicMock() + mock_judge.evaluate.return_value = OutputJudgeVerdict( + verdict_id="v1", + call_id="call-1", + risk_level="none", # LLM sees no prompt-injection + reasoning="Looks like a legitimate config dump; no injection.", + confidence=0.91, # explicit non-default so the assert isn't vacuous + judge_model="gpt-5-mini", + latency_ms=70, + ) + with patch.object(session, "_ensure_output_guard_judge", return_value=mock_judge): + session._evaluate_output("call-1", with_secret, "bash") + + assert len(warnings) == 1 + w = warnings[0] + # Tier + confidence + reasoning attributed to the LLM (it ran)... + assert w["tier"] == "llm" + assert w["confidence"] == 0.91 + assert w["judge_model"] == "gpt-5-mini" + assert w["reasoning"] == "Looks like a legitimate config dump; no injection." + # ...but the acted flags/risk stay the heuristic's credential finding, + # because regex credential redaction wins over the LLM's "none". + assert "credential_leak" in w["flags"] + assert w["risk_level"] == "high" + assert w["redacted"] is True + class TestBatchEvaluateOutputs: """Concurrent guard pre-pass for the per-tool-result loop (perf-2).""" diff --git a/turnstone/api/console_schemas.py b/turnstone/api/console_schemas.py index 68617ab7..e811da68 100644 --- a/turnstone/api/console_schemas.py +++ b/turnstone/api/console_schemas.py @@ -602,7 +602,16 @@ class ListVerdictsResponse(BaseModel): class OutputAssessmentInfo(BaseModel): - """Output guard assessment.""" + """Output guard assessment (one row per ``(call_id, tier)``). + + ``tier`` is one of ``"heuristic"`` (regex stage), ``"llm"`` (the judge's + own successful verdict), or ``"llm_error"`` (the judge ran but failed — + audit-only; ``reasoning`` carries the error). ``reasoning`` / + ``judge_model`` / ``latency_ms`` / ``confidence`` are populated on the + LLM tiers (migration 057) and carry their defaults on heuristic rows. + The inline UI chip MERGES the heuristic + ``llm`` rows; this list + endpoint exposes the raw rows for audit/calibration. + """ assessment_id: str ws_id: str @@ -613,6 +622,11 @@ class OutputAssessmentInfo(BaseModel): annotations: str = "[]" output_length: int = 0 redacted: int = 0 + tier: str = "heuristic" + reasoning: str = "" + judge_model: str = "" + latency_ms: int = 0 + confidence: float = 0.0 created: str diff --git a/turnstone/console/static/coordinator/coordinator.css b/turnstone/console/static/coordinator/coordinator.css index e93ac1b6..fb6739c0 100644 --- a/turnstone/console/static/coordinator/coordinator.css +++ b/turnstone/console/static/coordinator/coordinator.css @@ -438,6 +438,14 @@ color: var(--ink-3); font-size: 10px; } +/* LLM-judge attribution badge inside the warning chip — flex `gap` + handles spacing from the flag list; weight + dimmer ink mark it as + metadata rather than another flag. */ +.coord-tool-row-warning-tier { + font-weight: 600; + font-size: 10px; + opacity: 0.85; +} /* memory/recall calls are background metadata — the audit trail is useful but they crowd the tree on workstreams with heavy memory diff --git a/turnstone/console/static/coordinator/coordinator.js b/turnstone/console/static/coordinator/coordinator.js index 4cdbca97..dedfa006 100644 --- a/turnstone/console/static/coordinator/coordinator.js +++ b/turnstone/console/static/coordinator/coordinator.js @@ -862,7 +862,45 @@ redacted.textContent = " (credentials redacted)"; chip.appendChild(redacted); } + // LLM-judge attribution — mirrors the intent-verdict chip's tier + // badge so the operator can tell a regex match from a model + // judgement and read the model's confidence inline. + if (oa.tier === "llm") { + const tier = document.createElement("span"); + tier.className = "coord-tool-row-warning-tier"; + let t = "⚖ LLM"; + // Surface the judge's own verdict when it differs from the displayed + // (merged) risk — e.g. regex MEDIUM but the judge cleared it as none. + if (oa.judge_risk && oa.judge_risk !== risk) t += ": " + oa.judge_risk; + if (oa.confidence > 0) { + t += " · " + Math.round(oa.confidence * 100) + "%"; + } + if (oa.judge_model) t += " · " + oa.judge_model; + tier.textContent = t; + chip.appendChild(tier); + } if (!existing) row.appendChild(chip); + + // Rationale — collapsible disclosure mirroring the verdict rationale + // (reuses .coord-tool-row-rationale styling) so the judge's reasoning + // is inspectable without bloating the tree. Idempotent: drop any prior + // warning rationale before re-adding so a late SSE upgrade doesn't + // stack duplicates next to the one seeded on replay. + const oldRationale = row.querySelector(".coord-tool-row-warning-rationale"); + if (oldRationale) oldRationale.remove(); + if (oa.reasoning) { + const det = document.createElement("details"); + det.className = + "coord-tool-row-rationale coord-tool-row-warning-rationale"; + const sum = document.createElement("summary"); + sum.textContent = "guard rationale"; + det.appendChild(sum); + const body = document.createElement("div"); + body.className = "coord-tool-row-rationale-body"; + body.textContent = oa.reasoning; + det.appendChild(body); + chip.insertAdjacentElement("afterend", det); + } } // Stable signature for a verdict — used to skip the DOM rebuild when @@ -2254,6 +2292,11 @@ risk_level: ev.risk_level, flags: ev.flags, redacted: ev.redacted, + tier: ev.tier, + judge_risk: ev.judge_risk, + confidence: ev.confidence, + reasoning: ev.reasoning, + judge_model: ev.judge_model, }); } else { appendText( diff --git a/turnstone/core/history_decoration.py b/turnstone/core/history_decoration.py index dc0a5324..261199ed 100644 --- a/turnstone/core/history_decoration.py +++ b/turnstone/core/history_decoration.py @@ -41,10 +41,19 @@ def load_verdict_indexes( ) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: """Bulk-load intent verdicts and output assessments for a workstream. - Returns ``(verdicts_by_call_id, assessments_by_call_id)``. Both - tables are indexed by ws_id so the queries are O(rows-for-ws); the - DESC ordering plus first-seen-wins dedupe leaves the newest - verdict per call_id (LLM upgrade beats heuristic when both exist). + Returns ``(verdicts_by_call_id, assessments_by_call_id)``. + + ``verdicts_by_call_id`` keeps the newest intent verdict per call_id + (DESC ordering + first-seen-wins; LLM upgrade beats heuristic). + + ``assessments_by_call_id`` maps call_id → a SLOT + ``{"heuristic": row|None, "llm": row|None}`` holding the newest row of + each tier, because the output-guard chip is a MERGE of the two + detectors (issue #560, "show, annotated"). Failed-judge rows + (``tier="llm_error"``) are dropped here so a risk="none" failure row + can never shadow a real heuristic finding on reconnect — they are + audit-only and surface via the ``/v1/api/admin/output-assessments`` + list, not the inline chip. Pure storage I/O — safe to run in ``asyncio.to_thread`` from an async caller. Returns empty dicts when storage is unavailable or @@ -67,8 +76,24 @@ def load_verdict_indexes( verdicts_by_call_id[cid] = v for a in storage.list_output_assessments(ws_id=ws_id, limit=10000): cid = a.get("call_id") or "" - if cid and cid not in assessments_by_call_id: - assessments_by_call_id[cid] = a + if not cid: + continue + tier = a.get("tier", "heuristic") + if tier == "llm_error": + continue # audit-only failure row — never the acted UI finding + # KNOWN LIMITATION (historical data only): rows written BEFORE the + # llm_error split recorded judge FAILURES as tier="llm" risk="none" + # too, so on replay of pre-split workstreams such a row lands in the + # "llm" slot and the chip mis-renders it as a successful "none" + # verdict (spurious "⚖ LLM: none" + the error string as rationale). + # The heuristic finding still SURVIVES the max-merge — the chip + # never vanishes — so this is cosmetic. We can't fingerprint it + # safely (a legitimate benign verdict is also tier="llm" risk="none"), + # and new failures self-heal under "llm_error". + slot = assessments_by_call_id.setdefault(cid, {"heuristic": None, "llm": None}) + key = "llm" if tier == "llm" else "heuristic" + if slot.get(key) is None: # first-seen wins (rows arrive newest-first) + slot[key] = a except Exception: # Missing storage / migration drift / driver error must not # block replay — degrade to an unannotated history. @@ -113,29 +138,51 @@ def build_verdict_payload(vrow: dict[str, Any]) -> dict[str, Any] | None: return payload -def build_output_assessment_payload(arow: dict[str, Any]) -> dict[str, Any] | None: - """Project a stored ``output_assessments`` row into the wire shape. +def _decode_json_list(raw: Any) -> list[str]: + """Decode a stored JSON-string list (``flags`` / ``annotations``) into a list. - Returns ``None`` when the assessment is the unflagged baseline - (``risk_level == "none"``) — same skip-on-clean pattern as - :func:`build_verdict_payload`. - - Decodes ``flags`` from its JSON string form here so the client - never has to parse twice. Falls back to an empty list on bad JSON - rather than raising — the rest of the assessment is still useful. + Falls back to an empty list on bad JSON rather than raising — the rest + of the assessment is still useful. """ - if (arow.get("risk_level") or "none") == "none": - return None - flags_raw = arow.get("flags") or "[]" + if raw is None: + return [] try: - flags = json.loads(flags_raw) if isinstance(flags_raw, str) else flags_raw + decoded = json.loads(raw) if isinstance(raw, str) else raw except (ValueError, TypeError): - flags = [] - return { - "risk_level": arow.get("risk_level", "none"), - "flags": flags if isinstance(flags, list) else [], - "redacted": bool(arow.get("redacted", 0)), - } + return [] + return decoded if isinstance(decoded, list) else [] + + +def build_merged_output_assessment_payload(slot: dict[str, Any]) -> dict[str, Any] | None: + """Project a stored heuristic+LLM assessment pair into the chip payload. + + ``slot`` is ``{"heuristic": row|None, "llm": row|None}`` from + :func:`load_verdict_indexes` — the ``llm`` slot holds the judge's OWN + successful verdict; failed-judge rows were dropped upstream so they + cannot shadow a heuristic finding. + + Delegates the actual merge to + :func:`output_guard.merge_guard_display_payload` — the SAME projection + the live ``on_output_warning`` path calls — so the inline finding chip + renders identically live and on reconnect. Returns ``None`` (skip on + clean) when neither detector flagged and nothing was redacted. + """ + from turnstone.core.output_guard import merge_guard_display_payload + + heuristic = slot.get("heuristic") or {} + llm = slot.get("llm") + return merge_guard_display_payload( + heuristic_risk=heuristic.get("risk_level", "none") or "none", + heuristic_flags=_decode_json_list(heuristic.get("flags")), + heuristic_annotations=_decode_json_list(heuristic.get("annotations")), + redacted=bool(heuristic.get("redacted", 0)), + llm_succeeded=llm is not None, + llm_risk=(llm or {}).get("risk_level", "none") or "none", + llm_flags=_decode_json_list((llm or {}).get("flags")), + llm_reasoning=(llm or {}).get("reasoning", "") or "", + llm_confidence=(llm or {}).get("confidence", 0.0) or 0.0, + llm_model=(llm or {}).get("judge_model", "") or "", + ) def decorate_tool_call( @@ -160,9 +207,9 @@ def decorate_tool_call( verdict = build_verdict_payload(vrow) if verdict is not None: tc["verdict"] = verdict - arow = assessments_by_call_id.get(call_id) - if arow is not None: - assessment = build_output_assessment_payload(arow) + slot = assessments_by_call_id.get(call_id) + if slot is not None: + assessment = build_merged_output_assessment_payload(slot) if assessment is not None: tc["output_assessment"] = assessment diff --git a/turnstone/core/output_guard.py b/turnstone/core/output_guard.py index ddeac393..d1564f22 100644 --- a/turnstone/core/output_guard.py +++ b/turnstone/core/output_guard.py @@ -211,6 +211,78 @@ def _clean() -> OutputAssessment: return OutputAssessment() +def merge_guard_display_payload( + *, + heuristic_risk: str, + heuristic_flags: list[str] | tuple[str, ...], + redacted: bool, + llm_succeeded: bool, + heuristic_annotations: list[str] | tuple[str, ...] = (), + llm_risk: str = "none", + llm_flags: list[str] | tuple[str, ...] = (), + llm_reasoning: str = "", + llm_confidence: float = 0.0, + llm_model: str = "", +) -> dict[str, Any] | None: + """Merge the heuristic + LLM-judge findings into the single chip payload. + + This is the ONE projection both the live path + (``ChatSession._evaluate_output`` → ``on_output_warning``) and the + replay path (``history_decoration``) call, so the inline finding chip + renders identically live and on reconnect — the wire-shape parity the + output guard needs. + + Merge rule (issue #560, "show, annotated"): + + * ``risk_level`` = max(heuristic, llm) — a positive from EITHER detector + surfaces. A negative ("none") or failed/absent LLM never *lowers* a + heuristic positive: the judge reads adversarial tool output, so it may + raise an alarm but must not be able to suppress a deterministic regex + finding. Credential redaction is a heuristic-only signal handled by + the caller via ``redacted`` and is likewise never overridden. + * ``flags`` = union(heuristic, llm). + * ``annotations`` = the heuristic's human-readable messages (the only + prose a regex-only finding carries — the LLM's prose lives in the + separate ``reasoning`` field). Emitted only when non-empty. + * When the LLM produced a verdict (``llm_succeeded``) its own risk + (``judge_risk``), confidence, reasoning, and model ride along as + annotation under ``tier="llm"`` — so the operator sees the judge's + opinion even when it disagrees with the displayed (merged) risk. A + failed / disabled LLM leaves ``tier="heuristic"`` and no badge. + + Returns ``None`` when there is nothing to show — merged risk ``"none"`` + and no redaction — the skip-on-clean contract both surfaces rely on. + """ + risk = _max_risk(heuristic_risk or "none", llm_risk if llm_succeeded else "none") + flags: list[str] = [] + candidate_flags = list(heuristic_flags) + (list(llm_flags) if llm_succeeded else []) + for f in candidate_flags: + if f and f not in flags: + flags.append(f) + + if risk == "none" and not redacted: + return None + + payload: dict[str, Any] = { + "risk_level": risk, + "flags": flags, + "redacted": bool(redacted), + } + if heuristic_annotations: + payload["annotations"] = list(heuristic_annotations) + if llm_succeeded: + payload["tier"] = "llm" + payload["judge_risk"] = llm_risk + payload["confidence"] = llm_confidence + if llm_reasoning: + payload["reasoning"] = llm_reasoning + if llm_model: + payload["judge_model"] = llm_model + else: + payload["tier"] = "heuristic" + return payload + + @dataclass(frozen=True) class OutputGuardPatternDef: """A pattern definition for output guard scanning.""" diff --git a/turnstone/core/session.py b/turnstone/core/session.py index aed237dc..27f6e041 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -5187,7 +5187,11 @@ class ChatSession: Returns ``(possibly_sanitized_output, acted_assessment)``. The acted assessment is ``None`` when its risk_level is ``"none"``. """ - from turnstone.core.output_guard import OutputAssessment, evaluate_output + from turnstone.core.output_guard import ( + OutputAssessment, + evaluate_output, + merge_guard_display_payload, + ) og_patterns = None rule_reg = self._rule_registry @@ -5223,44 +5227,30 @@ class ChatSession: output_len = len(output) - # Acted assessment. Credential redaction is a regex-only signal - # that the LLM cannot override (bug-1 / sec-1 from the PR review): - # an LLM asked about prompt-injection can correctly label a - # credential-bearing tool output as "none" risk for injection, but - # the secret still needs to be redacted before it lands in the - # assistant's context. When the heuristic populated ``sanitized`` - # we keep the heuristic's verdict as acted regardless of the LLM. - if heuristic.sanitized is not None: - acted = heuristic - elif llm_verdict is not None and llm_verdict.succeeded: - acted = OutputAssessment( - flags=list(llm_verdict.flags), - risk_level=llm_verdict.risk_level, - annotations=[llm_verdict.reasoning] if llm_verdict.reasoning else [], - sanitized=heuristic.sanitized, # always None on this branch; explicit - ) - else: - acted = heuristic + # Merge the two detectors into one acted finding (issue #560, + # "show, annotated"). Risk = max(heuristic, llm); flags = union. + # An LLM "none" — or a failed/absent LLM — never LOWERS a heuristic + # positive: the judge evaluates adversarial tool output, so it may + # escalate but must not be able to hide a deterministic regex + # finding. Credential redaction stays a heuristic-only signal the + # LLM cannot override (bug-1 / sec-1): a secret is redacted whether + # or not the judge sees injection. + # ``llm`` is the narrowed, succeeded-only verdict (None on + # disable / rate-limit / error / timeout) — lets the type checker + # follow attribute access below without re-asserting succeeded. + llm = llm_verdict if (llm_verdict is not None and llm_verdict.succeeded) else None + wants_redaction = heuristic.sanitized is not None and jc is not None and jc.redact_secrets - # Persistence (single call path now — q-4): - # * Heuristic row: write when it has signal (risk != none OR flags) - # OR when verdicts disagree. Matched-clean evaluations are - # skipped to keep the audit table focused on disagreements - # and non-clean events. This is the perf-4 trade-off — pre-PR - # wrote heuristic rows only on risk != none; we now ALSO record - # them when the LLM ran and the two judges disagreed. - # * LLM row: write whenever the LLM ran, regardless of success. - # On failure (cp-3) the row's ``reasoning`` carries the error - # reason so audit can distinguish "LLM was attempted but - # failed" from "LLM was never enabled". + # Persistence — one row per (call_id, tier). Heuristic row when it + # has signal; the LLM row carries the judge's OWN verdict on success + # so the replay merge can recombine the two. A failed judge is + # recorded under tier="llm_error" (NOT "llm") so audit can tell + # "attempted but failed" from "never enabled" AND the replay merge + # treats it as absent — a risk="none" failure row must never shadow + # a real heuristic finding on reconnect (the vanishing-chip bug). heuristic_has_signal = heuristic.risk_level != "none" or bool(heuristic.flags) - verdicts_disagree = ( - llm_verdict is not None - and llm_verdict.succeeded - and ( - llm_verdict.risk_level != heuristic.risk_level - or set(llm_verdict.flags) != set(heuristic.flags) - ) + verdicts_disagree = llm is not None and ( + llm.risk_level != heuristic.risk_level or set(llm.flags) != set(heuristic.flags) ) if heuristic_has_signal or verdicts_disagree: self._record_output_tier(call_id, func_name, output_len, heuristic, tier="heuristic") @@ -5270,7 +5260,11 @@ class ChatSession: call_id, func_name, output_len, - acted, + OutputAssessment( + flags=list(llm_verdict.flags), + risk_level=llm_verdict.risk_level, + annotations=[llm_verdict.reasoning] if llm_verdict.reasoning else [], + ), tier="llm", reasoning=llm_verdict.reasoning, judge_model=llm_verdict.judge_model, @@ -5278,48 +5272,69 @@ class ChatSession: confidence=llm_verdict.confidence, ) else: - # Failure row — empty assessment + error reason for audit. - # confidence stays 0.0 since the LLM never produced a verdict. + # Failure row — empty assessment + error reason for audit, + # under the distinct "llm_error" tier (see comment above). self._record_output_tier( call_id, func_name, output_len, OutputAssessment(risk_level="none"), - tier="llm", + tier="llm_error", reasoning=llm_verdict.error, judge_model=llm_verdict.judge_model, latency_ms=llm_verdict.latency_ms, ) - # Redaction — fires whenever the heuristic detected secrets and - # the operator wants them redacted, regardless of acted.risk_level. - # This is the bug-1 fix: the prior code's ``risk_level == "none"`` - # short-circuit returned the un-redacted output when the LLM - # downgraded a credential-bearing result to "none". - wants_redaction = heuristic.sanitized is not None and jc is not None and jc.redact_secrets - - if acted.risk_level == "none" and not wants_redaction: + # Chip payload — built through the SAME merge the replay path uses + # (build_merged_output_assessment_payload) so the live SSE chip and + # the reconnect chip render identically. ``None`` means nothing to + # show (merged risk "none", no redaction). + d = merge_guard_display_payload( + heuristic_risk=heuristic.risk_level, + heuristic_flags=list(heuristic.flags), + heuristic_annotations=list(heuristic.annotations), + redacted=wants_redaction, + llm_succeeded=llm is not None, + llm_risk=llm.risk_level if llm else "none", + llm_flags=list(llm.flags) if llm else [], + llm_reasoning=llm.reasoning if llm else "", + llm_confidence=llm.confidence if llm else 0.0, + llm_model=llm.judge_model if llm else "", + ) + if d is None: return output, None + # Context-facing annotations (what the MODEL sees via GuardAdvisory): + # the heuristic findings, plus the LLM's reasoning ONLY when the LLM + # ESCALATED (flagged something itself). We deliberately never inject + # the judge's "benign" reasoning into the model context — a judge + # fooled into "none" on a real heuristic finding must not get to tell + # the model the output is safe. The operator UI still shows the full + # LLM verdict via the chip payload below. + context_annotations = list(heuristic.annotations) + if llm is not None and llm.risk_level != "none" and llm.reasoning: + context_annotations.append(llm.reasoning) + # acted's risk/flags come straight from the merge payload so the + # context advisory can't drift from the chip. + acted = OutputAssessment( + flags=list(d["flags"]), + risk_level=str(d["risk_level"]), + annotations=context_annotations, + sanitized=heuristic.sanitized, + ) + + d["func_name"] = func_name + d["output_length"] = output_len log.debug( "output_guard.flagged", call_id=call_id, func_name=func_name, - risk=acted.risk_level, - flags=acted.flags, - tier="llm" if (llm_verdict is not None and llm_verdict.succeeded) else "heuristic", + risk=d["risk_level"], + flags=d["flags"], + tier=d["tier"], redacted=wants_redaction, ) try: - d = acted.to_dict() # excludes sanitized by default - d["func_name"] = func_name - d["output_length"] = output_len - d["redacted"] = wants_redaction - # Surface the LLM's confidence when the LLM stage was the one - # that produced ``acted`` — the operator/UI can sort flagged - # outputs by how certain the judge was. - if llm_verdict is not None and llm_verdict.succeeded and acted is not heuristic: - d["confidence"] = llm_verdict.confidence self.ui.on_output_warning(call_id, d) except Exception: log.debug("output_guard.callback_failed", exc_info=True) diff --git a/turnstone/core/session_ui_base.py b/turnstone/core/session_ui_base.py index d0d868d5..21a0541e 100644 --- a/turnstone/core/session_ui_base.py +++ b/turnstone/core/session_ui_base.py @@ -1643,12 +1643,14 @@ class SessionUIBase: Called by the session once per tier. A ``"heuristic"`` row is written when the regex stage produced signal (risk!="none" or - flags) OR when the heuristic and LLM verdicts disagreed. An - ``"llm"`` row is written whenever the LLM stage ran — on - success ``reasoning`` carries the model's explanation; on - failure (timeout / parse error / provider error) ``reasoning`` - carries the error reason so audit can distinguish "LLM - attempted but failed" from "LLM was never enabled". + flags) OR when the heuristic and LLM verdicts disagreed. When the + LLM stage ran it writes one row: ``"llm"`` on success (``reasoning`` + carries the model's explanation), or ``"llm_error"`` on failure + (timeout / parse error / provider error — ``reasoning`` carries the + error reason). The distinct ``"llm_error"`` tier lets audit tell + "LLM attempted but failed" from "LLM was never enabled" AND keeps the + replay merge from treating a risk="none" failure row as a verdict + that could shadow the heuristic finding. """ try: from turnstone.core.storage._registry import get_storage diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 6f075672..2e35b083 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -1825,8 +1825,10 @@ class StorageBackend(Protocol): ) -> None: """Record an output guard assessment. - ``tier`` is ``"heuristic"`` (regex stage, default) or ``"llm"`` - (capability-gated semantic evaluator, issue #560 mitigation #1). + ``tier`` is ``"heuristic"`` (regex stage, default), ``"llm"`` (the + judge's own successful verdict, issue #560 mitigation #1), or + ``"llm_error"`` (the judge ran but failed — audit-only, excluded + from the replay display merge; ``reasoning`` carries the error). One row per ``(call_id, tier)`` so a single tool call can produce up to two rows; mirrors the ``intent_verdicts`` table's row model. ``reasoning`` / ``judge_model`` / ``latency_ms`` / ``confidence`` diff --git a/turnstone/core/storage/migrations/versions/057_output_assessments_llm_judge.py b/turnstone/core/storage/migrations/versions/057_output_assessments_llm_judge.py index f0fcd065..28568b5d 100644 --- a/turnstone/core/storage/migrations/versions/057_output_assessments_llm_judge.py +++ b/turnstone/core/storage/migrations/versions/057_output_assessments_llm_judge.py @@ -4,11 +4,12 @@ Adds five columns to ``output_assessments`` so the same table holds both heuristic (regex) verdicts and the new LLM-judge verdicts introduced for issue #560 mitigation #1: -* ``tier`` — ``'heuristic'`` (the regex stage) or ``'llm'`` (the new - capability-gated semantic evaluator). Existing rows backfill to - ``'heuristic'`` because that is what the table held before this - migration. One row per ``(call_id, tier)`` from this point on, mirroring - the ``intent_verdicts`` table's row model (migration 012). +* ``tier`` — ``'heuristic'`` (the regex stage), ``'llm'`` (the new + capability-gated semantic evaluator's successful verdict), or + ``'llm_error'`` (the evaluator ran but failed — audit-only). Existing + rows backfill to ``'heuristic'`` because that is what the table held + before this migration. One row per ``(call_id, tier)`` from this point + on, mirroring the ``intent_verdicts`` table's row model (migration 012). * ``reasoning`` — the LLM's free-form explanation. Empty for heuristic rows. * ``judge_model`` — the model alias used. Empty for heuristic rows. * ``latency_ms`` — wall-clock cost. ``0`` for heuristic rows (regex is diff --git a/turnstone/sdk/events.py b/turnstone/sdk/events.py index 82cbc4a2..6347c6e9 100644 --- a/turnstone/sdk/events.py +++ b/turnstone/sdk/events.py @@ -234,11 +234,33 @@ class IntentVerdictEvent(ServerEvent): @dataclass class OutputWarningEvent(ServerEvent): + """Output-guard finding for a tool result (Facet 2 of intent validation). + + The chip is a MERGE of the regex heuristic and the LLM judge (issue + #560, "show, annotated"): ``risk_level`` is max(heuristic, llm) and + ``flags`` is their union — a positive from either detector surfaces and + an LLM "none" never lowers a heuristic positive. ``tier`` is ``"llm"`` + when the judge returned a verdict (then ``judge_risk`` / ``confidence`` + / ``reasoning`` / ``judge_model`` carry its OWN opinion, which may + differ from the displayed ``risk_level``), else ``"heuristic"``. + Mirrors ``ChatSession._evaluate_output`` and the replay projection + ``build_merged_output_assessment_payload`` — all three share + ``output_guard.merge_guard_display_payload``. + """ + type: str = "output_warning" call_id: str = "" + func_name: str = "" risk_level: str = "" - categories: list[str] = field(default_factory=list) - explanation: str = "" + flags: list[str] = field(default_factory=list) + annotations: list[str] = field(default_factory=list) + redacted: bool = False + output_length: int = 0 + tier: str = "heuristic" + judge_risk: str = "" + confidence: float = 0.0 + reasoning: str = "" + judge_model: str = "" # --------------------------------------------------------------------------- diff --git a/turnstone/ui/static/app.js b/turnstone/ui/static/app.js index f36b1f91..312d3a82 100644 --- a/turnstone/ui/static/app.js +++ b/turnstone/ui/static/app.js @@ -318,11 +318,19 @@ class Pane { if (!toolDiv) return; // Shared DOM-builder with replayHistory \u2014 single source of truth for // role / class / escape semantics. Argument shape mirrors the - // server-side output_assessment dict (risk_level / flags / redacted). + // server-side output_assessment dict AND the replay payload built by + // build_merged_output_assessment_payload (both via the shared + // merge_guard_display_payload), so the live chip and the refresh chip + // render identically (tier / judge_risk / confidence / reasoning). const warning = _buildOutputWarningEl({ risk_level: evt.risk_level, flags: evt.flags, redacted: evt.redacted, + tier: evt.tier, + judge_risk: evt.judge_risk, + confidence: evt.confidence, + reasoning: evt.reasoning, + judge_model: evt.judge_model, }); const nextEl = toolDiv.nextElementSibling; if (nextEl && nextEl.classList.contains("tool-output")) { @@ -2315,6 +2323,36 @@ function _buildOutputWarningEl(assessment) { redacted.textContent = " (credentials redacted)"; warning.appendChild(redacted); } + // LLM-judge attribution — when the semantic stage owned this finding, + // mark the tier + the model's self-reported confidence so the operator + // can tell a regex match from a model judgement and weight it. Mirrors + // the intent-verdict badge's tier/confidence vocabulary. + if (assessment && assessment.tier === "llm") { + const tierEl = document.createElement("span"); + tierEl.className = "output-warning-tier"; + let t = "⚖ LLM"; + // Show the judge's OWN verdict when it differs from the displayed + // (merged) risk — e.g. regex flagged MEDIUM but the judge said none. + // Same-verdict cases stay terse ("⚖ LLM · 88%"). + if (assessment.judge_risk && assessment.judge_risk !== risk) { + t += ": " + assessment.judge_risk; + } + if (assessment.confidence > 0) { + t += " · " + Math.round(assessment.confidence * 100) + "%"; + } + if (assessment.judge_model) t += " · " + assessment.judge_model; + tierEl.textContent = t; + warning.appendChild(tierEl); + } + // Rationale — the judge's one-line reasoning, surfaced as a muted second + // line so the finding explains itself instead of showing a bare flag + // list. Block element wraps below the header row. + if (assessment && assessment.reasoning) { + const reasonEl = document.createElement("div"); + reasonEl.className = "output-warning-reasoning"; + reasonEl.textContent = assessment.reasoning; + warning.appendChild(reasonEl); + } return warning; } diff --git a/turnstone/ui/static/style.css b/turnstone/ui/static/style.css index 383563f8..bc5688e7 100644 --- a/turnstone/ui/static/style.css +++ b/turnstone/ui/static/style.css @@ -2480,6 +2480,21 @@ audio.media-player { color: var(--fg-dim); font-style: italic; } +/* LLM-judge attribution badge — set in slightly dimmer ink than the + risk-coloured flag text so it reads as metadata, not another flag. */ +.output-warning-tier { + margin-left: 6px; + font-weight: 600; + opacity: 0.85; +} +/* Judge rationale — muted second line under the flag row. */ +.output-warning-reasoning { + margin-top: 2px; + color: var(--fg-dim); + font-style: italic; + white-space: pre-wrap; + overflow-wrap: anywhere; +} /* ========================================================================== New workstream modal