mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-16 17:01:40 -06:00
Patrick Buckley
09ea3d164d
* feat: intent validation v1 — advisory LLM judge for tool approvals (#50) Two-tier evaluation pipeline for non-auto-approved tool calls: - Heuristic tier (instant): 23 pattern-based rules across 4 severity levels (critical/high/medium/low) with first-match-wins priority - LLM judge tier (async): multi-turn evaluation with read_file/ list_directory tool access, security-hardened path blocking, forcing message on final turn, four-stage JSON parsing with retry nudge Progressive UI: heuristic verdict badge + judge spinner, LLM verdict upgrade via intent_verdict SSE event, glow on action buttons. Verdict persisted to intent_verdicts table for audit. Prometheus metrics for verdict counts and LLM latency. Enabled by default (--no-judge to opt out). 132 new tests (1938 total). Integration: session, server/WebUI, CLI, MQ bridge, console admin API, Discord channel adapter. Config via [judge] in config.toml or CLI flags. * fix: address PR #50 Copilot review feedback - Fix double JSON encoding of func_args in both heuristic and LLM verdict persistence paths — use pre-serialized string from verdict - Fix confidence 0.0 treated as falsy in channel verdict formatter - Fix timestamp format inconsistency in storage backends (isoformat vs strftime) — now uses strftime consistently - Add on_intent_verdict to eval.py NullUI (mypy fix) - Fix late verdict after approval resolved — store last decision and apply immediately to late-arriving verdicts - Add permission rollback to migration 012 downgrade - Update docs to reflect judge enabled by default - Document confidence_threshold as reserved for v2 * fix: judge per-call timeout and credential recon heuristic - Wrap create_completion() in ThreadPoolExecutor with per-call timeout to prevent indefinite hangs on slow local models. On timeout, replace the executor so subsequent batch items don't queue behind lingering API calls - Add IntentJudge.shutdown() and wire into session.close() for cleanup - Add credential-recon heuristic rule: /etc/passwd, /etc/shadow, /etc/master.passwd access flagged as HIGH/review (reconnaissance pattern even though the command itself is read-only) - 3 new tests for credential file access patterns * fix: denied/blocked tool calls show correct badge on resume - _build_history() detects denied results ("Denied by user") and blocked results ("Blocked") and propagates denied flag to parent assistant entry for frontend consumption - Frontend history replay uses denied flag for badge-denied class instead of hardcoding badge-approved for all historical tool calls - Denial feedback always prefixed with "Denied by user:" so content detection works with custom user feedback - Denied tools visually muted (opacity 0.55, muted tool name) - role="status" on all approval badge elements (accessibility) - Broadened "Blocked" prefix match (catches "Blocked by tool policy")
232 lines
8.1 KiB
Python
232 lines
8.1 KiB
Python
"""Tests for turnstone.core.config — unified TOML config loading."""
|
|
|
|
import argparse
|
|
|
|
import turnstone.core.config as config_mod
|
|
from turnstone.core.config import apply_config, load_config
|
|
|
|
|
|
def _reset_cache():
|
|
"""Clear the module-level config cache between tests."""
|
|
config_mod._cache = None
|
|
|
|
|
|
def test_load_config_missing_file(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", tmp_path / "nope.toml")
|
|
assert load_config() == {}
|
|
|
|
|
|
def test_load_config_valid_toml(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[redis]\nhost = "10.0.0.1"\nport = 6380\npassword = "secret"\n')
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
result = load_config()
|
|
assert result["redis"]["host"] == "10.0.0.1"
|
|
assert result["redis"]["port"] == 6380
|
|
assert result["redis"]["password"] == "secret"
|
|
|
|
|
|
def test_load_config_section(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[api]\nbase_url = "http://x:8000/v1"\n[redis]\nhost = "y"\n')
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
assert load_config("redis") == {"host": "y"}
|
|
assert load_config("api") == {"base_url": "http://x:8000/v1"}
|
|
assert load_config("nonexistent") == {}
|
|
|
|
|
|
def test_load_config_invalid_toml(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text("this is not valid toml [[[")
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
assert load_config() == {}
|
|
|
|
|
|
def test_load_config_caches(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[api]\nbase_url = "http://first"\n')
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
first = load_config()
|
|
assert first["api"]["base_url"] == "http://first"
|
|
|
|
# Change file — should NOT be re-read (cached)
|
|
cfg.write_text('[api]\nbase_url = "http://second"\n')
|
|
second = load_config()
|
|
assert second["api"]["base_url"] == "http://first"
|
|
|
|
|
|
def test_apply_config_sets_defaults(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text(
|
|
'[redis]\nhost = "redis.local"\nport = 7777\npassword = "pw"\n'
|
|
'[bridge]\nserver_url = "http://bridge:9090"\n'
|
|
)
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--redis-host", default="localhost")
|
|
parser.add_argument("--redis-port", type=int, default=6379)
|
|
parser.add_argument("--redis-password", default=None)
|
|
parser.add_argument("--server-url", default="http://localhost:8080")
|
|
|
|
apply_config(parser, ["redis", "bridge"])
|
|
args = parser.parse_args([])
|
|
|
|
assert args.redis_host == "redis.local"
|
|
assert args.redis_port == 7777
|
|
assert args.redis_password == "pw"
|
|
assert args.server_url == "http://bridge:9090"
|
|
|
|
|
|
def test_apply_config_cli_overrides(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[redis]\nhost = "config-host"\nport = 7777\n')
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--redis-host", default="localhost")
|
|
parser.add_argument("--redis-port", type=int, default=6379)
|
|
|
|
apply_config(parser, ["redis"])
|
|
# CLI flag overrides config
|
|
args = parser.parse_args(["--redis-host", "cli-host"])
|
|
|
|
assert args.redis_host == "cli-host" # CLI wins
|
|
assert args.redis_port == 7777 # config wins (no CLI override)
|
|
|
|
|
|
def test_apply_config_missing_keys_keep_defaults(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[redis]\nhost = "only-host"\n') # no port, no password
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--redis-host", default="localhost")
|
|
parser.add_argument("--redis-port", type=int, default=6379)
|
|
parser.add_argument("--redis-password", default=None)
|
|
|
|
apply_config(parser, ["redis"])
|
|
args = parser.parse_args([])
|
|
|
|
assert args.redis_host == "only-host"
|
|
assert args.redis_port == 6379 # original default kept
|
|
assert args.redis_password is None # original default kept
|
|
|
|
|
|
def test_apply_config_no_file(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", tmp_path / "nope.toml")
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--redis-host", default="localhost")
|
|
|
|
apply_config(parser, ["redis"])
|
|
args = parser.parse_args([])
|
|
assert args.redis_host == "localhost"
|
|
|
|
|
|
def test_apply_config_model_section(tmp_path, monkeypatch):
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[model]\nname = "qwen-72b"\ntemperature = 0.3\n')
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--model", default=None)
|
|
parser.add_argument("--temperature", type=float, default=0.5)
|
|
|
|
apply_config(parser, ["model"])
|
|
args = parser.parse_args([])
|
|
|
|
assert args.model == "qwen-72b"
|
|
assert args.temperature == 0.3
|
|
|
|
|
|
def test_tavily_key_from_config(tmp_path, monkeypatch):
|
|
"""get_tavily_key() reads from config.toml [api] tavily_key."""
|
|
_reset_cache()
|
|
config_mod._tavily_key = None
|
|
config_mod._tavily_key_loaded = False
|
|
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('[api]\ntavily_key = "tvly-from-config"\n')
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
monkeypatch.delenv("TAVILY_API_KEY", raising=False)
|
|
|
|
key = config_mod.get_tavily_key()
|
|
assert key == "tvly-from-config"
|
|
|
|
|
|
def test_tavily_key_fallback_to_env(tmp_path, monkeypatch):
|
|
"""get_tavily_key() falls back to $TAVILY_API_KEY env var."""
|
|
_reset_cache()
|
|
config_mod._tavily_key = None
|
|
config_mod._tavily_key_loaded = False
|
|
|
|
# Config exists but no tavily_key in it
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text("[api]\n")
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
monkeypatch.setenv("TAVILY_API_KEY", "tvly-from-env")
|
|
|
|
key = config_mod.get_tavily_key()
|
|
assert key == "tvly-from-env"
|
|
|
|
|
|
def test_apply_config_judge_section(tmp_path, monkeypatch):
|
|
"""apply_config() loads [judge] section and maps to argparse dests."""
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text(
|
|
"[judge]\n"
|
|
"enabled = true\n"
|
|
'model = "gpt-5"\n'
|
|
"confidence_threshold = 0.85\n"
|
|
"timeout = 30.0\n"
|
|
"read_only_tools = false\n"
|
|
)
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--judge", dest="judge_enabled", action="store_true", default=False)
|
|
parser.add_argument("--judge-model", dest="judge_model", default="")
|
|
parser.add_argument("--judge-confidence", dest="judge_confidence", type=float, default=0.7)
|
|
parser.add_argument("--judge-timeout", dest="judge_timeout", type=float, default=60.0)
|
|
parser.add_argument("--judge-read-only-tools", dest="judge_read_only_tools", default=True)
|
|
|
|
apply_config(parser, ["judge"])
|
|
args = parser.parse_args([])
|
|
|
|
assert args.judge_enabled is True
|
|
assert args.judge_model == "gpt-5"
|
|
assert args.judge_confidence == 0.85
|
|
assert args.judge_timeout == 30.0
|
|
assert args.judge_read_only_tools is False
|
|
|
|
|
|
def test_apply_config_judge_cli_overrides(tmp_path, monkeypatch):
|
|
"""CLI flags override config.toml [judge] values."""
|
|
_reset_cache()
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text("[judge]\nenabled = true\nconfidence_threshold = 0.85\n")
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", cfg)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--judge", dest="judge_enabled", action="store_true", default=False)
|
|
parser.add_argument("--no-judge", dest="judge_enabled", action="store_false")
|
|
parser.add_argument("--judge-confidence", dest="judge_confidence", type=float, default=0.7)
|
|
|
|
apply_config(parser, ["judge"])
|
|
args = parser.parse_args(["--no-judge"])
|
|
|
|
assert args.judge_enabled is False # CLI wins
|
|
assert args.judge_confidence == 0.85 # config wins (no CLI override)
|