mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
9be155b97a
* Quality overhaul: code tooling, CI/CD, architecture diagrams, UI redesign, and legacy cleanup - Add ruff (lint+format) and mypy (strict) with zero errors across 37 source files - Add GitHub Actions CI (lint, typecheck, test matrix 3.11/3.12/3.13) and PyPI publish workflow - Create 12 PlantUML architecture diagrams with PNG renders covering all subsystems - Refresh README and docs with badges, diagram links, and current descriptions - Refactor test_server_live.py with mock streaming helpers for deterministic CI testing - Update dependencies to current versions (openai>=2.24, httpx>=0.28, redis>=7.2) Console dashboard: - Move state indicators from top cards to fixed bottom status bar with cluster metrics - Replace flat 50-node list with hostname-prefix grouped nodes (expand/collapse, up to 1000) - Apply "Instrument Panel" visual redesign: IBM Plex Mono + Outfit fonts, warm amber accent, LED glow state indicators, deep charcoal surfaces, WCAG AA contrast compliance - Add render cache, stale indicator, active filter highlight, loading states Server web UI: - Apply matching Instrument Panel aesthetic for visual consistency with console - Fix branding (pcode → turnstone), extract inline styles to CSS classes - Rename pcode localStorage keys and history state to turnstone Legacy cleanup: - Remove persona-model-specific --persona flag and /persona slash command - Remove model_identity from chat_template_kwargs (vLLM-specific mechanism) - Refactor plan agent to use standard developer message instead of model_identity - Remove dead code (unused date/has_tools variables, noqa suppressions) * Fix CI typecheck: add mypy overrides for optional sympy/numpy imports The math sandbox optionally imports sympy and numpy at runtime (try/except ImportError). In CI these packages are not installed, so mypy raises import-not-found rather than import-untyped. Add mypy overrides to ignore missing imports for these optional dependencies. * Fix Copilot review findings: ARIA role, status bar cache, and pulse opacity - Change #node-table from role="tree" to role="list" and group elements from role="treeitem" to role="listitem" (proper ARIA semantics) - Include currentView and currentFilter.state in renderStatusBar cache key so active pill highlight updates when switching views - Align pulse animation to 0.35 opacity (already applied in CSS)
167 lines
6.1 KiB
Python
167 lines
6.1 KiB
Python
"""Tests for turnstone.eval — score_run and _match_action."""
|
|
|
|
from turnstone.eval import _match_action, score_run
|
|
|
|
|
|
class TestMatchAction:
|
|
def test_tool_name_match(self):
|
|
actual = {"tool": "bash", "args": {"command": "ls"}}
|
|
expected = {"tool": "bash"}
|
|
assert _match_action(actual, expected) is True
|
|
|
|
def test_tool_name_mismatch(self):
|
|
actual = {"tool": "bash", "args": {"command": "ls"}}
|
|
expected = {"tool": "read_file"}
|
|
assert _match_action(actual, expected) is False
|
|
|
|
def test_exact_args_match(self):
|
|
actual = {"tool": "bash", "args": {"command": "ls -la"}}
|
|
expected = {"tool": "bash", "args": {"command": "ls -la"}}
|
|
assert _match_action(actual, expected) is True
|
|
|
|
def test_partial_key_matching(self):
|
|
# Expected only specifies a subset of actual args
|
|
actual = {"tool": "bash", "args": {"command": "ls", "extra": "val"}}
|
|
expected = {"tool": "bash", "args": {"command": "ls"}}
|
|
assert _match_action(actual, expected) is True
|
|
|
|
def test_args_value_mismatch(self):
|
|
actual = {"tool": "bash", "args": {"command": "ls"}}
|
|
expected = {"tool": "bash", "args": {"command": "pwd"}}
|
|
assert _match_action(actual, expected) is False
|
|
|
|
def test_args_missing_key(self):
|
|
actual = {"tool": "bash", "args": {"command": "ls"}}
|
|
expected = {"tool": "bash", "args": {"path": "/tmp"}}
|
|
assert _match_action(actual, expected) is False
|
|
|
|
def test_args_pattern_regex_match(self):
|
|
actual = {"tool": "bash", "args": {"command": "git log -5"}}
|
|
expected = {"tool": "bash", "args_pattern": {"command": r"git\s+log"}}
|
|
assert _match_action(actual, expected) is True
|
|
|
|
def test_args_pattern_regex_mismatch(self):
|
|
actual = {"tool": "bash", "args": {"command": "ls -la"}}
|
|
expected = {"tool": "bash", "args_pattern": {"command": r"^git"}}
|
|
assert _match_action(actual, expected) is False
|
|
|
|
def test_raw_fallback_no_expected_args(self):
|
|
actual = {"tool": "bash", "args": {"_raw": "something"}}
|
|
expected = {"tool": "bash"}
|
|
assert _match_action(actual, expected) is True
|
|
|
|
def test_raw_fallback_with_expected_args(self):
|
|
actual = {"tool": "bash", "args": {"_raw": "something"}}
|
|
expected = {"tool": "bash", "args": {"command": "ls"}}
|
|
assert _match_action(actual, expected) is False
|
|
|
|
|
|
class TestScoreRun:
|
|
def test_empty_expected_actions_passes(self):
|
|
result = score_run([{"tool": "bash", "args": {}}], [])
|
|
assert result["pass"] is True
|
|
assert result["score"] == 1.0
|
|
|
|
def test_ordered_subset_all_match(self):
|
|
tool_log = [
|
|
{"tool": "read_file", "args": {"path": "a.py"}},
|
|
{"tool": "bash", "args": {"command": "ls"}},
|
|
{"tool": "edit_file", "args": {"path": "a.py"}},
|
|
]
|
|
expected = [
|
|
{"tool": "read_file"},
|
|
{"tool": "edit_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="ordered_subset")
|
|
assert result["pass"] is True
|
|
assert result["score"] == 1.0
|
|
|
|
def test_ordered_subset_wrong_order(self):
|
|
tool_log = [
|
|
{"tool": "edit_file", "args": {"path": "a.py"}},
|
|
{"tool": "read_file", "args": {"path": "a.py"}},
|
|
]
|
|
expected = [
|
|
{"tool": "read_file"},
|
|
{"tool": "edit_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="ordered_subset")
|
|
# edit_file comes before read_file, so only one can match
|
|
assert result["pass"] is False
|
|
assert result["score"] == 0.5
|
|
|
|
def test_exact_mode_pass(self):
|
|
tool_log = [
|
|
{"tool": "bash", "args": {"command": "ls"}},
|
|
{"tool": "read_file", "args": {"path": "a.py"}},
|
|
]
|
|
expected = [
|
|
{"tool": "bash"},
|
|
{"tool": "read_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="exact")
|
|
assert result["pass"] is True
|
|
assert result["score"] == 1.0
|
|
|
|
def test_exact_mode_length_mismatch(self):
|
|
tool_log = [
|
|
{"tool": "bash", "args": {"command": "ls"}},
|
|
{"tool": "read_file", "args": {"path": "a.py"}},
|
|
{"tool": "edit_file", "args": {"path": "a.py"}},
|
|
]
|
|
expected = [
|
|
{"tool": "bash"},
|
|
{"tool": "read_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="exact")
|
|
# Length mismatch: 3 vs 2, so pass=False even though first 2 match
|
|
assert result["pass"] is False
|
|
|
|
def test_subset_mode_unordered(self):
|
|
tool_log = [
|
|
{"tool": "edit_file", "args": {"path": "a.py"}},
|
|
{"tool": "read_file", "args": {"path": "a.py"}},
|
|
]
|
|
expected = [
|
|
{"tool": "read_file"},
|
|
{"tool": "edit_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="subset")
|
|
assert result["pass"] is True
|
|
assert result["score"] == 1.0
|
|
|
|
def test_contains_any_mode_pass(self):
|
|
tool_log = [
|
|
{"tool": "bash", "args": {"command": "ls"}},
|
|
{"tool": "read_file", "args": {"path": "a.py"}},
|
|
]
|
|
expected = [
|
|
{"tool": "read_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="contains_any")
|
|
assert result["pass"] is True
|
|
assert result["score"] == 1.0
|
|
|
|
def test_contains_any_mode_fail(self):
|
|
tool_log = [
|
|
{"tool": "bash", "args": {"command": "ls"}},
|
|
]
|
|
expected = [
|
|
{"tool": "read_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="contains_any")
|
|
assert result["pass"] is False
|
|
assert result["score"] == 0.0
|
|
|
|
def test_score_partial(self):
|
|
tool_log = [
|
|
{"tool": "bash", "args": {"command": "ls"}},
|
|
]
|
|
expected = [
|
|
{"tool": "bash"},
|
|
{"tool": "read_file"},
|
|
]
|
|
result = score_run(tool_log, expected, match_mode="ordered_subset")
|
|
assert result["score"] == 0.5
|
|
assert len(result["unmatched"]) == 1
|