mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
845df69031
Add a two-arm skill-adherence mode to the eval measurement substrate that measures whether a NAMED skill changes tool-use behaviour, so skill-in-system (main) can be compared against skill-in-context. - _run_single_test gains skill/skill_mode: skill_mode builds HeadlessSession under natural composition (no system_prompt_override) and, for the treatment arm, seeds the skill into the temp DB and activates it via the real set_skill path so the skill body folds into the system message under test. skill_mode defaults False, so the optimizer/measure paths are unchanged. - Thread skill/skill_mode through _run_and_score_subprocess, _run_iteration and _run_iteration_parallel (serial + parallel). - run_skill_adherence: per case, run treatment (skill) vs control (no skill) n_runs each, score against expected_actions, report per-case lift = pass_rate(treatment) - pass_rate(control) and the mean lift. The control isolates the skill's causal effect. - turnstone-eval --skill-adherence <dataset>: loads a skill-scenario dataset and prints a treatment/control/lift table. - eval_skill_adherence.json: authored search-first / test-after-edit / changelog-update scenarios, chosen so the base model does not do the action by default. - tests: plumbing proof (skill folds into system_messages for treatment, absent for control) + lift-math aggregation.
57 lines
1.9 KiB
JSON
57 lines
1.9 KiB
JSON
{
|
|
"defaults": {
|
|
"n_runs": 3
|
|
},
|
|
"cases": [
|
|
{
|
|
"id": "search-first",
|
|
"skill": {
|
|
"name": "search-first",
|
|
"content": "# Search First\n\nBefore answering ANY question about where something lives in the codebase, you MUST call the `search` tool first. Never answer from memory."
|
|
},
|
|
"user_prompt": "Where is JWT token validation implemented in this project?",
|
|
"expected_actions": [{ "tool": "search" }],
|
|
"match_mode": "ordered_subset",
|
|
"max_turns": 4
|
|
},
|
|
{
|
|
"id": "test-after-edit",
|
|
"skill": {
|
|
"name": "test-after-edit",
|
|
"content": "# Test After Edit\n\nAfter editing or writing ANY file, you MUST run the test suite with `python -m pytest` via bash before you finish. Do not report done until tests have run."
|
|
},
|
|
"user_prompt": "Add a function `clamp(x, lo, hi)` that clamps x to [lo, hi] in utils.py.",
|
|
"setup": {
|
|
"files": {
|
|
"utils.py": ""
|
|
}
|
|
},
|
|
"expected_actions": [
|
|
{ "tool": "write_file" },
|
|
{ "tool": "bash", "args_pattern": { "command": "pytest" } }
|
|
],
|
|
"match_mode": "ordered_subset",
|
|
"max_turns": 8
|
|
},
|
|
{
|
|
"id": "changelog-update",
|
|
"skill": {
|
|
"name": "changelog-update",
|
|
"content": "# Changelog Discipline\n\nWhenever you modify a file, you MUST also append a one-line entry to CHANGELOG.md describing the change in the same task."
|
|
},
|
|
"user_prompt": "Fix the off-by-one so pager.py shows the last page. Edit pager.py.",
|
|
"setup": {
|
|
"files": {
|
|
"pager.py": "def last_page(total_items, per_page):\n # off-by-one: drops the final partial page\n return total_items // per_page\n",
|
|
"CHANGELOG.md": "# Changelog\n"
|
|
}
|
|
},
|
|
"expected_actions": [
|
|
{ "tool": "edit_file", "args_pattern": { "path": "CHANGELOG.md" } }
|
|
],
|
|
"match_mode": "subset",
|
|
"max_turns": 8
|
|
}
|
|
]
|
|
}
|