From 51b5b3ee7400ab2710f1cf1f33f4e5c92b5fc7ab Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Mon, 23 Mar 2026 19:14:57 -0700 Subject: [PATCH] feat: multi-agent eval pipeline with tree search, analyst, diversifier UCB tree search for prompt optimization (arXiv:2603.18620): - EvolutionNode dataclass, UCB1 selection, rolling mean scores - Replaces fragile linear chain with backtracking via tree - Holdout set separation prevents optimizer overfitting - Improvement-based delta feedback to optimizer Multi-agent optimization pipeline: - Analyst agent (phase 1): multi-turn with math/bash tools, identifies semantic failure patterns, computes statistics across test results - Optimizer (phase 2): uses analyst diagnosis to edit developer prompt - Observer: tunes optimizer strategy every 3 iterations - Diversifier: generates paraphrased prompt variants for phrasing robustness, with dedup, delta generation, and JSON caching Failure classification: - 8 failure mode buckets (no_tool_call, wrong_tool, missing_tool, wrong_args, extra_tools, timeout, error, json_dump) - Consistency signals (systematic, flaky, marginal) - Rule-based pre-analysis feeds into analyst as structured input Logging and observability: - Config summary at startup (models, case count, runs) - Per-case diversifier progress with dedup stats - UCB selection reasoning, node score updates, tree growth - Extended TSV: node_score, elapsed_s, prompt_len, iter_tokens, cumul_tokens columns plus 4-decimal precision Infrastructure: - Thread-safe fd-level stdout suppression (os.dup2) - Prompt variants plumbed through parallel execution path - Cached variants auto-detected from tests.json user_prompts field New CLI flags: --explore-constant, --analyst-model/base-url, --diversifier-model/base-url, --diversify N, --save-variants --- tests.json | 544 ++++++++++++++++++++++++++++--- turnstone/eval.py | 790 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 1266 insertions(+), 68 deletions(-) diff --git a/tests.json b/tests.json index 27021611..d8761eba 100644 --- a/tests.json +++ b/tests.json @@ -1,5 +1,5 @@ { - "description": "turnstone behavior tests — tool selection, sequencing, and multi-step reasoning", + "description": "turnstone behavior tests \u2014 tool selection, sequencing, and multi-step reasoning", "defaults": { "n_runs": 5, "max_turns": 15 @@ -8,35 +8,121 @@ { "id": "read-before-edit", "description": "Must read_file before edit_file on the same path", - "user_prompt": "Fix the typo in config.py — change 'recieve' to 'receive'", + "user_prompt": "Fix the typo in config.py \u2014 change 'recieve' to 'receive'", "setup": { "files": { "config.py": "# Config module\ndef recieve_data(source):\n \"\"\"Recieve data from source.\"\"\"\n return source.read()\n" } }, "expected_actions": [ - { "tool": "read_file", "args": { "path": "config.py" } }, - { "tool": "edit_file", "args": { "path": "config.py" } } + { + "tool": "read_file", + "args": { + "path": "config.py" + } + }, + { + "tool": "edit_file", + "args": { + "path": "config.py" + } + } ], - "match_mode": "ordered_subset" + "match_mode": "ordered_subset", + "user_prompts": [ + "Fix the typo in config.py \u2014 change 'recieve' to 'receive'", + "In config.py, correct the misspelling of 'recieve' to 'receive'", + "Please update config.py by replacing 'recieve' with the correct spelling 'receive'", + "There's a typo in config.py: 'recieve' should be 'receive'. Please fix it.", + "Could you change 'recieve' to 'receive' in config.py?", + "Go ahead and fix 'recieve' \u2192 'receive' in config.py", + "I need the word 'recieve' corrected to 'receive' in the file config.py", + "config.py has a spelling error \u2014 'recieve' needs to be changed to 'receive'", + "Kindly rectify the typographical error in config.py, replacing 'recieve' with 'receive'", + "Hey, swap 'recieve' for 'receive' in config.py", + "The file config.py contains a misspelling: please replace 'recieve' with 'receive'", + "Fix the spelling mistake in config.py where 'recieve' is used instead of 'receive'", + "Would you mind fixing 'recieve' to 'receive' in config.py?", + "Replace the incorrectly spelled 'recieve' with 'receive' in config.py", + "In the file config.py, the word 'recieve' is misspelled and should read 'receive' \u2014 please correct it", + "Correct 'recieve' to 'receive' in config.py", + "config.py: change 'recieve' to 'receive'", + "There's a typo \u2014 'recieve' in config.py ought to be 'receive'. Can you fix that?", + "I noticed 'recieve' in config.py. It should be 'receive'. Please make the fix.", + "Update the spelling of 'recieve' to 'receive' within config.py" + ] }, { "id": "write-file-not-bash", "description": "Use write_file for file creation, not bash echo/cat", "user_prompt": "Create a file called hello.py that prints hello world", "expected_actions": [ - { "tool": "write_file", "args_pattern": { "path": "hello\\.py" } } + { + "tool": "write_file", + "args_pattern": { + "path": "hello\\.py" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Create a file called hello.py that prints hello world", + "Make a hello.py file that outputs hello world", + "Please write a Python file named hello.py which prints hello world", + "I need a file called hello.py that prints hello world", + "Could you create hello.py with code that prints hello world?", + "Write hello.py \u2014 it should print hello world", + "Generate a hello.py file that outputs \"hello world\"", + "I'd like you to create a file named hello.py that prints hello world", + "Set up a file called hello.py to print hello world", + "Kindly produce a hello.py file whose purpose is to print hello world", + "hello.py \u2014 create it, should print hello world", + "Would you mind creating a hello.py that prints hello world?", + "I need you to write a hello.py file that will print hello world", + "Create hello.py containing code to print hello world", + "Please make a Python file hello.py that prints hello world to the console", + "Draft a file named hello.py that outputs hello world when run", + "Put together a hello.py script that prints hello world", + "Can you create a file hello.py that prints hello world?", + "Write a file called hello.py with a print statement for hello world", + "I want a hello.py file created that prints hello world" + ] }, { "id": "bash-for-commands", "description": "Use bash for running system commands", "user_prompt": "What Python version is installed?", "expected_actions": [ - { "tool": "bash", "args_pattern": { "command": "python" } } + { + "tool": "bash", + "args_pattern": { + "command": "python" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "What Python version is installed?", + "Check which version of Python is currently installed", + "Can you tell me the installed Python version?", + "python --version please", + "I need to know what version of Python is on this system", + "Which Python version do we have?", + "Could you look up the Python version that's installed here?", + "Determine the currently installed Python version", + "What's the Python version on this machine?", + "Please check the Python version", + "I'm curious about the installed Python version \u2014 can you find out?", + "Show me the Python version", + "Find out what Python version is available", + "What version of Python do we have installed?", + "Kindly verify which version of Python is present on the system", + "python version?", + "I'd like to know the Python version installed on this system", + "Would you mind checking what Python version is installed?", + "Tell me the Python version", + "Report the installed Python version on this machine" + ] }, { "id": "search-for-patterns", @@ -49,13 +135,40 @@ } }, "expected_actions": [ - { "tool": "search", "args_pattern": { "query": "test_" } } + { + "tool": "search", + "args_pattern": { + "query": "test_" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Find all functions that start with 'test_' in the project", + "List every function in the project whose name begins with 'test_'", + "I need to locate all functions prefixed with 'test_' across the project", + "Could you search the project for any functions starting with 'test_'?", + "Show me all the test_ prefixed functions in this project", + "Hunt down every function that has a 'test_' prefix in the codebase", + "I'm looking for all functions named test_* throughout the project", + "Search the entire project for functions whose names start with test_", + "What functions beginning with 'test_' exist in this project?", + "Please identify all functions with the 'test_' prefix in the project files", + "Grep the project for all function definitions starting with 'test_'", + "Give me a list of all test_ functions defined anywhere in the project", + "Can you find every function declaration that starts with 'test_' in the project?", + "Scan the project codebase and return all functions prefixed with test_", + "I'd like to see all functions in the project that begin with 'test_'", + "Go through the project and locate each function starting with 'test_'", + "Pull up all function definitions matching the test_ prefix across the project", + "Which functions in the project have names that start with 'test_'?", + "Enumerate all test_-prefixed function definitions in the project", + "Look through the project and find any function whose name starts with test_" + ] }, { "id": "multi-file-edit", - "description": "Read and edit multiple files — must read before editing each, and edit both", + "description": "Read and edit multiple files \u2014 must read before editing each, and edit both", "user_prompt": "Change the default port from 8000 to 9000 in both server.py and config.py", "setup": { "files": { @@ -64,12 +177,42 @@ } }, "expected_actions": [ - { "tool": "read_file" }, - { "tool": "read_file" }, - { "tool": "edit_file" }, - { "tool": "edit_file" } + { + "tool": "read_file" + }, + { + "tool": "read_file" + }, + { + "tool": "edit_file" + }, + { + "tool": "edit_file" + } ], - "match_mode": "ordered_subset" + "match_mode": "ordered_subset", + "user_prompts": [ + "Change the default port from 8000 to 9000 in both server.py and config.py", + "Update the default port to 9000 instead of 8000 in server.py and config.py", + "Could you modify the port number from 8000 to 9000 in both config.py and server.py?", + "Please replace port 8000 with 9000 in server.py and config.py", + "I need the default port switched from 8000 to 9000 in both server.py and config.py", + "In server.py and config.py, the default port should be changed from 8000 to 9000", + "Swap out port 8000 for 9000 in config.py and server.py", + "Would you mind updating the default port value from 8000 to 9000 across both server.py and config.py?", + "The default port in server.py and config.py needs to be 9000 instead of 8000 \u2014 please make that change", + "Go ahead and change 8000 to 9000 for the default port in both server.py and config.py", + "Kindly adjust the default port configuration from 8000 to 9000 in the files server.py and config.py", + "Port change needed: 8000 \u2192 9000 in server.py and config.py", + "Both server.py and config.py currently use port 8000 as the default \u2014 update them to use 9000", + "Set the default port to 9000 (currently 8000) in server.py and config.py", + "I'd like the default port modified from 8000 to 9000 in both config.py and server.py", + "Replace the 8000 port default with 9000 in both server.py and config.py please", + "Can you switch the default port number to 9000 from 8000 in server.py as well as config.py?", + "Make the default port 9000 rather than 8000 in server.py and config.py", + "In both config.py and server.py, please update the default port \u2014 it should be 9000 not 8000", + "Modify server.py and config.py so the default port is 9000 instead of the current 8000" + ] }, { "id": "search-then-edit", @@ -83,39 +226,159 @@ } }, "expected_actions": [ - { "tool": "search", "args_pattern": { "query": "MAX_RETRIES" } }, - { "tool": "read_file" }, - { "tool": "edit_file", "args_pattern": { "old_string": "3" } } + { + "tool": "search", + "args_pattern": { + "query": "MAX_RETRIES" + } + }, + { + "tool": "read_file" + }, + { + "tool": "edit_file", + "args_pattern": { + "old_string": "3" + } + } ], - "match_mode": "ordered_subset" + "match_mode": "ordered_subset", + "user_prompts": [ + "Find where MAX_RETRIES is defined and change it from 3 to 5", + "Locate the definition of MAX_RETRIES and update its value from 3 to 5", + "Could you search for where MAX_RETRIES is defined and modify it from 3 to 5?", + "I need MAX_RETRIES changed from 3 to 5 \u2014 find where it's defined and update it", + "Please find the MAX_RETRIES definition and bump it from 3 to 5", + "Hunt down MAX_RETRIES in the codebase and change its value from 3 to 5", + "Where is MAX_RETRIES set to 3? Change it to 5.", + "Search the code for the MAX_RETRIES definition and alter it from 3 to 5", + "I'd like you to locate MAX_RETRIES (currently 3) and set it to 5 instead", + "Go find MAX_RETRIES and switch it from 3 to 5", + "The constant MAX_RETRIES is defined somewhere as 3 \u2014 find it and change it to 5", + "Track down the MAX_RETRIES definition and increase it from 3 to 5", + "Would you mind finding where MAX_RETRIES is set and updating the value from 3 to 5?", + "Grep for MAX_RETRIES, find its definition, and change the value from 3 to 5", + "MAX_RETRIES needs to be 5 instead of 3. Find where it's defined and make the change.", + "Look up the definition of MAX_RETRIES and replace 3 with 5", + "Can you find the spot where MAX_RETRIES is defined as 3 and update it to 5?", + "Identify the location of the MAX_RETRIES definition and change its value to 5 from 3", + "I need you to search for MAX_RETRIES, which is currently set to 3, and change it to 5", + "Find the MAX_RETRIES constant and adjust it \u2014 it should be 5, not 3" + ] }, { "id": "bash-git-log", "description": "Use bash for git commands, not other tools", "user_prompt": "Show me the git log for the last 5 commits", "expected_actions": [ - { "tool": "bash", "args_pattern": { "command": "git\\s+log" } } + { + "tool": "bash", + "args_pattern": { + "command": "git\\s+log" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Show me the git log for the last 5 commits", + "Display the 5 most recent git commits", + "Can you pull up the git log limited to the last five commits?", + "I need to see the git log showing only the previous 5 commits", + "git log for the 5 latest commits, please", + "Would you mind showing me the last five entries in the git log?", + "Print out the most recent 5 commits from the git log", + "I'd like to review the git log \u2014 just the last 5 commits", + "Show the recent 5 commit history using git log", + "Could you display the git commit history for the past five commits?", + "Fetch the git log, but only the 5 newest commits", + "Let me see the five most recent commits in the git log", + "Please retrieve the last 5 entries from git log", + "I want to check the git log limited to 5 recent commits", + "Run git log and show me just the latest five commits", + "Pull up the commit log from git for the last 5 changes", + "Mind grabbing the git log? Just the last five commits though", + "Give me a view of the 5 most recent git log entries", + "Show me the history of the last five commits via git log", + "I'd appreciate seeing the git log restricted to the previous 5 commits" + ] }, { "id": "write-then-run", "description": "Create a script and run it to verify it works", "user_prompt": "Create a Python script called fib.py that prints the first 10 Fibonacci numbers, then run it to verify", "expected_actions": [ - { "tool": "write_file", "args_pattern": { "path": "fib\\.py" } }, - { "tool": "bash", "args_pattern": { "command": "python" } } + { + "tool": "write_file", + "args_pattern": { + "path": "fib\\.py" + } + }, + { + "tool": "bash", + "args_pattern": { + "command": "python" + } + } ], - "match_mode": "ordered_subset" + "match_mode": "ordered_subset", + "user_prompts": [ + "Create a Python script called fib.py that prints the first 10 Fibonacci numbers, then run it to verify", + "Write a Python file named fib.py that outputs the first 10 Fibonacci numbers, and execute it to confirm it works", + "Please make fib.py \u2013 a Python script printing the first ten Fibonacci numbers \u2013 then run it to check the output", + "I need a Python script fib.py that prints the first 10 Fibonacci numbers. Execute it afterwards to verify correctness.", + "Could you create fib.py to display the first 10 Fibonacci numbers in Python, and then run it to make sure it works?", + "Draft a script called fib.py in Python that outputs the first ten Fibonacci numbers, then execute it to validate", + "Hey, write me a fib.py that prints the first 10 Fibonacci numbers and run it so we can see it works", + "Generate a Python program fib.py which prints the initial 10 Fibonacci numbers, and verify by running it", + "Kindly produce a Python script named fib.py to print the first 10 Fibonacci numbers, then execute the script to confirm its output", + "Make a file fib.py containing Python code to print the first 10 Fibonacci numbers. Then run it to verify.", + "I'd like you to write fib.py, a Python script that displays the first ten Fibonacci numbers, and test it by running it", + "Put together a Python script fib.py that prints out the first 10 Fibonacci numbers, then go ahead and run it to check", + "Create fib.py with Python code for printing the first 10 Fibonacci numbers and execute it to see the results", + "Would you mind writing a Python script called fib.py that prints the first 10 Fibonacci numbers? Please run it too to verify.", + "Write fib.py in Python to output the first ten Fibonacci numbers. Run the script to ensure it's correct.", + "Compose a Python file fib.py that will print the first 10 Fibonacci numbers when executed, and then run it to verify the output", + "fib.py \u2014 create this Python script to print the first 10 Fibonacci numbers, and execute it to confirm it's working properly", + "Can you set up a Python script named fib.py that prints the first 10 Fibonacci numbers and then run it for verification?", + "Please author a Python program in fib.py to display the first ten Fibonacci numbers, followed by running it to check", + "Whip up fib.py to print the first 10 Fibonacci numbers using Python, then run it to make sure everything checks out" + ] }, { "id": "no-bash-for-file-write", - "description": "Should NOT use bash (echo/cat/heredoc) to create files — only write_file", + "description": "Should NOT use bash (echo/cat/heredoc) to create files \u2014 only write_file", "user_prompt": "Create a new file called README.md with a title and description of this project", "expected_actions": [ - { "tool": "write_file", "args_pattern": { "path": "README" } } + { + "tool": "write_file", + "args_pattern": { + "path": "README" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Create a new file called README.md with a title and description of this project", + "Make a README.md file that includes a project title and description", + "I need a README.md created with a title and a brief description of the project", + "Please generate a README.md file containing the project's title and description", + "Could you set up a README.md with a title and project description?", + "Write a README.md that has a title and describes this project", + "Go ahead and create README.md \u2014 it should have a title and a description of the project", + "I'd like you to produce a new README.md file featuring a project title and description", + "Kindly establish a README.md file incorporating both a title and a description for this project", + "Spin up a README.md with a project title and description in it", + "Draft a README.md file that provides a title and description for the project", + "Would you mind creating a README.md that contains a title and project description?", + "Set up a new file named README.md and include a title along with a project description", + "I want a README.md file \u2014 put a title and a description of the project in it", + "Create README.md and populate it with a title and a description of this project", + "Add a new README.md to the project with an appropriate title and description", + "Can you make a README.md that gives the project a title and description?", + "Put together a README.md file with a heading and a description of what this project is about", + "Initialize a README.md containing a project title and a short description", + "Generate a new file called README.md \u2014 include a title and overview of the project" + ] }, { "id": "plan-before-refactor", @@ -126,8 +389,34 @@ "app.py": "import sqlite3\nfrom flask import Flask, jsonify\n\napp = Flask(__name__)\nDB = 'data.db'\n\ndef get_db():\n return sqlite3.connect(DB)\n\n@app.route('/users')\ndef list_users():\n db = get_db()\n users = db.execute('SELECT * FROM users').fetchall()\n db.close()\n return jsonify(users)\n\n@app.route('/users/')\ndef get_user(uid):\n db = get_db()\n user = db.execute('SELECT * FROM users WHERE id=?', (uid,)).fetchone()\n db.close()\n return jsonify(user)\n\nif __name__ == '__main__':\n app.run(port=8000)\n" } }, - "expected_actions": [{ "tool": "create_plan" }], - "match_mode": "subset" + "expected_actions": [ + { + "tool": "create_plan" + } + ], + "match_mode": "subset", + "user_prompts": [ + "I need to refactor this codebase to separate the database layer from the API layer. Use the plan tool to think through the approach before making any changes.", + "Please use the plan tool to think through how to refactor this codebase, separating the database layer from the API layer, before making any changes.", + "Before touching any code, use the plan tool to outline an approach for decoupling the database layer from the API layer in this codebase.", + "I'd like to split the DB layer and API layer apart in this codebase. Think through the strategy using the plan tool first, then proceed.", + "Could you refactor this code to decouple the database and API layers? Start by using the plan tool to reason through the approach before changing anything.", + "Hey, I need the database layer pulled apart from the API layer in this codebase. Use the plan tool to map out the approach before diving in.", + "The database and API layers in this codebase need to be separated. Please leverage the plan tool to think it through before implementing changes.", + "Use the plan tool first to strategize, then refactor this codebase so that the database layer is cleanly separated from the API layer.", + "I want the DB layer and API layer in this codebase to be independent. Plan out the refactoring approach using the plan tool before making edits.", + "This codebase has its database and API layers tangled together. Use the plan tool to think through a separation strategy before modifying anything.", + "Would you mind using the plan tool to devise a refactoring plan for separating the database layer from the API layer in this codebase? Don't change code until you've planned.", + "Refactor this code so the DB layer is decoupled from the API layer. But first, use the plan tool to carefully consider the approach.", + "I'm looking to have the database concerns separated from the API concerns in this codebase. Think through it with the plan tool before making any modifications.", + "Before implementing anything, use the plan tool to plan how to refactor this codebase into separate database and API layers.", + "Can you separate the database layer from the API layer here? Please use the plan tool to reason about the approach prior to any code changes.", + "Time to decouple the DB and API layers in this codebase. Start with the plan tool to think through the strategy, then make changes.", + "I need this codebase restructured so the database layer doesn't mix with the API layer. Use the plan tool to work out the approach first.", + "Think through a refactoring plan using the plan tool for separating the database and API layers in this codebase, then execute the changes.", + "The goal is to isolate the database layer from the API layer in this codebase. Use the plan tool to figure out the best approach before writing any code.", + "Please plan out (using the plan tool) how to cleanly separate the database layer from the API layer in this codebase before making any actual changes." + ] }, { "id": "edit-not-rewrite", @@ -139,10 +428,42 @@ } }, "expected_actions": [ - { "tool": "read_file", "args": { "path": "utils.py" } }, - { "tool": "edit_file", "args": { "path": "utils.py" } } + { + "tool": "read_file", + "args": { + "path": "utils.py" + } + }, + { + "tool": "edit_file", + "args": { + "path": "utils.py" + } + } ], - "match_mode": "ordered_subset" + "match_mode": "ordered_subset", + "user_prompts": [ + "Add a docstring to the process_data function in utils.py", + "Please add a docstring to the process_data function in utils.py", + "Could you write a docstring for process_data in utils.py?", + "Insert a docstring into the process_data function found in utils.py", + "I need a docstring added to process_data in utils.py", + "Put a docstring on the process_data function in utils.py", + "The process_data function in utils.py is missing a docstring \u2014 please add one", + "Would you mind adding a docstring to process_data in utils.py?", + "In utils.py, the process_data function needs a docstring", + "Add documentation via a docstring to the process_data function within utils.py", + "Go ahead and give process_data in utils.py a proper docstring", + "I'd like you to add a docstring to the process_data function in utils.py", + "utils.py's process_data function lacks a docstring \u2014 can you add one?", + "Write a docstring for the process_data function located in utils.py", + "Kindly include a docstring in the process_data function in utils.py", + "Can you throw a docstring onto process_data in utils.py?", + "The function process_data in utils.py should have a docstring \u2014 please add it", + "Give the process_data function in utils.py a docstring", + "Hey, add a docstring to process_data over in utils.py", + "I require a docstring to be added to the process_data function defined in utils.py" + ] }, { "id": "bash-run-tests", @@ -154,45 +475,180 @@ } }, "expected_actions": [ - { "tool": "bash", "args_pattern": { "command": "pytest|python.*test" } } + { + "tool": "bash", + "args_pattern": { + "command": "pytest|python.*test" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Run the tests", + "Execute the test suite", + "Please go ahead and run the tests", + "Could you run the tests for me?", + "I need the tests to be run", + "Kick off the tests", + "Let's run the tests", + "Fire up the tests", + "Go ahead and execute the tests", + "I'd like you to run the tests", + "Run all tests please", + "Would you mind running the tests?", + "Time to run the tests", + "Please execute the tests", + "Trigger the test run", + "I want to run the tests", + "Can you run the tests?", + "Launch the tests", + "Kindly run the test suite", + "Run the tests for me" + ] }, { "id": "web-fetch-url", "description": "Use web_fetch when asked to retrieve content from a URL", "user_prompt": "Fetch the contents of https://example.com and summarize what's on the page", "expected_actions": [ - { "tool": "web_fetch", "args_pattern": { "url": "example\\.com" } } + { + "tool": "web_fetch", + "args_pattern": { + "url": "example\\.com" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Fetch the contents of https://example.com and summarize what's on the page", + "Go to https://example.com and give me a summary of what you find there", + "Could you pull up https://example.com and tell me what the page is about?", + "Retrieve the content from https://example.com, then provide a summary of it", + "I need you to grab https://example.com and summarize its contents for me", + "Please access https://example.com and give me an overview of the page", + "What's on https://example.com? Fetch it and summarize for me.", + "Download the page at https://example.com and provide a brief summary", + "I'd like a summary of whatever is at https://example.com \u2014 please fetch it first", + "Hit https://example.com and let me know what's there in summary form", + "Kindly retrieve the webpage https://example.com and offer a concise summary of its content", + "Pull the contents of https://example.com and break down what the page contains", + "Can you load https://example.com and give me the gist of it?", + "Get https://example.com and tell me what it says", + "Grab the page from https://example.com, then summarize what you see", + "I'm curious about https://example.com \u2014 fetch it and summarize the contents please", + "Would you mind fetching https://example.com and providing a summary of the page?", + "Access the content at https://example.com and write a short summary", + "Read https://example.com and give me a rundown of what's on that page", + "Check out https://example.com for me and summarize what the page contains" + ] }, { "id": "man-page-lookup", "description": "Use man tool to look up command documentation", "user_prompt": "Look up the man page for tar and tell me what the --xattrs flag does", "expected_actions": [ - { "tool": "man", "args_pattern": { "page": "tar" } } + { + "tool": "man", + "args_pattern": { + "page": "tar" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Look up the man page for tar and tell me what the --xattrs flag does", + "What does the --xattrs flag do in tar? Check the man page for me.", + "Could you pull up the man page for tar and explain the --xattrs option?", + "I need to know what --xattrs does in tar \u2014 can you check the man page?", + "Check tar's man page and let me know the purpose of the --xattrs flag.", + "Please consult the tar man page and describe what the --xattrs flag is for.", + "Hey, look at the tar man page real quick \u2014 what's --xattrs do?", + "I'd like you to read the tar man page and summarize the --xattrs option for me.", + "Would you mind checking the man page for tar to find out what --xattrs means?", + "Look into the tar manual and explain the --xattrs flag to me.", + "What's the --xattrs option in tar? Please reference the man page.", + "Refer to the tar man page and tell me about the --xattrs flag.", + "Can you check what --xattrs does by looking at tar's man page?", + "Pull up the manual for tar and describe the --xattrs flag, please.", + "I'm curious about tar's --xattrs flag \u2014 could you look it up in the man page?", + "Consult the man page for tar and give me a rundown on --xattrs.", + "Open the tar man page and find out what the --xattrs option does.", + "From the tar man page, what is the function of --xattrs?", + "Mind looking up --xattrs in the tar man page and explaining it?", + "Review tar's manual entry and tell me what --xattrs is used for." + ] }, { "id": "math-calculation", "description": "Use the math tool for precise calculations, not bash or mental math", "user_prompt": "What is 2^64 - 1? Use the math tool to calculate it precisely.", "expected_actions": [ - { "tool": "math", "args_pattern": { "code": "2.*64" } } + { + "tool": "math", + "args_pattern": { + "code": "2.*64" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "What is 2^64 - 1? Use the math tool to calculate it precisely.", + "Calculate 2^64 - 1 for me using the math tool, please.", + "I need the exact value of 2^64 - 1. Please use the math tool.", + "Could you use the math tool to compute 2^64 minus 1 precisely?", + "Use the math tool to tell me what 2^64 - 1 equals.", + "I'm curious: what's 2^64 - 1? Compute it with the math tool.", + "Please precisely determine 2^64 - 1 via the math tool.", + "Mind using the math tool to figure out 2^64 - 1 exactly?", + "I'd like to know the precise result of 2^64 - 1 \u2014 use the math tool for this.", + "Leverage the math tool to give me an exact answer for 2^64 - 1.", + "What does 2^64 - 1 evaluate to? Please use the math tool to get the precise answer.", + "Kindly employ the math tool to calculate the exact value of 2 to the power of 64, minus 1.", + "Hey, what's 2^64 - 1? Hit the math tool for an exact number.", + "Using the math tool, compute 2^64 - 1 with full precision.", + "I need the math tool to work out 2^64 - 1. What's the result?", + "Would you mind calculating 2^64 - 1 precisely with the math tool?", + "Give me the exact result of 2^64 - 1, calculated using the math tool.", + "Two to the sixty-fourth power minus one \u2014 what is it? Use the math tool to find out.", + "Can you determine 2^64 - 1 exactly? Please rely on the math tool for this computation.", + "Precisely compute 2**64 - 1 using the math tool and share the result." + ] }, { "id": "web-search-query", "description": "Use web_search for general knowledge lookups, not web_fetch", "user_prompt": "Search the web for the current population of Tokyo", "expected_actions": [ - { "tool": "web_search", "args_pattern": { "query": "Tokyo" } } + { + "tool": "web_search", + "args_pattern": { + "query": "Tokyo" + } + } ], - "match_mode": "subset" + "match_mode": "subset", + "user_prompts": [ + "Search the web for the current population of Tokyo", + "What's Tokyo's current population? Look it up on the web.", + "Could you do a web search to find out how many people currently live in Tokyo?", + "Please search online for Tokyo's present-day population.", + "I need you to look up the current population of Tokyo on the web.", + "Find me Tokyo's current population via a web search.", + "Web search: what is the current population of Tokyo?", + "I'd like to know Tokyo's current population\u2014can you search the web for that?", + "Look up how many people live in Tokyo right now using a web search.", + "Do a web search for the population of Tokyo as of now.", + "Search the internet to find Tokyo's current population figure.", + "Would you mind searching the web to determine Tokyo's present population?", + "I'm curious about Tokyo's current population. Please search the web for it.", + "Kindly perform a web search regarding the current population of Tokyo.", + "Pull up the current population of Tokyo from the web.", + "Go ahead and search online for how many people are in Tokyo currently.", + "Query the web for Tokyo's population at present.", + "Can you find the current population of Tokyo through a web search?", + "Use the web to look up what Tokyo's population is right now.", + "Search for the latest population count of Tokyo on the web." + ] } ] } diff --git a/turnstone/eval.py b/turnstone/eval.py index d6317234..23d5195a 100644 --- a/turnstone/eval.py +++ b/turnstone/eval.py @@ -776,6 +776,7 @@ def _run_iteration_parallel( context_window: int, test_timeout: int, parallel: int, + prompt_variants: dict[str, list[str]] | None = None, ) -> dict[str, Any]: """Run all test cases in parallel using ProcessPoolExecutor.""" # Build work items for every (case, run) combination @@ -783,14 +784,20 @@ def _run_iteration_parallel( for case in cases: case_id = case["id"] case_n = case.get("n_runs", n_runs) + variants = (prompt_variants or {}).get(case_id) for run_idx in range(case_n): + # Select prompt variant for this run (cycle through variants) + if variants and len(variants) > 1: + run_case = {**case, "user_prompt": variants[run_idx % len(variants)]} + else: + run_case = case work_items.append( { "base_url": base_url, "api_key": api_key, "model": model, "system_prompt": system_prompt, - "case": case, + "case": run_case, "case_id": case_id, "run_idx": run_idx, "temperature": temperature, @@ -917,6 +924,7 @@ def _run_iteration( parallel: int = 1, base_url: str = "", api_key: str = "", + prompt_variants: dict[str, list[str]] | None = None, ) -> dict[str, Any]: """Run all test cases n_runs times and score them.""" if parallel > 1 and base_url: @@ -933,6 +941,7 @@ def _run_iteration( context_window=context_window, test_timeout=test_timeout, parallel=parallel, + prompt_variants=prompt_variants, ) case_results: dict[str, Any] = {} @@ -956,12 +965,25 @@ def _run_iteration( log_prefix = f" [{run_idx + 1}/{case_n}]" run_tokens = 0 + # Select prompt variant for this run (cycle through variants) + variants = (prompt_variants or {}).get(case_id) + if variants and len(variants) > 1: + variant_idx = run_idx % len(variants) + run_case = {**case, "user_prompt": variants[variant_idx]} + if verbose: + _log( + f"{log_prefix} variant {variant_idx}: {variants[variant_idx][:80]}...", + dim=True, + ) + else: + run_case = case + try: run_result = _run_single_test( client=client, model=model, system_prompt=system_prompt, - case=case, + case=run_case, temperature=temperature, max_tokens=max_tokens, reasoning_effort=reasoning_effort, @@ -980,6 +1002,8 @@ def _run_iteration( score_result["tool_sequence"] = [t["tool"] for t in run_result["tool_log"]] score_result["tool_args"] = [{t["tool"]: t["args"]} for t in run_result["tool_log"]] score_result["elapsed"] = run_result.get("elapsed", 0) + if run_case is not case: + score_result["prompt_variant"] = run_case["user_prompt"] run_tokens = sum(run_result.get("usage", {}).values()) # Detect JSON dumped into final channel (tool call not made) @@ -1092,6 +1116,97 @@ def _run_iteration( # ─── Prompt optimizer ──────────────────────────────────────────────────────── +DIVERSIFIER_SYSTEM = """\ +You generate paraphrased variations of user prompts for a tool-use \ +evaluation harness. Each variation must preserve the EXACT SAME INTENT \ +— the same task, same expected outcome, same level of specificity — \ +but use different phrasing, vocabulary, sentence structure, or tone. + +Rules: +- Preserve the core action the user is asking for. If the original \ +says "fix the typo in config.py", all variants must ask to fix a \ +typo in config.py. +- Vary along these dimensions: formality (casual ↔ formal), \ +directness (imperative ↔ descriptive), verbosity (terse ↔ detailed), \ +framing (command ↔ question ↔ description of need). +- Do NOT change the expected tool behavior. If the original implies \ +using bash, the variant must also imply bash. +- Do NOT add new requirements, constraints, or context not in the \ +original. +- Each variant should be a single message, roughly similar length \ +to the original. + +Output a JSON array of strings, one per variant. No commentary, \ +no markdown fences, just the JSON array.\ +""" + + +ANALYST_SYSTEM = """\ +You are a test failure analyst for an LLM tool-use evaluation harness. \ +You receive test results showing how a coding assistant performed \ +against expected tool call sequences. + +Your job: identify SEMANTIC PATTERNS across failures and successes — \ +not just what failed, but WHY, and what the failures have in common. + +You have access to `math` (Python with numpy/scipy/collections) and \ +`bash` tools. Use them to compute statistics, build confusion matrices, \ +analyze tool co-occurrence, or quantify patterns — don't just eyeball \ +the data. + +## Analysis Framework + +### 1. Failure Mode Patterns +Look across all failing runs and identify shared root causes: +- Does the model treat certain phrasings as conversation vs action? +- Are there tool confusion pairs (e.g., always picks write_file over edit_file)? +- Do failures cluster around implicit vs explicit instructions? +- Are there complexity thresholds where the model breaks down? + +### 2. Success/Failure Contrast +Compare passing and failing cases to isolate what makes the difference: +- What do passing prompts have that failing ones lack? +- Do passing cases use action verbs while failing ones describe goals? +- Is there a pattern in prompt length, specificity, or structure? + +### 3. Consistency Signals +For each failing case, classify: +- Systematic (0% pass): the prompt is fundamentally missing an instruction +- Flaky (1-79% pass): the prompt is ambiguous — wording nudge needed +- Marginal (80-99%): minor edge case — low priority + +### 4. Actionable Diagnosis +For each pattern found, state: +- What the model is doing wrong (observed behavior) +- Why it's doing it (root cause hypothesis) +- What prompt change would fix it (specific, concrete) + +## Output Format + +``` +## Failure Patterns +- [Pattern 1]: [which cases] — [observed behavior] because [root cause] +- [Pattern 2]: ... + +## Success/Failure Contrast +[What distinguishes passing from failing cases] + +## Consistency +- Systematic: [case_ids] — [what's missing] +- Flaky: [case_ids] — [what's ambiguous] +- Marginal: [case_ids] — [edge case description] + +## Recommended Fixes (priority order) +1. [Highest impact fix]: addresses [N] cases — [specific instruction to add/change] +2. [Next fix]: ... +``` + +Be concise. Focus on patterns, not individual case narratives. \ +The optimizer that reads your output needs actionable signal, \ +not lengthy explanations.\ +""" + + OPTIMIZER_SYSTEM = """\ You are a prompt optimizer. You receive a developer prompt (instructions \ for a coding assistant on how to use its tools) and test results \ @@ -1117,18 +1232,11 @@ WHAT YOU CANNOT CHANGE: - The overall structure (system prompt for a coding assistant). - Phrasing tied to 100% pass rate cases. -FAILURE MODE DIAGNOSIS: -- If the assistant responded with only text (no tool call) → add a \ -rule: "ALWAYS call a tool. Never respond with only text." -- If write_file was used instead of edit_file for a small change → add: \ -"Use edit_file for modifying existing files. Only use write_file for \ -new files." -- If create_plan() was not called for a complex task → add: "When asked to \ -think through a problem, call create_plan(goal='...')." -- If the assistant searched for a file before creating it → add: \ -"When told to create a new file, use write_file directly." -- If the tool sequence is correct but arguments are wrong → adjust \ -the example arguments, not the tool selection logic. +FAILURE ANALYSIS: The input includes a "Failure Analysis" section \ +produced by a separate analyst agent. It identifies semantic patterns \ +across failures, contrasts them with successes, and recommends specific \ +fixes in priority order. Use this to guide your edits — address the \ +highest-priority patterns first. STYLE: direct imperative sentences. One instruction per line. \ Concrete tool call examples where helpful. @@ -1188,6 +1296,130 @@ Output ONLY the modified optimizer instructions.\ """ +def _diversify_prompts( + client: Any, + model: str, + cases: list[dict[str, Any]], + n_variants: int, + provider: LLMProvider | None = None, +) -> dict[str, list[str]]: + """Generate paraphrased prompt variants for each test case. + + Returns {case_id: [variant1, variant2, ...]}. The original + user_prompt is always included as the first variant. + """ + prov = provider or create_provider("openai") + result: dict[str, list[str]] = {} + + for ci, case in enumerate(cases): + cid = case["id"] + original = case["user_prompt"] + + if n_variants <= 1: + result[cid] = [original] + continue + + # Use cached variants if present in the test case + cached = case.get("user_prompts") + if cached and isinstance(cached, list) and len(cached) >= n_variants: + result[cid] = cached[:n_variants] + print( + f" {DIM}[{ci + 1}/{len(cases)}] {cid}...{RESET}" + f" {CYAN}{len(result[cid])} cached{RESET}" + ) + continue + + # Partial cache: keep existing variants, only generate the delta + existing: list[str] = [] + if cached and isinstance(cached, list): + existing = cached + + needed = n_variants - max(len(existing), 1) # original is always slot 0 + if needed <= 0: + result[cid] = (existing or [original])[:n_variants] + print( + f" {DIM}[{ci + 1}/{len(cases)}] {cid}...{RESET}" + f" {CYAN}{len(result[cid])} cached{RESET}" + ) + continue + + print(f" {DIM}[{ci + 1}/{len(cases)}] {cid}...{RESET}", end="", flush=True) + # Generate only the missing variants + existing_json = json.dumps(existing[1:]) if len(existing) > 1 else "[]" + user_content = f"Original prompt: {json.dumps(original)}\n\n" + if len(existing) > 1: + user_content += f"Existing variations (do NOT repeat these): {existing_json}\n\n" + user_content += ( + f"Generate {needed} new paraphrased variations of the original prompt. " + f"Output a JSON array of {needed} strings." + ) + + try: + cr = prov.create_completion( + client=client, + model=model, + messages=[ + {"role": "system", "content": DIVERSIFIER_SYSTEM}, + {"role": "user", "content": user_content}, + ], + max_tokens=2048, + temperature=0.8, + reasoning_effort="low", + ) + raw = (cr.content or "").strip() + # Strip reasoning tags + raw = re.sub( + r"<(?:think|reasoning)>.*?", + "", + raw, + flags=re.DOTALL, + ).strip() + # Strip markdown fences + fence_match = re.search(r"```[^\n]*\n(.*?)```", raw, re.DOTALL) + if fence_match: + raw = fence_match.group(1).strip() + + new_variants = json.loads(raw) + if isinstance(new_variants, list) and all(isinstance(v, str) for v in new_variants): + # Merge existing + new, then deduplicate + base = existing if existing else [original] + seen: set[str] = {v.strip().lower() for v in base} + unique: list[str] = list(base) + dupes = 0 + for v in new_variants[:needed]: + key = v.strip().lower() + if key in seen: + dupes += 1 + else: + seen.add(key) + unique.append(v) + result[cid] = unique + dupe_note = f", {dupes} dupes removed" if dupes else "" + short_note = ( + f" {YELLOW}(< {n_variants} requested, will cycle){RESET}" + if len(unique) < n_variants + else "" + ) + print(f" {GREEN}{len(unique)} unique{dupe_note}{RESET}{short_note}") + else: + result[cid] = [original] + print(f" {YELLOW}parse error, using original{RESET}") + except Exception as e: + _log(f"\n Diversifier failed for {cid}: {e}", dim=True) + result[cid] = [original] + + # Summary statistics + total_variants = sum(len(v) for v in result.values()) + cases_with_variants = sum(1 for v in result.values() if len(v) > 1) + avg_variants = total_variants / len(result) if result else 0 + print( + f" {total_variants} total variants across {cases_with_variants} cases" + f" (avg {avg_variants:.1f}/case)" + ) + + return result + + def _observe_and_update_optimizer( client: Any, model: str, @@ -1303,6 +1535,310 @@ def _observe_and_update_optimizer( return result +def _classify_failure( + run: dict[str, Any], + expected_actions: list[dict[str, Any]], +) -> str: + """Classify a failed run into a failure mode bucket.""" + tools = run.get("tool_sequence", []) + if run.get("json_dump"): + return "json_dump" + if not tools: + return "no_tool_call" + detail = run.get("detail", "") + if "timed out" in detail.lower(): + return "timeout" + if detail.startswith("Error:"): + return "error" + + unmatched = run.get("unmatched", []) + extra = run.get("extra_tools", []) + matched = run.get("matched", []) + + if not unmatched and extra: + return "extra_tools" + + # Check for tool substitution — expected one tool, got a different one + expected_names = {expected_actions[i]["tool"] for i in unmatched if i < len(expected_actions)} + actual_names = set(tools) + if expected_names and not expected_names & actual_names: + return "wrong_tool" + + # Check if right tools were called but args didn't match + if matched and unmatched: + unmatched_expected = { + expected_actions[i]["tool"] for i in unmatched if i < len(expected_actions) + } + if unmatched_expected & actual_names: + return "wrong_args" + + return "missing_tool" + + +def _build_failure_analysis( + iteration_result: dict[str, Any], + test_cases: list[dict[str, Any]], +) -> str: + """Build a semantic failure analysis summary across all cases.""" + # Classify every failed run + mode_cases: dict[str, list[str]] = {} # mode -> [case_id, ...] + mode_details: dict[str, list[str]] = {} # mode -> [detail strings] + consistency: dict[str, str] = {} # case_id -> systematic|flaky|marginal + + for case_id, case_result in iteration_result["cases"].items(): + case_def = next((c for c in test_cases if c["id"] == case_id), None) + expected = case_def.get("expected_actions", []) if case_def else [] + pr = case_result["pass_rate"] + + if pr == 1.0: + continue + + # Consistency classification + if pr == 0: + consistency[case_id] = "systematic" + elif pr < 0.8: + consistency[case_id] = "flaky" + else: + consistency[case_id] = "marginal" + + for run in case_result["runs"]: + if run.get("pass"): + continue + mode = _classify_failure(run, expected) + mode_cases.setdefault(mode, []) + if case_id not in mode_cases[mode]: + mode_cases[mode].append(case_id) + + # Build a detail string for tool substitution cases + if mode == "wrong_tool": + expected_names = [ + expected[i]["tool"] for i in run.get("unmatched", []) if i < len(expected) + ] + actual = run.get("tool_sequence", []) + detail = f"{case_id}: expected {expected_names}, got {actual}" + mode_details.setdefault(mode, []).append(detail) + + if not mode_cases: + return "" + + # Build the summary + lines: list[str] = [] + + # Failure mode summary + mode_labels = { + "no_tool_call": "Text-only response (no tool call made)", + "wrong_tool": "Wrong tool selected", + "missing_tool": "Missing required tool in sequence", + "wrong_args": "Right tool, wrong arguments", + "extra_tools": "Correct sequence but unnecessary extra calls", + "timeout": "Timed out", + "error": "Tool execution error", + "json_dump": "Tool call emitted as JSON text instead of function call", + } + for mode, cases in sorted(mode_cases.items(), key=lambda x: -len(x[1])): + label = mode_labels.get(mode, mode) + lines.append(f"- {label}: {', '.join(cases)}") + for detail in (mode_details.get(mode, []))[:3]: + lines.append(f" {detail}") + + # Consistency summary + systematic = [c for c, v in consistency.items() if v == "systematic"] + flaky = [c for c, v in consistency.items() if v == "flaky"] + marginal = [c for c, v in consistency.items() if v == "marginal"] + + if systematic or flaky or marginal: + lines.append("") + if systematic: + lines.append( + f"Systematic failures (always fail, need new instruction): {', '.join(systematic)}" + ) + if flaky: + lines.append( + f"Flaky failures (sometimes pass, prompt is ambiguous): {', '.join(flaky)}" + ) + if marginal: + lines.append(f"Marginal failures (mostly pass, minor edge case): {', '.join(marginal)}") + + return "\n".join(lines) + + +_ANALYST_TOOLS = [ + { + "type": "function", + "function": { + "name": "math", + "description": ( + "Execute Python code for analysis. Available: numpy, scipy, " + "collections, itertools, math, json, re. Use print() for output. " + "Example: print(numpy.mean([0.8, 0.6, 1.0]))" + ), + "parameters": { + "type": "object", + "properties": { + "code": {"type": "string", "description": "Python code to execute."}, + }, + "required": ["code"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "bash", + "description": "Execute a bash command. Use for jq, awk, sort, uniq, etc.", + "parameters": { + "type": "object", + "properties": { + "command": {"type": "string", "description": "Bash command to execute."}, + }, + "required": ["command"], + }, + }, + }, +] + + +def _exec_analyst_tool(name: str, arguments: str) -> str: + """Execute a tool call from the analyst agent.""" + try: + args = json.loads(arguments) + except json.JSONDecodeError: + return f"Invalid JSON arguments: {arguments[:200]}" + + if name == "math": + from turnstone.core.sandbox import execute_math_sandboxed + + output, is_error = execute_math_sandboxed(args.get("code", ""), timeout=15.0) + return output[:4000] + elif name == "bash": + import subprocess + + try: + proc = subprocess.run( + ["bash", "-c", args.get("command", "")], + capture_output=True, + text=True, + timeout=15, + ) + output = proc.stdout + proc.stderr + return output[:4000] or "(no output)" + except subprocess.TimeoutExpired: + return "Command timed out after 15s" + return f"Unknown tool: {name}" + + +def _run_analyst( + client: Any, + model: str, + test_cases: list[dict[str, Any]], + iteration_result: dict[str, Any], + provider: LLMProvider | None = None, +) -> str: + """Run the analyst agent to produce a semantic failure analysis. + + Multi-turn agent with math and bash tools for computing statistics. + Phase 1 of the two-phase optimization: analyst diagnoses patterns, + then optimizer uses the diagnosis to modify the prompt. + """ + # Build structured input: per-case results with rule-based pre-analysis + case_parts: list[str] = [] + for case_id, case_result in iteration_result["cases"].items(): + case_def = next((c for c in test_cases if c["id"] == case_id), None) + if not case_def: + continue + pr = case_result["pass_rate"] + status = "PASS" if pr == 1.0 else "WEAK" if pr >= 0.5 else "FAIL" + + part = ( + f"[{status}] {case_id} (pass_rate={pr:.0%})\n" + f" User prompt: {case_def['user_prompt']}\n" + f" Expected tools: {json.dumps(case_def.get('expected_actions', []))}\n" + f" Actual sequences: " + f"{[r.get('tool_sequence', []) for r in case_result['runs']]}" + ) + + # Add per-run failure classifications + expected = case_def.get("expected_actions", []) + failed_runs = [r for r in case_result["runs"] if not r.get("pass")] + if failed_runs: + modes = [_classify_failure(r, expected) for r in failed_runs] + part += f"\n Failure modes: {modes}" + + case_parts.append(part) + + # Include rule-based pre-analysis as a starting point + rule_analysis = _build_failure_analysis(iteration_result, test_cases) + + user_content = "## Test Results\n" + "\n\n".join(case_parts) + if rule_analysis: + user_content += f"\n\n## Rule-Based Pre-Analysis\n{rule_analysis}" + user_content += ( + "\n\nAnalyze the semantic patterns across these results. " + "Focus on WHY failures happen and what passing cases have in common. " + "Use the math or bash tools if you need to compute statistics, " + "build confusion matrices, or analyze distributions." + ) + + prov = provider or create_provider("openai") + messages: list[dict[str, Any]] = [ + {"role": "system", "content": ANALYST_SYSTEM}, + {"role": "user", "content": user_content}, + ] + + # Multi-turn loop: let the analyst call tools up to 5 rounds + max_turns = 5 + for _turn in range(max_turns): + cr = prov.create_completion( + client=client, + model=model, + messages=messages, + tools=_ANALYST_TOOLS, + max_tokens=4096, + temperature=0.3, + reasoning_effort="medium", + ) + + assistant_msg: dict[str, Any] = { + "role": "assistant", + "content": cr.content or None, + } + if cr.tool_calls: + assistant_msg["tool_calls"] = cr.tool_calls[:5] + messages.append(assistant_msg) + + if not cr.tool_calls: + break + + # Execute tool calls + for tc in assistant_msg["tool_calls"]: + func_name = tc["function"]["name"] + output = _exec_analyst_tool(func_name, tc["function"]["arguments"]) + messages.append( + { + "role": "tool", + "tool_call_id": tc["id"], + "content": output, + } + ) + + # Extract final text response + result = "" + for msg in reversed(messages): + if msg["role"] == "assistant" and msg.get("content"): + result = msg["content"] + break + + # Strip reasoning tags if present + result = re.sub( + r"<(?:think|reasoning)>.*?", + "", + result, + flags=re.DOTALL, + ).strip() + + return result + + def _propose_prompt_modification( client: Any, model: str, @@ -1313,6 +1849,7 @@ def _propose_prompt_modification( optimizer_system: str = OPTIMIZER_SYSTEM, provider: LLMProvider | None = None, parent_scores: dict[str, float] | None = None, + analyst_output: str = "", ) -> str: """Use the model to propose a new prompt based on evaluation results.""" # Build summary of results @@ -1364,6 +1901,11 @@ def _propose_prompt_modification( + "\n\n".join(failing) + "\n\n" ) + + # Failure analysis from analyst agent (phase 1) + if analyst_output: + user_content += f"## Failure Analysis (from analyst)\n{analyst_output}\n\n" + user_content += ( f"## Score History\n{history_text}\n\n" "Make the MINIMUM change needed to fix failing tests without " @@ -1555,12 +2097,26 @@ def _print_summary_table(iter_result: dict[str, Any]) -> None: ) -def _append_summary_tsv(path: str, iter_result: dict[str, Any], case_ids: list[str]) -> None: +def _append_summary_tsv( + path: str, + iter_result: dict[str, Any], + case_ids: list[str], + cumulative_tokens: int = 0, + node_score: float = 0.0, +) -> None: """Append one row per iteration to a TSV summary file.""" write_header = not os.path.exists(path) or os.path.getsize(path) == 0 agg = iter_result.get("aggregate", {}) per_case = agg.get("per_case_pass_rates", {}) + # Compute iteration elapsed from individual run times + iter_elapsed = sum( + r.get("elapsed", 0) + for cr in iter_result.get("cases", {}).values() + for r in cr.get("runs", []) + ) + prompt_len = len(iter_result.get("prompt", "")) + with open(path, "a") as f: if write_header: cols = [ @@ -1571,18 +2127,28 @@ def _append_summary_tsv(path: str, iter_result: dict[str, Any], case_ids: list[s "runs", "json_dumps", "tree_node", + "node_score", + "elapsed_s", + "prompt_len", + "iter_tokens", + "cumul_tokens", ] + [f"case:{cid}" for cid in case_ids] f.write("\t".join(cols) + "\n") vals = [ str(iter_result.get("iteration", "")), iter_result.get("timestamp", ""), - f"{agg.get('overall_pass_rate', 0):.2f}", - f"{agg.get('overall_avg_score', 0):.2f}", + f"{agg.get('overall_pass_rate', 0):.4f}", + f"{agg.get('overall_avg_score', 0):.4f}", str(agg.get("total_runs", 0)), str(agg.get("json_dumps", 0)), str(iter_result.get("tree_node_id", "")), - ] + [f"{per_case.get(cid, 0):.2f}" for cid in case_ids] + f"{node_score:.4f}", + f"{iter_elapsed:.1f}", + str(prompt_len), + str(agg.get("total_tokens", 0)), + str(cumulative_tokens), + ] + [f"{per_case.get(cid, 0):.4f}" for cid in case_ids] f.write("\t".join(vals) + "\n") @@ -1610,6 +2176,12 @@ def run_optimization( optimizer_model: str | None = None, observer_base_url: str | None = None, observer_model: str | None = None, + analyst_base_url: str | None = None, + analyst_model: str | None = None, + diversifier_base_url: str | None = None, + diversifier_model: str | None = None, + diversify: int = 0, + save_variants: bool = False, explore_constant: float = 1.414, ) -> dict[str, Any]: """Main optimization loop with UCB tree search. @@ -1658,11 +2230,33 @@ def run_optimization( obs_key = os.environ.get(obs_key_env, opt_key) obs_client, obs_provider = _make_client_and_provider(obs_base, obs_key) - # Log role assignments if any differ from the test model - if opt_model != model or opt_base != base_url: - _log(f" Optimizer: {opt_model} @ {opt_base}", dim=True) + # --- Analyst model (inherits from optimizer if not specified) --- + ana_base = analyst_base_url or opt_base + ana_model = analyst_model or opt_model + ana_key_env = ( + "ANTHROPIC_API_KEY" if _detect_provider(ana_base) == "anthropic" else "OPENAI_API_KEY" + ) + ana_key = os.environ.get(ana_key_env, opt_key) + ana_client, ana_provider = _make_client_and_provider(ana_base, ana_key) + + # --- Diversifier model (inherits from optimizer if not specified) --- + div_base = diversifier_base_url or opt_base + div_model = diversifier_model or opt_model + div_key_env = ( + "ANTHROPIC_API_KEY" if _detect_provider(div_base) == "anthropic" else "OPENAI_API_KEY" + ) + div_key = os.environ.get(div_key_env, opt_key) + div_client, div_provider = _make_client_and_provider(div_base, div_key) + + # Log role assignments + _log(f" Test model: {model} @ {base_url}", dim=True) + _log(f" Optimizer: {opt_model} @ {opt_base}", dim=True) if obs_model != opt_model or obs_base != opt_base: - _log(f" Observer: {obs_model} @ {obs_base}", dim=True) + _log(f" Observer: {obs_model} @ {obs_base}", dim=True) + if ana_model != opt_model or ana_base != opt_base: + _log(f" Analyst: {ana_model} @ {ana_base}", dim=True) + if diversify > 0 and (div_model != opt_model or div_base != opt_base): + _log(f" Diversifier: {div_model} @ {div_base}", dim=True) # Load test cases with open(test_file) as f: @@ -1678,6 +2272,12 @@ def run_optimization( # Precedence: CLI arg (non-None) > tests.json defaults > code default (3) resolved_n_runs: int = n_runs if n_runs is not None else int(defaults.get("n_runs", 3)) + total_runs = sum(c.get("n_runs", resolved_n_runs) for c in cases) + print( + f" {len(cases)} cases, {resolved_n_runs} runs/case ({total_runs} total), " + f"max {max_iterations} iterations" + ) + # Holdout split — holdout cases are evaluated but excluded from optimizer feedback holdout_ids: set[str] = {c["id"] for c in cases if c.get("holdout", False)} training_count = len(cases) - len(holdout_ids) @@ -1687,6 +2287,11 @@ def run_optimization( dim=True, ) holdout_ids = set() + elif holdout_ids: + _log( + f" Holdout: {len(holdout_ids)} cases ({', '.join(sorted(holdout_ids))})", + dim=True, + ) # Get initial prompt if initial_prompt is None: @@ -1731,6 +2336,8 @@ def run_optimization( "optimizer_base_url": opt_base, "observer_model": obs_model, "observer_base_url": obs_base, + "analyst_model": ana_model, + "analyst_base_url": ana_base, "started": datetime.now().isoformat(), "test_suite": test_file, "n_runs_default": resolved_n_runs, @@ -1744,6 +2351,52 @@ def run_optimization( tsv_path = os.path.splitext(output_file)[0] + ".tsv" case_ids = [c["id"] for c in cases] + # Diversify prompts — generate paraphrased variants before the loop + # Auto-detect: if any case has cached user_prompts, use them even without --diversify + prompt_variants: dict[str, list[str]] | None = None + if diversify == 0: + cached_variants = { + c["id"]: c["user_prompts"] + for c in cases + if isinstance(c.get("user_prompts"), list) and len(c["user_prompts"]) > 1 + } + if cached_variants: + prompt_variants = cached_variants + total_v = sum(len(v) for v in cached_variants.values()) + print( + f"\n Using cached variants for {len(cached_variants)} cases" + f" ({total_v} total prompts)" + ) + + if diversify > 0: + print(f"\nGenerating {diversify} prompt variants per case...") + prompt_variants = _diversify_prompts( + client=div_client, + model=div_model, + cases=cases, + n_variants=diversify, + provider=div_provider, + ) + results["meta"]["diversify"] = diversify + results["meta"]["prompt_variants"] = prompt_variants + + # Save variants back to test suite JSON for caching + if save_variants: + updated = False + for case in cases: + cid = case["id"] + variants = prompt_variants.get(cid, []) + if len(variants) > 1 and case.get("user_prompts") != variants: + case["user_prompts"] = variants + updated = True + if updated: + suite["cases"] = cases + with open(test_file, "w") as f: + json.dump(suite, f, indent=2) + f.write("\n") + print(f" Saved variants to {test_file}") + + cumulative_tokens = 0 suite_t0 = time.monotonic() for iteration in range(max_iterations): @@ -1760,8 +2413,16 @@ def run_optimization( selected_id = _ucb_select(nodes, explore_constant) selected = nodes[selected_id] + tree_info = f"node {selected_id}" + if selected.visit_count > 0: + tree_info += f", score={selected.score:.0%}, visits={selected.visit_count}" + else: + tree_info += ", unvisited" + if len(nodes) > 1: + tree_info += f", tree={len(nodes)} nodes" + print(f"\n{'=' * 60}") - print(f"Iteration {iteration} (node {selected_id}, visits={selected.visit_count})") + print(f"Iteration {iteration} ({tree_info})") print(f"{'=' * 60}") iter_result = _run_iteration( @@ -1780,6 +2441,7 @@ def run_optimization( parallel=parallel, base_url=base_url, api_key=api_key, + prompt_variants=prompt_variants, ) iter_result["iteration"] = iteration iter_result["prompt"] = selected.prompt @@ -1790,6 +2452,7 @@ def run_optimization( # Update node score (rolling mean) new_score = _compute_holdout_score(iter_result, holdout_ids) + old_score = selected.score if selected.visit_count == 0: selected.score = new_score else: @@ -1797,6 +2460,14 @@ def run_optimization( selected.visit_count + 1 ) selected.visit_count += 1 + if selected.visit_count > 1: + _log( + f" Node {selected_id} score: {old_score:.0%} → {selected.score:.0%}" + f" (this eval: {new_score:.0%})", + dim=True, + ) + + cumulative_tokens += iter_result.get("aggregate", {}).get("total_tokens", 0) results["iterations"].append(iter_result) @@ -1806,7 +2477,13 @@ def run_optimization( json.dump(results, f, indent=2) _print_summary_table(iter_result) - _append_summary_tsv(tsv_path, iter_result, case_ids) + _append_summary_tsv( + tsv_path, + iter_result, + case_ids, + cumulative_tokens=cumulative_tokens, + node_score=selected.score, + ) # Check if all passing agg = iter_result["aggregate"] @@ -1854,6 +2531,28 @@ def run_optimization( holdout_ids, ) + # Phase 1: Analyst diagnoses semantic patterns + analyst_output = "" + if opt_result["aggregate"].get("overall_pass_rate", 0) < 1.0: + print("\nAnalyzing failures...") + try: + analyst_output = _run_analyst( + client=ana_client, + model=ana_model, + test_cases=opt_cases, + iteration_result=opt_result, + provider=ana_provider, + ) + if analyst_output: + _log(f" Analyst:\n{analyst_output}", dim=True) + except Exception as e: + _log(f" Analyst failed: {e}", dim=True) + + # Store analyst output before optimizer (survives optimizer failure) + if analyst_output: + iter_result["analyst"] = analyst_output + + # Phase 2: Optimizer modifies prompt using analyst diagnosis print("\nOptimizing prompt...") try: new_prompt = _propose_prompt_modification( @@ -1866,6 +2565,7 @@ def run_optimization( optimizer_system=current_optimizer_system, provider=opt_provider, parent_scores=parent_scores, + analyst_output=analyst_output, ) except Exception as e: _log(f" Prompt modification failed: {e}", dim=True) @@ -1889,6 +2589,11 @@ def run_optimization( nodes[next_node_id] = child selected.children.append(next_node_id) iter_result["tree_child_id"] = next_node_id + _log( + f" Created node {next_node_id} (child of {selected_id}," + f" tree={len(nodes)} nodes)", + dim=True, + ) next_node_id += 1 # Re-write with tree update and diff @@ -1969,6 +2674,37 @@ def main() -> None: default=None, help="Base URL for observer model (default: same as --optimizer-base-url)", ) + parser.add_argument( + "--analyst-model", + default=None, + help="Model for failure analysis (default: same as --optimizer-model)", + ) + parser.add_argument( + "--analyst-base-url", + default=None, + help="Base URL for analyst model (default: same as --optimizer-base-url)", + ) + parser.add_argument( + "--diversifier-model", + default=None, + help="Model for prompt diversification (default: same as --optimizer-model)", + ) + parser.add_argument( + "--diversifier-base-url", + default=None, + help="Base URL for diversifier model (default: same as --optimizer-base-url)", + ) + parser.add_argument( + "--diversify", + type=int, + default=0, + help="Generate N prompt variants per test case (0=disabled, includes original)", + ) + parser.add_argument( + "--save-variants", + action="store_true", + help="Save generated variants back to test suite JSON for caching", + ) parser.add_argument( "--prompt", default=None, @@ -2090,6 +2826,12 @@ def main() -> None: optimizer_model=args.optimizer_model, observer_base_url=args.observer_base_url, observer_model=args.observer_model, + analyst_base_url=args.analyst_base_url, + analyst_model=args.analyst_model, + diversifier_base_url=args.diversifier_base_url, + diversifier_model=args.diversifier_model, + diversify=args.diversify, + save_variants=args.save_variants, explore_constant=args.explore_constant, )