Feat/eval improvements (#37)

* ci: add GitHub Release creation on tag push

* refactor: rename plan tool to create_plan

Rename plan → create_plan to resolve cross-provider tool selection
failures. Models consistently treated "plan" as a reasoning concept
rather than a callable tool. The new name is an unambiguous verb+noun
action. Also rename the parameter from prompt → goal for clarity,
add web_search to the default system prompt tool patterns

* feat: eval harness improvements inspired by autoresearch patterns

Major enhancements to turnstone-eval:

- Per-test timeout (--test-timeout, default 300s) and suite timeout
  (--suite-timeout) prevent stuck runs from blocking the suite
- Fast-fail skips remaining runs after ceil(n/2) consecutive zeros
- Summary table with colored PASS/WEAK/FAIL and append-only TSV output
- Progress reporting with running pass rate, token count, and ETA
- Parallel test execution via ProcessPoolExecutor (--parallel N)
- Per-role model assignment: test/optimizer/observer can use different
  models and providers (--optimizer-model, --observer-model, etc.)
  with auto-detection from base URL
- Improved optimizer and observer system prompts with structured
  failure-mode diagnosis, keep/discard rules, and trend analysis
- Fixed token counting (prompt tokens use last-turn value, not sum)
- Added math-calculation and web-search-query test cases
- Fixed multi-file-edit test (both files now contain the target string)

* fix: address Copilot review feedback

- Revert prompt token counting to sum (reflects billed usage)
- Add tool_args to fast-fail skipped run dicts for schema consistency
- Align approval_label with func_name ("create_plan")
- Add timeout to future.result() in parallel path (test_timeout + 30s)
- Document thread-leak trade-off on serial timeout path

.github/workflows/publish.yml

@@ -5,6 +5,7 @@ on:
     tags: ["v*"]
 
 permissions:
+  contents: write
   id-token: write
 
 jobs:
@@ -19,3 +20,10 @@ jobs:
       - run: pip install build
       - run: python -m build
       - uses: pypa/gh-action-pypi-publish@release/v1
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
+          draft: false
+          prerelease: ${{ contains(github.ref, '-') }}

tests.json

@@ -59,7 +59,7 @@
       "user_prompt": "Change the default port from 8000 to 9000 in both server.py and config.py",
       "setup": {
         "files": {
-          "server.py": "from config import PORT\n\ndef run():\n    print(f'Listening on port {PORT}')\n",
+          "server.py": "import socket\n\ndef run():\n    sock = socket.socket()\n    sock.bind(('localhost', 8000))\n    print('Server running on port 8000')\n",
           "config.py": "PORT = 8000\nHOST = 'localhost'\n"
         }
       },
@@ -126,7 +126,7 @@
           "app.py": "import sqlite3\nfrom flask import Flask, jsonify\n\napp = Flask(__name__)\nDB = 'data.db'\n\ndef get_db():\n    return sqlite3.connect(DB)\n\n@app.route('/users')\ndef list_users():\n    db = get_db()\n    users = db.execute('SELECT * FROM users').fetchall()\n    db.close()\n    return jsonify(users)\n\n@app.route('/users/<int:uid>')\ndef get_user(uid):\n    db = get_db()\n    user = db.execute('SELECT * FROM users WHERE id=?', (uid,)).fetchone()\n    db.close()\n    return jsonify(user)\n\nif __name__ == '__main__':\n    app.run(port=8000)\n"
         }
       },
-      "expected_actions": [{ "tool": "plan" }],
+      "expected_actions": [{ "tool": "create_plan" }],
       "match_mode": "subset"
     },
     {
@@ -175,6 +175,24 @@
         { "tool": "man", "args_pattern": { "page": "tar" } }
       ],
       "match_mode": "subset"
+    },
+    {
+      "id": "math-calculation",
+      "description": "Use the math tool for precise calculations, not bash or mental math",
+      "user_prompt": "What is 2^64 - 1? Use the math tool to calculate it precisely.",
+      "expected_actions": [
+        { "tool": "math", "args_pattern": { "code": "2.*64" } }
+      ],
+      "match_mode": "subset"
+    },
+    {
+      "id": "web-search-query",
+      "description": "Use web_search for general knowledge lookups, not web_fetch",
+      "user_prompt": "Search the web for the current population of Tokyo",
+      "expected_actions": [
+        { "tool": "web_search", "args_pattern": { "query": "Tokyo" } }
+      ],
+      "match_mode": "subset"
     }
   ]
 }

tests/test_session.py

@@ -203,8 +203,8 @@ class TestPlanExec:
                         "id": tc_id,
                         "type": "function",
                         "function": {
-                            "name": "plan",
-                            "arguments": json.dumps({"prompt": prior_prompt}),
+                            "name": "create_plan",
+                            "arguments": json.dumps({"goal": prior_prompt}),
                         },
                     }
                 ],
@@ -239,7 +239,7 @@ class TestPlanExec:
             m for m in messages if m["role"] == "assistant" and m.get("tool_calls")
         ]
         assert len(assistant_with_tc) == 1
-        assert assistant_with_tc[0]["tool_calls"][0]["function"]["name"] == "plan"
+        assert assistant_with_tc[0]["tool_calls"][0]["function"]["name"] == "create_plan"
 
         # The real tool result is forwarded with its original content
         tool_msgs = [m for m in messages if m["role"] == "tool"]

tests/test_tools_schema.py

@@ -97,7 +97,7 @@ class TestToolsMetadata:
             "web_fetch": "url",
             "web_search": "query",
             "task": "prompt",
-            "plan": "prompt",
+            "create_plan": "goal",
             "remember": "key",
             "recall": "query",
             "forget": "key",

turnstone/core/session.py

@@ -547,13 +547,15 @@ class ChatSession:
                 "   write_file(path='hello.py', content='...')\n\n"
                 "Find something across files → search:\n"
                 "   search(query='test_')\n\n"
-                "Complex or multi-step task → plan first:\n"
-                "   plan(prompt='refactor database from API')\n\n"
+                "Plan, design, or think through an approach → create_plan:\n"
+                "   create_plan(goal='refactor database from API')\n\n"
                 "Run a command, git, or tests → bash:\n"
                 "   bash(command='git log -5')\n"
                 "   bash(command='pytest')\n\n"
                 "Retrieve a URL → web_fetch:\n"
                 "   web_fetch(url='https://example.com')\n\n"
+                "Search the web for information → web_search:\n"
+                "   web_search(query='current population of Tokyo')\n\n"
                 "Look up documentation → man:\n"
                 "   man(page='tar')",
             ]
@@ -1464,7 +1466,7 @@ class ChatSession:
         # Post-plan gate: prompt user on main thread after plan completes
         for i, item in enumerate(items):
             if (
-                item.get("func_name") == "plan"
+                item.get("func_name") == "create_plan"
                 and not item.get("error")
                 and not item.get("denied")
                 and not self.auto_approve
@@ -1544,7 +1546,7 @@ class ChatSession:
             "web_search": self._prepare_web_search,
             "tool_search": self._prepare_tool_search,
             "task": self._prepare_task,
-            "plan": self._prepare_plan,
+            "create_plan": self._prepare_plan,
             "remember": self._prepare_remember,
             "recall": self._prepare_recall,
             "forget": self._prepare_forget,
@@ -2094,26 +2096,26 @@ class ChatSession:
 
     def _prepare_plan(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]:
         """Prepare a planning agent for approval."""
-        prompt = (args.get("prompt") or "").strip()
-        if not prompt:
+        goal = args.get("goal", "").strip()
+        if not goal:
             return {
                 "call_id": call_id,
-                "func_name": "plan",
-                "header": "\u2717 plan: empty prompt",
+                "func_name": "create_plan",
+                "header": "\u2717 create_plan: empty goal",
                 "preview": "",
                 "needs_approval": False,
-                "error": "Error: empty prompt",
+                "error": "Error: empty goal",
             }
-        preview_text = prompt[:300] + ("..." if len(prompt) > 300 else "")
+        preview_text = goal[:300] + ("..." if len(goal) > 300 else "")
         return {
             "call_id": call_id,
-            "func_name": "plan",
-            "header": "\u2699 plan (planning agent)",
+            "func_name": "create_plan",
+            "header": "\u2699 create_plan (planning agent)",
             "preview": f"    {DIM}{preview_text}{RESET}",
             "needs_approval": True,
-            "approval_label": "plan",
+            "approval_label": "create_plan",
             "execute": self._exec_plan,
-            "prompt": prompt,
+            "prompt": goal,
         }
 
     def _prepare_remember(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]:
@@ -2577,7 +2579,7 @@ class ChatSession:
                 tool_name = tc_dict["function"]["name"]
 
                 # Guard 1: block recursive agent calls.
-                if tool_name in ("task", "plan"):
+                if tool_name in ("task", "create_plan"):
                     output = "Error: agents cannot spawn further agents"
                 # Guard 2: tool not in this agent's API tool list.
                 elif tool_name not in tool_names:
@@ -2698,7 +2700,7 @@ class ChatSession:
         for i, msg in enumerate(self.messages):
             if msg.get("role") == "assistant" and msg.get("tool_calls"):
                 for tc in msg["tool_calls"]:
-                    if tc.get("function", {}).get("name") == "plan":
+                    if tc.get("function", {}).get("name") == "create_plan":
                         tc_id = tc["id"]
                         for j in range(i + 1, len(self.messages)):
                             if (

turnstone/tools/create_plan.json

@@ -0,0 +1,15 @@
+{
+  "name": "create_plan",
+  "description": "Create a structured plan before taking action. An autonomous agent explores the available context, identifies what needs to change, and writes a step-by-step plan. Call this tool when the user asks to plan, design, or think through an approach, or when a task is complex, touches multiple areas, or has unclear scope.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "goal": {
+        "type": "string",
+        "description": "The goal and scope of the plan, including any constraints."
+      }
+    },
+    "required": ["goal"]
+  },
+  "primary_key": "goal"
+}

turnstone/tools/plan.json

@@ -1,15 +0,0 @@
-{
-  "name": "plan",
-  "description": "Plan before implementing. An autonomous agent explores the codebase and writes a structured plan to .plan-<ws_id>.md (unique per workstream to avoid collisions). If a plan for this workstream already exists it is re-read and refined rather than overwritten from scratch. Use plan BEFORE writing code — when the user asks to build, add, refactor, or change something that touches multiple files or has unclear scope. The plan identifies files to modify, existing patterns to reuse, and risks to consider.",
-  "parameters": {
-    "type": "object",
-    "properties": {
-      "prompt": {
-        "type": "string",
-        "description": "What to plan — the goal, constraints, and scope."
-      }
-    },
-    "required": ["prompt"]
-  },
-  "primary_key": "prompt"
-}

turnstone/tools/web_fetch.json

@@ -1,6 +1,6 @@
 {
   "name": "web_fetch",
-  "description": "Fetch a URL and extract specific information from it. You must provide a question or extraction guidance — the page is fetched, analyzed, and only relevant information is returned (not raw page content).",
+  "description": "Fetch a URL and extract specific information from it. You must provide a question or extraction guidance. The page is fetched, analyzed, and only relevant information is returned (not raw page content).",
   "parameters": {
     "type": "object",
     "properties": {