From c7586abd0a1e01080e05c7e2cd01ab425cd70128 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Sun, 8 Mar 2026 01:43:38 -0800 Subject: [PATCH] Add dynamic tool search with native defer_loading for Anthropic/OpenAI (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add dynamic tool search with native defer_loading for Anthropic/OpenAI When MCP tools push the total tool count past a configurable threshold (default 20), tool definitions are deferred to reduce token overhead and improve tool selection accuracy. Three-tier approach mirrors the existing web search pattern: - Anthropic (Claude 4.x): native defer_loading + server-side BM25 search - OpenAI (GPT-5.4+): native defer_loading + hosted search - vLLM/llama/NIM: client-side BM25 fallback via synthetic tool_search tool New module turnstone/core/tool_search.py with BM25Index (pure-Python, zero deps) and ToolSearchManager (session-scoped visibility, expansion, server hint generation). Discovered tools persist for the session lifetime so the model only searches once per capability needed. Config: [tools] search/search_threshold/search_max_results CLI: --tool-search {auto,on,off}, --tool-search-threshold, --tool-search-max-results Agents (plan/task) exempt — their scoped tool sets are always small. 43 new tests (1253 total). All diagrams regenerated with PlantUML 1.2025.2. * Fix Copilot review feedback on tool search - Fix _MCP_PREFIX_RE to handle underscores in server names (non-greedy match) - Use ordered dict for _expanded to preserve tool discovery order - Avoid constructing ToolSearchManager when below threshold in auto mode - Return empty string from _mcp_server_summary when no servers (not "none") - Fix CLI help text to reference threshold generically, not hardcoded "20" - Fix agent exemption docs to accurately describe scoped tool sets - Fix README to not hardcode "30+" threshold number --- README.md | 10 +- docs/architecture.md | 3 +- docs/diagrams/02-package-structure.puml | 4 +- docs/diagrams/03-core-engine-classes.puml | 23 ++ docs/diagrams/05-tool-pipeline.puml | 46 ++-- docs/diagrams/png/01-system-context.png | 4 +- docs/diagrams/png/02-package-structure.png | 4 +- docs/diagrams/png/03-core-engine-classes.png | 4 +- docs/diagrams/png/04-conversation-turn.png | 4 +- docs/diagrams/png/05-tool-pipeline.png | 4 +- docs/diagrams/png/06-mq-protocol.png | 4 +- docs/diagrams/png/07-message-routing.png | 2 +- docs/diagrams/png/08-redis-key-schema.png | 4 +- docs/diagrams/png/09-workstream-states.png | 4 +- .../png/10-simulator-architecture.png | 4 +- docs/diagrams/png/11-console-data-flow.png | 4 +- docs/diagrams/png/12-deployment.png | 2 +- docs/diagrams/png/13-sdk-architecture.png | 4 +- docs/diagrams/png/14-storage-architecture.png | 4 +- docs/diagrams/png/15-auth-architecture.png | 4 +- docs/diagrams/png/16-channel-architecture.png | 4 +- docs/diagrams/png/17-notify-flow.png | 2 +- docs/tools.md | 80 +++++- tests/test_providers.py | 116 +++++++++ tests/test_tool_search.py | 246 ++++++++++++++++++ turnstone/cli.py | 21 ++ turnstone/core/config.py | 3 + turnstone/core/providers/_anthropic.py | 38 +++ turnstone/core/providers/_openai.py | 34 ++- turnstone/core/providers/_protocol.py | 3 + turnstone/core/session.py | 111 +++++++- turnstone/core/tool_search.py | 243 +++++++++++++++++ turnstone/core/tools.py | 1 + turnstone/server.py | 21 ++ 34 files changed, 1003 insertions(+), 62 deletions(-) create mode 100644 tests/test_tool_search.py create mode 100644 turnstone/core/tool_search.py diff --git a/README.md b/README.md index 1a56875b..6fb1e7cd 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) ## What it does -Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports. It runs as: +Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports. Native deferred tool loading for Anthropic and OpenAI APIs reduces token overhead and improves tool selection accuracy when MCP servers expose many tools; local models (vLLM, llama.cpp) get a transparent client-side BM25 fallback. It runs as: - **Interactive sessions** — terminal CLI or browser UI with parallel workstreams - **Queue-driven agents** — trigger workstreams via message queue, stream progress, approve or auto-approve tool use @@ -151,7 +151,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct ## Tools -14 built-in tools, 2 agent tools, plus external tools via MCP: +15 built-in tools, 2 agent tools, plus external tools via MCP: | Tool | Description | Auto-approved | |------|-------------|:---:| @@ -167,10 +167,13 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct | `remember` | Save persistent facts | yes | | `recall` | Search memories and history | yes | | `forget` | Remove a memory | yes | +| `notify` | Send notifications to linked channels | yes | | `task` | Spawn autonomous sub-agent | | | `plan` | Explore codebase, write .plan.md | | | `mcp__*` | External tools from MCP servers | | +When the total tool count exceeds a configurable threshold (default 20), MCP tools are automatically deferred using native `defer_loading` on Anthropic and OpenAI APIs, or a transparent client-side BM25 search for local models. The LLM discovers deferred tools on demand via a `tool_search` capability — no configuration needed beyond `--tool-search auto` (the default). + ### MCP Tool Servers Turnstone supports the [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) for connecting external tool servers. MCP tools are discovered at startup, converted to OpenAI function-calling format, and merged with built-in tools. Each MCP tool is prefixed with `mcp__{server}__{tool}` to avoid name collisions. @@ -248,6 +251,9 @@ agent_model = "" # model alias for plan/task sub-agents [tools] timeout = 30 skip_permissions = false +search = "auto" # "auto" (enable when >threshold tools), "on", "off" +search_threshold = 20 # min tools before tool search activates +search_max_results = 5 # max tools returned per search query [server] host = "0.0.0.0" diff --git a/docs/architecture.md b/docs/architecture.md index ec0f4047..e3edf026 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,6 +43,7 @@ turnstone/ workstream.py Parallel workstream manager (WorkstreamState, Workstream, WorkstreamManager) tools.py Tool schema loader (JSON -> OpenAI function-calling format) mcp_client.py MCPClientManager — MCP server connections, tool discovery, async-sync bridge + tool_search.py Dynamic tool search — BM25 index, session-scoped tool visibility model_registry.py ModelRegistry — named model configs, lazy client creation, fallback routing memory.py Persistence facade (delegates to storage backend) storage/ Pluggable storage: StorageBackend protocol, SQLite + PostgreSQL @@ -94,7 +95,7 @@ turnstone/ style.css Page-specific UI styles (dashboard layout, approval blocks) app.js Page-specific client-side JavaScript (SSE, workstreams, markdown) tools/ - *.json 14 tool schemas (OpenAI function-calling format + turnstone metadata) + *.json 15 tool schemas (OpenAI function-calling format + turnstone metadata) ``` Both UIs share a common design system extracted into `turnstone/shared_static/`: design tokens, login overlay, toast notifications, theme toggle, keyboard shortcuts, and utility functions. Each UI imports `base.css` and the shared JS modules at `/shared/`, then adds only page-specific code at `/static/`. diff --git a/docs/diagrams/02-package-structure.puml b/docs/diagrams/02-package-structure.puml index 088ad1ac..2a88b1b6 100644 --- a/docs/diagrams/02-package-structure.puml +++ b/docs/diagrams/02-package-structure.puml @@ -41,6 +41,7 @@ package "turnstone/core/" <> { component [healthcheck.py\nBackendHealthMonitor] as healthcheck <> component [ratelimit.py\nRateLimiter] as ratelimit <> component [mcp_client.py\nMCPClientManager] as mcp <> + component [tool_search.py\nToolSearchManager, BM25] as toolsearch <> component [model_registry.py\nModelRegistry] as registry <> } @@ -95,7 +96,7 @@ package "turnstone/sdk/" <> { ' Tool schemas package "turnstone/tools/" <> { - component [*.json\n14 tool schemas] as schemas <> + component [*.json\n15 tool schemas] as schemas <> } ' Entry point dependencies @@ -136,6 +137,7 @@ session --> edit session --> web session --> healthcheck session --> mcp : optional +session --> toolsearch : optional session --> registry : optional registry --> providers healthcheck --> metrics diff --git a/docs/diagrams/03-core-engine-classes.puml b/docs/diagrams/03-core-engine-classes.puml index af2cbd33..b0bef4de 100644 --- a/docs/diagrams/03-core-engine-classes.puml +++ b/docs/diagrams/03-core-engine-classes.puml @@ -108,6 +108,7 @@ class "ModelCapabilities" as ModelCaps <> { + thinking_mode: str + supports_effort: bool + supports_web_search: bool + + supports_tool_search: bool } ' ChatSession @@ -120,6 +121,7 @@ class "ChatSession" as ChatSession { - _msg_tokens: list[int] - _ws_id: str - _mcp_client: MCPClientManager | None + - _tool_search: ToolSearchManager | None - _registry: ModelRegistry | None + model_alias: str | None {property} - _tools: list[dict] @@ -139,6 +141,9 @@ class "ChatSession" as ChatSession { - _prepare_tool(tc) → item dict - _prepare_mcp_tool(call_id, name, args) → item dict - _exec_mcp_tool(item) → (call_id, output) + - _get_active_tools() → list[dict] + - _prepare_tool_search() → None + - _exec_tool_search(item) → (call_id, output) - _run_agent(messages, tools, ...) → str - _compact_messages(auto: bool) - _full_messages() → list[dict] @@ -217,6 +222,23 @@ class "MCPClientManager" as MCPMgr { core/mcp_client.py } +' ToolSearchManager +class "ToolSearchManager" as ToolSearchMgr { + - _all_tools: list[dict] + - _always_on: list[dict] + - _deferred: list[dict] + - _expanded: set[str] + - _index: BM25Index + -- + + should_activate() → bool + + get_visible_tools() → list[dict] + + get_deferred_tools() → list[dict] + + search(query, k) → list[dict] + + expand_visible(names) → list[dict] + + get_search_tool_definition() → dict + + format_search_results(tools) → str +} + ' ModelRegistry class "ModelRegistry" as ModelReg { - _models: dict[str, ModelConfig] @@ -317,6 +339,7 @@ LLMProvider <|.. AnthropicProv ChatSession --> SessionUI : uses ChatSession --> LLMProvider : delegates LLM calls ChatSession --> MCPMgr : optional +ChatSession --o ToolSearchMgr : _tool_search ChatSession --> ModelReg : optional ChatSession <|-- HeadlessSession diff --git a/docs/diagrams/05-tool-pipeline.puml b/docs/diagrams/05-tool-pipeline.puml index 32efaac5..32a37e90 100644 --- a/docs/diagrams/05-tool-pipeline.puml +++ b/docs/diagrams/05-tool-pipeline.puml @@ -24,27 +24,29 @@ partition "Phase 1: Prepare" #E8F5E9 { :Dispatch to _prepare_{func_name}(); note right - **Dispatch table (14 tools):** - ┌─────────────┬──────────────────┐ - │ Tool │ Needs Approval? │ - ├─────────────┼──────────────────┤ - │ bash │ ✓ Yes │ - │ read_file │ ✗ Auto-approve │ - │ write_file │ ✓ Yes │ - │ edit_file │ ✓ Yes │ - │ search │ ✗ Auto-approve │ - │ math │ ✓ Yes │ - │ man │ ✗ Auto-approve │ - │ web_fetch │ ✓ Yes │ - │ web_search │ ✓ Yes │ - │ task │ ✓ Yes │ - │ plan │ ✓ Yes │ - │ remember │ ✗ Auto-approve │ - │ recall │ ✗ Auto-approve │ - │ forget │ ✗ Auto-approve │ - ├─────────────┼──────────────────┤ - │ mcp__* │ ✓ Yes (external) │ - └─────────────┴──────────────────┘ + **Dispatch table (16 tools):** + ┌──────────────┬──────────────────┐ + │ Tool │ Needs Approval? │ + ├──────────────┼──────────────────┤ + │ bash │ ✓ Yes │ + │ read_file │ ✗ Auto-approve │ + │ write_file │ ✓ Yes │ + │ edit_file │ ✓ Yes │ + │ search │ ✗ Auto-approve │ + │ math │ ✓ Yes │ + │ man │ ✗ Auto-approve │ + │ web_fetch │ ✓ Yes │ + │ web_search │ ✓ Yes │ + │ tool_search │ ✗ Auto-approve │ + │ task │ ✓ Yes │ + │ plan │ ✓ Yes │ + │ remember │ ✗ Auto-approve │ + │ recall │ ✗ Auto-approve │ + │ forget │ ✗ Auto-approve │ + │ notify │ ✗ Auto-approve │ + ├──────────────┼──────────────────┤ + │ mcp__* │ ✓ Yes (external) │ + └──────────────┴──────────────────┘ end note :Build item dict: @@ -106,8 +108,10 @@ partition "Phase 3: Execute" #E3F2FD { ├─ _exec_man: man/info subprocess ├─ _exec_web_fetch: httpx.get + LLM summary ├─ _exec_web_search: Tavily API POST (fallback for local models) + ├─ _exec_tool_search: BM25 search + expand_visible() ├─ _exec_task: _run_agent(TASK_AGENT_TOOLS) ├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only) + ├─ _exec_notify: HTTP POST to channel gateway ├─ _exec_remember: SQLite INSERT OR REPLACE ├─ _exec_recall: SQLite FTS5/LIKE search ├─ _exec_forget: SQLite DELETE diff --git a/docs/diagrams/png/01-system-context.png b/docs/diagrams/png/01-system-context.png index a6e9d8aa..638a0f64 100644 --- a/docs/diagrams/png/01-system-context.png +++ b/docs/diagrams/png/01-system-context.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9a1b0361c466327d0011a488847ea3c0365983713537d4a7c27cd7f5538ba33c -size 164829 +oid sha256:d8ce6d2a43a991655c3f64a20b6e810fdb2f78eb767acc3d3d1b8d2c9f443181 +size 165011 diff --git a/docs/diagrams/png/02-package-structure.png b/docs/diagrams/png/02-package-structure.png index f6e94944..34e33b1c 100644 --- a/docs/diagrams/png/02-package-structure.png +++ b/docs/diagrams/png/02-package-structure.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d75da92a657525bcbb7a425dc6c8d3cafe074c3ac3bff3cf4b1d44aea607b50 -size 330156 +oid sha256:ab677009d526b1e9fdaadf30080c81161f2a59fc5b4090bfb3680de0444135a7 +size 321216 diff --git a/docs/diagrams/png/03-core-engine-classes.png b/docs/diagrams/png/03-core-engine-classes.png index 152f2f95..e83154b5 100644 --- a/docs/diagrams/png/03-core-engine-classes.png +++ b/docs/diagrams/png/03-core-engine-classes.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:637458e0d78df82752746e519cd7300a830c8ce211f21625694ad0c162ca316d -size 481637 +oid sha256:8f47555af0ff16c8a236a3edcb0d75ce1b2850d3ecb3cc4627526d5b9890327a +size 522131 diff --git a/docs/diagrams/png/04-conversation-turn.png b/docs/diagrams/png/04-conversation-turn.png index e40ec574..d4dc048e 100644 --- a/docs/diagrams/png/04-conversation-turn.png +++ b/docs/diagrams/png/04-conversation-turn.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dc3b64c9e48153641af62ed43fbc1d89a31d1a8a61e7e71cfc550c805000310d -size 288290 +oid sha256:24bdc6a83259e4db6aaa24f581bed59d351c7a83e1c282aac123c52b32d9f80d +size 288250 diff --git a/docs/diagrams/png/05-tool-pipeline.png b/docs/diagrams/png/05-tool-pipeline.png index 7b400901..47525813 100644 --- a/docs/diagrams/png/05-tool-pipeline.png +++ b/docs/diagrams/png/05-tool-pipeline.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b842683d238664a3e35d04358fecfc56cefd013f7dca5f13357b0376f881e1b3 -size 245043 +oid sha256:027ad99469d69f1d6b2e73ee802b50a75617cd286392375aa69133c13d3683dc +size 256347 diff --git a/docs/diagrams/png/06-mq-protocol.png b/docs/diagrams/png/06-mq-protocol.png index 79b8103d..15d48d55 100644 --- a/docs/diagrams/png/06-mq-protocol.png +++ b/docs/diagrams/png/06-mq-protocol.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b22d5980fe5cc4b8466ba0797113dc8fa83fab8df24b5dacceaf97e62e2e25b0 -size 187649 +oid sha256:32a0665cceffcc0517265bde12cfb227688aa8585284b5e946ab23bcc52daee6 +size 187650 diff --git a/docs/diagrams/png/07-message-routing.png b/docs/diagrams/png/07-message-routing.png index 6e15c426..b680bd63 100644 --- a/docs/diagrams/png/07-message-routing.png +++ b/docs/diagrams/png/07-message-routing.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:90e4f74be795b530e711faa87bc6eb2b3bf6abb68d8fac8ebff7aaf30c6fbe53 +oid sha256:09535722ba975e47cf0557a40b6c481f125ff2022c396f79715c3bba9f715871 size 222032 diff --git a/docs/diagrams/png/08-redis-key-schema.png b/docs/diagrams/png/08-redis-key-schema.png index dd4f80ba..3d02b113 100644 --- a/docs/diagrams/png/08-redis-key-schema.png +++ b/docs/diagrams/png/08-redis-key-schema.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d33b9b3affcdb07086b5aebca8a3b9c2b009cdfc6f360950a0e72e65fbcb8f17 -size 201602 +oid sha256:ed457b10b534b5fc2a5e190b281d7ded4dd1615da2229d67a373cf5dddccd059 +size 201601 diff --git a/docs/diagrams/png/09-workstream-states.png b/docs/diagrams/png/09-workstream-states.png index df49b5ee..e6dc2d97 100644 --- a/docs/diagrams/png/09-workstream-states.png +++ b/docs/diagrams/png/09-workstream-states.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:adac93a0bb062d7199b819a600a0983ff011a75d16928fb80322cbb41f9284ea -size 158866 +oid sha256:e0a3f48cca1b8408862dc4ba04fd340703346f44d84048c99e9900f48e9c7e22 +size 158867 diff --git a/docs/diagrams/png/10-simulator-architecture.png b/docs/diagrams/png/10-simulator-architecture.png index 2300ff67..6b247c45 100644 --- a/docs/diagrams/png/10-simulator-architecture.png +++ b/docs/diagrams/png/10-simulator-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:69f201cff948cb0a19810b7c4ad26d346f869ee2dd3141eba4f353332efa2e21 -size 373649 +oid sha256:35cf3a6942f62dabcbbe012ac2f9e6f155332c894692981b076de5a25c1f3330 +size 374055 diff --git a/docs/diagrams/png/11-console-data-flow.png b/docs/diagrams/png/11-console-data-flow.png index f8688057..858427e8 100644 --- a/docs/diagrams/png/11-console-data-flow.png +++ b/docs/diagrams/png/11-console-data-flow.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:97e7210cd8f1ad195f4d5e25e778d82df3c08c5c6e0f09722e84a7a453714867 -size 411664 +oid sha256:a74b4b8b5dbfb1a51a01100b731477968942b01218bad9451a3d5a9cb3003294 +size 411665 diff --git a/docs/diagrams/png/12-deployment.png b/docs/diagrams/png/12-deployment.png index cd712a08..7f114453 100644 --- a/docs/diagrams/png/12-deployment.png +++ b/docs/diagrams/png/12-deployment.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c3214ef416c1dfe4fa17834c2b6f4071a8093cfdb2b862848ca79938f726a13 +oid sha256:84524f4bc900708ac8adf081591d336f862830188eb8505e71a0f071b339d923 size 252599 diff --git a/docs/diagrams/png/13-sdk-architecture.png b/docs/diagrams/png/13-sdk-architecture.png index ed849653..27fc5563 100644 --- a/docs/diagrams/png/13-sdk-architecture.png +++ b/docs/diagrams/png/13-sdk-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c9823a41e09611c5c0530d9fc12ad4139cfcc3ae238dc665b2888ec94d7d6781 -size 195708 +oid sha256:435a58aa09d0e6615e78c0be62e5fd9aa6d7329b1e96619744355c42ade649c9 +size 196502 diff --git a/docs/diagrams/png/14-storage-architecture.png b/docs/diagrams/png/14-storage-architecture.png index 9bd0b0d3..00b0de54 100644 --- a/docs/diagrams/png/14-storage-architecture.png +++ b/docs/diagrams/png/14-storage-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:733aa17cbfdab60a601cac6adf439c657dd3535e3d6c33c69c2ef93ba8ec5989 -size 251042 +oid sha256:5faa5335152685cf1c8bf77ed93847d751cde59e1afed651e5991113f2f0f31b +size 242670 diff --git a/docs/diagrams/png/15-auth-architecture.png b/docs/diagrams/png/15-auth-architecture.png index 79d4e9b1..67ed2ae6 100644 --- a/docs/diagrams/png/15-auth-architecture.png +++ b/docs/diagrams/png/15-auth-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4b2a2010335f986511c8dabaf49ec046ac02e577f9bc9924897e045f860bb13 -size 248808 +oid sha256:af5ab3126bf685afe68e24bc4b0ed97371d0ebdb77bf4d76c0331ab120580cc0 +size 248809 diff --git a/docs/diagrams/png/16-channel-architecture.png b/docs/diagrams/png/16-channel-architecture.png index babc50fe..16e5b118 100644 --- a/docs/diagrams/png/16-channel-architecture.png +++ b/docs/diagrams/png/16-channel-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6049cc0b07480df88d0d93aa977a1e97f64b41588325ff41d98be0e39431fc5c -size 431712 +oid sha256:1380065cbb5f95b5ea7dc6b2a00986c455b82888af60784980dffbd936460dcf +size 431129 diff --git a/docs/diagrams/png/17-notify-flow.png b/docs/diagrams/png/17-notify-flow.png index f37f8139..8208bed1 100644 --- a/docs/diagrams/png/17-notify-flow.png +++ b/docs/diagrams/png/17-notify-flow.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f55e177e0838a16d9bc4f07b162b4b6a966cc596c9d0a022d35a3c84f23e7b02 +oid sha256:f0f6097840fccdbfe16cd5e4c9f5d063b2c36942460a944df68b8ec947e63ea3 size 221452 diff --git a/docs/tools.md b/docs/tools.md index e8cd0ce2..ea4b09f1 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -51,6 +51,7 @@ schema plus turnstone-specific metadata keys: | `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. | | `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. | | `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). | +| `BUILTIN_TOOL_NAMES`| Frozenset of all 15 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. | | `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. | --- @@ -68,6 +69,9 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools - Parses the JSON arguments (with fallback for malformed JSON). - If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string to the correct parameter. +- Dispatches to the matching `_prepare_{func_name}()` handler. There are 15 + built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and + the generic `_prepare_mcp_tool()` handler for MCP tools. - Validates arguments and builds a preview dict containing: - `call_id`, `func_name`, `header`, `preview` (for display) - `needs_approval` (bool) @@ -435,6 +439,77 @@ Provide either `username` for user-based targeting or `channel_type` + | `recall` | Memory | Yes | No | No | `query` | | `forget` | Memory | Yes | No | No | `key` | | `notify` | Notify | Yes | Yes | Yes | `message` | +| `tool_search`| Search | Yes | No | No | `query` | + +--- + +## Dynamic Tool Search + +When many MCP tools are connected, the total tool count can grow large enough to +consume significant context window tokens and reduce model accuracy. Dynamic tool +search addresses this by deferring tools the model is unlikely to need on the +current turn and letting it search for them on demand. + +### Three-tier approach + +Tool search uses the best available mechanism for each provider: + +1. **Anthropic (native)** -- Models that support it receive `defer_loading: true` + on deferred tool definitions plus the `tool_search_tool_bm25_20251119` server-side + search tool. Anthropic's API handles search and expansion transparently. + +2. **OpenAI GPT-5.4+ (native)** -- Models with hosted tool search receive + `defer_loading: true` on deferred definitions. The API handles search internally. + +3. **vLLM / llama.cpp / NIM (client-side BM25)** -- A synthetic `tool_search` + function tool is injected into the tool list. When the model calls it, + `_exec_tool_search()` runs a pure-Python BM25 index over tool names and + descriptions, then expands the matched tools into the visible set. + +### Configuration + +Tool search is configured in `config.toml` under the `[tools]` section: + +```toml +[tools] +search = "auto" # "auto", "on", or "off" +search_threshold = 20 # minimum total tool count to activate +search_max_results = 5 # max tools returned per search call +``` + +CLI flags override the config file: + +- `--tool-search {auto,on,off}` -- force tool search on or off, or let turnstone + decide based on threshold (default: `auto`). +- `--tool-search-threshold N` -- minimum tool count to activate (default: 20). +- `--tool-search-max-results N` -- max results per search (default: 5). + +### How it works + +1. **Threshold check**: At session startup, `ToolSearchManager.should_activate()` + counts total tools (built-in + MCP). If the count is below the threshold, tool + search stays off and all tools are sent to the model directly. + +2. **Partitioning**: When active, tools are split into two sets: + - **Always-on** -- the 15 built-in tools (members of `BUILTIN_TOOL_NAMES`). + These are always visible to the model. + - **Deferred** -- all MCP tools. These are not sent in the tool list unless + the model searches for them. + +3. **Search and expand**: When the model calls `tool_search` (client-side) or the + provider's native search returns results, the matched tools are added to the + visible set via `expand_visible()`. Once expanded, a tool stays visible for + the remainder of the session. + +4. **Multi-turn persistence**: Expanded tools are never removed. This avoids + confusing the model when it references a tool it discovered in an earlier turn. + +### Agent exemption + +Plan and task sub-agents do not use tool search. They operate on scoped tool +sets (`AGENT_TOOLS` for plan agents, `TASK_AGENT_TOOLS` for task agents) with +MCP tools merged in. Tool search is only active for the top-level session, +where the model can interactively search for tools it needs. --- @@ -456,8 +531,11 @@ MCP-compatible service. 3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`. -4. **Merging**: MCP tools are appended after the 14 built-in tools via +4. **Merging**: MCP tools are appended after the 15 built-in tools via `merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority. + When dynamic tool search is active, MCP tools are deferred rather than directly + visible -- the model discovers them via search as needed (see + [Dynamic Tool Search](#dynamic-tool-search) above). 5. **Dispatch**: When the LLM calls an MCP tool, `_prepare_mcp_tool()` builds a generic approval preview and `_exec_mcp_tool()` calls `MCPClientManager.call_tool_sync()`, diff --git a/tests/test_providers.py b/tests/test_providers.py index eb5ca6e0..abbd563c 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1933,3 +1933,119 @@ class TestAnthropicProviderBlocks: assert blocks[1]["input"] == {"query": "test"} # parsed from accumulated JSON assert blocks[2]["type"] == "web_search_tool_result" assert blocks[2]["encrypted_content"] == "enc_data" + + +# --------------------------------------------------------------------------- +# Tool search tests +# --------------------------------------------------------------------------- + + +class TestAnthropicToolSearch: + """Test Anthropic provider tool search injection.""" + + @pytest.fixture() + def provider(self): + from turnstone.core.providers._anthropic import AnthropicProvider + + return AnthropicProvider() + + def test_tool_search_capability_flag(self, provider): + caps = provider.get_capabilities("claude-opus-4-6-20260101") + assert caps.supports_tool_search is True + + def test_tool_search_not_supported_on_haiku(self, provider): + caps = provider.get_capabilities("claude-haiku-4-5-20251001") + assert caps.supports_tool_search is False + + def test_inject_tool_search_marks_deferred(self, provider): + caps = provider.get_capabilities("claude-opus-4-6-20260101") + tools = [ + {"name": "bash", "description": "Run commands", "input_schema": {}}, + { + "name": "mcp__github__create_issue", + "description": "Create issue", + "input_schema": {}, + }, + ] + deferred = frozenset(["mcp__github__create_issue"]) + result = provider._inject_tool_search(tools, caps, deferred) + # bash should not be deferred + assert result[0].get("defer_loading") is None or result[0].get("defer_loading") is False + # MCP tool should be deferred + assert result[1]["defer_loading"] is True + # Search tool should be appended + assert result[-1]["type"] == "tool_search_tool_bm25_20251119" + assert result[-1]["name"] == "tool_search" + + def test_inject_tool_search_no_op_without_deferred(self, provider): + caps = provider.get_capabilities("claude-opus-4-6-20260101") + tools = [{"name": "bash", "description": "Run commands", "input_schema": {}}] + result = provider._inject_tool_search(tools, caps, None) + assert result == tools + + def test_inject_tool_search_no_op_on_unsupported_model(self, provider): + caps = provider.get_capabilities("claude-haiku-4-5-20251001") + tools = [{"name": "bash", "description": "Run commands", "input_schema": {}}] + deferred = frozenset(["some_tool"]) + result = provider._inject_tool_search(tools, caps, deferred) + assert result == tools + + +class TestOpenAIToolSearch: + """Test OpenAI provider tool search injection.""" + + @pytest.fixture() + def provider(self): + return OpenAIProvider() + + def test_tool_search_capability_on_gpt54(self, provider): + caps = provider.get_capabilities("gpt-5.4") + assert caps.supports_tool_search is True + + def test_tool_search_not_supported_on_gpt5(self, provider): + caps = provider.get_capabilities("gpt-5") + assert caps.supports_tool_search is False + + def test_apply_tool_search_marks_deferred(self, provider): + caps = provider.get_capabilities("gpt-5.4") + tools = [ + {"type": "function", "function": {"name": "bash", "description": "Run commands"}}, + { + "type": "function", + "function": {"name": "mcp__slack__send", "description": "Send message"}, + }, + ] + deferred = frozenset(["mcp__slack__send"]) + result = provider._apply_tool_search(caps, tools, deferred) + assert result is not None + # bash not deferred + assert result[0].get("defer_loading") is None or result[0].get("defer_loading") is False + # slack tool deferred + assert result[1]["defer_loading"] is True + + def test_apply_tool_search_no_op_without_deferred(self, provider): + caps = provider.get_capabilities("gpt-5.4") + tools = [ + {"type": "function", "function": {"name": "bash", "description": "Run commands"}}, + ] + result = provider._apply_tool_search(caps, tools, None) + assert result == tools + + def test_apply_tool_search_no_op_on_unsupported_model(self, provider): + caps = provider.get_capabilities("gpt-5") + tools = [ + {"type": "function", "function": {"name": "bash", "description": "Run commands"}}, + ] + deferred = frozenset(["some_tool"]) + result = provider._apply_tool_search(caps, tools, deferred) + assert result == tools + + +class TestModelCapabilitiesToolSearch: + """Test supports_tool_search defaults and values.""" + + def test_default_is_false(self): + from turnstone.core.providers._protocol import ModelCapabilities + + caps = ModelCapabilities() + assert caps.supports_tool_search is False diff --git a/tests/test_tool_search.py b/tests/test_tool_search.py new file mode 100644 index 00000000..13aac747 --- /dev/null +++ b/tests/test_tool_search.py @@ -0,0 +1,246 @@ +"""Tests for turnstone.core.tool_search — BM25 index and tool search manager.""" + +from __future__ import annotations + +import pytest + +from turnstone.core.tool_search import ( + BM25Index, + ToolSearchManager, + _mcp_server_summary, + _tokenize, + _tool_name, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_tool(name: str, description: str = "") -> dict: + """Create a minimal OpenAI-format tool dict for testing.""" + return { + "type": "function", + "function": { + "name": name, + "description": description or f"Tool {name}", + "parameters": {"type": "object", "properties": {}}, + }, + } + + +# --------------------------------------------------------------------------- +# BM25Index tests +# --------------------------------------------------------------------------- + + +class TestTokenize: + def test_basic_split(self): + assert _tokenize("hello world") == ["hello", "world"] + + def test_underscore_split(self): + assert _tokenize("create_issue") == ["create", "issue"] + + def test_mixed_delimiters(self): + assert _tokenize("mcp__github__create-issue") == ["mcp", "github", "create", "issue"] + + def test_empty_string(self): + assert _tokenize("") == [] + + def test_lowercased(self): + assert _tokenize("GitHub Create") == ["github", "create"] + + +class TestBM25Index: + def test_empty_corpus(self): + idx = BM25Index([]) + assert idx.search("test") == [] + + def test_empty_query(self): + idx = BM25Index(["hello world", "foo bar"]) + assert idx.search("") == [] + + def test_single_document(self): + idx = BM25Index(["create github issue"]) + assert idx.search("github") == [0] + + def test_ranking_order(self): + docs = [ + "list_repos List all repositories", + "create_issue Create a new GitHub issue", + "get_issue Get details of a GitHub issue", + ] + idx = BM25Index(docs) + results = idx.search("github issue") + # Both issue-related docs should rank above list_repos + assert 1 in results[:2] + assert 2 in results[:2] + + def test_top_k_limit(self): + docs = [f"tool_{i} description {i}" for i in range(20)] + idx = BM25Index(docs) + results = idx.search("tool description", k=3) + assert len(results) <= 3 + + def test_no_match(self): + idx = BM25Index(["alpha beta gamma"]) + assert idx.search("zzzzz") == [] + + def test_exact_name_match_ranks_high(self): + docs = [ + "send_email Send an email message", + "send_slack Send a Slack message", + "read_email Read email inbox", + ] + idx = BM25Index(docs) + results = idx.search("send email") + assert results[0] == 0 # send_email should rank first + + +# --------------------------------------------------------------------------- +# ToolSearchManager tests +# --------------------------------------------------------------------------- + + +class TestToolSearchManager: + @pytest.fixture() + def builtin_tools(self): + return [ + _make_tool("bash", "Execute shell commands"), + _make_tool("read_file", "Read a file"), + _make_tool("edit_file", "Edit a file"), + ] + + @pytest.fixture() + def mcp_tools(self): + return [ + _make_tool("mcp__github__create_issue", "Create a new GitHub issue"), + _make_tool("mcp__github__list_issues", "List GitHub issues"), + _make_tool("mcp__github__get_repo", "Get repository details"), + _make_tool("mcp__slack__send_message", "Send a Slack message"), + _make_tool("mcp__slack__list_channels", "List Slack channels"), + _make_tool("mcp__jira__create_ticket", "Create a Jira ticket"), + ] + + @pytest.fixture() + def manager(self, builtin_tools, mcp_tools): + all_tools = builtin_tools + mcp_tools + return ToolSearchManager( + all_tools, + always_on_names={"bash", "read_file", "edit_file"}, + threshold=5, + max_results=3, + ) + + def test_should_activate_above_threshold(self, manager): + assert manager.should_activate() + + def test_should_not_activate_below_threshold(self, builtin_tools): + mgr = ToolSearchManager(builtin_tools, always_on_names={"bash", "read_file", "edit_file"}) + assert not mgr.should_activate() + + def test_visible_tools_initially_builtin_only(self, manager): + visible = manager.get_visible_tools() + names = {_tool_name(t) for t in visible} + assert names == {"bash", "read_file", "edit_file"} + + def test_deferred_tools_excludes_builtin(self, manager): + deferred = manager.get_deferred_tools() + names = {_tool_name(t) for t in deferred} + assert "bash" not in names + assert "mcp__github__create_issue" in names + + def test_search_returns_relevant_tools(self, manager): + results = manager.search("github issue") + names = {_tool_name(t) for t in results} + assert "mcp__github__create_issue" in names or "mcp__github__list_issues" in names + + def test_search_respects_max_results(self, manager): + results = manager.search("tool") + assert len(results) <= 3 + + def test_search_excludes_already_expanded(self, manager): + # Expand a github tool, then search for github — expanded tool should not appear + manager.expand_visible(["mcp__github__create_issue"]) + results = manager.search("github issue") + names = {_tool_name(t) for t in results} + assert "mcp__github__create_issue" not in names + + def test_expand_visible_adds_tools(self, manager): + manager.expand_visible(["mcp__github__create_issue"]) + visible = manager.get_visible_tools() + names = {_tool_name(t) for t in visible} + assert "mcp__github__create_issue" in names + + def test_expand_visible_returns_newly_added(self, manager): + added = manager.expand_visible(["mcp__github__create_issue", "mcp__slack__send_message"]) + assert len(added) == 2 + names = {_tool_name(t) for t in added} + assert names == {"mcp__github__create_issue", "mcp__slack__send_message"} + + def test_expand_visible_idempotent(self, manager): + manager.expand_visible(["mcp__github__create_issue"]) + added = manager.expand_visible(["mcp__github__create_issue"]) + assert added == [] + + def test_expand_visible_ignores_unknown(self, manager): + added = manager.expand_visible(["nonexistent_tool"]) + assert added == [] + + def test_deferred_excludes_expanded(self, manager): + manager.expand_visible(["mcp__github__create_issue"]) + deferred = manager.get_deferred_tools() + names = {_tool_name(t) for t in deferred} + assert "mcp__github__create_issue" not in names + + def test_get_all_tools_returns_everything(self, manager, builtin_tools, mcp_tools): + assert len(manager.get_all_tools()) == len(builtin_tools) + len(mcp_tools) + + def test_search_tool_definition_format(self, manager): + defn = manager.get_search_tool_definition() + assert defn["type"] == "function" + fn = defn["function"] + assert fn["name"] == "tool_search" + assert "query" in fn["parameters"]["properties"] + assert "query" in fn["parameters"]["required"] + + def test_search_tool_description_has_server_hint(self, manager): + defn = manager.get_search_tool_definition() + desc = defn["function"]["description"] + assert "github" in desc + assert "slack" in desc + assert "jira" in desc + + def test_format_search_results_empty(self, manager): + text = manager.format_search_results([]) + assert "No matching tools found" in text + + def test_format_search_results_with_tools(self, manager, mcp_tools): + text = manager.format_search_results(mcp_tools[:2]) + assert "Found 2" in text + assert "mcp__github__create_issue" in text + + +# --------------------------------------------------------------------------- +# Helper function tests +# --------------------------------------------------------------------------- + + +class TestMCPServerSummary: + def test_groups_by_server(self): + tools = [ + _make_tool("mcp__github__a"), + _make_tool("mcp__github__b"), + _make_tool("mcp__slack__c"), + ] + summary = _mcp_server_summary(tools) + assert "github (2 tools)" in summary + assert "slack (1 tool)" in summary + + def test_non_mcp_tools_counted_as_other(self): + tools = [_make_tool("custom_tool")] + summary = _mcp_server_summary(tools) + assert "other (1 tool)" in summary + + def test_empty_list(self): + assert _mcp_server_summary([]) == "" diff --git a/turnstone/cli.py b/turnstone/cli.py index c48248b2..fd9ea5a0 100644 --- a/turnstone/cli.py +++ b/turnstone/cli.py @@ -784,6 +784,24 @@ def main() -> None: default=0, help="Tool output truncation limit in chars, 0 for auto (50%% of context window) (default: 0)", ) + parser.add_argument( + "--tool-search", + choices=["auto", "on", "off"], + default="auto", + help="Dynamic tool search: auto (enable when tool count exceeds threshold), on, off (default: auto)", + ) + parser.add_argument( + "--tool-search-threshold", + type=int, + default=20, + help="Min tools before tool search activates (default: 20)", + ) + parser.add_argument( + "--tool-search-max-results", + type=int, + default=5, + help="Max tools returned per tool search query (default: 5)", + ) parser.add_argument( "--resume", default=None, @@ -917,6 +935,9 @@ def main() -> None: mcp_client=mcp_client, registry=registry, model_alias=model_alias or registry.default, + tool_search=args.tool_search, + tool_search_threshold=args.tool_search_threshold, + tool_search_max_results=args.tool_search_max_results, ) # Create workstream manager and initial workstream diff --git a/turnstone/core/config.py b/turnstone/core/config.py index ccc7dd52..c5fd4afe 100644 --- a/turnstone/core/config.py +++ b/turnstone/core/config.py @@ -70,6 +70,9 @@ _CONFIG_MAP: dict[str, dict[str, str]] = { "truncation": "tool_truncation", "agent_max_turns": "agent_max_turns", "skip_permissions": "skip_permissions", + "search": "tool_search", + "search_threshold": "tool_search_threshold", + "search_max_results": "tool_search_max_results", }, "server": { "host": "host", diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index 4fa5b178..6759d926 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -64,6 +64,9 @@ def _merge_consecutive(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: # Tool version for Anthropic's server-side web search (update when new version ships) _WEB_SEARCH_TOOL_TYPE = "web_search_20250305" +# Tool search: server-side BM25 tool discovery for deferred tools +_TOOL_SEARCH_TOOL_TYPE = "tool_search_tool_bm25_20251119" + # -- model capabilities ------------------------------------------------------- _ANTHROPIC_DEFAULT = ModelCapabilities( @@ -83,6 +86,7 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { supports_effort=True, effort_levels=("low", "medium", "high", "max"), supports_web_search=True, + supports_tool_search=True, ), "claude-sonnet-4-6": ModelCapabilities( context_window=200000, @@ -92,6 +96,7 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { supports_effort=True, effort_levels=("low", "medium", "high"), supports_web_search=True, + supports_tool_search=True, ), "claude-haiku-4-5": ModelCapabilities( context_window=200000, @@ -122,6 +127,7 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { token_param="max_tokens", thinking_mode="manual", supports_web_search=True, + supports_tool_search=True, ), "claude-sonnet-4": ModelCapabilities( context_window=200000, @@ -129,6 +135,7 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { token_param="max_tokens", thinking_mode="manual", supports_web_search=True, + supports_tool_search=True, ), } @@ -182,6 +189,31 @@ class AnthropicProvider: filtered.append({"type": _WEB_SEARCH_TOOL_TYPE, "name": "web_search"}) return filtered + # -- tool search injection ----------------------------------------------- + + def _inject_tool_search( + self, + tools: list[dict[str, Any]], + caps: ModelCapabilities, + deferred_names: frozenset[str] | None = None, + ) -> list[dict[str, Any]]: + """Mark deferred tools and add native server-side search tool. + + When the model supports tool search and ``deferred_names`` is provided, + tools whose name is in the deferred set get ``defer_loading: true``. + The BM25 search tool is appended so the model can discover them. + """ + if not caps.supports_tool_search or not deferred_names: + return tools + result = [] + for tool in tools: + if tool.get("name", "") in deferred_names: + result.append({**tool, "defer_loading": True}) + else: + result.append(tool) + result.append({"type": _TOOL_SEARCH_TOOL_TYPE, "name": "tool_search"}) + return result + # -- shared param logic -------------------------------------------------- def _build_thinking_and_kwargs( @@ -195,6 +227,7 @@ class AnthropicProvider: system_prompt: str, model: str, tools: list[dict[str, Any]] | None, + deferred_names: frozenset[str] | None = None, ) -> dict[str, Any]: """Build the full kwargs dict with thinking mode and effort params.""" thinking_params: dict[str, Any] = {} @@ -217,6 +250,7 @@ class AnthropicProvider: if tools: anthropic_tools = self.convert_tools(tools) anthropic_tools = self._inject_web_search(anthropic_tools, caps) + anthropic_tools = self._inject_tool_search(anthropic_tools, caps, deferred_names) kwargs["tools"] = anthropic_tools kwargs.update(thinking_params) @@ -371,6 +405,7 @@ class AnthropicProvider: temperature: float = 0.5, reasoning_effort: str = "medium", extra_params: dict[str, Any] | None = None, + deferred_names: frozenset[str] | None = None, ) -> Iterator[StreamChunk]: _ensure_anthropic() caps = self.get_capabilities(model) @@ -385,6 +420,7 @@ class AnthropicProvider: system_prompt, model, tools, + deferred_names, ) with client.messages.stream(**kwargs) as stream: @@ -536,6 +572,7 @@ class AnthropicProvider: temperature: float = 0.5, reasoning_effort: str = "medium", extra_params: dict[str, Any] | None = None, + deferred_names: frozenset[str] | None = None, ) -> CompletionResult: _ensure_anthropic() caps = self.get_capabilities(model) @@ -550,6 +587,7 @@ class AnthropicProvider: system_prompt, model, tools, + deferred_names, ) response = client.messages.create(**kwargs) diff --git a/turnstone/core/providers/_openai.py b/turnstone/core/providers/_openai.py index fd64b34f..b1fdde10 100644 --- a/turnstone/core/providers/_openai.py +++ b/turnstone/core/providers/_openai.py @@ -82,20 +82,22 @@ _OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), default_reasoning_effort="none", ), - # GPT-5.4 — 1M context window + # GPT-5.4 — 1M context window, native tool search "gpt-5.4": ModelCapabilities( context_window=1050000, max_output_tokens=128000, reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), default_reasoning_effort="none", + supports_tool_search=True, ), - # GPT-5.4 pro — always-reasoning, 1M context + # GPT-5.4 pro — always-reasoning, 1M context, native tool search "gpt-5.4-pro": ModelCapabilities( context_window=1050000, max_output_tokens=128000, supports_temperature=False, reasoning_effort_values=("medium", "high", "xhigh"), default_reasoning_effort="medium", + supports_tool_search=True, ), # O-series reasoning models "o1": ModelCapabilities( @@ -215,6 +217,30 @@ class OpenAIProvider: kwargs["web_search_options"] = {} return tools + # -- tool search --------------------------------------------------------- + + def _apply_tool_search( + self, + caps: ModelCapabilities, + tools: list[dict[str, Any]] | None, + deferred_names: frozenset[str] | None = None, + ) -> list[dict[str, Any]] | None: + """Mark deferred tools with ``defer_loading: true`` for native search. + + For GPT-5.4+ models that support tool search, OpenAI's API handles + discovery automatically — no explicit search tool is needed. + """ + if not caps.supports_tool_search or not deferred_names or not tools: + return tools + result = [] + for tool in tools: + name = tool.get("function", {}).get("name", "") + if name in deferred_names: + result.append({**tool, "defer_loading": True}) + else: + result.append(tool) + return result + # -- streaming ----------------------------------------------------------- def create_streaming( @@ -228,6 +254,7 @@ class OpenAIProvider: temperature: float = 0.5, reasoning_effort: str = "medium", extra_params: dict[str, Any] | None = None, + deferred_names: frozenset[str] | None = None, ) -> Iterator[StreamChunk]: caps = self.get_capabilities(model) kwargs: dict[str, Any] = { @@ -239,6 +266,7 @@ class OpenAIProvider: } self._apply_model_params(kwargs, caps, temperature, reasoning_effort) tools = self._apply_web_search(kwargs, caps, tools) + tools = self._apply_tool_search(caps, tools, deferred_names) if tools: kwargs["tools"] = tools if extra_params: @@ -332,6 +360,7 @@ class OpenAIProvider: temperature: float = 0.5, reasoning_effort: str = "medium", extra_params: dict[str, Any] | None = None, + deferred_names: frozenset[str] | None = None, ) -> CompletionResult: caps = self.get_capabilities(model) kwargs: dict[str, Any] = { @@ -342,6 +371,7 @@ class OpenAIProvider: } self._apply_model_params(kwargs, caps, temperature, reasoning_effort) tools = self._apply_web_search(kwargs, caps, tools) + tools = self._apply_tool_search(caps, tools, deferred_names) if tools: kwargs["tools"] = tools if extra_params: diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index 55be53a0..72ad13f3 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -76,6 +76,7 @@ class ModelCapabilities: reasoning_effort_values: tuple[str, ...] = () default_reasoning_effort: str = "medium" supports_web_search: bool = False + supports_tool_search: bool = False def _lookup_capabilities( @@ -119,6 +120,7 @@ class LLMProvider(Protocol): temperature: float = 0.5, reasoning_effort: str = "medium", extra_params: dict[str, Any] | None = None, + deferred_names: frozenset[str] | None = None, ) -> Iterator[StreamChunk]: """Create a streaming request, yielding normalized StreamChunks.""" ... @@ -134,6 +136,7 @@ class LLMProvider(Protocol): temperature: float = 0.5, reasoning_effort: str = "medium", extra_params: dict[str, Any] | None = None, + deferred_names: frozenset[str] | None = None, ) -> CompletionResult: """Create a non-streaming request, returning a normalized result.""" ... diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 39f98d70..3fe02857 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -50,9 +50,11 @@ from turnstone.core.providers import create_provider from turnstone.core.safety import is_command_blocked, sanitize_command from turnstone.core.sandbox import execute_math_sandboxed from turnstone.core.storage._registry import get_storage +from turnstone.core.tool_search import ToolSearchManager from turnstone.core.tools import ( AGENT_AUTO_TOOLS, AGENT_TOOLS, + BUILTIN_TOOL_NAMES, PRIMARY_KEY_MAP, TASK_AGENT_TOOLS, TASK_AUTO_TOOLS, @@ -158,6 +160,9 @@ class ChatSession: health_monitor: BackendHealthMonitor | None = None, node_id: str | None = None, ws_id: str | None = None, + tool_search: str = "auto", + tool_search_threshold: int = 20, + tool_search_max_results: int = 5, ): self.client = client self.model = model @@ -212,6 +217,17 @@ class ChatSession: self._tools = TOOLS self._task_tools = TASK_AGENT_TOOLS self._agent_tools = AGENT_TOOLS + # Dynamic tool search: defer MCP tools when tool count is high + self._tool_search: ToolSearchManager | None = None + if tool_search == "on" or ( + tool_search == "auto" and len(self._tools) > tool_search_threshold + ): + self._tool_search = ToolSearchManager( + self._tools, + always_on_names=set(BUILTIN_TOOL_NAMES), + threshold=tool_search_threshold, + max_results=tool_search_max_results, + ) self._init_system_messages() self._save_config() @@ -393,6 +409,14 @@ class ChatSession: "Look up documentation → man:\n" " man(page='tar')", ] + # Tool search hint (client-side mode only — native mode needs no hint) + if self._tool_search: + caps = self._provider.get_capabilities(self.model) + if not caps.supports_tool_search: + dev_parts.append( + "\n\nAdditional tools are available via tool_search. " + "Use it when you need a capability not in your current tool set." + ) if self.instructions: dev_parts.append("") dev_parts.append(self.instructions) @@ -429,6 +453,41 @@ class ChatSession: return {"chat_template_kwargs": kwargs} return None + # -- tool search helpers -------------------------------------------------- + + def _get_active_tools(self) -> list[dict[str, Any]] | None: + """Return the tool list to send to the LLM. + + When tool search is active: + - Native mode (provider supports it): send all tools (provider + marks deferred ones with defer_loading). + - Client-side fallback: send visible tools + synthetic tool_search. + + Without tool search: return self._tools unchanged. + """ + if self.creative_mode: + return None + if not self._tool_search: + return self._tools + # Check if provider supports native tool search + caps = self._provider.get_capabilities(self.model) + if caps.supports_tool_search: + # Provider handles defer_loading — send all tools + return self._tools + # Client-side fallback: visible tools + search tool + visible = self._tool_search.get_visible_tools() + return visible + [self._tool_search.get_search_tool_definition()] + + def _get_deferred_names(self) -> frozenset[str] | None: + """Return names of deferred tools for native provider search, or None.""" + if not self._tool_search: + return None + caps = self._provider.get_capabilities(self.model) + if not caps.supports_tool_search: + return None # Client-side mode — no deferred names for provider + deferred = self._tool_search.get_deferred_tools() + return frozenset(name for t in deferred if (name := t.get("function", {}).get("name", ""))) + # Retryable error names are now provided by LLMProvider.retryable_error_names. _MAX_RETRIES = 3 _RETRY_BASE_DELAY = 1.0 # seconds @@ -488,11 +547,12 @@ class ChatSession: client=client, model=model, messages=msgs, - tools=self._tools if not self.creative_mode else None, + tools=self._get_active_tools(), max_tokens=self.max_tokens, temperature=self.temperature, reasoning_effort=self.reasoning_effort, extra_params=self._provider_extra_params(provider=prov), + deferred_names=self._get_deferred_names(), ) except Exception as e: ename = type(e).__name__ @@ -883,7 +943,8 @@ class ChatSession: f"{GRAY}[request] model={self.model} " f"max_tokens={self.max_tokens} temp={self.temperature} " f"reasoning={self.reasoning_effort} " - f"tools={0 if self.creative_mode else len(self._tools)}{RESET}" + f"tools={0 if self.creative_mode else len(self._get_active_tools() or [])}" + f"{' (search)' if self._tool_search else ''}{RESET}" ) lines.append(f"{GRAY}[request] {len(msgs)} messages:{RESET}") for i, m in enumerate(msgs): @@ -937,7 +998,8 @@ class ChatSession: # Calibrate chars_per_token ratio from actual usage. all_msgs = self._full_messages() # system + self.messages (before append) - tool_def_chars = sum(len(json.dumps(t)) for t in self._tools) + active_tools = self._get_active_tools() or [] + tool_def_chars = sum(len(json.dumps(t)) for t in active_tools) total_chars = sum(self._msg_char_count(m) for m in all_msgs) + tool_def_chars if total_chars > 0 and prompt_tok > 0: self._chars_per_token = total_chars / prompt_tok @@ -1281,6 +1343,7 @@ class ChatSession: "man": self._prepare_man, "web_fetch": self._prepare_web_fetch, "web_search": self._prepare_web_search, + "tool_search": self._prepare_tool_search, "task": self._prepare_task, "plan": self._prepare_plan, "remember": self._prepare_remember, @@ -1763,6 +1826,48 @@ class ChatSession: "topic": topic, } + def _prepare_tool_search(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: + """Prepare a tool search query (client-side BM25 fallback).""" + query = (args.get("query") or "").strip() + if not query: + return { + "call_id": call_id, + "func_name": "tool_search", + "header": "\u2717 tool_search: empty query", + "preview": "", + "needs_approval": False, + "error": "Error: no query provided", + } + if not self._tool_search: + return { + "call_id": call_id, + "func_name": "tool_search", + "header": "\u2717 tool_search: not active", + "preview": "", + "needs_approval": False, + "error": "Tool search is not active.", + } + return { + "call_id": call_id, + "func_name": "tool_search", + "header": f"\u2699 tool_search: {query[:80]}", + "preview": f" {DIM}{query}{RESET}", + "needs_approval": False, + "execute": self._exec_tool_search, + "query": query, + } + + def _exec_tool_search(self, item: dict[str, Any]) -> tuple[str, str]: + """Execute a client-side tool search and expand visible tools.""" + assert self._tool_search is not None + query = item["query"] + results = self._tool_search.search(query) + # Expand discovered tools into the visible set + names = [t.get("function", {}).get("name", "") for t in results] + self._tool_search.expand_visible(names) + output = self._tool_search.format_search_results(results) + return item["call_id"], output + def _prepare_task(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: """Prepare a general-purpose sub-agent task for approval.""" prompt = (args.get("prompt") or "").strip() diff --git a/turnstone/core/tool_search.py b/turnstone/core/tool_search.py new file mode 100644 index 00000000..493e2855 --- /dev/null +++ b/turnstone/core/tool_search.py @@ -0,0 +1,243 @@ +"""Dynamic tool search — BM25 index and session-scoped visibility manager. + +When the total tool count exceeds a configurable threshold, deferred tools +are hidden from the LLM and discoverable via a ``tool_search`` function. +Native providers (Anthropic, OpenAI) handle search server-side; local +models (vLLM, llama.cpp) use the client-side BM25 fallback here. +""" + +from __future__ import annotations + +import math +import re +from collections import Counter +from typing import Any + +# --------------------------------------------------------------------------- +# BM25 index — lightweight, pure-Python, zero external deps +# --------------------------------------------------------------------------- + +_SPLIT_RE = re.compile(r"[_\-./\s]+") + + +def _tokenize(text: str) -> list[str]: + """Split text on whitespace, underscores, hyphens, dots.""" + return [t.lower() for t in _SPLIT_RE.split(text) if t] + + +class BM25Index: + """Okapi BM25 index over tool name + description text.""" + + def __init__(self, documents: list[str], *, k1: float = 1.5, b: float = 0.75) -> None: + self.k1 = k1 + self.b = b + self._docs = documents + self._doc_tokens: list[list[str]] = [_tokenize(d) for d in documents] + self._doc_lens = [len(t) for t in self._doc_tokens] + self._avgdl = sum(self._doc_lens) / max(len(self._doc_lens), 1) + self._n = len(documents) + # Document frequency per term + self._df: Counter[str] = Counter() + for tokens in self._doc_tokens: + for term in set(tokens): + self._df[term] += 1 + + def search(self, query: str, k: int = 5) -> list[int]: + """Return indices of top-k documents sorted by descending BM25 score.""" + q_tokens = _tokenize(query) + if not q_tokens: + return [] + scores: list[tuple[float, int]] = [] + for idx, doc_tokens in enumerate(self._doc_tokens): + score = self._score(q_tokens, doc_tokens, self._doc_lens[idx]) + if score > 0: + scores.append((score, idx)) + scores.sort(key=lambda x: (-x[0], x[1])) + return [idx for _, idx in scores[:k]] + + def _score(self, q_tokens: list[str], doc_tokens: list[str], dl: int) -> float: + tf_map: Counter[str] = Counter(doc_tokens) + score = 0.0 + for term in q_tokens: + if term not in tf_map: + continue + tf = tf_map[term] + df = self._df.get(term, 0) + idf = math.log((self._n - df + 0.5) / (df + 0.5) + 1.0) + numerator = tf * (self.k1 + 1) + denominator = tf + self.k1 * (1 - self.b + self.b * dl / self._avgdl) + score += idf * numerator / denominator + return score + + +# --------------------------------------------------------------------------- +# Tool search manager — partitions tools, tracks visibility +# --------------------------------------------------------------------------- + +_MCP_PREFIX_RE = re.compile(r"^mcp__(.+?)__") + + +def _tool_name(tool: dict[str, Any]) -> str: + """Extract function name from an OpenAI-format tool dict.""" + fn: dict[str, Any] = tool.get("function", {}) + name: str = fn.get("name", "") + return name + + +def _tool_text(tool: dict[str, Any]) -> str: + """Build searchable text from tool name + description.""" + fn = tool.get("function", {}) + return f"{fn.get('name', '')} {fn.get('description', '')}" + + +def _mcp_server_summary(tools: list[dict[str, Any]]) -> str: + """Summarise deferred tools by MCP server prefix for the hint.""" + servers: Counter[str] = Counter() + other = 0 + for tool in tools: + name = _tool_name(tool) + m = _MCP_PREFIX_RE.match(name) + if m: + servers[m.group(1)] += 1 + else: + other += 1 + parts = [f"{srv} ({cnt} tool{'s' if cnt != 1 else ''})" for srv, cnt in sorted(servers.items())] + if other: + parts.append(f"other ({other} tool{'s' if other != 1 else ''})") + return ", ".join(parts) + + +class ToolSearchManager: + """Session-scoped tool visibility manager with BM25 search. + + Partitions tools into always-on (built-in) and deferred (MCP) sets. + Tracks which deferred tools have been discovered and expanded into + the visible set for the current session. + """ + + def __init__( + self, + all_tools: list[dict[str, Any]], + always_on_names: set[str], + *, + threshold: int = 20, + max_results: int = 5, + ) -> None: + self._all_tools = all_tools + self._always_on: list[dict[str, Any]] = [] + self._deferred: list[dict[str, Any]] = [] + self._deferred_by_name: dict[str, dict[str, Any]] = {} + self._expanded: dict[str, None] = {} # ordered set (preserves discovery order) + self._threshold = threshold + self._max_results = max_results + + for tool in all_tools: + name = _tool_name(tool) + if name in always_on_names: + self._always_on.append(tool) + else: + self._deferred.append(tool) + self._deferred_by_name[name] = tool + + # BM25 index over deferred tools + texts = [_tool_text(t) for t in self._deferred] + self._index = BM25Index(texts) + + # Pre-compute server summary for the search tool description + self._server_hint = _mcp_server_summary(self._deferred) + + def should_activate(self) -> bool: + """Return True if tool search should be active (enough tools).""" + return len(self._all_tools) > self._threshold + + def get_visible_tools(self) -> list[dict[str, Any]]: + """Return always-on tools + any expanded (discovered) tools.""" + result = list(self._always_on) + for name in self._expanded: + tool = self._deferred_by_name.get(name) + if tool: + result.append(tool) + return result + + def get_deferred_tools(self) -> list[dict[str, Any]]: + """Return tools that are currently deferred (not yet discovered).""" + return [t for t in self._deferred if _tool_name(t) not in self._expanded] + + def get_all_tools(self) -> list[dict[str, Any]]: + """Return the full tool list (for native provider modes).""" + return list(self._all_tools) + + def search(self, query: str) -> list[dict[str, Any]]: + """Search deferred tools by query, return top-k matches. + + Already-expanded tools are excluded so every result is genuinely new. + """ + # Request extra results to compensate for filtering out expanded tools + indices = self._index.search(query, k=self._max_results + len(self._expanded)) + results = [] + for i in indices: + if _tool_name(self._deferred[i]) not in self._expanded: + results.append(self._deferred[i]) + if len(results) >= self._max_results: + break + return results + + def expand_visible(self, tool_names: list[str]) -> list[dict[str, Any]]: + """Promote discovered tools to the visible set. + + Returns the newly-expanded tool definitions (excludes tools + that were already visible). + """ + newly_added = [] + for name in tool_names: + if name not in self._expanded and name in self._deferred_by_name: + self._expanded[name] = None + newly_added.append(self._deferred_by_name[name]) + return newly_added + + def get_search_tool_definition(self) -> dict[str, Any]: + """Return the synthetic ``tool_search`` function tool definition. + + The description includes a dynamic hint listing available MCP + server names and tool counts so the model can craft specific queries. + """ + desc = ( + "Search for available tools by keyword. Returns matching tool " + "names and descriptions. Use this when you need a capability " + "not available in your current tool set." + ) + if self._server_hint: + desc += f" Available tool servers: {self._server_hint}." + return { + "type": "function", + "function": { + "name": "tool_search", + "description": desc, + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query describing the capability you need.", + }, + }, + "required": ["query"], + }, + }, + } + + def format_search_results(self, tools: list[dict[str, Any]]) -> str: + """Format search results as text for the tool_search response.""" + if not tools: + return "No matching tools found. Try a different search query." + lines = [] + for tool in tools: + fn = tool.get("function", {}) + name = fn.get("name", "") + desc = fn.get("description", "") + lines.append(f"- **{name}**: {desc}") + return ( + f"Found {len(tools)} matching tool(s):\n" + + "\n".join(lines) + + "\n\nThese tools are now available for use." + ) diff --git a/turnstone/core/tools.py b/turnstone/core/tools.py index 509245e5..6ce871b6 100644 --- a/turnstone/core/tools.py +++ b/turnstone/core/tools.py @@ -37,6 +37,7 @@ TASK_AGENT_TOOLS = [t for t in TOOLS if _META[t["function"]["name"]].get("task_a AGENT_AUTO_TOOLS = {n for n, m in _META.items() if m.get("auto_approve")} TASK_AUTO_TOOLS = {n for n, m in _META.items() if m.get("auto_approve")} PRIMARY_KEY_MAP = {n: m["primary_key"] for n, m in _META.items() if "primary_key" in m} +BUILTIN_TOOL_NAMES = frozenset(_META) def merge_mcp_tools( diff --git a/turnstone/server.py b/turnstone/server.py index 1b1f0fac..77380fe8 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1212,6 +1212,24 @@ def main() -> None: default=0, help="Tool output truncation limit in chars, 0 for auto (50%% of context window) (default: 0)", ) + parser.add_argument( + "--tool-search", + choices=["auto", "on", "off"], + default="auto", + help="Dynamic tool search: auto (enable when tool count exceeds threshold), on, off (default: auto)", + ) + parser.add_argument( + "--tool-search-threshold", + type=int, + default=20, + help="Min tools before tool search activates (default: 20)", + ) + parser.add_argument( + "--tool-search-max-results", + type=int, + default=5, + help="Max tools returned per tool search query (default: 5)", + ) parser.add_argument( "--resume", default=None, @@ -1462,6 +1480,9 @@ def main() -> None: health_monitor=health_monitor, node_id=_node_id, ws_id=ws_id, + tool_search=args.tool_search, + tool_search_threshold=args.tool_search_threshold, + tool_search_max_results=args.tool_search_max_results, ) # Create workstream manager and initial workstream