diff --git a/README.md b/README.md index 7ed8048d..c55a8e32 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ turnstone/ │ ├── session.py # ChatSession — multi-turn loop, tool dispatch, agents │ ├── tools.py # Tool definitions (auto-loaded from JSON) │ ├── workstream.py # WorkstreamManager — parallel independent sessions +│ ├── mcp_client.py # MCP client manager (external tool servers) │ ├── config.py # Unified TOML config (~/.config/turnstone/config.toml) │ ├── memory.py # SQLite persistence (memories, conversations, FTS5) │ ├── metrics.py # Prometheus-compatible metrics collector @@ -193,7 +194,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct ## Tools -14 built-in tools, 2 agent tools: +14 built-in tools, 2 agent tools, plus external tools via MCP: | Tool | Description | Auto-approved | |------|-------------|:---:| @@ -211,6 +212,31 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct | `forget` | Remove a memory | yes | | `task` | Spawn autonomous sub-agent | | | `plan` | Explore codebase, write .plan.md | | +| `mcp__*` | External tools from MCP servers | | + +### MCP Tool Servers + +Turnstone supports the [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) for connecting external tool servers. MCP tools are discovered at startup, converted to OpenAI function-calling format, and merged with built-in tools. Each MCP tool is prefixed with `mcp__{server}__{tool}` to avoid name collisions. + +Configure via `config.toml` or `--mcp-config`: + +```toml +[mcp.servers.github] +command = "npx" +args = ["-y", "@modelcontextprotocol/server-github"] + +[mcp.servers.github.env] +GITHUB_TOKEN = "ghp_..." +``` + +Or use a standard MCP JSON config file: + +```bash +turnstone --mcp-config ~/.config/turnstone/mcp.json +turnstone-server --mcp-config ~/.config/turnstone/mcp.json +``` + +Use `/mcp` in the REPL to list connected tools. MCP tools require user approval by default (overridden by `--skip-permissions` or UI auto-approve). ## Configuration @@ -249,6 +275,15 @@ host = "0.0.0.0" port = 8090 url = "http://localhost:8090" # used by CLI /cluster commands poll_interval = 10 + +[mcp] +config_path = "" # path to MCP JSON config file (alternative to TOML sections) + +[mcp.servers.example] # one section per MCP server +command = "npx" +args = ["-y", "@modelcontextprotocol/server-example"] +# type = "stdio" # "stdio" (default) or "http" +# url = "" # for HTTP transport ``` Precedence: CLI args > environment variables > config.toml > defaults. diff --git a/docs/architecture.md b/docs/architecture.md index 77e21b1d..92979a56 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,8 +2,8 @@ Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) and -gives the model 14 tools for reading, writing, searching, planning, and -executing code. +gives the model 14 built-in tools plus external tools via MCP (Model Context +Protocol) for reading, writing, searching, planning, and executing code. The core design principle is a **UI-agnostic engine with pluggable frontends**. The engine (`ChatSession`) drives the conversation loop -- streaming, tool @@ -34,6 +34,7 @@ turnstone/ session.py ChatSession engine, SessionUI protocol, tool dispatch workstream.py Parallel workstream manager (WorkstreamState, Workstream, WorkstreamManager) tools.py Tool schema loader (JSON -> OpenAI function-calling format) + mcp_client.py MCPClientManager — MCP server connections, tool discovery, async-sync bridge memory.py SQLite persistence (conversations, memories, FTS5 search) metrics.py Prometheus-compatible metrics collector (MetricsCollector) edit.py File edit utilities (find_occurrences, pick_nearest) @@ -379,6 +380,7 @@ from each schema and builds: - `TASK_AGENT_TOOLS` -- subset with `task_agent: true` - `AGENT_AUTO_TOOLS` / `TASK_AUTO_TOOLS` -- sets of tool names with `auto_approve: true` - `PRIMARY_KEY_MAP` -- `{name: primary_key}` for JSON fallback recovery +- `merge_mcp_tools(builtin, mcp_tools)` -- merges built-in + MCP tools at session init ### 14 Tools by Category @@ -425,8 +427,8 @@ separation allows the UI to show previews before any side effects occur. a subset of tools and its own system prompt. The sub-agent runs independently, then returns the final content as the tool result. -- **task**: uses `TASK_AGENT_TOOLS` (includes bash, read, write, edit, search) -- **plan**: uses `AGENT_TOOLS` (read-only subset for exploration). Writes output +- **task**: uses `self._task_tools` (`TASK_AGENT_TOOLS` + MCP tools) +- **plan**: uses `self._agent_tools` (`AGENT_TOOLS` + MCP tools). Writes output to `.plan-.md` — unique per `ChatSession` so concurrent workstreams don't collide. On repeat invocations the prior `plan` tool call and its result are forwarded from `self.messages` so the agent refines the existing plan rather @@ -442,6 +444,29 @@ independently, then returns the final content as the tool result. and returns whatever content was generated. `finish_reason: "content_filter"` returns a placeholder. +### MCP Tool Integration + +`MCPClientManager` (`turnstone/core/mcp_client.py`) connects to external MCP servers +and exposes their tools alongside built-in tools. The MCP SDK is fully async; turnstone +bridges this with a background asyncio event loop in a daemon thread. + +**Lifecycle:** +1. `create_mcp_client()` reads server configs from TOML or JSON +2. `MCPClientManager.start()` launches the background event loop thread +3. `_connect_all()` connects to each server (stdio subprocess or HTTP), runs + `initialize()` + `list_tools()`, converts schemas to OpenAI format +4. `ChatSession.__init__` receives the manager and builds `self._tools` (built-in + MCP) +5. `_prepare_tool()` routes MCP tools to `_prepare_mcp_tool()` / `_exec_mcp_tool()` +6. `_exec_mcp_tool()` calls `call_tool_sync()` which dispatches to the async loop + via `asyncio.run_coroutine_threadsafe()` + +**Tool naming:** `mcp__{server}__{tool}` — double underscore delimiter, validated +at connection time (server names with `__` are rejected). + +**Error isolation:** Per-server connection failures are caught and logged; other +servers still connect. Tool execution errors return error strings to the LLM +rather than crashing the session. + ### Tool Output Truncation Tool execution results (bash, read_file, search, math, man) are truncated by diff --git a/docs/diagrams/02-package-structure.puml b/docs/diagrams/02-package-structure.puml index 4e1bfe8c..6d4fed2f 100644 --- a/docs/diagrams/02-package-structure.puml +++ b/docs/diagrams/02-package-structure.puml @@ -34,6 +34,7 @@ package "turnstone/core/" <> { component [edit.py\nFile editing] as edit <> component [web.py\nWeb helpers] as web <> component [auth.py\nAuthentication] as auth <> + component [mcp_client.py\nMCPClientManager] as mcp <> } ' MQ subsystem @@ -105,6 +106,8 @@ session --> safety session --> sandbox session --> edit session --> web +session --> mcp : optional +mcp --> config tools --> schemas ' MQ dependencies diff --git a/docs/diagrams/03-core-engine-classes.puml b/docs/diagrams/03-core-engine-classes.puml index 2a00c61e..a309b503 100644 --- a/docs/diagrams/03-core-engine-classes.puml +++ b/docs/diagrams/03-core-engine-classes.puml @@ -71,6 +71,10 @@ class "ChatSession" as ChatSession { - messages: list[dict] - _msg_tokens: list[int] - _session_id: str + - _mcp_client: MCPClientManager | None + - _tools: list[dict] + - _task_tools: list[dict] + - _agent_tools: list[dict] - _read_files: set[str] - system_messages: list[dict] -- @@ -82,6 +86,8 @@ class "ChatSession" as ChatSession { - _create_stream_with_retry(msgs) → Stream - _execute_tools(tool_calls) → (results, feedback) - _prepare_tool(tc) → item dict + - _prepare_mcp_tool(call_id, name, args) → item dict + - _exec_mcp_tool(item) → (call_id, output) - _run_agent(messages, tools, ...) → str - _compact_messages(auto: bool) - _full_messages() → list[dict] @@ -142,6 +148,25 @@ enum "WorkstreamState" as WsState { ERROR } +' MCPClientManager +class "MCPClientManager" as MCPMgr { + - _sessions: dict[str, ClientSession] + - _tools: list[dict] + - _tool_map: dict[str, tuple] + -- + + start() + + get_tools() → list[dict] + + is_mcp_tool(name) → bool + + call_tool_sync(name, args) → str + + shutdown() + -- + Background asyncio event loop + bridges async MCP SDK to + sync ChatSession dispatch. + -- + core/mcp_client.py +} + ' Relationships SessionUI <|.. TerminalUI TerminalUI <|-- WsTermUI @@ -149,6 +174,7 @@ SessionUI <|.. WebUI SessionUI <|.. NullUI ChatSession --> SessionUI : uses +ChatSession --> MCPMgr : optional ChatSession <|-- HeadlessSession WsMgr --> "*" Ws : manages diff --git a/docs/diagrams/05-tool-pipeline.puml b/docs/diagrams/05-tool-pipeline.puml index d8143806..7c1e175e 100644 --- a/docs/diagrams/05-tool-pipeline.puml +++ b/docs/diagrams/05-tool-pipeline.puml @@ -42,6 +42,8 @@ partition "Phase 1: Prepare" #E8F5E9 { │ remember │ ✗ Auto-approve │ │ recall │ ✗ Auto-approve │ │ forget │ ✗ Auto-approve │ + ├─────────────┼──────────────────┤ + │ mcp__* │ ✓ Yes (external) │ └─────────────┴──────────────────┘ end note @@ -108,7 +110,8 @@ partition "Phase 3: Execute" #E3F2FD { ├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only) ├─ _exec_remember: SQLite INSERT OR REPLACE ├─ _exec_recall: SQLite FTS5/LIKE search - └─ _exec_forget: SQLite DELETE + ├─ _exec_forget: SQLite DELETE + └─ _exec_mcp_tool: MCPClientManager.call_tool_sync() end note :Collect results: [(call_id, output), ...]; diff --git a/docs/diagrams/png/02-package-structure.png b/docs/diagrams/png/02-package-structure.png index 4c9a2ee7..d6c31dad 100644 --- a/docs/diagrams/png/02-package-structure.png +++ b/docs/diagrams/png/02-package-structure.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d4b1db039e9edbed8b2b7246328b49ae87afbe58c1b0fca0812712678faee366 -size 252572 +oid sha256:e1aebded0994beccbf1af25c5c1c216775a59a4e343bcbafe69bcc1172907c71 +size 291899 diff --git a/docs/diagrams/png/03-core-engine-classes.png b/docs/diagrams/png/03-core-engine-classes.png index cf551e49..62d055f3 100644 --- a/docs/diagrams/png/03-core-engine-classes.png +++ b/docs/diagrams/png/03-core-engine-classes.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cca7c884004478d91d6d7672a59eadb370822750a4a2ecf2ff1e6ec9f4c3da89 -size 231642 +oid sha256:7456dcc6a68b22d841524062028298814293b0e4dfab4097c536847d1528bb8a +size 275572 diff --git a/docs/diagrams/png/05-tool-pipeline.png b/docs/diagrams/png/05-tool-pipeline.png index 42edba8e..bb89cf25 100644 --- a/docs/diagrams/png/05-tool-pipeline.png +++ b/docs/diagrams/png/05-tool-pipeline.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:335315486c253966d9004933d8c887016a4a9e7c1b168a82c6ff339a5172b458 -size 237025 +oid sha256:cb36b4924394cf54d6aaef454cced317b72e336e580cc6ac25cd4b9d0917bec5 +size 243422 diff --git a/docs/tools.md b/docs/tools.md index aa9cd30d..f88e251f 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -1,8 +1,10 @@ # Tools Reference -turnstone exposes 14 tools to the LLM via the OpenAI function-calling interface. -Each tool is defined as a JSON file under `turnstone/tools/` and loaded at startup -by `turnstone/core/tools.py`. +turnstone exposes 14 built-in tools plus any number of external MCP tools to the +LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON +files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`. +MCP tools are discovered from configured MCP servers at startup by +`turnstone/core/mcp_client.py`. --- @@ -400,3 +402,98 @@ Remove a persistent memory by key. | `remember` | Memory | Yes | No | No | `key` | | `recall` | Memory | Yes | No | No | `query` | | `forget` | Memory | Yes | No | No | `key` | + +--- + +## MCP Tools (External) + +Turnstone supports the [Model Context Protocol](https://modelcontextprotocol.io/) +(MCP) for connecting external tool servers — GitHub, databases, filesystems, or any +MCP-compatible service. + +### How it works + +1. **Configuration**: MCP servers are defined in `config.toml` under `[mcp.servers.*]` + sections, or via a standard MCP JSON config file (`--mcp-config`). + +2. **Discovery**: At startup, `MCPClientManager` connects to each configured server + (via stdio subprocess or HTTP), performs the MCP `initialize` handshake, and calls + `tools/list` to discover available tools. + +3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI + function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`. + +4. **Merging**: MCP tools are appended after the 14 built-in tools via + `merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority. + +5. **Dispatch**: When the LLM calls an MCP tool, `_prepare_mcp_tool()` builds a + generic approval preview and `_exec_mcp_tool()` calls `MCPClientManager.call_tool_sync()`, + which dispatches the call to the background asyncio event loop. + +### Approval behavior + +MCP tools **require user approval by default** (`needs_approval: True`). turnstone +does not auto-approve MCP tools based on their schema, since it cannot guarantee +that external tools are read-only. However, global overrides such as +`--skip-permissions` or the UI's "always allow" setting will auto-approve all +tools, including MCP tools. + +### Sub-agent availability + +MCP tools are available to: +- **Main session** — full access +- **Task sub-agents** — via `self._task_tools` (merged list) +- **Plan sub-agents** — via `self._agent_tools` (merged list) + +### Naming convention + +MCP tool names follow the pattern `mcp__{server}__{original}`: + +- `mcp__github__search_repos` — `search_repos` tool from `github` server +- `mcp__postgres__query` — `query` tool from `postgres` server + +Server names must not contain `__` (double underscore), which is reserved as the +delimiter. Servers with `__` in their name are rejected at connection time. + +### Configuration + +**TOML** (`~/.config/turnstone/config.toml`): + +```toml +[mcp.servers.github] +command = "npx" +args = ["-y", "@modelcontextprotocol/server-github"] + +[mcp.servers.github.env] +GITHUB_TOKEN = "ghp_..." + +[mcp.servers.remote] +type = "http" +url = "https://mcp.example.com/mcp" +``` + +**JSON** (standard `mcpServers` format, via `--mcp-config`): + +```json +{ + "mcpServers": { + "github": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-github"], + "env": {"GITHUB_TOKEN": "ghp_..."} + } + } +} +``` + +### Introspection + +Use the `/mcp` slash command to list all connected MCP tools: + +``` +/mcp +MCP tools (3): + mcp__github__search_repos [MCP: github] Search GitHub repositories + mcp__github__create_issue [MCP: github] Create a GitHub issue + mcp__postgres__query [MCP: postgres] Run a SQL query +``` diff --git a/pyproject.toml b/pyproject.toml index ae2508eb..da7859cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Artificial Intelligence", ] -dependencies = ["openai>=2.24", "httpx>=0.28"] +dependencies = ["openai>=2.24", "httpx>=0.28", "mcp>=1.6"] [project.urls] Homepage = "https://github.com/turnstonelabs/turnstone" @@ -35,6 +35,7 @@ mq = ["redis>=7.2"] console = ["redis>=7.2"] sim = ["redis>=7.2"] + [project.scripts] turnstone = "turnstone.cli:main" turnstone-eval = "turnstone.eval:main" @@ -99,6 +100,10 @@ exclude_lines = [ module = ["sympy", "sympy.*", "numpy", "numpy.*"] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ["mcp", "mcp.*"] +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "tests.*" disallow_untyped_defs = false diff --git a/tests/test_mcp_client.py b/tests/test_mcp_client.py new file mode 100644 index 00000000..3054091d --- /dev/null +++ b/tests/test_mcp_client.py @@ -0,0 +1,411 @@ +"""Tests for turnstone.core.mcp_client — MCP client manager and config loading.""" + +from __future__ import annotations + +import json +from contextlib import AsyncExitStack +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from turnstone.core.mcp_client import ( + MCPClientManager, + _mcp_to_openai, + load_mcp_config, +) +from turnstone.core.tools import TOOLS, merge_mcp_tools + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _fake_mcp_tool(name: str = "search", description: str = "Search stuff") -> MagicMock: + """Create a mock MCP tool object matching the SDK's Tool type.""" + tool = MagicMock() + tool.name = name + tool.description = description + tool.inputSchema = { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + } + return tool + + +def _fake_openai_tool(name: str = "mcp__test__search") -> dict[str, Any]: + """Create a fake OpenAI-format tool dict.""" + return { + "type": "function", + "function": { + "name": name, + "description": "[MCP: test] Search stuff", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } + + +# --------------------------------------------------------------------------- +# Schema conversion +# --------------------------------------------------------------------------- + + +class TestMcpToOpenai: + def test_basic_conversion(self): + tool = _fake_mcp_tool("search_repos", "Search GitHub repos") + result = _mcp_to_openai("github", tool) + + assert result["type"] == "function" + func = result["function"] + assert func["name"] == "mcp__github__search_repos" + assert "[MCP: github]" in func["description"] + assert func["parameters"]["type"] == "object" + assert "query" in func["parameters"]["properties"] + + def test_name_prefixing(self): + tool = _fake_mcp_tool("list_files") + result = _mcp_to_openai("fs", tool) + assert result["function"]["name"] == "mcp__fs__list_files" + + def test_missing_input_schema(self): + tool = MagicMock() + tool.name = "ping" + tool.description = "Ping the server" + tool.inputSchema = None + result = _mcp_to_openai("test", tool) + assert result["function"]["parameters"] == {"type": "object", "properties": {}} + + def test_empty_description(self): + tool = MagicMock() + tool.name = "noop" + tool.description = "" + tool.inputSchema = {"type": "object", "properties": {}} + result = _mcp_to_openai("test", tool) + assert result["function"]["description"] == "[MCP: test] " + + +# --------------------------------------------------------------------------- +# Config loading +# --------------------------------------------------------------------------- + + +class TestLoadMcpConfig: + def test_load_from_json_file(self, tmp_path): + config_file = tmp_path / "mcp.json" + config_file.write_text( + json.dumps( + { + "mcpServers": { + "github": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-github"], + "env": {"GITHUB_TOKEN": "test"}, + } + } + } + ) + ) + result = load_mcp_config(str(config_file)) + assert "github" in result + assert result["github"]["command"] == "npx" + assert result["github"]["env"]["GITHUB_TOKEN"] == "test" + + def test_load_from_toml(self): + mock_config = { + "servers": { + "postgres": { + "type": "http", + "url": "https://mcp.example.com/mcp", + } + } + } + with patch("turnstone.core.mcp_client.load_config", return_value=mock_config): + result = load_mcp_config(None) + assert "postgres" in result + assert result["postgres"]["url"] == "https://mcp.example.com/mcp" + + def test_empty_when_no_config(self): + with patch("turnstone.core.mcp_client.load_config", return_value={}): + result = load_mcp_config(None) + assert result == {} + + def test_json_file_not_found(self, tmp_path): + with patch("turnstone.core.mcp_client.load_config", return_value={}): + result = load_mcp_config(str(tmp_path / "nonexistent.json")) + assert result == {} + + def test_toml_config_path_redirect(self): + """TOML [mcp] config_path redirects to JSON file.""" + # load_config returns a section with config_path pointing to a nonexistent file + mock_config = {"config_path": "/tmp/nonexistent_mcp.json"} + with patch("turnstone.core.mcp_client.load_config", return_value=mock_config): + result = load_mcp_config(None) + assert result == {} + + def test_invalid_json(self, tmp_path): + config_file = tmp_path / "bad.json" + config_file.write_text("not json") + with patch("turnstone.core.mcp_client.load_config", return_value={}): + result = load_mcp_config(str(config_file)) + assert result == {} + + +# --------------------------------------------------------------------------- +# merge_mcp_tools +# --------------------------------------------------------------------------- + + +class TestMergeTools: + def test_merge_preserves_builtin(self): + mcp_tools = [_fake_openai_tool()] + merged = merge_mcp_tools(TOOLS, mcp_tools) + # First N should be built-in + for i, t in enumerate(TOOLS): + assert merged[i] is t + + def test_merge_appends_mcp(self): + mcp_tools = [_fake_openai_tool("mcp__a__x"), _fake_openai_tool("mcp__b__y")] + merged = merge_mcp_tools(TOOLS, mcp_tools) + assert len(merged) == len(TOOLS) + 2 + assert merged[-2]["function"]["name"] == "mcp__a__x" + assert merged[-1]["function"]["name"] == "mcp__b__y" + + def test_merge_empty_mcp(self): + merged = merge_mcp_tools(TOOLS, []) + assert merged == TOOLS + + def test_merge_does_not_mutate_input(self): + mcp_tools = [_fake_openai_tool()] + original_len = len(TOOLS) + merge_mcp_tools(TOOLS, mcp_tools) + assert len(TOOLS) == original_len + + +# --------------------------------------------------------------------------- +# MCPClientManager unit tests (no real MCP servers) +# --------------------------------------------------------------------------- + + +class TestMCPClientManager: + def test_init_state(self): + mgr = MCPClientManager({"test": {"command": "echo"}}) + assert mgr.get_tools() == [] + assert mgr.is_mcp_tool("anything") is False + assert mgr.server_count == 0 + + def test_get_tools_returns_copy(self): + mgr = MCPClientManager({}) + mgr._tools = [_fake_openai_tool()] + tools = mgr.get_tools() + assert len(tools) == 1 + tools.clear() # mutate the copy + assert len(mgr.get_tools()) == 1 # original unchanged + + def test_is_mcp_tool(self): + mgr = MCPClientManager({}) + mgr._tool_map["mcp__gh__search"] = ("gh", "search") + assert mgr.is_mcp_tool("mcp__gh__search") is True + assert mgr.is_mcp_tool("bash") is False + + def test_server_count(self): + mgr = MCPClientManager({}) + mgr._sessions["a"] = MagicMock() + mgr._sessions["b"] = MagicMock() + assert mgr.server_count == 2 + + def test_call_tool_sync_unknown_tool(self): + mgr = MCPClientManager({}) + with pytest.raises(ValueError, match="Unknown MCP tool"): + mgr.call_tool_sync("mcp__no__such", {}) + + def test_call_tool_sync_disconnected_server(self): + mgr = MCPClientManager({}) + mgr._tool_map["mcp__dead__ping"] = ("dead", "ping") + # No session registered for "dead" + with pytest.raises(RuntimeError, match="not connected"): + mgr.call_tool_sync("mcp__dead__ping", {}) + + def test_shutdown_on_unstarted_manager(self): + """shutdown() should not raise when called on a manager that was never started.""" + mgr = MCPClientManager({}) + mgr.shutdown() # should be a no-op + + +# --------------------------------------------------------------------------- +# Session integration (mock MCP client) +# --------------------------------------------------------------------------- + + +class TestSessionIntegration: + @pytest.fixture() + def tmp_db(self, tmp_path, monkeypatch): + monkeypatch.setenv("TURNSTONE_DB_PATH", str(tmp_path / "test.db")) + from turnstone.core.memory import open_db + + open_db() + + def _make_session(self, mcp_client=None, **kwargs): + from turnstone.core.session import ChatSession + + defaults: dict[str, Any] = dict( + client=MagicMock(), + model="test-model", + ui=MagicMock(), + instructions=None, + temperature=0.5, + max_tokens=4096, + tool_timeout=30, + mcp_client=mcp_client, + ) + defaults.update(kwargs) + return ChatSession(**defaults) + + def test_session_without_mcp(self, tmp_db): + session = self._make_session(mcp_client=None) + assert session._tools is TOOLS + assert session._mcp_client is None + + def test_session_with_mcp(self, tmp_db): + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + session = self._make_session(mcp_client=mock_mcp) + assert len(session._tools) == len(TOOLS) + 1 + assert session._tools[-1]["function"]["name"] == "mcp__test__search" + + def test_task_tools_include_mcp(self, tmp_db): + from turnstone.core.tools import TASK_AGENT_TOOLS + + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + session = self._make_session(mcp_client=mock_mcp) + assert len(session._task_tools) == len(TASK_AGENT_TOOLS) + 1 + + def test_agent_tools_include_mcp(self, tmp_db): + from turnstone.core.tools import AGENT_TOOLS + + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + session = self._make_session(mcp_client=mock_mcp) + assert len(session._agent_tools) == len(AGENT_TOOLS) + 1 + + def test_prepare_mcp_tool(self, tmp_db): + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + mock_mcp.is_mcp_tool.return_value = True + session = self._make_session(mcp_client=mock_mcp) + + tc = { + "id": "call_123", + "function": { + "name": "mcp__test__search", + "arguments": '{"query": "hello"}', + }, + } + prepared = session._prepare_tool(tc) + assert prepared["func_name"] == "mcp__test__search" + assert prepared["needs_approval"] is True + assert "mcp:test/search" in prepared["header"] + assert callable(prepared["execute"]) + + def test_unknown_tool_without_mcp(self, tmp_db): + session = self._make_session(mcp_client=None) + tc = { + "id": "call_456", + "function": {"name": "nonexistent", "arguments": "{}"}, + } + prepared = session._prepare_tool(tc) + assert "error" in prepared + assert "Unknown tool" in prepared["error"] + + def test_mcp_command_no_client(self, tmp_db): + session = self._make_session(mcp_client=None) + session.handle_command("/mcp") + session.ui.on_info.assert_called_once() + assert "No MCP servers" in session.ui.on_info.call_args[0][0] + + def test_mcp_command_with_tools(self, tmp_db): + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + session = self._make_session(mcp_client=mock_mcp) + session.handle_command("/mcp") + session.ui.on_info.assert_called_once() + output = session.ui.on_info.call_args[0][0] + assert "MCP tools (1)" in output + assert "mcp__test__search" in output + + def test_exec_mcp_tool(self, tmp_db): + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + mock_mcp.is_mcp_tool.return_value = True + mock_mcp.call_tool_sync.return_value = "result text" + session = self._make_session(mcp_client=mock_mcp) + + item = { + "call_id": "call_789", + "mcp_func_name": "mcp__test__search", + "mcp_args": {"query": "hello"}, + } + call_id, output = session._exec_mcp_tool(item) + assert call_id == "call_789" + assert output == "result text" + mock_mcp.call_tool_sync.assert_called_once_with( + "mcp__test__search", {"query": "hello"}, timeout=30 + ) + + def test_exec_mcp_tool_error(self, tmp_db): + mock_mcp = MagicMock() + mock_mcp.get_tools.return_value = [_fake_openai_tool()] + mock_mcp.is_mcp_tool.return_value = True + mock_mcp.call_tool_sync.side_effect = RuntimeError("server crashed") + session = self._make_session(mcp_client=mock_mcp) + + item = { + "call_id": "call_err", + "mcp_func_name": "mcp__test__search", + "mcp_args": {"query": "hello"}, + } + call_id, output = session._exec_mcp_tool(item) + assert call_id == "call_err" + assert "MCP tool error" in output + assert "server crashed" in output + + +# --------------------------------------------------------------------------- +# Server name validation +# --------------------------------------------------------------------------- + + +class TestServerNameValidation: + def test_double_underscore_in_name(self): + """Server names with __ should be rejected during _connect_one.""" + import asyncio + + async def _run() -> None: + mgr = MCPClientManager({"my__bad": {"command": "echo"}}) + async with AsyncExitStack() as stack: + mgr._exit_stack = stack + await mgr._connect_one("my__bad", {"command": "echo"}) + # Should not have connected + assert "my__bad" not in mgr._sessions + assert mgr.get_tools() == [] + + asyncio.run(_run()) + + +# --------------------------------------------------------------------------- +# create_mcp_client guard +# --------------------------------------------------------------------------- + + +class TestCreateMcpClient: + def test_returns_none_when_no_config(self): + with patch("turnstone.core.mcp_client.load_mcp_config", return_value={}): + from turnstone.core.mcp_client import create_mcp_client + + result = create_mcp_client() + assert result is None diff --git a/turnstone/cli.py b/turnstone/cli.py index 9504cb65..ff337475 100644 --- a/turnstone/cli.py +++ b/turnstone/cli.py @@ -54,6 +54,7 @@ SLASH_COMMANDS = [ "/compact", "/creative", "/debug", + "/mcp", "/help", "/exit", "/quit", @@ -825,9 +826,15 @@ def main() -> None: default=os.environ.get("TURNSTONE_AUTH_TOKEN", ""), help="Bearer token for authenticating to turnstone services (default: $TURNSTONE_AUTH_TOKEN)", ) + parser.add_argument( + "--mcp-config", + default=None, + metavar="PATH", + help="Path to MCP server config file (standard mcpServers JSON format)", + ) from turnstone.core.config import apply_config - apply_config(parser, ["api", "model", "session", "tools", "console", "auth"]) + apply_config(parser, ["api", "model", "session", "tools", "console", "auth", "mcp"]) args = parser.parse_args() # Prune stale / empty sessions on startup @@ -848,6 +855,11 @@ def main() -> None: # Detect or use provided model model = args.model or detect_model(client) + # Initialize MCP client (connects to configured MCP servers, if any) + from turnstone.core.mcp_client import create_mcp_client + + mcp_client = create_mcp_client(getattr(args, "mcp_config", None)) + # Session factory — captures shared config for creating workstream sessions def session_factory(ui: SessionUI | None) -> ChatSession: assert ui is not None, "session_factory requires a non-None UI" @@ -865,6 +877,7 @@ def main() -> None: auto_compact_pct=args.auto_compact_pct, agent_max_turns=args.agent_max_turns, tool_truncation=args.tool_truncation, + mcp_client=mcp_client, ) # Create workstream manager and initial workstream @@ -908,6 +921,10 @@ def main() -> None: # Print banner print(f"\n{bold('Chat')} with {cyan(model)}") + if mcp_client: + mcp_tools = mcp_client.get_tools() + if mcp_tools: + print(f"MCP tools: {len(mcp_tools)} from {mcp_client.server_count} server(s)") print("Type /help for commands, /ws for workstreams, /exit or Ctrl+D to quit.\n") # Prompt string -- use a short display name @@ -961,6 +978,9 @@ def main() -> None: except Exception as e: print(f"\n{red(f'Error: {e}')}") + if mcp_client: + mcp_client.shutdown() + print("Goodbye.") diff --git a/turnstone/core/config.py b/turnstone/core/config.py index 07f13d20..dae03707 100644 --- a/turnstone/core/config.py +++ b/turnstone/core/config.py @@ -99,6 +99,9 @@ _CONFIG_MAP: dict[str, dict[str, str]] = { "auth": { "token": "auth_token", }, + "mcp": { + "config_path": "mcp_config", + }, } diff --git a/turnstone/core/mcp_client.py b/turnstone/core/mcp_client.py new file mode 100644 index 00000000..1cea4822 --- /dev/null +++ b/turnstone/core/mcp_client.py @@ -0,0 +1,287 @@ +"""MCP (Model Context Protocol) client manager. + +Connects to external MCP tool servers and exposes their tools alongside +turnstone's built-in tools. + +Architecture: the MCP SDK is fully async, but turnstone's ChatSession is +synchronous. We bridge the two by running a dedicated asyncio event loop +in a daemon thread. ``call_tool_sync`` dispatches coroutines onto that loop +via ``asyncio.run_coroutine_threadsafe``. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import threading +from contextlib import AsyncExitStack +from pathlib import Path +from typing import Any + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.client.streamable_http import streamablehttp_client + +from turnstone.core.config import load_config + +log = logging.getLogger("turnstone.mcp") + + +# --------------------------------------------------------------------------- +# MCP ↔ OpenAI schema conversion +# --------------------------------------------------------------------------- + + +def _mcp_to_openai(server_name: str, tool: Any) -> dict[str, Any]: + """Convert a single MCP tool definition to OpenAI function-calling format. + + The tool name is prefixed ``mcp__{server}__{original}`` to avoid + collisions with built-in tools and to identify the owning server. + """ + input_schema = getattr(tool, "inputSchema", None) or { + "type": "object", + "properties": {}, + } + description = getattr(tool, "description", "") or "" + return { + "type": "function", + "function": { + "name": f"mcp__{server_name}__{tool.name}", + "description": f"[MCP: {server_name}] {description}", + "parameters": input_schema, + }, + } + + +# --------------------------------------------------------------------------- +# Client manager +# --------------------------------------------------------------------------- + + +class MCPClientManager: + """Manages connections to one or more MCP servers. + + Runs a background asyncio event loop in a daemon thread and exposes + synchronous methods for tool discovery and invocation. + """ + + def __init__(self, server_configs: dict[str, dict[str, Any]]) -> None: + self._server_configs = server_configs + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._exit_stack: AsyncExitStack | None = None + + self._sessions: dict[str, Any] = {} + self._tools: list[dict[str, Any]] = [] + # prefixed_name -> (server_name, original_tool_name) + self._tool_map: dict[str, tuple[str, str]] = {} + self._connected = threading.Event() + self._error: str | None = None + + # -- lifecycle ----------------------------------------------------------- + + def start(self) -> None: + """Launch background event loop and connect to all configured servers.""" + self._loop = asyncio.new_event_loop() + self._thread = threading.Thread(target=self._loop.run_forever, daemon=True, name="mcp-loop") + self._thread.start() + + future = asyncio.run_coroutine_threadsafe(self._connect_all(), self._loop) + self._connected.wait(timeout=30) + # Surface any exception from _connect_all (unlikely — per-server errors are caught) + if future.done() and future.exception(): + self._error = str(future.exception()) + log.error("MCP initialization error: %s", self._error) + + async def _connect_all(self) -> None: + """Connect to every configured server (runs on the background loop).""" + self._exit_stack = AsyncExitStack() + await self._exit_stack.__aenter__() + + for name, cfg in self._server_configs.items(): + try: + await self._connect_one(name, cfg) + except Exception: + log.warning("Failed to connect MCP server '%s'", name, exc_info=True) + + self._connected.set() + + async def _connect_one(self, name: str, cfg: dict[str, Any]) -> None: + """Connect to a single MCP server and discover its tools.""" + assert self._exit_stack is not None + + if "__" in name: + log.error("MCP server name '%s' contains '__' (reserved delimiter), skipping", name) + return + + transport = cfg.get("type", "stdio") + if transport in ("http", "streamable-http") or "url" in cfg: + read, write, _ = await self._exit_stack.enter_async_context( + streamablehttp_client(url=cfg["url"], headers=cfg.get("headers")) + ) + else: + # Default: stdio transport + command = cfg.get("command", "") + if not command: + log.warning("MCP server '%s' has no command configured", name) + return + env = {**os.environ, **cfg.get("env", {})} + params = StdioServerParameters( + command=command, + args=cfg.get("args", []), + env=env, + ) + read, write = await self._exit_stack.enter_async_context(stdio_client(params)) + + session = await self._exit_stack.enter_async_context(ClientSession(read, write)) + await session.initialize() + self._sessions[name] = session + + # Discover tools + result = await session.list_tools() + for tool in result.tools: + openai_def = _mcp_to_openai(name, tool) + prefixed = openai_def["function"]["name"] + self._tools.append(openai_def) + self._tool_map[prefixed] = (name, tool.name) + + log.info( + "Connected MCP server '%s' — %d tool(s)", + name, + len(result.tools), + ) + + def shutdown(self) -> None: + """Close all MCP sessions and stop the background loop.""" + if self._loop and self._exit_stack: + future = asyncio.run_coroutine_threadsafe(self._exit_stack.aclose(), self._loop) + try: + future.result(timeout=10) + except Exception: + log.debug("Error closing MCP sessions", exc_info=True) + + if self._loop: + self._loop.call_soon_threadsafe(self._loop.stop) + if self._thread: + self._thread.join(timeout=5) + + log.info("MCP client shut down") + + # -- query methods ------------------------------------------------------- + + def get_tools(self) -> list[dict[str, Any]]: + """Return MCP tools in OpenAI function-calling format.""" + return list(self._tools) + + def is_mcp_tool(self, func_name: str) -> bool: + """Check whether *func_name* belongs to an MCP server.""" + return func_name in self._tool_map + + @property + def server_count(self) -> int: + return len(self._sessions) + + # -- tool invocation ----------------------------------------------------- + + def call_tool_sync( + self, + func_name: str, + arguments: dict[str, Any], + timeout: int = 120, + ) -> str: + """Execute an MCP tool call synchronously (blocks the calling thread). + + Dispatches an async ``tools/call`` to the background event loop and + waits for the result. + """ + mapping = self._tool_map.get(func_name) + if mapping is None: + raise ValueError(f"Unknown MCP tool: {func_name}") + server_name, original_name = mapping + session = self._sessions.get(server_name) + if session is None: + raise RuntimeError(f"MCP server '{server_name}' is not connected") + assert self._loop is not None + + future = asyncio.run_coroutine_threadsafe( + session.call_tool(original_name, arguments), self._loop + ) + result = future.result(timeout=timeout) + + # Extract text from the content array + texts: list[str] = [] + for item in result.content: + if hasattr(item, "text"): + texts.append(item.text) + elif hasattr(item, "data"): + mime = getattr(item, "mimeType", "binary") + texts.append(f"[{mime} data, {len(item.data)} bytes]") + else: + texts.append(str(item)) + + output = "\n".join(texts) if texts else "(no output)" + if getattr(result, "isError", False): + output = f"Error: {output}" + return output + + +# --------------------------------------------------------------------------- +# Config loading +# --------------------------------------------------------------------------- + + +def load_mcp_config(config_path: str | None = None) -> dict[str, dict[str, Any]]: + """Load MCP server configurations. + + Sources (first match wins): + + 1. Explicit *config_path* (standard MCP JSON format). + 2. ``[mcp.servers.*]`` sections in ``config.toml``. + + Returns an empty dict if nothing is configured. + """ + # 1. Explicit JSON file + if config_path: + path = Path(config_path).expanduser() + if path.is_file(): + try: + data = json.loads(path.read_text(encoding="utf-8")) + servers: dict[str, Any] = data.get("mcpServers", {}) + if isinstance(servers, dict) and servers: + log.info("Loaded MCP config from %s (%d server(s))", path, len(servers)) + return servers + except Exception: + log.warning("Failed to parse MCP config file: %s", path, exc_info=True) + else: + log.warning("MCP config file not found: %s", path) + + # 2. TOML config + mcp_section = load_config("mcp") + servers_section = mcp_section.get("servers", {}) + + # If TOML has [mcp] config_path, try that JSON file + toml_config_path = mcp_section.get("config_path") + if toml_config_path and not config_path: + return load_mcp_config(toml_config_path) + + if isinstance(servers_section, dict) and servers_section: + log.info("Loaded MCP config from config.toml (%d server(s))", len(servers_section)) + return servers_section + + return {} + + +def create_mcp_client(config_path: str | None = None) -> MCPClientManager | None: + """Create and start an MCP client manager. + + Returns *None* if no servers are configured. + """ + servers = load_mcp_config(config_path) + if not servers: + return None + + mgr = MCPClientManager(servers) + mgr.start() + return mgr diff --git a/turnstone/core/session.py b/turnstone/core/session.py index cc28d487..859bcec0 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -54,6 +54,7 @@ from turnstone.core.tools import ( TASK_AGENT_TOOLS, TASK_AUTO_TOOLS, TOOLS, + merge_mcp_tools, ) from turnstone.core.web import check_ssrf, strip_html from turnstone.ui.colors import DIM, GRAY, GREEN, RED, RESET, YELLOW, bold, cyan, dim @@ -61,6 +62,8 @@ from turnstone.ui.colors import DIM, GRAY, GREEN, RED, RESET, YELLOW, bold, cyan if TYPE_CHECKING: from openai import OpenAI + from turnstone.core.mcp_client import MCPClientManager + # --------------------------------------------------------------------------- # SessionUI protocol — the contract every frontend must implement # --------------------------------------------------------------------------- @@ -104,6 +107,7 @@ class ChatSession: auto_compact_pct: float = 0.8, agent_max_turns: int = -1, tool_truncation: int = 0, + mcp_client: MCPClientManager | None = None, ): self.client = client self.model = model @@ -136,6 +140,17 @@ class ChatSession: self._system_tokens = 0 # tokens for system_messages self._assistant_pending_tokens = 0 self.creative_mode = False + # MCP tool integration: merge external tools with built-in + self._mcp_client = mcp_client + if mcp_client: + mcp_tools = mcp_client.get_tools() + self._tools = merge_mcp_tools(TOOLS, mcp_tools) + self._task_tools = merge_mcp_tools(TASK_AGENT_TOOLS, mcp_tools) + self._agent_tools = merge_mcp_tools(AGENT_TOOLS, mcp_tools) + else: + self._tools = TOOLS + self._task_tools = TASK_AGENT_TOOLS + self._agent_tools = AGENT_TOOLS self._init_system_messages() self._save_config() @@ -361,7 +376,7 @@ class ChatSession: return self.client.chat.completions.create( # type: ignore[call-overload] model=self.model, messages=msgs, - **({"tools": TOOLS} if not self.creative_mode else {}), + **({"tools": self._tools} if not self.creative_mode else {}), max_completion_tokens=self.max_tokens, temperature=self.temperature, stream=True, @@ -741,7 +756,7 @@ class ChatSession: f"{GRAY}[request] model={self.model} " f"max_tokens={self.max_tokens} temp={self.temperature} " f"reasoning={self.reasoning_effort} " - f"tools={0 if self.creative_mode else len(TOOLS)}{RESET}" + f"tools={0 if self.creative_mode else len(self._tools)}{RESET}" ) lines.append(f"{GRAY}[request] {len(msgs)} messages:{RESET}") for i, m in enumerate(msgs): @@ -795,7 +810,7 @@ class ChatSession: # Calibrate chars_per_token ratio from actual usage. all_msgs = self._full_messages() # system + self.messages (before append) - tool_def_chars = sum(len(json.dumps(t)) for t in TOOLS) + tool_def_chars = sum(len(json.dumps(t)) for t in self._tools) total_chars = sum(self._msg_char_count(m) for m in all_msgs) + tool_def_chars if total_chars > 0 and prompt_tok > 0: self._chars_per_token = total_chars / prompt_tok @@ -1149,6 +1164,9 @@ class ChatSession: } preparer = preparers.get(func_name) if not preparer: + # Check if this is an MCP tool + if self._mcp_client and self._mcp_client.is_mcp_tool(func_name): + return self._prepare_mcp_tool(call_id, func_name, args) return { "call_id": call_id, "func_name": func_name, @@ -1733,6 +1751,56 @@ class ChatSession: "limit": min(limit, 50), } + # -- MCP tool prepare/execute ---------------------------------------------- + + def _prepare_mcp_tool( + self, call_id: str, func_name: str, args: dict[str, Any] + ) -> dict[str, Any]: + """Prepare an MCP tool call for approval.""" + # Parse prefixed name for display: mcp__github__search → github/search + parts = func_name.split("__", 2) + display = f"{parts[1]}/{parts[2]}" if len(parts) == 3 else func_name + + preview_lines = [] + for key, val in args.items(): + val_str = str(val) + if len(val_str) > 200: + val_str = val_str[:200] + "..." + preview_lines.append(f" {key}: {val_str}") + preview = "\n".join(preview_lines) if preview_lines else " (no arguments)" + + return { + "call_id": call_id, + "func_name": func_name, + "header": f"\u2699 mcp:{display}", + "preview": f"{DIM}{preview}{RESET}", + "needs_approval": True, + "approval_label": "mcp_tool", + "execute": self._exec_mcp_tool, + "mcp_func_name": func_name, + "mcp_args": args, + } + + def _exec_mcp_tool(self, item: dict[str, Any]) -> tuple[str, str]: + """Execute an MCP tool call via the MCPClientManager.""" + call_id: str = item["call_id"] + func_name: str = item["mcp_func_name"] + args: dict[str, Any] = item["mcp_args"] + + assert self._mcp_client is not None + try: + output = self._mcp_client.call_tool_sync(func_name, args, timeout=self.tool_timeout) + except TimeoutError: + output = f"MCP tool timed out after {self.tool_timeout}s" + self.ui.on_error(output) + except Exception as e: + output = f"MCP tool error: {e}" + self.ui.on_error(output) + + output = self._truncate_output(output) + self.ui.on_tool_result(call_id, func_name, output) + return call_id, output + # -- Execute methods (do the work, report output via UI) ------------------- def _exec_bash(self, item: dict[str, Any]) -> tuple[str, str]: @@ -1933,7 +2001,7 @@ class ChatSession: Final content string from the agent. """ if tools is None: - tools = AGENT_TOOLS + tools = self._agent_tools if auto_tools is None: auto_tools = self._AGENT_AUTO_TOOLS max_tool_turns = self.agent_max_turns @@ -2112,7 +2180,7 @@ class ChatSession: return call_id, self._run_agent( agent_messages, label="task", - tools=TASK_AGENT_TOOLS, + tools=self._task_tools, auto_tools=self._TASK_AUTO_TOOLS, ) except KeyboardInterrupt: @@ -2715,6 +2783,21 @@ class ChatSession: state = "on" if self.debug else "off" self.ui.on_info(f"Debug mode: {bold(state)} (prints raw SSE deltas)") + elif cmd == "/mcp": + if not self._mcp_client: + self.ui.on_info("No MCP servers configured.") + else: + tools = self._mcp_client.get_tools() + if not tools: + self.ui.on_info("MCP client connected but no tools available.") + else: + lines = [f"MCP tools ({len(tools)}):"] + for t in tools: + name = t["function"]["name"] + desc = t["function"].get("description", "")[:80] + lines.append(f" {name} {dim(desc)}") + self.ui.on_info("\n".join(lines)) + elif cmd == "/help": self.ui.on_info( "\n".join( @@ -2737,6 +2820,7 @@ class ChatSession: " /reason [low|med|high] Set/show reasoning effort", " /creative Toggle creative writing mode (no tools)", " /debug Toggle raw SSE delta logging", + " /mcp List connected MCP tools", " /help Show this help", " /exit Exit (also: Ctrl+D)", "────────────────────────────────────────────────────────", diff --git a/turnstone/core/tools.py b/turnstone/core/tools.py index abff45d7..509245e5 100644 --- a/turnstone/core/tools.py +++ b/turnstone/core/tools.py @@ -37,3 +37,14 @@ TASK_AGENT_TOOLS = [t for t in TOOLS if _META[t["function"]["name"]].get("task_a AGENT_AUTO_TOOLS = {n for n, m in _META.items() if m.get("auto_approve")} TASK_AUTO_TOOLS = {n for n, m in _META.items() if m.get("auto_approve")} PRIMARY_KEY_MAP = {n: m["primary_key"] for n, m in _META.items() if "primary_key" in m} + + +def merge_mcp_tools( + builtin: list[dict[str, Any]], mcp_tools: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Merge built-in tools with MCP tools. + + Built-in tools come first so the LLM sees them with natural priority. + Returns a new list; neither input is mutated. + """ + return builtin + mcp_tools diff --git a/turnstone/server.py b/turnstone/server.py index f951685b..4d662c4c 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1041,9 +1041,15 @@ def main() -> None: metavar="MINUTES", help="Close IDLE workstreams after MINUTES of inactivity, 0 to disable (default: 120)", ) + parser.add_argument( + "--mcp-config", + default=None, + metavar="PATH", + help="Path to MCP server config file (standard mcpServers JSON format)", + ) from turnstone.core.config import apply_config - apply_config(parser, ["api", "model", "session", "tools", "server"]) + apply_config(parser, ["api", "model", "session", "tools", "server", "mcp"]) args = parser.parse_args() # Prune stale / empty sessions on startup @@ -1061,6 +1067,11 @@ def main() -> None: # Detect or use provided model model = args.model or detect_model(client) + # Initialize MCP client (connects to configured MCP servers, if any) + from turnstone.core.mcp_client import create_mcp_client + + mcp_client = create_mcp_client(getattr(args, "mcp_config", None)) + # Set up global event queue for state-change broadcasts global_queue: queue.Queue[dict[str, Any]] = queue.Queue() global_listeners: list[queue.Queue[dict[str, Any]]] = [] @@ -1084,6 +1095,7 @@ def main() -> None: auto_compact_pct=args.auto_compact_pct, agent_max_turns=args.agent_max_turns, tool_truncation=args.tool_truncation, + mcp_client=mcp_client, ) # Create workstream manager and initial workstream @@ -1147,12 +1159,18 @@ def main() -> None: print(f"turnstone web server running on http://{args.host}:{args.port}") print(f"Model: {model}") + if mcp_client: + mcp_tools = mcp_client.get_tools() + if mcp_tools: + print(f"MCP tools: {len(mcp_tools)} from {mcp_client.server_count} server(s)") print("Press Ctrl+C to stop.") try: server.serve_forever() except KeyboardInterrupt: print("\nShutting down.") + if mcp_client: + mcp_client.shutdown() server.shutdown()