diff --git a/docs/api-reference.md b/docs/api-reference.md index 16567fa8..3674dc4d 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1391,6 +1391,24 @@ the `admin.settings` permission. --- +### MCP Servers + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/v1/api/admin/mcp-servers` | List all MCP server definitions with live node status. Query: `?reveal=true` to show env/header secrets. | +| POST | `/v1/api/admin/mcp-servers` | Create an MCP server definition. Body: `{name, transport, command?, args?, url?, headers?, env?, auto_approve?, enabled?}` | +| GET | `/v1/api/admin/mcp-servers/{server_id}` | Get a single MCP server with per-node connection status. | +| PUT | `/v1/api/admin/mcp-servers/{server_id}` | Update an MCP server definition. Partial updates supported. | +| DELETE | `/v1/api/admin/mcp-servers/{server_id}` | Delete an MCP server definition. | +| POST | `/v1/api/admin/mcp-servers/reload` | Tell all cluster nodes to re-read the `mcp_servers` DB table and reconcile (add new, remove stale, reconnect changed). | +| POST | `/v1/api/admin/mcp-servers/import` | Import servers from a pasted JSON config. Body: `{config: {mcpServers: {...}}}`. Skips existing names. | + +Permission: `admin.mcp` + +Secrets (`env`, `headers` fields) are masked with `***` by default. Use `?reveal=true` on GET endpoints to see actual values. + +--- + ### `OPTIONS` (any path) Handles CORS preflight requests. diff --git a/docs/architecture.md b/docs/architecture.md index 09c30eea..91506131 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -506,8 +506,18 @@ independently, then returns the final content as the tool result. and exposes their tools alongside built-in tools. The MCP SDK is fully async; turnstone bridges this with a background asyncio event loop in a daemon thread. +**Configuration sources:** MCP servers can be defined in config files (TOML/JSON) +or in the database via the admin UI. Database-backed definitions are managed +through the console admin panel's MCP Servers tab and stored in the +`mcp_servers` table. On startup, `load_mcp_config(storage=)` uses +first-match-wins priority: DB rows (if any enabled) take precedence over +config files. The console can trigger a cluster-wide reload (`POST +/_internal/mcp-reload`) that causes each node to call `reconcile_sync()`, +which diffs the running MCP connections against the current DB state and +adds, removes, or reconnects servers as needed. + **Lifecycle:** -1. `create_mcp_client()` reads server configs from TOML or JSON +1. `create_mcp_client()` reads server configs from TOML/JSON and database 2. `MCPClientManager.start()` launches the background event loop thread 3. `_connect_all()` connects to each server (stdio subprocess or HTTP), runs `initialize()` + `list_tools()`, converts schemas to OpenAI format, detects @@ -1018,8 +1028,8 @@ Three hierarchical scopes control endpoint access: - **Console** is the auth management hub — it hosts the admin endpoints for creating users, issuing API tokens, and managing channel mappings. User records and token hashes live in the shared storage backend. The console - dashboard includes an **admin panel** (13 tabs) for managing - credentials, governance, and runtime settings through the browser. + dashboard includes an **admin panel** (14 tabs) for managing + credentials, governance, MCP servers, and runtime settings through the browser. - **Server** is a JWT validator only — it validates tokens on each request but never creates users or tokens. Both processes share the same `jwt_secret` (via `TURNSTONE_JWT_SECRET` env var or `[auth].jwt_secret` config). @@ -1393,8 +1403,10 @@ enforcement tracks consumption in `session.send()` with 80% warning and 100% approval gate via the `__budget_override__` synthetic tool name. The console admin panel adds 6 governance tabs (Roles, Policies, Templates, -WS Templates, Usage, Audit), a Memories tab, and a Settings tab (form-based -editor for all ConfigStore settings) for a total of 13 tabs, all permission-gated. +WS Templates, Usage, Audit), a Memories tab, a Settings tab (form-based +editor for all ConfigStore settings), and an MCP Servers tab (database-backed +server definitions with live connection status and cluster-wide reload) for a +total of 14 tabs, all permission-gated. Both Python and TypeScript SDKs expose governance methods on the console client. diff --git a/docs/diagrams/20-mcp-architecture.puml b/docs/diagrams/20-mcp-architecture.puml index f61064f4..0a5c5da6 100644 --- a/docs/diagrams/20-mcp-architecture.puml +++ b/docs/diagrams/20-mcp-architecture.puml @@ -16,8 +16,27 @@ participant "ChatSession\n(session.py)" as Session <> participant "StorageBackend\n(governance)" as Storage <> participant "Server / Console\n(health + UI)" as UI <> +participant "Console Admin UI\n(admin panel)" as Admin <> +participant "Database\n(mcp_servers table)" as DB <> + +== Admin-Driven Configuration == + +Admin -> DB : CRUD MCP server definitions\n(POST/PUT/DELETE /v1/api/admin/mcp-servers) + +Admin -> UI : POST /v1/api/admin/mcp-servers/reload +UI -> MCPMgr : POST /_internal/mcp-reload\n(forwarded to each node) +MCPMgr -> MCPMgr : reconcile_sync() +note right + Diffs running servers against DB: + - New entries → connect + - Removed entries → disconnect + - Changed entries → reconnect +end note + == Startup: Connection & Discovery == +MCPMgr -> DB : load_mcp_config(storage=)\n(merge config file + DB) + MCPMgr -> MCPSrv : initialize (stdio or HTTP) MCPSrv --> MCPMgr : capabilities\n(tools, resources, prompts) diff --git a/docs/diagrams/png/20-mcp-architecture.png b/docs/diagrams/png/20-mcp-architecture.png index b5dc770e..45b040a1 100644 --- a/docs/diagrams/png/20-mcp-architecture.png +++ b/docs/diagrams/png/20-mcp-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8e6dc5142c7908314ce01229b3c4f13bf9450adcbb62a178838bd4cf81d9f4da -size 250417 +oid sha256:e4593873599342b2830fedd5d783e9a28eab0bb0d6589798ef6ef2649eeee80f +size 324518 diff --git a/sdk/typescript/openapi-console.json b/sdk/typescript/openapi-console.json index b700cedf..f4eecde2 100644 --- a/sdk/typescript/openapi-console.json +++ b/sdk/typescript/openapi-console.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Console API", - "version": "0.6.0", + "version": "0.6.1", "description": "Cluster-wide visibility and control across all turnstone nodes." }, "paths": { @@ -2661,6 +2661,298 @@ } } }, + "/v1/api/admin/mcp-servers": { + "get": { + "summary": "List MCP server definitions with live status", + "operationId": "v1_api_admin_mcp-servers_get", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "reveal", + "in": "query", + "required": false, + "schema": { + "type": "boolean" + }, + "description": "Show secret env/header values" + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListMcpServersResponse" + } + } + } + } + } + }, + "post": { + "summary": "Create an MCP server definition", + "operationId": "v1_api_admin_mcp-servers_post", + "tags": [ + "Admin" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateMcpServerRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-servers/{server_id}": { + "get": { + "summary": "Get a single MCP server with status", + "operationId": "v1_api_admin_mcp-servers_{server_id}_get", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "server_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, + "put": { + "summary": "Update an MCP server definition", + "operationId": "v1_api_admin_mcp-servers_{server_id}_put", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "server_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMcpServerRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, + "delete": { + "summary": "Delete an MCP server definition", + "operationId": "v1_api_admin_mcp-servers_{server_id}_delete", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "server_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-servers/reload": { + "post": { + "summary": "Tell all nodes to re-read MCP server config from DB and reconcile", + "operationId": "v1_api_admin_mcp-servers_reload_post", + "tags": [ + "Admin" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpReloadResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-servers/import": { + "post": { + "summary": "Import MCP servers from a JSON config file", + "operationId": "v1_api_admin_mcp-servers_import_post", + "tags": [ + "Admin" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportMcpConfigRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportMcpConfigResponse" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/health": { "get": { "summary": "Console health check", @@ -5264,6 +5556,397 @@ ], "title": "UpdateSettingRequest", "type": "object" + }, + "McpServerDetail": { + "properties": { + "server_id": { + "title": "Server Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "transport": { + "title": "Transport", + "type": "string" + }, + "command": { + "default": "", + "title": "Command", + "type": "string" + }, + "args": { + "default": "[]", + "title": "Args", + "type": "string" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + }, + "headers": { + "default": "{}", + "title": "Headers", + "type": "string" + }, + "env": { + "default": "{}", + "title": "Env", + "type": "string" + }, + "auto_approve": { + "default": false, + "title": "Auto Approve", + "type": "boolean" + }, + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "created_by": { + "default": "", + "title": "Created By", + "type": "string" + }, + "created": { + "title": "Created", + "type": "string" + }, + "updated": { + "title": "Updated", + "type": "string" + }, + "status": { + "additionalProperties": { + "$ref": "#/components/schemas/McpServerStatus" + }, + "title": "Status", + "type": "object" + }, + "source": { + "default": "", + "title": "Source", + "type": "string" + } + }, + "required": [ + "server_id", + "name", + "transport", + "created", + "updated" + ], + "title": "McpServerDetail", + "type": "object" + }, + "McpServerStatus": { + "properties": { + "connected": { + "default": false, + "title": "Connected", + "type": "boolean" + }, + "tools": { + "default": 0, + "title": "Tools", + "type": "integer" + }, + "resources": { + "default": 0, + "title": "Resources", + "type": "integer" + }, + "prompts": { + "default": 0, + "title": "Prompts", + "type": "integer" + }, + "error": { + "default": "", + "title": "Error", + "type": "string" + }, + "transport": { + "default": "", + "title": "Transport", + "type": "string" + }, + "command": { + "default": "", + "title": "Command", + "type": "string" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + } + }, + "title": "McpServerStatus", + "type": "object" + }, + "CreateMcpServerRequest": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "transport": { + "title": "Transport", + "type": "string" + }, + "command": { + "default": "", + "title": "Command", + "type": "string" + }, + "args": { + "default": [], + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + }, + "headers": { + "additionalProperties": { + "type": "string" + }, + "title": "Headers", + "type": "object" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "auto_approve": { + "default": false, + "title": "Auto Approve", + "type": "boolean" + }, + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + } + }, + "required": [ + "name", + "transport" + ], + "title": "CreateMcpServerRequest", + "type": "object" + }, + "UpdateMcpServerRequest": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "transport": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Transport" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "args": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Args" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Url" + }, + "headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Headers" + }, + "env": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Env" + }, + "auto_approve": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auto Approve" + }, + "enabled": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enabled" + } + }, + "title": "UpdateMcpServerRequest", + "type": "object" + }, + "ListMcpServersResponse": { + "properties": { + "servers": { + "items": { + "$ref": "#/components/schemas/McpServerDetail" + }, + "title": "Servers", + "type": "array" + } + }, + "required": [ + "servers" + ], + "title": "ListMcpServersResponse", + "type": "object" + }, + "ImportMcpConfigRequest": { + "properties": { + "config": { + "additionalProperties": true, + "description": "JSON config object with mcpServers key", + "title": "Config", + "type": "object" + } + }, + "required": [ + "config" + ], + "title": "ImportMcpConfigRequest", + "type": "object" + }, + "ImportMcpConfigResponse": { + "properties": { + "imported": { + "default": [], + "items": { + "type": "string" + }, + "title": "Imported", + "type": "array" + }, + "skipped": { + "default": [], + "items": { + "type": "string" + }, + "title": "Skipped", + "type": "array" + }, + "errors": { + "default": [], + "items": { + "type": "string" + }, + "title": "Errors", + "type": "array" + } + }, + "title": "ImportMcpConfigResponse", + "type": "object" + }, + "McpReloadResponse": { + "properties": { + "status": { + "default": "ok", + "title": "Status", + "type": "string" + }, + "results": { + "additionalProperties": true, + "title": "Results", + "type": "object" + } + }, + "title": "McpReloadResponse", + "type": "object" } } } diff --git a/sdk/typescript/openapi-server.json b/sdk/typescript/openapi-server.json index 34bb3f20..b1edbd5d 100644 --- a/sdk/typescript/openapi-server.json +++ b/sdk/typescript/openapi-server.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Server API", - "version": "0.6.0", + "version": "0.6.1", "description": "Single-node workstream management, chat interaction, and real-time streaming." }, "paths": { diff --git a/sdk/typescript/src/console.ts b/sdk/typescript/src/console.ts index ed93d45f..9adbe432 100644 --- a/sdk/typescript/src/console.ts +++ b/sdk/typescript/src/console.ts @@ -16,16 +16,20 @@ import type { ConsoleCreateWsRequest, ConsoleCreateWsResponse, ConsoleHealthResponse, + CreateMcpServerRequest, CreatePolicyOptions, CreateRoleOptions, CreateScheduleRequest, CreateTemplateOptions, CreateWsTemplateOptions, + ImportMcpConfigResponse, ListAdminMemoriesResponse, + ListMcpServersResponse, ListScheduleRunsResponse, ListSchedulesResponse, ListSettingSchemaResponse, ListSettingsResponse, + McpServerDetail, NodeDetailResponse, NodesOptions, OrgInfo, @@ -35,6 +39,7 @@ import type { SettingInfo, StatusResponse, ToolPolicyInfo, + UpdateMcpServerRequest, UpdateOrgOptions, UpdatePolicyOptions, UpdateRoleOptions, @@ -410,4 +415,49 @@ export class TurnstoneConsole extends BaseClient { params, }); } + + // -- MCP servers ---------------------------------------------------------- + + async listMcpServers(opts?: { + reveal?: boolean; + }): Promise { + const params: Record = {}; + if (opts?.reveal) params.reveal = "true"; + return this.request("GET", "/v1/api/admin/mcp-servers", { params }); + } + + async createMcpServer( + body: CreateMcpServerRequest, + ): Promise { + return this.request("POST", "/v1/api/admin/mcp-servers", { json: body }); + } + + async getMcpServer(serverId: string): Promise { + return this.request("GET", `/v1/api/admin/mcp-servers/${serverId}`); + } + + async updateMcpServer( + serverId: string, + body: UpdateMcpServerRequest, + ): Promise { + return this.request("PUT", `/v1/api/admin/mcp-servers/${serverId}`, { + json: body, + }); + } + + async deleteMcpServer(serverId: string): Promise { + return this.request("DELETE", `/v1/api/admin/mcp-servers/${serverId}`); + } + + async reloadMcpServers(): Promise { + return this.request("POST", "/v1/api/admin/mcp-servers/reload"); + } + + async importMcpConfig( + config: Record, + ): Promise { + return this.request("POST", "/v1/api/admin/mcp-servers/import", { + json: { config }, + }); + } } diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 3af85d89..9d4265b5 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -160,6 +160,13 @@ export type { SettingSchemaInfo, ListSettingSchemaResponse, UpdateSettingOptions, + // MCP server types + McpServerStatus, + McpServerDetail, + ListMcpServersResponse, + CreateMcpServerRequest, + UpdateMcpServerRequest, + ImportMcpConfigResponse, } from "./types.js"; // SSE parser (for advanced usage) diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts index 53dbcad7..7577e153 100644 --- a/sdk/typescript/src/types.ts +++ b/sdk/typescript/src/types.ts @@ -728,6 +728,67 @@ export interface AdminSearchMemoriesOptions { limit?: number; } +// -- Console API: MCP Servers ----------------------------------------------- + +export interface McpServerStatus { + connected: boolean; + tools: number; + resources: number; + prompts: number; + error: string; +} + +export interface McpServerDetail { + server_id: string; + name: string; + transport: string; + command: string; + args: string; + url: string; + headers: string; + env: string; + auto_approve: boolean; + enabled: boolean; + created_by: string; + created: string; + updated: string; + status: Record; +} + +export interface ListMcpServersResponse { + servers: McpServerDetail[]; +} + +export interface CreateMcpServerRequest { + name: string; + transport: string; + command?: string; + args?: string[]; + url?: string; + headers?: Record; + env?: Record; + auto_approve?: boolean; + enabled?: boolean; +} + +export interface UpdateMcpServerRequest { + name?: string; + transport?: string; + command?: string; + args?: string[]; + url?: string; + headers?: Record; + env?: Record; + auto_approve?: boolean; + enabled?: boolean; +} + +export interface ImportMcpConfigResponse { + imported: string[]; + skipped: string[]; + errors: string[]; +} + // -- Console API: System Settings ------------------------------------------- export interface SettingInfo { diff --git a/tests/test_mcp_admin_api.py b/tests/test_mcp_admin_api.py new file mode 100644 index 00000000..eafc734f --- /dev/null +++ b/tests/test_mcp_admin_api.py @@ -0,0 +1,539 @@ +"""Tests for MCP server admin API endpoints.""" + +from __future__ import annotations + +import json +import uuid +from typing import TYPE_CHECKING, Any +from unittest.mock import AsyncMock, patch + +import pytest +from starlette.applications import Starlette +from starlette.middleware import Middleware +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.routing import Mount, Route +from starlette.testclient import TestClient + +if TYPE_CHECKING: + from starlette.requests import Request + from starlette.responses import Response + +from turnstone.console.server import ( + admin_create_mcp_server, + admin_delete_mcp_server, + admin_get_mcp_server, + admin_import_mcp_config, + admin_list_mcp_servers, + admin_update_mcp_server, +) +from turnstone.core.auth import AuthResult +from turnstone.core.storage._sqlite import SQLiteBackend + +# --------------------------------------------------------------------------- +# Auth middleware variants +# --------------------------------------------------------------------------- + + +class _InjectAuthMiddleware(BaseHTTPMiddleware): + """Inject an admin auth result with admin.mcp permission.""" + + async def dispatch(self, request: Request, call_next: Any) -> Response: + request.state.auth_result = AuthResult( + user_id="test-user", + scopes=frozenset({"approve"}), + token_source="config", + permissions=frozenset( + { + "read", + "write", + "approve", + "admin.mcp", + } + ), + ) + resp: Response = await call_next(request) + return resp + + +class _InjectAuthNoMcpMiddleware(BaseHTTPMiddleware): + """Inject an auth result WITHOUT admin.mcp permission.""" + + async def dispatch(self, request: Request, call_next: Any) -> Response: + request.state.auth_result = AuthResult( + user_id="test-user", + scopes=frozenset({"approve"}), + token_source="jwt", + permissions=frozenset( + { + "read", + "write", + "approve", + } + ), + ) + resp: Response = await call_next(request) + return resp + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +_ROUTES = [ + Mount( + "/v1", + routes=[ + Route("/api/admin/mcp-servers", admin_list_mcp_servers), + Route( + "/api/admin/mcp-servers", + admin_create_mcp_server, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/import", + admin_import_mcp_config, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_get_mcp_server, + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_update_mcp_server, + methods=["PUT"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_delete_mcp_server, + methods=["DELETE"], + ), + ], + ), +] + + +@pytest.fixture +def storage(tmp_path): + return SQLiteBackend(str(tmp_path / "test.db")) + + +@pytest.fixture +def client(storage): + """TestClient wired to console admin MCP endpoints with full permissions.""" + app = Starlette( + routes=_ROUTES, + middleware=[Middleware(_InjectAuthMiddleware)], + ) + app.state.auth_storage = storage + return TestClient(app) + + +@pytest.fixture +def client_no_perm(storage): + """TestClient without admin.mcp permission.""" + app = Starlette( + routes=_ROUTES, + middleware=[Middleware(_InjectAuthNoMcpMiddleware)], + ) + app.state.auth_storage = storage + return TestClient(app) + + +def _create_server( + client: TestClient, + *, + name: str = "test-server", + transport: str = "stdio", + command: str = "npx", + args: list[str] | None = None, + env: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + url: str = "", +) -> dict[str, Any]: + """Helper to create a server via the API and return the response dict.""" + body: dict[str, Any] = {"name": name, "transport": transport} + if transport == "stdio": + body["command"] = command + body["args"] = args or ["-y", "@modelcontextprotocol/server-test"] + else: + body["url"] = url or "http://localhost:8080/mcp" + if env is not None: + body["env"] = env + if headers is not None: + body["headers"] = headers + r = client.post("/v1/api/admin/mcp-servers", json=body) + assert r.status_code == 200 + data: dict[str, Any] = r.json() + return data + + +# --------------------------------------------------------------------------- +# Mock _collect_mcp_status to avoid real HTTP calls +# --------------------------------------------------------------------------- + +_PATCH_MCP_STATUS = patch( + "turnstone.console.server._collect_mcp_status", + new_callable=AsyncMock, + return_value={}, +) + + +# --------------------------------------------------------------------------- +# List +# --------------------------------------------------------------------------- + + +class TestListMcpServers: + def test_list_empty(self, client): + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers") + assert r.status_code == 200 + assert r.json()["servers"] == [] + + def test_list_returns_created_servers(self, client): + _create_server(client, name="server-a") + _create_server(client, name="server-b") + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers") + assert r.status_code == 200 + names = [s["name"] for s in r.json()["servers"]] + assert "server-a" in names + assert "server-b" in names + + +# --------------------------------------------------------------------------- +# Create +# --------------------------------------------------------------------------- + + +class TestCreateMcpServer: + def test_create_stdio_server(self, client): + data = _create_server(client, name="my-mcp", transport="stdio", command="node") + assert data["name"] == "my-mcp" + assert data["transport"] == "stdio" + assert data["command"] == "node" + assert data["server_id"] + assert data["enabled"] is True + + def test_create_http_server(self, client): + data = _create_server( + client, + name="remote-mcp", + transport="streamable-http", + url="http://mcp.example.com/sse", + ) + assert data["name"] == "remote-mcp" + assert data["transport"] == "streamable-http" + assert data["url"] == "http://mcp.example.com/sse" + + def test_create_invalid_name_spaces(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "bad name!", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 400 + assert "name" in r.json()["error"].lower() + + def test_create_invalid_name_double_underscore(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "bad__name", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 400 + assert "__" in r.json()["error"] + + def test_create_invalid_transport(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "ok-name", "transport": "grpc"}, + ) + assert r.status_code == 400 + assert "transport" in r.json()["error"].lower() + + def test_create_duplicate_name(self, client): + _create_server(client, name="dup-test") + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "dup-test", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 409 + assert "already exists" in r.json()["error"] + + def test_create_missing_name(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"transport": "stdio", "command": "x"}, + ) + assert r.status_code == 400 + assert "name" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Get single +# --------------------------------------------------------------------------- + + +class TestGetMcpServer: + def test_get_existing(self, client): + created = _create_server(client, name="get-test") + sid = created["server_id"] + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{sid}") + assert r.status_code == 200 + assert r.json()["name"] == "get-test" + + def test_get_not_found(self, client): + fake_id = uuid.uuid4().hex + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{fake_id}") + assert r.status_code == 404 + assert "not found" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Update +# --------------------------------------------------------------------------- + + +class TestUpdateMcpServer: + def test_update_name(self, client): + created = _create_server(client, name="old-name") + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"name": "new-name"}, + ) + assert r.status_code == 200 + assert r.json()["name"] == "new-name" + + def test_update_transport(self, client): + created = _create_server( + client, + name="update-transport", + transport="streamable-http", + url="http://localhost/mcp", + ) + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"transport": "stdio", "command": "node"}, + ) + assert r.status_code == 200 + assert r.json()["transport"] == "stdio" + + def test_update_enabled(self, client): + created = _create_server(client, name="toggle-enabled") + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"enabled": False}, + ) + assert r.status_code == 200 + assert r.json()["enabled"] is False + + def test_update_not_found(self, client): + fake_id = uuid.uuid4().hex + r = client.put( + f"/v1/api/admin/mcp-servers/{fake_id}", + json={"name": "x"}, + ) + assert r.status_code == 404 + + def test_update_invalid_transport(self, client): + created = _create_server(client, name="bad-transport-update") + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"transport": "websocket"}, + ) + assert r.status_code == 400 + assert "transport" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Delete +# --------------------------------------------------------------------------- + + +class TestDeleteMcpServer: + def test_delete_existing(self, client): + created = _create_server(client, name="del-test") + sid = created["server_id"] + r = client.delete(f"/v1/api/admin/mcp-servers/{sid}") + assert r.status_code == 200 + assert r.json()["status"] == "ok" + + # Confirm it's gone + with _PATCH_MCP_STATUS: + r2 = client.get(f"/v1/api/admin/mcp-servers/{sid}") + assert r2.status_code == 404 + + def test_delete_not_found(self, client): + fake_id = uuid.uuid4().hex + r = client.delete(f"/v1/api/admin/mcp-servers/{fake_id}") + assert r.status_code == 404 + assert "not found" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Secret masking +# --------------------------------------------------------------------------- + + +class TestSecretMasking: + def test_list_masks_secrets(self, client): + _create_server( + client, + name="secret-test", + env={"API_KEY": "sk-real-secret-123"}, + headers={"Authorization": "Bearer tok-xyz"}, + transport="streamable-http", + url="http://localhost/mcp", + ) + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers") + assert r.status_code == 200 + server = r.json()["servers"][0] + env = json.loads(server["env"]) + headers = json.loads(server["headers"]) + assert env["API_KEY"] == "***" + assert headers["Authorization"] == "***" + + def test_list_reveals_secrets(self, client): + _create_server( + client, + name="reveal-test", + env={"API_KEY": "sk-real-secret-123"}, + headers={"Authorization": "Bearer tok-xyz"}, + transport="streamable-http", + url="http://localhost/mcp", + ) + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers?reveal=true") + assert r.status_code == 200 + server = r.json()["servers"][0] + env = json.loads(server["env"]) + headers = json.loads(server["headers"]) + assert env["API_KEY"] == "sk-real-secret-123" + assert headers["Authorization"] == "Bearer tok-xyz" + + def test_get_masks_secrets_by_default(self, client): + created = _create_server( + client, + name="mask-get-test", + env={"SECRET": "value"}, + ) + sid = created["server_id"] + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{sid}") + assert r.status_code == 200 + env = json.loads(r.json()["env"]) + assert env["SECRET"] == "***" + + def test_get_reveals_secrets(self, client): + created = _create_server( + client, + name="reveal-get-test", + env={"SECRET": "real-value"}, + ) + sid = created["server_id"] + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{sid}?reveal=true") + assert r.status_code == 200 + env = json.loads(r.json()["env"]) + assert env["SECRET"] == "real-value" + + +# --------------------------------------------------------------------------- +# Import +# --------------------------------------------------------------------------- + + +class TestImportMcpConfig: + def test_import_inline_config(self, client): + config = { + "mcpServers": { + "filesystem": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"], + }, + "remote": { + "url": "http://remote.example.com/mcp", + }, + }, + } + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": config}, + ) + assert r.status_code == 200 + data = r.json() + assert "filesystem" in data["imported"] + assert "remote" in data["imported"] + assert data["skipped"] == [] + assert data["errors"] == [] + + def test_import_not_a_dict(self, client): + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": "not-a-dict"}, + ) + assert r.status_code == 400 + + def test_import_skips_duplicates(self, client): + _create_server(client, name="existing-srv") + config = { + "mcpServers": { + "existing-srv": {"command": "node", "args": []}, + "new-srv": {"command": "node", "args": []}, + }, + } + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": config}, + ) + assert r.status_code == 200 + data = r.json() + assert "new-srv" in data["imported"] + assert "existing-srv" in data["skipped"] + + def test_import_empty_body(self, client): + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={}, + ) + assert r.status_code == 400 + assert "config" in r.json()["error"].lower() + + def test_import_no_mcp_servers_key(self, client): + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": {"other": "data"}}, + ) + assert r.status_code == 400 + assert "mcpServers" in r.json()["error"] or "No" in r.json()["error"] + + +# --------------------------------------------------------------------------- +# Permission check +# --------------------------------------------------------------------------- + + +class TestPermission: + def test_list_without_permission(self, client_no_perm): + with _PATCH_MCP_STATUS: + r = client_no_perm.get("/v1/api/admin/mcp-servers") + assert r.status_code == 403 + assert "admin.mcp" in r.json()["error"] + + def test_create_without_permission(self, client_no_perm): + r = client_no_perm.post( + "/v1/api/admin/mcp-servers", + json={"name": "test", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 403 + + def test_delete_without_permission(self, client_no_perm): + r = client_no_perm.delete(f"/v1/api/admin/mcp-servers/{uuid.uuid4().hex}") + assert r.status_code == 403 diff --git a/tests/test_mcp_hot_reload.py b/tests/test_mcp_hot_reload.py new file mode 100644 index 00000000..e26167b9 --- /dev/null +++ b/tests/test_mcp_hot_reload.py @@ -0,0 +1,353 @@ +"""Tests for MCPClientManager hot-reload methods.""" + +from __future__ import annotations + +from typing import Any + +from turnstone.core.mcp_client import MCPClientManager + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _fake_openai_tool(name: str = "mcp__test__search") -> dict[str, Any]: + """Create a fake OpenAI-format tool dict.""" + return { + "type": "function", + "function": { + "name": name, + "description": "[MCP: test] Search stuff", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } + + +def _fake_resource_dict( + uri: str = "file:///README.md", + name: str = "readme", + server: str = "test", +) -> dict[str, Any]: + """Create a fake resource dict as stored in per-server state.""" + return { + "uri": uri, + "name": name, + "description": "A resource", + "mimeType": "text/plain", + "server": server, + } + + +def _fake_prompt_dict( + name: str = "mcp__test__code_review", + original_name: str = "code_review", + server: str = "test", +) -> dict[str, Any]: + """Create a fake prompt dict as stored in per-server state.""" + return { + "name": name, + "original_name": original_name, + "server": server, + "description": "Generate a code review", + "arguments": [ + {"name": "language", "description": "Programming language", "required": True} + ], + } + + +# --------------------------------------------------------------------------- +# add_server_sync +# --------------------------------------------------------------------------- + + +class TestAddServerSync: + def test_rejects_double_underscore_name(self) -> None: + """Names containing __ should be rejected.""" + mgr = MCPClientManager({}) + result = mgr.add_server_sync("bad__name", {"command": "echo"}) + assert result["connected"] is False + assert "__" in result["error"] + assert result["tools"] == 0 + assert result["resources"] == 0 + assert result["prompts"] == 0 + + def test_fails_without_event_loop(self) -> None: + """Adding a server without starting the event loop should fail gracefully.""" + mgr = MCPClientManager({}) + result = mgr.add_server_sync("test", {"command": "echo"}) + assert result["connected"] is False + assert "loop" in result["error"].lower() + + def test_config_removed_on_failure(self) -> None: + """add_server_sync removes the config entry when connection fails.""" + mgr = MCPClientManager({}) + mgr.add_server_sync("new-srv", {"command": "echo"}) + # Since the loop isn't running, it fails and config is cleaned up + assert "new-srv" not in mgr._server_configs + + +# --------------------------------------------------------------------------- +# remove_server_sync +# --------------------------------------------------------------------------- + + +class TestRemoveServerSync: + def test_returns_false_for_nonexistent(self) -> None: + """Removing a non-connected server returns False.""" + mgr = MCPClientManager({}) + assert mgr.remove_server_sync("nonexistent") is False + + def test_cleans_up_per_server_state(self) -> None: + """remove_server_sync cleans up all per-server state dicts.""" + mgr = MCPClientManager({"test": {"command": "echo"}}) + # Simulate state as if the server was connected + mgr._per_server_tools["test"] = [_fake_openai_tool()] + mgr._per_server_resources["test"] = [_fake_resource_dict()] + mgr._per_server_prompts["test"] = [_fake_prompt_dict()] + mgr._supports_list_changed["test"] = True + mgr._supports_resources["test"] = True + mgr._supports_resource_list_changed["test"] = True + mgr._supports_prompts["test"] = True + mgr._supports_prompt_list_changed["test"] = True + mgr._rebuild_tools() + mgr._rebuild_resources() + mgr._rebuild_prompts() + + # Verify preconditions + assert len(mgr.get_tools()) == 1 + assert mgr.resource_count == 1 + assert mgr.prompt_count == 1 + + mgr.remove_server_sync("test") + + assert len(mgr.get_tools()) == 0 + assert mgr.resource_count == 0 + assert mgr.prompt_count == 0 + assert "test" not in mgr._per_server_tools + assert "test" not in mgr._per_server_resources + assert "test" not in mgr._per_server_prompts + assert "test" not in mgr._supports_list_changed + assert "test" not in mgr._supports_resources + assert "test" not in mgr._supports_resource_list_changed + assert "test" not in mgr._supports_prompts + assert "test" not in mgr._supports_prompt_list_changed + + def test_removes_config_to_prevent_reconnect(self) -> None: + """remove_server_sync removes from _server_configs to prevent reconnect.""" + mgr = MCPClientManager({"test": {"command": "echo"}}) + assert "test" in mgr._server_configs + mgr.remove_server_sync("test") + assert "test" not in mgr._server_configs + + def test_preserves_other_servers(self) -> None: + """Removing one server does not affect another server's state.""" + mgr = MCPClientManager({"srv_a": {}, "srv_b": {}}) + mgr._per_server_tools["srv_a"] = [_fake_openai_tool("mcp__srv_a__foo")] + mgr._per_server_tools["srv_b"] = [_fake_openai_tool("mcp__srv_b__bar")] + mgr._rebuild_tools() + + assert len(mgr.get_tools()) == 2 + + mgr.remove_server_sync("srv_a") + + assert len(mgr.get_tools()) == 1 + assert mgr.get_tools()[0]["function"]["name"] == "mcp__srv_b__bar" + assert "srv_b" in mgr._server_configs + + +# --------------------------------------------------------------------------- +# get_server_status +# --------------------------------------------------------------------------- + + +class TestGetServerStatus: + def test_disconnected_server_in_config(self) -> None: + """Status of a configured but not connected server shows disconnected.""" + mgr = MCPClientManager({"test": {"command": "echo"}}) + status = mgr.get_server_status("test") + assert status["connected"] is False + assert status["tools"] == 0 + assert status["resources"] == 0 + assert status["prompts"] == 0 + assert status["error"] == "" + + def test_connected_server_with_tools(self) -> None: + """Status of a connected server reports correct tool/resource/prompt counts.""" + mgr = MCPClientManager({"test": {}}) + # Simulate connected state + mgr._sessions["test"] = object() # any truthy value + mgr._per_server_tools["test"] = [ + _fake_openai_tool("mcp__test__a"), + _fake_openai_tool("mcp__test__b"), + ] + mgr._per_server_resources["test"] = [_fake_resource_dict()] + mgr._per_server_prompts["test"] = [_fake_prompt_dict()] + + status = mgr.get_server_status("test") + assert status["connected"] is True + assert status["tools"] == 2 + assert status["resources"] == 1 + assert status["prompts"] == 1 + + def test_unknown_server(self) -> None: + """Status of a server not in config or sessions shows disconnected.""" + mgr = MCPClientManager({}) + status = mgr.get_server_status("unknown") + assert status["connected"] is False + assert status["tools"] == 0 + + +# --------------------------------------------------------------------------- +# get_all_server_status +# --------------------------------------------------------------------------- + + +class TestGetAllServerStatus: + def test_empty_manager(self) -> None: + """Empty manager returns empty status dict.""" + mgr = MCPClientManager({}) + assert mgr.get_all_server_status() == {} + + def test_multiple_servers(self) -> None: + """Manager with configs but no connections returns status for each.""" + mgr = MCPClientManager({"alpha": {}, "bravo": {}}) + statuses = mgr.get_all_server_status() + assert len(statuses) == 2 + assert "alpha" in statuses + assert "bravo" in statuses + assert statuses["alpha"]["connected"] is False + assert statuses["bravo"]["connected"] is False + + def test_mixed_connected_and_disconnected(self) -> None: + """Status correctly reflects a mix of connected and disconnected servers.""" + mgr = MCPClientManager({"up": {}, "down": {}}) + mgr._sessions["up"] = object() + mgr._per_server_tools["up"] = [_fake_openai_tool("mcp__up__x")] + + statuses = mgr.get_all_server_status() + assert statuses["up"]["connected"] is True + assert statuses["up"]["tools"] == 1 + assert statuses["down"]["connected"] is False + assert statuses["down"]["tools"] == 0 + + +# --------------------------------------------------------------------------- +# reconcile_sync +# --------------------------------------------------------------------------- + + +class _FakeStorage: + """Minimal mock storage for reconcile tests.""" + + def __init__(self, rows: list[dict[str, Any]]) -> None: + self._rows = rows + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + if enabled_only: + return [r for r in self._rows if r.get("enabled", True)] + return list(self._rows) + + +def _db_row( + name: str, + transport: str = "stdio", + command: str = "echo", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + enabled: bool = True, +) -> dict[str, Any]: + return { + "name": name, + "transport": transport, + "command": command, + "args": args, + "url": url, + "headers": headers, + "env": env, + "enabled": enabled, + } + + +class TestReconcileSync: + def test_adds_new_servers(self) -> None: + mgr = MCPClientManager({}) + storage = _FakeStorage([_db_row("new-srv")]) + # Can't actually connect (no loop), but config should be attempted + result = mgr.reconcile_sync(storage) + # add_server_sync fails without a loop, but the method shouldn't crash + assert "new-srv" not in result["added"] # fails gracefully + assert result["removed"] == [] + assert result["updated"] == [] + + def test_removes_stale_db_servers(self) -> None: + mgr = MCPClientManager({"old-srv": {"command": "echo"}}) + mgr._db_managed.add("old-srv") # mark as DB-managed + storage = _FakeStorage([]) # DB is empty + result = mgr.reconcile_sync(storage) + assert "old-srv" in result["removed"] + assert "old-srv" not in mgr._server_configs + + def test_preserves_config_file_servers(self) -> None: + """Config-file servers (not in _db_managed) survive reconcile.""" + mgr = MCPClientManager({"env-srv": {"command": "echo"}}) + # NOT in _db_managed — loaded from MCP_CONFIG env + storage = _FakeStorage([]) # DB is empty + result = mgr.reconcile_sync(storage) + assert result["removed"] == [] + assert "env-srv" in mgr._server_configs # still there + + def test_config_server_not_overwritten_by_db_name_collision(self) -> None: + """DB server with same name as config-file server does not replace it.""" + original_cfg = {"type": "stdio", "command": "config-echo", "args": [], "env": {}} + mgr = MCPClientManager({"shared-name": dict(original_cfg)}) + # NOT in _db_managed — this is a config-file server + # DB has a server with the same name but different config + storage = _FakeStorage([_db_row("shared-name", command="db-echo")]) + result = mgr.reconcile_sync(storage) + # Config-file server should NOT be updated + assert result["updated"] == [] + assert "shared-name" in mgr._server_configs + assert mgr._server_configs["shared-name"]["command"] == "config-echo" + + def test_updates_changed_config(self) -> None: + original_cfg = {"type": "stdio", "command": "echo", "args": [], "env": {}} + mgr = MCPClientManager({"srv": dict(original_cfg)}) + mgr._db_managed.add("srv") # mark as DB-managed + # DB has updated command — config differs + storage = _FakeStorage([_db_row("srv", command="cat")]) + result = mgr.reconcile_sync(storage) + # remove_server_sync ran (old config cleared), add_server_sync attempted + # but fails without a running event loop — that's expected in unit tests. + # The key assertion: the old config was evicted (not left stale). + assert "srv" not in mgr._server_configs + # Not in "removed" (that's for servers absent from DB) + assert "srv" not in result["removed"] + + def test_no_change_is_noop(self) -> None: + cfg = {"type": "stdio", "command": "echo", "args": [], "env": {}} + mgr = MCPClientManager({"srv": dict(cfg)}) + storage = _FakeStorage([_db_row("srv", command="echo")]) + result = mgr.reconcile_sync(storage) + assert result["added"] == [] + assert result["removed"] == [] + assert result["updated"] == [] + # Config unchanged + assert "srv" in mgr._server_configs + + def test_storage_failure_graceful(self) -> None: + mgr = MCPClientManager({"srv": {}}) + + class _BrokenStorage: + def list_mcp_servers(self, **kw: Any) -> list[dict[str, Any]]: + raise RuntimeError("DB down") + + result = mgr.reconcile_sync(_BrokenStorage()) + assert result == {"added": [], "removed": [], "updated": []} + # Existing server untouched + assert "srv" in mgr._server_configs diff --git a/tests/test_mcp_server_storage.py b/tests/test_mcp_server_storage.py new file mode 100644 index 00000000..a8ab48d1 --- /dev/null +++ b/tests/test_mcp_server_storage.py @@ -0,0 +1,152 @@ +"""Tests for MCP server storage CRUD operations.""" + +from __future__ import annotations + +import uuid + +import pytest + +from turnstone.core.storage._sqlite import SQLiteBackend + + +@pytest.fixture +def db(tmp_path): + """Fresh SQLite backend for each test.""" + return SQLiteBackend(str(tmp_path / "test.db")) + + +def _make_id() -> str: + return uuid.uuid4().hex + + +class TestMcpServerStorage: + def test_create_and_get(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server( + server_id=sid, + name="test-server", + transport="stdio", + command="echo", + args='["hello"]', + ) + s = db.get_mcp_server(sid) + assert s is not None + assert s["name"] == "test-server" + assert s["transport"] == "stdio" + assert s["command"] == "echo" + assert s["args"] == '["hello"]' + assert s["enabled"] is True + assert s["auto_approve"] is False + + def test_get_by_name(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="named-srv", transport="stdio") + s = db.get_mcp_server_by_name("named-srv") + assert s is not None + assert s["server_id"] == sid + + def test_get_by_name_not_found(self, db: SQLiteBackend) -> None: + assert db.get_mcp_server_by_name("nope") is None + + def test_get_not_found(self, db: SQLiteBackend) -> None: + assert db.get_mcp_server("nonexistent") is None + + def test_list_empty(self, db: SQLiteBackend) -> None: + assert db.list_mcp_servers() == [] + + def test_list_all(self, db: SQLiteBackend) -> None: + db.create_mcp_server(server_id=_make_id(), name="alpha", transport="stdio") + db.create_mcp_server( + server_id=_make_id(), name="beta", transport="streamable-http", url="http://x" + ) + servers = db.list_mcp_servers() + assert len(servers) == 2 + assert servers[0]["name"] == "alpha" # ordered by name + assert servers[1]["name"] == "beta" + + def test_list_enabled_only(self, db: SQLiteBackend) -> None: + sid1 = _make_id() + sid2 = _make_id() + db.create_mcp_server(server_id=sid1, name="enabled-srv", transport="stdio", enabled=True) + db.create_mcp_server(server_id=sid2, name="disabled-srv", transport="stdio", enabled=False) + enabled = db.list_mcp_servers(enabled_only=True) + assert len(enabled) == 1 + assert enabled[0]["name"] == "enabled-srv" + + def test_update_basic_fields(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="orig", transport="stdio", command="echo") + ok = db.update_mcp_server(sid, name="renamed", command="cat") + assert ok is True + s = db.get_mcp_server(sid) + assert s is not None + assert s["name"] == "renamed" + assert s["command"] == "cat" + + def test_update_boolean_conversion(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="booltest", transport="stdio") + db.update_mcp_server(sid, auto_approve=True, enabled=False) + s = db.get_mcp_server(sid) + assert s is not None + assert s["auto_approve"] is True + assert s["enabled"] is False + + def test_update_not_found(self, db: SQLiteBackend) -> None: + ok = db.update_mcp_server("nonexistent", name="x") + assert ok is False + + def test_update_ignores_disallowed_fields(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="guard", transport="stdio", created_by="admin") + original = db.get_mcp_server(sid) + assert original is not None + original_created = original["created"] + # created_by and created are not in the mutable allowlist + db.update_mcp_server(sid, created_by="evil", created="2000-01-01T00:00:00") + s = db.get_mcp_server(sid) + assert s is not None + assert s["created_by"] == "admin" # unchanged + assert s["created"] == original_created # unchanged + + def test_delete(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="delme", transport="stdio") + ok = db.delete_mcp_server(sid) + assert ok is True + assert db.get_mcp_server(sid) is None + + def test_delete_not_found(self, db: SQLiteBackend) -> None: + ok = db.delete_mcp_server("nonexistent") + assert ok is False + + def test_create_duplicate_name(self, db: SQLiteBackend) -> None: + db.create_mcp_server(server_id=_make_id(), name="unique", transport="stdio") + # Second create with same name but different ID should be no-op (OR IGNORE) + sid2 = _make_id() + db.create_mcp_server(server_id=sid2, name="unique", transport="stdio") + # OR IGNORE silently drops the conflicting insert + assert db.get_mcp_server(sid2) is None + + def test_create_idempotent_same_id(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="idem", transport="stdio", command="v1") + db.create_mcp_server(server_id=sid, name="idem", transport="stdio", command="v2") + s = db.get_mcp_server(sid) + assert s is not None + assert s["command"] == "v1" # original preserved, second ignored + + def test_http_transport_fields(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server( + server_id=sid, + name="http-srv", + transport="streamable-http", + url="https://example.com/mcp", + headers='{"Authorization":"Bearer xyz"}', + ) + s = db.get_mcp_server(sid) + assert s is not None + assert s["transport"] == "streamable-http" + assert s["url"] == "https://example.com/mcp" + assert "Authorization" in s["headers"] diff --git a/turnstone/api/console_schemas.py b/turnstone/api/console_schemas.py index a406a9e8..21a68a4e 100644 --- a/turnstone/api/console_schemas.py +++ b/turnstone/api/console_schemas.py @@ -570,3 +570,83 @@ class ListSettingSchemaResponse(BaseModel): class UpdateSettingRequest(BaseModel): value: Any node_id: str = "" + + +# --------------------------------------------------------------------------- +# Admin: MCP Servers +# --------------------------------------------------------------------------- + + +class McpServerInfo(BaseModel): + server_id: str + name: str + transport: str + command: str = "" + args: str = "[]" + url: str = "" + headers: str = "{}" + env: str = "{}" + auto_approve: bool = False + enabled: bool = True + created_by: str = "" + created: str + updated: str + + +class McpServerStatus(BaseModel): + connected: bool = False + tools: int = 0 + resources: int = 0 + prompts: int = 0 + error: str = "" + transport: str = "" + command: str = "" + url: str = "" + + +class McpServerDetail(McpServerInfo): + status: dict[str, McpServerStatus] = Field(default_factory=dict) + source: str = "" # "config" for config-file servers, empty for DB-managed + + +class CreateMcpServerRequest(BaseModel): + name: str + transport: str # "stdio" | "streamable-http" + command: str = "" + args: list[str] = [] + url: str = "" + headers: dict[str, str] = Field(default_factory=dict) + env: dict[str, str] = Field(default_factory=dict) + auto_approve: bool = False + enabled: bool = True + + +class UpdateMcpServerRequest(BaseModel): + name: str | None = None + transport: str | None = None + command: str | None = None + args: list[str] | None = None + url: str | None = None + headers: dict[str, str] | None = None + env: dict[str, str] | None = None + auto_approve: bool | None = None + enabled: bool | None = None + + +class ListMcpServersResponse(BaseModel): + servers: list[McpServerDetail] + + +class ImportMcpConfigRequest(BaseModel): + config: dict[str, Any] = Field(..., description="JSON config object with mcpServers key") + + +class ImportMcpConfigResponse(BaseModel): + imported: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + + +class McpReloadResponse(BaseModel): + status: str = "ok" + results: dict[str, Any] = Field(default_factory=dict) diff --git a/turnstone/api/console_spec.py b/turnstone/api/console_spec.py index ade0f2e0..05a9a1f5 100644 --- a/turnstone/api/console_spec.py +++ b/turnstone/api/console_spec.py @@ -20,13 +20,17 @@ from turnstone.api.console_schemas import ( ConsoleCreateWsResponse, ConsoleHealthResponse, CreateChannelUserRequest, + CreateMcpServerRequest, CreatePromptTemplateRequest, CreateRoleRequest, CreateToolPolicyRequest, CreateWsTemplateRequest, + ImportMcpConfigRequest, + ImportMcpConfigResponse, ListAdminMemoriesResponse, ListAuditEventsResponse, ListChannelUsersResponse, + ListMcpServersResponse, ListOrgsResponse, ListPromptTemplatesResponse, ListRolesResponse, @@ -38,6 +42,8 @@ from turnstone.api.console_schemas import ( ListWsTemplatesResponse, ListWsTemplateSummaryResponse, ListWsTemplateVersionsResponse, + McpReloadResponse, + McpServerDetail, NodeDetailResponse, OrgInfo, PromptTemplateInfo, @@ -45,6 +51,7 @@ from turnstone.api.console_schemas import ( SettingInfo, SettingSchemaInfo, ToolPolicyInfo, + UpdateMcpServerRequest, UpdateOrgRequest, UpdatePromptTemplateRequest, UpdateRoleRequest, @@ -661,6 +668,67 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ error_codes=[400, 404], tags=["Admin"], ), + # --- Admin: MCP Servers --- + EndpointSpec( + "/v1/api/admin/mcp-servers", + "GET", + "List MCP server definitions with live status", + response_model=ListMcpServersResponse, + query_params=[ + QueryParam("reveal", "Show secret env/header values", schema_type="boolean"), + ], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers", + "POST", + "Create an MCP server definition", + request_model=CreateMcpServerRequest, + response_model=McpServerDetail, + error_codes=[400, 409], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/{server_id}", + "GET", + "Get a single MCP server with status", + response_model=McpServerDetail, + error_codes=[404], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/{server_id}", + "PUT", + "Update an MCP server definition", + request_model=UpdateMcpServerRequest, + response_model=McpServerDetail, + error_codes=[400, 404, 409], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/{server_id}", + "DELETE", + "Delete an MCP server definition", + response_model=StatusResponse, + error_codes=[404], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/reload", + "POST", + "Tell all nodes to re-read MCP server config from DB and reconcile", + response_model=McpReloadResponse, + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/import", + "POST", + "Import MCP servers from a JSON config file", + request_model=ImportMcpConfigRequest, + response_model=ImportMcpConfigResponse, + error_codes=[400], + tags=["Admin"], + ), # --- Observability --- EndpointSpec( "/health", @@ -732,6 +800,13 @@ _ALL_MODELS: list[type[BaseModel]] = [ SettingSchemaInfo, ListSettingSchemaResponse, UpdateSettingRequest, + McpServerDetail, + CreateMcpServerRequest, + UpdateMcpServerRequest, + ListMcpServersResponse, + ImportMcpConfigRequest, + ImportMcpConfigResponse, + McpReloadResponse, ] diff --git a/turnstone/cli.py b/turnstone/cli.py index efa71bd5..865aa0d8 100644 --- a/turnstone/cli.py +++ b/turnstone/cli.py @@ -1006,10 +1006,12 @@ def main() -> None: # Initialize MCP client (connects to configured MCP servers, if any) from turnstone.core.mcp_client import create_mcp_client + from turnstone.core.storage._registry import get_storage as _get_storage mcp_client = create_mcp_client( getattr(args, "mcp_config", None), refresh_interval=getattr(args, "mcp_refresh_interval", 14400), + storage=_get_storage(), ) # ChatSession factory — captures shared config for creating workstreams diff --git a/turnstone/console/server.py b/turnstone/console/server.py index f48f61e7..b8d74f7b 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -1513,6 +1513,7 @@ _VALID_PERMISSIONS = frozenset( "admin.judge", "admin.memories", "admin.settings", + "admin.mcp", "tools.approve", "workstreams.create", "workstreams.close", @@ -2968,6 +2969,534 @@ async def admin_delete_setting(request: Request) -> JSONResponse: return JSONResponse({"status": "ok", "key": key}) +# --------------------------------------------------------------------------- +# Admin: MCP Servers +# --------------------------------------------------------------------------- + +_MCP_NAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$") +_MCP_MAX_SERVERS = 50 + + +def _mask_mcp_secrets(server: dict[str, Any], reveal: bool = False) -> dict[str, Any]: + """Replace env/headers values with '***' unless reveal is True.""" + if reveal: + return server + s = dict(server) + if s.get("env") and s["env"] != "{}": + try: + env_dict = json.loads(s["env"]) if isinstance(s["env"], str) else s["env"] + s["env"] = json.dumps({k: "***" for k in env_dict}) + except (json.JSONDecodeError, TypeError): + s["env"] = "{}" + if s.get("headers") and s["headers"] != "{}": + try: + hdr_dict = json.loads(s["headers"]) if isinstance(s["headers"], str) else s["headers"] + s["headers"] = json.dumps({k: "***" for k in hdr_dict}) + except (json.JSONDecodeError, TypeError): + s["headers"] = "{}" + return s + + +def _mcp_server_to_detail( + server: dict[str, Any], + node_statuses: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Convert a storage dict to a McpServerDetail-shaped dict.""" + d = dict(server) + d["status"] = node_statuses or {} + return d + + +async def _collect_mcp_status( + request: Request, +) -> dict[str, dict[str, dict[str, Any]]]: + """Query all nodes for MCP status. Returns {node_id: {server_name: status}}.""" + collector: ClusterCollector = request.app.state.collector + nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + result: dict[str, dict[str, dict[str, Any]]] = {} + for node in nodes: + node_id = node.get("node_id", "") + url = node.get("server_url", "") + if not url: + continue + try: + headers = _proxy_auth_headers(request) + async with httpx.AsyncClient(timeout=httpx.Timeout(10)) as client: + resp = await client.get( + f"{url.rstrip('/')}/v1/api/_internal/mcp-status", + headers=headers, + ) + if resp.status_code == 200: + result[node_id] = resp.json().get("servers", {}) + except Exception: + pass + return result + + +async def admin_list_mcp_servers(request: Request) -> JSONResponse: + """GET /v1/api/admin/mcp-servers — list all MCP server definitions.""" + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + reveal = str(request.query_params.get("reveal", "")).lower() in ("true", "1") + servers = storage.list_mcp_servers() + + # Collect live status from all nodes + node_statuses = await _collect_mcp_status(request) + + db_names: set[str] = set() + result = [] + for s in servers: + db_names.add(s["name"]) + # Build per-node status for this server + per_node: dict[str, dict[str, Any]] = {} + for node_id, node_servers in node_statuses.items(): + status = node_servers.get(s["name"]) + if status: + per_node[node_id] = status + s = _mask_mcp_secrets(s, reveal) + result.append(_mcp_server_to_detail(s, per_node)) + + # Merge config-sourced servers visible on nodes but not in DB + config_names: set[str] = set() + for node_servers in node_statuses.values(): + for name in node_servers: + if name not in db_names: + config_names.add(name) + for name in sorted(config_names): + # Build a synthetic read-only entry from node-reported data + per_node = {} + transport = "stdio" + command = "" + url = "" + for node_id, node_servers in node_statuses.items(): + ns = node_servers.get(name) + if ns: + per_node[node_id] = ns + transport = ns.get("transport", "stdio") + command = ns.get("command", "") + url = ns.get("url", "") + result.append( + { + "server_id": "", + "name": name, + "transport": transport, + "command": command, + "args": "[]", + "url": url, + "headers": "{}", + "env": "{}", + "auto_approve": False, + "enabled": True, + "created_by": "", + "created": "", + "updated": "", + "source": "config", + "status": per_node, + } + ) + + return JSONResponse({"servers": result}) + + +async def admin_create_mcp_server(request: Request) -> JSONResponse: + """POST /v1/api/admin/mcp-servers — create an MCP server definition.""" + import uuid + + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + + name = str(body.get("name", "")).strip()[:64] + transport = str(body.get("transport", "")).strip() + if not name: + return JSONResponse({"error": "name is required"}, status_code=400) + if not _MCP_NAME_RE.match(name): + return JSONResponse( + {"error": "name must match [a-zA-Z0-9._-]+"}, + status_code=400, + ) + if "__" in name: + return JSONResponse( + {"error": "name must not contain '__' (reserved delimiter)"}, + status_code=400, + ) + if transport not in ("stdio", "streamable-http"): + return JSONResponse( + {"error": "transport must be 'stdio' or 'streamable-http'"}, + status_code=400, + ) + if transport == "stdio" and not str(body.get("command", "")).strip(): + return JSONResponse({"error": "command is required for stdio transport"}, status_code=400) + if transport == "streamable-http" and not str(body.get("url", "")).strip(): + return JSONResponse( + {"error": "url is required for streamable-http transport"}, status_code=400 + ) + + # Check max servers + existing = storage.list_mcp_servers() + if len(existing) >= _MCP_MAX_SERVERS: + return JSONResponse( + {"error": f"Maximum {_MCP_MAX_SERVERS} servers"}, + status_code=400, + ) + + # Check name uniqueness + if storage.get_mcp_server_by_name(name): + return JSONResponse( + {"error": f"Server '{name}' already exists"}, + status_code=409, + ) + + server_id = uuid.uuid4().hex + audit_uid, ip = _audit_context(request) + + args_list = body.get("args", []) + headers_dict = body.get("headers", {}) + env_dict = body.get("env", {}) + + storage.create_mcp_server( + server_id=server_id, + name=name, + transport=transport, + command=str(body.get("command", "")).strip(), + args=json.dumps(args_list) if isinstance(args_list, list) else "[]", + url=str(body.get("url", "")).strip(), + headers=json.dumps(headers_dict) if isinstance(headers_dict, dict) else "{}", + env=json.dumps(env_dict) if isinstance(env_dict, dict) else "{}", + auto_approve=bool(body.get("auto_approve", False)), + enabled=bool(body.get("enabled", True)), + created_by=audit_uid, + ) + + record_audit( + storage, + audit_uid, + "mcp_server.create", + "mcp_server", + server_id, + {"name": name}, + ip, + ) + + server = storage.get_mcp_server(server_id) + return JSONResponse(_mcp_server_to_detail(_mask_mcp_secrets(server or {}))) + + +async def admin_get_mcp_server(request: Request) -> JSONResponse: + """GET /v1/api/admin/mcp-servers/{server_id} — get single MCP server.""" + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + server_id = request.path_params["server_id"] + server = storage.get_mcp_server(server_id) + if server is None: + return JSONResponse({"error": "MCP server not found"}, status_code=404) + + node_statuses = await _collect_mcp_status(request) + per_node: dict[str, dict[str, Any]] = {} + for node_id, node_servers in node_statuses.items(): + status = node_servers.get(server["name"]) + if status: + per_node[node_id] = status + + reveal = str(request.query_params.get("reveal", "")).lower() in ("true", "1") + server = _mask_mcp_secrets(server, reveal) + return JSONResponse(_mcp_server_to_detail(server, per_node)) + + +async def admin_update_mcp_server(request: Request) -> JSONResponse: + """PUT /v1/api/admin/mcp-servers/{server_id} — update an MCP server.""" + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + server_id = request.path_params["server_id"] + existing = storage.get_mcp_server(server_id) + if existing is None: + return JSONResponse({"error": "MCP server not found"}, status_code=404) + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + + updates: dict[str, Any] = {} + if "name" in body: + name = str(body["name"]).strip()[:64] + if not name: + return JSONResponse({"error": "name cannot be empty"}, status_code=400) + if not _MCP_NAME_RE.match(name): + return JSONResponse( + {"error": "name must match [a-zA-Z0-9._-]+"}, + status_code=400, + ) + if "__" in name: + return JSONResponse( + {"error": "name must not contain '__'"}, + status_code=400, + ) + if name != existing["name"] and storage.get_mcp_server_by_name(name): + return JSONResponse( + {"error": f"Server '{name}' already exists"}, + status_code=409, + ) + updates["name"] = name + if "transport" in body: + transport = str(body["transport"]).strip() + if transport not in ("stdio", "streamable-http"): + return JSONResponse( + {"error": "transport must be 'stdio' or 'streamable-http'"}, + status_code=400, + ) + updates["transport"] = transport + if "command" in body: + updates["command"] = str(body["command"]).strip() + if "args" in body: + updates["args"] = json.dumps(body["args"]) if isinstance(body["args"], list) else "[]" + if "url" in body: + updates["url"] = str(body["url"]).strip() + if "headers" in body: + updates["headers"] = ( + json.dumps(body["headers"]) if isinstance(body["headers"], dict) else "{}" + ) + if "env" in body: + updates["env"] = json.dumps(body["env"]) if isinstance(body["env"], dict) else "{}" + if "auto_approve" in body: + updates["auto_approve"] = bool(body["auto_approve"]) + if "enabled" in body: + updates["enabled"] = bool(body["enabled"]) + + if updates: + storage.update_mcp_server(server_id, **updates) + + audit_uid, ip = _audit_context(request) + audit_detail = dict(updates) + for _secret_key in ("env", "headers"): + if _secret_key in audit_detail: + audit_detail[_secret_key] = "(updated)" + record_audit( + storage, + audit_uid, + "mcp_server.update", + "mcp_server", + server_id, + audit_detail, + ip, + ) + + server = storage.get_mcp_server(server_id) + return JSONResponse(_mcp_server_to_detail(_mask_mcp_secrets(server or {}))) + + +async def admin_delete_mcp_server(request: Request) -> JSONResponse: + """DELETE /v1/api/admin/mcp-servers/{server_id}.""" + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + server_id = request.path_params["server_id"] + existing = storage.get_mcp_server(server_id) + if existing is None: + return JSONResponse({"error": "MCP server not found"}, status_code=404) + + storage.delete_mcp_server(server_id) + + audit_uid, ip = _audit_context(request) + record_audit( + storage, + audit_uid, + "mcp_server.delete", + "mcp_server", + server_id, + {"name": existing.get("name", "")}, + ip, + ) + + return JSONResponse({"status": "ok"}) + + +async def _notify_nodes_mcp_reload(request: Request) -> dict[str, Any]: + """Tell all nodes to re-read the mcp_servers DB table and reconcile.""" + collector: ClusterCollector = request.app.state.collector + nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + results: dict[str, Any] = {} + + for node in nodes: + node_id = node.get("node_id", "") + url = node.get("server_url", "") + if not url: + continue + try: + headers = _proxy_auth_headers(request) + async with httpx.AsyncClient(timeout=httpx.Timeout(30)) as client: + resp = await client.post( + f"{url.rstrip('/')}/v1/api/_internal/mcp-reload", + headers=headers, + ) + results[node_id] = resp.json() + except Exception as exc: + results[node_id] = {"error": str(exc)} + + return results + + +async def admin_mcp_reload(request: Request) -> JSONResponse: + """POST /v1/api/admin/mcp-servers/reload — tell nodes to re-read DB.""" + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + results = await _notify_nodes_mcp_reload(request) + return JSONResponse({"status": "ok", "results": results}) + + +async def admin_import_mcp_config(request: Request) -> JSONResponse: + """POST /v1/api/admin/mcp-servers/import — import from pasted JSON config.""" + import uuid + + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + + data = body.get("config") + if not isinstance(data, dict): + return JSONResponse( + {"error": "config is required (JSON object with mcpServers key)"}, status_code=400 + ) + + servers = data.get("mcpServers", {}) + if not isinstance(servers, dict) or not servers: + return JSONResponse( + {"error": "No mcpServers found in config"}, + status_code=400, + ) + + imported: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + audit_uid, ip = _audit_context(request) + current_count = len(storage.list_mcp_servers()) + + for srv_name, cfg in servers.items(): + srv_name = str(srv_name).strip()[:64] + if not srv_name or not _MCP_NAME_RE.match(srv_name) or "__" in srv_name: + errors.append(f"{srv_name}: invalid server name") + continue + if storage.get_mcp_server_by_name(srv_name): + skipped.append(srv_name) + continue + if current_count >= _MCP_MAX_SERVERS: + errors.append(f"{srv_name}: max servers reached") + break + + transport = "stdio" + if "url" in cfg or cfg.get("type") in ("http", "streamable-http"): + transport = "streamable-http" + + # Coerce fields to expected types + raw_args = cfg.get("args", []) + raw_headers = cfg.get("headers", {}) + raw_env = cfg.get("env", {}) + if not isinstance(raw_args, list): + errors.append(f"{srv_name}: args must be a list") + continue + if not isinstance(raw_headers, dict): + errors.append(f"{srv_name}: headers must be an object") + continue + if not isinstance(raw_env, dict): + errors.append(f"{srv_name}: env must be an object") + continue + + server_id = uuid.uuid4().hex + try: + storage.create_mcp_server( + server_id=server_id, + name=srv_name, + transport=transport, + command=str(cfg.get("command", "")), + args=json.dumps(raw_args), + url=str(cfg.get("url", "")), + headers=json.dumps(raw_headers), + env=json.dumps(raw_env), + auto_approve=False, + enabled=True, + created_by=audit_uid, + ) + imported.append(srv_name) + current_count += 1 + except Exception as exc: + errors.append(f"{srv_name}: {exc}") + + if imported: + record_audit( + storage, + audit_uid, + "mcp_server.import", + "mcp_server", + "", + {"imported": imported, "skipped": skipped}, + ip, + ) + + return JSONResponse({"imported": imported, "skipped": skipped, "errors": errors}) + + # --------------------------------------------------------------------------- # App factory # --------------------------------------------------------------------------- @@ -3131,6 +3660,37 @@ def create_app( admin_delete_setting, methods=["DELETE"], ), + # System: MCP Servers + Route("/api/admin/mcp-servers", admin_list_mcp_servers), + Route( + "/api/admin/mcp-servers", + admin_create_mcp_server, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/import", + admin_import_mcp_config, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/reload", + admin_mcp_reload, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_get_mcp_server, + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_update_mcp_server, + methods=["PUT"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_delete_mcp_server, + methods=["DELETE"], + ), # Governance: Usage & Audit Route("/api/admin/usage", admin_usage), Route("/api/admin/audit", admin_audit), diff --git a/turnstone/console/static/admin.js b/turnstone/console/static/admin.js index e224551e..00ce7683 100644 --- a/turnstone/console/static/admin.js +++ b/turnstone/console/static/admin.js @@ -64,7 +64,8 @@ function showAdmin() { usage: "admin.usage", audit: "admin.audit", memories: "admin.memories", - settings: "admin.users", + settings: "admin.settings", + mcp: "admin.mcp", }; if (perms) { var permSet = perms.split(","); @@ -194,6 +195,7 @@ function switchAdminTab(tab) { "audit", "memories", "settings", + "mcp", ]; for (var p = 0; p < panels.length; p++) { var el = document.getElementById("admin-" + panels[p]); @@ -216,6 +218,7 @@ function switchAdminTab(tab) { } if (tab === "memories") loadAdminMemories(); if (tab === "settings") loadSettings(); + if (tab === "mcp") loadAdminMcp(); // Update breadcrumb with active tab label var activeNav = document.querySelector('.admin-nav[data-tab="' + tab + '"]'); @@ -1593,6 +1596,9 @@ function _installTrap(overlayId, boxId, trapRef) { else if (overlayId === "edit-wst-overlay") hideEditWsTemplateModal(); else if (overlayId === "wst-history-overlay") hideWstHistoryModal(); else if (overlayId === "memory-detail-overlay") hideMemoryDetailModal(); + else if (overlayId === "mcp-create-overlay") hideCreateMcpModal(); + else if (overlayId === "mcp-import-overlay") hideImportMcpModal(); + else if (overlayId === "mcp-detail-overlay") hideMcpDetailModal(); } }; } @@ -1679,6 +1685,9 @@ document.addEventListener("keydown", function (e) { ["edit-wst-overlay", hideEditWsTemplateModal], ["wst-history-overlay", hideWstHistoryModal], ["memory-detail-overlay", hideMemoryDetailModal], + ["mcp-detail-overlay", hideMcpDetailModal], + ["mcp-import-overlay", hideImportMcpModal], + ["mcp-create-overlay", hideCreateMcpModal], ]; for (var gi = 0; gi < govOverlays.length; gi++) { var govEl = document.getElementById(govOverlays[gi][0]); @@ -2354,3 +2363,565 @@ function _showModalError(el, msg) { el.textContent = msg; el.style.display = "block"; } + +/* ── MCP Servers tab ─────────────────────────────────────────────────────── */ + +var _mcpServers = []; +var _mcpCreateTrap = null; +var _mcpCreateTrigger = null; +var _mcpImportTrap = null; +var _mcpImportTrigger = null; +var _mcpDetailTrap = null; +var _mcpDetailTrigger = null; + +function loadAdminMcp() { + authFetch("/v1/api/admin/mcp-servers") + .then(function (r) { + if (!r.ok) throw new Error("Failed"); + return r.json(); + }) + .then(function (data) { + _mcpServers = data.servers || []; + _renderMcpServers(_mcpServers); + }) + .catch(function () { + document.getElementById("admin-mcp-table").innerHTML = + '
Failed to load MCP servers
'; + }); +} + +function _renderMcpServers(items) { + var el = document.getElementById("admin-mcp-table"); + if (!items.length) { + el.innerHTML = + '
No MCP servers configured
'; + return; + } + var html = ""; + for (var i = 0; i < items.length; i++) { + var s = items[i]; + var statusEntries = s.status || {}; + var nodeIds = Object.keys(statusEntries); + var anyConnected = false; + var anyError = false; + var totalTools = 0, + totalRes = 0, + totalPrompts = 0; + for (var j = 0; j < nodeIds.length; j++) { + var ns = statusEntries[nodeIds[j]]; + if (ns.connected) { + anyConnected = true; + totalTools += ns.tools || 0; + totalRes += ns.resources || 0; + totalPrompts += ns.prompts || 0; + } + if (ns.error) anyError = true; + } + + var dotClass = "mcp-status-dot disabled"; + var rowClass = "mcp-row-disabled"; + var statusText = "disabled"; + if (!s.enabled) { + statusText = "disabled"; + } else if (anyConnected) { + dotClass = "mcp-status-dot connected"; + rowClass = "mcp-row-connected"; + statusText = "connected"; + } else if (anyError) { + dotClass = "mcp-status-dot error"; + rowClass = "mcp-row-error"; + statusText = "error"; + } else { + dotClass = "mcp-status-dot disabled"; + rowClass = "mcp-row-disabled"; + statusText = "idle"; + } + + var transportCls = + s.transport === "stdio" ? "mcp-transport-stdio" : "mcp-transport-http"; + var toolsVal = anyConnected + ? totalTools + : '--'; + var resVal = anyConnected + ? totalRes + : '--'; + var promptsVal = anyConnected + ? totalPrompts + : '--'; + + var isConfig = s.source === "config"; + var nameBadge = isConfig + ? ' config' + : ""; + var detailAttr = isConfig + ? 'data-mcp-detail-name="' + escapeHtml(s.name) + '"' + : 'data-mcp-detail="' + escapeHtml(s.server_id) + '"'; + var actions = isConfig + ? "" + : '' + + ''; + + html += + '
' + + '" + + escapeHtml(s.name) + + "" + + nameBadge + + "" + + '' + + escapeHtml(s.transport) + + "" + + '' + + toolsVal + + "" + + '' + + resVal + + "" + + '' + + promptsVal + + "" + + '' + + escapeHtml(statusText) + + "" + + '' + + actions + + "
"; + } + el.innerHTML = html; + + // Bind event handlers + el.querySelectorAll("[data-mcp-detail]").forEach(function (a) { + a.addEventListener("click", function (e) { + e.preventDefault(); + showMcpDetailModal(this.getAttribute("data-mcp-detail")); + }); + }); + el.querySelectorAll("[data-mcp-detail-name]").forEach(function (a) { + a.addEventListener("click", function (e) { + e.preventDefault(); + showMcpDetailByName(this.getAttribute("data-mcp-detail-name")); + }); + }); + el.querySelectorAll("[data-mcp-edit]").forEach(function (btn) { + btn.addEventListener("click", function () { + showEditMcpModal(this.getAttribute("data-mcp-edit")); + }); + }); + el.querySelectorAll("[data-mcp-delete]").forEach(function (btn) { + btn.addEventListener("click", function () { + var sid = this.getAttribute("data-mcp-delete"); + var sname = this.getAttribute("data-mcp-name"); + showConfirmModal( + "Delete MCP Server", + 'Delete server "' + sname + '"?', + "Delete", + function () { + authFetch("/v1/api/admin/mcp-servers/" + sid, { method: "DELETE" }) + .then(function (r) { + if (!r.ok) throw new Error(); + return r.json(); + }) + .then(function () { + showToast("Server deleted"); + loadAdminMcp(); + }) + .catch(function () { + showToast("Failed to delete server"); + }); + }, + ); + }); + }); +} + +function toggleMcpTransport() { + var v = document.getElementById("mcp-transport").value; + document.getElementById("mcp-stdio-fields").style.display = + v === "stdio" ? "" : "none"; + document.getElementById("mcp-http-fields").style.display = + v === "streamable-http" ? "" : "none"; +} + +function showCreateMcpModal() { + _mcpCreateTrigger = document.activeElement; + var ov = document.getElementById("mcp-create-overlay"); + ov.style.display = "flex"; + document.getElementById("mcp-edit-id").value = ""; + document.getElementById("mcp-create-title").textContent = "Add MCP Server"; + document.getElementById("mcp-create-submit").textContent = "Create"; + document.getElementById("mcp-name").value = ""; + document.getElementById("mcp-transport").value = "stdio"; + document.getElementById("mcp-command").value = ""; + document.getElementById("mcp-args").value = ""; + document.getElementById("mcp-env").value = ""; + document.getElementById("mcp-url").value = ""; + document.getElementById("mcp-headers").value = ""; + document.getElementById("mcp-auto-approve").checked = false; + document.getElementById("mcp-enabled").checked = true; + document.getElementById("mcp-create-error").style.display = "none"; + toggleMcpTransport(); + document.getElementById("mcp-name").focus(); + _mcpCreateTrap = _installTrap("mcp-create-overlay", "mcp-create-box"); +} + +function showEditMcpModal(serverId) { + // Fetch with reveal=true to get actual secret values for editing + authFetch("/v1/api/admin/mcp-servers/" + serverId + "?reveal=true") + .then(function (r) { + if (!r.ok) throw new Error("Failed to load server"); + return r.json(); + }) + .then(function (s) { + showCreateMcpModal(); + document.getElementById("mcp-edit-id").value = serverId; + document.getElementById("mcp-create-title").textContent = + "Edit MCP Server"; + document.getElementById("mcp-create-submit").textContent = "Save"; + document.getElementById("mcp-name").value = s.name; + document.getElementById("mcp-transport").value = s.transport; + document.getElementById("mcp-command").value = s.command || ""; + try { + var argsList = JSON.parse(s.args || "[]"); + document.getElementById("mcp-args").value = argsList.join("\n"); + } catch (e) { + document.getElementById("mcp-args").value = ""; + } + try { + var envObj = JSON.parse(s.env || "{}"); + document.getElementById("mcp-env").value = Object.keys(envObj) + .map(function (k) { + return k + "=" + envObj[k]; + }) + .join("\n"); + } catch (e) { + document.getElementById("mcp-env").value = ""; + } + document.getElementById("mcp-url").value = s.url || ""; + try { + var hdrObj = JSON.parse(s.headers || "{}"); + document.getElementById("mcp-headers").value = Object.keys(hdrObj) + .map(function (k) { + return k + ": " + hdrObj[k]; + }) + .join("\n"); + } catch (e) { + document.getElementById("mcp-headers").value = ""; + } + document.getElementById("mcp-auto-approve").checked = + s.auto_approve || false; + document.getElementById("mcp-enabled").checked = s.enabled !== false; + toggleMcpTransport(); + }) + .catch(function () { + showToast("Failed to load server details"); + }); +} + +function hideCreateMcpModal() { + document.getElementById("mcp-create-overlay").style.display = "none"; + _mcpCreateTrap = _removeTrap(_mcpCreateTrap); + if (_mcpCreateTrigger && _mcpCreateTrigger.focus) _mcpCreateTrigger.focus(); + _mcpCreateTrigger = null; +} + +function _parseMcpForm() { + var name = document.getElementById("mcp-name").value.trim(); + var transport = document.getElementById("mcp-transport").value; + if (!name) return { error: "Name is required" }; + if (!/^[a-zA-Z0-9._-]+$/.test(name)) + return { error: "Name must match [a-zA-Z0-9._-]+" }; + if (name.indexOf("__") >= 0) return { error: "Name must not contain '__'" }; + + var payload = { + name: name, + transport: transport, + auto_approve: document.getElementById("mcp-auto-approve").checked, + enabled: document.getElementById("mcp-enabled").checked, + }; + + if (transport === "stdio") { + payload.command = document.getElementById("mcp-command").value.trim(); + var argsText = document.getElementById("mcp-args").value.trim(); + payload.args = argsText + ? argsText + .split("\n") + .map(function (l) { + return l.trim(); + }) + .filter(Boolean) + : []; + var envText = document.getElementById("mcp-env").value.trim(); + var envObj = {}; + if (envText) { + envText.split("\n").forEach(function (line) { + var eq = line.indexOf("="); + if (eq > 0) + envObj[line.substring(0, eq).trim()] = line.substring(eq + 1).trim(); + }); + } + payload.env = envObj; + } else { + payload.url = document.getElementById("mcp-url").value.trim(); + var hdrText = document.getElementById("mcp-headers").value.trim(); + var hdrObj = {}; + if (hdrText) { + hdrText.split("\n").forEach(function (line) { + var colon = line.indexOf(":"); + if (colon > 0) + hdrObj[line.substring(0, colon).trim()] = line + .substring(colon + 1) + .trim(); + }); + } + payload.headers = hdrObj; + } + return payload; +} + +function submitCreateMcp() { + var form = _parseMcpForm(); + if (form.error) { + var e = document.getElementById("mcp-create-error"); + e.textContent = form.error; + e.style.display = ""; + return; + } + var editId = document.getElementById("mcp-edit-id").value; + var method = editId ? "PUT" : "POST"; + var url = editId + ? "/v1/api/admin/mcp-servers/" + editId + : "/v1/api/admin/mcp-servers"; + + document.getElementById("mcp-create-submit").disabled = true; + authFetch(url, { + method: method, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(form), + }) + .then(function (r) { + if (!r.ok) + return r.json().then(function (d) { + throw new Error(d.error || "Failed"); + }); + return r.json(); + }) + .then(function () { + hideCreateMcpModal(); + showToast(editId ? "Server updated" : "Server created"); + loadAdminMcp(); + }) + .catch(function (e) { + var el = document.getElementById("mcp-create-error"); + el.textContent = e.message; + el.style.display = ""; + }) + .finally(function () { + document.getElementById("mcp-create-submit").disabled = false; + }); +} + +function reloadMcpNodes() { + authFetch("/v1/api/admin/mcp-servers/reload", { method: "POST" }) + .then(function (r) { + if (!r.ok) throw new Error(); + return r.json(); + }) + .then(function (data) { + var results = data.results || {}; + var nodeIds = Object.keys(results); + var totalAdded = 0, + totalRemoved = 0; + for (var i = 0; i < nodeIds.length; i++) { + var nr = results[nodeIds[i]]; + totalAdded += (nr.added || []).length; + totalRemoved += (nr.removed || []).length; + } + var msg = "Reload sent to " + nodeIds.length + " node(s)"; + if (totalAdded) msg += ", +" + totalAdded + " added"; + if (totalRemoved) msg += ", -" + totalRemoved + " removed"; + showToast(msg); + setTimeout(loadAdminMcp, 1500); + }) + .catch(function () { + showToast("Failed to reload nodes"); + }); +} + +function showMcpDetailByName(name) { + for (var i = 0; i < _mcpServers.length; i++) { + if (_mcpServers[i].name === name) { + return _openMcpDetail(_mcpServers[i]); + } + } +} + +function showMcpDetailModal(serverId) { + for (var i = 0; i < _mcpServers.length; i++) { + if (_mcpServers[i].server_id === serverId) { + return _openMcpDetail(_mcpServers[i]); + } + } +} + +function _openMcpDetail(s) { + if (!s) return; + _mcpDetailTrigger = document.activeElement; + + var html = '
'; + html += '
'; + html += '

Configuration

'; + html += + '

Transport: ' + + escapeHtml(s.transport) + + "

"; + if (s.transport === "stdio") { + html += + '

Command: ' + + escapeHtml(s.command || "") + + "

"; + try { + var a = JSON.parse(s.args || "[]"); + if (a.length) + html += + '

Args: ' + + escapeHtml(a.join(" ")) + + "

"; + } catch (e) {} + } else { + html += + '

URL: ' + + escapeHtml(s.url || "") + + "

"; + } + html += "
"; + + html += '
'; + var statusEntries = s.status || {}; + var nodeIds = Object.keys(statusEntries); + html += '

Node Status

'; + if (nodeIds.length === 0) { + html += + '

Not connected on any node

'; + } else { + html += '
    '; + for (var j = 0; j < nodeIds.length; j++) { + var ns = statusEntries[nodeIds[j]]; + var dot = ns.connected + ? '' + : ''; + html += + "
  • " + + dot + + escapeHtml(nodeIds[j]) + + " — " + + (ns.tools || 0) + + " tools, " + + (ns.resources || 0) + + " resources, " + + (ns.prompts || 0) + + " prompts
    • "; + } + html += "
"; + } + html += "
"; + + document.getElementById("mcp-detail-title").textContent = s.name; + document.getElementById("mcp-detail-content").innerHTML = html; + document.getElementById("mcp-detail-overlay").style.display = "flex"; + _mcpDetailTrap = _installTrap("mcp-detail-overlay", "mcp-detail-box"); +} + +function hideMcpDetailModal() { + document.getElementById("mcp-detail-overlay").style.display = "none"; + _mcpDetailTrap = _removeTrap(_mcpDetailTrap); + if (_mcpDetailTrigger && _mcpDetailTrigger.focus) _mcpDetailTrigger.focus(); + _mcpDetailTrigger = null; +} + +function showImportMcpModal() { + _mcpImportTrigger = document.activeElement; + document.getElementById("mcp-import-overlay").style.display = "flex"; + document.getElementById("mcp-import-json").value = ""; + document.getElementById("mcp-import-error").style.display = "none"; + document.getElementById("mcp-import-json").focus(); + _mcpImportTrap = _installTrap("mcp-import-overlay", "mcp-import-box"); +} + +function hideImportMcpModal() { + document.getElementById("mcp-import-overlay").style.display = "none"; + _mcpImportTrap = _removeTrap(_mcpImportTrap); + if (_mcpImportTrigger && _mcpImportTrigger.focus) _mcpImportTrigger.focus(); + _mcpImportTrigger = null; +} + +function submitImportMcp() { + var raw = document.getElementById("mcp-import-json").value.trim(); + if (!raw) { + var e = document.getElementById("mcp-import-error"); + e.textContent = "Paste a JSON config"; + e.style.display = ""; + return; + } + var parsed; + try { + parsed = JSON.parse(raw); + } catch (ex) { + var e2 = document.getElementById("mcp-import-error"); + e2.textContent = "Invalid JSON: " + ex.message; + e2.style.display = ""; + return; + } + if (!parsed.mcpServers || typeof parsed.mcpServers !== "object") { + var e3 = document.getElementById("mcp-import-error"); + e3.textContent = 'No "mcpServers" key found in JSON'; + e3.style.display = ""; + return; + } + document.getElementById("mcp-import-submit").disabled = true; + authFetch("/v1/api/admin/mcp-servers/import", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ config: parsed }), + }) + .then(function (r) { + if (!r.ok) + return r.json().then(function (d) { + throw new Error(d.error || "Failed"); + }); + return r.json(); + }) + .then(function (data) { + hideImportMcpModal(); + var msg = "Imported " + (data.imported || []).length; + if ((data.skipped || []).length) + msg += ", skipped " + data.skipped.length; + if ((data.errors || []).length) + msg += ", " + data.errors.length + " error(s)"; + showToast(msg); + loadAdminMcp(); + }) + .catch(function (e) { + var el = document.getElementById("mcp-import-error"); + el.textContent = e.message; + el.style.display = ""; + }) + .finally(function () { + document.getElementById("mcp-import-submit").disabled = false; + }); +} diff --git a/turnstone/console/static/governance.js b/turnstone/console/static/governance.js index 1a2b6c6c..913dc6e7 100644 --- a/turnstone/console/static/governance.js +++ b/turnstone/console/static/governance.js @@ -152,10 +152,15 @@ var _ALL_PERMISSIONS = [ "admin.orgs", "admin.policies", "admin.templates", + "admin.ws_templates", "admin.audit", "admin.usage", "admin.schedules", "admin.watches", + "admin.judge", + "admin.memories", + "admin.settings", + "admin.mcp", "tools.approve", "workstreams.create", "workstreams.close", diff --git a/turnstone/console/static/index.html b/turnstone/console/static/index.html index 2ded62cc..cd5ab8c6 100644 --- a/turnstone/console/static/index.html +++ b/turnstone/console/static/index.html @@ -107,6 +107,7 @@
+
@@ -393,6 +394,27 @@ + + @@ -972,6 +994,67 @@ window.TURNSTONE_KB_SHORTCUTS = [ + + + + + + diff --git a/turnstone/console/static/style.css b/turnstone/console/static/style.css index 670e5158..094fa202 100644 --- a/turnstone/console/static/style.css +++ b/turnstone/console/static/style.css @@ -1173,7 +1173,8 @@ #create-policy-overlay, #edit-policy-overlay, #create-template-overlay, #edit-template-overlay, #create-wst-overlay, #edit-wst-overlay, #wst-history-overlay, -#memory-detail-overlay { +#memory-detail-overlay, +#mcp-create-overlay, #mcp-import-overlay, #mcp-detail-overlay { position: fixed; inset: 0; background: rgba(0, 0, 0, 0.7); @@ -1851,6 +1852,45 @@ .settings-input select { max-width: 100%; } } +/* -- MCP Servers grid ----------------------------------------------------- */ +.admin-col-mname a{color:var(--fg);text-decoration:none;transition:color .15s} +.admin-col-mname a:hover{color:var(--magenta)} +.admin-col-mname a:focus-visible{outline:2px solid var(--magenta);outline-offset:2px} +.mcp-grid{grid-template-columns:1.5fr 80px 55px 45px 80px 95px 120px;gap:0 6px} +@media(max-width:700px){ + .mcp-grid{grid-template-columns:1fr 100px 130px} + .admin-col-mtransport,.admin-col-mtools,.admin-col-mres,.admin-col-mprompts{display:none} +} + +.mcp-status-dot{display:inline-block;width:8px;height:8px;border-radius:50%;vertical-align:middle;margin-right:6px} +.mcp-status-dot.connected{background:var(--magenta);box-shadow:0 0 6px var(--magenta-glow, rgba(192,132,252,.45))} +.mcp-status-dot.error{background:var(--red);box-shadow:0 0 6px var(--red-glow);border-radius:1px} +.mcp-status-dot.disabled{background:var(--fg-dim);opacity:.35} +.mcp-status-dot.connecting{background:var(--magenta);animation:mcp-pulse 1.2s ease-in-out infinite} +@keyframes mcp-pulse{0%,100%{opacity:.3}50%{opacity:1}} + +.mcp-row-connected{border-left:3px solid var(--magenta)} +.mcp-row-error{border-left:3px solid var(--red)} +.mcp-row-disabled{border-left:3px solid transparent} + +.mcp-transport-badge{display:inline-block;font-size:9px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;padding:1px 6px;border-radius:2px;background:var(--bg-highlight);border:1px solid var(--border)} +.mcp-transport-stdio{color:var(--cyan);border-color:rgba(103,232,249,.2)} +.mcp-transport-http{color:var(--magenta);border-color:rgba(192,132,252,.25)} + +.admin-col-mtools,.admin-col-mres,.admin-col-mprompts{text-align:right;font-variant-numeric:tabular-nums} +.mcp-count-dim{opacity:.4} + +.mcp-detail-modal::before{background:linear-gradient(90deg,transparent,var(--magenta),transparent)!important} +.mcp-detail-modal h2{color:var(--magenta)!important} +.mcp-detail-section{margin-top:16px} +.mcp-detail-section h3{font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.08em;color:var(--magenta);margin-bottom:8px} +.mcp-detail-list{list-style:none;padding:0;margin:0} +.mcp-detail-list li{font-size:12px;padding:3px 0;border-bottom:1px solid var(--border);color:var(--fg-dim)} +.mcp-detail-list li:last-child{border-bottom:none} + +.admin-action-btn-ghost{background:transparent;color:var(--fg-dim);border:1px solid var(--border-strong)} +.admin-action-btn-ghost:hover{color:var(--fg);background:var(--bg-highlight)} + /* ========================================================================== Reduced motion — console-specific ========================================================================== */ @@ -1868,4 +1908,5 @@ #view-admin { animation: none; } .admin-action-btn, .modal-cancel, .modal-submit { transition: none; } .admin-modal input, .admin-modal select { transition: none; } + .mcp-status-dot.connecting { animation: none; } } diff --git a/turnstone/core/auth.py b/turnstone/core/auth.py index f8140e05..3bd01853 100644 --- a/turnstone/core/auth.py +++ b/turnstone/core/auth.py @@ -167,7 +167,9 @@ WRITE_PATHS: frozenset[str] = frozenset( } ) -APPROVE_PATHS: frozenset[str] = frozenset({"/api/approve", "/api/_internal/config-reload"}) +APPROVE_PATHS: frozenset[str] = frozenset( + {"/api/approve", "/api/_internal/config-reload", "/api/_internal/mcp-reload"} +) ADMIN_PREFIX = "/api/admin/" diff --git a/turnstone/core/mcp_client.py b/turnstone/core/mcp_client.py index 496c3fdf..96ec6abb 100644 --- a/turnstone/core/mcp_client.py +++ b/turnstone/core/mcp_client.py @@ -97,6 +97,7 @@ class MCPClientManager: self._loop: asyncio.AbstractEventLoop | None = None self._thread: threading.Thread | None = None self._exit_stack: AsyncExitStack | None = None + self._per_server_stacks: dict[str, AsyncExitStack] = {} self._sessions: dict[str, Any] = {} self._tools: list[dict[str, Any]] = [] @@ -104,6 +105,10 @@ class MCPClientManager: self._tool_map: dict[str, tuple[str, str]] = {} self._connected = threading.Event() self._error: str | None = None + # Names managed by the DB (added via reconcile_sync / add_server_sync). + # Config-file servers loaded at startup are NOT in this set and + # will never be removed by reconcile_sync. + self._db_managed: set[str] = set() # Per-server tool storage for surgical refresh self._per_server_tools: dict[str, list[dict[str, Any]]] = {} @@ -189,30 +194,37 @@ class MCPClientManager: async def _connect_one(self, name: str, cfg: dict[str, Any]) -> None: """Connect to a single MCP server and discover its tools.""" - assert self._exit_stack is not None - if "__" in name: log.error("MCP server name '%s' contains '__' (reserved delimiter), skipping", name) return + # Per-server exit stack for clean per-server lifecycle management + stack = AsyncExitStack() + await stack.__aenter__() + transport = cfg.get("type", "stdio") - if transport in ("http", "streamable-http") or "url" in cfg: - read, write, _ = await self._exit_stack.enter_async_context( - streamablehttp_client(url=cfg["url"], headers=cfg.get("headers")) - ) - else: - # Default: stdio transport - command = cfg.get("command", "") - if not command: - log.warning("MCP server '%s' has no command configured", name) - return - env = {**os.environ, **cfg.get("env", {})} - params = StdioServerParameters( - command=command, - args=cfg.get("args", []), - env=env, - ) - read, write = await self._exit_stack.enter_async_context(stdio_client(params)) + try: + if transport in ("http", "streamable-http") or "url" in cfg: + read, write, _ = await stack.enter_async_context( + streamablehttp_client(url=cfg["url"], headers=cfg.get("headers")) + ) + else: + # Default: stdio transport + command = cfg.get("command", "") + if not command: + log.warning("MCP server '%s' has no command configured", name) + await stack.aclose() + return + env = {**os.environ, **cfg.get("env", {})} + params = StdioServerParameters( + command=command, + args=cfg.get("args", []), + env=env, + ) + read, write = await stack.enter_async_context(stdio_client(params)) + except Exception: + await stack.aclose() + raise # Register notification handler — dispatches tool, resource, and # prompt list-change notifications to the appropriate refresh method. @@ -235,10 +247,22 @@ class MCPClientManager: except Exception: log.warning("Refresh after notification failed for '%s'", name, exc_info=True) - session = await self._exit_stack.enter_async_context( - ClientSession(read, write, message_handler=_on_notification) # type: ignore[arg-type] - ) - await session.initialize() + try: + session = await stack.enter_async_context( + ClientSession(read, write, message_handler=_on_notification) # type: ignore[arg-type] + ) + except Exception: + await stack.aclose() + raise + + self._per_server_stacks[name] = stack + try: + await session.initialize() + except Exception: + self._per_server_stacks.pop(name, None) + with contextlib.suppress(Exception): + await stack.aclose() + raise self._sessions[name] = session # Check push notification support for each capability @@ -807,12 +831,27 @@ class MCPClientManager: if self._refresh_task and self._loop: self._loop.call_soon_threadsafe(self._refresh_task.cancel) + # Close all per-server stacks (transports + sessions) + if self._loop and self._per_server_stacks: + + async def _close_all_stacks() -> None: + for stack in self._per_server_stacks.values(): + with contextlib.suppress(Exception): + await stack.aclose() + + future = asyncio.run_coroutine_threadsafe(_close_all_stacks(), self._loop) + try: + future.result(timeout=10) + except Exception: + log.debug("Error closing MCP sessions", exc_info=True) + + # Close legacy shared stack (if any resources were registered on it) if self._loop and self._exit_stack: future = asyncio.run_coroutine_threadsafe(self._exit_stack.aclose(), self._loop) try: future.result(timeout=10) except Exception: - log.debug("Error closing MCP sessions", exc_info=True) + log.debug("Error closing MCP exit stack", exc_info=True) if self._loop: self._loop.call_soon_threadsafe(self._loop.stop) @@ -821,6 +860,8 @@ class MCPClientManager: # Clear all state self._sessions.clear() + self._per_server_stacks.clear() + self._db_managed.clear() self._tools = [] self._tool_map = {} self._per_server_tools.clear() @@ -843,6 +884,209 @@ class MCPClientManager: log.info("MCP client shut down") + # -- hot-reload (add/remove servers) ------------------------------------ + + def add_server_sync(self, name: str, cfg: dict[str, Any], timeout: int = 30) -> dict[str, Any]: + """Connect a new MCP server at runtime (blocks the calling thread). + + Returns status dict with keys: connected, tools, resources, prompts, error. + """ + if "__" in name: + return { + "connected": False, + "tools": 0, + "resources": 0, + "prompts": 0, + "error": f"Server name '{name}' contains '__' (reserved delimiter)", + } + if self._loop is None: + return { + "connected": False, + "tools": 0, + "resources": 0, + "prompts": 0, + "error": "MCP event loop not running", + } + + # Add to config so _refresh_all can reconnect on failure + self._server_configs[name] = cfg + + future = asyncio.run_coroutine_threadsafe(self._connect_one(name, cfg), self._loop) + try: + future.result(timeout=timeout) + except Exception as exc: + # Remove from configs on failure + self._server_configs.pop(name, None) + return {"connected": False, "tools": 0, "resources": 0, "prompts": 0, "error": str(exc)} + + return { + "connected": name in self._sessions, + "tools": len(self._per_server_tools.get(name, [])), + "resources": len(self._per_server_resources.get(name, [])), + "prompts": len(self._per_server_prompts.get(name, [])), + "error": "", + } + + def remove_server_sync(self, name: str, timeout: int = 15) -> bool: + """Disconnect and remove an MCP server at runtime (blocks the calling thread). + + All state mutations run on the MCP event loop thread to avoid races + with notification handlers and refresh tasks. + + Returns True if the server was connected and successfully removed. + """ + was_connected = name in self._sessions + + # Remove from config to prevent reconnection + self._server_configs.pop(name, None) + + if self._loop is not None: + + async def _remove() -> None: + # Close session + transport via per-server stack + self._sessions.pop(name, None) + stack = self._per_server_stacks.pop(name, None) + if stack is not None: + with contextlib.suppress(Exception): + await stack.aclose() + # Clean up per-server state (on the event loop thread) + self._per_server_tools.pop(name, None) + self._per_server_resources.pop(name, None) + self._per_server_prompts.pop(name, None) + self._supports_list_changed.pop(name, None) + self._supports_resources.pop(name, None) + self._supports_resource_list_changed.pop(name, None) + self._supports_prompts.pop(name, None) + self._supports_prompt_list_changed.pop(name, None) + # Rebuild merged state (serialized with notification handlers) + self._rebuild_tools() + self._rebuild_resources() + self._rebuild_prompts() + + future = asyncio.run_coroutine_threadsafe(_remove(), self._loop) + try: + future.result(timeout=timeout) + except Exception: + log.warning("Error removing MCP server '%s'", name, exc_info=True) + else: + # No event loop (tests / pre-start) — mutate directly + self._sessions.pop(name, None) + self._per_server_tools.pop(name, None) + self._per_server_resources.pop(name, None) + self._per_server_prompts.pop(name, None) + self._supports_list_changed.pop(name, None) + self._supports_resources.pop(name, None) + self._supports_resource_list_changed.pop(name, None) + self._supports_prompts.pop(name, None) + self._supports_prompt_list_changed.pop(name, None) + self._rebuild_tools() + self._rebuild_resources() + self._rebuild_prompts() + + # Clean up governance templates from this server + try: + self.sync_prompts_to_storage() + except Exception: + log.warning("Prompt sync after remove failed for '%s'", name, exc_info=True) + + log.info("Removed MCP server '%s'", name) + return was_connected + + def get_server_status(self, name: str) -> dict[str, Any]: + """Return live status for a single server, including config details.""" + connected = name in self._sessions + cfg = self._server_configs.get(name, {}) + transport = cfg.get("type", "stdio") + return { + "connected": connected, + "tools": len(self._per_server_tools.get(name, [])) if connected else 0, + "resources": len(self._per_server_resources.get(name, [])) if connected else 0, + "prompts": len(self._per_server_prompts.get(name, [])) if connected else 0, + "error": "", + "transport": transport, + "command": cfg.get("command", "") if transport == "stdio" else "", + "url": cfg.get("url", "") if transport != "stdio" else "", + } + + def get_all_server_status(self) -> dict[str, dict[str, Any]]: + """Return live status for all configured servers.""" + result: dict[str, dict[str, Any]] = {} + for name in list(self._server_configs): + result[name] = self.get_server_status(name) + return result + + def reconcile_sync(self, storage: Any, timeout: int = 30) -> dict[str, Any]: + """Reconcile DB-managed servers against DB state. + + Reads enabled ``mcp_servers`` rows from *storage*, then: + - Connects servers in DB but not currently running. + - Disconnects DB-managed servers no longer in DB (or disabled). + - Reconnects DB-managed servers whose config has changed. + + Config-file servers (loaded at startup, not in ``_db_managed``) + are never touched — only servers previously added via DB are + eligible for removal. + + Returns ``{"added": [...], "removed": [...], "updated": [...]}``. + """ + try: + rows = storage.list_mcp_servers(enabled_only=True) + except Exception: + log.warning("reconcile_sync: failed to read mcp_servers table", exc_info=True) + return {"added": [], "removed": [], "updated": []} + + desired = _db_servers_to_config(rows) + desired_names = set(desired) + + added: list[str] = [] + removed: list[str] = [] + updated: list[str] = [] + + # Remove DB-managed servers no longer in DB (or disabled). + # Config-file servers (not in _db_managed) are left untouched. + for name in list(self._db_managed - desired_names): + self.remove_server_sync(name, timeout=timeout) + self._db_managed.discard(name) + removed.append(name) + + # Add servers in DB but not running + for name in desired_names - set(self._server_configs): + result = self.add_server_sync(name, desired[name], timeout=timeout) + if result.get("connected"): + added.append(name) + self._db_managed.add(name) + else: + log.warning("reconcile_sync: failed to add '%s': %s", name, result.get("error", "")) + + # Update DB-managed servers whose config has changed (cycle: remove + add). + # Config-file servers with the same name as a DB server are left untouched. + for name in desired_names & set(self._server_configs): + if name not in self._db_managed: + continue # config-file server — DB doesn't own it + if desired[name] != self._server_configs.get(name): + log.info("Config changed for MCP server '%s', reconnecting", name) + self.remove_server_sync(name, timeout=timeout) + result = self.add_server_sync(name, desired[name], timeout=timeout) + if result.get("connected"): + updated.append(name) + self._db_managed.add(name) + else: + self._db_managed.discard(name) + log.warning( + "reconcile_sync: failed to reconnect '%s': %s", + name, + result.get("error", ""), + ) + + if added or removed or updated: + log.info( + "MCP reconcile: +%d added, -%d removed, ~%d updated", + len(added), + len(removed), + len(updated), + ) + return {"added": added, "removed": removed, "updated": updated} + # -- query methods ------------------------------------------------------- def get_tools(self) -> list[dict[str, Any]]: @@ -1024,23 +1268,64 @@ class MCPClientManager: # --------------------------------------------------------------------------- -def load_mcp_config(config_path: str | None = None) -> dict[str, dict[str, Any]]: +def _db_servers_to_config(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Convert mcp_servers DB rows to the config dict format.""" + result: dict[str, dict[str, Any]] = {} + for row in rows: + name = row["name"] + cfg: dict[str, Any] = {"type": row["transport"]} + if row["transport"] == "stdio": + cfg["command"] = row.get("command", "") + try: + cfg["args"] = json.loads(row.get("args", "[]")) + except (json.JSONDecodeError, TypeError): + cfg["args"] = [] + try: + cfg["env"] = json.loads(row.get("env", "{}")) + except (json.JSONDecodeError, TypeError): + cfg["env"] = {} + else: + cfg["url"] = row.get("url", "") + try: + cfg["headers"] = json.loads(row.get("headers", "{}")) + except (json.JSONDecodeError, TypeError): + cfg["headers"] = {} + result[name] = cfg + return result + + +def load_mcp_config( + config_path: str | None = None, + storage: Any = None, +) -> dict[str, dict[str, Any]]: """Load MCP server configurations. Sources (first match wins): - 1. Explicit *config_path* (standard MCP JSON format). - 2. ``[mcp.servers.*]`` sections in ``config.toml``. + 1. DB ``mcp_servers`` table (if *storage* provided and has enabled rows). + 2. Explicit *config_path* (standard MCP JSON format). + 3. ``[mcp.servers.*]`` sections in ``config.toml``. Returns an empty dict if nothing is configured. """ - # 1. Explicit JSON file + # 1. Database + if storage is not None: + try: + rows = storage.list_mcp_servers(enabled_only=True) + if rows: + servers = _db_servers_to_config(rows) + log.info("Loaded MCP config from database (%d server(s))", len(servers)) + return servers + except Exception: + log.debug("DB MCP config lookup failed (table may not exist yet)", exc_info=True) + + # 2. Explicit JSON file if config_path: path = Path(config_path).expanduser() if path.is_file(): try: data = json.loads(path.read_text(encoding="utf-8")) - servers: dict[str, Any] = data.get("mcpServers", {}) + servers = data.get("mcpServers", {}) if isinstance(servers, dict) and servers: log.info("Loaded MCP config from %s (%d server(s))", path, len(servers)) return servers @@ -1049,7 +1334,7 @@ def load_mcp_config(config_path: str | None = None) -> dict[str, dict[str, Any]] else: log.warning("MCP config file not found: %s", path) - # 2. TOML config + # 3. TOML config mcp_section = load_config("mcp") servers_section = mcp_section.get("servers", {}) @@ -1069,15 +1354,28 @@ def create_mcp_client( config_path: str | None = None, *, refresh_interval: float = _DEFAULT_REFRESH_INTERVAL, + storage: Any = None, ) -> MCPClientManager | None: """Create and start an MCP client manager. Returns *None* if no servers are configured. """ - servers = load_mcp_config(config_path) + # Check DB first to know which servers are DB-managed + db_names: set[str] = set() + if storage is not None: + try: + rows = storage.list_mcp_servers(enabled_only=True) + if rows: + db_names = {r["name"] for r in rows} + except Exception: + pass + + servers = load_mcp_config(config_path, storage=storage) if not servers: return None mgr = MCPClientManager(servers, refresh_interval=refresh_interval) + # Mark DB-sourced servers so reconcile_sync won't remove config-file servers + mgr._db_managed = {name for name in servers if name in db_names} mgr.start() return mgr diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index b3b01a3a..87b7e8ad 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -229,7 +229,8 @@ def _build_registry() -> dict[str, SettingDef]: "mcp", restart_required=True, help="Model Context Protocol (MCP) lets the AI connect to external tool servers. " - "This points to a JSON file listing which MCP servers to connect to on startup.", + "This points to a JSON file listing which MCP servers to connect to on startup. " + "Tip: use the MCP Servers tab to manage servers via the database instead.", reference_url="https://modelcontextprotocol.io", ), SettingDef( diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index 29f65c2e..18a0cd29 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -14,6 +14,7 @@ from turnstone.core.storage._schema import ( audit_events, conversations, intent_verdicts, + mcp_servers, metadata, orgs, prompt_templates, @@ -29,6 +30,9 @@ from turnstone.core.storage._schema import ( workstream_templates, workstreams, ) +from turnstone.core.storage._utils import ( + MCP_SERVER_MUTABLE as _MCP_SERVER_MUTABLE, +) from turnstone.core.storage._utils import ( ORG_MUTABLE as _ORG_MUTABLE, ) @@ -2364,6 +2368,98 @@ class PostgreSQLBackend: ).fetchall() return {r.key: r.value for r in rows} + # -- MCP server definitions ------------------------------------------------ + + def create_mcp_server( + self, + server_id: str, + name: str, + transport: str, + command: str = "", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + auto_approve: bool = False, + enabled: bool = True, + created_by: str = "", + ) -> None: + from sqlalchemy.dialects import postgresql + + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + with self._engine.connect() as conn: + conn.execute( + postgresql.insert(mcp_servers) + .values( + server_id=server_id, + name=name, + transport=transport, + command=command, + args=args, + url=url, + headers=headers, + env=env, + auto_approve=1 if auto_approve else 0, + enabled=1 if enabled else 0, + created_by=created_by, + created=now, + updated=now, + ) + .on_conflict_do_nothing() + ) + conn.commit() + + def get_mcp_server(self, server_id: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute( + sa.select(mcp_servers).where(mcp_servers.c.server_id == server_id) + ).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def get_mcp_server_by_name(self, name: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute(sa.select(mcp_servers).where(mcp_servers.c.name == name)).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + + with self._engine.connect() as conn: + q = sa.select(mcp_servers).order_by(mcp_servers.c.name) + if enabled_only: + q = q.where(mcp_servers.c.enabled == 1) + rows = conn.execute(q).fetchall() + return [_row_to_dict(r, "auto_approve", "enabled") for r in rows] + + def update_mcp_server(self, server_id: str, **fields: Any) -> bool: + + fields = {k: v for k, v in fields.items() if k in _MCP_SERVER_MUTABLE} + fields["updated"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + if "auto_approve" in fields: + fields["auto_approve"] = 1 if fields["auto_approve"] else 0 + if "enabled" in fields: + fields["enabled"] = 1 if fields["enabled"] else 0 + with self._engine.connect() as conn: + result = conn.execute( + sa.update(mcp_servers).where(mcp_servers.c.server_id == server_id).values(**fields) + ) + conn.commit() + return result.rowcount > 0 + + def delete_mcp_server(self, server_id: str) -> bool: + + with self._engine.connect() as conn: + result = conn.execute( + sa.delete(mcp_servers).where(mcp_servers.c.server_id == server_id) + ) + conn.commit() + return result.rowcount > 0 + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 6af64722..b1adb9c6 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -784,6 +784,45 @@ class StorageBackend(Protocol): """ ... + # -- MCP server definitions ------------------------------------------------ + + def create_mcp_server( + self, + server_id: str, + name: str, + transport: str, + command: str = "", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + auto_approve: bool = False, + enabled: bool = True, + created_by: str = "", + ) -> None: + """Create an MCP server definition. No-op if server_id already exists.""" + ... + + def get_mcp_server(self, server_id: str) -> dict[str, Any] | None: + """Return MCP server dict or None.""" + ... + + def get_mcp_server_by_name(self, name: str) -> dict[str, Any] | None: + """Return MCP server dict by name or None.""" + ... + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + """Return MCP servers ordered by name.""" + ... + + def update_mcp_server(self, server_id: str, **fields: Any) -> bool: + """Update specified fields on an MCP server. Returns True if found.""" + ... + + def delete_mcp_server(self, server_id: str) -> bool: + """Delete an MCP server definition. Returns True if existed.""" + ... + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_schema.py b/turnstone/core/storage/_schema.py index 9ac62b42..2a8a2406 100644 --- a/turnstone/core/storage/_schema.py +++ b/turnstone/core/storage/_schema.py @@ -445,3 +445,27 @@ system_settings = sa.Table( ) sa.Index("idx_system_settings_node", system_settings.c.node_id) + +# --------------------------------------------------------------------------- +# MCP server definitions — database-backed MCP configuration +# --------------------------------------------------------------------------- + +mcp_servers = sa.Table( + "mcp_servers", + metadata, + sa.Column("server_id", sa.Text, primary_key=True), + sa.Column("name", sa.Text, nullable=False, unique=True), + sa.Column("transport", sa.Text, nullable=False), # "stdio" | "streamable-http" + sa.Column("command", sa.Text, nullable=False, server_default=""), + sa.Column("args", sa.Text, nullable=False, server_default="[]"), # JSON array + sa.Column("url", sa.Text, nullable=False, server_default=""), + sa.Column("headers", sa.Text, nullable=False, server_default="{}"), # JSON object + sa.Column("env", sa.Text, nullable=False, server_default="{}"), # JSON object + sa.Column("auto_approve", sa.Integer, nullable=False, server_default="0"), + sa.Column("enabled", sa.Integer, nullable=False, server_default="1"), + sa.Column("created_by", sa.Text, nullable=False, server_default=""), + sa.Column("created", sa.Text, nullable=False), + sa.Column("updated", sa.Text, nullable=False), +) + +sa.Index("idx_mcp_servers_enabled", mcp_servers.c.enabled) diff --git a/turnstone/core/storage/_sqlite.py b/turnstone/core/storage/_sqlite.py index a8b4a758..364c02b3 100644 --- a/turnstone/core/storage/_sqlite.py +++ b/turnstone/core/storage/_sqlite.py @@ -14,6 +14,7 @@ from turnstone.core.storage._schema import ( audit_events, conversations, intent_verdicts, + mcp_servers, metadata, orgs, prompt_templates, @@ -29,6 +30,9 @@ from turnstone.core.storage._schema import ( workstream_templates, workstreams, ) +from turnstone.core.storage._utils import ( + MCP_SERVER_MUTABLE as _MCP_SERVER_MUTABLE, +) from turnstone.core.storage._utils import ( ORG_MUTABLE as _ORG_MUTABLE, ) @@ -2389,6 +2393,96 @@ class SQLiteBackend: ).fetchall() return {r.key: r.value for r in rows} + # -- MCP server definitions ------------------------------------------------ + + def create_mcp_server( + self, + server_id: str, + name: str, + transport: str, + command: str = "", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + auto_approve: bool = False, + enabled: bool = True, + created_by: str = "", + ) -> None: + + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + with self._engine.connect() as conn: + conn.execute( + sa.insert(mcp_servers).prefix_with("OR IGNORE"), + { + "server_id": server_id, + "name": name, + "transport": transport, + "command": command, + "args": args, + "url": url, + "headers": headers, + "env": env, + "auto_approve": 1 if auto_approve else 0, + "enabled": 1 if enabled else 0, + "created_by": created_by, + "created": now, + "updated": now, + }, + ) + conn.commit() + + def get_mcp_server(self, server_id: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute( + sa.select(mcp_servers).where(mcp_servers.c.server_id == server_id) + ).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def get_mcp_server_by_name(self, name: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute(sa.select(mcp_servers).where(mcp_servers.c.name == name)).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + + with self._engine.connect() as conn: + q = sa.select(mcp_servers).order_by(mcp_servers.c.name) + if enabled_only: + q = q.where(mcp_servers.c.enabled == 1) + rows = conn.execute(q).fetchall() + return [_row_to_dict(r, "auto_approve", "enabled") for r in rows] + + def update_mcp_server(self, server_id: str, **fields: Any) -> bool: + + fields = {k: v for k, v in fields.items() if k in _MCP_SERVER_MUTABLE} + fields["updated"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + if "auto_approve" in fields: + fields["auto_approve"] = 1 if fields["auto_approve"] else 0 + if "enabled" in fields: + fields["enabled"] = 1 if fields["enabled"] else 0 + with self._engine.connect() as conn: + result = conn.execute( + sa.update(mcp_servers).where(mcp_servers.c.server_id == server_id).values(**fields) + ) + conn.commit() + return result.rowcount > 0 + + def delete_mcp_server(self, server_id: str) -> bool: + + with self._engine.connect() as conn: + result = conn.execute( + sa.delete(mcp_servers).where(mcp_servers.c.server_id == server_id) + ) + conn.commit() + return result.rowcount > 0 + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_utils.py b/turnstone/core/storage/_utils.py index d6864e34..15495db3 100644 --- a/turnstone/core/storage/_utils.py +++ b/turnstone/core/storage/_utils.py @@ -48,6 +48,19 @@ WS_TEMPLATE_MUTABLE = frozenset( } ) STRUCTURED_MEMORY_MUTABLE = frozenset({"content", "description", "type"}) +MCP_SERVER_MUTABLE = frozenset( + { + "name", + "transport", + "command", + "args", + "url", + "headers", + "env", + "auto_approve", + "enabled", + } +) VERDICT_MUTABLE = frozenset( { "user_decision", diff --git a/turnstone/core/storage/migrations/versions/016_mcp_servers.py b/turnstone/core/storage/migrations/versions/016_mcp_servers.py new file mode 100644 index 00000000..3f2d2cb4 --- /dev/null +++ b/turnstone/core/storage/migrations/versions/016_mcp_servers.py @@ -0,0 +1,55 @@ +"""Create mcp_servers table and grant admin.mcp permission. + +Revision ID: 016 +Revises: 015 +Create Date: 2026-03-14 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "016" +down_revision = "015" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "mcp_servers", + sa.Column("server_id", sa.Text, primary_key=True), + sa.Column("name", sa.Text, nullable=False, unique=True), + sa.Column("transport", sa.Text, nullable=False), + sa.Column("command", sa.Text, nullable=False, server_default=""), + sa.Column("args", sa.Text, nullable=False, server_default="[]"), + sa.Column("url", sa.Text, nullable=False, server_default=""), + sa.Column("headers", sa.Text, nullable=False, server_default="{}"), + sa.Column("env", sa.Text, nullable=False, server_default="{}"), + sa.Column("auto_approve", sa.Integer, nullable=False, server_default="0"), + sa.Column("enabled", sa.Integer, nullable=False, server_default="1"), + sa.Column("created_by", sa.Text, nullable=False, server_default=""), + sa.Column("created", sa.Text, nullable=False), + sa.Column("updated", sa.Text, nullable=False), + ) + op.create_index("idx_mcp_servers_enabled", "mcp_servers", ["enabled"]) + + # Grant admin.mcp permission to the built-in admin role + conn = op.get_bind() + conn.execute( + sa.text( + "UPDATE roles SET permissions = permissions || ',admin.mcp' " + "WHERE role_id = 'builtin-admin' " + "AND permissions NOT LIKE '%admin.mcp%'" + ) + ) + + +def downgrade() -> None: + conn = op.get_bind() + conn.execute( + sa.text( + "UPDATE roles SET permissions = REPLACE(permissions, ',admin.mcp', '') " + "WHERE role_id = 'builtin-admin'" + ) + ) + op.drop_table("mcp_servers") diff --git a/turnstone/sdk/console.py b/turnstone/sdk/console.py index d472b19a..69dac665 100644 --- a/turnstone/sdk/console.py +++ b/turnstone/sdk/console.py @@ -21,8 +21,10 @@ from turnstone.api.console_schemas import ( ClusterWorkstreamsResponse, ConsoleCreateWsResponse, ConsoleHealthResponse, + ImportMcpConfigResponse, ListAdminMemoriesResponse, ListAuditEventsResponse, + ListMcpServersResponse, ListOrgsResponse, ListPromptTemplatesResponse, ListRolesResponse, @@ -32,6 +34,7 @@ from turnstone.api.console_schemas import ( ListUserRolesResponse, ListWsTemplatesResponse, ListWsTemplateVersionsResponse, + McpServerDetail, NodeDetailResponse, OrgInfo, PromptTemplateInfo, @@ -666,6 +669,98 @@ class AsyncTurnstoneConsole(_BaseClient): "DELETE", f"/v1/api/admin/settings/{key}", params=params, response_model=StatusResponse ) + # -- MCP servers ------------------------------------------------------- + + async def list_mcp_servers(self, reveal: bool = False) -> ListMcpServersResponse: + """List MCP server definitions with live status.""" + params: dict[str, str] = {} + if reveal: + params["reveal"] = "true" + return await self._request( + "GET", + "/v1/api/admin/mcp-servers", + params=params, + response_model=ListMcpServersResponse, + ) + + async def create_mcp_server( + self, + name: str, + transport: str, + *, + command: str = "", + args: list[str] | None = None, + url: str = "", + headers: dict[str, str] | None = None, + env: dict[str, str] | None = None, + auto_approve: bool = False, + enabled: bool = True, + ) -> McpServerDetail: + """Create an MCP server definition.""" + body: dict[str, Any] = {"name": name, "transport": transport} + if command: + body["command"] = command + if args: + body["args"] = args + if url: + body["url"] = url + if headers: + body["headers"] = headers + if env: + body["env"] = env + if auto_approve: + body["auto_approve"] = True + if not enabled: + body["enabled"] = False + return await self._request( + "POST", + "/v1/api/admin/mcp-servers", + json_body=body, + response_model=McpServerDetail, + ) + + async def get_mcp_server(self, server_id: str) -> McpServerDetail: + """Get a single MCP server with status.""" + return await self._request( + "GET", + f"/v1/api/admin/mcp-servers/{server_id}", + response_model=McpServerDetail, + ) + + async def update_mcp_server(self, server_id: str, **fields: Any) -> McpServerDetail: + """Update an MCP server definition.""" + return await self._request( + "PUT", + f"/v1/api/admin/mcp-servers/{server_id}", + json_body=fields, + response_model=McpServerDetail, + ) + + async def delete_mcp_server(self, server_id: str) -> StatusResponse: + """Delete an MCP server definition.""" + return await self._request( + "DELETE", + f"/v1/api/admin/mcp-servers/{server_id}", + response_model=StatusResponse, + ) + + async def reload_mcp_servers(self) -> StatusResponse: + """Tell all nodes to re-read MCP server config from DB.""" + return await self._request( + "POST", + "/v1/api/admin/mcp-servers/reload", + response_model=StatusResponse, + ) + + async def import_mcp_config(self, config: dict[str, Any]) -> ImportMcpConfigResponse: + """Import MCP servers from a config dict with mcpServers key.""" + return await self._request( + "POST", + "/v1/api/admin/mcp-servers/import", + json_body={"config": config}, + response_model=ImportMcpConfigResponse, + ) + class TurnstoneConsole: """Synchronous client for the turnstone console API. @@ -1041,6 +1136,53 @@ class TurnstoneConsole: def delete_setting(self, key: str, *, node_id: str = "") -> StatusResponse: return self._runner.run(self._async.delete_setting(key, node_id=node_id)) + # -- MCP servers ------------------------------------------------------- + + def list_mcp_servers(self, reveal: bool = False) -> ListMcpServersResponse: + return self._runner.run(self._async.list_mcp_servers(reveal=reveal)) + + def create_mcp_server( + self, + name: str, + transport: str, + *, + command: str = "", + args: list[str] | None = None, + url: str = "", + headers: dict[str, str] | None = None, + env: dict[str, str] | None = None, + auto_approve: bool = False, + enabled: bool = True, + ) -> McpServerDetail: + return self._runner.run( + self._async.create_mcp_server( + name, + transport, + command=command, + args=args, + url=url, + headers=headers, + env=env, + auto_approve=auto_approve, + enabled=enabled, + ) + ) + + def get_mcp_server(self, server_id: str) -> McpServerDetail: + return self._runner.run(self._async.get_mcp_server(server_id)) + + def update_mcp_server(self, server_id: str, **fields: Any) -> McpServerDetail: + return self._runner.run(self._async.update_mcp_server(server_id, **fields)) + + def delete_mcp_server(self, server_id: str) -> StatusResponse: + return self._runner.run(self._async.delete_mcp_server(server_id)) + + def reload_mcp_servers(self) -> StatusResponse: + return self._runner.run(self._async.reload_mcp_servers()) + + def import_mcp_config(self, config: dict[str, Any]) -> ImportMcpConfigResponse: + return self._runner.run(self._async.import_mcp_config(config)) + # -- lifecycle ----------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/server.py b/turnstone/server.py index d2a9e91f..c218003c 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1556,6 +1556,36 @@ def config_reload(request: Request) -> JSONResponse: return JSONResponse({"status": "ok"}) +# -- internal MCP management ----------------------------------------------- + + +def internal_mcp_reload(request: Request) -> JSONResponse: + """POST /v1/api/_internal/mcp-reload — re-read mcp_servers table and reconcile.""" + from turnstone.core.storage._registry import get_storage + + storage = get_storage() + mcp_mgr = getattr(request.app.state, "mcp_client", None) + if mcp_mgr is None: + # Create a new manager if none exists + from turnstone.core.mcp_client import MCPClientManager + + mcp_mgr = MCPClientManager({}) + mcp_mgr.start() + request.app.state.mcp_client = mcp_mgr + + result = mcp_mgr.reconcile_sync(storage) + return JSONResponse({"status": "ok", **result}) + + +def internal_mcp_status(request: Request) -> JSONResponse: + """GET /v1/api/_internal/mcp-status — return MCP server status.""" + mcp_mgr = getattr(request.app.state, "mcp_client", None) + if mcp_mgr is None: + return JSONResponse({"servers": {}}) + + return JSONResponse({"servers": mcp_mgr.get_all_server_status()}) + + # --------------------------------------------------------------------------- # Global SSE fan-out # --------------------------------------------------------------------------- @@ -1723,6 +1753,8 @@ def create_app( Route("/api/auth/status", auth_status), Route("/api/auth/setup", auth_setup, methods=["POST"]), Route("/api/_internal/config-reload", config_reload, methods=["POST"]), + Route("/api/_internal/mcp-reload", internal_mcp_reload, methods=["POST"]), + Route("/api/_internal/mcp-status", internal_mcp_status), ], ), Route("/health", health), @@ -1936,11 +1968,13 @@ def main() -> None: # Initialize MCP client (connects to configured MCP servers, if any) from turnstone.core.mcp_client import create_mcp_client + from turnstone.core.storage._registry import get_storage as _get_storage mcp_config_cli = args.mcp_config # CLI-only (no config.toml for this) mcp_client = create_mcp_client( mcp_config_cli or config_store.get("mcp.config_path") or None, refresh_interval=config_store.get("mcp.refresh_interval"), + storage=_get_storage(), ) # Backend health monitor with circuit breaker