From 19abc0cc653c6f0673f87f23708bb8aeb60757c0 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Sat, 14 Mar 2026 17:02:50 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20admin=20MCP=20Servers=20tab=20=E2=80=94?= =?UTF-8?q?=20database-backed=20MCP=20server=20management=20w=E2=80=A6=20(?= =?UTF-8?q?#62)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: admin MCP Servers tab — database-backed MCP server management with live status Add MCP Servers admin tab (14th tab, System group) for managing MCP server definitions via the database instead of static JSON config files. Storage: `mcp_servers` table (migration 016), 6 CRUD methods on both SQLite and PostgreSQL backends, `MCP_SERVER_MUTABLE` field allowlist. Config priority chain: DB rows (if any enabled) → CLI `--mcp-config` → `mcp.config_path` setting → none. Nodes auto-load from DB on startup via `load_mcp_config(storage=)`. Hot-reload: `reconcile_sync(storage)` diffs running servers against DB — adds missing, removes stale, reconnects changed. `_db_managed` set tracks DB-sourced servers so config-file servers (MCP_CONFIG env) are never removed by reconcile. Per-server `AsyncExitStack` for clean teardown. Reload pattern: console writes to DB then signals nodes via `POST /_internal/mcp-reload` (update by reference, no config payload). Console admin API: 7 endpoints under `/v1/api/admin/mcp-servers` (CRUD + reload + import), `admin.mcp` permission, secret masking (env/headers replaced with *** unless ?reveal=true), audit log sanitization. Unified view: tab merges DB-managed servers with config-sourced servers detected on nodes. Config servers shown as read-only rows with "config" badge — no edit/delete. Admin UI: 7-column grid with magenta status dots, transport badges, single-column create/edit modal, paste-based JSON import (mcpServers format), detail modal with per-node status. Mobile 3-column collapse, reduced-motion support, backdrop-click dismiss, focus trapping. SDKs: 7 methods on Python (async+sync) and TypeScript SDKs. Also fixes: Settings tab permission gate (admin.users → admin.settings), _ALL_PERMISSIONS list in governance.js (5 missing permissions added), _internal/mcp-reload added to APPROVE_PATHS. Docs: architecture.md (14 tabs), api-reference.md (7 endpoints), 20-mcp-architecture.puml updated with admin-driven lifecycle. 66 new tests (2232 total). * fix: address Copilot review feedback on MCP admin PR - Docs: fix "merges both sources" → "first-match-wins priority" (architecture.md) - Validation: require command for stdio, url for streamable-http transport - Validation: check args/headers/env types in import handler before storing - Schema: add transport/command/url to McpServerStatus, source to McpServerDetail - Thread safety: move all remove_server_sync mutations onto MCP event loop thread - Regenerate OpenAPI JSON snapshots for TypeScript SDK --- docs/api-reference.md | 18 + docs/architecture.md | 22 +- docs/diagrams/20-mcp-architecture.puml | 19 + docs/diagrams/png/20-mcp-architecture.png | 4 +- sdk/typescript/openapi-console.json | 685 +++++++++++++++++- sdk/typescript/openapi-server.json | 2 +- sdk/typescript/src/console.ts | 50 ++ sdk/typescript/src/index.ts | 7 + sdk/typescript/src/types.ts | 61 ++ tests/test_mcp_admin_api.py | 539 ++++++++++++++ tests/test_mcp_hot_reload.py | 353 +++++++++ tests/test_mcp_server_storage.py | 152 ++++ turnstone/api/console_schemas.py | 80 ++ turnstone/api/console_spec.py | 75 ++ turnstone/cli.py | 2 + turnstone/console/server.py | 560 ++++++++++++++ turnstone/console/static/admin.js | 573 ++++++++++++++- turnstone/console/static/governance.js | 5 + turnstone/console/static/index.html | 83 +++ turnstone/console/static/style.css | 43 +- turnstone/core/auth.py | 4 +- turnstone/core/mcp_client.py | 360 ++++++++- turnstone/core/settings_registry.py | 3 +- turnstone/core/storage/_postgresql.py | 96 +++ turnstone/core/storage/_protocol.py | 39 + turnstone/core/storage/_schema.py | 24 + turnstone/core/storage/_sqlite.py | 94 +++ turnstone/core/storage/_utils.py | 13 + .../migrations/versions/016_mcp_servers.py | 55 ++ turnstone/sdk/console.py | 142 ++++ turnstone/server.py | 34 + 31 files changed, 4153 insertions(+), 44 deletions(-) create mode 100644 tests/test_mcp_admin_api.py create mode 100644 tests/test_mcp_hot_reload.py create mode 100644 tests/test_mcp_server_storage.py create mode 100644 turnstone/core/storage/migrations/versions/016_mcp_servers.py diff --git a/docs/api-reference.md b/docs/api-reference.md index 16567fa8..3674dc4d 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1391,6 +1391,24 @@ the `admin.settings` permission. --- +### MCP Servers + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/v1/api/admin/mcp-servers` | List all MCP server definitions with live node status. Query: `?reveal=true` to show env/header secrets. | +| POST | `/v1/api/admin/mcp-servers` | Create an MCP server definition. Body: `{name, transport, command?, args?, url?, headers?, env?, auto_approve?, enabled?}` | +| GET | `/v1/api/admin/mcp-servers/{server_id}` | Get a single MCP server with per-node connection status. | +| PUT | `/v1/api/admin/mcp-servers/{server_id}` | Update an MCP server definition. Partial updates supported. | +| DELETE | `/v1/api/admin/mcp-servers/{server_id}` | Delete an MCP server definition. | +| POST | `/v1/api/admin/mcp-servers/reload` | Tell all cluster nodes to re-read the `mcp_servers` DB table and reconcile (add new, remove stale, reconnect changed). | +| POST | `/v1/api/admin/mcp-servers/import` | Import servers from a pasted JSON config. Body: `{config: {mcpServers: {...}}}`. Skips existing names. | + +Permission: `admin.mcp` + +Secrets (`env`, `headers` fields) are masked with `***` by default. Use `?reveal=true` on GET endpoints to see actual values. + +--- + ### `OPTIONS` (any path) Handles CORS preflight requests. diff --git a/docs/architecture.md b/docs/architecture.md index 09c30eea..91506131 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -506,8 +506,18 @@ independently, then returns the final content as the tool result. and exposes their tools alongside built-in tools. The MCP SDK is fully async; turnstone bridges this with a background asyncio event loop in a daemon thread. +**Configuration sources:** MCP servers can be defined in config files (TOML/JSON) +or in the database via the admin UI. Database-backed definitions are managed +through the console admin panel's MCP Servers tab and stored in the +`mcp_servers` table. On startup, `load_mcp_config(storage=)` uses +first-match-wins priority: DB rows (if any enabled) take precedence over +config files. The console can trigger a cluster-wide reload (`POST +/_internal/mcp-reload`) that causes each node to call `reconcile_sync()`, +which diffs the running MCP connections against the current DB state and +adds, removes, or reconnects servers as needed. + **Lifecycle:** -1. `create_mcp_client()` reads server configs from TOML or JSON +1. `create_mcp_client()` reads server configs from TOML/JSON and database 2. `MCPClientManager.start()` launches the background event loop thread 3. `_connect_all()` connects to each server (stdio subprocess or HTTP), runs `initialize()` + `list_tools()`, converts schemas to OpenAI format, detects @@ -1018,8 +1028,8 @@ Three hierarchical scopes control endpoint access: - **Console** is the auth management hub — it hosts the admin endpoints for creating users, issuing API tokens, and managing channel mappings. User records and token hashes live in the shared storage backend. The console - dashboard includes an **admin panel** (13 tabs) for managing - credentials, governance, and runtime settings through the browser. + dashboard includes an **admin panel** (14 tabs) for managing + credentials, governance, MCP servers, and runtime settings through the browser. - **Server** is a JWT validator only — it validates tokens on each request but never creates users or tokens. Both processes share the same `jwt_secret` (via `TURNSTONE_JWT_SECRET` env var or `[auth].jwt_secret` config). @@ -1393,8 +1403,10 @@ enforcement tracks consumption in `session.send()` with 80% warning and 100% approval gate via the `__budget_override__` synthetic tool name. The console admin panel adds 6 governance tabs (Roles, Policies, Templates, -WS Templates, Usage, Audit), a Memories tab, and a Settings tab (form-based -editor for all ConfigStore settings) for a total of 13 tabs, all permission-gated. +WS Templates, Usage, Audit), a Memories tab, a Settings tab (form-based +editor for all ConfigStore settings), and an MCP Servers tab (database-backed +server definitions with live connection status and cluster-wide reload) for a +total of 14 tabs, all permission-gated. Both Python and TypeScript SDKs expose governance methods on the console client. diff --git a/docs/diagrams/20-mcp-architecture.puml b/docs/diagrams/20-mcp-architecture.puml index f61064f4..0a5c5da6 100644 --- a/docs/diagrams/20-mcp-architecture.puml +++ b/docs/diagrams/20-mcp-architecture.puml @@ -16,8 +16,27 @@ participant "ChatSession\n(session.py)" as Session <> participant "StorageBackend\n(governance)" as Storage <> participant "Server / Console\n(health + UI)" as UI <> +participant "Console Admin UI\n(admin panel)" as Admin <> +participant "Database\n(mcp_servers table)" as DB <> + +== Admin-Driven Configuration == + +Admin -> DB : CRUD MCP server definitions\n(POST/PUT/DELETE /v1/api/admin/mcp-servers) + +Admin -> UI : POST /v1/api/admin/mcp-servers/reload +UI -> MCPMgr : POST /_internal/mcp-reload\n(forwarded to each node) +MCPMgr -> MCPMgr : reconcile_sync() +note right + Diffs running servers against DB: + - New entries → connect + - Removed entries → disconnect + - Changed entries → reconnect +end note + == Startup: Connection & Discovery == +MCPMgr -> DB : load_mcp_config(storage=)\n(merge config file + DB) + MCPMgr -> MCPSrv : initialize (stdio or HTTP) MCPSrv --> MCPMgr : capabilities\n(tools, resources, prompts) diff --git a/docs/diagrams/png/20-mcp-architecture.png b/docs/diagrams/png/20-mcp-architecture.png index b5dc770e..45b040a1 100644 --- a/docs/diagrams/png/20-mcp-architecture.png +++ b/docs/diagrams/png/20-mcp-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8e6dc5142c7908314ce01229b3c4f13bf9450adcbb62a178838bd4cf81d9f4da -size 250417 +oid sha256:e4593873599342b2830fedd5d783e9a28eab0bb0d6589798ef6ef2649eeee80f +size 324518 diff --git a/sdk/typescript/openapi-console.json b/sdk/typescript/openapi-console.json index b700cedf..f4eecde2 100644 --- a/sdk/typescript/openapi-console.json +++ b/sdk/typescript/openapi-console.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Console API", - "version": "0.6.0", + "version": "0.6.1", "description": "Cluster-wide visibility and control across all turnstone nodes." }, "paths": { @@ -2661,6 +2661,298 @@ } } }, + "/v1/api/admin/mcp-servers": { + "get": { + "summary": "List MCP server definitions with live status", + "operationId": "v1_api_admin_mcp-servers_get", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "reveal", + "in": "query", + "required": false, + "schema": { + "type": "boolean" + }, + "description": "Show secret env/header values" + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListMcpServersResponse" + } + } + } + } + } + }, + "post": { + "summary": "Create an MCP server definition", + "operationId": "v1_api_admin_mcp-servers_post", + "tags": [ + "Admin" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateMcpServerRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-servers/{server_id}": { + "get": { + "summary": "Get a single MCP server with status", + "operationId": "v1_api_admin_mcp-servers_{server_id}_get", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "server_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, + "put": { + "summary": "Update an MCP server definition", + "operationId": "v1_api_admin_mcp-servers_{server_id}_put", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "server_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMcpServerRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, + "delete": { + "summary": "Delete an MCP server definition", + "operationId": "v1_api_admin_mcp-servers_{server_id}_delete", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "server_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-servers/reload": { + "post": { + "summary": "Tell all nodes to re-read MCP server config from DB and reconcile", + "operationId": "v1_api_admin_mcp-servers_reload_post", + "tags": [ + "Admin" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpReloadResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-servers/import": { + "post": { + "summary": "Import MCP servers from a JSON config file", + "operationId": "v1_api_admin_mcp-servers_import_post", + "tags": [ + "Admin" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportMcpConfigRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportMcpConfigResponse" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/health": { "get": { "summary": "Console health check", @@ -5264,6 +5556,397 @@ ], "title": "UpdateSettingRequest", "type": "object" + }, + "McpServerDetail": { + "properties": { + "server_id": { + "title": "Server Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "transport": { + "title": "Transport", + "type": "string" + }, + "command": { + "default": "", + "title": "Command", + "type": "string" + }, + "args": { + "default": "[]", + "title": "Args", + "type": "string" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + }, + "headers": { + "default": "{}", + "title": "Headers", + "type": "string" + }, + "env": { + "default": "{}", + "title": "Env", + "type": "string" + }, + "auto_approve": { + "default": false, + "title": "Auto Approve", + "type": "boolean" + }, + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "created_by": { + "default": "", + "title": "Created By", + "type": "string" + }, + "created": { + "title": "Created", + "type": "string" + }, + "updated": { + "title": "Updated", + "type": "string" + }, + "status": { + "additionalProperties": { + "$ref": "#/components/schemas/McpServerStatus" + }, + "title": "Status", + "type": "object" + }, + "source": { + "default": "", + "title": "Source", + "type": "string" + } + }, + "required": [ + "server_id", + "name", + "transport", + "created", + "updated" + ], + "title": "McpServerDetail", + "type": "object" + }, + "McpServerStatus": { + "properties": { + "connected": { + "default": false, + "title": "Connected", + "type": "boolean" + }, + "tools": { + "default": 0, + "title": "Tools", + "type": "integer" + }, + "resources": { + "default": 0, + "title": "Resources", + "type": "integer" + }, + "prompts": { + "default": 0, + "title": "Prompts", + "type": "integer" + }, + "error": { + "default": "", + "title": "Error", + "type": "string" + }, + "transport": { + "default": "", + "title": "Transport", + "type": "string" + }, + "command": { + "default": "", + "title": "Command", + "type": "string" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + } + }, + "title": "McpServerStatus", + "type": "object" + }, + "CreateMcpServerRequest": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "transport": { + "title": "Transport", + "type": "string" + }, + "command": { + "default": "", + "title": "Command", + "type": "string" + }, + "args": { + "default": [], + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + }, + "headers": { + "additionalProperties": { + "type": "string" + }, + "title": "Headers", + "type": "object" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "auto_approve": { + "default": false, + "title": "Auto Approve", + "type": "boolean" + }, + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + } + }, + "required": [ + "name", + "transport" + ], + "title": "CreateMcpServerRequest", + "type": "object" + }, + "UpdateMcpServerRequest": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "transport": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Transport" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "args": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Args" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Url" + }, + "headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Headers" + }, + "env": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Env" + }, + "auto_approve": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auto Approve" + }, + "enabled": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enabled" + } + }, + "title": "UpdateMcpServerRequest", + "type": "object" + }, + "ListMcpServersResponse": { + "properties": { + "servers": { + "items": { + "$ref": "#/components/schemas/McpServerDetail" + }, + "title": "Servers", + "type": "array" + } + }, + "required": [ + "servers" + ], + "title": "ListMcpServersResponse", + "type": "object" + }, + "ImportMcpConfigRequest": { + "properties": { + "config": { + "additionalProperties": true, + "description": "JSON config object with mcpServers key", + "title": "Config", + "type": "object" + } + }, + "required": [ + "config" + ], + "title": "ImportMcpConfigRequest", + "type": "object" + }, + "ImportMcpConfigResponse": { + "properties": { + "imported": { + "default": [], + "items": { + "type": "string" + }, + "title": "Imported", + "type": "array" + }, + "skipped": { + "default": [], + "items": { + "type": "string" + }, + "title": "Skipped", + "type": "array" + }, + "errors": { + "default": [], + "items": { + "type": "string" + }, + "title": "Errors", + "type": "array" + } + }, + "title": "ImportMcpConfigResponse", + "type": "object" + }, + "McpReloadResponse": { + "properties": { + "status": { + "default": "ok", + "title": "Status", + "type": "string" + }, + "results": { + "additionalProperties": true, + "title": "Results", + "type": "object" + } + }, + "title": "McpReloadResponse", + "type": "object" } } } diff --git a/sdk/typescript/openapi-server.json b/sdk/typescript/openapi-server.json index 34bb3f20..b1edbd5d 100644 --- a/sdk/typescript/openapi-server.json +++ b/sdk/typescript/openapi-server.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Server API", - "version": "0.6.0", + "version": "0.6.1", "description": "Single-node workstream management, chat interaction, and real-time streaming." }, "paths": { diff --git a/sdk/typescript/src/console.ts b/sdk/typescript/src/console.ts index ed93d45f..9adbe432 100644 --- a/sdk/typescript/src/console.ts +++ b/sdk/typescript/src/console.ts @@ -16,16 +16,20 @@ import type { ConsoleCreateWsRequest, ConsoleCreateWsResponse, ConsoleHealthResponse, + CreateMcpServerRequest, CreatePolicyOptions, CreateRoleOptions, CreateScheduleRequest, CreateTemplateOptions, CreateWsTemplateOptions, + ImportMcpConfigResponse, ListAdminMemoriesResponse, + ListMcpServersResponse, ListScheduleRunsResponse, ListSchedulesResponse, ListSettingSchemaResponse, ListSettingsResponse, + McpServerDetail, NodeDetailResponse, NodesOptions, OrgInfo, @@ -35,6 +39,7 @@ import type { SettingInfo, StatusResponse, ToolPolicyInfo, + UpdateMcpServerRequest, UpdateOrgOptions, UpdatePolicyOptions, UpdateRoleOptions, @@ -410,4 +415,49 @@ export class TurnstoneConsole extends BaseClient { params, }); } + + // -- MCP servers ---------------------------------------------------------- + + async listMcpServers(opts?: { + reveal?: boolean; + }): Promise { + const params: Record = {}; + if (opts?.reveal) params.reveal = "true"; + return this.request("GET", "/v1/api/admin/mcp-servers", { params }); + } + + async createMcpServer( + body: CreateMcpServerRequest, + ): Promise { + return this.request("POST", "/v1/api/admin/mcp-servers", { json: body }); + } + + async getMcpServer(serverId: string): Promise { + return this.request("GET", `/v1/api/admin/mcp-servers/${serverId}`); + } + + async updateMcpServer( + serverId: string, + body: UpdateMcpServerRequest, + ): Promise { + return this.request("PUT", `/v1/api/admin/mcp-servers/${serverId}`, { + json: body, + }); + } + + async deleteMcpServer(serverId: string): Promise { + return this.request("DELETE", `/v1/api/admin/mcp-servers/${serverId}`); + } + + async reloadMcpServers(): Promise { + return this.request("POST", "/v1/api/admin/mcp-servers/reload"); + } + + async importMcpConfig( + config: Record, + ): Promise { + return this.request("POST", "/v1/api/admin/mcp-servers/import", { + json: { config }, + }); + } } diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 3af85d89..9d4265b5 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -160,6 +160,13 @@ export type { SettingSchemaInfo, ListSettingSchemaResponse, UpdateSettingOptions, + // MCP server types + McpServerStatus, + McpServerDetail, + ListMcpServersResponse, + CreateMcpServerRequest, + UpdateMcpServerRequest, + ImportMcpConfigResponse, } from "./types.js"; // SSE parser (for advanced usage) diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts index 53dbcad7..7577e153 100644 --- a/sdk/typescript/src/types.ts +++ b/sdk/typescript/src/types.ts @@ -728,6 +728,67 @@ export interface AdminSearchMemoriesOptions { limit?: number; } +// -- Console API: MCP Servers ----------------------------------------------- + +export interface McpServerStatus { + connected: boolean; + tools: number; + resources: number; + prompts: number; + error: string; +} + +export interface McpServerDetail { + server_id: string; + name: string; + transport: string; + command: string; + args: string; + url: string; + headers: string; + env: string; + auto_approve: boolean; + enabled: boolean; + created_by: string; + created: string; + updated: string; + status: Record; +} + +export interface ListMcpServersResponse { + servers: McpServerDetail[]; +} + +export interface CreateMcpServerRequest { + name: string; + transport: string; + command?: string; + args?: string[]; + url?: string; + headers?: Record; + env?: Record; + auto_approve?: boolean; + enabled?: boolean; +} + +export interface UpdateMcpServerRequest { + name?: string; + transport?: string; + command?: string; + args?: string[]; + url?: string; + headers?: Record; + env?: Record; + auto_approve?: boolean; + enabled?: boolean; +} + +export interface ImportMcpConfigResponse { + imported: string[]; + skipped: string[]; + errors: string[]; +} + // -- Console API: System Settings ------------------------------------------- export interface SettingInfo { diff --git a/tests/test_mcp_admin_api.py b/tests/test_mcp_admin_api.py new file mode 100644 index 00000000..eafc734f --- /dev/null +++ b/tests/test_mcp_admin_api.py @@ -0,0 +1,539 @@ +"""Tests for MCP server admin API endpoints.""" + +from __future__ import annotations + +import json +import uuid +from typing import TYPE_CHECKING, Any +from unittest.mock import AsyncMock, patch + +import pytest +from starlette.applications import Starlette +from starlette.middleware import Middleware +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.routing import Mount, Route +from starlette.testclient import TestClient + +if TYPE_CHECKING: + from starlette.requests import Request + from starlette.responses import Response + +from turnstone.console.server import ( + admin_create_mcp_server, + admin_delete_mcp_server, + admin_get_mcp_server, + admin_import_mcp_config, + admin_list_mcp_servers, + admin_update_mcp_server, +) +from turnstone.core.auth import AuthResult +from turnstone.core.storage._sqlite import SQLiteBackend + +# --------------------------------------------------------------------------- +# Auth middleware variants +# --------------------------------------------------------------------------- + + +class _InjectAuthMiddleware(BaseHTTPMiddleware): + """Inject an admin auth result with admin.mcp permission.""" + + async def dispatch(self, request: Request, call_next: Any) -> Response: + request.state.auth_result = AuthResult( + user_id="test-user", + scopes=frozenset({"approve"}), + token_source="config", + permissions=frozenset( + { + "read", + "write", + "approve", + "admin.mcp", + } + ), + ) + resp: Response = await call_next(request) + return resp + + +class _InjectAuthNoMcpMiddleware(BaseHTTPMiddleware): + """Inject an auth result WITHOUT admin.mcp permission.""" + + async def dispatch(self, request: Request, call_next: Any) -> Response: + request.state.auth_result = AuthResult( + user_id="test-user", + scopes=frozenset({"approve"}), + token_source="jwt", + permissions=frozenset( + { + "read", + "write", + "approve", + } + ), + ) + resp: Response = await call_next(request) + return resp + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +_ROUTES = [ + Mount( + "/v1", + routes=[ + Route("/api/admin/mcp-servers", admin_list_mcp_servers), + Route( + "/api/admin/mcp-servers", + admin_create_mcp_server, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/import", + admin_import_mcp_config, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_get_mcp_server, + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_update_mcp_server, + methods=["PUT"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_delete_mcp_server, + methods=["DELETE"], + ), + ], + ), +] + + +@pytest.fixture +def storage(tmp_path): + return SQLiteBackend(str(tmp_path / "test.db")) + + +@pytest.fixture +def client(storage): + """TestClient wired to console admin MCP endpoints with full permissions.""" + app = Starlette( + routes=_ROUTES, + middleware=[Middleware(_InjectAuthMiddleware)], + ) + app.state.auth_storage = storage + return TestClient(app) + + +@pytest.fixture +def client_no_perm(storage): + """TestClient without admin.mcp permission.""" + app = Starlette( + routes=_ROUTES, + middleware=[Middleware(_InjectAuthNoMcpMiddleware)], + ) + app.state.auth_storage = storage + return TestClient(app) + + +def _create_server( + client: TestClient, + *, + name: str = "test-server", + transport: str = "stdio", + command: str = "npx", + args: list[str] | None = None, + env: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + url: str = "", +) -> dict[str, Any]: + """Helper to create a server via the API and return the response dict.""" + body: dict[str, Any] = {"name": name, "transport": transport} + if transport == "stdio": + body["command"] = command + body["args"] = args or ["-y", "@modelcontextprotocol/server-test"] + else: + body["url"] = url or "http://localhost:8080/mcp" + if env is not None: + body["env"] = env + if headers is not None: + body["headers"] = headers + r = client.post("/v1/api/admin/mcp-servers", json=body) + assert r.status_code == 200 + data: dict[str, Any] = r.json() + return data + + +# --------------------------------------------------------------------------- +# Mock _collect_mcp_status to avoid real HTTP calls +# --------------------------------------------------------------------------- + +_PATCH_MCP_STATUS = patch( + "turnstone.console.server._collect_mcp_status", + new_callable=AsyncMock, + return_value={}, +) + + +# --------------------------------------------------------------------------- +# List +# --------------------------------------------------------------------------- + + +class TestListMcpServers: + def test_list_empty(self, client): + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers") + assert r.status_code == 200 + assert r.json()["servers"] == [] + + def test_list_returns_created_servers(self, client): + _create_server(client, name="server-a") + _create_server(client, name="server-b") + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers") + assert r.status_code == 200 + names = [s["name"] for s in r.json()["servers"]] + assert "server-a" in names + assert "server-b" in names + + +# --------------------------------------------------------------------------- +# Create +# --------------------------------------------------------------------------- + + +class TestCreateMcpServer: + def test_create_stdio_server(self, client): + data = _create_server(client, name="my-mcp", transport="stdio", command="node") + assert data["name"] == "my-mcp" + assert data["transport"] == "stdio" + assert data["command"] == "node" + assert data["server_id"] + assert data["enabled"] is True + + def test_create_http_server(self, client): + data = _create_server( + client, + name="remote-mcp", + transport="streamable-http", + url="http://mcp.example.com/sse", + ) + assert data["name"] == "remote-mcp" + assert data["transport"] == "streamable-http" + assert data["url"] == "http://mcp.example.com/sse" + + def test_create_invalid_name_spaces(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "bad name!", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 400 + assert "name" in r.json()["error"].lower() + + def test_create_invalid_name_double_underscore(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "bad__name", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 400 + assert "__" in r.json()["error"] + + def test_create_invalid_transport(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "ok-name", "transport": "grpc"}, + ) + assert r.status_code == 400 + assert "transport" in r.json()["error"].lower() + + def test_create_duplicate_name(self, client): + _create_server(client, name="dup-test") + r = client.post( + "/v1/api/admin/mcp-servers", + json={"name": "dup-test", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 409 + assert "already exists" in r.json()["error"] + + def test_create_missing_name(self, client): + r = client.post( + "/v1/api/admin/mcp-servers", + json={"transport": "stdio", "command": "x"}, + ) + assert r.status_code == 400 + assert "name" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Get single +# --------------------------------------------------------------------------- + + +class TestGetMcpServer: + def test_get_existing(self, client): + created = _create_server(client, name="get-test") + sid = created["server_id"] + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{sid}") + assert r.status_code == 200 + assert r.json()["name"] == "get-test" + + def test_get_not_found(self, client): + fake_id = uuid.uuid4().hex + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{fake_id}") + assert r.status_code == 404 + assert "not found" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Update +# --------------------------------------------------------------------------- + + +class TestUpdateMcpServer: + def test_update_name(self, client): + created = _create_server(client, name="old-name") + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"name": "new-name"}, + ) + assert r.status_code == 200 + assert r.json()["name"] == "new-name" + + def test_update_transport(self, client): + created = _create_server( + client, + name="update-transport", + transport="streamable-http", + url="http://localhost/mcp", + ) + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"transport": "stdio", "command": "node"}, + ) + assert r.status_code == 200 + assert r.json()["transport"] == "stdio" + + def test_update_enabled(self, client): + created = _create_server(client, name="toggle-enabled") + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"enabled": False}, + ) + assert r.status_code == 200 + assert r.json()["enabled"] is False + + def test_update_not_found(self, client): + fake_id = uuid.uuid4().hex + r = client.put( + f"/v1/api/admin/mcp-servers/{fake_id}", + json={"name": "x"}, + ) + assert r.status_code == 404 + + def test_update_invalid_transport(self, client): + created = _create_server(client, name="bad-transport-update") + sid = created["server_id"] + r = client.put( + f"/v1/api/admin/mcp-servers/{sid}", + json={"transport": "websocket"}, + ) + assert r.status_code == 400 + assert "transport" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Delete +# --------------------------------------------------------------------------- + + +class TestDeleteMcpServer: + def test_delete_existing(self, client): + created = _create_server(client, name="del-test") + sid = created["server_id"] + r = client.delete(f"/v1/api/admin/mcp-servers/{sid}") + assert r.status_code == 200 + assert r.json()["status"] == "ok" + + # Confirm it's gone + with _PATCH_MCP_STATUS: + r2 = client.get(f"/v1/api/admin/mcp-servers/{sid}") + assert r2.status_code == 404 + + def test_delete_not_found(self, client): + fake_id = uuid.uuid4().hex + r = client.delete(f"/v1/api/admin/mcp-servers/{fake_id}") + assert r.status_code == 404 + assert "not found" in r.json()["error"].lower() + + +# --------------------------------------------------------------------------- +# Secret masking +# --------------------------------------------------------------------------- + + +class TestSecretMasking: + def test_list_masks_secrets(self, client): + _create_server( + client, + name="secret-test", + env={"API_KEY": "sk-real-secret-123"}, + headers={"Authorization": "Bearer tok-xyz"}, + transport="streamable-http", + url="http://localhost/mcp", + ) + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers") + assert r.status_code == 200 + server = r.json()["servers"][0] + env = json.loads(server["env"]) + headers = json.loads(server["headers"]) + assert env["API_KEY"] == "***" + assert headers["Authorization"] == "***" + + def test_list_reveals_secrets(self, client): + _create_server( + client, + name="reveal-test", + env={"API_KEY": "sk-real-secret-123"}, + headers={"Authorization": "Bearer tok-xyz"}, + transport="streamable-http", + url="http://localhost/mcp", + ) + with _PATCH_MCP_STATUS: + r = client.get("/v1/api/admin/mcp-servers?reveal=true") + assert r.status_code == 200 + server = r.json()["servers"][0] + env = json.loads(server["env"]) + headers = json.loads(server["headers"]) + assert env["API_KEY"] == "sk-real-secret-123" + assert headers["Authorization"] == "Bearer tok-xyz" + + def test_get_masks_secrets_by_default(self, client): + created = _create_server( + client, + name="mask-get-test", + env={"SECRET": "value"}, + ) + sid = created["server_id"] + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{sid}") + assert r.status_code == 200 + env = json.loads(r.json()["env"]) + assert env["SECRET"] == "***" + + def test_get_reveals_secrets(self, client): + created = _create_server( + client, + name="reveal-get-test", + env={"SECRET": "real-value"}, + ) + sid = created["server_id"] + with _PATCH_MCP_STATUS: + r = client.get(f"/v1/api/admin/mcp-servers/{sid}?reveal=true") + assert r.status_code == 200 + env = json.loads(r.json()["env"]) + assert env["SECRET"] == "real-value" + + +# --------------------------------------------------------------------------- +# Import +# --------------------------------------------------------------------------- + + +class TestImportMcpConfig: + def test_import_inline_config(self, client): + config = { + "mcpServers": { + "filesystem": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"], + }, + "remote": { + "url": "http://remote.example.com/mcp", + }, + }, + } + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": config}, + ) + assert r.status_code == 200 + data = r.json() + assert "filesystem" in data["imported"] + assert "remote" in data["imported"] + assert data["skipped"] == [] + assert data["errors"] == [] + + def test_import_not_a_dict(self, client): + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": "not-a-dict"}, + ) + assert r.status_code == 400 + + def test_import_skips_duplicates(self, client): + _create_server(client, name="existing-srv") + config = { + "mcpServers": { + "existing-srv": {"command": "node", "args": []}, + "new-srv": {"command": "node", "args": []}, + }, + } + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": config}, + ) + assert r.status_code == 200 + data = r.json() + assert "new-srv" in data["imported"] + assert "existing-srv" in data["skipped"] + + def test_import_empty_body(self, client): + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={}, + ) + assert r.status_code == 400 + assert "config" in r.json()["error"].lower() + + def test_import_no_mcp_servers_key(self, client): + r = client.post( + "/v1/api/admin/mcp-servers/import", + json={"config": {"other": "data"}}, + ) + assert r.status_code == 400 + assert "mcpServers" in r.json()["error"] or "No" in r.json()["error"] + + +# --------------------------------------------------------------------------- +# Permission check +# --------------------------------------------------------------------------- + + +class TestPermission: + def test_list_without_permission(self, client_no_perm): + with _PATCH_MCP_STATUS: + r = client_no_perm.get("/v1/api/admin/mcp-servers") + assert r.status_code == 403 + assert "admin.mcp" in r.json()["error"] + + def test_create_without_permission(self, client_no_perm): + r = client_no_perm.post( + "/v1/api/admin/mcp-servers", + json={"name": "test", "transport": "stdio", "command": "x"}, + ) + assert r.status_code == 403 + + def test_delete_without_permission(self, client_no_perm): + r = client_no_perm.delete(f"/v1/api/admin/mcp-servers/{uuid.uuid4().hex}") + assert r.status_code == 403 diff --git a/tests/test_mcp_hot_reload.py b/tests/test_mcp_hot_reload.py new file mode 100644 index 00000000..e26167b9 --- /dev/null +++ b/tests/test_mcp_hot_reload.py @@ -0,0 +1,353 @@ +"""Tests for MCPClientManager hot-reload methods.""" + +from __future__ import annotations + +from typing import Any + +from turnstone.core.mcp_client import MCPClientManager + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _fake_openai_tool(name: str = "mcp__test__search") -> dict[str, Any]: + """Create a fake OpenAI-format tool dict.""" + return { + "type": "function", + "function": { + "name": name, + "description": "[MCP: test] Search stuff", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } + + +def _fake_resource_dict( + uri: str = "file:///README.md", + name: str = "readme", + server: str = "test", +) -> dict[str, Any]: + """Create a fake resource dict as stored in per-server state.""" + return { + "uri": uri, + "name": name, + "description": "A resource", + "mimeType": "text/plain", + "server": server, + } + + +def _fake_prompt_dict( + name: str = "mcp__test__code_review", + original_name: str = "code_review", + server: str = "test", +) -> dict[str, Any]: + """Create a fake prompt dict as stored in per-server state.""" + return { + "name": name, + "original_name": original_name, + "server": server, + "description": "Generate a code review", + "arguments": [ + {"name": "language", "description": "Programming language", "required": True} + ], + } + + +# --------------------------------------------------------------------------- +# add_server_sync +# --------------------------------------------------------------------------- + + +class TestAddServerSync: + def test_rejects_double_underscore_name(self) -> None: + """Names containing __ should be rejected.""" + mgr = MCPClientManager({}) + result = mgr.add_server_sync("bad__name", {"command": "echo"}) + assert result["connected"] is False + assert "__" in result["error"] + assert result["tools"] == 0 + assert result["resources"] == 0 + assert result["prompts"] == 0 + + def test_fails_without_event_loop(self) -> None: + """Adding a server without starting the event loop should fail gracefully.""" + mgr = MCPClientManager({}) + result = mgr.add_server_sync("test", {"command": "echo"}) + assert result["connected"] is False + assert "loop" in result["error"].lower() + + def test_config_removed_on_failure(self) -> None: + """add_server_sync removes the config entry when connection fails.""" + mgr = MCPClientManager({}) + mgr.add_server_sync("new-srv", {"command": "echo"}) + # Since the loop isn't running, it fails and config is cleaned up + assert "new-srv" not in mgr._server_configs + + +# --------------------------------------------------------------------------- +# remove_server_sync +# --------------------------------------------------------------------------- + + +class TestRemoveServerSync: + def test_returns_false_for_nonexistent(self) -> None: + """Removing a non-connected server returns False.""" + mgr = MCPClientManager({}) + assert mgr.remove_server_sync("nonexistent") is False + + def test_cleans_up_per_server_state(self) -> None: + """remove_server_sync cleans up all per-server state dicts.""" + mgr = MCPClientManager({"test": {"command": "echo"}}) + # Simulate state as if the server was connected + mgr._per_server_tools["test"] = [_fake_openai_tool()] + mgr._per_server_resources["test"] = [_fake_resource_dict()] + mgr._per_server_prompts["test"] = [_fake_prompt_dict()] + mgr._supports_list_changed["test"] = True + mgr._supports_resources["test"] = True + mgr._supports_resource_list_changed["test"] = True + mgr._supports_prompts["test"] = True + mgr._supports_prompt_list_changed["test"] = True + mgr._rebuild_tools() + mgr._rebuild_resources() + mgr._rebuild_prompts() + + # Verify preconditions + assert len(mgr.get_tools()) == 1 + assert mgr.resource_count == 1 + assert mgr.prompt_count == 1 + + mgr.remove_server_sync("test") + + assert len(mgr.get_tools()) == 0 + assert mgr.resource_count == 0 + assert mgr.prompt_count == 0 + assert "test" not in mgr._per_server_tools + assert "test" not in mgr._per_server_resources + assert "test" not in mgr._per_server_prompts + assert "test" not in mgr._supports_list_changed + assert "test" not in mgr._supports_resources + assert "test" not in mgr._supports_resource_list_changed + assert "test" not in mgr._supports_prompts + assert "test" not in mgr._supports_prompt_list_changed + + def test_removes_config_to_prevent_reconnect(self) -> None: + """remove_server_sync removes from _server_configs to prevent reconnect.""" + mgr = MCPClientManager({"test": {"command": "echo"}}) + assert "test" in mgr._server_configs + mgr.remove_server_sync("test") + assert "test" not in mgr._server_configs + + def test_preserves_other_servers(self) -> None: + """Removing one server does not affect another server's state.""" + mgr = MCPClientManager({"srv_a": {}, "srv_b": {}}) + mgr._per_server_tools["srv_a"] = [_fake_openai_tool("mcp__srv_a__foo")] + mgr._per_server_tools["srv_b"] = [_fake_openai_tool("mcp__srv_b__bar")] + mgr._rebuild_tools() + + assert len(mgr.get_tools()) == 2 + + mgr.remove_server_sync("srv_a") + + assert len(mgr.get_tools()) == 1 + assert mgr.get_tools()[0]["function"]["name"] == "mcp__srv_b__bar" + assert "srv_b" in mgr._server_configs + + +# --------------------------------------------------------------------------- +# get_server_status +# --------------------------------------------------------------------------- + + +class TestGetServerStatus: + def test_disconnected_server_in_config(self) -> None: + """Status of a configured but not connected server shows disconnected.""" + mgr = MCPClientManager({"test": {"command": "echo"}}) + status = mgr.get_server_status("test") + assert status["connected"] is False + assert status["tools"] == 0 + assert status["resources"] == 0 + assert status["prompts"] == 0 + assert status["error"] == "" + + def test_connected_server_with_tools(self) -> None: + """Status of a connected server reports correct tool/resource/prompt counts.""" + mgr = MCPClientManager({"test": {}}) + # Simulate connected state + mgr._sessions["test"] = object() # any truthy value + mgr._per_server_tools["test"] = [ + _fake_openai_tool("mcp__test__a"), + _fake_openai_tool("mcp__test__b"), + ] + mgr._per_server_resources["test"] = [_fake_resource_dict()] + mgr._per_server_prompts["test"] = [_fake_prompt_dict()] + + status = mgr.get_server_status("test") + assert status["connected"] is True + assert status["tools"] == 2 + assert status["resources"] == 1 + assert status["prompts"] == 1 + + def test_unknown_server(self) -> None: + """Status of a server not in config or sessions shows disconnected.""" + mgr = MCPClientManager({}) + status = mgr.get_server_status("unknown") + assert status["connected"] is False + assert status["tools"] == 0 + + +# --------------------------------------------------------------------------- +# get_all_server_status +# --------------------------------------------------------------------------- + + +class TestGetAllServerStatus: + def test_empty_manager(self) -> None: + """Empty manager returns empty status dict.""" + mgr = MCPClientManager({}) + assert mgr.get_all_server_status() == {} + + def test_multiple_servers(self) -> None: + """Manager with configs but no connections returns status for each.""" + mgr = MCPClientManager({"alpha": {}, "bravo": {}}) + statuses = mgr.get_all_server_status() + assert len(statuses) == 2 + assert "alpha" in statuses + assert "bravo" in statuses + assert statuses["alpha"]["connected"] is False + assert statuses["bravo"]["connected"] is False + + def test_mixed_connected_and_disconnected(self) -> None: + """Status correctly reflects a mix of connected and disconnected servers.""" + mgr = MCPClientManager({"up": {}, "down": {}}) + mgr._sessions["up"] = object() + mgr._per_server_tools["up"] = [_fake_openai_tool("mcp__up__x")] + + statuses = mgr.get_all_server_status() + assert statuses["up"]["connected"] is True + assert statuses["up"]["tools"] == 1 + assert statuses["down"]["connected"] is False + assert statuses["down"]["tools"] == 0 + + +# --------------------------------------------------------------------------- +# reconcile_sync +# --------------------------------------------------------------------------- + + +class _FakeStorage: + """Minimal mock storage for reconcile tests.""" + + def __init__(self, rows: list[dict[str, Any]]) -> None: + self._rows = rows + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + if enabled_only: + return [r for r in self._rows if r.get("enabled", True)] + return list(self._rows) + + +def _db_row( + name: str, + transport: str = "stdio", + command: str = "echo", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + enabled: bool = True, +) -> dict[str, Any]: + return { + "name": name, + "transport": transport, + "command": command, + "args": args, + "url": url, + "headers": headers, + "env": env, + "enabled": enabled, + } + + +class TestReconcileSync: + def test_adds_new_servers(self) -> None: + mgr = MCPClientManager({}) + storage = _FakeStorage([_db_row("new-srv")]) + # Can't actually connect (no loop), but config should be attempted + result = mgr.reconcile_sync(storage) + # add_server_sync fails without a loop, but the method shouldn't crash + assert "new-srv" not in result["added"] # fails gracefully + assert result["removed"] == [] + assert result["updated"] == [] + + def test_removes_stale_db_servers(self) -> None: + mgr = MCPClientManager({"old-srv": {"command": "echo"}}) + mgr._db_managed.add("old-srv") # mark as DB-managed + storage = _FakeStorage([]) # DB is empty + result = mgr.reconcile_sync(storage) + assert "old-srv" in result["removed"] + assert "old-srv" not in mgr._server_configs + + def test_preserves_config_file_servers(self) -> None: + """Config-file servers (not in _db_managed) survive reconcile.""" + mgr = MCPClientManager({"env-srv": {"command": "echo"}}) + # NOT in _db_managed — loaded from MCP_CONFIG env + storage = _FakeStorage([]) # DB is empty + result = mgr.reconcile_sync(storage) + assert result["removed"] == [] + assert "env-srv" in mgr._server_configs # still there + + def test_config_server_not_overwritten_by_db_name_collision(self) -> None: + """DB server with same name as config-file server does not replace it.""" + original_cfg = {"type": "stdio", "command": "config-echo", "args": [], "env": {}} + mgr = MCPClientManager({"shared-name": dict(original_cfg)}) + # NOT in _db_managed — this is a config-file server + # DB has a server with the same name but different config + storage = _FakeStorage([_db_row("shared-name", command="db-echo")]) + result = mgr.reconcile_sync(storage) + # Config-file server should NOT be updated + assert result["updated"] == [] + assert "shared-name" in mgr._server_configs + assert mgr._server_configs["shared-name"]["command"] == "config-echo" + + def test_updates_changed_config(self) -> None: + original_cfg = {"type": "stdio", "command": "echo", "args": [], "env": {}} + mgr = MCPClientManager({"srv": dict(original_cfg)}) + mgr._db_managed.add("srv") # mark as DB-managed + # DB has updated command — config differs + storage = _FakeStorage([_db_row("srv", command="cat")]) + result = mgr.reconcile_sync(storage) + # remove_server_sync ran (old config cleared), add_server_sync attempted + # but fails without a running event loop — that's expected in unit tests. + # The key assertion: the old config was evicted (not left stale). + assert "srv" not in mgr._server_configs + # Not in "removed" (that's for servers absent from DB) + assert "srv" not in result["removed"] + + def test_no_change_is_noop(self) -> None: + cfg = {"type": "stdio", "command": "echo", "args": [], "env": {}} + mgr = MCPClientManager({"srv": dict(cfg)}) + storage = _FakeStorage([_db_row("srv", command="echo")]) + result = mgr.reconcile_sync(storage) + assert result["added"] == [] + assert result["removed"] == [] + assert result["updated"] == [] + # Config unchanged + assert "srv" in mgr._server_configs + + def test_storage_failure_graceful(self) -> None: + mgr = MCPClientManager({"srv": {}}) + + class _BrokenStorage: + def list_mcp_servers(self, **kw: Any) -> list[dict[str, Any]]: + raise RuntimeError("DB down") + + result = mgr.reconcile_sync(_BrokenStorage()) + assert result == {"added": [], "removed": [], "updated": []} + # Existing server untouched + assert "srv" in mgr._server_configs diff --git a/tests/test_mcp_server_storage.py b/tests/test_mcp_server_storage.py new file mode 100644 index 00000000..a8ab48d1 --- /dev/null +++ b/tests/test_mcp_server_storage.py @@ -0,0 +1,152 @@ +"""Tests for MCP server storage CRUD operations.""" + +from __future__ import annotations + +import uuid + +import pytest + +from turnstone.core.storage._sqlite import SQLiteBackend + + +@pytest.fixture +def db(tmp_path): + """Fresh SQLite backend for each test.""" + return SQLiteBackend(str(tmp_path / "test.db")) + + +def _make_id() -> str: + return uuid.uuid4().hex + + +class TestMcpServerStorage: + def test_create_and_get(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server( + server_id=sid, + name="test-server", + transport="stdio", + command="echo", + args='["hello"]', + ) + s = db.get_mcp_server(sid) + assert s is not None + assert s["name"] == "test-server" + assert s["transport"] == "stdio" + assert s["command"] == "echo" + assert s["args"] == '["hello"]' + assert s["enabled"] is True + assert s["auto_approve"] is False + + def test_get_by_name(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="named-srv", transport="stdio") + s = db.get_mcp_server_by_name("named-srv") + assert s is not None + assert s["server_id"] == sid + + def test_get_by_name_not_found(self, db: SQLiteBackend) -> None: + assert db.get_mcp_server_by_name("nope") is None + + def test_get_not_found(self, db: SQLiteBackend) -> None: + assert db.get_mcp_server("nonexistent") is None + + def test_list_empty(self, db: SQLiteBackend) -> None: + assert db.list_mcp_servers() == [] + + def test_list_all(self, db: SQLiteBackend) -> None: + db.create_mcp_server(server_id=_make_id(), name="alpha", transport="stdio") + db.create_mcp_server( + server_id=_make_id(), name="beta", transport="streamable-http", url="http://x" + ) + servers = db.list_mcp_servers() + assert len(servers) == 2 + assert servers[0]["name"] == "alpha" # ordered by name + assert servers[1]["name"] == "beta" + + def test_list_enabled_only(self, db: SQLiteBackend) -> None: + sid1 = _make_id() + sid2 = _make_id() + db.create_mcp_server(server_id=sid1, name="enabled-srv", transport="stdio", enabled=True) + db.create_mcp_server(server_id=sid2, name="disabled-srv", transport="stdio", enabled=False) + enabled = db.list_mcp_servers(enabled_only=True) + assert len(enabled) == 1 + assert enabled[0]["name"] == "enabled-srv" + + def test_update_basic_fields(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="orig", transport="stdio", command="echo") + ok = db.update_mcp_server(sid, name="renamed", command="cat") + assert ok is True + s = db.get_mcp_server(sid) + assert s is not None + assert s["name"] == "renamed" + assert s["command"] == "cat" + + def test_update_boolean_conversion(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="booltest", transport="stdio") + db.update_mcp_server(sid, auto_approve=True, enabled=False) + s = db.get_mcp_server(sid) + assert s is not None + assert s["auto_approve"] is True + assert s["enabled"] is False + + def test_update_not_found(self, db: SQLiteBackend) -> None: + ok = db.update_mcp_server("nonexistent", name="x") + assert ok is False + + def test_update_ignores_disallowed_fields(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="guard", transport="stdio", created_by="admin") + original = db.get_mcp_server(sid) + assert original is not None + original_created = original["created"] + # created_by and created are not in the mutable allowlist + db.update_mcp_server(sid, created_by="evil", created="2000-01-01T00:00:00") + s = db.get_mcp_server(sid) + assert s is not None + assert s["created_by"] == "admin" # unchanged + assert s["created"] == original_created # unchanged + + def test_delete(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="delme", transport="stdio") + ok = db.delete_mcp_server(sid) + assert ok is True + assert db.get_mcp_server(sid) is None + + def test_delete_not_found(self, db: SQLiteBackend) -> None: + ok = db.delete_mcp_server("nonexistent") + assert ok is False + + def test_create_duplicate_name(self, db: SQLiteBackend) -> None: + db.create_mcp_server(server_id=_make_id(), name="unique", transport="stdio") + # Second create with same name but different ID should be no-op (OR IGNORE) + sid2 = _make_id() + db.create_mcp_server(server_id=sid2, name="unique", transport="stdio") + # OR IGNORE silently drops the conflicting insert + assert db.get_mcp_server(sid2) is None + + def test_create_idempotent_same_id(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server(server_id=sid, name="idem", transport="stdio", command="v1") + db.create_mcp_server(server_id=sid, name="idem", transport="stdio", command="v2") + s = db.get_mcp_server(sid) + assert s is not None + assert s["command"] == "v1" # original preserved, second ignored + + def test_http_transport_fields(self, db: SQLiteBackend) -> None: + sid = _make_id() + db.create_mcp_server( + server_id=sid, + name="http-srv", + transport="streamable-http", + url="https://example.com/mcp", + headers='{"Authorization":"Bearer xyz"}', + ) + s = db.get_mcp_server(sid) + assert s is not None + assert s["transport"] == "streamable-http" + assert s["url"] == "https://example.com/mcp" + assert "Authorization" in s["headers"] diff --git a/turnstone/api/console_schemas.py b/turnstone/api/console_schemas.py index a406a9e8..21a68a4e 100644 --- a/turnstone/api/console_schemas.py +++ b/turnstone/api/console_schemas.py @@ -570,3 +570,83 @@ class ListSettingSchemaResponse(BaseModel): class UpdateSettingRequest(BaseModel): value: Any node_id: str = "" + + +# --------------------------------------------------------------------------- +# Admin: MCP Servers +# --------------------------------------------------------------------------- + + +class McpServerInfo(BaseModel): + server_id: str + name: str + transport: str + command: str = "" + args: str = "[]" + url: str = "" + headers: str = "{}" + env: str = "{}" + auto_approve: bool = False + enabled: bool = True + created_by: str = "" + created: str + updated: str + + +class McpServerStatus(BaseModel): + connected: bool = False + tools: int = 0 + resources: int = 0 + prompts: int = 0 + error: str = "" + transport: str = "" + command: str = "" + url: str = "" + + +class McpServerDetail(McpServerInfo): + status: dict[str, McpServerStatus] = Field(default_factory=dict) + source: str = "" # "config" for config-file servers, empty for DB-managed + + +class CreateMcpServerRequest(BaseModel): + name: str + transport: str # "stdio" | "streamable-http" + command: str = "" + args: list[str] = [] + url: str = "" + headers: dict[str, str] = Field(default_factory=dict) + env: dict[str, str] = Field(default_factory=dict) + auto_approve: bool = False + enabled: bool = True + + +class UpdateMcpServerRequest(BaseModel): + name: str | None = None + transport: str | None = None + command: str | None = None + args: list[str] | None = None + url: str | None = None + headers: dict[str, str] | None = None + env: dict[str, str] | None = None + auto_approve: bool | None = None + enabled: bool | None = None + + +class ListMcpServersResponse(BaseModel): + servers: list[McpServerDetail] + + +class ImportMcpConfigRequest(BaseModel): + config: dict[str, Any] = Field(..., description="JSON config object with mcpServers key") + + +class ImportMcpConfigResponse(BaseModel): + imported: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + + +class McpReloadResponse(BaseModel): + status: str = "ok" + results: dict[str, Any] = Field(default_factory=dict) diff --git a/turnstone/api/console_spec.py b/turnstone/api/console_spec.py index ade0f2e0..05a9a1f5 100644 --- a/turnstone/api/console_spec.py +++ b/turnstone/api/console_spec.py @@ -20,13 +20,17 @@ from turnstone.api.console_schemas import ( ConsoleCreateWsResponse, ConsoleHealthResponse, CreateChannelUserRequest, + CreateMcpServerRequest, CreatePromptTemplateRequest, CreateRoleRequest, CreateToolPolicyRequest, CreateWsTemplateRequest, + ImportMcpConfigRequest, + ImportMcpConfigResponse, ListAdminMemoriesResponse, ListAuditEventsResponse, ListChannelUsersResponse, + ListMcpServersResponse, ListOrgsResponse, ListPromptTemplatesResponse, ListRolesResponse, @@ -38,6 +42,8 @@ from turnstone.api.console_schemas import ( ListWsTemplatesResponse, ListWsTemplateSummaryResponse, ListWsTemplateVersionsResponse, + McpReloadResponse, + McpServerDetail, NodeDetailResponse, OrgInfo, PromptTemplateInfo, @@ -45,6 +51,7 @@ from turnstone.api.console_schemas import ( SettingInfo, SettingSchemaInfo, ToolPolicyInfo, + UpdateMcpServerRequest, UpdateOrgRequest, UpdatePromptTemplateRequest, UpdateRoleRequest, @@ -661,6 +668,67 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [ error_codes=[400, 404], tags=["Admin"], ), + # --- Admin: MCP Servers --- + EndpointSpec( + "/v1/api/admin/mcp-servers", + "GET", + "List MCP server definitions with live status", + response_model=ListMcpServersResponse, + query_params=[ + QueryParam("reveal", "Show secret env/header values", schema_type="boolean"), + ], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers", + "POST", + "Create an MCP server definition", + request_model=CreateMcpServerRequest, + response_model=McpServerDetail, + error_codes=[400, 409], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/{server_id}", + "GET", + "Get a single MCP server with status", + response_model=McpServerDetail, + error_codes=[404], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/{server_id}", + "PUT", + "Update an MCP server definition", + request_model=UpdateMcpServerRequest, + response_model=McpServerDetail, + error_codes=[400, 404, 409], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/{server_id}", + "DELETE", + "Delete an MCP server definition", + response_model=StatusResponse, + error_codes=[404], + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/reload", + "POST", + "Tell all nodes to re-read MCP server config from DB and reconcile", + response_model=McpReloadResponse, + tags=["Admin"], + ), + EndpointSpec( + "/v1/api/admin/mcp-servers/import", + "POST", + "Import MCP servers from a JSON config file", + request_model=ImportMcpConfigRequest, + response_model=ImportMcpConfigResponse, + error_codes=[400], + tags=["Admin"], + ), # --- Observability --- EndpointSpec( "/health", @@ -732,6 +800,13 @@ _ALL_MODELS: list[type[BaseModel]] = [ SettingSchemaInfo, ListSettingSchemaResponse, UpdateSettingRequest, + McpServerDetail, + CreateMcpServerRequest, + UpdateMcpServerRequest, + ListMcpServersResponse, + ImportMcpConfigRequest, + ImportMcpConfigResponse, + McpReloadResponse, ] diff --git a/turnstone/cli.py b/turnstone/cli.py index efa71bd5..865aa0d8 100644 --- a/turnstone/cli.py +++ b/turnstone/cli.py @@ -1006,10 +1006,12 @@ def main() -> None: # Initialize MCP client (connects to configured MCP servers, if any) from turnstone.core.mcp_client import create_mcp_client + from turnstone.core.storage._registry import get_storage as _get_storage mcp_client = create_mcp_client( getattr(args, "mcp_config", None), refresh_interval=getattr(args, "mcp_refresh_interval", 14400), + storage=_get_storage(), ) # ChatSession factory — captures shared config for creating workstreams diff --git a/turnstone/console/server.py b/turnstone/console/server.py index f48f61e7..b8d74f7b 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -1513,6 +1513,7 @@ _VALID_PERMISSIONS = frozenset( "admin.judge", "admin.memories", "admin.settings", + "admin.mcp", "tools.approve", "workstreams.create", "workstreams.close", @@ -2968,6 +2969,534 @@ async def admin_delete_setting(request: Request) -> JSONResponse: return JSONResponse({"status": "ok", "key": key}) +# --------------------------------------------------------------------------- +# Admin: MCP Servers +# --------------------------------------------------------------------------- + +_MCP_NAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$") +_MCP_MAX_SERVERS = 50 + + +def _mask_mcp_secrets(server: dict[str, Any], reveal: bool = False) -> dict[str, Any]: + """Replace env/headers values with '***' unless reveal is True.""" + if reveal: + return server + s = dict(server) + if s.get("env") and s["env"] != "{}": + try: + env_dict = json.loads(s["env"]) if isinstance(s["env"], str) else s["env"] + s["env"] = json.dumps({k: "***" for k in env_dict}) + except (json.JSONDecodeError, TypeError): + s["env"] = "{}" + if s.get("headers") and s["headers"] != "{}": + try: + hdr_dict = json.loads(s["headers"]) if isinstance(s["headers"], str) else s["headers"] + s["headers"] = json.dumps({k: "***" for k in hdr_dict}) + except (json.JSONDecodeError, TypeError): + s["headers"] = "{}" + return s + + +def _mcp_server_to_detail( + server: dict[str, Any], + node_statuses: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Convert a storage dict to a McpServerDetail-shaped dict.""" + d = dict(server) + d["status"] = node_statuses or {} + return d + + +async def _collect_mcp_status( + request: Request, +) -> dict[str, dict[str, dict[str, Any]]]: + """Query all nodes for MCP status. Returns {node_id: {server_name: status}}.""" + collector: ClusterCollector = request.app.state.collector + nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + result: dict[str, dict[str, dict[str, Any]]] = {} + for node in nodes: + node_id = node.get("node_id", "") + url = node.get("server_url", "") + if not url: + continue + try: + headers = _proxy_auth_headers(request) + async with httpx.AsyncClient(timeout=httpx.Timeout(10)) as client: + resp = await client.get( + f"{url.rstrip('/')}/v1/api/_internal/mcp-status", + headers=headers, + ) + if resp.status_code == 200: + result[node_id] = resp.json().get("servers", {}) + except Exception: + pass + return result + + +async def admin_list_mcp_servers(request: Request) -> JSONResponse: + """GET /v1/api/admin/mcp-servers — list all MCP server definitions.""" + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + reveal = str(request.query_params.get("reveal", "")).lower() in ("true", "1") + servers = storage.list_mcp_servers() + + # Collect live status from all nodes + node_statuses = await _collect_mcp_status(request) + + db_names: set[str] = set() + result = [] + for s in servers: + db_names.add(s["name"]) + # Build per-node status for this server + per_node: dict[str, dict[str, Any]] = {} + for node_id, node_servers in node_statuses.items(): + status = node_servers.get(s["name"]) + if status: + per_node[node_id] = status + s = _mask_mcp_secrets(s, reveal) + result.append(_mcp_server_to_detail(s, per_node)) + + # Merge config-sourced servers visible on nodes but not in DB + config_names: set[str] = set() + for node_servers in node_statuses.values(): + for name in node_servers: + if name not in db_names: + config_names.add(name) + for name in sorted(config_names): + # Build a synthetic read-only entry from node-reported data + per_node = {} + transport = "stdio" + command = "" + url = "" + for node_id, node_servers in node_statuses.items(): + ns = node_servers.get(name) + if ns: + per_node[node_id] = ns + transport = ns.get("transport", "stdio") + command = ns.get("command", "") + url = ns.get("url", "") + result.append( + { + "server_id": "", + "name": name, + "transport": transport, + "command": command, + "args": "[]", + "url": url, + "headers": "{}", + "env": "{}", + "auto_approve": False, + "enabled": True, + "created_by": "", + "created": "", + "updated": "", + "source": "config", + "status": per_node, + } + ) + + return JSONResponse({"servers": result}) + + +async def admin_create_mcp_server(request: Request) -> JSONResponse: + """POST /v1/api/admin/mcp-servers — create an MCP server definition.""" + import uuid + + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + + name = str(body.get("name", "")).strip()[:64] + transport = str(body.get("transport", "")).strip() + if not name: + return JSONResponse({"error": "name is required"}, status_code=400) + if not _MCP_NAME_RE.match(name): + return JSONResponse( + {"error": "name must match [a-zA-Z0-9._-]+"}, + status_code=400, + ) + if "__" in name: + return JSONResponse( + {"error": "name must not contain '__' (reserved delimiter)"}, + status_code=400, + ) + if transport not in ("stdio", "streamable-http"): + return JSONResponse( + {"error": "transport must be 'stdio' or 'streamable-http'"}, + status_code=400, + ) + if transport == "stdio" and not str(body.get("command", "")).strip(): + return JSONResponse({"error": "command is required for stdio transport"}, status_code=400) + if transport == "streamable-http" and not str(body.get("url", "")).strip(): + return JSONResponse( + {"error": "url is required for streamable-http transport"}, status_code=400 + ) + + # Check max servers + existing = storage.list_mcp_servers() + if len(existing) >= _MCP_MAX_SERVERS: + return JSONResponse( + {"error": f"Maximum {_MCP_MAX_SERVERS} servers"}, + status_code=400, + ) + + # Check name uniqueness + if storage.get_mcp_server_by_name(name): + return JSONResponse( + {"error": f"Server '{name}' already exists"}, + status_code=409, + ) + + server_id = uuid.uuid4().hex + audit_uid, ip = _audit_context(request) + + args_list = body.get("args", []) + headers_dict = body.get("headers", {}) + env_dict = body.get("env", {}) + + storage.create_mcp_server( + server_id=server_id, + name=name, + transport=transport, + command=str(body.get("command", "")).strip(), + args=json.dumps(args_list) if isinstance(args_list, list) else "[]", + url=str(body.get("url", "")).strip(), + headers=json.dumps(headers_dict) if isinstance(headers_dict, dict) else "{}", + env=json.dumps(env_dict) if isinstance(env_dict, dict) else "{}", + auto_approve=bool(body.get("auto_approve", False)), + enabled=bool(body.get("enabled", True)), + created_by=audit_uid, + ) + + record_audit( + storage, + audit_uid, + "mcp_server.create", + "mcp_server", + server_id, + {"name": name}, + ip, + ) + + server = storage.get_mcp_server(server_id) + return JSONResponse(_mcp_server_to_detail(_mask_mcp_secrets(server or {}))) + + +async def admin_get_mcp_server(request: Request) -> JSONResponse: + """GET /v1/api/admin/mcp-servers/{server_id} — get single MCP server.""" + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + server_id = request.path_params["server_id"] + server = storage.get_mcp_server(server_id) + if server is None: + return JSONResponse({"error": "MCP server not found"}, status_code=404) + + node_statuses = await _collect_mcp_status(request) + per_node: dict[str, dict[str, Any]] = {} + for node_id, node_servers in node_statuses.items(): + status = node_servers.get(server["name"]) + if status: + per_node[node_id] = status + + reveal = str(request.query_params.get("reveal", "")).lower() in ("true", "1") + server = _mask_mcp_secrets(server, reveal) + return JSONResponse(_mcp_server_to_detail(server, per_node)) + + +async def admin_update_mcp_server(request: Request) -> JSONResponse: + """PUT /v1/api/admin/mcp-servers/{server_id} — update an MCP server.""" + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + server_id = request.path_params["server_id"] + existing = storage.get_mcp_server(server_id) + if existing is None: + return JSONResponse({"error": "MCP server not found"}, status_code=404) + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + + updates: dict[str, Any] = {} + if "name" in body: + name = str(body["name"]).strip()[:64] + if not name: + return JSONResponse({"error": "name cannot be empty"}, status_code=400) + if not _MCP_NAME_RE.match(name): + return JSONResponse( + {"error": "name must match [a-zA-Z0-9._-]+"}, + status_code=400, + ) + if "__" in name: + return JSONResponse( + {"error": "name must not contain '__'"}, + status_code=400, + ) + if name != existing["name"] and storage.get_mcp_server_by_name(name): + return JSONResponse( + {"error": f"Server '{name}' already exists"}, + status_code=409, + ) + updates["name"] = name + if "transport" in body: + transport = str(body["transport"]).strip() + if transport not in ("stdio", "streamable-http"): + return JSONResponse( + {"error": "transport must be 'stdio' or 'streamable-http'"}, + status_code=400, + ) + updates["transport"] = transport + if "command" in body: + updates["command"] = str(body["command"]).strip() + if "args" in body: + updates["args"] = json.dumps(body["args"]) if isinstance(body["args"], list) else "[]" + if "url" in body: + updates["url"] = str(body["url"]).strip() + if "headers" in body: + updates["headers"] = ( + json.dumps(body["headers"]) if isinstance(body["headers"], dict) else "{}" + ) + if "env" in body: + updates["env"] = json.dumps(body["env"]) if isinstance(body["env"], dict) else "{}" + if "auto_approve" in body: + updates["auto_approve"] = bool(body["auto_approve"]) + if "enabled" in body: + updates["enabled"] = bool(body["enabled"]) + + if updates: + storage.update_mcp_server(server_id, **updates) + + audit_uid, ip = _audit_context(request) + audit_detail = dict(updates) + for _secret_key in ("env", "headers"): + if _secret_key in audit_detail: + audit_detail[_secret_key] = "(updated)" + record_audit( + storage, + audit_uid, + "mcp_server.update", + "mcp_server", + server_id, + audit_detail, + ip, + ) + + server = storage.get_mcp_server(server_id) + return JSONResponse(_mcp_server_to_detail(_mask_mcp_secrets(server or {}))) + + +async def admin_delete_mcp_server(request: Request) -> JSONResponse: + """DELETE /v1/api/admin/mcp-servers/{server_id}.""" + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + server_id = request.path_params["server_id"] + existing = storage.get_mcp_server(server_id) + if existing is None: + return JSONResponse({"error": "MCP server not found"}, status_code=404) + + storage.delete_mcp_server(server_id) + + audit_uid, ip = _audit_context(request) + record_audit( + storage, + audit_uid, + "mcp_server.delete", + "mcp_server", + server_id, + {"name": existing.get("name", "")}, + ip, + ) + + return JSONResponse({"status": "ok"}) + + +async def _notify_nodes_mcp_reload(request: Request) -> dict[str, Any]: + """Tell all nodes to re-read the mcp_servers DB table and reconcile.""" + collector: ClusterCollector = request.app.state.collector + nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + results: dict[str, Any] = {} + + for node in nodes: + node_id = node.get("node_id", "") + url = node.get("server_url", "") + if not url: + continue + try: + headers = _proxy_auth_headers(request) + async with httpx.AsyncClient(timeout=httpx.Timeout(30)) as client: + resp = await client.post( + f"{url.rstrip('/')}/v1/api/_internal/mcp-reload", + headers=headers, + ) + results[node_id] = resp.json() + except Exception as exc: + results[node_id] = {"error": str(exc)} + + return results + + +async def admin_mcp_reload(request: Request) -> JSONResponse: + """POST /v1/api/admin/mcp-servers/reload — tell nodes to re-read DB.""" + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + results = await _notify_nodes_mcp_reload(request) + return JSONResponse({"status": "ok", "results": results}) + + +async def admin_import_mcp_config(request: Request) -> JSONResponse: + """POST /v1/api/admin/mcp-servers/import — import from pasted JSON config.""" + import uuid + + from turnstone.core.audit import record_audit + from turnstone.core.auth import require_permission + from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503 + + storage, err = require_storage_or_503(request) + if err: + return err + err = require_permission(request, "admin.mcp") + if err: + return err + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + + data = body.get("config") + if not isinstance(data, dict): + return JSONResponse( + {"error": "config is required (JSON object with mcpServers key)"}, status_code=400 + ) + + servers = data.get("mcpServers", {}) + if not isinstance(servers, dict) or not servers: + return JSONResponse( + {"error": "No mcpServers found in config"}, + status_code=400, + ) + + imported: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + audit_uid, ip = _audit_context(request) + current_count = len(storage.list_mcp_servers()) + + for srv_name, cfg in servers.items(): + srv_name = str(srv_name).strip()[:64] + if not srv_name or not _MCP_NAME_RE.match(srv_name) or "__" in srv_name: + errors.append(f"{srv_name}: invalid server name") + continue + if storage.get_mcp_server_by_name(srv_name): + skipped.append(srv_name) + continue + if current_count >= _MCP_MAX_SERVERS: + errors.append(f"{srv_name}: max servers reached") + break + + transport = "stdio" + if "url" in cfg or cfg.get("type") in ("http", "streamable-http"): + transport = "streamable-http" + + # Coerce fields to expected types + raw_args = cfg.get("args", []) + raw_headers = cfg.get("headers", {}) + raw_env = cfg.get("env", {}) + if not isinstance(raw_args, list): + errors.append(f"{srv_name}: args must be a list") + continue + if not isinstance(raw_headers, dict): + errors.append(f"{srv_name}: headers must be an object") + continue + if not isinstance(raw_env, dict): + errors.append(f"{srv_name}: env must be an object") + continue + + server_id = uuid.uuid4().hex + try: + storage.create_mcp_server( + server_id=server_id, + name=srv_name, + transport=transport, + command=str(cfg.get("command", "")), + args=json.dumps(raw_args), + url=str(cfg.get("url", "")), + headers=json.dumps(raw_headers), + env=json.dumps(raw_env), + auto_approve=False, + enabled=True, + created_by=audit_uid, + ) + imported.append(srv_name) + current_count += 1 + except Exception as exc: + errors.append(f"{srv_name}: {exc}") + + if imported: + record_audit( + storage, + audit_uid, + "mcp_server.import", + "mcp_server", + "", + {"imported": imported, "skipped": skipped}, + ip, + ) + + return JSONResponse({"imported": imported, "skipped": skipped, "errors": errors}) + + # --------------------------------------------------------------------------- # App factory # --------------------------------------------------------------------------- @@ -3131,6 +3660,37 @@ def create_app( admin_delete_setting, methods=["DELETE"], ), + # System: MCP Servers + Route("/api/admin/mcp-servers", admin_list_mcp_servers), + Route( + "/api/admin/mcp-servers", + admin_create_mcp_server, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/import", + admin_import_mcp_config, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/reload", + admin_mcp_reload, + methods=["POST"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_get_mcp_server, + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_update_mcp_server, + methods=["PUT"], + ), + Route( + "/api/admin/mcp-servers/{server_id}", + admin_delete_mcp_server, + methods=["DELETE"], + ), # Governance: Usage & Audit Route("/api/admin/usage", admin_usage), Route("/api/admin/audit", admin_audit), diff --git a/turnstone/console/static/admin.js b/turnstone/console/static/admin.js index e224551e..00ce7683 100644 --- a/turnstone/console/static/admin.js +++ b/turnstone/console/static/admin.js @@ -64,7 +64,8 @@ function showAdmin() { usage: "admin.usage", audit: "admin.audit", memories: "admin.memories", - settings: "admin.users", + settings: "admin.settings", + mcp: "admin.mcp", }; if (perms) { var permSet = perms.split(","); @@ -194,6 +195,7 @@ function switchAdminTab(tab) { "audit", "memories", "settings", + "mcp", ]; for (var p = 0; p < panels.length; p++) { var el = document.getElementById("admin-" + panels[p]); @@ -216,6 +218,7 @@ function switchAdminTab(tab) { } if (tab === "memories") loadAdminMemories(); if (tab === "settings") loadSettings(); + if (tab === "mcp") loadAdminMcp(); // Update breadcrumb with active tab label var activeNav = document.querySelector('.admin-nav[data-tab="' + tab + '"]'); @@ -1593,6 +1596,9 @@ function _installTrap(overlayId, boxId, trapRef) { else if (overlayId === "edit-wst-overlay") hideEditWsTemplateModal(); else if (overlayId === "wst-history-overlay") hideWstHistoryModal(); else if (overlayId === "memory-detail-overlay") hideMemoryDetailModal(); + else if (overlayId === "mcp-create-overlay") hideCreateMcpModal(); + else if (overlayId === "mcp-import-overlay") hideImportMcpModal(); + else if (overlayId === "mcp-detail-overlay") hideMcpDetailModal(); } }; } @@ -1679,6 +1685,9 @@ document.addEventListener("keydown", function (e) { ["edit-wst-overlay", hideEditWsTemplateModal], ["wst-history-overlay", hideWstHistoryModal], ["memory-detail-overlay", hideMemoryDetailModal], + ["mcp-detail-overlay", hideMcpDetailModal], + ["mcp-import-overlay", hideImportMcpModal], + ["mcp-create-overlay", hideCreateMcpModal], ]; for (var gi = 0; gi < govOverlays.length; gi++) { var govEl = document.getElementById(govOverlays[gi][0]); @@ -2354,3 +2363,565 @@ function _showModalError(el, msg) { el.textContent = msg; el.style.display = "block"; } + +/* ── MCP Servers tab ─────────────────────────────────────────────────────── */ + +var _mcpServers = []; +var _mcpCreateTrap = null; +var _mcpCreateTrigger = null; +var _mcpImportTrap = null; +var _mcpImportTrigger = null; +var _mcpDetailTrap = null; +var _mcpDetailTrigger = null; + +function loadAdminMcp() { + authFetch("/v1/api/admin/mcp-servers") + .then(function (r) { + if (!r.ok) throw new Error("Failed"); + return r.json(); + }) + .then(function (data) { + _mcpServers = data.servers || []; + _renderMcpServers(_mcpServers); + }) + .catch(function () { + document.getElementById("admin-mcp-table").innerHTML = + '
Failed to load MCP servers
'; + }); +} + +function _renderMcpServers(items) { + var el = document.getElementById("admin-mcp-table"); + if (!items.length) { + el.innerHTML = + '
No MCP servers configured
'; + return; + } + var html = ""; + for (var i = 0; i < items.length; i++) { + var s = items[i]; + var statusEntries = s.status || {}; + var nodeIds = Object.keys(statusEntries); + var anyConnected = false; + var anyError = false; + var totalTools = 0, + totalRes = 0, + totalPrompts = 0; + for (var j = 0; j < nodeIds.length; j++) { + var ns = statusEntries[nodeIds[j]]; + if (ns.connected) { + anyConnected = true; + totalTools += ns.tools || 0; + totalRes += ns.resources || 0; + totalPrompts += ns.prompts || 0; + } + if (ns.error) anyError = true; + } + + var dotClass = "mcp-status-dot disabled"; + var rowClass = "mcp-row-disabled"; + var statusText = "disabled"; + if (!s.enabled) { + statusText = "disabled"; + } else if (anyConnected) { + dotClass = "mcp-status-dot connected"; + rowClass = "mcp-row-connected"; + statusText = "connected"; + } else if (anyError) { + dotClass = "mcp-status-dot error"; + rowClass = "mcp-row-error"; + statusText = "error"; + } else { + dotClass = "mcp-status-dot disabled"; + rowClass = "mcp-row-disabled"; + statusText = "idle"; + } + + var transportCls = + s.transport === "stdio" ? "mcp-transport-stdio" : "mcp-transport-http"; + var toolsVal = anyConnected + ? totalTools + : '--'; + var resVal = anyConnected + ? totalRes + : '--'; + var promptsVal = anyConnected + ? totalPrompts + : '--'; + + var isConfig = s.source === "config"; + var nameBadge = isConfig + ? ' config' + : ""; + var detailAttr = isConfig + ? 'data-mcp-detail-name="' + escapeHtml(s.name) + '"' + : 'data-mcp-detail="' + escapeHtml(s.server_id) + '"'; + var actions = isConfig + ? "" + : '' + + ''; + + html += + '
' + + '" + + escapeHtml(s.name) + + "" + + nameBadge + + "" + + '' + + escapeHtml(s.transport) + + "" + + '' + + toolsVal + + "" + + '' + + resVal + + "" + + '' + + promptsVal + + "" + + '' + + escapeHtml(statusText) + + "" + + '' + + actions + + "
"; + } + el.innerHTML = html; + + // Bind event handlers + el.querySelectorAll("[data-mcp-detail]").forEach(function (a) { + a.addEventListener("click", function (e) { + e.preventDefault(); + showMcpDetailModal(this.getAttribute("data-mcp-detail")); + }); + }); + el.querySelectorAll("[data-mcp-detail-name]").forEach(function (a) { + a.addEventListener("click", function (e) { + e.preventDefault(); + showMcpDetailByName(this.getAttribute("data-mcp-detail-name")); + }); + }); + el.querySelectorAll("[data-mcp-edit]").forEach(function (btn) { + btn.addEventListener("click", function () { + showEditMcpModal(this.getAttribute("data-mcp-edit")); + }); + }); + el.querySelectorAll("[data-mcp-delete]").forEach(function (btn) { + btn.addEventListener("click", function () { + var sid = this.getAttribute("data-mcp-delete"); + var sname = this.getAttribute("data-mcp-name"); + showConfirmModal( + "Delete MCP Server", + 'Delete server "' + sname + '"?', + "Delete", + function () { + authFetch("/v1/api/admin/mcp-servers/" + sid, { method: "DELETE" }) + .then(function (r) { + if (!r.ok) throw new Error(); + return r.json(); + }) + .then(function () { + showToast("Server deleted"); + loadAdminMcp(); + }) + .catch(function () { + showToast("Failed to delete server"); + }); + }, + ); + }); + }); +} + +function toggleMcpTransport() { + var v = document.getElementById("mcp-transport").value; + document.getElementById("mcp-stdio-fields").style.display = + v === "stdio" ? "" : "none"; + document.getElementById("mcp-http-fields").style.display = + v === "streamable-http" ? "" : "none"; +} + +function showCreateMcpModal() { + _mcpCreateTrigger = document.activeElement; + var ov = document.getElementById("mcp-create-overlay"); + ov.style.display = "flex"; + document.getElementById("mcp-edit-id").value = ""; + document.getElementById("mcp-create-title").textContent = "Add MCP Server"; + document.getElementById("mcp-create-submit").textContent = "Create"; + document.getElementById("mcp-name").value = ""; + document.getElementById("mcp-transport").value = "stdio"; + document.getElementById("mcp-command").value = ""; + document.getElementById("mcp-args").value = ""; + document.getElementById("mcp-env").value = ""; + document.getElementById("mcp-url").value = ""; + document.getElementById("mcp-headers").value = ""; + document.getElementById("mcp-auto-approve").checked = false; + document.getElementById("mcp-enabled").checked = true; + document.getElementById("mcp-create-error").style.display = "none"; + toggleMcpTransport(); + document.getElementById("mcp-name").focus(); + _mcpCreateTrap = _installTrap("mcp-create-overlay", "mcp-create-box"); +} + +function showEditMcpModal(serverId) { + // Fetch with reveal=true to get actual secret values for editing + authFetch("/v1/api/admin/mcp-servers/" + serverId + "?reveal=true") + .then(function (r) { + if (!r.ok) throw new Error("Failed to load server"); + return r.json(); + }) + .then(function (s) { + showCreateMcpModal(); + document.getElementById("mcp-edit-id").value = serverId; + document.getElementById("mcp-create-title").textContent = + "Edit MCP Server"; + document.getElementById("mcp-create-submit").textContent = "Save"; + document.getElementById("mcp-name").value = s.name; + document.getElementById("mcp-transport").value = s.transport; + document.getElementById("mcp-command").value = s.command || ""; + try { + var argsList = JSON.parse(s.args || "[]"); + document.getElementById("mcp-args").value = argsList.join("\n"); + } catch (e) { + document.getElementById("mcp-args").value = ""; + } + try { + var envObj = JSON.parse(s.env || "{}"); + document.getElementById("mcp-env").value = Object.keys(envObj) + .map(function (k) { + return k + "=" + envObj[k]; + }) + .join("\n"); + } catch (e) { + document.getElementById("mcp-env").value = ""; + } + document.getElementById("mcp-url").value = s.url || ""; + try { + var hdrObj = JSON.parse(s.headers || "{}"); + document.getElementById("mcp-headers").value = Object.keys(hdrObj) + .map(function (k) { + return k + ": " + hdrObj[k]; + }) + .join("\n"); + } catch (e) { + document.getElementById("mcp-headers").value = ""; + } + document.getElementById("mcp-auto-approve").checked = + s.auto_approve || false; + document.getElementById("mcp-enabled").checked = s.enabled !== false; + toggleMcpTransport(); + }) + .catch(function () { + showToast("Failed to load server details"); + }); +} + +function hideCreateMcpModal() { + document.getElementById("mcp-create-overlay").style.display = "none"; + _mcpCreateTrap = _removeTrap(_mcpCreateTrap); + if (_mcpCreateTrigger && _mcpCreateTrigger.focus) _mcpCreateTrigger.focus(); + _mcpCreateTrigger = null; +} + +function _parseMcpForm() { + var name = document.getElementById("mcp-name").value.trim(); + var transport = document.getElementById("mcp-transport").value; + if (!name) return { error: "Name is required" }; + if (!/^[a-zA-Z0-9._-]+$/.test(name)) + return { error: "Name must match [a-zA-Z0-9._-]+" }; + if (name.indexOf("__") >= 0) return { error: "Name must not contain '__'" }; + + var payload = { + name: name, + transport: transport, + auto_approve: document.getElementById("mcp-auto-approve").checked, + enabled: document.getElementById("mcp-enabled").checked, + }; + + if (transport === "stdio") { + payload.command = document.getElementById("mcp-command").value.trim(); + var argsText = document.getElementById("mcp-args").value.trim(); + payload.args = argsText + ? argsText + .split("\n") + .map(function (l) { + return l.trim(); + }) + .filter(Boolean) + : []; + var envText = document.getElementById("mcp-env").value.trim(); + var envObj = {}; + if (envText) { + envText.split("\n").forEach(function (line) { + var eq = line.indexOf("="); + if (eq > 0) + envObj[line.substring(0, eq).trim()] = line.substring(eq + 1).trim(); + }); + } + payload.env = envObj; + } else { + payload.url = document.getElementById("mcp-url").value.trim(); + var hdrText = document.getElementById("mcp-headers").value.trim(); + var hdrObj = {}; + if (hdrText) { + hdrText.split("\n").forEach(function (line) { + var colon = line.indexOf(":"); + if (colon > 0) + hdrObj[line.substring(0, colon).trim()] = line + .substring(colon + 1) + .trim(); + }); + } + payload.headers = hdrObj; + } + return payload; +} + +function submitCreateMcp() { + var form = _parseMcpForm(); + if (form.error) { + var e = document.getElementById("mcp-create-error"); + e.textContent = form.error; + e.style.display = ""; + return; + } + var editId = document.getElementById("mcp-edit-id").value; + var method = editId ? "PUT" : "POST"; + var url = editId + ? "/v1/api/admin/mcp-servers/" + editId + : "/v1/api/admin/mcp-servers"; + + document.getElementById("mcp-create-submit").disabled = true; + authFetch(url, { + method: method, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(form), + }) + .then(function (r) { + if (!r.ok) + return r.json().then(function (d) { + throw new Error(d.error || "Failed"); + }); + return r.json(); + }) + .then(function () { + hideCreateMcpModal(); + showToast(editId ? "Server updated" : "Server created"); + loadAdminMcp(); + }) + .catch(function (e) { + var el = document.getElementById("mcp-create-error"); + el.textContent = e.message; + el.style.display = ""; + }) + .finally(function () { + document.getElementById("mcp-create-submit").disabled = false; + }); +} + +function reloadMcpNodes() { + authFetch("/v1/api/admin/mcp-servers/reload", { method: "POST" }) + .then(function (r) { + if (!r.ok) throw new Error(); + return r.json(); + }) + .then(function (data) { + var results = data.results || {}; + var nodeIds = Object.keys(results); + var totalAdded = 0, + totalRemoved = 0; + for (var i = 0; i < nodeIds.length; i++) { + var nr = results[nodeIds[i]]; + totalAdded += (nr.added || []).length; + totalRemoved += (nr.removed || []).length; + } + var msg = "Reload sent to " + nodeIds.length + " node(s)"; + if (totalAdded) msg += ", +" + totalAdded + " added"; + if (totalRemoved) msg += ", -" + totalRemoved + " removed"; + showToast(msg); + setTimeout(loadAdminMcp, 1500); + }) + .catch(function () { + showToast("Failed to reload nodes"); + }); +} + +function showMcpDetailByName(name) { + for (var i = 0; i < _mcpServers.length; i++) { + if (_mcpServers[i].name === name) { + return _openMcpDetail(_mcpServers[i]); + } + } +} + +function showMcpDetailModal(serverId) { + for (var i = 0; i < _mcpServers.length; i++) { + if (_mcpServers[i].server_id === serverId) { + return _openMcpDetail(_mcpServers[i]); + } + } +} + +function _openMcpDetail(s) { + if (!s) return; + _mcpDetailTrigger = document.activeElement; + + var html = '
'; + html += '
'; + html += '

Configuration

'; + html += + '

Transport: ' + + escapeHtml(s.transport) + + "

"; + if (s.transport === "stdio") { + html += + '

Command: ' + + escapeHtml(s.command || "") + + "

"; + try { + var a = JSON.parse(s.args || "[]"); + if (a.length) + html += + '

Args: ' + + escapeHtml(a.join(" ")) + + "

"; + } catch (e) {} + } else { + html += + '

URL: ' + + escapeHtml(s.url || "") + + "

"; + } + html += "
"; + + html += '
'; + var statusEntries = s.status || {}; + var nodeIds = Object.keys(statusEntries); + html += '

Node Status

'; + if (nodeIds.length === 0) { + html += + '

Not connected on any node

'; + } else { + html += '
    '; + for (var j = 0; j < nodeIds.length; j++) { + var ns = statusEntries[nodeIds[j]]; + var dot = ns.connected + ? '' + : ''; + html += + "
  • " + + dot + + escapeHtml(nodeIds[j]) + + " — " + + (ns.tools || 0) + + " tools, " + + (ns.resources || 0) + + " resources, " + + (ns.prompts || 0) + + " prompts
    • "; + } + html += "
"; + } + html += "
"; + + document.getElementById("mcp-detail-title").textContent = s.name; + document.getElementById("mcp-detail-content").innerHTML = html; + document.getElementById("mcp-detail-overlay").style.display = "flex"; + _mcpDetailTrap = _installTrap("mcp-detail-overlay", "mcp-detail-box"); +} + +function hideMcpDetailModal() { + document.getElementById("mcp-detail-overlay").style.display = "none"; + _mcpDetailTrap = _removeTrap(_mcpDetailTrap); + if (_mcpDetailTrigger && _mcpDetailTrigger.focus) _mcpDetailTrigger.focus(); + _mcpDetailTrigger = null; +} + +function showImportMcpModal() { + _mcpImportTrigger = document.activeElement; + document.getElementById("mcp-import-overlay").style.display = "flex"; + document.getElementById("mcp-import-json").value = ""; + document.getElementById("mcp-import-error").style.display = "none"; + document.getElementById("mcp-import-json").focus(); + _mcpImportTrap = _installTrap("mcp-import-overlay", "mcp-import-box"); +} + +function hideImportMcpModal() { + document.getElementById("mcp-import-overlay").style.display = "none"; + _mcpImportTrap = _removeTrap(_mcpImportTrap); + if (_mcpImportTrigger && _mcpImportTrigger.focus) _mcpImportTrigger.focus(); + _mcpImportTrigger = null; +} + +function submitImportMcp() { + var raw = document.getElementById("mcp-import-json").value.trim(); + if (!raw) { + var e = document.getElementById("mcp-import-error"); + e.textContent = "Paste a JSON config"; + e.style.display = ""; + return; + } + var parsed; + try { + parsed = JSON.parse(raw); + } catch (ex) { + var e2 = document.getElementById("mcp-import-error"); + e2.textContent = "Invalid JSON: " + ex.message; + e2.style.display = ""; + return; + } + if (!parsed.mcpServers || typeof parsed.mcpServers !== "object") { + var e3 = document.getElementById("mcp-import-error"); + e3.textContent = 'No "mcpServers" key found in JSON'; + e3.style.display = ""; + return; + } + document.getElementById("mcp-import-submit").disabled = true; + authFetch("/v1/api/admin/mcp-servers/import", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ config: parsed }), + }) + .then(function (r) { + if (!r.ok) + return r.json().then(function (d) { + throw new Error(d.error || "Failed"); + }); + return r.json(); + }) + .then(function (data) { + hideImportMcpModal(); + var msg = "Imported " + (data.imported || []).length; + if ((data.skipped || []).length) + msg += ", skipped " + data.skipped.length; + if ((data.errors || []).length) + msg += ", " + data.errors.length + " error(s)"; + showToast(msg); + loadAdminMcp(); + }) + .catch(function (e) { + var el = document.getElementById("mcp-import-error"); + el.textContent = e.message; + el.style.display = ""; + }) + .finally(function () { + document.getElementById("mcp-import-submit").disabled = false; + }); +} diff --git a/turnstone/console/static/governance.js b/turnstone/console/static/governance.js index 1a2b6c6c..913dc6e7 100644 --- a/turnstone/console/static/governance.js +++ b/turnstone/console/static/governance.js @@ -152,10 +152,15 @@ var _ALL_PERMISSIONS = [ "admin.orgs", "admin.policies", "admin.templates", + "admin.ws_templates", "admin.audit", "admin.usage", "admin.schedules", "admin.watches", + "admin.judge", + "admin.memories", + "admin.settings", + "admin.mcp", "tools.approve", "workstreams.create", "workstreams.close", diff --git a/turnstone/console/static/index.html b/turnstone/console/static/index.html index 2ded62cc..cd5ab8c6 100644 --- a/turnstone/console/static/index.html +++ b/turnstone/console/static/index.html @@ -107,6 +107,7 @@
+
@@ -393,6 +394,27 @@ + + @@ -972,6 +994,67 @@ window.TURNSTONE_KB_SHORTCUTS = [ + + + + + + diff --git a/turnstone/console/static/style.css b/turnstone/console/static/style.css index 670e5158..094fa202 100644 --- a/turnstone/console/static/style.css +++ b/turnstone/console/static/style.css @@ -1173,7 +1173,8 @@ #create-policy-overlay, #edit-policy-overlay, #create-template-overlay, #edit-template-overlay, #create-wst-overlay, #edit-wst-overlay, #wst-history-overlay, -#memory-detail-overlay { +#memory-detail-overlay, +#mcp-create-overlay, #mcp-import-overlay, #mcp-detail-overlay { position: fixed; inset: 0; background: rgba(0, 0, 0, 0.7); @@ -1851,6 +1852,45 @@ .settings-input select { max-width: 100%; } } +/* -- MCP Servers grid ----------------------------------------------------- */ +.admin-col-mname a{color:var(--fg);text-decoration:none;transition:color .15s} +.admin-col-mname a:hover{color:var(--magenta)} +.admin-col-mname a:focus-visible{outline:2px solid var(--magenta);outline-offset:2px} +.mcp-grid{grid-template-columns:1.5fr 80px 55px 45px 80px 95px 120px;gap:0 6px} +@media(max-width:700px){ + .mcp-grid{grid-template-columns:1fr 100px 130px} + .admin-col-mtransport,.admin-col-mtools,.admin-col-mres,.admin-col-mprompts{display:none} +} + +.mcp-status-dot{display:inline-block;width:8px;height:8px;border-radius:50%;vertical-align:middle;margin-right:6px} +.mcp-status-dot.connected{background:var(--magenta);box-shadow:0 0 6px var(--magenta-glow, rgba(192,132,252,.45))} +.mcp-status-dot.error{background:var(--red);box-shadow:0 0 6px var(--red-glow);border-radius:1px} +.mcp-status-dot.disabled{background:var(--fg-dim);opacity:.35} +.mcp-status-dot.connecting{background:var(--magenta);animation:mcp-pulse 1.2s ease-in-out infinite} +@keyframes mcp-pulse{0%,100%{opacity:.3}50%{opacity:1}} + +.mcp-row-connected{border-left:3px solid var(--magenta)} +.mcp-row-error{border-left:3px solid var(--red)} +.mcp-row-disabled{border-left:3px solid transparent} + +.mcp-transport-badge{display:inline-block;font-size:9px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;padding:1px 6px;border-radius:2px;background:var(--bg-highlight);border:1px solid var(--border)} +.mcp-transport-stdio{color:var(--cyan);border-color:rgba(103,232,249,.2)} +.mcp-transport-http{color:var(--magenta);border-color:rgba(192,132,252,.25)} + +.admin-col-mtools,.admin-col-mres,.admin-col-mprompts{text-align:right;font-variant-numeric:tabular-nums} +.mcp-count-dim{opacity:.4} + +.mcp-detail-modal::before{background:linear-gradient(90deg,transparent,var(--magenta),transparent)!important} +.mcp-detail-modal h2{color:var(--magenta)!important} +.mcp-detail-section{margin-top:16px} +.mcp-detail-section h3{font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.08em;color:var(--magenta);margin-bottom:8px} +.mcp-detail-list{list-style:none;padding:0;margin:0} +.mcp-detail-list li{font-size:12px;padding:3px 0;border-bottom:1px solid var(--border);color:var(--fg-dim)} +.mcp-detail-list li:last-child{border-bottom:none} + +.admin-action-btn-ghost{background:transparent;color:var(--fg-dim);border:1px solid var(--border-strong)} +.admin-action-btn-ghost:hover{color:var(--fg);background:var(--bg-highlight)} + /* ========================================================================== Reduced motion — console-specific ========================================================================== */ @@ -1868,4 +1908,5 @@ #view-admin { animation: none; } .admin-action-btn, .modal-cancel, .modal-submit { transition: none; } .admin-modal input, .admin-modal select { transition: none; } + .mcp-status-dot.connecting { animation: none; } } diff --git a/turnstone/core/auth.py b/turnstone/core/auth.py index f8140e05..3bd01853 100644 --- a/turnstone/core/auth.py +++ b/turnstone/core/auth.py @@ -167,7 +167,9 @@ WRITE_PATHS: frozenset[str] = frozenset( } ) -APPROVE_PATHS: frozenset[str] = frozenset({"/api/approve", "/api/_internal/config-reload"}) +APPROVE_PATHS: frozenset[str] = frozenset( + {"/api/approve", "/api/_internal/config-reload", "/api/_internal/mcp-reload"} +) ADMIN_PREFIX = "/api/admin/" diff --git a/turnstone/core/mcp_client.py b/turnstone/core/mcp_client.py index 496c3fdf..96ec6abb 100644 --- a/turnstone/core/mcp_client.py +++ b/turnstone/core/mcp_client.py @@ -97,6 +97,7 @@ class MCPClientManager: self._loop: asyncio.AbstractEventLoop | None = None self._thread: threading.Thread | None = None self._exit_stack: AsyncExitStack | None = None + self._per_server_stacks: dict[str, AsyncExitStack] = {} self._sessions: dict[str, Any] = {} self._tools: list[dict[str, Any]] = [] @@ -104,6 +105,10 @@ class MCPClientManager: self._tool_map: dict[str, tuple[str, str]] = {} self._connected = threading.Event() self._error: str | None = None + # Names managed by the DB (added via reconcile_sync / add_server_sync). + # Config-file servers loaded at startup are NOT in this set and + # will never be removed by reconcile_sync. + self._db_managed: set[str] = set() # Per-server tool storage for surgical refresh self._per_server_tools: dict[str, list[dict[str, Any]]] = {} @@ -189,30 +194,37 @@ class MCPClientManager: async def _connect_one(self, name: str, cfg: dict[str, Any]) -> None: """Connect to a single MCP server and discover its tools.""" - assert self._exit_stack is not None - if "__" in name: log.error("MCP server name '%s' contains '__' (reserved delimiter), skipping", name) return + # Per-server exit stack for clean per-server lifecycle management + stack = AsyncExitStack() + await stack.__aenter__() + transport = cfg.get("type", "stdio") - if transport in ("http", "streamable-http") or "url" in cfg: - read, write, _ = await self._exit_stack.enter_async_context( - streamablehttp_client(url=cfg["url"], headers=cfg.get("headers")) - ) - else: - # Default: stdio transport - command = cfg.get("command", "") - if not command: - log.warning("MCP server '%s' has no command configured", name) - return - env = {**os.environ, **cfg.get("env", {})} - params = StdioServerParameters( - command=command, - args=cfg.get("args", []), - env=env, - ) - read, write = await self._exit_stack.enter_async_context(stdio_client(params)) + try: + if transport in ("http", "streamable-http") or "url" in cfg: + read, write, _ = await stack.enter_async_context( + streamablehttp_client(url=cfg["url"], headers=cfg.get("headers")) + ) + else: + # Default: stdio transport + command = cfg.get("command", "") + if not command: + log.warning("MCP server '%s' has no command configured", name) + await stack.aclose() + return + env = {**os.environ, **cfg.get("env", {})} + params = StdioServerParameters( + command=command, + args=cfg.get("args", []), + env=env, + ) + read, write = await stack.enter_async_context(stdio_client(params)) + except Exception: + await stack.aclose() + raise # Register notification handler — dispatches tool, resource, and # prompt list-change notifications to the appropriate refresh method. @@ -235,10 +247,22 @@ class MCPClientManager: except Exception: log.warning("Refresh after notification failed for '%s'", name, exc_info=True) - session = await self._exit_stack.enter_async_context( - ClientSession(read, write, message_handler=_on_notification) # type: ignore[arg-type] - ) - await session.initialize() + try: + session = await stack.enter_async_context( + ClientSession(read, write, message_handler=_on_notification) # type: ignore[arg-type] + ) + except Exception: + await stack.aclose() + raise + + self._per_server_stacks[name] = stack + try: + await session.initialize() + except Exception: + self._per_server_stacks.pop(name, None) + with contextlib.suppress(Exception): + await stack.aclose() + raise self._sessions[name] = session # Check push notification support for each capability @@ -807,12 +831,27 @@ class MCPClientManager: if self._refresh_task and self._loop: self._loop.call_soon_threadsafe(self._refresh_task.cancel) + # Close all per-server stacks (transports + sessions) + if self._loop and self._per_server_stacks: + + async def _close_all_stacks() -> None: + for stack in self._per_server_stacks.values(): + with contextlib.suppress(Exception): + await stack.aclose() + + future = asyncio.run_coroutine_threadsafe(_close_all_stacks(), self._loop) + try: + future.result(timeout=10) + except Exception: + log.debug("Error closing MCP sessions", exc_info=True) + + # Close legacy shared stack (if any resources were registered on it) if self._loop and self._exit_stack: future = asyncio.run_coroutine_threadsafe(self._exit_stack.aclose(), self._loop) try: future.result(timeout=10) except Exception: - log.debug("Error closing MCP sessions", exc_info=True) + log.debug("Error closing MCP exit stack", exc_info=True) if self._loop: self._loop.call_soon_threadsafe(self._loop.stop) @@ -821,6 +860,8 @@ class MCPClientManager: # Clear all state self._sessions.clear() + self._per_server_stacks.clear() + self._db_managed.clear() self._tools = [] self._tool_map = {} self._per_server_tools.clear() @@ -843,6 +884,209 @@ class MCPClientManager: log.info("MCP client shut down") + # -- hot-reload (add/remove servers) ------------------------------------ + + def add_server_sync(self, name: str, cfg: dict[str, Any], timeout: int = 30) -> dict[str, Any]: + """Connect a new MCP server at runtime (blocks the calling thread). + + Returns status dict with keys: connected, tools, resources, prompts, error. + """ + if "__" in name: + return { + "connected": False, + "tools": 0, + "resources": 0, + "prompts": 0, + "error": f"Server name '{name}' contains '__' (reserved delimiter)", + } + if self._loop is None: + return { + "connected": False, + "tools": 0, + "resources": 0, + "prompts": 0, + "error": "MCP event loop not running", + } + + # Add to config so _refresh_all can reconnect on failure + self._server_configs[name] = cfg + + future = asyncio.run_coroutine_threadsafe(self._connect_one(name, cfg), self._loop) + try: + future.result(timeout=timeout) + except Exception as exc: + # Remove from configs on failure + self._server_configs.pop(name, None) + return {"connected": False, "tools": 0, "resources": 0, "prompts": 0, "error": str(exc)} + + return { + "connected": name in self._sessions, + "tools": len(self._per_server_tools.get(name, [])), + "resources": len(self._per_server_resources.get(name, [])), + "prompts": len(self._per_server_prompts.get(name, [])), + "error": "", + } + + def remove_server_sync(self, name: str, timeout: int = 15) -> bool: + """Disconnect and remove an MCP server at runtime (blocks the calling thread). + + All state mutations run on the MCP event loop thread to avoid races + with notification handlers and refresh tasks. + + Returns True if the server was connected and successfully removed. + """ + was_connected = name in self._sessions + + # Remove from config to prevent reconnection + self._server_configs.pop(name, None) + + if self._loop is not None: + + async def _remove() -> None: + # Close session + transport via per-server stack + self._sessions.pop(name, None) + stack = self._per_server_stacks.pop(name, None) + if stack is not None: + with contextlib.suppress(Exception): + await stack.aclose() + # Clean up per-server state (on the event loop thread) + self._per_server_tools.pop(name, None) + self._per_server_resources.pop(name, None) + self._per_server_prompts.pop(name, None) + self._supports_list_changed.pop(name, None) + self._supports_resources.pop(name, None) + self._supports_resource_list_changed.pop(name, None) + self._supports_prompts.pop(name, None) + self._supports_prompt_list_changed.pop(name, None) + # Rebuild merged state (serialized with notification handlers) + self._rebuild_tools() + self._rebuild_resources() + self._rebuild_prompts() + + future = asyncio.run_coroutine_threadsafe(_remove(), self._loop) + try: + future.result(timeout=timeout) + except Exception: + log.warning("Error removing MCP server '%s'", name, exc_info=True) + else: + # No event loop (tests / pre-start) — mutate directly + self._sessions.pop(name, None) + self._per_server_tools.pop(name, None) + self._per_server_resources.pop(name, None) + self._per_server_prompts.pop(name, None) + self._supports_list_changed.pop(name, None) + self._supports_resources.pop(name, None) + self._supports_resource_list_changed.pop(name, None) + self._supports_prompts.pop(name, None) + self._supports_prompt_list_changed.pop(name, None) + self._rebuild_tools() + self._rebuild_resources() + self._rebuild_prompts() + + # Clean up governance templates from this server + try: + self.sync_prompts_to_storage() + except Exception: + log.warning("Prompt sync after remove failed for '%s'", name, exc_info=True) + + log.info("Removed MCP server '%s'", name) + return was_connected + + def get_server_status(self, name: str) -> dict[str, Any]: + """Return live status for a single server, including config details.""" + connected = name in self._sessions + cfg = self._server_configs.get(name, {}) + transport = cfg.get("type", "stdio") + return { + "connected": connected, + "tools": len(self._per_server_tools.get(name, [])) if connected else 0, + "resources": len(self._per_server_resources.get(name, [])) if connected else 0, + "prompts": len(self._per_server_prompts.get(name, [])) if connected else 0, + "error": "", + "transport": transport, + "command": cfg.get("command", "") if transport == "stdio" else "", + "url": cfg.get("url", "") if transport != "stdio" else "", + } + + def get_all_server_status(self) -> dict[str, dict[str, Any]]: + """Return live status for all configured servers.""" + result: dict[str, dict[str, Any]] = {} + for name in list(self._server_configs): + result[name] = self.get_server_status(name) + return result + + def reconcile_sync(self, storage: Any, timeout: int = 30) -> dict[str, Any]: + """Reconcile DB-managed servers against DB state. + + Reads enabled ``mcp_servers`` rows from *storage*, then: + - Connects servers in DB but not currently running. + - Disconnects DB-managed servers no longer in DB (or disabled). + - Reconnects DB-managed servers whose config has changed. + + Config-file servers (loaded at startup, not in ``_db_managed``) + are never touched — only servers previously added via DB are + eligible for removal. + + Returns ``{"added": [...], "removed": [...], "updated": [...]}``. + """ + try: + rows = storage.list_mcp_servers(enabled_only=True) + except Exception: + log.warning("reconcile_sync: failed to read mcp_servers table", exc_info=True) + return {"added": [], "removed": [], "updated": []} + + desired = _db_servers_to_config(rows) + desired_names = set(desired) + + added: list[str] = [] + removed: list[str] = [] + updated: list[str] = [] + + # Remove DB-managed servers no longer in DB (or disabled). + # Config-file servers (not in _db_managed) are left untouched. + for name in list(self._db_managed - desired_names): + self.remove_server_sync(name, timeout=timeout) + self._db_managed.discard(name) + removed.append(name) + + # Add servers in DB but not running + for name in desired_names - set(self._server_configs): + result = self.add_server_sync(name, desired[name], timeout=timeout) + if result.get("connected"): + added.append(name) + self._db_managed.add(name) + else: + log.warning("reconcile_sync: failed to add '%s': %s", name, result.get("error", "")) + + # Update DB-managed servers whose config has changed (cycle: remove + add). + # Config-file servers with the same name as a DB server are left untouched. + for name in desired_names & set(self._server_configs): + if name not in self._db_managed: + continue # config-file server — DB doesn't own it + if desired[name] != self._server_configs.get(name): + log.info("Config changed for MCP server '%s', reconnecting", name) + self.remove_server_sync(name, timeout=timeout) + result = self.add_server_sync(name, desired[name], timeout=timeout) + if result.get("connected"): + updated.append(name) + self._db_managed.add(name) + else: + self._db_managed.discard(name) + log.warning( + "reconcile_sync: failed to reconnect '%s': %s", + name, + result.get("error", ""), + ) + + if added or removed or updated: + log.info( + "MCP reconcile: +%d added, -%d removed, ~%d updated", + len(added), + len(removed), + len(updated), + ) + return {"added": added, "removed": removed, "updated": updated} + # -- query methods ------------------------------------------------------- def get_tools(self) -> list[dict[str, Any]]: @@ -1024,23 +1268,64 @@ class MCPClientManager: # --------------------------------------------------------------------------- -def load_mcp_config(config_path: str | None = None) -> dict[str, dict[str, Any]]: +def _db_servers_to_config(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Convert mcp_servers DB rows to the config dict format.""" + result: dict[str, dict[str, Any]] = {} + for row in rows: + name = row["name"] + cfg: dict[str, Any] = {"type": row["transport"]} + if row["transport"] == "stdio": + cfg["command"] = row.get("command", "") + try: + cfg["args"] = json.loads(row.get("args", "[]")) + except (json.JSONDecodeError, TypeError): + cfg["args"] = [] + try: + cfg["env"] = json.loads(row.get("env", "{}")) + except (json.JSONDecodeError, TypeError): + cfg["env"] = {} + else: + cfg["url"] = row.get("url", "") + try: + cfg["headers"] = json.loads(row.get("headers", "{}")) + except (json.JSONDecodeError, TypeError): + cfg["headers"] = {} + result[name] = cfg + return result + + +def load_mcp_config( + config_path: str | None = None, + storage: Any = None, +) -> dict[str, dict[str, Any]]: """Load MCP server configurations. Sources (first match wins): - 1. Explicit *config_path* (standard MCP JSON format). - 2. ``[mcp.servers.*]`` sections in ``config.toml``. + 1. DB ``mcp_servers`` table (if *storage* provided and has enabled rows). + 2. Explicit *config_path* (standard MCP JSON format). + 3. ``[mcp.servers.*]`` sections in ``config.toml``. Returns an empty dict if nothing is configured. """ - # 1. Explicit JSON file + # 1. Database + if storage is not None: + try: + rows = storage.list_mcp_servers(enabled_only=True) + if rows: + servers = _db_servers_to_config(rows) + log.info("Loaded MCP config from database (%d server(s))", len(servers)) + return servers + except Exception: + log.debug("DB MCP config lookup failed (table may not exist yet)", exc_info=True) + + # 2. Explicit JSON file if config_path: path = Path(config_path).expanduser() if path.is_file(): try: data = json.loads(path.read_text(encoding="utf-8")) - servers: dict[str, Any] = data.get("mcpServers", {}) + servers = data.get("mcpServers", {}) if isinstance(servers, dict) and servers: log.info("Loaded MCP config from %s (%d server(s))", path, len(servers)) return servers @@ -1049,7 +1334,7 @@ def load_mcp_config(config_path: str | None = None) -> dict[str, dict[str, Any]] else: log.warning("MCP config file not found: %s", path) - # 2. TOML config + # 3. TOML config mcp_section = load_config("mcp") servers_section = mcp_section.get("servers", {}) @@ -1069,15 +1354,28 @@ def create_mcp_client( config_path: str | None = None, *, refresh_interval: float = _DEFAULT_REFRESH_INTERVAL, + storage: Any = None, ) -> MCPClientManager | None: """Create and start an MCP client manager. Returns *None* if no servers are configured. """ - servers = load_mcp_config(config_path) + # Check DB first to know which servers are DB-managed + db_names: set[str] = set() + if storage is not None: + try: + rows = storage.list_mcp_servers(enabled_only=True) + if rows: + db_names = {r["name"] for r in rows} + except Exception: + pass + + servers = load_mcp_config(config_path, storage=storage) if not servers: return None mgr = MCPClientManager(servers, refresh_interval=refresh_interval) + # Mark DB-sourced servers so reconcile_sync won't remove config-file servers + mgr._db_managed = {name for name in servers if name in db_names} mgr.start() return mgr diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index b3b01a3a..87b7e8ad 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -229,7 +229,8 @@ def _build_registry() -> dict[str, SettingDef]: "mcp", restart_required=True, help="Model Context Protocol (MCP) lets the AI connect to external tool servers. " - "This points to a JSON file listing which MCP servers to connect to on startup.", + "This points to a JSON file listing which MCP servers to connect to on startup. " + "Tip: use the MCP Servers tab to manage servers via the database instead.", reference_url="https://modelcontextprotocol.io", ), SettingDef( diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index 29f65c2e..18a0cd29 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -14,6 +14,7 @@ from turnstone.core.storage._schema import ( audit_events, conversations, intent_verdicts, + mcp_servers, metadata, orgs, prompt_templates, @@ -29,6 +30,9 @@ from turnstone.core.storage._schema import ( workstream_templates, workstreams, ) +from turnstone.core.storage._utils import ( + MCP_SERVER_MUTABLE as _MCP_SERVER_MUTABLE, +) from turnstone.core.storage._utils import ( ORG_MUTABLE as _ORG_MUTABLE, ) @@ -2364,6 +2368,98 @@ class PostgreSQLBackend: ).fetchall() return {r.key: r.value for r in rows} + # -- MCP server definitions ------------------------------------------------ + + def create_mcp_server( + self, + server_id: str, + name: str, + transport: str, + command: str = "", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + auto_approve: bool = False, + enabled: bool = True, + created_by: str = "", + ) -> None: + from sqlalchemy.dialects import postgresql + + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + with self._engine.connect() as conn: + conn.execute( + postgresql.insert(mcp_servers) + .values( + server_id=server_id, + name=name, + transport=transport, + command=command, + args=args, + url=url, + headers=headers, + env=env, + auto_approve=1 if auto_approve else 0, + enabled=1 if enabled else 0, + created_by=created_by, + created=now, + updated=now, + ) + .on_conflict_do_nothing() + ) + conn.commit() + + def get_mcp_server(self, server_id: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute( + sa.select(mcp_servers).where(mcp_servers.c.server_id == server_id) + ).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def get_mcp_server_by_name(self, name: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute(sa.select(mcp_servers).where(mcp_servers.c.name == name)).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + + with self._engine.connect() as conn: + q = sa.select(mcp_servers).order_by(mcp_servers.c.name) + if enabled_only: + q = q.where(mcp_servers.c.enabled == 1) + rows = conn.execute(q).fetchall() + return [_row_to_dict(r, "auto_approve", "enabled") for r in rows] + + def update_mcp_server(self, server_id: str, **fields: Any) -> bool: + + fields = {k: v for k, v in fields.items() if k in _MCP_SERVER_MUTABLE} + fields["updated"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + if "auto_approve" in fields: + fields["auto_approve"] = 1 if fields["auto_approve"] else 0 + if "enabled" in fields: + fields["enabled"] = 1 if fields["enabled"] else 0 + with self._engine.connect() as conn: + result = conn.execute( + sa.update(mcp_servers).where(mcp_servers.c.server_id == server_id).values(**fields) + ) + conn.commit() + return result.rowcount > 0 + + def delete_mcp_server(self, server_id: str) -> bool: + + with self._engine.connect() as conn: + result = conn.execute( + sa.delete(mcp_servers).where(mcp_servers.c.server_id == server_id) + ) + conn.commit() + return result.rowcount > 0 + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 6af64722..b1adb9c6 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -784,6 +784,45 @@ class StorageBackend(Protocol): """ ... + # -- MCP server definitions ------------------------------------------------ + + def create_mcp_server( + self, + server_id: str, + name: str, + transport: str, + command: str = "", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + auto_approve: bool = False, + enabled: bool = True, + created_by: str = "", + ) -> None: + """Create an MCP server definition. No-op if server_id already exists.""" + ... + + def get_mcp_server(self, server_id: str) -> dict[str, Any] | None: + """Return MCP server dict or None.""" + ... + + def get_mcp_server_by_name(self, name: str) -> dict[str, Any] | None: + """Return MCP server dict by name or None.""" + ... + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + """Return MCP servers ordered by name.""" + ... + + def update_mcp_server(self, server_id: str, **fields: Any) -> bool: + """Update specified fields on an MCP server. Returns True if found.""" + ... + + def delete_mcp_server(self, server_id: str) -> bool: + """Delete an MCP server definition. Returns True if existed.""" + ... + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_schema.py b/turnstone/core/storage/_schema.py index 9ac62b42..2a8a2406 100644 --- a/turnstone/core/storage/_schema.py +++ b/turnstone/core/storage/_schema.py @@ -445,3 +445,27 @@ system_settings = sa.Table( ) sa.Index("idx_system_settings_node", system_settings.c.node_id) + +# --------------------------------------------------------------------------- +# MCP server definitions — database-backed MCP configuration +# --------------------------------------------------------------------------- + +mcp_servers = sa.Table( + "mcp_servers", + metadata, + sa.Column("server_id", sa.Text, primary_key=True), + sa.Column("name", sa.Text, nullable=False, unique=True), + sa.Column("transport", sa.Text, nullable=False), # "stdio" | "streamable-http" + sa.Column("command", sa.Text, nullable=False, server_default=""), + sa.Column("args", sa.Text, nullable=False, server_default="[]"), # JSON array + sa.Column("url", sa.Text, nullable=False, server_default=""), + sa.Column("headers", sa.Text, nullable=False, server_default="{}"), # JSON object + sa.Column("env", sa.Text, nullable=False, server_default="{}"), # JSON object + sa.Column("auto_approve", sa.Integer, nullable=False, server_default="0"), + sa.Column("enabled", sa.Integer, nullable=False, server_default="1"), + sa.Column("created_by", sa.Text, nullable=False, server_default=""), + sa.Column("created", sa.Text, nullable=False), + sa.Column("updated", sa.Text, nullable=False), +) + +sa.Index("idx_mcp_servers_enabled", mcp_servers.c.enabled) diff --git a/turnstone/core/storage/_sqlite.py b/turnstone/core/storage/_sqlite.py index a8b4a758..364c02b3 100644 --- a/turnstone/core/storage/_sqlite.py +++ b/turnstone/core/storage/_sqlite.py @@ -14,6 +14,7 @@ from turnstone.core.storage._schema import ( audit_events, conversations, intent_verdicts, + mcp_servers, metadata, orgs, prompt_templates, @@ -29,6 +30,9 @@ from turnstone.core.storage._schema import ( workstream_templates, workstreams, ) +from turnstone.core.storage._utils import ( + MCP_SERVER_MUTABLE as _MCP_SERVER_MUTABLE, +) from turnstone.core.storage._utils import ( ORG_MUTABLE as _ORG_MUTABLE, ) @@ -2389,6 +2393,96 @@ class SQLiteBackend: ).fetchall() return {r.key: r.value for r in rows} + # -- MCP server definitions ------------------------------------------------ + + def create_mcp_server( + self, + server_id: str, + name: str, + transport: str, + command: str = "", + args: str = "[]", + url: str = "", + headers: str = "{}", + env: str = "{}", + auto_approve: bool = False, + enabled: bool = True, + created_by: str = "", + ) -> None: + + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + with self._engine.connect() as conn: + conn.execute( + sa.insert(mcp_servers).prefix_with("OR IGNORE"), + { + "server_id": server_id, + "name": name, + "transport": transport, + "command": command, + "args": args, + "url": url, + "headers": headers, + "env": env, + "auto_approve": 1 if auto_approve else 0, + "enabled": 1 if enabled else 0, + "created_by": created_by, + "created": now, + "updated": now, + }, + ) + conn.commit() + + def get_mcp_server(self, server_id: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute( + sa.select(mcp_servers).where(mcp_servers.c.server_id == server_id) + ).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def get_mcp_server_by_name(self, name: str) -> dict[str, Any] | None: + + with self._engine.connect() as conn: + row = conn.execute(sa.select(mcp_servers).where(mcp_servers.c.name == name)).fetchone() + if row is None: + return None + return _row_to_dict(row, "auto_approve", "enabled") + + def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]: + + with self._engine.connect() as conn: + q = sa.select(mcp_servers).order_by(mcp_servers.c.name) + if enabled_only: + q = q.where(mcp_servers.c.enabled == 1) + rows = conn.execute(q).fetchall() + return [_row_to_dict(r, "auto_approve", "enabled") for r in rows] + + def update_mcp_server(self, server_id: str, **fields: Any) -> bool: + + fields = {k: v for k, v in fields.items() if k in _MCP_SERVER_MUTABLE} + fields["updated"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + if "auto_approve" in fields: + fields["auto_approve"] = 1 if fields["auto_approve"] else 0 + if "enabled" in fields: + fields["enabled"] = 1 if fields["enabled"] else 0 + with self._engine.connect() as conn: + result = conn.execute( + sa.update(mcp_servers).where(mcp_servers.c.server_id == server_id).values(**fields) + ) + conn.commit() + return result.rowcount > 0 + + def delete_mcp_server(self, server_id: str) -> bool: + + with self._engine.connect() as conn: + result = conn.execute( + sa.delete(mcp_servers).where(mcp_servers.c.server_id == server_id) + ) + conn.commit() + return result.rowcount > 0 + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_utils.py b/turnstone/core/storage/_utils.py index d6864e34..15495db3 100644 --- a/turnstone/core/storage/_utils.py +++ b/turnstone/core/storage/_utils.py @@ -48,6 +48,19 @@ WS_TEMPLATE_MUTABLE = frozenset( } ) STRUCTURED_MEMORY_MUTABLE = frozenset({"content", "description", "type"}) +MCP_SERVER_MUTABLE = frozenset( + { + "name", + "transport", + "command", + "args", + "url", + "headers", + "env", + "auto_approve", + "enabled", + } +) VERDICT_MUTABLE = frozenset( { "user_decision", diff --git a/turnstone/core/storage/migrations/versions/016_mcp_servers.py b/turnstone/core/storage/migrations/versions/016_mcp_servers.py new file mode 100644 index 00000000..3f2d2cb4 --- /dev/null +++ b/turnstone/core/storage/migrations/versions/016_mcp_servers.py @@ -0,0 +1,55 @@ +"""Create mcp_servers table and grant admin.mcp permission. + +Revision ID: 016 +Revises: 015 +Create Date: 2026-03-14 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "016" +down_revision = "015" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "mcp_servers", + sa.Column("server_id", sa.Text, primary_key=True), + sa.Column("name", sa.Text, nullable=False, unique=True), + sa.Column("transport", sa.Text, nullable=False), + sa.Column("command", sa.Text, nullable=False, server_default=""), + sa.Column("args", sa.Text, nullable=False, server_default="[]"), + sa.Column("url", sa.Text, nullable=False, server_default=""), + sa.Column("headers", sa.Text, nullable=False, server_default="{}"), + sa.Column("env", sa.Text, nullable=False, server_default="{}"), + sa.Column("auto_approve", sa.Integer, nullable=False, server_default="0"), + sa.Column("enabled", sa.Integer, nullable=False, server_default="1"), + sa.Column("created_by", sa.Text, nullable=False, server_default=""), + sa.Column("created", sa.Text, nullable=False), + sa.Column("updated", sa.Text, nullable=False), + ) + op.create_index("idx_mcp_servers_enabled", "mcp_servers", ["enabled"]) + + # Grant admin.mcp permission to the built-in admin role + conn = op.get_bind() + conn.execute( + sa.text( + "UPDATE roles SET permissions = permissions || ',admin.mcp' " + "WHERE role_id = 'builtin-admin' " + "AND permissions NOT LIKE '%admin.mcp%'" + ) + ) + + +def downgrade() -> None: + conn = op.get_bind() + conn.execute( + sa.text( + "UPDATE roles SET permissions = REPLACE(permissions, ',admin.mcp', '') " + "WHERE role_id = 'builtin-admin'" + ) + ) + op.drop_table("mcp_servers") diff --git a/turnstone/sdk/console.py b/turnstone/sdk/console.py index d472b19a..69dac665 100644 --- a/turnstone/sdk/console.py +++ b/turnstone/sdk/console.py @@ -21,8 +21,10 @@ from turnstone.api.console_schemas import ( ClusterWorkstreamsResponse, ConsoleCreateWsResponse, ConsoleHealthResponse, + ImportMcpConfigResponse, ListAdminMemoriesResponse, ListAuditEventsResponse, + ListMcpServersResponse, ListOrgsResponse, ListPromptTemplatesResponse, ListRolesResponse, @@ -32,6 +34,7 @@ from turnstone.api.console_schemas import ( ListUserRolesResponse, ListWsTemplatesResponse, ListWsTemplateVersionsResponse, + McpServerDetail, NodeDetailResponse, OrgInfo, PromptTemplateInfo, @@ -666,6 +669,98 @@ class AsyncTurnstoneConsole(_BaseClient): "DELETE", f"/v1/api/admin/settings/{key}", params=params, response_model=StatusResponse ) + # -- MCP servers ------------------------------------------------------- + + async def list_mcp_servers(self, reveal: bool = False) -> ListMcpServersResponse: + """List MCP server definitions with live status.""" + params: dict[str, str] = {} + if reveal: + params["reveal"] = "true" + return await self._request( + "GET", + "/v1/api/admin/mcp-servers", + params=params, + response_model=ListMcpServersResponse, + ) + + async def create_mcp_server( + self, + name: str, + transport: str, + *, + command: str = "", + args: list[str] | None = None, + url: str = "", + headers: dict[str, str] | None = None, + env: dict[str, str] | None = None, + auto_approve: bool = False, + enabled: bool = True, + ) -> McpServerDetail: + """Create an MCP server definition.""" + body: dict[str, Any] = {"name": name, "transport": transport} + if command: + body["command"] = command + if args: + body["args"] = args + if url: + body["url"] = url + if headers: + body["headers"] = headers + if env: + body["env"] = env + if auto_approve: + body["auto_approve"] = True + if not enabled: + body["enabled"] = False + return await self._request( + "POST", + "/v1/api/admin/mcp-servers", + json_body=body, + response_model=McpServerDetail, + ) + + async def get_mcp_server(self, server_id: str) -> McpServerDetail: + """Get a single MCP server with status.""" + return await self._request( + "GET", + f"/v1/api/admin/mcp-servers/{server_id}", + response_model=McpServerDetail, + ) + + async def update_mcp_server(self, server_id: str, **fields: Any) -> McpServerDetail: + """Update an MCP server definition.""" + return await self._request( + "PUT", + f"/v1/api/admin/mcp-servers/{server_id}", + json_body=fields, + response_model=McpServerDetail, + ) + + async def delete_mcp_server(self, server_id: str) -> StatusResponse: + """Delete an MCP server definition.""" + return await self._request( + "DELETE", + f"/v1/api/admin/mcp-servers/{server_id}", + response_model=StatusResponse, + ) + + async def reload_mcp_servers(self) -> StatusResponse: + """Tell all nodes to re-read MCP server config from DB.""" + return await self._request( + "POST", + "/v1/api/admin/mcp-servers/reload", + response_model=StatusResponse, + ) + + async def import_mcp_config(self, config: dict[str, Any]) -> ImportMcpConfigResponse: + """Import MCP servers from a config dict with mcpServers key.""" + return await self._request( + "POST", + "/v1/api/admin/mcp-servers/import", + json_body={"config": config}, + response_model=ImportMcpConfigResponse, + ) + class TurnstoneConsole: """Synchronous client for the turnstone console API. @@ -1041,6 +1136,53 @@ class TurnstoneConsole: def delete_setting(self, key: str, *, node_id: str = "") -> StatusResponse: return self._runner.run(self._async.delete_setting(key, node_id=node_id)) + # -- MCP servers ------------------------------------------------------- + + def list_mcp_servers(self, reveal: bool = False) -> ListMcpServersResponse: + return self._runner.run(self._async.list_mcp_servers(reveal=reveal)) + + def create_mcp_server( + self, + name: str, + transport: str, + *, + command: str = "", + args: list[str] | None = None, + url: str = "", + headers: dict[str, str] | None = None, + env: dict[str, str] | None = None, + auto_approve: bool = False, + enabled: bool = True, + ) -> McpServerDetail: + return self._runner.run( + self._async.create_mcp_server( + name, + transport, + command=command, + args=args, + url=url, + headers=headers, + env=env, + auto_approve=auto_approve, + enabled=enabled, + ) + ) + + def get_mcp_server(self, server_id: str) -> McpServerDetail: + return self._runner.run(self._async.get_mcp_server(server_id)) + + def update_mcp_server(self, server_id: str, **fields: Any) -> McpServerDetail: + return self._runner.run(self._async.update_mcp_server(server_id, **fields)) + + def delete_mcp_server(self, server_id: str) -> StatusResponse: + return self._runner.run(self._async.delete_mcp_server(server_id)) + + def reload_mcp_servers(self) -> StatusResponse: + return self._runner.run(self._async.reload_mcp_servers()) + + def import_mcp_config(self, config: dict[str, Any]) -> ImportMcpConfigResponse: + return self._runner.run(self._async.import_mcp_config(config)) + # -- lifecycle ----------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/server.py b/turnstone/server.py index d2a9e91f..c218003c 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1556,6 +1556,36 @@ def config_reload(request: Request) -> JSONResponse: return JSONResponse({"status": "ok"}) +# -- internal MCP management ----------------------------------------------- + + +def internal_mcp_reload(request: Request) -> JSONResponse: + """POST /v1/api/_internal/mcp-reload — re-read mcp_servers table and reconcile.""" + from turnstone.core.storage._registry import get_storage + + storage = get_storage() + mcp_mgr = getattr(request.app.state, "mcp_client", None) + if mcp_mgr is None: + # Create a new manager if none exists + from turnstone.core.mcp_client import MCPClientManager + + mcp_mgr = MCPClientManager({}) + mcp_mgr.start() + request.app.state.mcp_client = mcp_mgr + + result = mcp_mgr.reconcile_sync(storage) + return JSONResponse({"status": "ok", **result}) + + +def internal_mcp_status(request: Request) -> JSONResponse: + """GET /v1/api/_internal/mcp-status — return MCP server status.""" + mcp_mgr = getattr(request.app.state, "mcp_client", None) + if mcp_mgr is None: + return JSONResponse({"servers": {}}) + + return JSONResponse({"servers": mcp_mgr.get_all_server_status()}) + + # --------------------------------------------------------------------------- # Global SSE fan-out # --------------------------------------------------------------------------- @@ -1723,6 +1753,8 @@ def create_app( Route("/api/auth/status", auth_status), Route("/api/auth/setup", auth_setup, methods=["POST"]), Route("/api/_internal/config-reload", config_reload, methods=["POST"]), + Route("/api/_internal/mcp-reload", internal_mcp_reload, methods=["POST"]), + Route("/api/_internal/mcp-status", internal_mcp_status), ], ), Route("/health", health), @@ -1936,11 +1968,13 @@ def main() -> None: # Initialize MCP client (connects to configured MCP servers, if any) from turnstone.core.mcp_client import create_mcp_client + from turnstone.core.storage._registry import get_storage as _get_storage mcp_config_cli = args.mcp_config # CLI-only (no config.toml for this) mcp_client = create_mcp_client( mcp_config_cli or config_store.get("mcp.config_path") or None, refresh_interval=config_store.get("mcp.refresh_interval"), + storage=_get_storage(), ) # Backend health monitor with circuit breaker