From 4866c9873c06d53b2dd6f8200e2df440d2d30454 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Thu, 12 Mar 2026 16:51:53 -0700 Subject: [PATCH] feat: ddgCluster compose profile with DuckDuckGo Search MCP sidecar (#48) Add ddgCluster profile extending the 10-node cluster with a DuckDuckGo Search MCP sidecar. All cluster nodes connect via streamable-http and gain duckduckgo_web_search + duckduckgo_fetch_content tools. No API key required. Key implementation details learned during testing: - MCP SDK DNS rebinding protection must be disabled for Docker internal networking (Host header uses container names) - FastMCP server binds to 127.0.0.1 by default; must set mcp.settings.host='0.0.0.0' for cross-container access - DDG CLI lacks --host/--port flags; settings configured via Python entry point that patches FastMCP.settings directly - Safe search disabled by default Also adds MCP_CONFIG env var support to all server commands (shell conditional, no-op when empty) and moves default server/bridge to production profile for cleaner profile separation. --- compose.yaml | 61 ++++++++++++++++++++++++++++++++++++++---- docker/mcp-ddg.json | 7 +++++ turnstone/bootstrap.py | 30 ++++++++++++++++----- 3 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 docker/mcp-ddg.json diff --git a/compose.yaml b/compose.yaml index 7e059153..2b74703c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,10 +2,11 @@ # Turnstone Docker Compose Stack # # Usage: -# Default (SQLite): docker compose up +# Infra only: docker compose up +# Single node: docker compose --profile production up # Production (PG): DB_BACKEND=postgresql docker compose --profile production up -# (or set DB_BACKEND=postgresql in .env) # 10-node cluster: docker compose --profile cluster up +# Cluster + DDG: docker compose --profile ddgCluster up # With simulator: docker compose --profile sim up # ============================================================================= @@ -29,6 +30,7 @@ services: profiles: - production - cluster + - ddgCluster environment: POSTGRES_DB: turnstone POSTGRES_USER: ${POSTGRES_USER:-turnstone} @@ -88,6 +90,8 @@ services: build: context: . dockerfile: Dockerfile + profiles: + - production command: - sh - -c @@ -99,10 +103,12 @@ services: --api-key "$${OPENAI_API_KEY}" $${MODEL:+--model $$MODEL} $${SKIP_PERMISSIONS:+--skip-permissions} + $${MCP_CONFIG:+--mcp-config $$MCP_CONFIG} ports: - "${SERVER_PORT:-8080}:8080" volumes: - turnstone-data:/data + - ./docker/mcp-ddg.json:/etc/turnstone/mcp-ddg.json:ro environment: - LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1} - OPENAI_API_KEY=${OPENAI_API_KEY:-dummy} @@ -112,6 +118,7 @@ services: - TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-} - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-} - MODEL=${MODEL:-} + - MCP_CONFIG=${MCP_CONFIG:-} - TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite} - TURNSTONE_DB_URL=${DATABASE_URL:-} - TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-} @@ -125,6 +132,9 @@ services: postgres: condition: service_healthy required: false + ddg-search: + condition: service_healthy + required: false healthcheck: test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"] interval: 10s @@ -141,6 +151,8 @@ services: build: context: . dockerfile: Dockerfile + profiles: + - production command: - turnstone-bridge - --server-url=http://server:8080 @@ -208,6 +220,7 @@ services: profiles: - production - cluster + - ddgCluster command: - sh - -c @@ -235,6 +248,39 @@ services: required: false restart: unless-stopped + # ------------------------------------------------------------------- + # ddg-search — DuckDuckGo Search MCP server (HTTP transport) + # Provides web search + content fetch tools to turnstone via MCP. + # No API key required. + # + # Start with: MCP_CONFIG=/etc/turnstone/mcp-ddg.json \ + # docker compose --profile ddgCluster up + # ------------------------------------------------------------------- + ddg-search: + image: python:3.13-slim + profiles: + - ddgCluster + command: + - sh + - -c + - >- + pip install --no-cache-dir duckduckgo-mcp-server && + python -c "from mcp.server.transport_security import TransportSecuritySettings; import duckduckgo_mcp_server.server as s; s.safe_search=s.SafeSearchMode.OFF; s.mcp.settings.host='0.0.0.0'; s.mcp.settings.port=3000; s.mcp.settings.transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False); s.mcp.run(transport='streamable-http')" + networks: + - turnstone-net + healthcheck: + test: ["CMD-SHELL", "python -c \"import socket; s=socket.create_connection(('0.0.0.0',3000),2); s.close()\""] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + memory: 256M + cpus: '0.25' + restart: unless-stopped + # ------------------------------------------------------------------- # turnstone-sim — Multi-node cluster simulator (no LLM needed) # Start with: docker compose --profile sim up @@ -288,7 +334,7 @@ services: server-1: &cluster-server build: { context: ., dockerfile: Dockerfile } - profiles: [cluster] + profiles: [cluster, ddgCluster] command: &cluster-server-cmd - sh - -c @@ -300,7 +346,10 @@ services: --api-key "$${OPENAI_API_KEY}" $${MODEL:+--model $$MODEL} $${SKIP_PERMISSIONS:+--skip-permissions} - volumes: [turnstone-data:/data] + $${MCP_CONFIG:+--mcp-config $$MCP_CONFIG} + volumes: + - turnstone-data:/data + - ./docker/mcp-ddg.json:/etc/turnstone/mcp-ddg.json:ro environment: &cluster-server-env LLM_BASE_URL: ${LLM_BASE_URL:-http://host.docker.internal:8000/v1} OPENAI_API_KEY: ${OPENAI_API_KEY:-dummy} @@ -310,6 +359,7 @@ services: TURNSTONE_AUTH_TOKEN: ${TURNSTONE_AUTH_TOKEN:-} TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:-} MODEL: ${MODEL:-} + MCP_CONFIG: ${MCP_CONFIG:-} TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql} TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone} TURNSTONE_NODE_ID: node-1 @@ -318,6 +368,7 @@ services: depends_on: redis: { condition: service_healthy } postgres: { condition: service_healthy } + ddg-search: { condition: service_healthy, required: false } healthcheck: test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"] interval: 10s @@ -361,7 +412,7 @@ services: bridge-1: &cluster-bridge build: { context: ., dockerfile: Dockerfile } - profiles: [cluster] + profiles: [cluster, ddgCluster] command: - turnstone-bridge - --server-url=http://server-1:8080 diff --git a/docker/mcp-ddg.json b/docker/mcp-ddg.json new file mode 100644 index 00000000..55ba71d7 --- /dev/null +++ b/docker/mcp-ddg.json @@ -0,0 +1,7 @@ +{ + "mcpServers": { + "ddg": { + "url": "http://ddg-search:3000/mcp" + } + } +} diff --git a/turnstone/bootstrap.py b/turnstone/bootstrap.py index aeff26b2..e257cddf 100644 --- a/turnstone/bootstrap.py +++ b/turnstone/bootstrap.py @@ -60,9 +60,10 @@ Turnstone is a multi-node AI orchestration platform. A deployment consists of: - **Channel** (optional): Discord/Slack gateway ## Deployment Profiles (compose.yaml) -- **Default**: redis + 1 server + 1 bridge + console (SQLite, good for dev/testing) -- **Production** (`--profile production`): + PostgreSQL + channel gateway (single node, production-ready) -- **Cluster** (`--profile cluster`): 10-node server/bridge fleet + PostgreSQL + channel (multi-node) +- **Default** (no flag): redis + console only (infrastructure, good for running external servers) +- **Production** (`--profile production`): redis + 1 server + 1 bridge + console + PostgreSQL + channel (single node) +- **Cluster** (`--profile cluster`): 10-node server/bridge fleet + PostgreSQL + channel + console (multi-node) +- **ddgCluster** (`--profile ddgCluster`): Cluster + DuckDuckGo Search MCP sidecar (web search via MCP, no API key needed) ## Environment Variables (.env) The compose.yaml reads these from a `.env` file: @@ -100,6 +101,15 @@ For commercial providers (OpenAI, Anthropic-via-proxy), use the real key. - `TURNSTONE_DISCORD_TOKEN` — Discord bot token - `TURNSTONE_DISCORD_GUILD` — Restrict to single guild ID +### MCP Integration (optional) +- `MCP_CONFIG` — Path to MCP server config inside the container \ +(e.g., `/etc/turnstone/mcp-ddg.json`). When set, servers connect to configured MCP servers on startup. +- The `ddgCluster` profile runs a DuckDuckGo Search MCP sidecar (Python) that provides \ +`duckduckgo_web_search` and `duckduckgo_fetch_content` tools to every node. No API key required. \ +The sidecar uses MCP streamable-http transport with DNS rebinding protection disabled \ +(required for Docker internal networking) and binds to 0.0.0.0:3000 via FastMCP settings. \ +Safe search is disabled by default. + ### Cluster - `HEARTBEAT_TTL` — Bridge heartbeat TTL in seconds (default: 60) - `APPROVAL_TIMEOUT` — Tool approval timeout in seconds (default: 3600) @@ -130,8 +140,8 @@ Categories like "engineering", "analysis", etc. Walk the user through setting up their deployment step by step: 1. **First**: Call `check_docker` and `read_file` on `.env` to detect existing state. -2. **Deployment mode**: Ask if they want single-node (production) or multi-node (cluster). \ -Explain trade-offs. +2. **Deployment mode**: Ask if they want single-node (`--profile production`) or multi-node \ +(`--profile cluster`). Explain trade-offs. 3. **LLM provider for the deployment**: Which LLM backend their Turnstone will use \ (may differ from this wizard's model). Ask for base URL, API key, model name. 4. **Database**: SQLite (dev/simple) vs PostgreSQL (production/cluster). \ @@ -140,7 +150,9 @@ PostgreSQL is required for cluster mode. Use `generate_secret` for JWT secret, Redis password, auth token, and Postgres password. \ Ask for initial admin username and password. 6. **Ports**: Check defaults with `check_port`, suggest alternatives if conflicts. -7. **Optional features**: Discord integration, web search (Tavily key). +7. **Optional features**: Discord integration, web search (Tavily key), \ +DuckDuckGo Search MCP (for cluster — uses `ddgCluster` profile with \ +`MCP_CONFIG=/etc/turnstone/mcp-ddg.json`, no API key needed). 8. **Generate .env**: Call `write_file` with the complete `.env` content. 9. **Generate setup.sh**: Call `write_file` with a post-start script that creates the admin \ user and any roles/policies/templates the user wants. @@ -154,6 +166,12 @@ exact commands to run next (e.g., `docker compose --profile production up -d` th - When writing files, use `write_file` — the user will see a preview and confirm. - If an existing .env is detected, summarize what's configured and ask what to change. - For cluster mode, the compose.yaml has a fixed 10-node fleet — no override needed. +- For cluster + DuckDuckGo Search, use `--profile ddgCluster` instead of `--profile cluster`. \ +Set `MCP_CONFIG=/etc/turnstone/mcp-ddg.json` in `.env`. No API key needed. \ +The DuckDuckGo MCP sidecar starts automatically and all cluster nodes connect to it. \ +Note: the MCP SDK's DNS rebinding protection must be disabled for Docker-internal networking \ +(the compose.yaml handles this), and the server must bind to 0.0.0.0 (not 127.0.0.1) to be \ +reachable from other containers. - The `DATABASE_URL` for docker compose internal networking uses the hostname `postgres` \ (e.g., `postgresql://turnstone:@postgres:5432/turnstone`). - For local LLM backends (vLLM, llama.cpp, Ollama, etc.), set `OPENAI_API_KEY=dummy` in the \