Files
turnstone/compose.yaml
T
Patrick Buckley 4866c9873c feat: ddgCluster compose profile with DuckDuckGo Search MCP sidecar (#48)
Add ddgCluster profile extending the 10-node cluster with a DuckDuckGo
Search MCP sidecar. All cluster nodes connect via streamable-http and
gain duckduckgo_web_search + duckduckgo_fetch_content tools. No API
key required.

Key implementation details learned during testing:
- MCP SDK DNS rebinding protection must be disabled for Docker
  internal networking (Host header uses container names)
- FastMCP server binds to 127.0.0.1 by default; must set
  mcp.settings.host='0.0.0.0' for cross-container access
- DDG CLI lacks --host/--port flags; settings configured via Python
  entry point that patches FastMCP.settings directly
- Safe search disabled by default

Also adds MCP_CONFIG env var support to all server commands (shell
conditional, no-op when empty) and moves default server/bridge to
production profile for cleaner profile separation.
2026-03-12 16:51:53 -07:00

544 lines
17 KiB
YAML

# =============================================================================
# Turnstone Docker Compose Stack
#
# Usage:
# Infra only: docker compose up
# Single node: docker compose --profile production up
# Production (PG): DB_BACKEND=postgresql docker compose --profile production up
# 10-node cluster: docker compose --profile cluster up
# Cluster + DDG: docker compose --profile ddgCluster up
# With simulator: docker compose --profile sim up
# =============================================================================
name: turnstone
networks:
turnstone-net:
driver: bridge
volumes:
redis-data:
turnstone-data:
postgres-data:
services:
# -------------------------------------------------------------------
# PostgreSQL — production database (profile: production)
# -------------------------------------------------------------------
postgres:
image: postgres:17-alpine
profiles:
- production
- cluster
- ddgCluster
environment:
POSTGRES_DB: turnstone
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required for production profile}
volumes:
- postgres-data:/var/lib/postgresql/data
networks:
- turnstone-net
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-turnstone}"]
interval: 5s
timeout: 3s
retries: 5
start_period: 5s
deploy:
resources:
limits:
memory: 512M
cpus: '1.0'
restart: unless-stopped
# -------------------------------------------------------------------
# Redis — message broker, pub/sub, node registry
# -------------------------------------------------------------------
redis:
image: redis:7.4-alpine
command:
- sh
- -c
- >-
redis-server
--save 60 1
--loglevel warning
$${REDIS_PASSWORD:+--requirepass $$REDIS_PASSWORD}
ports:
- "${REDIS_PORT:-6379}:6379"
environment:
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
volumes:
- redis-data:/data
networks:
- turnstone-net
healthcheck:
test:
- CMD-SHELL
- redis-cli $${REDIS_PASSWORD:+-a $$REDIS_PASSWORD} ping | grep -q PONG
interval: 5s
timeout: 3s
retries: 5
start_period: 5s
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-server — Web UI + chat workstreams + LLM interaction
# -------------------------------------------------------------------
server:
build:
context: .
dockerfile: Dockerfile
profiles:
- production
command:
- sh
- -c
- >-
turnstone-server
--host 0.0.0.0
--port 8080
--base-url "$${LLM_BASE_URL}"
--api-key "$${OPENAI_API_KEY}"
$${MODEL:+--model $$MODEL}
$${SKIP_PERMISSIONS:+--skip-permissions}
$${MCP_CONFIG:+--mcp-config $$MCP_CONFIG}
ports:
- "${SERVER_PORT:-8080}:8080"
volumes:
- turnstone-data:/data
- ./docker/mcp-ddg.json:/etc/turnstone/mcp-ddg.json:ro
environment:
- LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
- OPENAI_API_KEY=${OPENAI_API_KEY:-dummy}
- TAVILY_API_KEY=${TAVILY_API_KEY:-}
- SKIP_PERMISSIONS=${SKIP_PERMISSIONS:-}
- TURNSTONE_AUTH_ENABLED=${TURNSTONE_AUTH_ENABLED:-}
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
- MODEL=${MODEL:-}
- MCP_CONFIG=${MCP_CONFIG:-}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
- TURNSTONE_DB_URL=${DATABASE_URL:-}
- TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-}
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- turnstone-net
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
required: false
ddg-search:
condition: service_healthy
required: false
healthcheck:
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-bridge — Redis <-> HTTP bridge for multi-node routing
# Node ID auto-generated from container hostname (no --node-id needed)
# -------------------------------------------------------------------
bridge:
build:
context: .
dockerfile: Dockerfile
profiles:
- production
command:
- turnstone-bridge
- --server-url=http://server:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
environment:
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
networks:
- turnstone-net
depends_on:
server:
condition: service_healthy
redis:
condition: service_healthy
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-console — Cluster dashboard
# -------------------------------------------------------------------
console:
build:
context: .
dockerfile: Dockerfile
command:
- turnstone-console
- --host=0.0.0.0
- --port=8090
- --redis-host=redis
- --redis-port=6379
- --poll-interval=${CONSOLE_POLL_INTERVAL:-10}
ports:
- "${CONSOLE_PORT:-8090}:8090"
environment:
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
- TURNSTONE_AUTH_ENABLED=${TURNSTONE_AUTH_ENABLED:-}
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
- TURNSTONE_DB_URL=${DATABASE_URL:-}
networks:
- turnstone-net
depends_on:
redis:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8090/health"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-channel — Channel gateway (Discord, Slack, etc.)
# Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter
# -------------------------------------------------------------------
channel:
build:
context: .
dockerfile: Dockerfile
profiles:
- production
- cluster
- ddgCluster
command:
- sh
- -c
- >-
turnstone-channel
--redis-host=redis
--redis-port=6379
--http-host=0.0.0.0
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
environment:
- TURNSTONE_DISCORD_TOKEN=${TURNSTONE_DISCORD_TOKEN:-}
- TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0}
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
- TURNSTONE_DB_URL=${DATABASE_URL:-}
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
networks:
- turnstone-net
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
required: false
restart: unless-stopped
# -------------------------------------------------------------------
# ddg-search — DuckDuckGo Search MCP server (HTTP transport)
# Provides web search + content fetch tools to turnstone via MCP.
# No API key required.
#
# Start with: MCP_CONFIG=/etc/turnstone/mcp-ddg.json \
# docker compose --profile ddgCluster up
# -------------------------------------------------------------------
ddg-search:
image: python:3.13-slim
profiles:
- ddgCluster
command:
- sh
- -c
- >-
pip install --no-cache-dir duckduckgo-mcp-server &&
python -c "from mcp.server.transport_security import TransportSecuritySettings; import duckduckgo_mcp_server.server as s; s.safe_search=s.SafeSearchMode.OFF; s.mcp.settings.host='0.0.0.0'; s.mcp.settings.port=3000; s.mcp.settings.transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False); s.mcp.run(transport='streamable-http')"
networks:
- turnstone-net
healthcheck:
test: ["CMD-SHELL", "python -c \"import socket; s=socket.create_connection(('0.0.0.0',3000),2); s.close()\""]
interval: 10s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
memory: 256M
cpus: '0.25'
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-sim — Multi-node cluster simulator (no LLM needed)
# Start with: docker compose --profile sim up
# -------------------------------------------------------------------
sim:
build:
context: .
dockerfile: Dockerfile
profiles:
- sim
command:
- sh
- -c
- >-
turnstone-sim
--nodes "$${SIM_NODES}"
--scenario "$${SIM_SCENARIO}"
--duration "$${SIM_DURATION}"
--mps "$${SIM_MPS}"
--redis-host redis
--redis-port 6379
--log-level "$${SIM_LOG_LEVEL}"
$${SIM_SEED:+--seed $$SIM_SEED}
$${SIM_METRICS_FILE:+--metrics-file $$SIM_METRICS_FILE}
environment:
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
- SIM_NODES=${SIM_NODES:-100}
- SIM_SCENARIO=${SIM_SCENARIO:-steady}
- SIM_DURATION=${SIM_DURATION:-60}
- SIM_MPS=${SIM_MPS:-5.0}
- SIM_LOG_LEVEL=${SIM_LOG_LEVEL:-INFO}
- SIM_SEED=${SIM_SEED:-}
- SIM_METRICS_FILE=${SIM_METRICS_FILE:-}
networks:
- turnstone-net
depends_on:
redis:
condition: service_healthy
restart: "no"
# ===================================================================
# 10-node cluster (profile: cluster)
#
# Each node is a server + bridge pair. All share the same PostgreSQL
# and Redis instances. Access via console at :8090.
#
# Start: docker compose --profile cluster up
# ===================================================================
# -- cluster servers ------------------------------------------------
server-1: &cluster-server
build: { context: ., dockerfile: Dockerfile }
profiles: [cluster, ddgCluster]
command: &cluster-server-cmd
- sh
- -c
- >-
turnstone-server
--host 0.0.0.0
--port 8080
--base-url "$${LLM_BASE_URL}"
--api-key "$${OPENAI_API_KEY}"
$${MODEL:+--model $$MODEL}
$${SKIP_PERMISSIONS:+--skip-permissions}
$${MCP_CONFIG:+--mcp-config $$MCP_CONFIG}
volumes:
- turnstone-data:/data
- ./docker/mcp-ddg.json:/etc/turnstone/mcp-ddg.json:ro
environment: &cluster-server-env
LLM_BASE_URL: ${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
OPENAI_API_KEY: ${OPENAI_API_KEY:-dummy}
TAVILY_API_KEY: ${TAVILY_API_KEY:-}
SKIP_PERMISSIONS: ${SKIP_PERMISSIONS:-}
TURNSTONE_AUTH_ENABLED: ${TURNSTONE_AUTH_ENABLED:-}
TURNSTONE_AUTH_TOKEN: ${TURNSTONE_AUTH_TOKEN:-}
TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:-}
MODEL: ${MODEL:-}
MCP_CONFIG: ${MCP_CONFIG:-}
TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql}
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
TURNSTONE_NODE_ID: node-1
extra_hosts: ["host.docker.internal:host-gateway"]
networks: [turnstone-net]
depends_on:
redis: { condition: service_healthy }
postgres: { condition: service_healthy }
ddg-search: { condition: service_healthy, required: false }
healthcheck:
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
deploy:
resources:
limits: { memory: 384M, cpus: '0.5' }
restart: unless-stopped
server-2:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-2 }
server-3:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-3 }
server-4:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-4 }
server-5:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-5 }
server-6:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-6 }
server-7:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-7 }
server-8:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-8 }
server-9:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-9 }
server-10:
<<: *cluster-server
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-10 }
# -- cluster bridges ------------------------------------------------
bridge-1: &cluster-bridge
build: { context: ., dockerfile: Dockerfile }
profiles: [cluster, ddgCluster]
command:
- turnstone-bridge
- --server-url=http://server-1:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
environment: &cluster-bridge-env
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
TURNSTONE_AUTH_TOKEN: ${TURNSTONE_AUTH_TOKEN:-}
TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:-}
networks: [turnstone-net]
depends_on:
server-1: { condition: service_healthy }
redis: { condition: service_healthy }
deploy:
resources:
limits: { memory: 256M, cpus: '0.25' }
restart: unless-stopped
bridge-2:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-2:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-2: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-3:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-3:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-3: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-4:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-4:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-4: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-5:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-5:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-5: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-6:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-6:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-6: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-7:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-7:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-7: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-8:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-8:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-8: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-9:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-9:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-9: { condition: service_healthy }
redis: { condition: service_healthy }
bridge-10:
<<: *cluster-bridge
command:
- turnstone-bridge
- --server-url=http://server-10:8080
- --redis-host=redis
- --redis-port=6379
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
depends_on:
server-10: { condition: service_healthy }
redis: { condition: service_healthy }