Compare commits

...

2 Commits

Author SHA1 Message Date
Patrick Buckley 79eec8194f chore: bump version to 1.0.3 2026-04-04 12:28:47 -07:00
Patrick Buckley 425d1a7d8f fix: bundle production compose.yaml for pipx users (#293) (#294)
* fix: bundle production compose.yaml for pipx users (#293)

Users who install via pipx don't have a git clone, so there's no
compose.yaml or Dockerfile. Bootstrap now extracts a bundled production
compose file that uses pre-built ghcr.io images instead of local builds.

- Add turnstone/deploy/compose.yaml (ghcr.io images, no build blocks,
  single-node production profile only)
- Add write_compose tool to bootstrap wizard
- Update bootstrap system prompt to check for and write compose.yaml
- Remove stale ddgCluster profile references from system prompt
- Include turnstone/deploy/*.yaml in wheel

* fix: use postgresql+psycopg:// DSN scheme in compose fallbacks

The Docker image ships psycopg3, not psycopg2, so the bare
postgresql:// scheme fails. Also clarify PG usage comment in
production compose.
2026-04-04 12:27:20 -07:00
8 changed files with 335 additions and 45 deletions
+8 -3
View File
@@ -1,5 +1,10 @@
# =============================================================================
# Turnstone Docker Compose Stack
# Turnstone Docker Compose Stack — Development
#
# This file is for local development from a git clone. It builds images
# locally from the Dockerfile. If you installed via pip/pipx, run
# `turnstone-bootstrap` instead — it writes a production compose.yaml
# that pulls pre-built images from ghcr.io.
#
# Usage:
# Infra only: docker compose up
@@ -161,7 +166,7 @@ services:
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql}
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
networks:
- turnstone-net
@@ -211,7 +216,7 @@ services:
MODEL: ${MODEL:-}
MCP_CONFIG: ${MCP_CONFIG:-}
TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql}
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
TURNSTONE_NODE_ID: node-1
TURNSTONE_ADVERTISE_URL: http://server-1:8080
extra_hosts: ["host.docker.internal:host-gateway"]
+2 -1
View File
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "turnstone"
version = "1.0.2"
version = "1.0.3"
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
readme = "README.md"
license = "BUSL-1.1"
@@ -81,6 +81,7 @@ include = [
"turnstone/shared_static/hljs-11.11.1/**/*",
"turnstone/shared_static/mermaid-11.14.0/**/*",
"turnstone/sdk/py.typed",
"turnstone/deploy/*.yaml",
]
[tool.pytest.ini_options]
+48 -1
View File
@@ -19,6 +19,7 @@ from turnstone.bootstrap import (
_tool_generate_secret,
_tool_read_file,
_tool_validate_api_key,
_tool_write_compose,
_tool_write_file,
execute_tool,
)
@@ -103,6 +104,52 @@ class TestWriteFile:
assert (tmp_path / "changed.txt").read_text() == "new\n"
class TestWriteCompose:
def test_writes_compose_file(self, tmp_path: Path) -> None:
with patch("builtins.input", return_value="y"):
result = _tool_write_compose(tmp_path, {})
assert "written successfully" in result
assert "ghcr.io" in result
content = (tmp_path / "compose.yaml").read_text()
assert "ghcr.io/turnstonelabs/turnstone" in content
assert "TURNSTONE_IMAGE_TAG" in content
def test_user_declines(self, tmp_path: Path) -> None:
with patch("builtins.input", return_value="n"):
result = _tool_write_compose(tmp_path, {})
assert "declined" in result
assert not (tmp_path / "compose.yaml").exists()
def test_identical_content_skipped(self, tmp_path: Path) -> None:
# Write it once
with patch("builtins.input", return_value="y"):
_tool_write_compose(tmp_path, {})
# Second call should skip
result = _tool_write_compose(tmp_path, {})
assert "already exists" in result
def test_no_build_blocks(self, tmp_path: Path) -> None:
with patch("builtins.input", return_value="y"):
_tool_write_compose(tmp_path, {})
content = (tmp_path / "compose.yaml").read_text()
assert "build:" not in content
assert "dockerfile:" not in content.lower()
def test_overwrites_different_content(self, tmp_path: Path) -> None:
(tmp_path / "compose.yaml").write_text("old content\n")
with patch("builtins.input", return_value="y"):
result = _tool_write_compose(tmp_path, {})
assert "written successfully" in result
content = (tmp_path / "compose.yaml").read_text()
assert "ghcr.io" in content
def test_no_local_image_references(self, tmp_path: Path) -> None:
with patch("builtins.input", return_value="y"):
_tool_write_compose(tmp_path, {})
content = (tmp_path / "compose.yaml").read_text()
assert "turnstone:local" not in content
class TestGenerateSecret:
def test_default_length(self) -> None:
secret = _tool_generate_secret({})
@@ -620,7 +667,7 @@ class TestConstants:
assert func["parameters"]["type"] == "object"
def test_tool_count(self) -> None:
assert len(TOOLS) == 7
assert len(TOOLS) == 8
def test_all_tools_have_implementations(self) -> None:
from turnstone.bootstrap import TOOL_FUNCTIONS
+1 -1
View File
@@ -1,3 +1,3 @@
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
__version__ = "1.0.2"
__version__ = "1.0.3"
+97 -38
View File
@@ -2,14 +2,15 @@
Entry point: turnstone-bootstrap
Walks users through configuring a single-node or multi-node Turnstone
deployment via a conversational AI assistant. Generates .env files,
docker-compose overrides, and post-start setup scripts.
Walks users through configuring a Turnstone deployment via a conversational
AI assistant. Generates compose.yaml, .env files, and post-start setup
scripts.
"""
from __future__ import annotations
import getpass
import importlib.resources
import json
import os
import secrets
@@ -60,8 +61,6 @@ Turnstone is a multi-node AI orchestration platform. A deployment consists of:
## Deployment Profiles (compose.yaml)
- **Default** (no flag): console only (infrastructure, good for running external servers)
- **Production** (`--profile production`): 1 server + console + PostgreSQL + channel (single node)
- **Cluster** (`--profile cluster`): 10-node server fleet + PostgreSQL + channel + console (multi-node)
- **ddgCluster** (`--profile ddgCluster`): Cluster + DuckDuckGo Search MCP sidecar (web search via MCP, no API key needed)
## Environment Variables (.env)
The compose.yaml reads these from a `.env` file:
@@ -78,9 +77,9 @@ For commercial providers (OpenAI, Anthropic-via-proxy), use the real key.
### Database
- `DB_BACKEND` — `sqlite` (default) or `postgresql`
- `DATABASE_URL` — PostgreSQL connection string (production/cluster only)
- `DATABASE_URL` — PostgreSQL connection string (production only)
- `POSTGRES_USER` — PostgreSQL username (default: turnstone)
- `POSTGRES_PASSWORD` — PostgreSQL password (required for production/cluster)
- `POSTGRES_PASSWORD` — PostgreSQL password (required for production)
### Authentication (always enabled)
- `TURNSTONE_JWT_SECRET` — JWT signing secret (required). All services must share the same secret. \
@@ -104,16 +103,15 @@ Generate with: `python -c "import secrets; print(secrets.token_hex(32))"`
- `TURNSTONE_DISCORD_TOKEN` — Discord bot token
- `TURNSTONE_DISCORD_GUILD` — Restrict to single guild ID
### MCP Integration (optional)
- `MCP_CONFIG` — Path to MCP server config inside the container \
(e.g., `/etc/turnstone/mcp-ddg.json`). When set, servers connect to configured MCP servers on startup.
- The `ddgCluster` profile runs a DuckDuckGo Search MCP sidecar (Python) that provides \
`duckduckgo_web_search` and `duckduckgo_fetch_content` tools to every node. No API key required. \
The sidecar uses MCP streamable-http transport with DNS rebinding protection disabled \
(required for Docker internal networking) and binds to 0.0.0.0:3000 via FastMCP settings. \
Safe search is disabled by default.
### Docker Image
- `TURNSTONE_IMAGE_TAG` — Docker image tag (default: `latest`). \
Set this to pin the image version (e.g., `1.1.0a3`, `stable`, `experimental`).
### Cluster
### MCP Integration (optional)
- `MCP_CONFIG` — Path to MCP server config inside the container. \
When set, servers connect to configured MCP servers on startup.
### Other
- `APPROVAL_TIMEOUT` — Tool approval timeout in seconds (default: 3600)
## Auth Setup Flow
@@ -154,28 +152,28 @@ Categories like "engineering", "analysis", etc.
## Your Task
Walk the user through setting up their deployment step by step:
1. **First**: Call `check_docker` and `read_file` on `.env` to detect existing state.
2. **Deployment mode**: Ask if they want single-node (`--profile production`) or multi-node \
(`--profile cluster`). Explain trade-offs.
3. **LLM provider for the deployment**: Which LLM backend their Turnstone will use \
1. **First**: Call `check_docker`, `read_file` on `.env`, and `read_file` on `compose.yaml` \
to detect existing state. If `compose.yaml` does not exist, call `write_compose` to \
extract the bundled production compose file. This is essential — without it, \
`docker compose` will fail.
2. **LLM provider for the deployment**: Which LLM backend their Turnstone will use \
(may differ from this wizard's model). Ask for base URL, API key, model name.
4. **Database**: SQLite (dev/simple) vs PostgreSQL (production/cluster). \
PostgreSQL is required for cluster mode.
5. **Security**: Auth is always enabled and requires `TURNSTONE_JWT_SECRET`. \
3. **Database**: SQLite (dev/simple) vs PostgreSQL (production). \
PostgreSQL is recommended for production use.
4. **Security**: Auth is always enabled and requires `TURNSTONE_JWT_SECRET`. \
Use `generate_secret` for JWT secret and Postgres password. \
Always set `TURNSTONE_JWT_SECRET` in the .env. \
Ask for initial admin username and password. \
If the user's deployment will use an external identity provider (Okta, Azure AD, Google, etc.), \
offer to configure OIDC SSO. Ask for the issuer URL, client ID, and client secret. \
Optionally configure role mapping and OIDC-only mode.
6. **Ports**: Check defaults with `check_port`, suggest alternatives if conflicts.
7. **Optional features**: Discord integration, web search (Tavily key), \
DuckDuckGo Search MCP (for cluster — uses `ddgCluster` profile with \
`MCP_CONFIG=/etc/turnstone/mcp-ddg.json`, no API key needed).
8. **Generate .env**: Call `write_file` with the complete `.env` content.
9. **Generate setup.sh**: Call `write_file` with a post-start script that creates the admin \
5. **Ports**: Check defaults with `check_port`, suggest alternatives if conflicts.
6. **Optional features**: Discord integration, web search (Tavily key).
7. **Generate .env**: Call `write_file` with the complete `.env` content. \
Include `TURNSTONE_IMAGE_TAG` set to the version matching the installed package.
8. **Generate setup.sh**: Call `write_file` with a post-start script that creates the admin \
user and any roles/policies/skills the user wants.
10. **Finish**: Call the `finish` tool with a summary of what was configured and the \
9. **Finish**: Call the `finish` tool with a summary of what was configured and the \
exact commands to run next (e.g., `docker compose --profile production up -d` then `./setup.sh`).
## Rules
@@ -183,14 +181,9 @@ exact commands to run next (e.g., `docker compose --profile production up -d` th
- NEVER echo API keys or passwords back to the user in your text responses.
- ALWAYS use `generate_secret` for passwords and secrets — never invent them.
- When writing files, use `write_file` — the user will see a preview and confirm.
- If `compose.yaml` is missing, call `write_compose` before anything else. \
The compose file uses pre-built images from ghcr.io — no local Docker build is needed.
- If an existing .env is detected, summarize what's configured and ask what to change.
- For cluster mode, the compose.yaml has a fixed 10-node fleet — no override needed.
- For cluster + DuckDuckGo Search, use `--profile ddgCluster` instead of `--profile cluster`. \
Set `MCP_CONFIG=/etc/turnstone/mcp-ddg.json` in `.env`. No API key needed. \
The DuckDuckGo MCP sidecar starts automatically and all cluster nodes connect to it. \
Note: the MCP SDK's DNS rebinding protection must be disabled for Docker-internal networking \
(the compose.yaml handles this), and the server must bind to 0.0.0.0 (not 127.0.0.1) to be \
reachable from other containers.
- The `DATABASE_URL` for docker compose internal networking uses the hostname `postgres` \
(e.g., `postgresql+psycopg://turnstone:<password>@postgres:5432/turnstone`).
- For local LLM backends (vLLM, llama.cpp, Ollama, etc.), set `OPENAI_API_KEY=dummy` in the \
@@ -342,6 +335,23 @@ TOOLS: list[dict[str, Any]] = [
},
},
},
{
"type": "function",
"function": {
"name": "write_compose",
"description": (
"Write the production Docker Compose file to the project directory. "
"This extracts the compose.yaml bundled with Turnstone, which uses "
"pre-built images from ghcr.io (no local Docker build required). "
"The user will be shown a preview and asked to confirm."
),
"parameters": {
"type": "object",
"properties": {},
"required": [],
},
},
},
{
"type": "function",
"function": {
@@ -561,6 +571,54 @@ def _tool_check_docker(args: dict[str, Any]) -> str:
return "\n".join(results)
def _tool_write_compose(project_dir: Path, args: dict[str, Any]) -> str:
"""Extract the bundled production compose.yaml to the project directory."""
dest = project_dir / "compose.yaml"
# Read the bundled template
try:
ref = importlib.resources.files("turnstone.deploy").joinpath("compose.yaml")
content = ref.read_text(encoding="utf-8")
except Exception as exc:
return f"Error: could not read bundled compose template: {exc}"
# Skip if identical
if dest.exists():
try:
existing = dest.read_text(encoding="utf-8")
if existing == content:
return "compose.yaml already exists with identical content."
except (OSError, UnicodeDecodeError):
pass
line_count = content.count("\n") + (1 if content and not content.endswith("\n") else 0)
# Show preview
print(f"\n{YELLOW} Writing compose.yaml ({line_count} lines){RESET}")
print(f"{DIM}{'' * 50}{RESET}")
for line in content.split("\n")[:30]:
print(f" {DIM}{line}{RESET}")
if line_count > 30:
print(f" {DIM}... ({line_count - 30} more lines){RESET}")
print(f"{DIM}{'' * 50}{RESET}")
try:
choice = input(f"{BOLD}Write this file? [Y/n]{RESET} ").strip().lower()
except (EOFError, KeyboardInterrupt):
return "User cancelled the write."
if choice in ("n", "no"):
return "User declined to write compose.yaml."
dest.write_text(content, encoding="utf-8")
return (
f"compose.yaml written successfully. "
f"It uses ghcr.io/turnstonelabs/turnstone images. "
f"Add TURNSTONE_IMAGE_TAG={__version__} to .env to pin the image "
f"to the currently installed version, or omit it to use 'latest'."
)
class _FinishError(Exception):
"""Raised by the finish tool to signal the wizard is done."""
@@ -581,11 +639,12 @@ TOOL_FUNCTIONS: dict[str, Any] = {
"check_port": _tool_check_port,
"validate_api_key": _tool_validate_api_key,
"check_docker": _tool_check_docker,
"write_compose": _tool_write_compose,
"finish": _tool_finish,
}
# Tools that need the project_dir argument
_PROJECT_DIR_TOOLS = frozenset({"read_file", "write_file"})
_PROJECT_DIR_TOOLS = frozenset({"read_file", "write_file", "write_compose"})
def execute_tool(name: str, args: dict[str, Any], project_dir: Path) -> str:
+5
View File
@@ -0,0 +1,5 @@
"""Bundled deployment templates (compose files, overlays).
These files are included in the wheel so that ``turnstone-bootstrap`` can
extract them for users who install via pip/pipx and don't have a git clone.
"""
+173
View File
@@ -0,0 +1,173 @@
# =============================================================================
# Turnstone Docker Compose Stack — Production
#
# This file is bundled with the turnstone wheel and written by
# turnstone-bootstrap for users who install via pip/pipx.
# It pulls pre-built images from ghcr.io instead of building locally.
#
# Usage:
# Infra only: docker compose up
# Single node: docker compose --profile production up
# Production (PG): docker compose --profile production up
# (set DB_BACKEND, DATABASE_URL, POSTGRES_PASSWORD in .env)
#
# Set TURNSTONE_IMAGE_TAG in .env to pin the image version (default: latest).
# =============================================================================
name: turnstone
networks:
turnstone-net:
driver: bridge
volumes:
turnstone-data:
workspace:
postgres-data:
services:
# -------------------------------------------------------------------
# PostgreSQL — production database (profile: production)
# -------------------------------------------------------------------
postgres:
image: pgautoupgrade/pgautoupgrade:18-alpine
profiles:
- production
command:
- postgres
- -c
- max_connections=${POSTGRES_MAX_CONNECTIONS:-300}
- -c
- shared_buffers=128MB
environment:
POSTGRES_DB: turnstone
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required for production profile}
PGDATA: /var/lib/postgresql/data
volumes:
- postgres-data:/var/lib/postgresql/data
networks:
- turnstone-net
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-turnstone}"]
interval: 5s
timeout: 3s
retries: 5
start_period: 30s
deploy:
resources:
limits:
memory: 1G
cpus: '1.0'
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-server — Web UI + chat workstreams + LLM interaction
# -------------------------------------------------------------------
server:
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
profiles:
- production
command:
- sh
- -c
- >-
turnstone-server
--host 0.0.0.0
--port 8080
--base-url "$${LLM_BASE_URL}"
--api-key "$${OPENAI_API_KEY}"
$${MODEL:+--model $$MODEL}
$${SKIP_PERMISSIONS:+--skip-permissions}
$${MCP_CONFIG:+--mcp-config $$MCP_CONFIG}
ports:
- "${SERVER_PORT:-8080}:8080"
volumes:
- turnstone-data:/data
- ${WORKSPACE_MOUNT:-workspace}:/workspace
environment:
- LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
- OPENAI_API_KEY=${OPENAI_API_KEY:-dummy}
- TAVILY_API_KEY=${TAVILY_API_KEY:-}
- SKIP_PERMISSIONS=${SKIP_PERMISSIONS:-}
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
- MODEL=${MODEL:-}
- MCP_CONFIG=${MCP_CONFIG:-}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
- TURNSTONE_DB_URL=${DATABASE_URL:-}
- TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-}
- TURNSTONE_ADVERTISE_URL=${TURNSTONE_ADVERTISE_URL:-http://server:8080}
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- turnstone-net
depends_on:
postgres:
condition: service_healthy
required: false
healthcheck:
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
interval: 10s
timeout: 5s
retries: 5
start_period: 60s
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-console — Cluster dashboard
# -------------------------------------------------------------------
console:
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
command:
- turnstone-console
- --host=0.0.0.0
- --port=8090
ports:
- "${CONSOLE_PORT:-8090}:8090"
environment:
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
- TURNSTONE_DB_URL=${DATABASE_URL:-}
- TURNSTONE_CONSOLE_URL=http://console:8090
networks:
- turnstone-net
healthcheck:
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8090/health"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
restart: unless-stopped
# -------------------------------------------------------------------
# turnstone-channel — Channel gateway (Discord, Slack, etc.)
# Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter
# -------------------------------------------------------------------
channel:
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
profiles:
- production
command:
- sh
- -c
- >-
turnstone-channel
--http-host=0.0.0.0
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
environment:
- TURNSTONE_DISCORD_TOKEN=${TURNSTONE_DISCORD_TOKEN:-}
- TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0}
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql}
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
networks:
- turnstone-net
depends_on:
postgres:
condition: service_healthy
required: false
restart: unless-stopped
Generated
+1 -1
View File
@@ -2496,7 +2496,7 @@ wheels = [
[[package]]
name = "turnstone"
version = "1.0.2"
version = "1.0.3"
source = { editable = "." }
dependencies = [
{ name = "alembic" },