From 2a05ba5915b2d1b1d73b1f63e1744acc9019f167 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Mon, 13 Apr 2026 14:48:51 -0700 Subject: [PATCH] fix: standardize database env vars on TURNSTONE_DB_* naming (#348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: standardize database env vars on TURNSTONE_DB_* naming compose.yaml used DB_BACKEND/DATABASE_URL in .env which got mapped to TURNSTONE_DB_BACKEND/TURNSTONE_DB_URL inside containers. Running bare- metal required the TURNSTONE_ prefix, but docs didn't explain this. Eliminate the indirection — use TURNSTONE_DB_BACKEND and TURNSTONE_DB_URL everywhere (compose, .env, bare-metal, docs, bootstrap wizard). * fix: update .env.example to use TURNSTONE_DB_* naming --- .env.example | 4 ++-- README.md | 9 +++++++++ compose.yaml | 18 +++++++++--------- docs/docker.md | 6 +++++- docs/pgbouncer.md | 2 +- turnstone/bootstrap.py | 7 ++++--- turnstone/core/env.py | 2 +- turnstone/deploy/compose.yaml | 14 +++++++------- 8 files changed, 38 insertions(+), 24 deletions(-) diff --git a/.env.example b/.env.example index d3e7a5b8..ac4611f9 100644 --- a/.env.example +++ b/.env.example @@ -20,10 +20,10 @@ TURNSTONE_JWT_SECRET=changeme-to-32-bytes-of-hex # -- Database ------------------------------------------------------------------ # Single-node default is SQLite (zero config). Set these for PostgreSQL: -# DB_BACKEND=postgresql +# TURNSTONE_DB_BACKEND=postgresql # POSTGRES_USER=turnstone # POSTGRES_PASSWORD=changeme -# DATABASE_URL=postgresql+psycopg://turnstone:changeme@postgres:5432/turnstone +# TURNSTONE_DB_URL=postgresql+psycopg://turnstone:changeme@postgres:5432/turnstone # -- Ports --------------------------------------------------------------------- # SERVER_PORT=8080 diff --git a/README.md b/README.md index 0fafbdfd..3116d9f6 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,15 @@ pip install turnstone[console] turnstone-console --port 8090 ``` +For PostgreSQL (recommended for production): + +```bash +pip install turnstone[postgres] +export TURNSTONE_DB_BACKEND=postgresql +export TURNSTONE_DB_URL="postgresql+psycopg://user:pass@localhost:5432/turnstone" +turnstone-server --port 8080 --base-url http://localhost:8000/v1 +``` + ### Docker ```bash diff --git a/compose.yaml b/compose.yaml index 234b141f..3c3e519d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,7 +9,7 @@ # Usage: # Infra only: docker compose up # Single node: docker compose --profile production up -# Production (PG): DB_BACKEND=postgresql docker compose --profile production up +# Production (PG): TURNSTONE_DB_BACKEND=postgresql docker compose --profile production up # 10-node cluster: docker compose --profile cluster up # ============================================================================= @@ -94,8 +94,8 @@ services: - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} - MODEL=${MODEL:-} - MCP_CONFIG=${MCP_CONFIG:-} - - TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite} - - TURNSTONE_DB_URL=${DATABASE_URL:-} + - TURNSTONE_DB_BACKEND=${TURNSTONE_DB_BACKEND:-sqlite} + - TURNSTONE_DB_URL=${TURNSTONE_DB_URL:-} - TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-} - TURNSTONE_ADVERTISE_URL=${TURNSTONE_ADVERTISE_URL:-http://server:8080} extra_hosts: @@ -131,8 +131,8 @@ services: environment: # Generate with: python -c "import secrets; print(secrets.token_hex(32))" - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} - - TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite} - - TURNSTONE_DB_URL=${DATABASE_URL:-} + - TURNSTONE_DB_BACKEND=${TURNSTONE_DB_BACKEND:-sqlite} + - TURNSTONE_DB_URL=${TURNSTONE_DB_URL:-} - TURNSTONE_CONSOLE_URL=http://console:8090 networks: - turnstone-net @@ -165,8 +165,8 @@ services: - TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0} # Generate with: python -c "import secrets; print(secrets.token_hex(32))" - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} - - TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql} - - TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone} + - TURNSTONE_DB_BACKEND=${TURNSTONE_DB_BACKEND:-postgresql} + - TURNSTONE_DB_URL=${TURNSTONE_DB_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone} - TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091 networks: - turnstone-net @@ -215,8 +215,8 @@ services: TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} MODEL: ${MODEL:-} MCP_CONFIG: ${MCP_CONFIG:-} - TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql} - TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone} + TURNSTONE_DB_BACKEND: ${TURNSTONE_DB_BACKEND:-postgresql} + TURNSTONE_DB_URL: ${TURNSTONE_DB_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone} TURNSTONE_NODE_ID: node-1 TURNSTONE_ADVERTISE_URL: http://server-1:8080 extra_hosts: ["host.docker.internal:host-gateway"] diff --git a/docs/docker.md b/docs/docker.md index 846f1106..e5a373a3 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -83,11 +83,15 @@ Auth is always enabled. `TURNSTONE_JWT_SECRET` is required. | Variable | Default | Description | |----------|---------|-------------| | `TURNSTONE_DB_BACKEND` | `sqlite` | Storage backend: `sqlite` or `postgresql` | -| `TURNSTONE_DB_URL` | — | Database URL (e.g. `postgresql://user:pass@db:5432/turnstone`). For SQLite, defaults to `/data/.turnstone.db` | +| `TURNSTONE_DB_URL` | — | Database URL (e.g. `postgresql+psycopg://user:pass@postgres:5432/turnstone`). For SQLite, defaults to `/data/.turnstone.db` | | `TURNSTONE_DB_POOL_SIZE` | `2` | PostgreSQL connection pool size per process (default: 2 base + 3 overflow = 5 max) | +| `POSTGRES_USER` | `turnstone` | PostgreSQL container username (used in default `TURNSTONE_DB_URL` for cluster/channel) | +| `POSTGRES_PASSWORD` | — | PostgreSQL container password (required for production and cluster profiles) | The database stores workstream history, user accounts, and API tokens. When using JWT auth, a database backend is required for user storage. +> **Upgrading from <1.3.0a4:** Earlier versions used `DB_BACKEND` and `DATABASE_URL` in `.env`, which `compose.yaml` mapped to the `TURNSTONE_`-prefixed names internally. These short aliases have been removed. Rename `DB_BACKEND` → `TURNSTONE_DB_BACKEND` and `DATABASE_URL` → `TURNSTONE_DB_URL` in your `.env` file. + > **Large clusters:** Each turnstone process maintains a small connection pool (5 max). At hundreds of nodes this adds up — use [PgBouncer](pgbouncer.md) in transaction pooling mode between turnstone and PostgreSQL. > **First-time setup:** After deploying with auth enabled, create an initial admin user by running `turnstone-admin create-user` inside the container: diff --git a/docs/pgbouncer.md b/docs/pgbouncer.md index 82ec08d2..4d3fda01 100644 --- a/docs/pgbouncer.md +++ b/docs/pgbouncer.md @@ -67,7 +67,7 @@ services: ``` Then point turnstone services at PgBouncer instead of PostgreSQL -directly by changing the `DATABASE_URL` (or `TURNSTONE_DB_URL`): +directly by changing `TURNSTONE_DB_URL`: ```bash # Before (direct) diff --git a/turnstone/bootstrap.py b/turnstone/bootstrap.py index 559ba9b4..132a3dcb 100644 --- a/turnstone/bootstrap.py +++ b/turnstone/bootstrap.py @@ -76,8 +76,9 @@ For commercial providers (OpenAI, Anthropic-via-proxy), use the real key. - `TAVILY_API_KEY` — Web search API key (optional) ### Database -- `DB_BACKEND` — `sqlite` (default) or `postgresql` -- `DATABASE_URL` — PostgreSQL connection string (production only) +- `TURNSTONE_DB_BACKEND` — `sqlite` (default) or `postgresql` +- `TURNSTONE_DB_URL` — PostgreSQL connection URL (production only), \ +e.g. `postgresql+psycopg://turnstone:@postgres:5432/turnstone` - `POSTGRES_USER` — PostgreSQL username (default: turnstone) - `POSTGRES_PASSWORD` — PostgreSQL password (required for production) @@ -184,7 +185,7 @@ exact commands to run next (e.g., `docker compose --profile production up -d` th - If `compose.yaml` is missing, call `write_compose` before anything else. \ The compose file uses pre-built images from ghcr.io — no local Docker build is needed. - If an existing .env is detected, summarize what's configured and ask what to change. -- The `DATABASE_URL` for docker compose internal networking uses the hostname `postgres` \ +- The `TURNSTONE_DB_URL` for docker compose internal networking uses the hostname `postgres` \ (e.g., `postgresql+psycopg://turnstone:@postgres:5432/turnstone`). - For local LLM backends (vLLM, llama.cpp, Ollama, etc.), set `OPENAI_API_KEY=dummy` in the \ .env file — local servers typically don't require authentication. The `LLM_BASE_URL` should \ diff --git a/turnstone/core/env.py b/turnstone/core/env.py index 151f275b..b87e5301 100644 --- a/turnstone/core/env.py +++ b/turnstone/core/env.py @@ -73,7 +73,7 @@ _EXPLICIT_SCRUB: frozenset[str] = frozenset( "AZURE_CLIENT_SECRET", "GCP_SERVICE_ACCOUNT_KEY", "GOOGLE_APPLICATION_CREDENTIALS", - "DATABASE_URL", + "DATABASE_URL", # conventional name (Heroku, Railway, etc.) — kept for defence-in-depth "TURNSTONE_DB_URL", } ) diff --git a/turnstone/deploy/compose.yaml b/turnstone/deploy/compose.yaml index dca17cb2..77cd0a42 100644 --- a/turnstone/deploy/compose.yaml +++ b/turnstone/deploy/compose.yaml @@ -9,7 +9,7 @@ # Infra only: docker compose up # Single node: docker compose --profile production up # Production (PG): docker compose --profile production up -# (set DB_BACKEND, DATABASE_URL, POSTGRES_PASSWORD in .env) +# (set TURNSTONE_DB_BACKEND, TURNSTONE_DB_URL, POSTGRES_PASSWORD in .env) # # Set TURNSTONE_IMAGE_TAG in .env to pin the image version (default: latest). # ============================================================================= @@ -94,8 +94,8 @@ services: - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} - MODEL=${MODEL:-} - MCP_CONFIG=${MCP_CONFIG:-} - - TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite} - - TURNSTONE_DB_URL=${DATABASE_URL:-} + - TURNSTONE_DB_BACKEND=${TURNSTONE_DB_BACKEND:-sqlite} + - TURNSTONE_DB_URL=${TURNSTONE_DB_URL:-} - TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-} - TURNSTONE_ADVERTISE_URL=${TURNSTONE_ADVERTISE_URL:-http://server:8080} extra_hosts: @@ -128,8 +128,8 @@ services: environment: # Generate with: python -c "import secrets; print(secrets.token_hex(32))" - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} - - TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite} - - TURNSTONE_DB_URL=${DATABASE_URL:-} + - TURNSTONE_DB_BACKEND=${TURNSTONE_DB_BACKEND:-sqlite} + - TURNSTONE_DB_URL=${TURNSTONE_DB_URL:-} - TURNSTONE_CONSOLE_URL=http://console:8090 networks: - turnstone-net @@ -161,8 +161,8 @@ services: - TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0} # Generate with: python -c "import secrets; print(secrets.token_hex(32))" - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env} - - TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql} - - TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone} + - TURNSTONE_DB_BACKEND=${TURNSTONE_DB_BACKEND:-postgresql} + - TURNSTONE_DB_URL=${TURNSTONE_DB_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone} - TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091 networks: - turnstone-net