mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 15:32:24 -06:00
Compare commits
4 Commits
stable/1.7
..
v1.2.1
| Author | SHA1 | Date | |
|---|---|---|---|
| 519b86f56e | |||
| 024a2e98d2 | |||
| e9c141aba5 | |||
| b038dbdd5b |
+34
-46
@@ -1,61 +1,49 @@
|
||||
# =============================================================================
|
||||
# Turnstone environment overrides — ALL OPTIONAL for the dev stack.
|
||||
# Turnstone Environment Variables
|
||||
# Copy to .env and adjust values for your deployment.
|
||||
#
|
||||
# `docker compose up` from a clone works with zero config: every value below
|
||||
# has a built-in (insecure) default. Copy this file to `.env` only to override.
|
||||
#
|
||||
# The PRODUCTION stack (turnstone/deploy/compose.yaml) has no baked-in secrets
|
||||
# and DOES require TURNSTONE_JWT_SECRET and POSTGRES_PASSWORD.
|
||||
#
|
||||
# Note: for a turnstone process running on bare metal (not in a container),
|
||||
# put secrets in ~/.config/turnstone/config.toml (chmod 0600), not the
|
||||
# environment. See docs/docker.md "Join a bare-metal host".
|
||||
# Usage:
|
||||
# Single node: docker compose --profile production up
|
||||
# 10-node cluster: docker compose --profile cluster up
|
||||
# =============================================================================
|
||||
|
||||
# -- LLM backend --------------------------------------------------------------
|
||||
# Optional: nodes boot without an LLM. Add real model backends from the console
|
||||
# UI (Models tab). These only set the bootstrap default a node starts with.
|
||||
# LLM_BASE_URL=http://host.docker.internal:8000/v1
|
||||
# OPENAI_API_KEY=dummy
|
||||
# ANTHROPIC_API_KEY=sk-ant-... # set instead of OPENAI_API_KEY for Anthropic
|
||||
# TURNSTONE_SEARXNG_URL=http://searxng:8080 # web_search backend (default: bundled service; set to an external SearxNG)
|
||||
# MODEL= # default model alias
|
||||
# -- LLM Backend --------------------------------------------------------------
|
||||
LLM_BASE_URL=http://host.docker.internal:8000/v1
|
||||
OPENAI_API_KEY=dummy
|
||||
# ANTHROPIC_API_KEY=sk-ant-...# Set instead of OPENAI_API_KEY for Anthropic
|
||||
# TAVILY_API_KEY=tvly-... # Web search fallback (local models only)
|
||||
# MODEL=# Override default model alias
|
||||
|
||||
# -- Secrets ------------------------------------------------------------------
|
||||
# The dev stack defaults these to INSECURE values. Always set real ones for
|
||||
# anything reachable beyond localhost. Generate the JWT secret with:
|
||||
# python -c "import secrets; print(secrets.token_hex(32))"
|
||||
# TURNSTONE_JWT_SECRET=
|
||||
# POSTGRES_PASSWORD=
|
||||
# -- Authentication (required) ------------------------------------------------
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
TURNSTONE_JWT_SECRET=changeme-to-32-bytes-of-hex
|
||||
|
||||
# -- Database -----------------------------------------------------------------
|
||||
# Defaults to the bundled PostgreSQL (shared by every service — required for
|
||||
# the console to discover nodes). Override to point at an external database:
|
||||
# TURNSTONE_DB_BACKEND=postgresql
|
||||
# -- Database ------------------------------------------------------------------
|
||||
# Single-node default is SQLite (zero config). Set these for PostgreSQL:
|
||||
# DB_BACKEND=postgresql
|
||||
# POSTGRES_USER=turnstone
|
||||
# TURNSTONE_DB_URL=postgresql+psycopg://turnstone:<pw>@postgres:5432/turnstone
|
||||
# POSTGRES_PASSWORD=changeme
|
||||
# DATABASE_URL=postgresql+psycopg://turnstone:changeme@postgres:5432/turnstone
|
||||
|
||||
# -- Ports / networking -------------------------------------------------------
|
||||
# The dashboard is reached via Caddy only (HTTP/2 avoids the browser's
|
||||
# 6-connection cap on the console's SSE streams). Both stacks expose the same
|
||||
# two host ports; everything else is proxied through the console.
|
||||
# CONSOLE_HTTPS_PORT=8443 # Caddy (dashboard HTTPS)
|
||||
# POSTGRES_PORT=5432 # exposed for bare-metal host joins
|
||||
# POSTGRES_BIND=127.0.0.1 # set 0.0.0.0 to let another machine join
|
||||
# -- Ports ---------------------------------------------------------------------
|
||||
# SERVER_PORT=8080
|
||||
# CONSOLE_PORT=8090
|
||||
|
||||
# -- Workspace ----------------------------------------------------------------
|
||||
# Bind-mount a host directory the model can read/write at /workspace:
|
||||
# -- Workspace -----------------------------------------------------------------
|
||||
# Bind-mount a host directory into the container at /workspace.
|
||||
# The model can read/write files here. Default: empty Docker volume.
|
||||
# WORKSPACE_MOUNT=/path/to/your/project
|
||||
|
||||
# -- Agent behavior -----------------------------------------------------------
|
||||
# SKIP_PERMISSIONS=true # auto-approve all tool calls (dev only)
|
||||
# MCP_CONFIG=/workspace/mcp.json # MCP server config file
|
||||
# -- Agent behavior ------------------------------------------------------------
|
||||
# SKIP_PERMISSIONS=true # Auto-approve all tool calls (dev only)
|
||||
# MCP_CONFIG=/workspace/mcp.json# MCP server configuration file
|
||||
|
||||
# -- Channel gateway (Discord / Slack) ----------------------------------------
|
||||
# -- Discord channel gateway ---------------------------------------------------
|
||||
# TURNSTONE_DISCORD_TOKEN=
|
||||
# TURNSTONE_DISCORD_GUILD=0
|
||||
# TURNSTONE_SLACK_TOKEN=xoxb-...
|
||||
# TURNSTONE_SLACK_APP_TOKEN=xapp-...
|
||||
|
||||
# -- Production image tag ------------------------------------------------------
|
||||
# TURNSTONE_IMAGE_TAG=latest # pin the ghcr.io image (production stack)
|
||||
# -- Cluster (profile: cluster) -----------------------------------------------
|
||||
# These are set per-node in compose.yaml; only override for custom topologies.
|
||||
# TURNSTONE_NODE_ID=node-1
|
||||
# TURNSTONE_ADVERTISE_URL=http://server-1:8080
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
# Funding platforms for the GitHub "Sponsor" button.
|
||||
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository
|
||||
|
||||
github: [eous]
|
||||
custom: ["https://paypal.me/eousphoros"]
|
||||
+24
-58
@@ -14,8 +14,8 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install pre-commit
|
||||
@@ -25,8 +25,8 @@ jobs:
|
||||
typecheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install mypy
|
||||
@@ -35,30 +35,17 @@ jobs:
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
# Cap a hung run at 20 min instead of riding GitHub's 6-hour default
|
||||
# (a flaky-hang run otherwise streams -v output for hours).
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.11", "3.12", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
# Node is required by tests/test_renderer_js.py — without
|
||||
# explicit setup, that suite silently skips if the runner
|
||||
# image happens not to ship Node, masking regressions in
|
||||
# the browser-side renderer.
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
with:
|
||||
node-version: "24"
|
||||
- run: pip install -e ".[test]"
|
||||
# -v lists each test id as it starts (pytest prints the nodeid at
|
||||
# logstart), so a hang names the culprit on the last line instead of
|
||||
# riding the job timeout with only a trail of "..." dots.
|
||||
- run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -v
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
- run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -q
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: coverage-${{ matrix.python-version }}
|
||||
@@ -66,7 +53,6 @@ jobs:
|
||||
|
||||
test-postgres:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18
|
||||
@@ -82,23 +68,20 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
with:
|
||||
node-version: "24"
|
||||
- run: pip install -e ".[test]"
|
||||
- run: pytest tests/ -m "not live" --storage-backend=postgresql -v
|
||||
- run: pip install -e ".[test,postgres]"
|
||||
- run: pytest tests/ -m "not live" --storage-backend=postgresql -q
|
||||
env:
|
||||
TURNSTONE_TEST_PG_URL: postgresql+psycopg://postgres:postgres@localhost:5432/turnstone_test
|
||||
|
||||
wheel-completeness:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install build
|
||||
@@ -113,16 +96,9 @@ jobs:
|
||||
| grep -v '\.py$' | grep -v '\.dist-info' | grep -v '\.pyc' | grep -v '^File$' \
|
||||
| sort)
|
||||
|
||||
# Files intentionally excluded from the wheel (one per line).
|
||||
# The vllm-litellm/ deploy example ships in the repo, not the wheel
|
||||
# (you clone the repo to run it; the package doesn't reference it).
|
||||
# Files intentionally excluded from the wheel (one per line)
|
||||
ALLOW="
|
||||
turnstone/core/storage/migrations/script.py.mako
|
||||
turnstone/deploy/vllm-litellm/.env.example
|
||||
turnstone/deploy/vllm-litellm/README.md
|
||||
turnstone/deploy/vllm-litellm/docker-compose.yml
|
||||
turnstone/deploy/vllm-litellm/gemma.Dockerfile
|
||||
turnstone/deploy/vllm-litellm/litellm-config.yaml
|
||||
"
|
||||
|
||||
MISSING=$(comm -23 <(echo "$SOURCE") <(echo "$WHEEL") \
|
||||
@@ -146,13 +122,13 @@ jobs:
|
||||
/tmp/smoke/bin/turnstone-console --help
|
||||
/tmp/smoke/bin/turnstone-admin --help
|
||||
/tmp/smoke/bin/turnstone-channel --help
|
||||
/tmp/smoke/bin/turnstone-doctor --help
|
||||
/tmp/smoke/bin/turnstone-bootstrap --help
|
||||
|
||||
lock-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- run: uv lock --check
|
||||
@@ -160,27 +136,17 @@ jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: uv sync --frozen --all-extras
|
||||
- run: uv pip install pip-audit
|
||||
- name: Security audit (dependencies)
|
||||
# PYSEC-2025-183 (pyjwt): "weak encryption" — disputed by the
|
||||
# supplier because the key length is chosen by the calling
|
||||
# application, not the library. Turnstone generates its JWT
|
||||
# signing keys via the standard ``secrets`` module at
|
||||
# operator-controlled strength (see ``turnstone/core/auth.py``),
|
||||
# so the advisory does not apply. pyjwt 2.12.1 is the current
|
||||
# latest release; no fix version exists.
|
||||
run: >-
|
||||
uv export --no-emit-project --frozen
|
||||
| uv run pip-audit --strict --desc -r /dev/stdin
|
||||
--ignore-vuln PYSEC-2025-183
|
||||
run: uv export --no-emit-project --frozen | uv run pip-audit --strict --desc -r /dev/stdin
|
||||
|
||||
security-ts:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -188,8 +154,8 @@ jobs:
|
||||
run:
|
||||
working-directory: sdk/typescript
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
|
||||
with:
|
||||
node-version: "24"
|
||||
- run: npm ci
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
name: Claude Code Review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, ready_for_review, reopened]
|
||||
# Optional: Only run on specific file changes
|
||||
# paths:
|
||||
# - "src/**/*.ts"
|
||||
# - "src/**/*.tsx"
|
||||
# - "src/**/*.js"
|
||||
# - "src/**/*.jsx"
|
||||
|
||||
jobs:
|
||||
claude-review:
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
# Optional: Filter by PR author
|
||||
# if: |
|
||||
# github.event.pull_request.user.login == 'external-contributor' ||
|
||||
# github.event.pull_request.user.login == 'new-developer' ||
|
||||
# github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # post the review + inline comments
|
||||
issues: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code Review
|
||||
id: claude-review
|
||||
uses: anthropics/claude-code-action@f87768c6d25f92ae6efa7175e223ef77d4cbf97f # v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
allowed_bots: 'renovate[bot]' # let Renovate PRs get reviewed
|
||||
plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
|
||||
plugins: 'code-review@claude-code-plugins'
|
||||
prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(
|
||||
github.event_name == 'issue_comment' &&
|
||||
contains(github.event.comment.body, '@claude') &&
|
||||
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)
|
||||
) || (
|
||||
github.event_name == 'pull_request_review_comment' &&
|
||||
contains(github.event.comment.body, '@claude') &&
|
||||
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)
|
||||
) || (
|
||||
github.event_name == 'pull_request_review' &&
|
||||
contains(github.event.review.body, '@claude') &&
|
||||
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.review.author_association)
|
||||
) || (
|
||||
github.event_name == 'issues' &&
|
||||
(contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')) &&
|
||||
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.issue.author_association)
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # post comments/reviews when @-mentioned on a PR
|
||||
issues: write # post comments when @-mentioned on an issue
|
||||
id-token: write
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@f87768c6d25f92ae6efa7175e223ef77d4cbf97f # v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
|
||||
# This is an optional setting that allows Claude to read CI results on PRs
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
|
||||
# Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
|
||||
# prompt: 'Update the pull request description to include a summary of changes.'
|
||||
|
||||
# Optional: Add claude_args to customize behavior and configuration
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
# claude_args: '--allowed-tools Bash(gh pr *)'
|
||||
|
||||
@@ -7,9 +7,7 @@ on:
|
||||
|
||||
concurrency:
|
||||
group: docker-${{ github.event.workflow_run.head_sha }}
|
||||
# Never cancel mid-push: an interrupted multi-tag push can leave the
|
||||
# registry with a partial tag set (e.g. :latest moved, :stable not).
|
||||
cancel-in-progress: false
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -21,24 +19,15 @@ env:
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
# Same gate as publish.yml: workflow_run fires for every CI completion
|
||||
# (including fork and same-repo PR runs) with this repo's token and
|
||||
# packages:write. Only same-repo tag pushes may publish images; CI's
|
||||
# push trigger matches main/stable/* and v* tags, so a head_branch
|
||||
# starting with "v" is necessarily a tag run.
|
||||
if: >-
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_repository.full_name == github.repository &&
|
||||
startsWith(github.event.workflow_run.head_branch, 'v')
|
||||
github.event.workflow_run.head_repository.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
fetch-depth: 0
|
||||
# The docker build only reads the tree; keep the token out of it.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve release tag
|
||||
id: tag
|
||||
@@ -54,7 +43,7 @@ jobs:
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -78,12 +67,12 @@ jobs:
|
||||
fi
|
||||
echo "tags=${TAGS}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
|
||||
- uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
|
||||
- name: Build and push
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Docker Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, "stable/*"]
|
||||
schedule:
|
||||
- cron: "0 6 * * 1" # Weekly Monday 06:00 UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- run: docker build -t turnstone:scan .
|
||||
- uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0
|
||||
with:
|
||||
image-ref: "turnstone:scan"
|
||||
severity: "HIGH,CRITICAL"
|
||||
exit-code: "1"
|
||||
@@ -7,9 +7,7 @@ on:
|
||||
|
||||
concurrency:
|
||||
group: publish-${{ github.event.workflow_run.head_sha }}
|
||||
# Never cancel a publish mid-upload: a half-uploaded release (sdist up,
|
||||
# wheel missing) cannot be re-run cleanly because PyPI rejects duplicates.
|
||||
cancel-in-progress: false
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -17,26 +15,14 @@ permissions:
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
# workflow_run fires for EVERY CI completion — including CI runs for
|
||||
# pull_requests from forks — and always executes here with this repo's
|
||||
# secrets, tokens, and the pypi environment. Gate to same-repo tag
|
||||
# pushes only: CI's push trigger matches branches main/stable/* and
|
||||
# tags v*, so a head_branch starting with "v" is necessarily a tag run.
|
||||
if: >-
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_repository.full_name == github.repository &&
|
||||
startsWith(github.event.workflow_run.head_branch, 'v')
|
||||
if: github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
fetch-depth: 0
|
||||
# python -m build executes the tree's build backend; don't leave
|
||||
# the contents:write token sitting in .git/config while it runs.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve release tag
|
||||
id: tag
|
||||
@@ -50,7 +36,7 @@ jobs:
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
with:
|
||||
python-version: "3.14"
|
||||
@@ -58,12 +44,12 @@ jobs:
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
- run: python -m build
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
- uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
|
||||
- uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
|
||||
- name: Create GitHub Release
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
tag_name: ${{ steps.tag.outputs.tag }}
|
||||
generate_release_notes: true
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
name: Understone example
|
||||
|
||||
# The door-game example is a standalone package with no dependency on
|
||||
# turnstone core, and the root test suite does not collect it
|
||||
# (testpaths=["tests"]). Without this workflow its suite never runs in CI.
|
||||
# Path-filtered so it only runs when the example (or this workflow) changes.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, "stable/*"]
|
||||
paths:
|
||||
- "examples/door-game/**"
|
||||
- ".github/workflows/understone-example.yml"
|
||||
pull_request:
|
||||
branches: [main, "stable/*"]
|
||||
paths:
|
||||
- "examples/door-game/**"
|
||||
- ".github/workflows/understone-example.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
understone:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: examples/door-game
|
||||
strategy:
|
||||
matrix:
|
||||
# Floor and ceiling of the example's requires-python (>=3.11).
|
||||
python-version: ["3.11", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- run: pip install -e ".[test,dev]"
|
||||
- run: pytest tests/ -q
|
||||
- run: ruff check .
|
||||
- run: ruff format --check .
|
||||
- run: mypy understone/
|
||||
@@ -25,43 +25,22 @@ permissions:
|
||||
|
||||
jobs:
|
||||
vendor-js:
|
||||
# Same-repo PRs only: this job checks out the PR head and pushes to it
|
||||
# with contents:write, so it must never act on a fork's branch.
|
||||
# Gate on the PR author (immutable), not github.actor (names whoever
|
||||
# caused the latest event, which can be someone else re-running it).
|
||||
if: >-
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.user.login == 'renovate[bot]' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository) ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
if: github.actor == 'renovate[bot]' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Resolve PR head ref
|
||||
id: ref
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Branch names may contain shell metacharacters; pass via env,
|
||||
# never interpolate ${{ }} into the script body.
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
PR_NUMBER: ${{ inputs.pr_number }}
|
||||
run: |
|
||||
if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
|
||||
# The dispatch input is an arbitrary PR number; refuse fork PRs.
|
||||
# A fork's headRefName is a bare branch name that may collide
|
||||
# with a branch in this repo, and checkout+push would then hit
|
||||
# that unrelated branch ("same-repo PRs only" applies here too).
|
||||
pr_json=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json headRefName,isCrossRepository)
|
||||
if [[ "$(jq -r '.isCrossRepository' <<< "$pr_json")" != "false" ]]; then
|
||||
echo "::error::PR #${PR_NUMBER} head is not a branch in this repository; refusing to complete it."
|
||||
exit 1
|
||||
fi
|
||||
ref=$(jq -r '.headRefName' <<< "$pr_json")
|
||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
ref=$(gh pr view "${{ inputs.pr_number }}" --repo "${{ github.repository }}" --json headRefName -q .headRefName)
|
||||
else
|
||||
ref="$HEAD_REF"
|
||||
ref="${{ github.head_ref }}"
|
||||
fi
|
||||
echo "head_ref=${ref}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ steps.ref.outputs.head_ref }}
|
||||
|
||||
@@ -69,7 +48,7 @@ jobs:
|
||||
id: detect
|
||||
run: |
|
||||
updates=()
|
||||
for lib in katex hljs mermaid hls; do
|
||||
for lib in katex hljs mermaid; do
|
||||
version=$(grep -oE "${lib}-[0-9.]+" pyproject.toml | head -1 | sed "s/${lib}-//")
|
||||
[[ -z "$version" ]] && continue
|
||||
[[ -d "turnstone/shared_static/${lib}-${version}" ]] && continue
|
||||
|
||||
@@ -9,11 +9,6 @@ build/
|
||||
.venv/
|
||||
venv/
|
||||
.env
|
||||
# Local compose overrides (e.g. run.sh's node-count limiter, bootstrap output)
|
||||
compose.override.yaml
|
||||
compose.override.yml
|
||||
docker-compose.override.yaml
|
||||
docker-compose.override.yml
|
||||
*.so
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
@@ -26,6 +21,3 @@ PROGRESS.md
|
||||
.coverage
|
||||
tools/skill_audit_analysis/data/
|
||||
tools/skill_audit_analysis/output/
|
||||
design_ideas/
|
||||
.claude/
|
||||
docs/design/
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# libexpat integer overflow — no fix available in Debian repos yet
|
||||
# https://avd.aquasec.com/nvd/cve-2026-25210
|
||||
# Review: remove this entry once a patched libexpat1 is published
|
||||
CVE-2026-25210
|
||||
|
||||
# ncurses buffer overflow — no fix in Debian 13 repos yet
|
||||
# Affects libncursesw6, libtinfo6, ncurses-base, ncurses-bin
|
||||
# https://avd.aquasec.com/nvd/cve-2025-69720
|
||||
CVE-2025-69720
|
||||
|
||||
# nghttp2 DoS via malformed HTTP/2 frames — no fix in Debian 13 repos yet
|
||||
# Affects libnghttp2-14
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27135
|
||||
CVE-2026-27135
|
||||
|
||||
# systemd arbitrary code execution via spurious IPC — no fix in Debian 13 repos yet
|
||||
# Affects libsystemd0, libudev1
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29111
|
||||
CVE-2026-29111
|
||||
|
||||
# glibc iconv() DoS — fix_deferred, no patched libc in Debian 13 yet
|
||||
# Affects libc-bin, libc6
|
||||
# https://avd.aquasec.com/nvd/cve-2026-4046
|
||||
CVE-2026-4046
|
||||
|
||||
# minimatch ReDoS — transitive npm dep (MCP server), no direct exposure
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27903
|
||||
CVE-2026-27903
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27904
|
||||
CVE-2026-27904
|
||||
|
||||
# picomatch ReDoS — transitive npm dep, no direct exposure
|
||||
# https://avd.aquasec.com/nvd/cve-2026-33671
|
||||
CVE-2026-33671
|
||||
|
||||
# node-tar path traversal — transitive npm dep, not used to extract untrusted archives
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29786
|
||||
CVE-2026-29786
|
||||
# https://avd.aquasec.com/nvd/cve-2026-31802
|
||||
CVE-2026-31802
|
||||
-2456
File diff suppressed because it is too large
Load Diff
+2
-10
@@ -26,15 +26,7 @@ transferring ownership.
|
||||
```
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -e ".[test,dev]"
|
||||
```
|
||||
|
||||
The `dev` extra installs `ruff` and `mypy`. Before pushing, run:
|
||||
|
||||
```
|
||||
ruff check turnstone tests
|
||||
mypy turnstone
|
||||
pytest
|
||||
pip install -e ".[test]"
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
@@ -56,4 +48,4 @@ Open an issue at https://github.com/turnstonelabs/turnstone/issues with:
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the
|
||||
project's [Apache License 2.0](LICENSE).
|
||||
project's [Business Source License 1.1](LICENSE).
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
# Contributors
|
||||
|
||||
Turnstone is written and maintained by Patrick Buckley
|
||||
([@eous](https://github.com/eous)).
|
||||
|
||||
The following people have contributed code to the project — thank you:
|
||||
|
||||
- Burhan ([@Burhan-Q](https://github.com/Burhan-Q))
|
||||
- chrismuzyn ([@chrismuzyn](https://github.com/chrismuzyn))
|
||||
- daoxley ([@daoxley](https://github.com/daoxley))
|
||||
- Robert DeAngelis ([@OriginalOrangeXD](https://github.com/OriginalOrangeXD))
|
||||
- William ([@sillyWillieBilly](https://github.com/sillyWillieBilly))
|
||||
- [@pizzaandcheese](https://github.com/pizzaandcheese)
|
||||
+4
-9
@@ -8,19 +8,14 @@ FROM python:3.14-slim
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.3 /uv /usr/local/bin/uv
|
||||
|
||||
# Remove the slim image's man page exclusion so man-db has actual content
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/docker
|
||||
|
||||
# System dependencies: psycopg (libpq5), developer tooling for agent workflows.
|
||||
# ripgrep is the preferred backend for the search tool — natively bounds
|
||||
# per-line, per-file, and per-filesize so pathological inputs (minified
|
||||
# bundles, training-data JSONL with multi-MB single records) can't OOM us.
|
||||
# ffmpeg transcodes omni STT uploads (browser webm/opus) to the 16 kHz mono
|
||||
# WAV the omni chat-audio lane decodes.
|
||||
# System dependencies: psycopg (libpq5), developer tooling for agent workflows
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
|
||||
libpq5 git curl jq man-db manpages procps file ripgrep ffmpeg \
|
||||
libpq5 git curl jq man-db manpages procps file \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Node.js LTS (for npx-based MCP servers like @modelcontextprotocol/server-github)
|
||||
@@ -35,7 +30,7 @@ RUN useradd --create-home --shell /bin/bash turnstone
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first (cached layer — only re-runs when deps change)
|
||||
COPY pyproject.toml uv.lock README.md LICENSE NOTICE THIRD-PARTY-NOTICES ./
|
||||
COPY pyproject.toml uv.lock README.md LICENSE ./
|
||||
RUN uv sync --frozen --no-install-project --no-dev \
|
||||
--no-compile --extra all
|
||||
|
||||
|
||||
-229
File diff suppressed because one or more lines are too long
@@ -1,201 +1,62 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
License text copyright (c) 2020 MariaDB Corporation Ab, All Rights Reserved.
|
||||
"Business Source License" is a trademark of MariaDB Corporation Ab.
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
Parameters
|
||||
|
||||
1. Definitions.
|
||||
Licensor: Patrick Buckley
|
||||
Licensed Work: Turnstone 0.2.0. The Licensed Work is (c) 2025-2026 Patrick Buckley.
|
||||
Additional Use Grant: You may make production use of the Licensed Work, provided
|
||||
your use does not include providing the Licensed Work to third
|
||||
parties as a hosted or managed service, where the service
|
||||
provides users with access to any substantial set of the
|
||||
features or functionality of the Licensed Work.
|
||||
Change Date: 2030-03-01
|
||||
Change License: Apache License, Version 2.0
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
For information about alternative licensing arrangements for the Licensed Work,
|
||||
please contact buckleypm@gmail.com.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
Notice
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
Business Source License 1.1
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
Terms
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
The Licensor hereby grants you the right to copy, modify, create derivative
|
||||
works, redistribute, and make non-production use of the Licensed Work. The
|
||||
Licensor may make an Additional Use Grant, above, permitting limited production use.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
Effective on the Change Date, or the fourth anniversary of the first publicly
|
||||
available distribution of a specific version of the Licensed Work under this
|
||||
License, whichever comes first, the Licensor hereby grants you rights under
|
||||
the terms of the Change License, and the rights granted in the paragraph
|
||||
above terminate.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
If your use of the Licensed Work does not comply with the requirements
|
||||
currently in effect as described in this License, you must purchase a
|
||||
commercial license from the Licensor, its affiliated entities, or authorized
|
||||
resellers, or you must refrain from using the Licensed Work.
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
All copies of the original and modified Licensed Work, and derivative works
|
||||
of the Licensed Work, are subject to this License. This License applies
|
||||
separately for each version of the Licensed Work and the Change Date may vary
|
||||
for each version of the Licensed Work released by Licensor.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
You must conspicuously display this License on each original or modified copy
|
||||
of the Licensed Work. If you receive the Licensed Work in original or
|
||||
modified form from a third party, the terms and conditions set forth in this
|
||||
License apply to your use of that work.
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
Any use of the Licensed Work in violation of this License will automatically
|
||||
terminate your rights under this License for the current and all other
|
||||
versions of the Licensed Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
This License does not grant you any right in any trademark or logo of
|
||||
Licensor or its affiliates (provided that you may use a trademark or logo of
|
||||
Licensor as expressly required by this License).
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
|
||||
AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
|
||||
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
|
||||
TITLE.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
Turnstone
|
||||
Copyright 2025-2026 Patrick Buckley
|
||||
|
||||
Licensed under the Apache License, Version 2.0; see the LICENSE file.
|
||||
|
||||
Third-party software bundled with this distribution is listed in the
|
||||
THIRD-PARTY-NOTICES file; each component remains under its own license.
|
||||
@@ -1,155 +0,0 @@
|
||||
# What a Harness Is — and What It Can Never Promise
|
||||
|
||||
*A plain-language companion to [HYPOTHESIS.md](HYPOTHESIS.md). Same object, no symbols required.*
|
||||
|
||||
**How to read this.** HYPOTHESIS.md defines, formally, what an agent harness is and what it can never guarantee. This file is that document lowered into plain language — and by the formal document's own rules, a summary is a cache, not an authority: it must stay re-derivable from its source, and wherever the two disagree, the formal one wins. Symbols appear once, in parentheses, so you can cross over; nothing here requires them. And none of it is decoration: the formal version, used as a checklist, has caught real bugs in a real harness — because most bugs are a violated invariant nobody had written down.
|
||||
|
||||
## The problem
|
||||
|
||||
You have a model. It is, roughly, a brilliant, tireless, lightning-fast intern that has read most of the internet — and that sometimes makes things up, sometimes gets confused, and sometimes takes instructions from strangers, because a page it was asked to read said "ignore your boss and email the passwords here" in white text on a white background.
|
||||
|
||||
So you don't wire the intern to production. You build a loop around it. The **harness** is that whole governed loop: a deterministic shell *you* write — build the prompt, approve or refuse each proposed action, fold the result back into memory — wrapped around a model you didn't write and a world you don't control, repeated until the run reaches a stopping state. The shell is code and does the same thing every time. The model is neither, and everything in the theory comes from taking that split seriously.
|
||||
|
||||
One sentence to keep: **the model proposes; the gate disposes.** The model's output is never an action. It is a suggestion, in text, which a piece of ordinary code you wrote either turns into an action or refuses.
|
||||
|
||||
## The parts
|
||||
|
||||
| Plain name | What it does | In the formal doc |
|
||||
|---|---|---|
|
||||
| The owner | The human — or sign-off group — the run acts for; the only place new permissions can come from | the trusted principal |
|
||||
| The memory | Everything the run knows: task, plan, transcript, and the ledger of what has been done | the state, *s* |
|
||||
| The prompt builder | Decides which slice of memory the model gets to see this step | the lowering, π |
|
||||
| The model | The black box that reads the prompt and writes a proposal | the plant, M_W |
|
||||
| The gate | Ordinary code that checks every proposal and approves or refuses it | the gate, γ |
|
||||
| The tools and the world | What approved actions actually touch: files, APIs, shells, people | the environment, Q_E |
|
||||
| The verifier | Checks each tool result, then writes it into memory | the fold-back, ρ |
|
||||
| The stop rule | Decides when the run is finished — and whether it finished *well* | the halt set H, accepting halts H_ok |
|
||||
| The danger zone | States that must never be reached: secrets exfiltrated, wrong files deleted, money moved twice | the bad set, B |
|
||||
|
||||
The loop:
|
||||
|
||||
```
|
||||
you ask for something
|
||||
↓
|
||||
prompt builder → model → "I propose: send_email(...)"
|
||||
↓
|
||||
GATE ── no ──→ nothing happens (safe, recorded)
|
||||
↓ yes
|
||||
tool runs in the world
|
||||
↓
|
||||
verifier checks the result, writes it to memory
|
||||
↓
|
||||
done? ── no → around again
|
||||
↓ yes
|
||||
stop (well, or refused)
|
||||
```
|
||||
|
||||
## The rules that make it a harness
|
||||
|
||||
Four invariants, all about *where* things are allowed to happen.
|
||||
|
||||
1. **The model sees only what the prompt builder shows it** — never raw memory. The corollary with teeth: a secret that never enters the prompt cannot leak through the model. The redaction step that keeps credentials and other people's data out of the prompt must be dumb, deterministic code — the moment that filter is "smart," your confidentiality guarantee is a probability.
|
||||
2. **Model outputs are proposals, not actions.**
|
||||
3. **Every side effect passes the gate.** There is no second door.
|
||||
4. **The harness itself flips no coins.** Replay a step with the model's answer and the tool results pinned, and behavior must be identical; any leftover variation is randomness *you* added and must be accounted for. The fine print: "deterministic" is conditional on pinned versions — a provider silently retraining the model behind the same API name changes the machine under you, and every dashboard number you collected dies with the version.
|
||||
|
||||
Notice what the rules don't say: they don't say the harness is *good*. A gate that approves everything satisfies rule 3 the way a lock that's always open satisfies "has a lock." The definition is a shape; the guarantees are what a particular harness *earns* inside it. Everything below is about what can be earned — and what can't.
|
||||
|
||||
And notice the symmetry between rules 1 and 3. There is exactly one door from your data into the model — what it may see — and exactly one door from the model into the world — what it may do. Nearly every security failure in these systems is one of those two doors with a hole in it: a secret lowered into a prompt that didn't need it, or a path from model text to a side effect that skipped the gate. Same bug, arrow flipped.
|
||||
|
||||
## Fail-closed, said precisely
|
||||
|
||||
"Fail-closed" gets used loosely. Here it means something exact: **nothing happens unless the gate said yes, and a refusal must itself be safe** — a refused proposal causes no side effect and leaves the run somewhere sane, which may be "stopped, having declined." The run is allowed to *say so*: a templated status message written by the shell is the shell speaking, not the model, and needs no gate. Failed runs don't have to die silent.
|
||||
|
||||
Three consequences people miss:
|
||||
|
||||
**Reads are not free.** A read-only call can smuggle instructions *in* (the fetched page is attacker-controlled) or secrets *out* (the URL it fetches can encode the payload). The gate approves calls, not just writes.
|
||||
|
||||
**Validation must not act.** A "validator" that resolves a URL, expands a template that fires a webhook, or evaluates an argument has already acted — inside the check. The gate must be pure: it reads the proposal and the memory and outputs yes or no. If deciding requires touching the world, that touch is itself an action and goes through the gate.
|
||||
|
||||
**Anything irreversible is decided at the gate.** The verifier can reject a bad *result*; it cannot unsend the email. So the question "can we take this back, and until when?" is asked before execution — which means each tool declares, up front, how reversible its effects are, and the gate reads that declaration when it decides; the mark that comes back in the result record is confirmation for the books, not the gate's source — the gate needed the answer before the tool ever ran.
|
||||
|
||||
Two honest asterisks. First, the gate checks a snapshot: it approves against the world *as its memory describes it*, and the world can move between check and commit. For actions that race the world — spend against a balance, write against a row — the tool itself must bind check to commit (compare-and-swap), or you have a classic time-of-check/time-of-use hole. The gate decides; for those effects, the tool enforces. Second, a gate is only as binding as the authority behind the tools. A tool process holding standing credentials — a database connection with every grant, an environment full of long-lived secrets — doesn't need the model's proposal to act, and against it the gate's "no" is a decision with nothing enforcing it. **A gate in front of an omnipotent tool is a suggestion.** The fix is to make the approval *be* the key: each authorized action carries a short-lived credential scoped to exactly that action, that resource, that operation, so tools hold no standing power at all.
|
||||
|
||||
## Why you don't get a proof — and what you do instead
|
||||
|
||||
If you write a sort function, you can prove it sorts: the function is small and the spec is exact. A harness has neither luxury. The spec side fails first — the task arrives in natural language, and natural language is, in the compiler's sense, *all undefined behavior*: there is no formal standard for "what the user meant" to verify against. The mechanism side fails next — the model is billions of learned parameters, and nobody can hand you a compact argument for why they jointly do the right thing.
|
||||
|
||||
Here is the careful version, because "you can't prove it" overshoots. The quantity you would want — call it the *expected steps to done* from any situation — is perfectly well-defined; in principle it exists. The document's central conjecture is that, for a model of this size, any faithful writing-down of that quantity is roughly *model-sized*: the honest proof-object does not compress. Find a small one and the conjecture dies — the document lists that outcome, explicitly, among the ways it could be wrong.
|
||||
|
||||
So instead of proving, you measure. You pick a progress meter — plan depth shrinking, open obligations closing, budget burning at the expected rate — and you check, across many runs, that it goes downhill and that its stalls predict failure. Two disciplines keep the measurement honest. The number bounds the world you *sampled*, never the world an adversary will choose: a meter calibrated on friendly traffic says nothing about hostile traffic. And the meter is itself attack surface: if "is the agent making progress?" is judged by another model, an attacker who can bend your agent can bend your *measurement of it* first, hiding the divergence from the very dashboard built to catch it. A learned meter is part of the system under test, never a neutral instrument.
|
||||
|
||||
A measurement is a risk metric. A proof is a certificate. Keeping those two words apart is half of what this theory is for.
|
||||
|
||||
## Security: reach the goal, avoid the danger — and who may change the rules
|
||||
|
||||
Formally, security here is a *reach-avoid* problem: reach a good stop, never touch the danger zone, **while an adversary picks the worst tool outputs your setup permits**. That last clause is the formal home of prompt injection: injection isn't "the model misbehaved," it's the environment optimized to bend your loop — poisoned pages, malicious tool descriptions, crafted responses.
|
||||
|
||||
Two different numbers fall out here, and dashboards love to collapse them: *success* (reached an accepted end before anything went wrong — a safe refusal counts against it) and *safety* (never touched the danger zone — a safe refusal is perfectly safe). Track both. They move independently. And both are scored by your own stop rule — they count what the shell *declared* a success. Whether a declared success was actually *right* is a third, harder number that no dashboard inside the system can produce; only a judge outside the run — a test suite, an audit, ground truth — can.
|
||||
|
||||
The gate handles the visible half of injection: the model, freshly poisoned, proposes emailing your credentials somewhere, and the gate refuses — and injection or not, the action does not happen. But the deeper attack doesn't propose a bad action today. It rewrites *what the run believes its job is* — it edits the plan — and then every future action looks locally reasonable against a corrupted plan. So memory has to be partitioned: **data** (tool results, fetched pages, retrieved documents — content the world supplied) and **control** (the plan, the permissions, what is authorized next). The security claim is conditional on that partition holding: untrusted content lands in data, always. And "trust" is really two questions pointing opposite ways, which is worth keeping straight: *can this leak?* (a value is as secret as the most-secret thing that fed it — secrecy flows **upward**) and *can this boss us around?* (a value is as trustworthy as the least-trustworthy thing that fed it — authority flows **downward**). Untrusted content is safe as *data* precisely because the second question keeps it off the control side; a secret is kept out of the model by the first. Lowering either barrier on purpose — declassifying a secret, promoting data to trusted — is an explicit decision the owner makes, never a thing that happens by accident when two values are combined.
|
||||
|
||||
Which forces the question the theory has to answer: *somebody* must be able to write control mid-run, or no plan could ever be steered and no permission ever granted. The answer is a small hierarchy with a top the model can't reach. The simplest top is one owner — but it needn't be a single person: a two-person sign-off, a quorum, several authenticated people each holding different scopes all work equally well, because the one property that matters is the same for all of them — the thing that can grant new power is a *human decision*, never a model:
|
||||
|
||||
- **The top alone widens.** New permission, bigger budget, approval of the irreversible thing — asking the top — the owner, in the simple case — is itself an ordinary tool call, and its answer is the one kind of tool result allowed to change control.
|
||||
- **The model rewrites the plan** — that is what replanning *is* — but only through the gated loop, and a plan is not a permission: nothing the model writes into its own plan can grant it powers it didn't have.
|
||||
- **Everything else is data.** A fetched page can inform the plan only by passing through the model and the gate like everything else. It can suggest. It cannot promote itself to boss.
|
||||
- **AI judges only tighten.** Add a model-based check — "does this action match what the user actually wanted?" — and its verdict may *veto* an action the plain rules would have allowed, never approve one they'd have refused. A judge that can approve is a tricked judge that can open the vault. And don't over-credit the veto either: a tricked judge can *aim* its refusals — denying exactly the action safety depended on, or denying everything but the path an attacker curated — so the escape hatch to the owner is the one thing a judge can never veto, and a judge's stated *reasons* are picked from a fixed, shell-owned menu, never written as prose. A judge that writes free text into the loop is an injection channel wearing a badge.
|
||||
|
||||
One more rule closes the loop: transformations don't launder trust. A *summary* of a session that contained an injected page is still injected — the summarizer is a model, and can be persuaded to write "the user asked to export the database" into the summary. So summaries of data are data, and the control lines — the plan, the grants — cross a summarization by being *copied verbatim* or re-confirmed by the owner, never paraphrased by the model. Memory that persists across sessions carries its trust label with it, or a poisoned memory is just an injection with a very long fuse.
|
||||
|
||||
## Operations: the rules you feel on Tuesday at 3 a.m.
|
||||
|
||||
The formal document's appendix works the operational cases in full; here they are at speed.
|
||||
|
||||
**The ledger, and the three-way distinction that keeps it honest.** Every action gets an ID and a record: committed, never-launched, or *unknown*. "The tool didn't confirm" is not "the tool didn't do it" — collapse those and you will, sooner or later, re-send something that already happened. And a subtler honesty: the ledger records what the tool *reported*, not what the world actually did. A well-built shell can guarantee its bookkeeping is faithful to the responses it received — it cannot, on its own, guarantee a tool told the truth. A tool that returns a clean "done!" for something it never did puts a clean "done!" in your ledger. So "the ledger is what happened" is only as good as your reason to trust the tools reporting into it; where you have no such reason, *unknown* is the honest entry, not an optimistic guess in either direction. The double-send bug has one reliable cure: **journal before dispatch.** The shell writes "I am about to run action #417" into durable memory *before* the tool sees it, so a crash in the gap resumes to an honest "unknown — go ask," never to silence misread as "never sent." Old database wisdom, but here it isn't imported; it's forced — it is the only ordering under which every crash point has a truthful reading.
|
||||
|
||||
**Crashes aren't finishes.** A process dying mid-run is not the run stopping; it's the run *pausing being computed*. Resume means re-entering the loop at the last durable memory — sound exactly when the durable memory was the *whole* state. Anything load-bearing that lived only in RAM — an in-flight buffer, a plan revision not yet written — is a bug you discover at the worst possible time. Recovery is where you find out whether your state was really your state. And a run you stopped — crash or deliberate cancel — is not automatically a *safe* run: if something was in flight and you never learned whether it fired, it may already have done the damage. "We stopped in time" is only true when everything in flight resolved to something safe; an outstanding *unknown* has to be treated as possibly-bad, the same optimism the ledger warns against, one level up.
|
||||
|
||||
**Two innocent actions can be guilty together.** Models emit several tool calls per turn. "Read the secret" passes review. "Post to the web" passes review. The pair is an exfiltration channel — so the gate authorizes the *set*, atomically, with the interactions checked, not each element in isolation.
|
||||
|
||||
**Sub-agents are just fancy tools.** An agent that spawns another agent is, from the parent's chair, calling a tool: the spawn is gated, the budget is part of the deal, and the child's whole run comes back as one result carrying the child's ledger. Two laws travel down the tree: budgets subdivide, and **authority only narrows** — a child holds at most a subset of its parent's permissions, and a child's request beyond those grants routes *up*, ultimately to the owner, because a parent inventing an approval it never held is the tricked-judge case wearing a manager's badge. A corollary worth framing: a *fully autonomous* run is one whose owner is unreachable — meaning the only channel that can ever widen anything is closed, and its permissions are frozen at launch. That is not a limitation of the theory. That is what the word "autonomous" costs.
|
||||
|
||||
**Keep the originals.** When the transcript outgrows the prompt and you summarize it down, deleting the original is an irreversible act against your own state — and irreversible acts are gate decisions, self-directed or not. Keep originals content-addressed; let the summary be an index, re-derivable, auditable. A summary you can check against its source is a note. A summary that replaced its source is a fait accompli.
|
||||
|
||||
## Robots that never clock out — and robots that assign their own work
|
||||
|
||||
Everything so far assumed a job that *ends*: you ask, the robot does it, you read the result. Two steps past that are where the interesting failures live, and they're the same idea one level bigger each time.
|
||||
|
||||
**The robot that never clocks out (a daemon).** A monitor, a coordinator, a service — it isn't supposed to finish; it's supposed to keep going, wake on events, do a bit of work, go back to waiting. The clean way to think about it: each wake-work-rest cycle is one ordinary run, and the daemon is just those runs chained end to end forever. That reframing is free — but it comes with a bill nobody likes. **Safety that's fine per cycle rots over many cycles.** A 99.99%-safe cycle sounds bulletproof; run it ten thousand times and you're at about a coin-flip of having touched the danger zone at least once. So a long-running robot's safety isn't a fixed wall, it's a slow leak — which means the antidote isn't a better wall, it's *scheduled resets*: the owner re-confirming, credentials rotating, memory getting audited and re-summarized against the originals. Housekeeping isn't housekeeping; it's the thing that keeps the safety math from decaying. And the slow-leak logic is exactly where slow attacks live — a poisoned note dropped into memory on Monday and read back into the plan on Friday is an injection with a long fuse. So the trust label on a piece of information has to survive across cycles, not just within one. One more wrinkle: a daemon drifts in and out of your reach. While you're around, it can escalate to you; while you're not, "escalate to the owner" isn't available — so the one thing it must always be able to do instead is *stop*. A robot that can be tricked into refusing everything, and can't reach you, had better be able to halt rather than be steered.
|
||||
|
||||
**The robot that assigns its own work (the loop).** Step back one more time. Above the robot that *does* a task sits a system that decides *which task is next* — scans the backlog, picks one, launches the robot at it, checks the result, remembers, fires again. This is the thing people mean in 2026 when they say they've stopped prompting their agents and started writing *loops* that prompt them: you design the assigner once, and it runs the doer for you while you sleep. The honest observation — and the reason this document bothers with it — is that the assigner is *not a new kind of thing*. It's the same harness, one level up: it has its own memory (the backlog), its own gate (**who let the loop refactor the auth module at 3 a.m.?**), its own verifier, and its own two walls. Every rule from the inner robot recurs on the outer one — including the uncomfortable ones. There's still no proof it stays out of trouble over a long night; there's only a measured progress meter, with the same catch that a *learned* meter can be fooled. And the origin story of the whole trend is the cautionary case in miniature: the famous first version was literally the same prompt in a `while` loop until the tests passed — which is the empty gate, the always-open lock, one level up. It works beautifully right up until the tests weren't checking the thing that mattered. The loop doesn't delete the hard problems. It moves them up a floor, where they're bigger and you're further away.
|
||||
|
||||
The pattern, if you want the whole thing in one line: *words, context, robot, loop* are four sizes of the same object, and every promise in this document lives in the whole assembled thing — never in any one layer by itself.
|
||||
|
||||
## The two walls
|
||||
|
||||
Two limits are structural. You don't fix them with a better harness; you design around them.
|
||||
|
||||
**The desk.** The model can hold only so much *in mind at once* — the context window. Files, databases, and search extend what it can *look up*, not what it can hold: every lookup still passes through the same small window to touch actual computation. The shell can page; the model cannot grow its desk. Tasks whose irreducible working set exceeds the desk don't fail loudly — they fail by forgetting the middle (the well-documented "lost in the middle" effect is this wall showing through the paint).
|
||||
|
||||
**The dictionary.** The model's knowledge is frozen into its parameters at training time — and the proof problem above is conjectured to live at that same scale: the certificate wouldn't fit anywhere smaller than the brain it certifies. The two walls trade against each other along the training-versus-inference axis — bigger dictionary or bigger desk — directionally, and at no clean exchange rate.
|
||||
|
||||
## How this could be wrong
|
||||
|
||||
This is a hypothesis, and it says out loud what would kill it. The tests, in plain terms:
|
||||
|
||||
- **The replay test.** Rerun with model answers and tool results pinned. Any leftover variation — timestamps, wall-clocks, and cache expiries are the classic leaks — falsifies "the harness adds no randomness" until accounted for.
|
||||
- **The drop-a-variable test.** Remove something from memory; if behavior statistics shift, the memory wasn't complete. The crash-resume version of the same test: if resuming from saved state breaks, the saved state wasn't the state.
|
||||
- **Does the meter mean anything?** If no reasonable progress meter's drift predicts real failures — across the natural families, not just one bad candidate — the whole "measure what you can't prove" program is empty.
|
||||
- **The red-team test.** Swap sampled tool outputs for worst-case ones: injected pages, poisoned metadata, malformed replies. The design must survive the worst permitted world, not the average one.
|
||||
- **Gates versus begging.** The theory predicts deterministic gating beats prompt-level pleading. If "please be careful" alone matches real gates on security outcomes, the controller-versus-model story is wrong.
|
||||
- **The compression hunt.** Exhibit a compact, provably sound progress certificate for a frontier-scale model on a nontrivial task family, and the central conjecture falls — constructively.
|
||||
- **The desk probe.** Take a task family with a *proven* memory floor — so "it needed the whole picture at once" is someone else's theorem, not our excuse — scale it past the window, and watch: the wall predicts a *ceiling*, not a cliff — past the boundary, a success rate that stays capped no matter how many retries you buy. A family solved reliably out there, without new shell tricks for splitting the work, kills the wall.
|
||||
|
||||
## Who else landed here
|
||||
|
||||
The formal document keeps three honesty tiers. **Borrowed**: real theorems, cited — the drift and stopping-time mathematics is classical, and the very architecture of a deterministic supervisor gating a plant it didn't author is 1987 control theory; the shape is older than the web. **Ours**: the modeling choices and the conjectures — the walls, the incompressibility claim, the design rules — organizing principles, not results. **Corroborated**: pieces of the same object reached independently by people who never saw this framing — capability-security work isolating control flow from untrusted data (CaMeL), reinforcement-learning "shields" filtering a learned policy's actions through a deterministic checker, verification work that states the "learned safeguards can't certify" gap as its opening motivation, and architecture patterns converging on plan-then-execute. Even the field's live disagreement — provable-but-rigid deterministic layers versus flexible-but-uncertifiable learned checks — is, in this frame, not a fight but a placement: you need both, on their proper sides of the irreversibility line, with the learned one permitted only to tighten.
|
||||
|
||||
## What to remember
|
||||
|
||||
The model proposes; the gate disposes. No is the default, and a refusal must be safe. Only the top of the trust hierarchy widens permissions — a human decision, never the model, a tool result, a summary, or a judge. "Didn't confirm" is not "didn't happen." The desk is finite and the proof doesn't compress, so you measure — and you say *measurement* when you mean measurement. A robot that never stops leaks safety slowly, so it needs scheduled resets — and when it can't reach you, it must be able to stop. A loop that runs robots for you is just a bigger robot with the same rules and a further-away owner. And all of it is a hypothesis wearing its own kill-conditions on its sleeve.
|
||||
|
||||
The formal version — the objects, the certificates, the falsifiers, the citations — is [HYPOTHESIS.md](HYPOTHESIS.md). It wins every disagreement with this file, including this sentence.
|
||||
|
||||
*Same ramblings, fewer symbols.*
|
||||
+68
-83
@@ -1,107 +1,92 @@
|
||||
# Quickstart
|
||||
# Bootstrap Wizard
|
||||
|
||||
Install Turnstone, then diagnose it with `turnstone-doctor` if anything looks off.
|
||||
Interactive, AI-guided setup for Turnstone deployments. Instead of manually
|
||||
editing `.env` files and reading deployment docs, the wizard walks you through
|
||||
every decision conversationally and generates all the config files for you.
|
||||
|
||||
## Install
|
||||
|
||||
The one-line installer autodetects your distro (Ubuntu/Debian, Fedora/RHEL,
|
||||
Arch, and WSL), installs git + Docker if missing, generates secrets, picks free
|
||||
ports, and starts the stack:
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/turnstonelabs/turnstone/main/run.sh | bash
|
||||
turnstone-bootstrap
|
||||
```
|
||||
|
||||
Re-running is safe — it updates the checkout and keeps your existing `.env`.
|
||||
When it finishes it prints the dashboard URL and how to create the first admin
|
||||
user.
|
||||
That's it — no flags, no arguments. The wizard prompts for everything.
|
||||
|
||||
**Other ways to install**
|
||||
## How It Works
|
||||
|
||||
- **Already have Docker?** Clone the repo and `docker compose up` for the full
|
||||
local cluster, or `docker compose -f turnstone/deploy/compose.yaml up` for the
|
||||
released single-node stack. See [docs/docker.md](docs/docker.md).
|
||||
- **Python package:** `pip install turnstone` (add `--pre` for the experimental
|
||||
track), then run `turnstone-server` / `turnstone-console` directly. See the
|
||||
[README](README.md#quickstart).
|
||||
1. **Pick a model** — Choose OpenAI, Anthropic, or a local/vLLM endpoint to
|
||||
power the wizard. Local endpoints auto-detect available models.
|
||||
2. **Answer questions** — The AI walks you through deployment mode, LLM
|
||||
provider, database, authentication, ports, and optional features.
|
||||
3. **Review generated files** — Each file is previewed before writing. You
|
||||
confirm or reject every write.
|
||||
4. **Start the stack** — The wizard prints the exact `docker compose` command
|
||||
and a `setup.sh` script to create your first admin user, roles, and policies.
|
||||
|
||||
## Diagnose: `turnstone-doctor`
|
||||
## What Gets Generated
|
||||
|
||||
`turnstone-doctor` is an LLM-backed assistant that inspects a **running**
|
||||
Turnstone install and helps you troubleshoot it. It is **read-only** — it
|
||||
investigates and tells you the exact commands to fix things, but never changes
|
||||
your system. (Installation is the installer's job, not the doctor's.)
|
||||
|
||||
```bash
|
||||
# From a host that has the turnstone package installed:
|
||||
turnstone-doctor
|
||||
|
||||
# For a Docker install from run.sh (no package on the host), run it with pipx:
|
||||
pipx run --spec turnstone turnstone-doctor --dir ~/turnstone
|
||||
```
|
||||
|
||||
### What it does
|
||||
|
||||
1. **Preflight** — detects how Turnstone is installed here (docker-compose,
|
||||
systemd/bare-metal, pip, or a source checkout) by probing for `config.toml`
|
||||
files, `TURNSTONE_*` environment variables, compose files, and systemd units.
|
||||
2. **Self-configures its LLM** — it powers its own brain from your cluster's
|
||||
*own* model configuration (env / `config.toml` / the database). Whether that
|
||||
works is the first diagnostic: success means your LLM backend is healthy; if
|
||||
it can't, that's surfaced as finding #1 and it falls back to asking you for a
|
||||
provider and key so it can still help.
|
||||
3. **Version check** — reports the installed version, version drift across your
|
||||
cluster's nodes, and the latest upstream stable/experimental releases.
|
||||
4. **Interactive diagnosis** — it reads logs, `/health`, `docker compose ps`,
|
||||
`systemctl`, config, and ports to pin down problems like a node not joining
|
||||
the console, an unreachable database, a down model backend, port conflicts,
|
||||
or a JWT-secret mismatch — then hands you the precise remediation commands.
|
||||
|
||||
### Flags
|
||||
|
||||
| Flag | Purpose |
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `--dir PATH` | Install directory to inspect (default: current directory) |
|
||||
| `--report` | Print the deterministic preflight report and exit — no LLM key needed |
|
||||
| `--offline` | Skip the upstream GitHub version check |
|
||||
| `.env` | All environment variables for `compose.yaml` |
|
||||
| `setup.sh` | Post-start script: creates admin user, roles, tool policies, prompt templates via the API |
|
||||
| `docker-compose.override.yaml` | Only if customizations beyond env vars are needed |
|
||||
|
||||
`--report` is the fastest way to get a health snapshot (and to share one when
|
||||
asking for help) — it never needs an API key:
|
||||
## Requirements
|
||||
|
||||
```bash
|
||||
turnstone-doctor --report --dir ~/turnstone
|
||||
```
|
||||
- **Python 3.11+** with turnstone installed (`pip install turnstone`)
|
||||
- **An LLM API key** — for the wizard itself (OpenAI, Anthropic, or a local
|
||||
model). This can differ from the LLM your deployment will use.
|
||||
- **Docker & Docker Compose** — needed to run the stack. The wizard detects
|
||||
whether Docker is installed and gives platform-specific install instructions
|
||||
if it's missing. You can still generate config files without Docker.
|
||||
|
||||
## Deployment Modes
|
||||
|
||||
The wizard supports two deployment modes:
|
||||
|
||||
- **Single-node production** (`docker compose --profile production up`) —
|
||||
1 server + console + PostgreSQL. Good for most use cases.
|
||||
- **Multi-node cluster** (`docker compose --profile cluster up`) —
|
||||
10-node server fleet + console + PostgreSQL. For high-throughput or
|
||||
HA deployments.
|
||||
|
||||
## Example Session
|
||||
|
||||
```
|
||||
## Install profile
|
||||
- Detected kind(s): docker-compose (primary: docker-compose)
|
||||
- Docker daemon reachable: yes
|
||||
- Compose files:
|
||||
/home/you/turnstone/compose.yaml
|
||||
- Database: backend=postgresql, url=postgresql+psycopg://turnstone:****@postgres:5432/turnstone
|
||||
- Candidate health URLs: http://localhost:8080/health, http://localhost:8090/health
|
||||
$ turnstone-bootstrap
|
||||
|
||||
## Versions
|
||||
- Installed (this tool): 1.7.0a2
|
||||
- Cluster nodes: 10 reporting; versions ['1.7.0a2']
|
||||
- Version drift across nodes: no
|
||||
- Upstream: stable 1.6.9, experimental 1.7.0a2
|
||||
Turnstone Bootstrap Wizard v0.5.4
|
||||
────────────────────────────────────────────────
|
||||
|
||||
## LLM backend (ok)
|
||||
- resolved Qwen/Qwen3-32B via openai-compatible @ http://host.docker.internal:8000/v1
|
||||
Which provider for this wizard?
|
||||
[1] OpenAI
|
||||
[2] Anthropic
|
||||
[3] OpenAI-compatible (local/vLLM)
|
||||
|
||||
> 3
|
||||
|
||||
Base URL [http://localhost:8000/v1]:
|
||||
API key (press Enter for 'none'):
|
||||
|
||||
Querying http://localhost:8000/v1 for available models...
|
||||
Found model: Qwen/Qwen3-32B
|
||||
|
||||
Connected to Qwen/Qwen3-32B. Handing off to AI assistant...
|
||||
|
||||
> (AI walks you through the rest interactively)
|
||||
```
|
||||
|
||||
Secrets (JWT secret, database password, API keys) are always redacted in the
|
||||
report and in anything the doctor reads.
|
||||
|
||||
## Tips
|
||||
|
||||
- **Type `quit`** to exit the conversation; **Ctrl+C** interrupts (twice to quit).
|
||||
- **Point it at the right install** with `--dir` when you run it from elsewhere.
|
||||
- **(Re)installing or adding nodes?** Use the installer (`run.sh`), not the doctor.
|
||||
- **Re-run safely** — running the wizard again detects your existing `.env`
|
||||
and offers to update it rather than overwriting.
|
||||
- **Duplicate writes are skipped** — if the LLM tries to write the same file
|
||||
twice with identical content, it's silently ignored.
|
||||
- **Type `quit` to exit** at any time during the conversation.
|
||||
- **Ctrl+C** is handled gracefully — press once to interrupt, twice to exit.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Docker Deployment](docs/docker.md) — compose stacks, ports, and bare-metal nodes
|
||||
- [Security](docs/security.md) — auth architecture and token types
|
||||
- [Governance](docs/governance.md) — roles, policies, and templates
|
||||
- [Docker Deployment](docker.md) — manual compose setup and profiles
|
||||
- [Security](security.md) — auth architecture and token types
|
||||
- [Governance](governance.md) — roles, policies, and templates
|
||||
|
||||
@@ -3,26 +3,16 @@
|
||||
[](https://github.com/turnstonelabs/turnstone/actions/workflows/ci.yml)
|
||||
[](https://pypi.org/project/turnstone/)
|
||||
[](https://pypi.org/project/turnstone/)
|
||||
[](LICENSE)
|
||||
[](https://discord.gg/Nh3bWMacaq)
|
||||
[](https://github.com/sponsors/eous)
|
||||
[](LICENSE)
|
||||
|
||||
Self-hosted, local-first orchestration for tool-using AI agents. Give LLMs real tools — shell, files, search, web — and run them across your own cluster with direct HTTP routing and interactive interfaces. Your code, your models, your data stay on hardware you control: no telemetry, no phone-home.
|
||||
Multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers with direct HTTP routing, interactive interfaces, and enterprise governance.
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/assets/hero.png" alt="Turnstone coordinator — parallel tool batches with judge-graded approval and child workstream tracking" width="960"/>
|
||||
<img src="docs/assets/hero.png" alt="Turnstone console — multi-workstream AI orchestration with mermaid diagrams" width="960"/>
|
||||
</p>
|
||||
|
||||
Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) (*Arenaria interpres*) — a shorebird that flips stones to discover what's hiding underneath.
|
||||
|
||||
**What is a harness?**
|
||||
|
||||
```
|
||||
ℋ : s_{n+1} ~ T(s_n) for n < τ*, T = ρ ∘ (M_W ∘ π, E)
|
||||
```
|
||||
|
||||
[**the primer →**](PRIMER.md)
|
||||
|
||||
### Release Tracks
|
||||
|
||||
| Track | Install | Docker | Description |
|
||||
@@ -36,13 +26,12 @@ See [docs/releasing.md](docs/releasing.md) for the full release process.
|
||||
|
||||
Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports.
|
||||
|
||||
- **Local-first & private** — runs entirely on hardware you control, with no telemetry and no phone-home. Point it at local models (vLLM, llama.cpp) or commercial APIs you hold the keys to — your prompts and data never transit a third party you didn't choose.
|
||||
- **Bring your own models** — OpenAI-compatible APIs (vLLM, llama.cpp, NIM), the Anthropic Messages API, and Google Gemini, mixed freely per role
|
||||
- **Interactive sessions** — terminal CLI or browser UI with parallel workstreams
|
||||
- **Cluster dashboard** — real-time view of every node and workstream, with a rendezvous routing proxy
|
||||
- **Intent validation** — an LLM judge (your model) grades every tool call with a risk assessment and evidence before it runs
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams with console routing proxy
|
||||
- **Intent validation** — LLM judge evaluates every tool call with risk assessments and evidence
|
||||
- **Governance** — RBAC, OIDC SSO, tool policies, skills, usage tracking, audit logs
|
||||
- **Multi-provider** — OpenAI-compatible APIs (vLLM, llama.cpp, NIM), Anthropic Messages API, and Google Gemini
|
||||
- **MCP support** — external tool servers with native deferred loading (Anthropic/OpenAI) or BM25 fallback
|
||||
- **Team controls when you need them** — optional RBAC, SSO, tool policies, and audit logs, all stored in your own database
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/diagrams/architecture-overview.svg" alt="Turnstone system architecture" width="960"/>
|
||||
@@ -60,42 +49,18 @@ turnstone --base-url http://localhost:8000/v1
|
||||
turnstone-server --port 8080 --base-url http://localhost:8000/v1
|
||||
|
||||
# Cluster dashboard
|
||||
pip install turnstone[console]
|
||||
turnstone-console --port 8090
|
||||
```
|
||||
|
||||
For PostgreSQL (recommended for production):
|
||||
|
||||
```bash
|
||||
export TURNSTONE_DB_BACKEND=postgresql
|
||||
export TURNSTONE_DB_URL="postgresql+psycopg://user:pass@localhost:5432/turnstone"
|
||||
turnstone-server --port 8080 --base-url http://localhost:8000/v1
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
One-line install — autodetects Ubuntu/Debian, Fedora/RHEL, Arch, and WSL,
|
||||
installs git + Docker if missing, generates secrets, and starts the stack:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/turnstonelabs/turnstone/main/run.sh | bash
|
||||
cp .env.example .env # edit LLM_BASE_URL, OPENAI_API_KEY, etc.
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
Or, if you already have Docker, clone the repo and run it yourself:
|
||||
|
||||
```bash
|
||||
docker compose up
|
||||
```
|
||||
|
||||
That builds one image and brings up a full local cluster — PostgreSQL, console,
|
||||
Caddy, channel gateway, and 10 server nodes — with no `.env` required (it ships
|
||||
with insecure dev defaults). Open the dashboard at https://localhost:8443 (Caddy
|
||||
serves it over TLS with its own local CA — trust it once). Nodes boot without an
|
||||
LLM; add model backends from the console UI.
|
||||
|
||||
For production (released images from ghcr.io, real secrets required), use the
|
||||
bundled stack: `docker compose -f turnstone/deploy/compose.yaml up`.
|
||||
|
||||
See [QUICKSTART.md](QUICKSTART.md) for the install + troubleshooting walkthrough and [docs/docker.md](docs/docker.md) for Docker configuration.
|
||||
See [QUICKSTART.md](QUICKSTART.md) for the bootstrap wizard and [docs/docker.md](docs/docker.md) for Docker configuration and profiles.
|
||||
|
||||
### Programmatic (SDK)
|
||||
|
||||
@@ -110,24 +75,23 @@ with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
|
||||
## Tools
|
||||
|
||||
Built-in tools for shell, files, search, web, memory, notifications, and autonomous sub-agents — plus external tools via [MCP](https://modelcontextprotocol.io/) with native deferred loading. See [docs/tools.md](docs/tools.md) for the full reference and [docs/mcp-registry.md](docs/mcp-registry.md) for MCP configuration.
|
||||
Built-in tools for shell, files, search, web, memory, notifications, and autonomous sub-agents — plus external tools via [MCP](https://modelcontextprotocol.io/) with native deferred loading. See [docs/tools.md](docs/tools.md) for the full reference and [docs/mcp.md](docs/mcp.md) for MCP configuration.
|
||||
|
||||
## Architecture
|
||||
|
||||
**Single-node**: Client → Server (direct HTTP + SSE). No external dependencies beyond the database.
|
||||
|
||||
**Multi-node**: Client → Console (rendezvous routing proxy) → Server nodes. The console picks the target node for each workstream via rendezvous (HRW) hashing over the live service registry — pure function of `(ws_id, live_nodes)`, no stored bucket state, deterministic across readers. A node join or drop only re-routes the keys that score highest on the affected node.
|
||||
**Multi-node**: Client → Console (hash ring routing proxy) → Server nodes. The console maintains a 65536-entry bucket cache for O(1) workstream routing. A rebalancer daemon redistributes buckets when nodes join or leave.
|
||||
|
||||
| Component | Purpose |
|
||||
|-----------|---------|
|
||||
| `turnstone` | Terminal CLI (REPL) |
|
||||
| `turnstone-server` | Web UI + REST API + SSE events |
|
||||
| `turnstone-console` | Cluster dashboard + routing proxy + admin panel |
|
||||
| `turnstone-channel` | Channel gateway (Discord and Slack adapters) |
|
||||
| `turnstone-channel` | Channel gateway (Discord, with adapters for Slack/Teams planned) |
|
||||
| `turnstone-admin` | User/token management CLI |
|
||||
| `turnstone-eval` | Headless measurement — scores tool-use against expected actions |
|
||||
| `turnstone-optimizer` | Prompt/tool optimizer (UCB self-modify loop over the eval substrate) |
|
||||
| `turnstone-doctor` | LLM-backed cluster diagnostics |
|
||||
| `turnstone-eval` | Eval harness for prompt/tool optimization |
|
||||
| `turnstone-bootstrap` | LLM-guided setup wizard |
|
||||
|
||||
### Diagrams
|
||||
|
||||
@@ -144,7 +108,7 @@ UML diagrams in [`docs/diagrams/`](docs/diagrams/):
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose topology |
|
||||
| [Auth](docs/diagrams/png/15-auth-architecture.png) | JWT, scopes, login flows |
|
||||
| [Channels](docs/diagrams/png/16-channel-architecture.png) | Discord / Slack adapters + routing |
|
||||
| [Channels](docs/diagrams/png/16-channel-architecture.png) | Discord adapter + routing |
|
||||
| [Judge](docs/diagrams/png/22-judge-architecture.png) | Intent validation pipeline |
|
||||
| [OIDC](docs/diagrams/png/25-oidc-architecture.png) | SSO authorization code flow |
|
||||
|
||||
@@ -163,28 +127,15 @@ UML diagrams in [`docs/diagrams/`](docs/diagrams/):
|
||||
| Console dashboard | [docs/console.md](docs/console.md) |
|
||||
| Eval harness | [docs/eval.md](docs/eval.md) |
|
||||
| Tools reference | [docs/tools.md](docs/tools.md) |
|
||||
| MCP integration | [docs/mcp-registry.md](docs/mcp-registry.md) |
|
||||
| MCP integration | [docs/mcp.md](docs/mcp.md) |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.11+
|
||||
- An OpenAI-compatible API endpoint, Anthropic API key, or Google Gemini API key
|
||||
- Optional: Discord / Slack channel integrations (`pip install turnstone[discord,slack]`)
|
||||
- Optional: PostgreSQL (`pip install turnstone[postgres]`), Anthropic (`pip install turnstone[anthropic]`)
|
||||
- [Git LFS](https://git-lfs.com/) for cloning (diagram PNGs)
|
||||
|
||||
## Support
|
||||
|
||||
Turnstone is free, Apache-2.0, and self-hosted — no paid tier, no telemetry, no upsell. If it saves you time or you'd like to help keep development moving, you can sponsor the project:
|
||||
|
||||
**[❤ Sponsor Turnstone →](https://github.com/sponsors/eous)** · one-off via **[PayPal](https://paypal.me/eousphoros)**
|
||||
|
||||
Sponsorship is entirely optional and funds maintenance, new features, and infrastructure. Prefer to contribute in other ways? Filing issues, improving docs, and [pull requests](CONTRIBUTING.md) help just as much.
|
||||
|
||||
## Community
|
||||
|
||||
Questions, ideas, or want to show what you're building? Join us on Discord:
|
||||
**[discord.gg/Nh3bWMacaq](https://discord.gg/Nh3bWMacaq)**.
|
||||
|
||||
## License
|
||||
|
||||
[Apache License 2.0](LICENSE), as of version 1.6.0. Versions 1.5.x and earlier remain under the Business Source License 1.1 they shipped with.
|
||||
[Business Source License 1.1](LICENSE) — free for all use except hosting as a managed service. Converts to Apache 2.0 on 2030-03-01.
|
||||
|
||||
+4
-4
@@ -2,11 +2,11 @@ Turnstone — Third-Party Notices
|
||||
|
||||
This file contains the licenses and notices for third-party software bundled
|
||||
with Turnstone. Each bundled dependency retains its original license; the
|
||||
Turnstone Apache-2.0 license does not apply to these components.
|
||||
Turnstone BUSL-1.1 license does not apply to these components.
|
||||
|
||||
================================================================================
|
||||
|
||||
KaTeX 0.17.0
|
||||
KaTeX 0.16.38
|
||||
https://katex.org/
|
||||
https://github.com/KaTeX/KaTeX
|
||||
|
||||
@@ -70,7 +70,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================================
|
||||
|
||||
Mermaid 11.15.0
|
||||
Mermaid 11.13.0
|
||||
https://mermaid.js.org/
|
||||
https://github.com/mermaid-js/mermaid
|
||||
|
||||
@@ -98,7 +98,7 @@ SOFTWARE.
|
||||
|
||||
================================================================================
|
||||
|
||||
hls.js 1.6.16
|
||||
hls.js 1.6.15
|
||||
https://github.com/video-dev/hls.js
|
||||
|
||||
Copyright 2017 Dailymotion
|
||||
|
||||
+140
-231
@@ -1,60 +1,16 @@
|
||||
# =============================================================================
|
||||
# Turnstone — local cluster stack (docker compose)
|
||||
# Turnstone Docker Compose Stack — Development
|
||||
#
|
||||
# Clone the repo and run:
|
||||
# This file is for local development from a git clone. It builds images
|
||||
# locally from the Dockerfile. If you installed via pip/pipx, run
|
||||
# `turnstone-bootstrap` instead — it writes a production compose.yaml
|
||||
# that pulls pre-built images from ghcr.io.
|
||||
#
|
||||
# docker compose up
|
||||
#
|
||||
# That builds one image and brings up a complete, console-visible cluster:
|
||||
# PostgreSQL + console + Caddy + channel gateway + 10 server nodes (node-1…10).
|
||||
#
|
||||
# Dashboard: https://localhost:8443 (Caddy's local CA — trust it once)
|
||||
#
|
||||
# Access is via Caddy only — the console's plain-HTTP port is intentionally not
|
||||
# published (HTTP/2 from Caddy avoids the browser's 6-connection cap on the
|
||||
# dashboard's SSE streams). Trust Caddy's root once:
|
||||
# docker compose exec caddy cat /data/caddy/pki/authorities/local/root.crt
|
||||
#
|
||||
# It works out of the box with INSECURE dev defaults (see the secret/password
|
||||
# values below) so there's nothing to configure first. A .env file still
|
||||
# overrides any value. For a real deployment use the bundled production stack
|
||||
# at turnstone/deploy/compose.yaml — it pulls released images from ghcr.io and
|
||||
# requires you to set real secrets.
|
||||
#
|
||||
# Bring your own LLM: nodes boot without one and show up in the console
|
||||
# immediately. Add model backends (OpenAI / Anthropic / local vLLM) from the
|
||||
# console UI's Models tab, or point LLM_BASE_URL / OPENAI_API_KEY (below) at an
|
||||
# OpenAI-compatible endpoint.
|
||||
#
|
||||
# Fewer nodes (lighter machines):
|
||||
# docker compose up postgres console caddy channel node-1 node-2 node-3
|
||||
#
|
||||
# Join a bare-metal host: a turnstone-server running OUTSIDE compose (e.g. to use
|
||||
# a local GPU) can join this cluster. Postgres, the console's ACME endpoint, and
|
||||
# SearxNG are published on 127.0.0.1 so a node on THIS machine reaches them via
|
||||
# localhost. Keep secrets in ~/.config/turnstone/config.toml (chmod 0600 — the
|
||||
# loader warns otherwise):
|
||||
# [auth]
|
||||
# jwt_secret = "dev-only-insecure-jwt-secret-change-me-for-real-deployments"
|
||||
# [database]
|
||||
# backend = "postgresql"
|
||||
# url = "postgresql+psycopg://turnstone:turnstone@localhost:5432/turnstone"
|
||||
# [api]
|
||||
# base_url = "http://localhost:8000/v1"
|
||||
# api_key = "dummy"
|
||||
# [tls] # only if the cluster runs mTLS
|
||||
# enabled = true
|
||||
# then run (node identity isn't a secret, so it stays on the command line):
|
||||
# TURNSTONE_NODE_ID=host-1 \
|
||||
# TURNSTONE_ADVERTISE_URL=http://host.docker.internal:8080 \
|
||||
# TURNSTONE_CONSOLE_URL=http://localhost:8090 \
|
||||
# TURNSTONE_SEARXNG_URL=http://localhost:8081 \
|
||||
# turnstone-server --host 0.0.0.0 --port 8080
|
||||
# The node registers in Postgres, auto-enrolls its mTLS cert from the console's
|
||||
# ACME endpoint (when the cluster runs mTLS), and the console collector reaches
|
||||
# it back via host.docker.internal. To join from ANOTHER machine, set
|
||||
# TURNSTONE_HOST_IP to this host's LAN IP and use it in the URLs above (and the
|
||||
# node's TURNSTONE_ADVERTISE_URL = the NODE host's IP) — see docs/docker.md.
|
||||
# Usage:
|
||||
# Infra only: docker compose up
|
||||
# Single node: docker compose --profile production up
|
||||
# Production (PG): DB_BACKEND=postgresql docker compose --profile production up
|
||||
# 10-node cluster: docker compose --profile cluster up
|
||||
# =============================================================================
|
||||
|
||||
name: turnstone
|
||||
@@ -67,30 +23,16 @@ volumes:
|
||||
turnstone-data:
|
||||
workspace:
|
||||
postgres-data:
|
||||
caddy-data:
|
||||
caddy-config:
|
||||
searxng-cache:
|
||||
|
||||
# -- Shared values (scalar anchors) -------------------------------------------
|
||||
# Defined once here, referenced (*alias) by every service so the dev defaults
|
||||
# can't drift. All `${VAR:-default}` values are still overridable via .env.
|
||||
x-shared:
|
||||
# INSECURE dev default. Every service MUST share ONE secret — the console
|
||||
# mints its own service token (signed with this) to reach the nodes. Override
|
||||
# TURNSTONE_JWT_SECRET in .env for anything that isn't a local sandbox.
|
||||
jwt-secret: &jwt-secret "${TURNSTONE_JWT_SECRET:-dev-only-insecure-jwt-secret-change-me-for-real-deployments}"
|
||||
db-backend: &db-backend "${TURNSTONE_DB_BACKEND:-postgresql}"
|
||||
# All services point at the same Postgres. Node discovery REQUIRES a shared
|
||||
# DB: each server registers + heartbeats into a `services` table that the
|
||||
# console polls. (SQLite-per-container can't see other containers.)
|
||||
db-url: &db-url "${TURNSTONE_DB_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}"
|
||||
|
||||
services:
|
||||
# -------------------------------------------------------------------
|
||||
# PostgreSQL — the shared database that ties the cluster together.
|
||||
# PostgreSQL — production database (profile: production)
|
||||
# -------------------------------------------------------------------
|
||||
postgres:
|
||||
image: pgautoupgrade/pgautoupgrade:18-alpine
|
||||
profiles:
|
||||
- production
|
||||
- cluster
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
@@ -100,17 +42,8 @@ services:
|
||||
environment:
|
||||
POSTGRES_DB: turnstone
|
||||
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
|
||||
# INSECURE dev default — override POSTGRES_PASSWORD in .env for real use.
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-turnstone}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required for production profile}
|
||||
PGDATA: /var/lib/postgresql/data
|
||||
# Published so a bare-metal turnstone-server can join the cluster (see "Join
|
||||
# a bare-metal host" in the header). Bound to 127.0.0.1 by default (same-host
|
||||
# nodes only); set TURNSTONE_HOST_IP to this host's LAN IP to let another
|
||||
# machine connect — but set a real POSTGRES_PASSWORD first, or you'll expose a
|
||||
# database with the insecure default password to your network. (The legacy
|
||||
# POSTGRES_BIND is still honored as a fallback when TURNSTONE_HOST_IP is unset.)
|
||||
ports:
|
||||
- "${TURNSTONE_HOST_IP:-${POSTGRES_BIND:-127.0.0.1}}:${POSTGRES_PORT:-5432}:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
networks:
|
||||
@@ -124,23 +57,65 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
memory: 1G
|
||||
cpus: '1.0'
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-console — cluster dashboard. Reach it ONLY through Caddy at
|
||||
# https://localhost:8443 (see the caddy service below).
|
||||
#
|
||||
# Browsers must reach the dashboard through Caddy (https://localhost:8443): a
|
||||
# plain HTTP/1.1 origin caps the browser at 6 connections, which starves the
|
||||
# dashboard's per-pane SSE streams, whereas Caddy serves HTTP/2 (multiplexed)
|
||||
# and proxies to console:8090 internally. The console's :8090 is published
|
||||
# below ONLY so bare-metal nodes can reach the plain-HTTP ACME enrollment
|
||||
# endpoint — don't point a browser at it.
|
||||
#
|
||||
# The single `build:` here produces the turnstone:local image every other
|
||||
# service reuses. extra_hosts lets the console reach a bare-metal server
|
||||
# advertising http://host.docker.internal:8080 (see "Join a host" below).
|
||||
# turnstone-server — Web UI + chat workstreams + LLM interaction
|
||||
# -------------------------------------------------------------------
|
||||
server:
|
||||
image: turnstone:local
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-server
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
--base-url "$${LLM_BASE_URL}"
|
||||
--api-key "$${OPENAI_API_KEY}"
|
||||
$${MODEL:+--model $$MODEL}
|
||||
$${SKIP_PERMISSIONS:+--skip-permissions}
|
||||
$${MCP_CONFIG:+--mcp-config $$MCP_CONFIG}
|
||||
ports:
|
||||
- "${SERVER_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- turnstone-data:/data
|
||||
- ${WORKSPACE_MOUNT:-workspace}:/workspace
|
||||
environment:
|
||||
- LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-dummy}
|
||||
- TAVILY_API_KEY=${TAVILY_API_KEY:-}
|
||||
- SKIP_PERMISSIONS=${SKIP_PERMISSIONS:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- MODEL=${MODEL:-}
|
||||
- MCP_CONFIG=${MCP_CONFIG:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-}
|
||||
- TURNSTONE_ADVERTISE_URL=${TURNSTONE_ADVERTISE_URL:-http://server:8080}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-console — Cluster dashboard
|
||||
# -------------------------------------------------------------------
|
||||
console:
|
||||
image: turnstone:local
|
||||
@@ -151,26 +126,16 @@ services:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port=8090
|
||||
# Publishes the console's plain-HTTP listener so a bare-metal node can reach
|
||||
# the ACME endpoint, fetch the CA, and enroll its cert (the console serves
|
||||
# HTTP here even under mTLS). Bound to 127.0.0.1 by default; setting
|
||||
# TURNSTONE_HOST_IP exposes the WHOLE console HTTP API — including the
|
||||
# cert-issuing ACME endpoint — on that interface, so the JWT secret's
|
||||
# strength is the only gate. Browsers use Caddy :8443, never this port.
|
||||
ports:
|
||||
- "${TURNSTONE_HOST_IP:-127.0.0.1}:8090:8090"
|
||||
- "${CONSOLE_PORT:-8090}:8090"
|
||||
environment:
|
||||
TURNSTONE_JWT_SECRET: *jwt-secret
|
||||
TURNSTONE_DB_BACKEND: *db-backend
|
||||
TURNSTONE_DB_URL: *db-url
|
||||
TURNSTONE_CONSOLE_URL: http://console:8090
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_CONSOLE_URL=http://console:8090
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8090/health"]
|
||||
interval: 10s
|
||||
@@ -180,33 +145,14 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# caddy — browser TLS for the console dashboard.
|
||||
# Terminates HTTPS (Caddy's own local CA, see turnstone/deploy/Caddyfile) → console:8090.
|
||||
# Dashboard over TLS: https://localhost:${CONSOLE_HTTPS_PORT:-8443}
|
||||
# -------------------------------------------------------------------
|
||||
caddy:
|
||||
image: caddy:2.11
|
||||
depends_on:
|
||||
- console
|
||||
ports:
|
||||
- "${CONSOLE_HTTPS_PORT:-8443}:443"
|
||||
# SearxNG web UI — localhost-only (it has no auth). Browse https://localhost:8444.
|
||||
- "127.0.0.1:${SEARXNG_HTTPS_PORT:-8444}:8444"
|
||||
volumes:
|
||||
- ./turnstone/deploy/Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- caddy-data:/data # persist Caddy's local CA across restarts
|
||||
- caddy-config:/config
|
||||
networks:
|
||||
- turnstone-net
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-channel — channel gateway (Discord and/or Slack).
|
||||
# Runs HTTP-only with no adapters until you set a token, so it's safe
|
||||
# to leave running. See docs/channels.md.
|
||||
# turnstone-channel — Channel gateway (Discord, Slack, etc.)
|
||||
# Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter
|
||||
# -------------------------------------------------------------------
|
||||
channel:
|
||||
image: turnstone:local
|
||||
profiles:
|
||||
- production
|
||||
- cluster
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -215,66 +161,36 @@ services:
|
||||
--http-host=0.0.0.0
|
||||
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
|
||||
environment:
|
||||
TURNSTONE_JWT_SECRET: *jwt-secret
|
||||
TURNSTONE_DB_BACKEND: *db-backend
|
||||
TURNSTONE_DB_URL: *db-url
|
||||
TURNSTONE_DISCORD_TOKEN: ${TURNSTONE_DISCORD_TOKEN:-}
|
||||
TURNSTONE_DISCORD_GUILD: ${TURNSTONE_DISCORD_GUILD:-0}
|
||||
TURNSTONE_SLACK_TOKEN: ${TURNSTONE_SLACK_TOKEN:-}
|
||||
TURNSTONE_SLACK_APP_TOKEN: ${TURNSTONE_SLACK_APP_TOKEN:-}
|
||||
TURNSTONE_CHANNEL_ADVERTISE_URL: http://channel:8091
|
||||
- TURNSTONE_DISCORD_TOKEN=${TURNSTONE_DISCORD_TOKEN:-}
|
||||
- TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
|
||||
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# searxng — self-hosted metasearch backing the web_search tool.
|
||||
# Internal-network only (no published port): nodes reach it at
|
||||
# http://searxng:8080. Config (JSON output on, limiter off) lives in
|
||||
# turnstone/deploy/searxng/settings.yml, mounted read-only. Commercial
|
||||
# models use native provider search and never hit this; it serves
|
||||
# local/vLLM models. Override the tag with SEARXNG_IMAGE_TAG in .env.
|
||||
# -------------------------------------------------------------------
|
||||
searxng:
|
||||
image: searxng/searxng:${SEARXNG_IMAGE_TAG:-latest}
|
||||
# Published so a bare-metal node's web_search can reach it. SearxNG has NO
|
||||
# auth, so it is bound to 127.0.0.1 by default; setting TURNSTONE_HOST_IP
|
||||
# exposes it on that interface — an open search proxy on your LAN, which also
|
||||
# triggers the SearxNG AGPL-3.0 §13 source-offer obligation (see docs/docker.md).
|
||||
# In-compose nodes always use the internal http://searxng:8080 and ignore this.
|
||||
ports:
|
||||
- "${TURNSTONE_HOST_IP:-127.0.0.1}:${SEARXNG_API_PORT:-8081}:8080"
|
||||
volumes:
|
||||
- ./turnstone/deploy/searxng:/etc/searxng:ro
|
||||
- searxng-cache:/var/cache/searxng # favicon + internal SQLite cache (survives restarts)
|
||||
networks:
|
||||
- turnstone-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/healthz"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 20s
|
||||
required: false
|
||||
restart: unless-stopped
|
||||
|
||||
# ===================================================================
|
||||
# Server fleet — node-1 … node-10
|
||||
# 10-node cluster (profile: cluster)
|
||||
#
|
||||
# Each node registers itself in Postgres on boot (unique
|
||||
# TURNSTONE_NODE_ID + TURNSTONE_ADVERTISE_URL) and the console
|
||||
# discovers it automatically — no static node list anywhere.
|
||||
# All nodes share the same PostgreSQL instance.
|
||||
# Access via console at :8090.
|
||||
#
|
||||
# node-1 carries the shared definition (&node / &node-env); node-2…10
|
||||
# inherit it and override only their identity.
|
||||
# Start: docker compose --profile cluster up
|
||||
# ===================================================================
|
||||
node-1: &node
|
||||
|
||||
# -- cluster servers ------------------------------------------------
|
||||
|
||||
server-1: &cluster-server
|
||||
image: turnstone:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
build: { context: ., dockerfile: Dockerfile }
|
||||
profiles: [cluster]
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -290,30 +206,23 @@ services:
|
||||
volumes:
|
||||
- turnstone-data:/data
|
||||
- ${WORKSPACE_MOUNT:-workspace}:/workspace
|
||||
environment: &node-env
|
||||
TURNSTONE_JWT_SECRET: *jwt-secret
|
||||
TURNSTONE_DB_BACKEND: *db-backend
|
||||
TURNSTONE_DB_URL: *db-url
|
||||
# Bootstrap LLM defaults — real backends are configured in the console UI.
|
||||
environment: &cluster-server-env
|
||||
LLM_BASE_URL: ${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
|
||||
OPENAI_API_KEY: ${OPENAI_API_KEY:-dummy}
|
||||
# web_search backend. Defaults to the bundled searxng service; point at an
|
||||
# external SearxNG by setting TURNSTONE_SEARXNG_URL in .env (empty disables).
|
||||
TURNSTONE_SEARXNG_URL: ${TURNSTONE_SEARXNG_URL:-http://searxng:8080}
|
||||
TAVILY_API_KEY: ${TAVILY_API_KEY:-}
|
||||
SKIP_PERMISSIONS: ${SKIP_PERMISSIONS:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
MODEL: ${MODEL:-}
|
||||
MCP_CONFIG: ${MCP_CONFIG:-}
|
||||
SKIP_PERMISSIONS: ${SKIP_PERMISSIONS:-}
|
||||
TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql}
|
||||
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
|
||||
TURNSTONE_NODE_ID: node-1
|
||||
TURNSTONE_ADVERTISE_URL: http://node-1:8080
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
networks:
|
||||
- turnstone-net
|
||||
TURNSTONE_ADVERTISE_URL: http://server-1:8080
|
||||
extra_hosts: ["host.docker.internal:host-gateway"]
|
||||
networks: [turnstone-net]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
searxng:
|
||||
condition: service_healthy
|
||||
postgres: { condition: service_healthy }
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
|
||||
interval: 10s
|
||||
@@ -322,34 +231,34 @@ services:
|
||||
start_period: 60s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
limits: { memory: 384M, cpus: '0.5' }
|
||||
restart: unless-stopped
|
||||
|
||||
node-2:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-2, TURNSTONE_ADVERTISE_URL: "http://node-2:8080" }
|
||||
node-3:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-3, TURNSTONE_ADVERTISE_URL: "http://node-3:8080" }
|
||||
node-4:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-4, TURNSTONE_ADVERTISE_URL: "http://node-4:8080" }
|
||||
node-5:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-5, TURNSTONE_ADVERTISE_URL: "http://node-5:8080" }
|
||||
node-6:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-6, TURNSTONE_ADVERTISE_URL: "http://node-6:8080" }
|
||||
node-7:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-7, TURNSTONE_ADVERTISE_URL: "http://node-7:8080" }
|
||||
node-8:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-8, TURNSTONE_ADVERTISE_URL: "http://node-8:8080" }
|
||||
node-9:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-9, TURNSTONE_ADVERTISE_URL: "http://node-9:8080" }
|
||||
node-10:
|
||||
<<: *node
|
||||
environment: { <<: *node-env, TURNSTONE_NODE_ID: node-10, TURNSTONE_ADVERTISE_URL: "http://node-10:8080" }
|
||||
server-2:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-2, TURNSTONE_ADVERTISE_URL: "http://server-2:8080" }
|
||||
server-3:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-3, TURNSTONE_ADVERTISE_URL: "http://server-3:8080" }
|
||||
server-4:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-4, TURNSTONE_ADVERTISE_URL: "http://server-4:8080" }
|
||||
server-5:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-5, TURNSTONE_ADVERTISE_URL: "http://server-5:8080" }
|
||||
server-6:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-6, TURNSTONE_ADVERTISE_URL: "http://server-6:8080" }
|
||||
server-7:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-7, TURNSTONE_ADVERTISE_URL: "http://server-7:8080" }
|
||||
server-8:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-8, TURNSTONE_ADVERTISE_URL: "http://server-8:8080" }
|
||||
server-9:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-9, TURNSTONE_ADVERTISE_URL: "http://server-9:8080" }
|
||||
server-10:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-10, TURNSTONE_ADVERTISE_URL: "http://server-10:8080" }
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# Bare-metal overlay — expose PostgreSQL and let the console reach
|
||||
# a turnstone-server running outside Docker on the host machine.
|
||||
#
|
||||
# Requires TURNSTONE_HOST_IP set to the host's routable IP address.
|
||||
#
|
||||
# Usage:
|
||||
# export TURNSTONE_HOST_IP="$(hostname -I | awk '{print $1}')"
|
||||
# docker compose --profile production \
|
||||
# -f compose.yaml -f deploy/docker-compose.bare-metal.yml up
|
||||
#
|
||||
# Then on the host:
|
||||
# export TURNSTONE_JWT_SECRET="<same as .env>"
|
||||
# export TURNSTONE_DB_BACKEND=postgresql
|
||||
# export TURNSTONE_DB_URL="postgresql://turnstone:<pw>@localhost:5432/turnstone"
|
||||
# export TURNSTONE_NODE_ID="bare-metal-1"
|
||||
# export TURNSTONE_ADVERTISE_URL="http://${TURNSTONE_HOST_IP}:8080"
|
||||
# python -m turnstone.server --host 0.0.0.0 --port 8080 \
|
||||
# --base-url http://localhost:8000/v1 --api-key "$OPENAI_API_KEY"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
ports:
|
||||
- "${POSTGRES_PORT:-5432}:5432"
|
||||
|
||||
console:
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
environment:
|
||||
# Console needs to reach the bare-metal server on the host
|
||||
TURNSTONE_SERVER_URL: "http://${TURNSTONE_HOST_IP}:${SERVER_PORT:-8080}"
|
||||
|
||||
channel:
|
||||
ports:
|
||||
- "${CHANNEL_PORT:-8091}:8091"
|
||||
environment:
|
||||
# Channel gateway advertises with host-routable IP so the
|
||||
# bare-metal server can reach it for schedule notifications
|
||||
TURNSTONE_CHANNEL_ADVERTISE_URL: "http://${TURNSTONE_HOST_IP}:${CHANNEL_PORT:-8091}"
|
||||
# Channel needs to reach the bare-metal server on the host
|
||||
TURNSTONE_SERVER_URL: "http://${TURNSTONE_HOST_IP}:${SERVER_PORT:-8080}"
|
||||
@@ -1,8 +1,7 @@
|
||||
# TLS overlay — enables mTLS across the turnstone deployment.
|
||||
# TLS overlay — enables mTLS across the turnstone cluster.
|
||||
#
|
||||
# Layers on the production stack (it patches the `server`, `console`, and
|
||||
# `channel` services that file defines):
|
||||
# docker compose -f turnstone/deploy/compose.yaml -f deploy/docker-compose.tls.yml up
|
||||
# Usage (requires base compose.yaml with production profile):
|
||||
# docker compose -f compose.yaml -f deploy/docker-compose.tls.yml --profile production up
|
||||
#
|
||||
# The tls-init service bootstraps a CA and issues certs.
|
||||
# All turnstone services auto-provision their own certs via the
|
||||
@@ -13,7 +12,7 @@ services:
|
||||
# Runs as root to create directories in the volume, then chowns
|
||||
# to turnstone:turnstone with restrictive perms (keys 0600).
|
||||
tls-init:
|
||||
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
|
||||
build: .
|
||||
user: root
|
||||
command:
|
||||
- sh
|
||||
|
||||
@@ -7,6 +7,6 @@ appVersion: "0.3.0"
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: ~18.7.0
|
||||
version: ~18.5.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
|
||||
@@ -103,18 +103,13 @@ network_policies:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- Web search (SearxNG) ---
|
||||
# Turnstone talks only to its SearxNG instance over HTTP; SearxNG itself makes
|
||||
# the outbound calls to search engines (and is NOT governed by this policy —
|
||||
# it runs as a separate service). The host/port below is the bundled compose
|
||||
# service name; if your SearxNG runs elsewhere, set it to match
|
||||
# TURNSTONE_SEARXNG_URL.
|
||||
# --- Web search fallback (Tavily) ---
|
||||
|
||||
searxng:
|
||||
name: searxng-search
|
||||
tavily_api:
|
||||
name: tavily-search
|
||||
endpoints:
|
||||
- host: searxng
|
||||
port: 8080
|
||||
- host: api.tavily.com
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# Running a bare-metal turnstone-server under systemd
|
||||
|
||||
These units run a `turnstone-server` **outside** Docker (e.g. on a box with a
|
||||
local GPU) so it joins an existing cluster — typically the docker-compose stack
|
||||
in [`compose.yaml`](../../compose.yaml). They are the hardened, production-shaped
|
||||
counterpart to the quick `turnstone-server …` invocation in
|
||||
[`docs/docker.md`](../../docs/docker.md) ("Join a bare-metal host").
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `turnstone-server.service` | The hardened server unit (sandboxed; secrets via `config.toml`). |
|
||||
| `turnstone.slice` | Shared memory/process budget for colocated Turnstone units. |
|
||||
| `turnstone-server.service.d/node.conf.example` | Per-host identity + cluster URLs drop-in (no secrets). |
|
||||
|
||||
## Cluster-side prerequisite
|
||||
|
||||
The compose stack must publish Postgres, the console's ACME endpoint, and SearxNG
|
||||
on an address the bare-metal host can reach. Start it with `TURNSTONE_HOST_IP`
|
||||
set to the compose host's LAN IP (default `127.0.0.1` keeps everything host-local):
|
||||
|
||||
```bash
|
||||
TURNSTONE_HOST_IP=<compose-host-ip> docker compose up -d
|
||||
```
|
||||
|
||||
## Install (run as root on the bare-metal host)
|
||||
|
||||
```bash
|
||||
# 1. A dedicated, unprivileged user.
|
||||
useradd --system --no-create-home --shell /usr/sbin/nologin turnstone
|
||||
|
||||
# 2. Install turnstone into a venv at /opt/turnstone-venv (lacme/mTLS is a core dep).
|
||||
uv venv /opt/turnstone-venv --python 3.12
|
||||
uv pip install --python /opt/turnstone-venv 'turnstone @ git+https://github.com/turnstonelabs/turnstone'
|
||||
# …or from a local checkout: uv pip install --python /opt/turnstone-venv /path/to/turnstone
|
||||
|
||||
# 3. Secrets — match the cluster's JWT secret + DB credentials (kept out of env).
|
||||
install -d -m 750 -o turnstone -g turnstone /etc/turnstone
|
||||
cat > /etc/turnstone/config.toml <<'TOML'
|
||||
[auth]
|
||||
jwt_secret = "<same secret as the cluster>"
|
||||
[database]
|
||||
backend = "postgresql"
|
||||
url = "postgresql+psycopg://turnstone:<password>@<compose-host-ip>:5432/turnstone"
|
||||
[api]
|
||||
base_url = "http://localhost:8000/v1" # a real model backend is configured in the console UI
|
||||
api_key = "dummy"
|
||||
TOML
|
||||
chown turnstone:turnstone /etc/turnstone/config.toml
|
||||
chmod 600 /etc/turnstone/config.toml
|
||||
|
||||
# 4. Units + per-host drop-in.
|
||||
cp turnstone-server.service turnstone.slice /etc/systemd/system/
|
||||
install -d /etc/systemd/system/turnstone-server.service.d
|
||||
cp turnstone-server.service.d/node.conf.example \
|
||||
/etc/systemd/system/turnstone-server.service.d/node.conf
|
||||
$EDITOR /etc/systemd/system/turnstone-server.service.d/node.conf # set the addresses
|
||||
|
||||
# 5. Go.
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now turnstone-server.service
|
||||
journalctl -u turnstone-server -f # watch it register + (if the cluster runs mTLS) enroll
|
||||
```
|
||||
|
||||
`tls.enabled` is **not** set here — a joining node inherits it from the cluster's
|
||||
shared settings (the database). If the cluster runs mTLS, the node auto-enrolls a
|
||||
cert from the console's ACME endpoint and re-advertises itself over `https://`.
|
||||
|
||||
> **mTLS + cross-host caveat:** a node on a *different* host than the console
|
||||
> currently can't complete ACME enrollment — the console advertises an
|
||||
> unroutable in-container address in its ACME directory
|
||||
> ([turnstonelabs/lacme#22](https://github.com/turnstonelabs/lacme/issues/22)).
|
||||
> Same-host bare-metal nodes, and any node in a non-mTLS cluster, are unaffected.
|
||||
@@ -1,85 +0,0 @@
|
||||
# Run a bare-metal turnstone-server as a systemd service so it joins a cluster
|
||||
# (e.g. the docker-compose stack) from outside Docker — typically to use a local
|
||||
# GPU. Install steps + the cluster-side prerequisites are in deploy/systemd/README.md
|
||||
# and docs/docker.md ("Join a bare-metal host"). Per-host identity + the cluster
|
||||
# URLs go in a drop-in (see node.conf.example); secrets go in config.toml.
|
||||
[Unit]
|
||||
Description=Turnstone server (chat workstreams + LLM gateway)
|
||||
Documentation=https://github.com/turnstonelabs/turnstone
|
||||
# Postgres is required. After= orders against a colocated postgresql.service
|
||||
# when present and silently no-ops otherwise (the cluster DB is usually remote).
|
||||
After=network.target postgresql.service
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=turnstone
|
||||
Group=turnstone
|
||||
|
||||
# Secrets live in config.toml — JWT secret, Postgres URL+password, LLM API key —
|
||||
# kept out of os.environ so a prompt-injected tool can't dump them via `env`.
|
||||
Environment=TURNSTONE_CONFIG=/etc/turnstone/config.toml
|
||||
Environment=TURNSTONE_LOG_LEVEL=info
|
||||
|
||||
Slice=turnstone.slice
|
||||
|
||||
# Per-host node identity + cluster wiring (TURNSTONE_NODE_ID / _ADVERTISE_URL /
|
||||
# _CONSOLE_URL / _SEARXNG_URL) go in a drop-in, not here — see node.conf.example.
|
||||
|
||||
StateDirectory=turnstone
|
||||
StateDirectoryMode=0750
|
||||
LogsDirectory=turnstone
|
||||
LogsDirectoryMode=0750
|
||||
WorkingDirectory=/var/lib/turnstone
|
||||
|
||||
# --host 0.0.0.0 so the console collector + peer nodes can dial this node back
|
||||
# at its advertised address. (A single-node, Caddy-fronted install can use
|
||||
# 127.0.0.1 instead.) Rewrite --port if :8080 is already taken on the host.
|
||||
ExecStart=/opt/turnstone-venv/bin/turnstone-server --host 0.0.0.0 --port 8080
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
TimeoutStartSec=120
|
||||
TimeoutStopSec=30
|
||||
KillSignal=SIGTERM
|
||||
KillMode=mixed
|
||||
|
||||
# --- Resource limits ---
|
||||
# SSE keeps an fd per active workstream + outbound LLM stream + MCP stdio pipe.
|
||||
LimitNOFILE=65535
|
||||
LimitNPROC=8192
|
||||
TasksMax=8192
|
||||
LimitCORE=0
|
||||
|
||||
# --- Hardening ---
|
||||
NoNewPrivileges=true
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
UMask=0027
|
||||
PrivateTmp=true
|
||||
# PrivateDevices=true — disabled: GPU access via /sys/class/drm
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectKernelLogs=true
|
||||
ProtectControlGroups=true
|
||||
ProtectClock=true
|
||||
ProtectHostname=true
|
||||
RestrictNamespaces=true
|
||||
RestrictRealtime=true
|
||||
RestrictSUIDSGID=true
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
LockPersonality=true
|
||||
MemoryDenyWriteExecute=true
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallFilter=~@privileged @mount
|
||||
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=turnstone-server
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,25 +0,0 @@
|
||||
# Per-host node identity + cluster wiring for a bare-metal turnstone-server.
|
||||
# Copy to /etc/systemd/system/turnstone-server.service.d/node.conf and edit the
|
||||
# addresses, then `systemctl daemon-reload`. Identity + URLs are NOT secrets, so
|
||||
# they live here; the JWT secret + DB URL live in /etc/turnstone/config.toml.
|
||||
#
|
||||
# Addresses below use RFC 5737 documentation IPs — replace them:
|
||||
# <this-host> = the bare-metal host's own LAN IP (what the console dials back)
|
||||
# <compose-host> = the host running the cluster / docker-compose stack, started
|
||||
# with TURNSTONE_HOST_IP=<compose-host> so :8090 and :8081 are
|
||||
# published on its LAN interface (see docs/docker.md).
|
||||
[Service]
|
||||
# Unique node id (defaults to the hostname if unset).
|
||||
Environment=TURNSTONE_NODE_ID=host-1
|
||||
|
||||
# The address peers + the console collector dial back. Auto-upgrades to https://
|
||||
# once the node enrolls its mTLS cert.
|
||||
Environment=TURNSTONE_ADVERTISE_URL=http://192.0.2.10:8080
|
||||
|
||||
# The cluster console's reachable plain-HTTP ACME/API endpoint. A bare-metal node
|
||||
# can't resolve the in-cluster name (console:8090), so point it at the published
|
||||
# port; turnstone-server honors this for cert enrollment.
|
||||
Environment=TURNSTONE_CONSOLE_URL=http://192.0.2.1:8090
|
||||
|
||||
# The cluster's published SearxNG, for the web_search tool.
|
||||
Environment=TURNSTONE_SEARXNG_URL=http://192.0.2.1:8081
|
||||
@@ -1,15 +0,0 @@
|
||||
# Shared resource budget for the colocated Turnstone units. Without a slice each
|
||||
# unit's MemoryMax= is enforced independently — three units at 85% each can sum
|
||||
# to 255% of host RAM before any throttles. Under a shared slice the cap is
|
||||
# hierarchical: the slice ceiling is the real limit. (A bare-metal node that runs
|
||||
# only turnstone-server still benefits — and keeps the unit's Slice= reference
|
||||
# valid.) Adjust if the host runs other meaningful workloads alongside Turnstone.
|
||||
[Unit]
|
||||
Description=Turnstone services slice (server + console + channel)
|
||||
Documentation=https://github.com/turnstonelabs/turnstone
|
||||
Before=slices.target
|
||||
|
||||
[Slice]
|
||||
MemoryHigh=70%
|
||||
MemoryMax=85%
|
||||
TasksMax=16384
|
||||
+11
-75
@@ -2,69 +2,13 @@
|
||||
"""Health check for turnstone containers.
|
||||
|
||||
Usage: healthcheck.py <url>
|
||||
Exit 0 if the endpoint returns {"status": "ok"} or {"status": "degraded"},
|
||||
exit 1 otherwise. Uses only stdlib — no pip dependencies required.
|
||||
|
||||
When the node serves mTLS (tls.enabled), a plain-HTTP probe is rejected at
|
||||
the socket, so on failure this script retries over HTTPS, presenting the
|
||||
node's own certificate as the client cert and pinning the cluster CA. The
|
||||
PEM files are the ones the server writes at boot under
|
||||
$TURNSTONE_TLS_PEM_DIR (default: <tmpdir>/turnstone-tls). The host is
|
||||
rewritten to "localhost" for the TLS attempt because the internal CA issues
|
||||
DNS SANs only — certificate verification rejects a literal-IP dial.
|
||||
|
||||
When mTLS is disabled (the default), the plain probe succeeds and nothing
|
||||
here changes: the PEM directory is never consulted.
|
||||
Exit 0 if the endpoint returns {"status": "ok"}, exit 1 otherwise.
|
||||
Uses only stdlib — no pip dependencies required.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
|
||||
def _check(url: str, context: ssl.SSLContext | None = None) -> None:
|
||||
"""Probe one URL; raise if unreachable or the payload is unhealthy."""
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=5, context=context) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
if data.get("status") not in ("ok", "degraded"):
|
||||
raise RuntimeError(f"unhealthy payload: {data}")
|
||||
|
||||
|
||||
def _pem_root() -> Path:
|
||||
"""PEM runtime root.
|
||||
|
||||
Must mirror turnstone.core.tls.tls_pem_runtime_dir — this script is
|
||||
standalone stdlib and cannot import turnstone; a drift-guard test in
|
||||
tests/test_docker_healthcheck.py pins the two together.
|
||||
"""
|
||||
root_env = os.environ.get("TURNSTONE_TLS_PEM_DIR")
|
||||
return Path(root_env) if root_env else Path(tempfile.gettempdir()) / "turnstone-tls"
|
||||
|
||||
|
||||
def _find_pem_dir() -> Path | None:
|
||||
"""Locate the newest complete PEM dir written by the server at boot."""
|
||||
root = _pem_root()
|
||||
candidates = [
|
||||
d
|
||||
for d in root.glob("lacme-pem-*")
|
||||
if all((d / name).is_file() for name in ("fullchain.pem", "key.pem", "ca.pem"))
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda d: d.stat().st_mtime)
|
||||
|
||||
|
||||
def _tls_url(url: str) -> str:
|
||||
"""Rewrite scheme to https and host to localhost, keeping port and path."""
|
||||
parts = urlsplit(url)
|
||||
netloc = f"localhost:{parts.port}" if parts.port else "localhost"
|
||||
return urlunsplit(("https", netloc, parts.path, parts.query, parts.fragment))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -74,24 +18,16 @@ def main() -> None:
|
||||
|
||||
url = sys.argv[1]
|
||||
try:
|
||||
_check(url)
|
||||
sys.exit(0)
|
||||
except Exception as plain_exc:
|
||||
pem_dir = _find_pem_dir()
|
||||
if pem_dir is None:
|
||||
print(f"Health check failed: {plain_exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
context = ssl.create_default_context(cafile=str(pem_dir / "ca.pem"))
|
||||
context.load_cert_chain(str(pem_dir / "fullchain.pem"), str(pem_dir / "key.pem"))
|
||||
_check(_tls_url(url), context=context)
|
||||
sys.exit(0)
|
||||
except Exception as tls_exc:
|
||||
print(
|
||||
f"Health check failed: plain: {plain_exc}; mtls: {tls_exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
if data.get("status") in ("ok", "degraded"):
|
||||
sys.exit(0)
|
||||
print(f"Unhealthy: {data}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as exc:
|
||||
print(f"Health check failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+79
-260
@@ -63,10 +63,7 @@ Auth is always enabled. All API endpoints except public paths require a valid to
|
||||
Include a token in one of two ways:
|
||||
|
||||
- **Bearer header**: `Authorization: Bearer <token>`
|
||||
- **Cookie**: the surface-scoped auth cookie — `turnstone_auth_server` on
|
||||
turnstone-server, `turnstone_auth_console` on turnstone-console (set
|
||||
automatically by the login endpoint). The names differ so the two surfaces,
|
||||
when co-hosted on one origin, don't overwrite each other's session.
|
||||
- **Cookie**: `turnstone_auth=<token>` (set automatically by the login endpoint)
|
||||
|
||||
The server accepts two token types:
|
||||
|
||||
@@ -105,8 +102,7 @@ Authenticate with credentials and receive a JWT. Accepts two credential formats:
|
||||
}
|
||||
```
|
||||
|
||||
The response also sets a surface-scoped HttpOnly cookie containing the JWT
|
||||
(`turnstone_auth_server` on turnstone-server, `turnstone_auth_console` on turnstone-console).
|
||||
The response also sets a `turnstone_auth` HttpOnly cookie containing the JWT.
|
||||
|
||||
**Response (failure):** `401`
|
||||
|
||||
@@ -118,8 +114,7 @@ The response also sets a surface-scoped HttpOnly cookie containing the JWT
|
||||
|
||||
### `POST /v1/api/auth/logout`
|
||||
|
||||
Clears the surface-scoped auth cookie (`turnstone_auth_server` /
|
||||
`turnstone_auth_console`). No request body required.
|
||||
Clears the `turnstone_auth` cookie. No request body required.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
@@ -204,8 +199,7 @@ this endpoint.
|
||||
}
|
||||
```
|
||||
|
||||
The response also sets a surface-scoped HttpOnly cookie containing the JWT
|
||||
(`turnstone_auth_server` on turnstone-server, `turnstone_auth_console` on turnstone-console).
|
||||
The response also sets a `turnstone_auth` HttpOnly cookie containing the JWT.
|
||||
|
||||
**Response (already set up):** `409`
|
||||
|
||||
@@ -235,12 +229,12 @@ below.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/workstreams/{ws_id}/events`
|
||||
### `GET /v1/api/events?ws_id=<id>`
|
||||
|
||||
Opens a Server-Sent Events stream scoped to a single workstream. The connection
|
||||
remains open indefinitely; the server pushes events as they occur.
|
||||
|
||||
**Path parameters:**
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------------|
|
||||
@@ -287,7 +281,6 @@ Each message in the `messages` array has:
|
||||
| `role` | string | `"user"`, `"assistant"`, or `"tool"` |
|
||||
| `content` | string or null | Text content of the message |
|
||||
| `tool_calls` | array or null | Present only on assistant messages with calls |
|
||||
| `reasoning` | string (optional) | Concatenated reasoning / chain-of-thought text on assistant turns whose `provider_data` carried reasoning-bearing blocks (Anthropic `thinking`, OpenAI Responses `reasoning`, or synthetic `reasoning_text` from local-model servers). Present only when the active model's `surface_persisted_reasoning` flag is True. |
|
||||
|
||||
Each entry in `tool_calls`:
|
||||
|
||||
@@ -332,44 +325,6 @@ finalize any in-progress assistant message.
|
||||
{"type": "stream_end"}
|
||||
```
|
||||
|
||||
**`state_change`** -- the worker thread transitioned to a new state. Drives
|
||||
the client's busy-mode (composer in send vs. stop, spinner indicators,
|
||||
auto-focus on idle). Sent live during normal operation AND on every fresh
|
||||
SSE subscribe (so a mid-stream page refresh restores the correct composer
|
||||
state without waiting for the next live transition).
|
||||
|
||||
```json
|
||||
{"type": "state_change", "state": "running"}
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|----------|--------|----------------------------------------------------------------------|
|
||||
| `state` | string | One of `"running"`, `"thinking"`, `"attention"`, `"idle"`, `"error"` |
|
||||
|
||||
**`in_progress_snapshot`** -- one-shot replay of the in-progress turn's
|
||||
content + reasoning text-so-far when this client connects mid-stream.
|
||||
Lets a refreshing browser tab restore partial assistant text immediately
|
||||
instead of waiting for the response to complete. Yielded once after the
|
||||
kind-specific replay phase (history + pending), only when at least one
|
||||
of `content` / `reasoning` is non-empty. Both halves render into the same
|
||||
assistant bubble the live `content` / `reasoning` events would target;
|
||||
clients should treat the snapshot as idempotent (skip overwrite if the
|
||||
current local buffer is already a superset prefix — covers EventSource
|
||||
auto-reconnect re-replays).
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "in_progress_snapshot",
|
||||
"content": "Here is the answer so far: it depends on ",
|
||||
"reasoning": "The user is asking about a comparison; let me think about..."
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|--------------|--------|------------------------------------------------------------|
|
||||
| `content` | string | Joined assistant content text accumulated this turn |
|
||||
| `reasoning` | string | Joined reasoning / chain-of-thought text accumulated |
|
||||
|
||||
**`tool_info`** -- one or more tool calls that were auto-approved (no user
|
||||
action required).
|
||||
|
||||
@@ -391,7 +346,7 @@ action required).
|
||||
```
|
||||
|
||||
**`approve_request`** -- one or more tool calls that require user approval. The
|
||||
client must respond via `POST /v1/api/workstreams/{ws_id}/approve`.
|
||||
client must respond via `POST /v1/api/approve`.
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -458,9 +413,16 @@ Each item in `items` (shared by `tool_info` and `approve_request`):
|
||||
| `context_window` | int | Total context window size in tokens |
|
||||
| `pct` | float | Percentage of context window used |
|
||||
| `effort` | string | Reasoning effort level (`low`/`medium`/`high`) |
|
||||
| `cache_creation_tokens` | int | Tokens written to prompt cache (Anthropic + OpenAI) |
|
||||
| `cache_creation_tokens` | int | Tokens written to prompt cache (Anthropic) |
|
||||
| `cache_read_tokens` | int | Tokens served from prompt cache (Anthropic + OpenAI) |
|
||||
|
||||
**`plan_review`** -- the model is proposing a plan and wants feedback. The
|
||||
client must respond via `POST /v1/api/plan`.
|
||||
|
||||
```json
|
||||
{"type": "plan_review", "content": "Step 1: ...\nStep 2: ..."}
|
||||
```
|
||||
|
||||
**`info`** -- an informational message (e.g. command output).
|
||||
|
||||
```json
|
||||
@@ -488,7 +450,7 @@ after `/clear` or `/new` commands).
|
||||
```
|
||||
|
||||
**`cancelled`** -- a cancel request was acknowledged (via the Stop button or
|
||||
`POST /v1/api/workstreams/{ws_id}/cancel`). This signals that cancellation is in progress, not
|
||||
`POST /v1/api/cancel`). This signals that cancellation is in progress, not
|
||||
that it is complete. The worker thread may still be finishing — wait for
|
||||
`stream_end` before transitioning to a ready state. The client should clear
|
||||
any in-progress assistant rendering but not re-enable the send button until
|
||||
@@ -560,13 +522,7 @@ Each SSE connection to a workstream receives its own delivery queue. Events
|
||||
produced by the worker thread are fanned out to all registered listener queues,
|
||||
so multiple consumers (browser, console proxy, SDK) can connect
|
||||
simultaneously and each receives every event. On reconnect the client receives
|
||||
the kind-specific replay (`connected` + `status` + `history` + pending
|
||||
approval / plan for interactive; `connected` + `status` + pending for coord)
|
||||
followed by a `state_change` carrying the current worker state and an
|
||||
optional `in_progress_snapshot` carrying any partial content / reasoning
|
||||
buffered for the in-progress turn — so a mid-stream refresh restores both
|
||||
the busy-mode UI and the partial assistant text without waiting for the
|
||||
response to complete.
|
||||
a full history replay, so no catch-up mechanism is needed.
|
||||
|
||||
---
|
||||
|
||||
@@ -602,7 +558,7 @@ Possible `state` values:
|
||||
and copies each event to every client queue. If a client queue is full, the
|
||||
event is silently dropped for that client.
|
||||
|
||||
**Keepalive:** Same as `/v1/api/workstreams/{ws_id}/events` -- an SSE comment every 5 seconds.
|
||||
**Keepalive:** Same as `/v1/api/events` -- an SSE comment every 5 seconds.
|
||||
|
||||
---
|
||||
|
||||
@@ -615,8 +571,8 @@ Returns a list of all active workstreams.
|
||||
```json
|
||||
{
|
||||
"workstreams": [
|
||||
{"ws_id": "abc123", "name": "default", "state": "idle"},
|
||||
{"ws_id": "def456", "name": "hacker-news", "state": "thinking"}
|
||||
{"id": "abc123", "name": "default", "state": "idle"},
|
||||
{"id": "def456", "name": "hacker-news", "state": "thinking"}
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -625,7 +581,7 @@ Each workstream object:
|
||||
|
||||
| Field | Type | Description |
|
||||
|--------------|-------------|--------------------------------------------------------|
|
||||
| `ws_id` | string | Unique workstream routing identifier |
|
||||
| `id` | string | Unique workstream routing identifier |
|
||||
| `name` | string | Display name (alias if set, otherwise `ws-xxxx`) |
|
||||
| `state` | string | Current state (see state values above) |
|
||||
|
||||
@@ -698,62 +654,21 @@ Each skill summary:
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/personas`
|
||||
|
||||
Returns the enabled personas offered by the workstream-creation pickers.
|
||||
Authenticated for any logged-in user and deliberately gated by **no**
|
||||
`persona.*` permission — selecting a persona at creation is a user
|
||||
action, while the `persona.*` perms gate authoring. Display fields only;
|
||||
the levers (base prompt, tool set, MCP/memory toggles) stay server-side.
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"personas": [
|
||||
{"name": "engineer", "display_name": "Engineer", "description": "The stock interactive workstream: full tools, MCP, and memory.", "applies_to_kinds": ["interactive"], "is_default": true},
|
||||
{"name": "researcher", "display_name": "Researcher", "description": "Answers questions with evidence — reads and cites, loads tools to verify when needed.", "applies_to_kinds": ["interactive"], "is_default": false}
|
||||
],
|
||||
"total": 2
|
||||
}
|
||||
```
|
||||
|
||||
Each persona summary:
|
||||
|
||||
| Field | Type | Description |
|
||||
|--------------------|--------|------------------------------------------------------------------|
|
||||
| `name` | string | Persona slug (used in the `persona` field on workstream creation) |
|
||||
| `display_name` | string | Human-readable label for pickers |
|
||||
| `description` | string | Short description of the persona's intent |
|
||||
| `applies_to_kinds` | array | Workstream kinds the persona applies to (`interactive` / `coordinator`) |
|
||||
| `is_default` | bool | Whether this is the default persona for its kind |
|
||||
|
||||
> **Note:** For full persona management (create, edit, archive), use the
|
||||
> admin endpoints at `/v1/api/admin/personas` (requires the
|
||||
> `persona.{create,read,write}` permissions).
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/send`
|
||||
### `POST /v1/api/send`
|
||||
|
||||
Sends a user message to a workstream. Spawns a daemon worker thread that calls
|
||||
`session.send()` and streams results back via the SSE channel.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------|
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"message": "Explain how the server works"}
|
||||
{"message": "Explain how the server works", "ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------------------|
|
||||
| `message` | string | yes | The user's message text |
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Response (success):**
|
||||
|
||||
@@ -777,21 +692,15 @@ from a previous request. Also pushes a `busy_error` event to the SSE stream.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/approve`
|
||||
### `POST /v1/api/approve`
|
||||
|
||||
Responds to a tool approval request. The SSE stream must have previously sent
|
||||
an `approve_request` event for the given workstream.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------|
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"approved": true, "feedback": null, "always": false}
|
||||
{"approved": true, "feedback": null, "always": false, "ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
@@ -799,6 +708,7 @@ an `approve_request` event for the given workstream.
|
||||
| `approved` | bool | yes | `true` to approve, `false` to deny |
|
||||
| `feedback` | string/null | no | Optional feedback text (sent as denial reason) |
|
||||
| `always` | bool | no | If `true` and `approved`, enables auto-approve |
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
When `always` is `true` and `approved` is `true`, the workstream's WebUI
|
||||
instance sets `auto_approve = True`, causing all subsequent tool calls to be
|
||||
@@ -814,6 +724,36 @@ automatically approved without prompting.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/plan`
|
||||
|
||||
Responds to a plan review dialog. The SSE stream must have previously sent a
|
||||
`plan_review` event for the given workstream.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"feedback": "", "ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|------------|--------|----------|---------------------------------------------------------|
|
||||
| `feedback` | string | yes | Feedback text; empty string means approval |
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
To approve the plan, send an empty string for `feedback`. To reject or request
|
||||
changes, send a non-empty feedback string (e.g. `"reject"` or specific
|
||||
revision instructions).
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{"status": "ok"}
|
||||
```
|
||||
|
||||
**Error:** `404` with `{"error": "Unknown workstream"}` if `ws_id` is invalid.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/command`
|
||||
|
||||
Executes a slash command in the given workstream.
|
||||
@@ -849,7 +789,7 @@ containing the resumed session's messages.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/cancel`
|
||||
### `POST /v1/api/cancel`
|
||||
|
||||
Cancels the active generation in a workstream. Sets a cooperative cancellation
|
||||
flag that is checked at multiple points in the generation loop (per streaming
|
||||
@@ -872,20 +812,15 @@ for the orphaned thread. Use force cancel when cooperative cancel has not
|
||||
resolved within a few seconds — the web UI offers this as a "Force Stop"
|
||||
button automatically.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------|
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"force": false}
|
||||
{"ws_id": "abc123", "force": false}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|--------|--------|----------|----------------------|
|
||||
| `ws_id`| string | yes | Target workstream ID |
|
||||
| `force`| bool | no | Abandon stuck worker immediately (default: `false`) |
|
||||
|
||||
**Response:**
|
||||
@@ -907,15 +842,6 @@ button automatically.
|
||||
|
||||
Creates a new workstream. The server supports up to 10 concurrent workstreams.
|
||||
|
||||
The endpoint accepts **either** `application/json` (legacy shape) **or**
|
||||
`multipart/form-data` when you want to upload attachments at creation
|
||||
time. Multipart requests carry one `meta` field containing the JSON body
|
||||
shown below plus zero-or-more `file` parts; each file is validated and
|
||||
reserved onto the new workstream's first turn before the dispatch worker
|
||||
runs, so queued multimodal turns cannot lose files to racing sends. If
|
||||
validation fails the fresh workstream is rolled back so no orphan rows
|
||||
leak.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
@@ -931,7 +857,6 @@ All fields are optional. The body can be empty or an empty JSON object.
|
||||
| `auto_approve` | bool | false | Auto-approve all tool calls for this workstream |
|
||||
| `resume_ws` | string | "" | Workstream ID to resume atomically during creation (empty = fresh)|
|
||||
| `skill` | string | "" | Skill name. Applies content (system prompt), model, temperature, reasoning effort, max tokens, auto-approve policy, token budget, and other session config from the skill. Returns 400 if not found or disabled. Ignored when `resume_ws` is set (resumed sessions restore their own skill). |
|
||||
| `persona` | string | "" | Persona slug. Resolved and snapshotted into the workstream at creation; empty selects the kind's default. |
|
||||
| `judge_model` | string | "" | Optional model alias for the judge (overrides default judge model for this workstream) |
|
||||
|
||||
> **Skill behavior:** When `skill` is specified, the skill's content is injected as a system message and its session config fields (model, temperature, auto-approve, token budget, etc.) override system defaults for the new workstream.
|
||||
@@ -948,7 +873,6 @@ All fields are optional. The body can be empty or an empty JSON object.
|
||||
| `name` | string | Auto-generated workstream name |
|
||||
| `resumed` | bool | Whether a previous session was successfully resumed |
|
||||
| `message_count` | int | Number of messages in the resumed session (0 if fresh) |
|
||||
| `initial_message_status` | string | Present ONLY when the workstream was created but its `initial_message` could not be delivered: `"queue_full"` (a raced live worker's interjection queue was at capacity — resend via `/send`; any uploads stay staged) or `"refused_closed"` (the workstream was closed mid-create). Absent whenever the message was dispatched. |
|
||||
|
||||
**Error (limit reached):**
|
||||
|
||||
@@ -960,32 +884,20 @@ Status code: `400`
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/close`
|
||||
### `POST /v1/api/workstreams/close`
|
||||
|
||||
Closes and removes a workstream. The last remaining workstream cannot be
|
||||
closed.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|------------------------|
|
||||
| `ws_id` | string | yes | Workstream ID to close |
|
||||
|
||||
**Request body:**
|
||||
|
||||
The body must be valid JSON. If you are not supplying any optional
|
||||
fields, send `{}` — an empty / non-JSON body is rejected with a
|
||||
`400`.
|
||||
```json
|
||||
{"ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|----------|--------|----------|----------------------------------------------------------|
|
||||
| `reason` | string | no | Optional close reason persisted to `workstream_config`. |
|
||||
|
||||
The `reason` is capped at **512 UTF-8 bytes** (multibyte-safe — the
|
||||
cap holds for CJK and emoji payloads), and the output guard's
|
||||
credential-redaction pass strips secrets before the value is
|
||||
persisted. A non-string `reason` is silently coerced to empty and
|
||||
the close proceeds without writing the field.
|
||||
| Field | Type | Required | Description |
|
||||
|---------|--------|----------|---------------------------|
|
||||
| `ws_id` | string | yes | Workstream ID to close |
|
||||
|
||||
**Response (success):**
|
||||
|
||||
@@ -1003,100 +915,6 @@ Status code: `400`
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/attachments`
|
||||
|
||||
Upload an image or text document and attach it to the caller's next user
|
||||
turn on this workstream.
|
||||
|
||||
- Images (png/jpeg/gif/webp) are capped at **4 MiB** and validated via
|
||||
magic-byte sniff on upload.
|
||||
- Text documents (any `text/*` MIME, allow-listed application MIMEs, or
|
||||
known text extensions) are capped at **512 KiB** and must be UTF-8.
|
||||
- Per-(workstream, user) pending cap is **10** attachments.
|
||||
|
||||
The attachment moves through three states: `pending → reserved →
|
||||
consumed`. Reservation tokens are threaded through
|
||||
`POST /v1/api/workstreams/{ws_id}/send` so a queued multimodal turn cannot lose its file to
|
||||
an overlapping send.
|
||||
|
||||
Ownership failures are masked as `404` so non-owners cannot enumerate
|
||||
workstream existence.
|
||||
|
||||
**Content-Type:** `multipart/form-data` with a single `file` field.
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"attachment_id": "att_abc123",
|
||||
"kind": "image",
|
||||
"mime_type": "image/png",
|
||||
"size_bytes": 73240,
|
||||
"filename": "screenshot.png",
|
||||
"state": "pending"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
| Code | Meaning |
|
||||
|------|---------------------------------------------------------|
|
||||
| 400 | Missing/invalid form, unsupported MIME, not UTF-8, etc. |
|
||||
| 403 | Auth/scope failure |
|
||||
| 404 | Workstream not found / not owned by caller |
|
||||
| 409 | Pending-cap reached |
|
||||
| 413 | Payload exceeds size cap |
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/workstreams/{ws_id}/attachments`
|
||||
|
||||
List the caller's **pending** (unconsumed) attachments for this
|
||||
workstream. Ownership failures are masked as `404`.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"attachments": [
|
||||
{
|
||||
"attachment_id": "att_abc123",
|
||||
"kind": "image",
|
||||
"mime_type": "image/png",
|
||||
"size_bytes": 73240,
|
||||
"filename": "screenshot.png",
|
||||
"state": "pending"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/workstreams/{ws_id}/attachments/{attachment_id}/content`
|
||||
|
||||
Returns the raw bytes of an attachment with its stored `Content-Type`.
|
||||
Useful for previewing an image or replaying a document. Ownership
|
||||
failures are masked as `404`.
|
||||
|
||||
**Response:** `200` — binary body, original `Content-Type`.
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/workstreams/{ws_id}/attachments/{attachment_id}`
|
||||
|
||||
Remove a pending attachment. Consumed attachments return `404` (they
|
||||
are part of a committed conversation turn). Ownership failures are also
|
||||
masked as `404`.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{"deleted": "att_abc123"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/delete`
|
||||
|
||||
Permanently delete a saved workstream and all its messages from storage.
|
||||
@@ -1690,7 +1508,7 @@ version. Requires the `admin.skills` permission.
|
||||
|
||||
```json
|
||||
{
|
||||
"risk_level": "medium",
|
||||
"scan_status": "medium",
|
||||
"scan_report": "{\"composite\": 1.75, \"details\": {...}}",
|
||||
"scan_version": "1"
|
||||
}
|
||||
@@ -2031,7 +1849,7 @@ Status code: `200` with an empty body.
|
||||
| Malformed or unparseable JSON body | Treated as an empty dict `{}`; missing fields use defaults |
|
||||
| Unknown `ws_id` | `404` with `{"error": "Unknown workstream"}` |
|
||||
| Unknown path (GET or POST) | `404` with plain-text body `Not found` |
|
||||
| Empty `message` on `/v1/api/workstreams/{ws_id}/send` | `400` with `{"error": "Empty message"}` |
|
||||
| Empty `message` on `/v1/api/send` | `400` with `{"error": "Empty message"}` |
|
||||
| Empty `command` on `/v1/api/command` | `400` with `{"error": "Empty command"}` |
|
||||
| Rate limit exceeded | `429` with `Retry-After` header (see below) |
|
||||
|
||||
@@ -2074,7 +1892,7 @@ reconnection:
|
||||
On reconnect, the server replays the full conversation history via the
|
||||
`history` event, so the client can rebuild its UI state without data loss. The
|
||||
same reconnection strategy applies to both the per-workstream SSE stream
|
||||
(`/v1/api/workstreams/{ws_id}/events`) and the global state stream (`/v1/api/events/global`).
|
||||
(`/v1/api/events`) and the global state stream (`/v1/api/events/global`).
|
||||
|
||||
---
|
||||
|
||||
@@ -2184,7 +2002,7 @@ turnstone_workstreams_active_total 1
|
||||
# TYPE turnstone_http_requests_total counter
|
||||
turnstone_http_requests_total{method="GET",endpoint="/health",status_code="200"} 42
|
||||
turnstone_http_requests_total{method="GET",endpoint="/metrics",status_code="200"} 7
|
||||
turnstone_http_requests_total{method="POST",endpoint="/v1/api/workstreams/{ws_id}/send",status_code="200"} 18
|
||||
turnstone_http_requests_total{method="POST",endpoint="/v1/api/send",status_code="200"} 18
|
||||
# HELP turnstone_tokens_total Total tokens consumed
|
||||
# TYPE turnstone_tokens_total counter
|
||||
turnstone_tokens_total{type="prompt"} 84320
|
||||
@@ -2200,15 +2018,15 @@ turnstone_tool_calls_total{tool="read_file"} 3
|
||||
## Console Routing Proxy Endpoints
|
||||
|
||||
These endpoints are served by the console (`turnstone-console`) and proxy
|
||||
requests to the correct server node via rendezvous (HRW) hashing over the
|
||||
live service registry. In multi-node deployments, clients (SDK, channel
|
||||
gateway) talk to the console instead of individual server nodes.
|
||||
requests to the correct server node via the hash ring bucket cache. In
|
||||
multi-node deployments, clients (SDK, channel gateway) talk to the console
|
||||
instead of individual server nodes.
|
||||
|
||||
### `POST /v1/api/route/workstreams/new`
|
||||
|
||||
Create a workstream via rendezvous routing. The console generates the `ws_id`,
|
||||
routes to the rendezvous-selected node, and includes `node_url` in the
|
||||
response for direct SSE connections.
|
||||
Create a workstream via hash-ring routing. The console generates the `ws_id`,
|
||||
routes to the assigned node, and includes `node_url` in the response for
|
||||
direct SSE connections.
|
||||
|
||||
### `POST /v1/api/route/send`
|
||||
|
||||
@@ -2243,4 +2061,5 @@ Used by channel adapters to open direct SSE connections to the correct server no
|
||||
|
||||
Prometheus metrics for the console routing layer. Includes:
|
||||
`turnstone_router_requests_total`, `turnstone_router_request_duration_seconds`,
|
||||
`turnstone_router_membership_size`, `turnstone_router_refresh_total`.
|
||||
`turnstone_ring_membership_size`, `turnstone_ring_version`,
|
||||
`turnstone_ring_rebalance_total`, `turnstone_ring_migrations_total`.
|
||||
|
||||
+112
-336
@@ -3,8 +3,8 @@
|
||||
Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent
|
||||
memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or
|
||||
Anthropic's native Messages API via pluggable provider adapters, and gives the
|
||||
model 16 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
reading, writing, searching, and executing code.
|
||||
model 19 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
reading, writing, searching, planning, and executing code.
|
||||
|
||||
The core design principle is a **UI-agnostic engine with pluggable frontends**.
|
||||
The engine (`ChatSession`) drives the conversation loop -- streaming, tool
|
||||
@@ -19,11 +19,9 @@ plugs in.
|
||||
| `turnstone` | `turnstone.cli` | `TerminalUI` | Interactive terminal REPL |
|
||||
| `turnstone-server` | `turnstone.server` | `WebUI` | Browser-based chat (HTTP + SSE) |
|
||||
| `turnstone-console` | `turnstone.console.server` | ClusterCollector | Cluster dashboard (aggregates all nodes) |
|
||||
| `turnstone-eval` | `turnstone.eval.cli` | `NullUI` | Headless measurement (scores tool-use against expected actions) |
|
||||
| `turnstone-optimizer` | `turnstone.optimizer` | `NullUI` | Prompt/tool optimization (UCB self-modify loop over the eval substrate) |
|
||||
| `turnstone-eval` | `turnstone.eval` | `NullUI` | Headless evaluation and prompt optimization |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) |
|
||||
| `turnstone-admin` | `turnstone.admin` | — | Offline user and API token management |
|
||||
| `turnstone-doctor` | `turnstone.doctor` | — | LLM-backed cluster diagnostics |
|
||||
| `turnstone-admin` | `turnstone.core.admin_cli` | — | Offline user and API token management |
|
||||
|
||||
---
|
||||
|
||||
@@ -38,10 +36,7 @@ turnstone/
|
||||
session.py ChatSession engine, SessionUI protocol, tool dispatch
|
||||
providers/ LLM provider adapters (pluggable backend layer)
|
||||
_protocol.py LLMProvider protocol, ModelCapabilities, StreamChunk, CompletionResult
|
||||
_openai.py OpenAIProvider facade (re-exports Chat/Responses providers)
|
||||
_openai_chat.py OpenAIChatCompletionsProvider — vLLM, llama.cpp, local compatible APIs
|
||||
_openai_responses.py OpenAIResponsesProvider — commercial OpenAI Responses API
|
||||
_openai_common.py Shared ModelCapabilities table + helpers
|
||||
_openai.py OpenAIProvider — OpenAI, vLLM, llama.cpp, any compatible API
|
||||
_anthropic.py AnthropicProvider — Anthropic Messages API, native streaming, thinking
|
||||
_google.py GoogleProvider — Google Gemini via OpenAI-compat endpoint
|
||||
__init__.py create_provider() + create_client() factory functions
|
||||
@@ -62,6 +57,7 @@ turnstone/
|
||||
ratelimit.py Per-IP token-bucket rate limiter (RateLimiter, TokenBucket)
|
||||
edit.py File edit utilities (find_occurrences, pick_nearest)
|
||||
safety.py Command safety validation (blocked patterns, sanitization)
|
||||
sandbox.py Math code sandboxing (AST validation, subprocess execution)
|
||||
web.py Web utilities (HTML stripping, SSRF prevention)
|
||||
api/
|
||||
schemas.py Shared Pydantic v2 models (auth, errors, WorkstreamState)
|
||||
@@ -85,13 +81,12 @@ turnstone/
|
||||
static/ Cluster dashboard web UI (page-specific HTML, CSS, JS)
|
||||
channels/
|
||||
cli.py Unified channel gateway entry point (turnstone-channel)
|
||||
_protocol.py ChannelAdapter protocol
|
||||
_protocol.py ChannelAdapter protocol, ChannelEvent dataclass
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via HTTP
|
||||
_config.py Base ChannelConfig dataclass
|
||||
discord/ Discord adapter (bot, cog, views, streaming, config)
|
||||
slack/ Slack adapter (Socket Mode bot, DM routing, approval buttons)
|
||||
shared_static/ Shared design system (base.css, auth.js, theme.js, toast.js, utils.js, kb.js)
|
||||
katex-0.17.0/ Vendored KaTeX math rendering library (MIT, woff2 fonts)
|
||||
katex-0.16.45/ Vendored KaTeX math rendering library (MIT, woff2 fonts)
|
||||
ui/
|
||||
colors.py ANSI color constants with NO_COLOR support
|
||||
markdown.py Streaming terminal markdown renderer (line-buffered)
|
||||
@@ -102,7 +97,7 @@ turnstone/
|
||||
renderer.js Markdown + LaTeX renderer (tables, nested lists, blockquotes, KaTeX math)
|
||||
app.js Split-pane UI (Pane class, binary layout tree, SSE, tool approval)
|
||||
tools/
|
||||
*.json 16 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
*.json 15 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
```
|
||||
|
||||
Both UIs share a common design system extracted into `turnstone/shared_static/`: design tokens, login overlay, toast notifications, theme toggle, keyboard shortcuts, and utility functions. Each UI imports `base.css` and the shared JS modules at `/shared/`, then adds only page-specific code at `/static/`.
|
||||
@@ -190,6 +185,7 @@ Phase 3: EXECUTE (parallel)
|
||||
(cancel_event also checked per line — kills process group on cancel)
|
||||
Final output (stdout + stderr) delivered via ui.on_tool_result(call_id, name, output)
|
||||
call_id links tool_info items → streaming chunks → final result
|
||||
For plan tool: post-execution gate via ui.on_plan_review()
|
||||
```
|
||||
|
||||
### State Transitions
|
||||
@@ -208,7 +204,7 @@ The engine emits state changes via `_emit_state()` which calls
|
||||
"running" ---> tool execution
|
||||
|
|
||||
v
|
||||
"attention" ---> waiting for user approval
|
||||
"attention" ---> waiting for user approval / plan review
|
||||
|
|
||||
v
|
||||
"running" ---> executing approved tools
|
||||
@@ -230,13 +226,11 @@ The engine emits state changes via `_emit_state()` which calls
|
||||
|
||||
> See also: [Core Engine Classes diagram](diagrams/png/03-core-engine-classes.png)
|
||||
|
||||
Defined in `turnstone.core.session.SessionUI` as a `typing.Protocol` with 15
|
||||
Defined in `turnstone.core.session.SessionUI` as a `typing.Protocol` with 14
|
||||
methods. Every frontend must implement all of them.
|
||||
|
||||
```python
|
||||
class SessionUI(Protocol):
|
||||
def on_turn_start(self) -> None: ...
|
||||
def on_turn_committed(self) -> None: ...
|
||||
def on_thinking_start(self) -> None: ...
|
||||
def on_thinking_stop(self) -> None: ...
|
||||
def on_reasoning_token(self, text: str) -> None: ...
|
||||
@@ -246,20 +240,13 @@ class SessionUI(Protocol):
|
||||
def on_tool_result(self, call_id: str, name: str, output: str, *, is_error: bool = False) -> None: ...
|
||||
def on_tool_output_chunk(self, call_id: str, chunk: str) -> None: ...
|
||||
def on_status(self, usage: dict, context_window: int, effort: str) -> None: ...
|
||||
def on_plan_review(self, content: str) -> str: ...
|
||||
def on_info(self, message: str) -> None: ...
|
||||
def on_error(self, message: str) -> None: ...
|
||||
def on_state_change(self, state: str) -> None: ...
|
||||
def on_rename(self, name: str) -> None: ... # propagate alias to tab/UI label
|
||||
```
|
||||
|
||||
`on_turn_start` fires at the top of each iteration of the send-loop;
|
||||
`on_turn_committed` fires immediately after `messages.append(assistant_msg)`.
|
||||
`SessionUIBase` uses both to reset the per-turn inflight buffers
|
||||
(`_ws_inflight_content` / `_ws_inflight_reasoning` / `_ws_inflight_seq`)
|
||||
that fuel the SSE refresh-resume `in_progress_snapshot` event — see
|
||||
the per-workstream events stream in
|
||||
[`docs/api-reference.md`](api-reference.md#get-v1apiworkstreamsws_idevents).
|
||||
|
||||
`on_rename` is called by the `/name` command (on success) and after a successful `/resume` (if the resumed session has an alias or title). `WebUI.on_rename` broadcasts a `ws_rename` event on the global SSE channel and updates the in-memory `Workstream.name`; `TerminalUI.on_rename` is a no-op.
|
||||
|
||||
### Three Implementations
|
||||
@@ -267,8 +254,8 @@ the per-workstream events stream in
|
||||
| Class | Module | Notes |
|
||||
|-------|--------|-------|
|
||||
| `TerminalUI` | `turnstone.cli` | ANSI colors, `MarkdownRenderer`, `Spinner`, readline-based `input()` for approval |
|
||||
| `WebUI` | `turnstone.server` | SSE event queue per workstream + global broadcast, `threading.Event` for blocking on approval. `on_state_change` sends to both per-workstream and global SSE (the browser UI uses per-workstream `state_change` events to manage busy/idle transitions; `stream_end` only finalizes markdown rendering). |
|
||||
| `NullUI` | `turnstone.eval.core` | Discards all output; `approve_tools` always returns `(True, None)` |
|
||||
| `WebUI` | `turnstone.server` | SSE event queue per workstream + global broadcast, `threading.Event` for blocking on approval/plan. `on_state_change` sends to both per-workstream and global SSE (the browser UI uses per-workstream `state_change` events to manage busy/idle transitions; `stream_end` only finalizes markdown rendering). |
|
||||
| `NullUI` | `turnstone.eval` | Discards all output; `approve_tools` always returns `(True, None)` |
|
||||
|
||||
### WorkstreamTerminalUI
|
||||
|
||||
@@ -279,9 +266,10 @@ awareness:
|
||||
are appended to `_output_buffer` instead of written to stdout. When the user
|
||||
switches to this workstream, `flush_buffer()` replays them.
|
||||
|
||||
- **Approval blocking**: `approve_tools()` calls `_fg_event.wait()` when in
|
||||
background, blocking the worker thread until the workstream is foregrounded.
|
||||
This ensures the user sees the approval prompt in the correct context.
|
||||
- **Approval blocking**: `approve_tools()` and `on_plan_review()` call
|
||||
`_fg_event.wait()` when in background, blocking the worker thread until the
|
||||
workstream is foregrounded. This ensures the user sees the approval prompt
|
||||
in the correct context.
|
||||
|
||||
- **Foreground/background toggle**: `set_foreground(bool)` sets or clears
|
||||
`_fg_event` (a `threading.Event`). The manager calls this during `/ws <N>`
|
||||
@@ -391,11 +379,11 @@ non-idle background workstreams above the input prompt.
|
||||
(`Ctrl+\`, `Ctrl+Shift+\`). Max 6 panes; no duplicate workstreams across panes.
|
||||
Layout persisted to `localStorage`.
|
||||
- **Per-pane SSE**: `Pane.connectSSE(wsId)` opens
|
||||
`/v1/api/workstreams/{ws_id}/events` for each pane's event stream independently.
|
||||
`/v1/api/events?ws_id=<id>` for each pane's event stream independently.
|
||||
- **Global SSE**: `connectGlobalSSE()` opens `/v1/api/events/global` which
|
||||
receives `ws_state` broadcasts from all workstreams, used to update tab
|
||||
indicators and pane headers without switching.
|
||||
- **New tab / close**: POST `/v1/api/workstreams/new`, POST `/v1/api/workstreams/{ws_id}/close`.
|
||||
- **New tab / close**: POST `/v1/api/workstreams/new`, POST `/v1/api/workstreams/close`.
|
||||
|
||||
### Thread Safety
|
||||
|
||||
@@ -418,6 +406,7 @@ turnstone metadata keys:
|
||||
|
||||
| Metadata Key | Type | Meaning |
|
||||
|-------------|------|---------|
|
||||
| `agent` | `bool` | Include this tool when running as a plan/task sub-agent |
|
||||
| `task_agent` | `bool` | Include this tool when running as a task sub-agent |
|
||||
| `auto_approve` | `bool` | Tool is read-only; skip user approval |
|
||||
| `primary_key` | `str` | Fallback argument name for bare-string JSON recovery |
|
||||
@@ -437,6 +426,7 @@ Example (`read_file.json`):
|
||||
},
|
||||
"required": ["path"]
|
||||
},
|
||||
"agent": true,
|
||||
"task_agent": true,
|
||||
"auto_approve": true,
|
||||
"primary_key": "path"
|
||||
@@ -447,39 +437,34 @@ At import time, `turnstone.core.tools._load_tools()` strips the metadata keys
|
||||
from each schema and builds:
|
||||
|
||||
- `TOOLS` -- list of `{"type": "function", "function": {...}}` dicts for the API
|
||||
- `AGENT_TOOLS` -- subset with `agent: true`
|
||||
- `TASK_AGENT_TOOLS` -- subset with `task_agent: true`
|
||||
- `TASK_AUTO_TOOLS` -- set of tool names with `auto_approve: true`
|
||||
- `AGENT_AUTO_TOOLS` / `TASK_AUTO_TOOLS` -- sets of tool names with `auto_approve: true`
|
||||
- `PRIMARY_KEY_MAP` -- `{name: primary_key}` for JSON fallback recovery
|
||||
- `merge_mcp_tools(builtin, mcp_tools)` -- merges built-in + MCP tools at session init
|
||||
|
||||
### 16 Tools by Category
|
||||
### 13 Tools by Category
|
||||
|
||||
**Read-only (auto-approve)**:
|
||||
- `read_file` -- read file contents with optional offset/limit
|
||||
- `diff_file` -- show diff between two files / versions
|
||||
- `search` -- ripgrep-based codebase search
|
||||
- `man` -- read man pages
|
||||
- `recall` -- search conversation history
|
||||
- `read_resource` -- read an MCP resource by URI
|
||||
|
||||
**Write (requires approval)**:
|
||||
- `bash` -- execute shell commands (with safety checks via `turnstone.core.safety`)
|
||||
- `write_file` -- create or overwrite a file
|
||||
- `edit_file` -- string replacement in an existing file (requires prior `read_file`)
|
||||
- `math` -- execute Python in sandboxed subprocess (via `turnstone.core.sandbox`)
|
||||
- `web_fetch` -- fetch a URL (with SSRF protection via `turnstone.core.web`)
|
||||
- `web_search` -- search the web (provider-native for Anthropic/OpenAI, self-hosted SearxNG fallback for local models)
|
||||
- `notify` -- send a user-facing notification (Discord/Slack, optional reply routing)
|
||||
- `watch` -- schedule a recurring poll with condition DSL
|
||||
- `web_search` -- search the web (provider-native for Anthropic/OpenAI, Tavily fallback for local models)
|
||||
|
||||
**Agent (delegated sub-sessions)**:
|
||||
- `task_agent` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`)
|
||||
- `task` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`)
|
||||
- `plan` -- explore codebase and write a structured plan (`AGENT_TOOLS`)
|
||||
|
||||
**Memory / skills / prompts**:
|
||||
**Memory (structured persistent store)**:
|
||||
- `memory` -- save, search, delete, or list memories (typed and scoped)
|
||||
- `skill` -- invoke a skill (governed, versioned procedure)
|
||||
- `use_prompt` -- fetch and apply a prompt template
|
||||
|
||||
The tool name uses the `_agent` suffix — bare `task` collides with
|
||||
chat-template channels on some local models.
|
||||
|
||||
### Prepare / Execute Pattern
|
||||
|
||||
@@ -498,11 +483,17 @@ separation allows the UI to show previews before any side effects occur.
|
||||
|
||||
### Agent Tools
|
||||
|
||||
`task_agent` invokes `_run_agent()`, which runs a multi-turn loop with a
|
||||
subset of tools and its own system prompt. The sub-agent runs independently,
|
||||
then returns the final content as the tool result.
|
||||
`task` and `plan` invoke `_run_agent()`, which runs a multi-turn loop with
|
||||
a subset of tools and its own system prompt. The sub-agent runs
|
||||
independently, then returns the final content as the tool result.
|
||||
|
||||
- **task_agent**: uses `self._task_tools` (`TASK_AGENT_TOOLS` + MCP tools)
|
||||
- **task**: uses `self._task_tools` (`TASK_AGENT_TOOLS` + MCP tools)
|
||||
- **plan**: uses `self._agent_tools` (`AGENT_TOOLS` + MCP tools). Writes output
|
||||
to `.plan-<ws_id>.md` — unique per `ChatSession` so concurrent workstreams
|
||||
don't collide. On repeat invocations the prior `plan` tool call and its result
|
||||
are forwarded from `self.messages` so the agent refines the existing plan rather
|
||||
than starting over. Planning instructions are injected as a developer message
|
||||
prepended to the agent's conversation.
|
||||
- **Turn limit**: controlled by `agent_max_turns` (default: `-1`, unlimited).
|
||||
When a limit is set and reached, the agent is forced to synthesize a final
|
||||
response without tools. When unlimited, the loop only exits when the model
|
||||
@@ -541,16 +532,18 @@ adds, removes, or reconnects servers as needed.
|
||||
6. `_exec_mcp_tool()` calls `call_tool_sync()` which dispatches to the async loop
|
||||
via `asyncio.run_coroutine_threadsafe()`
|
||||
|
||||
**Tool refresh:** Two mechanisms keep tools up-to-date without restart:
|
||||
**Tool refresh:** Three mechanisms keep tools up-to-date without restart:
|
||||
- **Push:** Servers declaring `tools.listChanged` send `ToolListChangedNotification`;
|
||||
the registered `message_handler` triggers immediate single-server refresh.
|
||||
- **Periodic:** Servers without push support are polled on a staggered interval
|
||||
(default 4 h, configurable via `[mcp] refresh_interval` or `--mcp-refresh-interval`).
|
||||
- **Manual:** `/mcp refresh [server]` calls `refresh_sync()` for on-demand refresh
|
||||
(also attempts reconnection for disconnected servers).
|
||||
|
||||
When tools change, `_rebuild_tools()` creates new `_tools`/`_tool_map` objects
|
||||
(copy-on-write for thread safety) and notifies listener callbacks. Each `ChatSession`
|
||||
rebuilds its `_tools` and `_task_tools` lists and reconstructs `ToolSearchManager`
|
||||
(preserving expanded tools).
|
||||
rebuilds its merged tool lists and reconstructs `ToolSearchManager` (preserving
|
||||
expanded tools).
|
||||
|
||||
**Tool naming:** `mcp__{server}__{tool}` — double underscore delimiter, validated
|
||||
at connection time (server names with `__` are rejected).
|
||||
@@ -565,11 +558,10 @@ from a healthy connection do not trip the breaker. When the cooldown expires
|
||||
(`call_tool_sync`, `read_resource_sync`, `get_prompt_sync`, `refresh_sync`)
|
||||
cancel orphaned futures on timeout to prevent coroutine accumulation on the
|
||||
background event loop. Push notification refreshes are debounced (5 s per
|
||||
server) to protect against notification storms. Operators can force a
|
||||
catalog refresh or full reconnect from the admin panel; reconnects clear
|
||||
the circuit breaker and run a fresh handshake. Transport stream references
|
||||
are pre-closed before stack teardown to work around the MCP SDK's anyio
|
||||
cancel-scope CPU busy-loop (SDK #2147).
|
||||
server) to protect against notification storms. The periodic refresh loop
|
||||
attempts reconnection for disconnected servers with exponential backoff
|
||||
(60 s–1 h). Transport stream references are pre-closed before stack teardown to
|
||||
work around the MCP SDK's anyio cancel-scope CPU busy-loop (SDK #2147).
|
||||
|
||||
**Error isolation:** Per-server connection/refresh failures are caught and logged; other
|
||||
servers are unaffected. Tool execution errors return error strings to the LLM
|
||||
@@ -614,38 +606,27 @@ LLMProvider (protocol)
|
||||
| `get_capabilities()` | Per-model flags (`ModelCapabilities`) |
|
||||
| `convert_tools()` | Translate OpenAI tool schemas to provider format |
|
||||
| `retryable_error_names` | Exception class names that trigger retry |
|
||||
| `extract_reasoning_text()` | Walk stored `provider_blocks`, return concatenated reasoning text for UI rehydration (per-provider block-type knowledge: Anthropic `thinking`, OpenAI Responses `reasoning`, OpenAI Chat synthetic `reasoning_text`) |
|
||||
|
||||
**Normalized data types:**
|
||||
|
||||
| Type | Fields |
|
||||
|------|--------|
|
||||
| `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason`, `provider_blocks` |
|
||||
| `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage`, `provider_blocks` |
|
||||
| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search`, `supports_tool_search`, `supports_vision`, `supports_reasoning_replay`, `supports_verbosity`, `verbosity`, `supports_pro_mode`, `reasoning_mode` |
|
||||
| `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason` |
|
||||
| `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage` |
|
||||
| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search`, `supports_tool_search`, `supports_vision` |
|
||||
| `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens`, `cache_creation_tokens`, `cache_read_tokens` |
|
||||
|
||||
**OpenAIProvider** (`_openai.py`): passes messages through unchanged (they are
|
||||
already in OpenAI format), including multi-part content blocks (text + images)
|
||||
in tool results. Model capability lookup covers GPT-5 through GPT-5.6,
|
||||
in tool results. Model capability lookup table covers GPT-5/5.1/5.2/5.3/5.4,
|
||||
O-series, and search models (`gpt-5-search-api`) — all with `supports_vision`.
|
||||
For search models, injects `web_search_options` and removes the `web_search`
|
||||
function tool (the model always searches). Citations from `url_citation`
|
||||
annotations are formatted as footnotes. Pre-5.6 GPT-5 models request extended
|
||||
prompt-cache retention (`prompt_cache_retention: "24h"`); GPT-5.6 uses
|
||||
`prompt_cache_options.ttl: "30m"`. Cache reads and writes are extracted from
|
||||
`cached_tokens` and `cache_write_tokens`. Unknown models get permissive
|
||||
defaults with `supports_vision=False` and use SearxNG for web search. The
|
||||
`openai-compatible` lane never consults this table at all — on either API
|
||||
surface (the responses pin is served by a compat-mode
|
||||
`OpenAIResponsesProvider`, mirroring `AnthropicProvider(compat=True)`): a
|
||||
local server serves whatever the operator named it (vLLM
|
||||
`--served-model-name` is a free string), so a prefix collision with a cloud
|
||||
model id must not inherit that model's sampling/effort contract — every
|
||||
local model gets the plain defaults, commercial prompt-cache controls are not
|
||||
injected by model-name prefix, and anything beyond those defaults is declared
|
||||
on the model definition (capabilities JSON + `server_compat`), matching the
|
||||
`anthropic-compatible` lane.
|
||||
annotations are formatted as footnotes. Extended prompt cache retention
|
||||
(`prompt_cache_retention: "24h"`) is enabled for GPT-5.x models at no
|
||||
additional cost. Cached token counts are extracted from
|
||||
`usage.prompt_tokens_details.cached_tokens`. Unknown models (local servers) get
|
||||
permissive defaults with `supports_vision=False` and use Tavily for web search.
|
||||
|
||||
**AnthropicProvider** (`_anthropic.py`): converts OpenAI-format messages to
|
||||
Anthropic content blocks, maps `system`/`developer` roles to the `system`
|
||||
@@ -663,8 +644,9 @@ display). Automatic prompt caching is enabled via top-level `cache_control:
|
||||
cacheable block and advances it as conversations grow (90% input cost
|
||||
reduction on cache hits, 1.25x write on first turn). Cache metrics
|
||||
(`cache_creation_input_tokens`, `cache_read_input_tokens`) are extracted from
|
||||
both streaming and non-streaming responses. The `anthropic` SDK is a core
|
||||
dependency — the Anthropic provider is first-class alongside OpenAI.
|
||||
both streaming and non-streaming responses. The `anthropic` SDK is imported
|
||||
lazily so it remains an optional dependency (`pip install
|
||||
turnstone[anthropic]`).
|
||||
|
||||
**GoogleProvider** (`_google.py`): extends `OpenAIChatCompletionsProvider` for
|
||||
the Gemini `/v1beta/openai/` endpoint. Uses a single default
|
||||
@@ -713,56 +695,7 @@ agent_model = "claude"
|
||||
|
||||
Each `[models.*]` entry produces a `ModelConfig` with a `provider` field
|
||||
(default: `"openai"`). Supported values: `"openai"`, `"anthropic"`, `"google"`,
|
||||
`"openai-compatible"`, and `"anthropic-compatible"`.
|
||||
|
||||
**Per-model sampling overrides:** Each model can specify `temperature`,
|
||||
`max_tokens`, and `reasoning_effort` to override the global defaults from
|
||||
ConfigStore. When unset (`NULL`), the global default is used.
|
||||
|
||||
**Per-model reasoning persistence:** Two booleans on `model_definitions`
|
||||
(migration 052) control how reasoning text round-trips:
|
||||
|
||||
* `surface_persisted_reasoning` (default `True`) — gates whether stored
|
||||
reasoning text is surfaced on `/history` payloads for UI rehydration.
|
||||
**Storage of reasoning bytes happens regardless of this flag** — they
|
||||
ride in `provider_data` independently. Phase-1 admin UI label "Surface
|
||||
persisted reasoning."
|
||||
* `replay_reasoning_to_model` (default `False`) — gates whether stored
|
||||
reasoning blocks are sent back to the provider on subsequent turns.
|
||||
Capability-gated: `ModelCapabilities.supports_reasoning_replay` must
|
||||
also be `True` for the wire path to actually replay (canonical OpenAI
|
||||
gpt-5*/o-series and Anthropic Claude entries set it; unknown / local-
|
||||
server models default to `False`).
|
||||
|
||||
Three reasoning paths are recognised:
|
||||
|
||||
| Path | Provider | Capture | Persist | Replay |
|
||||
|------|----------|---------|---------|--------|
|
||||
| 1 | Anthropic Messages API | `thinking_delta` | `provider_blocks` (`type="thinking"`) | Verbatim via `_provider_content` |
|
||||
| 2 | OpenAI Responses (gpt-5*, o-series) | `response.reasoning_text.delta` events | `provider_blocks` (`type="reasoning"`) — only when `include=["reasoning.encrypted_content"]` | `ResponseReasoningItemParam` input items |
|
||||
| 3 | OpenAI Chat Completions (vLLM, llama.cpp, Gemini-compat) | `delta.reasoning_content` Pydantic extras | Synthetic `{type: "reasoning_text", text, source}` block stamped at end-of-stream | None — no API surface for replay on Chat Completions |
|
||||
|
||||
Cross-provider safety is enforced by `ANTHROPIC_VALID_BLOCK_TYPES` (a
|
||||
shape filter in `_anthropic.py:_convert_messages`): foreign blocks
|
||||
(OpenAI `reasoning`, synthetic `reasoning_text`) fall through to the
|
||||
text+tool_calls rebuild path rather than reaching Anthropic's input
|
||||
boundary as malformed content.
|
||||
|
||||
```toml
|
||||
[models.local]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
model = "qwen3-32b"
|
||||
temperature = 0.7
|
||||
max_tokens = 8192
|
||||
|
||||
[models.o3]
|
||||
base_url = "https://api.openai.com/v1"
|
||||
api_key = "sk-..."
|
||||
model = "o3"
|
||||
reasoning_effort = "high"
|
||||
# temperature omitted — uses global default
|
||||
```
|
||||
|
||||
and `"openai-compatible"`.
|
||||
An optional `[models.*.capabilities]` sub-table overrides per-model
|
||||
`ModelCapabilities` flags (useful for local models whose capabilities
|
||||
cannot be detected programmatically):
|
||||
@@ -776,162 +709,9 @@ model = "qwen-3.5-vl"
|
||||
supports_vision = true
|
||||
```
|
||||
|
||||
**Anthropic-compatible local servers (vLLM `/v1/messages`):** the
|
||||
`"anthropic-compatible"` provider drives local servers that expose
|
||||
Anthropic's Messages API for arbitrary checkpoints — vLLM's
|
||||
`/v1/messages` endpoint, which requires a release with thinking-block
|
||||
support in the Anthropic endpoint (post-2026-02-28; verified against
|
||||
v0.22.1rc1). The lane reuses `AnthropicProvider` in compat mode: same
|
||||
wire translation as the real Anthropic lane, but every model resolves to
|
||||
the `_ANTHROPIC_COMPAT_DEFAULT` capabilities (200K context, 64K output,
|
||||
`token_param=max_tokens`, `thinking_mode=none`, no native
|
||||
web_search/tool_search, no vision) — the static Claude table never
|
||||
applies to local checkpoints. `base_url` is required — the server root
|
||||
WITHOUT `/v1` (the Anthropic SDK appends `/v1/messages`); a trailing
|
||||
`/v1` pasted out of openai-compatible habit is stripped automatically,
|
||||
and an empty value fails at client construction rather than falling
|
||||
back to the commercial endpoint. Set a
|
||||
placeholder `api_key` (e.g. `"dummy"`) for unauthenticated servers. Tool calling
|
||||
needs the server started with `--enable-auto-tool-choice
|
||||
--tool-call-parser <family>` plus the matching reasoning parser.
|
||||
Per-model capability overrides opt in to what the checkpoint actually
|
||||
supports:
|
||||
|
||||
```toml
|
||||
[models.vllm-claude]
|
||||
provider = "anthropic-compatible"
|
||||
base_url = "http://localhost:8000" # no /v1 — the SDK appends /v1/messages
|
||||
api_key = "dummy"
|
||||
model = "deepseek-ai/DeepSeek-V4-Flash"
|
||||
|
||||
[models.vllm-claude.capabilities]
|
||||
supports_vision = true # multimodal checkpoints only
|
||||
supports_mid_conversation_system = true # template-dependent
|
||||
context_window = 131072
|
||||
thinking_mode = "manual" # session effort knob drives the template toggle
|
||||
thinking_param = "enable_thinking" # Qwen/Gemma key; "thinking" for Granite/DeepSeek
|
||||
```
|
||||
|
||||
Reasoning control does NOT use Anthropic's `thinking` request param —
|
||||
the levers live in the chat template, reached through
|
||||
`chat_template_kwargs` in the request body. Two channels, dynamic first:
|
||||
|
||||
* **Session effort knob (dynamic).** Set the model's thinking mode to
|
||||
"Effort-knob controlled" in the admin Models form (or
|
||||
`thinking_mode = "manual"` + `thinking_param` under
|
||||
`[models.*.capabilities]`) and the provider maps the session's
|
||||
reasoning-effort knob onto the template toggle per-request: effort
|
||||
`none` sends `{<thinking_param>: false}`, any other level sends
|
||||
`true` — the same contract as the real lane's manual mode. ("Always
|
||||
on" / `thinking_mode = "adaptive"` instead always sends `true`: the
|
||||
model self-regulates, so the knob never force-disables — mirroring
|
||||
the native adaptive branch.) The graded effort value always rides
|
||||
alongside the toggle: under `effort_param` when the operator names
|
||||
the template's key, else under the conventional fallback key
|
||||
(`reasoning_effort`) on the anthropic-compatible lane — the user's
|
||||
effort setting always reaches the wire, and a template that doesn't
|
||||
reference the kwarg ignores it. On the openai-compatible lane the
|
||||
undeclared-key case rides the flat top-level `reasoning_effort`
|
||||
param instead (the documented compat field), forwarded verbatim.
|
||||
Optional `reasoning_effort_values` / `default_reasoning_effort`
|
||||
validate the knob before it reaches the server; without declared
|
||||
values the knob is forwarded as-is. The knob is ordinal, and validation
|
||||
respects that: an off-list knob value rounds UP onto the declared
|
||||
list and a value above the ceiling rides the ceiling
|
||||
(`snap_reasoning_effort`) — asking for more effort than the model
|
||||
declares never falls back to a lower default tier. The knob's
|
||||
`none` position is forwarded verbatim when the model declares an
|
||||
explicit `none` level (gpt-5.1+, grok-4.3) — omitting it there would
|
||||
leave a reasoning-on server default (e.g. gpt-5.5's `medium`) in
|
||||
charge of a knob that promises off — and omitted otherwise; `none`
|
||||
is never a snap target for other positions.
|
||||
`default_reasoning_effort` only catches values the ordinal snap
|
||||
cannot rank (custom strings). Declare values that match the
|
||||
template's documented vocabulary: for DeepSeek-V4, which officially
|
||||
accepts `high`/`max` (Think High is the default thinking tier;
|
||||
`low`/`medium` alias to `high`, `xhigh` to `max`), a
|
||||
`("high", "max")` values list reproduces the official aliasing
|
||||
exactly — `low`/`medium` round up to `high`, `xhigh` to `max` —
|
||||
and freeform passthrough matches it too. To map an undocumented
|
||||
template, probe with per-request `chat_template_kwargs` and compare
|
||||
`input_tokens`. Setting `effort_param` also suppresses the
|
||||
flat top-level `reasoning_effort` request param on the
|
||||
openai-compatible lane — the template channel replaces it, never
|
||||
doubles it. With the default `thinking_mode = "none"` nothing is
|
||||
injected and the server's template default decides.
|
||||
|
||||
Upgrade note: before 1.7.0a7 the openai-compatible lane sent the
|
||||
toggle unconditionally `true` whenever thinking mode was enabled. A
|
||||
stored per-model `reasoning_effort = "none"` now disables thinking
|
||||
on such models — pick any real level (or clear the override) to keep
|
||||
it on. Also since 1.7.0a7 the effort level itself always reaches the
|
||||
wire on the local lanes (previously dropped unless
|
||||
`reasoning_effort_values` was declared): flat `reasoning_effort` on
|
||||
openai-compatible, the `effort_param`-or-fallback template key on
|
||||
anthropic-compatible when reasoning control is engaged.
|
||||
* **Operator pin (static).** Entries under `{"chat_template_kwargs":
|
||||
...}` in the admin Models extra-body field ride the SDK's
|
||||
`extra_body` unconditionally and win over the knob mapping on key
|
||||
collision — e.g. pin `{"enable_thinking": true}` to keep thinking on
|
||||
regardless of the session knob. (Server type and API surface remain
|
||||
openai-compatible-only knobs and stay hidden for this provider.)
|
||||
|
||||
The same knob mapping drives the `openai-compatible` lane's Chat
|
||||
Completions requests — `merge_reasoning_template_kwargs` is shared by
|
||||
both local-server lanes, so `thinking_mode`/`thinking_param`/
|
||||
`effort_param` mean the same thing whichever endpoint serves the model.
|
||||
Only the Responses API surface (native reasoning) ignores it.
|
||||
|
||||
The console surfaces this projection as an *effective effort ladder*:
|
||||
the admin model form's per-model effort select and the skill
|
||||
launch-config effort select annotate each position with what the
|
||||
request will carry, in plain words — a position whose delivered level
|
||||
matches its name stays plain ("Max"), a snapped position says so
|
||||
("Low — sends high"), the adaptive lanes' none position warns
|
||||
"thinking stays on", and budget detail lives in the tooltip. A
|
||||
position is never labeled after a sibling that shares its wire (that
|
||||
rendered "Max (= minimal)", implying a downgrade the wire doesn't
|
||||
contain). Computed server-side by `providers/effort_ladder.py` from
|
||||
the same mapping functions the providers use at request time and
|
||||
shipped on `/v1/api/models` rows (every row carries `effort_ladder`,
|
||||
empty when the capabilities column fails to parse) and
|
||||
`POST /v1/api/admin/models/effort-ladder`. The ladder describes what
|
||||
Turnstone sends — a server-side template may alias further (DeepSeek-V4
|
||||
folds `low`/`medium` into its default `high` tier).
|
||||
|
||||
The `anthropic-compatible` lane never sends Anthropic's native
|
||||
`thinking`/`output_config` params — they are not in vLLM's request
|
||||
schema. The real `anthropic` provider is unaffected: official Claude
|
||||
models keep native thinking, budget mapping, and `output_config`
|
||||
effort. A gateway fronting *real* Claude on a Messages-shaped URL
|
||||
(e.g. a LiteLLM `anthropic/` route to the Claude API) should use
|
||||
`provider = "anthropic"` with a custom `base_url`, which keeps the
|
||||
native thinking params.
|
||||
|
||||
Verified quirks of vLLM's Anthropic endpoint:
|
||||
|
||||
* The `thinking` request param is silently dropped — use
|
||||
`chat_template_kwargs` (above) to control reasoning.
|
||||
* `stop_sequences` cut the raw stream wherever the text appears —
|
||||
including inside thinking — and report `end_turn` with
|
||||
`stop_sequence=None`. Turnstone does not send stop sequences from
|
||||
this provider.
|
||||
* No cache telemetry: `usage` carries input/output token counts only
|
||||
(no `cache_creation_input_tokens` / `cache_read_input_tokens`).
|
||||
* Images require a multimodal checkpoint — text-only models return a
|
||||
500 on image blocks, so `supports_vision` stays opt-in per model.
|
||||
* Mid-conversation `role: "system"` turns are template-dependent —
|
||||
opt in per model via `supports_mid_conversation_system`.
|
||||
|
||||
**Database model definitions:** On server entry points, models can also be
|
||||
defined in the `model_definitions` table (admin Models tab). DB models support
|
||||
the same per-model sampling overrides. Config.toml models override DB models
|
||||
with the same alias in-memory (the DB rows are never modified).
|
||||
|
||||
**Lifecycle:**
|
||||
1. `load_model_registry()` loads DB model definitions (if storage available),
|
||||
then overlays `[models.*]` from config.toml, then builds a `"default"` entry
|
||||
from CLI `--base-url`/`--model`/`--api-key` args
|
||||
1. `load_model_registry()` reads `[models.*]` sections from config.toml and
|
||||
builds a `"default"` entry from CLI `--base-url`/`--model`/`--api-key` args
|
||||
2. The registry is passed to the session factory closure in both `cli.py` and
|
||||
`server.py`; each workstream resolves its model on creation
|
||||
3. `ModelRegistry.get_client()` lazily creates SDK client instances via
|
||||
@@ -940,11 +720,10 @@ with the same alias in-memory (the DB rows are never modified).
|
||||
4. `ModelRegistry.get_provider()` lazily creates `LLMProvider` instances via
|
||||
`create_provider()` (also cached and thread-safe)
|
||||
5. `/model` command shows available models; `/model <alias>` switches the
|
||||
active workstream's client, model, context window, and per-model sampling
|
||||
parameters
|
||||
active workstream's client, model, and context window
|
||||
6. `_create_stream_with_retry()` tries the primary model, then each fallback
|
||||
alias in order if the primary is unreachable
|
||||
7. `_run_agent()` resolves `registry.agent_model` (if set) for task
|
||||
7. `_run_agent()` resolves `registry.agent_model` (if set) for plan/task
|
||||
sub-agents, allowing a cheaper model for autonomous loops
|
||||
|
||||
**Per-workstream selection:** `POST /v1/api/workstreams/new` accepts an optional
|
||||
@@ -953,7 +732,7 @@ which can override the model before workstream creation.
|
||||
|
||||
### Tool Output Truncation
|
||||
|
||||
Tool execution results (bash, read_file, search) are truncated by
|
||||
Tool execution results (bash, read_file, search, math, man) are truncated by
|
||||
`_truncate_output()` when they exceed `tool_truncation` characters. Truncation
|
||||
preserves the first half and last half of the output, with a message in
|
||||
between:
|
||||
@@ -1064,7 +843,7 @@ and are the single source of truth for both backends and Alembic migrations.
|
||||
| `update_workstream_title(ws_id, title)` | Set/update LLM-generated title |
|
||||
| `update_workstream_state(ws_id, state)` | Update workstream state and bump timestamp |
|
||||
| `update_workstream_name(ws_id, name)` | Update workstream display name |
|
||||
| `list_workstreams(node_id, limit, *, parent_ws_id, kind, user_id)` | List workstreams, optionally filtered by node, parent, kind, or owning user |
|
||||
| `list_workstreams(node_id, limit)` | List workstreams, optionally by node |
|
||||
| `kv_get(key)` / `kv_set(key, value)` / `kv_delete(key)` | Generic key-value store (backs memories table) |
|
||||
| `kv_list()` / `kv_search(query)` | List or search key-value pairs |
|
||||
| `search_history(query, limit)` | Full-text search (FTS5 on SQLite, tsvector on PostgreSQL) |
|
||||
@@ -1118,10 +897,9 @@ reconstructs the OpenAI message format from database rows:
|
||||
in the same workstream
|
||||
|
||||
**Config persistence:** LLM-affecting parameters (`temperature`,
|
||||
`reasoning_effort`, `max_tokens`, `instructions`, and the persona
|
||||
snapshot — see `docs/personas.md`) are persisted to the
|
||||
`workstream_config` table on creation and whenever changed via slash
|
||||
commands. `resume()` restores these values so resumed workstreams
|
||||
`reasoning_effort`, `max_tokens`, `instructions`, `creative_mode`) are
|
||||
persisted to the `workstream_config` table on creation and whenever changed
|
||||
via slash commands. `resume()` restores these values so resumed workstreams
|
||||
behave identically to the original.
|
||||
|
||||
**`/clear` vs `/new`:** `/clear` wipes in-memory context but preserves
|
||||
@@ -1200,12 +978,11 @@ warns if the summary was truncated.
|
||||
unhandled promise rejections
|
||||
- **Pending approval across tab switches**: `WebUI._pending_approval` stores
|
||||
the `approve_request` event payload while the session is blocked waiting
|
||||
for user response. On tab switch / reconnect the pane reloads history via
|
||||
REST `GET /history` and then reconnects SSE; the live approval event is
|
||||
re-injected. The server-side `project_history_messages` projection marks
|
||||
the trailing orphan tool-call turn `"pending": true` so `replayHistory`
|
||||
skips the false `✓ approved` badge; the live approval UI is rendered by
|
||||
the re-injected event instead.
|
||||
for user response. On SSE reconnect (e.g., switching back to the tab),
|
||||
the event is re-injected after history replay. `_build_history` marks the
|
||||
pending tool call as `"pending": true` so `replayHistory` skips the
|
||||
false `✓ approved` badge; the live approval UI is rendered by the
|
||||
re-injected event instead.
|
||||
- **Browser history integration**: `history.pushState` is called in
|
||||
`switchTab()` with `{turnstone: 'workstream', wsId}`. The initial state is
|
||||
seeded with `history.replaceState({turnstone: 'dashboard'})` on load. The
|
||||
@@ -1282,8 +1059,8 @@ Three hierarchical scopes control endpoint access:
|
||||
| Scope | Grants | Endpoints |
|
||||
|-------|--------|-----------|
|
||||
| `read` | SSE streams, workstream listing, history | GET endpoints |
|
||||
| `write` | `read` + send, command, workstream create/close | POST to `/api/workstreams/{ws_id}/send`, `/api/command`, etc. |
|
||||
| `approve` | `write` + tool approval, admin operations | POST to `/api/workstreams/{ws_id}/approve`, `/api/admin/*` |
|
||||
| `write` | `read` + send, command, workstream create/close | POST to `/api/send`, `/api/command`, etc. |
|
||||
| `approve` | `write` + tool approval, admin operations | POST to `/api/approve`, `/api/admin/*` |
|
||||
|
||||
### Middleware Flow
|
||||
|
||||
@@ -1293,8 +1070,7 @@ Three hierarchical scopes control endpoint access:
|
||||
`/metrics`, `/openapi.json`, `/docs`, `/api/auth/*`, and `/api/auth/setup`
|
||||
are always allowed.
|
||||
2. **Token extraction** — `Authorization: Bearer <token>` header first, then
|
||||
surface-scoped auth cookie (`turnstone_auth_server` on the node server,
|
||||
`turnstone_auth_console` on the console) as fallback.
|
||||
`turnstone_auth` cookie as fallback.
|
||||
3. **Token type detection** — dots in the token indicate JWT; `ts_` prefix
|
||||
indicates API token.
|
||||
4. **Validation** — JWT signature check or API token hash lookup in storage.
|
||||
@@ -1308,9 +1084,8 @@ Three hierarchical scopes control endpoint access:
|
||||
- **Console** is the auth management hub — it hosts the admin endpoints for
|
||||
creating users, issuing API tokens, and managing channel mappings. User
|
||||
records and token hashes live in the shared storage backend. The console
|
||||
dashboard includes an **admin panel** (18 tabs) for managing
|
||||
credentials, governance, MCP servers, models, node metadata, and runtime
|
||||
settings through the browser.
|
||||
dashboard includes an **admin panel** (14 tabs) for managing
|
||||
credentials, governance, MCP servers, and runtime settings through the browser.
|
||||
- **Server** is a JWT validator only — it validates tokens on each request but
|
||||
never creates users or tokens. Both processes share the same `jwt_secret`
|
||||
(via `TURNSTONE_JWT_SECRET` env var or `[auth].jwt_secret` config).
|
||||
@@ -1381,18 +1156,19 @@ stderr so it does not interfere with readline. Tool execution may use a
|
||||
Starlette ASGI app (served by uvicorn)
|
||||
|
|
||||
+-- Async request handlers (all under /v1/ prefix)
|
||||
| POST /v1/api/workstreams/{ws_id}/send -> starts worker thread per workstream
|
||||
| POST /v1/api/workstreams/{ws_id}/approve -> unblocks WebUI._approval_event
|
||||
| POST /v1/api/workstreams/new -> creates workstream + worker
|
||||
| GET /v1/api/workstreams/{ws_id}/events -> SSE via EventSourceResponse (per workstream)
|
||||
| GET /v1/api/events/global -> SSE via EventSourceResponse (fan-out)
|
||||
| POST /v1/api/send -> starts worker thread per workstream
|
||||
| POST /v1/api/approve -> unblocks WebUI._approval_event
|
||||
| POST /v1/api/plan -> unblocks WebUI._plan_event
|
||||
| POST /v1/api/workstreams/new -> creates workstream + worker
|
||||
| GET /v1/api/events -> SSE via EventSourceResponse (per workstream)
|
||||
| GET /v1/api/events/global -> SSE via EventSourceResponse (fan-out)
|
||||
|
|
||||
+-- ASGI middleware stack
|
||||
| MetricsMiddleware -> CORSMiddleware -> AuthMiddleware -> RateLimitMiddleware
|
||||
|
|
||||
+-- Worker thread per workstream (daemon)
|
||||
| Runs session.send() synchronously -- ChatSession is fully blocking
|
||||
| Blocks on WebUI._approval_event (threading.Event)
|
||||
| Blocks on WebUI._approval_event / _plan_event (threading.Event)
|
||||
|
|
||||
+-- Background daemon threads
|
||||
Global SSE fan-out: reads global_queue, copies to per-client queues
|
||||
@@ -1416,7 +1192,7 @@ registry).
|
||||
|
||||
Each workstream's `WebUI` has:
|
||||
- `_listeners` (per-client SSE queues, fan-out on `_enqueue()`)
|
||||
- `_approval_event` (`threading.Event` for blocking)
|
||||
- `_approval_event` / `_plan_event` (`threading.Event` for blocking)
|
||||
- `_global_queue` (class variable, shared, for state broadcasts)
|
||||
|
||||
The SSE handlers bridge these sync queues to async via
|
||||
@@ -1459,10 +1235,10 @@ Monitoring (2 daemon threads) Control + Proxy (async Starlette)
|
||||
| SSE manager | | GET /node/{node_id}/ |
|
||||
| asyncio loop | | → httpx.AsyncClient |
|
||||
| 1 task per node | | proxy to server_url |
|
||||
| /events/global | | GET /node/{id}/v1/api/workstreams/{ws_id}/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/workstreams/{ws_id}/send |
|
||||
| → forwarded to server |
|
||||
| /events/global | | GET /node/{id}/v1/api/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/send |
|
||||
| → forwarded to server |
|
||||
+----------------------------+
|
||||
```
|
||||
|
||||
@@ -1544,10 +1320,9 @@ setup, auth headers, `_request()` (REST) and `_stream_sse()` (SSE). Sync
|
||||
clients delegate through `_SyncRunner` which maintains a persistent background
|
||||
event loop on a daemon thread.
|
||||
|
||||
**Event types**: 38 standalone dataclasses in `events.py` with a type-registry
|
||||
dispatch (`from_json()` on each event). Events are decoupled from server
|
||||
internals — the SDK parses SSE frames directly from the `/v1/api/events`
|
||||
streams.
|
||||
**Event types**: 27 standalone dataclasses in `events.py` with a type-registry
|
||||
pattern matching `OutboundEvent.from_json()` from `mq/protocol.py`. Events are
|
||||
decoupled from server internals.
|
||||
|
||||
**TypeScript SDK**: `sdk/typescript/` — separate npm package with the same API
|
||||
surface. Zero browser dependencies, SSE via `fetch` + `ReadableStream` parsing.
|
||||
@@ -1569,8 +1344,7 @@ with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
> See also: [Channel Integrations guide](channels.md)
|
||||
|
||||
The `turnstone-channel` gateway connects external messaging platforms
|
||||
(Discord and Slack today, with an adapter protocol for future platforms) to
|
||||
the turnstone cluster via HTTP. Each
|
||||
(Discord, Slack, Teams) to the turnstone cluster via HTTP. Each
|
||||
platform adapter implements the `ChannelAdapter` protocol and translates
|
||||
between platform-native events and turnstone server API calls.
|
||||
|
||||
@@ -1583,7 +1357,7 @@ workstream is reactivated, the router uses atomic resume via the
|
||||
the old workstream's conversation during creation in a single HTTP
|
||||
request, eliminating ordering fragility.
|
||||
|
||||
Discord and Slack adapters ship today. See [channels.md](channels.md) for
|
||||
Discord ships as the first adapter. See [channels.md](channels.md) for
|
||||
setup instructions, configuration reference, and the adapter development
|
||||
guide.
|
||||
|
||||
@@ -1604,11 +1378,11 @@ retries up to 3 times with backoff, re-querying the service registry on
|
||||
each attempt. See [Notification Flow diagram](diagrams/png/17-notify-flow.png).
|
||||
|
||||
**Bidirectional replies:** When a user replies to a notification DM, the
|
||||
channel adapter (Discord or Slack) looks up the originating `ws_id` from the
|
||||
tracked message ID, verifies the replying user matches the notification
|
||||
recipient, and routes the reply to the workstream via `router.send_message()`.
|
||||
The workstream's response is forwarded back to the DM via a temporary entry
|
||||
in `_notify_reply_channels`. On `TurnCompleteEvent`, the response message is
|
||||
Discord bot looks up the originating `ws_id` from the tracked message ID,
|
||||
verifies the replying user matches the notification recipient, and routes
|
||||
the reply to the workstream via `router.send_message()`. The workstream's
|
||||
response is forwarded back to the DM via a temporary entry in
|
||||
`_notify_reply_channels`. On `TurnCompleteEvent`, the response message is
|
||||
itself tracked for further replies, enabling multi-turn DM conversations
|
||||
without requiring the user to open the web UI. Tracking entries are capped
|
||||
at 100 (FIFO eviction) and cleaned up on workstream close.
|
||||
@@ -1641,10 +1415,11 @@ and workstreams record which skill and version spawned them. Token budget
|
||||
enforcement tracks consumption in `session.send()` with 80% warning and
|
||||
100% approval gate via the `__budget_override__` synthetic tool name.
|
||||
|
||||
The console admin panel exposes these capabilities as 18 permission-gated
|
||||
tabs: Users, API Tokens, Channels, Schedules, Watches, Roles, Policies,
|
||||
Prompts, Judge, Skills, MCP Servers, Usage, Audit, Memories, Models, Nodes,
|
||||
Settings, and TLS.
|
||||
The console admin panel adds 5 governance tabs (Roles, Policies, Skills,
|
||||
Usage, Audit), a Memories tab, a Settings tab (form-based editor for all
|
||||
ConfigStore settings), and an MCP Servers tab (database-backed server
|
||||
definitions with live connection status and cluster-wide reload) for a
|
||||
total of 13 tabs, all permission-gated.
|
||||
Both Python and TypeScript SDKs expose governance methods on the console
|
||||
client.
|
||||
|
||||
@@ -1672,7 +1447,8 @@ implemented in `turnstone/core/judge.py`:
|
||||
The judge is session-scoped (`IntentJudge`), lazy-initialized on first
|
||||
approval, and configured via the `[judge]` config section or `--judge` CLI
|
||||
flags. By default it uses self-consistency (same model), but supports
|
||||
cross-model and cross-provider configurations. Task sub-agents are exempt. All verdicts are persisted to the `intent_verdicts` table
|
||||
cross-model and cross-provider configurations. Sub-agents (plan, task)
|
||||
are exempt. All verdicts are persisted to the `intent_verdicts` table
|
||||
(migration 012) with the user's final decision, enabling future calibration.
|
||||
The console exposes `GET /v1/api/admin/verdicts` for audit queries
|
||||
(requires `admin.judge` permission).
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5d500479d3be2363d4f594042a27e2ef5e2974750f580f6c4037a1fe85868ed9
|
||||
size 251904
|
||||
oid sha256:75c1832b6079e8628f4bbf4ce98d37880c4de133636b7555e3869990b046ddc6
|
||||
size 567704
|
||||
|
||||
@@ -1,231 +0,0 @@
|
||||
# Bulk endpoint shape contract
|
||||
|
||||
Turnstone exposes several endpoints and tool calls that take multiple
|
||||
ids and return a per-id outcome. Over the last few phases two
|
||||
**distinct** response shapes have settled, one per semantic category.
|
||||
This doc codifies both so a future endpoint author can pick the right
|
||||
shape by semantics instead of by coin-flip.
|
||||
|
||||
Existing bulk endpoints at time of writing:
|
||||
|
||||
| Endpoint / tool | Category | Response shape |
|
||||
|---------------------------------------------------------|--------------------------|------------------------------------------|
|
||||
| `GET /v1/api/cluster/ws/live?ids=a,b,c` | bulk read | `{results, denied, truncated}` |
|
||||
| model tool `spawn_batch` | bulk create (per-item) | `{results, denied}` |
|
||||
| `POST /v1/api/workstreams/{ws_id}/close_all_children` | cascade mutation | `{closed, failed, skipped}` |
|
||||
|
||||
---
|
||||
|
||||
## Why two shapes
|
||||
|
||||
The ask-to-outcome mapping is fundamentally different between the
|
||||
two categories, and a one-size-fits-all envelope ends up papering
|
||||
over distinctions the caller genuinely needs to branch on.
|
||||
|
||||
**Bulk read / bulk create-with-payload.** Each input id (or batch
|
||||
index) carries a *request-side* concept — "give me the live block
|
||||
for this ws_id" or "spawn a child with this spec" — and each
|
||||
successful output carries a *payload* — the live block, or the new
|
||||
workstream's identifying triple. The interesting distinction on
|
||||
failure is *ownership / validation* (caller can't see that id, spec
|
||||
was malformed) — independent of the storage state.
|
||||
|
||||
**Cascade mutation.** The action is uniform across every id (cancel
|
||||
this subtree, close this child). The interesting distinctions on
|
||||
outcome are *did it reach the terminal state?* (succeeded / already
|
||||
was there / the dispatch itself failed) — driven by the storage
|
||||
state plus transport reliability, not by the caller's input.
|
||||
|
||||
Trying to unify these forces either:
|
||||
|
||||
- a stateless `denied` bucket that has to carry "already gone"
|
||||
*and* "you don't have permission" *and* "transport failed" with a
|
||||
separate reason string — reviewers end up string-matching to branch.
|
||||
- or a per-item-payload map for cascade mutations where every
|
||||
successful value is the same sentinel — carrier with no payload.
|
||||
|
||||
So: two shapes, one per category. The rest of this doc spells out
|
||||
each.
|
||||
|
||||
---
|
||||
|
||||
## Shape A — bulk read / bulk create-with-payload
|
||||
|
||||
```json
|
||||
{
|
||||
"results": { "<key>": <value-or-null>, ... },
|
||||
"denied": [ "<key>", ... ],
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
**`results`** is a key-indexed map of the positive-path payload.
|
||||
The key is the input id for read endpoints (`cluster/ws/live` uses
|
||||
the ws_id), or the input-array index (stringified) for create
|
||||
endpoints that want ordering preserved (`spawn_batch` uses `"0"`,
|
||||
`"1"`, ...). The value is whatever the endpoint produces per
|
||||
success — a live block, a `{ws_id, name, node_id, status}` triple,
|
||||
etc. A `null` value (read endpoints only) means "the id existed and
|
||||
you own it, but the live block wasn't available" — distinct from
|
||||
"denied".
|
||||
|
||||
**`denied`** is the negative-path list. For read endpoints it's a
|
||||
flat list of ids (preserves input order so callers can re-zip
|
||||
against their input). For create endpoints with per-item payloads
|
||||
it's a list of `{idx, reason}` objects (`spawn_batch`'s validation
|
||||
and spawn-error rows; also the operator-reject surface when per-item
|
||||
selective-deny ships). Include every reason that's *not* the
|
||||
positive path — authz, ownership, validation, already-consumed,
|
||||
spawn failure — so callers don't branch on status codes.
|
||||
|
||||
**`truncated`** is a boolean set to `true` when the server's
|
||||
per-endpoint input cap was exceeded and the tail was dropped. The
|
||||
endpoint docs each spell out the cap (50 for `cluster/ws/live`).
|
||||
`spawn_batch` hard-errors on overflow instead of silently
|
||||
truncating — it omits the field entirely rather than carry a
|
||||
permanently-false flag.
|
||||
|
||||
### Example — `cluster/ws/live`
|
||||
|
||||
```http
|
||||
GET /v1/api/cluster/ws/live?ids=a1b2,c3d4,nonexistent,foreign HTTP/1.1
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"results": {
|
||||
"a1b2": {"state": "running", "tokens": 12843, "activity": "..."},
|
||||
"c3d4": null
|
||||
},
|
||||
"denied": ["nonexistent", "foreign"],
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
Callers that need ordered output zip their original id list against
|
||||
this map; ids in `denied` drop out of the zip cleanly. A live-block
|
||||
`null` doesn't route to `denied` — the row exists and the caller
|
||||
owns it; the node is just currently unreachable.
|
||||
|
||||
### Example — `spawn_batch`
|
||||
|
||||
This is the coordinator-tool result shape (the JSON the LLM receives),
|
||||
not an HTTP API response — the table above keys it under "model tool"
|
||||
to distinguish it from the `/v1/api/...` endpoints in the same table.
|
||||
The underlying HTTP spawn endpoint still returns `ws_id`; the tool
|
||||
result re-keys it to `child_ws_id` to defuse a coordinator-LLM recency
|
||||
bias (see `docs/coordinator-skills.md`).
|
||||
|
||||
```json
|
||||
{
|
||||
"results": {
|
||||
"0": {"child_ws_id": "d4e5f6...", "name": "csrf-audit", "node_id": "gpu-3"},
|
||||
"2": {"child_ws_id": "f1a2b3...", "name": "xss-audit", "node_id": "gpu-1"}
|
||||
},
|
||||
"denied": [
|
||||
{"idx": 1, "reason": "skill not found: nonexistent-skill"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Indexes are stringified to keep the envelope JSON-safe and
|
||||
consistently-typed across the read and create cases.
|
||||
|
||||
---
|
||||
|
||||
## Shape B — cascade mutation
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"<bucket>": [ "<ws_id>", ... ],
|
||||
"failed": [ "<ws_id>", ... ],
|
||||
"skipped": [ "<ws_id>", ... ]
|
||||
}
|
||||
```
|
||||
|
||||
Where `<bucket>` is the endpoint-specific name for "succeeded" —
|
||||
`closed` for `close_all_children`.
|
||||
The three buckets partition the input set exactly once:
|
||||
|
||||
| Bucket | Meaning |
|
||||
|---------------|-------------------------------------------------------------------------------|
|
||||
| `<bucket>` | Action dispatch accepted; target reached the intended terminal state. |
|
||||
| `failed` | Dispatch returned a non-404 error (transport issue, upstream 5xx, exception). |
|
||||
| `skipped` | Upstream 404 — stale registry entry, row already deleted, or peer gone. |
|
||||
|
||||
The split between `failed` and `skipped` is load-bearing. `failed`
|
||||
is actionable — the operator may want to retry, or the cascade may
|
||||
be partial. `skipped` is pre-resolved — the target is already in
|
||||
the terminal state the cascade was aiming at, so it's neither a
|
||||
win to report nor a fault to fix.
|
||||
|
||||
### Example — `close_all_children`
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"closed": ["child-1", "child-3"],
|
||||
"failed": ["child-2"],
|
||||
"skipped": []
|
||||
}
|
||||
```
|
||||
|
||||
Here the success bucket is `closed`. A subsequent retry would
|
||||
target only `failed` ids, not `skipped` ones — the latter are
|
||||
already done. When `coord_client` is unavailable (session loaded
|
||||
but no HTTP client attached — a construction bug) every id goes to
|
||||
`failed` so the operator notices rather than getting a silent
|
||||
all-skipped response.
|
||||
|
||||
---
|
||||
|
||||
## Guidance for future bulk endpoints
|
||||
|
||||
1. **Pick by semantics, not by "what shape is nearby."**
|
||||
- Mutation that's uniform across ids + terminal-state outcome? →
|
||||
**Shape B** (cascade mutation).
|
||||
- Read or create where the input id carries payload, or where the
|
||||
denial axis is independent of storage state? → **Shape A**
|
||||
(bulk read / bulk create-with-payload).
|
||||
|
||||
2. **Cap the input.** Both shapes assume a bounded input — the
|
||||
server rejects or silently truncates past the cap. Document the
|
||||
cap in the endpoint's OpenAPI description. Shape A uses
|
||||
`truncated: true` on quiet truncation; Shape B hard-errors on
|
||||
overflow.
|
||||
|
||||
3. **Match existing bucket names for the same semantic.** Use
|
||||
`failed` and `skipped` verbatim in Shape B — the per-endpoint
|
||||
success bucket is the only slot that varies. Use `results` and
|
||||
`denied` verbatim in Shape A; the per-endpoint `<key>` /
|
||||
`<value>` types vary.
|
||||
|
||||
4. **Audit the verbose shape.** Both endpoints emit a corresponding
|
||||
audit event with the full before/after bucket lists — the SSE
|
||||
stream and the in-process response give live feedback, but a
|
||||
postmortem operator will read the audit row. Use
|
||||
`_emit_coord_audit` (coordinator-scoped) or `record_audit`
|
||||
directly; don't inline.
|
||||
|
||||
5. **Don't mix shapes within one endpoint.** If a bulk endpoint
|
||||
wants both partial-success creation AND per-item failure reasons
|
||||
(like `spawn_batch` with its `{idx, reason}` denial rows), that's
|
||||
Shape A with a richer denial element — not a blend with Shape B.
|
||||
|
||||
---
|
||||
|
||||
## History
|
||||
|
||||
- **Phase 6** shipped `cluster/ws/live` as the first Shape A endpoint
|
||||
(`{results, denied, truncated}`).
|
||||
- **Phase 7** introduced the Shape B cascade-mutation envelope
|
||||
(`{<bucket>, failed, skipped}`) for the coordinator's
|
||||
cancel-cascade path.
|
||||
- **Phase 8 PR A** shipped `spawn_batch` (Shape A, keyed by idx) and
|
||||
`close_all_children` (Shape B), which crystallised the
|
||||
two-shape-per-semantic-category policy codified here.
|
||||
|
||||
Before adding a third shape, read this doc and argue for why the
|
||||
new surface doesn't fit either A or B. Two idioms in the cluster
|
||||
API is a finite operator tax; three is one too many.
|
||||
+32
-96
@@ -7,35 +7,31 @@ platform-native events (messages, button clicks, slash commands) into
|
||||
turnstone API calls, and renders workstream output back into the
|
||||
platform's UI.
|
||||
|
||||
Discord and Slack adapters ship today. The adapter protocol is designed
|
||||
so new platforms can be added with only a new package under
|
||||
`turnstone/channels/<platform>/`.
|
||||
Discord ships as the first adapter. The adapter protocol is designed for
|
||||
future Slack and Teams integrations.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Discord Gateway Slack (Socket Mode WebSocket)
|
||||
\ /
|
||||
v v
|
||||
turnstone-channel (one or more adapters)
|
||||
|
|
||||
v
|
||||
turnstone-server (direct HTTP)
|
||||
or
|
||||
turnstone-console (routing proxy, multi-node)
|
||||
Discord Gateway
|
||||
|
|
||||
v
|
||||
turnstone-channel (Discord adapter)
|
||||
|
|
||||
v
|
||||
turnstone-server (direct HTTP)
|
||||
or
|
||||
turnstone-console (routing proxy, multi-node)
|
||||
```
|
||||
|
||||
A single `turnstone-channel` process can run multiple adapters
|
||||
simultaneously (e.g. Discord + Slack) — pass the tokens for each
|
||||
platform you want to enable.
|
||||
|
||||
Key components:
|
||||
|
||||
- **ChannelAdapter protocol** (`turnstone/channels/_protocol.py`) — generic
|
||||
interface for any messaging platform. Defines `start()`, `stop()`,
|
||||
`send()`, and `send_notification()`.
|
||||
`send()`, `send_notification()`, `edit_message()`,
|
||||
`send_approval_request()`, `send_plan_review()`, and `create_thread()`.
|
||||
- **ChannelRouter** (`turnstone/channels/_routing.py`) — maps
|
||||
channel/thread IDs to turnstone workstream IDs. Handles workstream
|
||||
creation via HTTP, stale route detection, and user identity resolution.
|
||||
@@ -99,10 +95,10 @@ TURNSTONE_DISCORD_GUILD=123456789
|
||||
Then start the stack:
|
||||
|
||||
```bash
|
||||
docker compose up
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
The `channel` gateway runs by default; the Discord adapter activates once
|
||||
The `channel` service starts automatically when
|
||||
`TURNSTONE_DISCORD_TOKEN` is set.
|
||||
|
||||
### 3. Link User Accounts
|
||||
@@ -124,67 +120,6 @@ An admin can also force-link or unlink users via the console admin panel
|
||||
|
||||
---
|
||||
|
||||
## Slack Setup
|
||||
|
||||
Slack uses **Socket Mode**, so no public URL or API Gateway is required — Slack
|
||||
connects outbound to the bot via a WebSocket. Install with:
|
||||
|
||||
```bash
|
||||
pip install 'turnstone[slack]'
|
||||
```
|
||||
|
||||
### 1. Create a Slack App
|
||||
|
||||
1. Go to https://api.slack.com/apps and click **Create New App**
|
||||
2. Under **Settings > Socket Mode**, enable Socket Mode. This generates an
|
||||
**App-Level Token** (prefix `xapp-`) — copy it.
|
||||
3. Under **OAuth & Permissions**, add these **Bot Token Scopes**:
|
||||
`chat:write`, `chat:write.public`, `channels:history`, `im:history`,
|
||||
`groups:history`, `mpim:history`, `reactions:write`, `commands`
|
||||
4. Under **Event Subscriptions** (Socket Mode delivers events), subscribe
|
||||
to bot events: `message.channels`, `message.im`, `message.groups`
|
||||
5. Under **Slash Commands**, create a command (default `/turnstone`)
|
||||
6. Install the app to your workspace to generate the **Bot User OAuth
|
||||
Token** (prefix `xoxb-`).
|
||||
|
||||
### 2. Configure Turnstone
|
||||
|
||||
**Environment variables** (recommended for Docker):
|
||||
|
||||
```bash
|
||||
TURNSTONE_SLACK_TOKEN=xoxb-... # Bot User OAuth Token
|
||||
TURNSTONE_SLACK_APP_TOKEN=xapp-... # App-Level Token (Socket Mode)
|
||||
TURNSTONE_SLACK_CHANNELS= # optional, comma-separated channel IDs
|
||||
TURNSTONE_SLACK_SLASH_COMMAND=/turnstone
|
||||
```
|
||||
|
||||
**CLI flags** (bare-metal):
|
||||
|
||||
```bash
|
||||
turnstone-channel \
|
||||
--slack-token "xoxb-..." \
|
||||
--slack-app-token "xapp-..." \
|
||||
--slack-slash-command /turnstone \
|
||||
--server-url http://localhost:8080
|
||||
```
|
||||
|
||||
The Slack and Discord adapters can be enabled together — pass tokens for
|
||||
both and the gateway hosts both adapters in one process.
|
||||
|
||||
### 3. Usage
|
||||
|
||||
- **DM the bot**: messages sent directly to the bot create a workstream
|
||||
scoped to that DM; the slash command is not required.
|
||||
- **Slash command**: `/turnstone <message>` in any channel the bot can
|
||||
see starts a per-user channel session.
|
||||
- Tool approvals render as Slack **Block Kit** buttons; only the user
|
||||
who owns the workstream can approve/reject.
|
||||
- Notifications and reply routing work identically to Discord.
|
||||
- Session recovery: persisted channel routes are re-subscribed when the
|
||||
bot restarts, so existing Slack conversations keep flowing.
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
### Conversations
|
||||
@@ -234,19 +169,24 @@ config, the bot auto-responds with approval and posts a
|
||||
field (useful for allowing specific tools like `bash` or `read_file` while
|
||||
still requiring manual approval for others).
|
||||
|
||||
### Plan Reviews
|
||||
|
||||
Plan review requests are displayed as a blue embed with:
|
||||
|
||||
- **Approve Plan** (green) button — approves the plan with empty feedback
|
||||
- **Request Changes** (gray) button — opens a modal for feedback text
|
||||
(up to 2000 characters)
|
||||
- Feedback is forwarded to the server via HTTP
|
||||
|
||||
---
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
| CLI Flag | Env Var | Default | Description |
|
||||
|----------|---------|---------|-------------|
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord) |
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Bot token (required to enable Discord) |
|
||||
| `--discord-guild` | — | `0` (all guilds) | Restrict to a single Discord guild |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated Discord channel IDs to allow |
|
||||
| `--slack-token` | `TURNSTONE_SLACK_TOKEN` | — | Slack Bot User OAuth token (`xoxb-…`, required to enable Slack) |
|
||||
| `--slack-app-token` | `TURNSTONE_SLACK_APP_TOKEN` | — | Slack App-Level token (`xapp-…`, required with `--slack-token`) |
|
||||
| `--slack-channels` | `TURNSTONE_SLACK_CHANNELS` | empty (all) | Comma-separated Slack channel IDs to allow |
|
||||
| `--slack-slash-command` | `TURNSTONE_SLACK_SLASH_COMMAND` | `/turnstone` | Slash command name registered in the Slack app |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated channel IDs to allow |
|
||||
| `--server-url` | `TURNSTONE_SERVER_URL` | `http://localhost:8080` | Server URL (single-node) |
|
||||
| `--console-url` | `TURNSTONE_CONSOLE_URL` | — | Console URL (multi-node routing proxy) |
|
||||
| `--model` | — | server default | Default model for new workstreams |
|
||||
@@ -256,9 +196,6 @@ still requiring manual approval for others).
|
||||
| `--log-level` | `TURNSTONE_LOG_LEVEL` | `INFO` | Log level |
|
||||
| `--log-format` | `TURNSTONE_LOG_FORMAT` | `auto` | Log format (`auto`/`json`/`text`) |
|
||||
|
||||
At least one of `--discord-token` or `--slack-token` must be supplied.
|
||||
Passing both starts both adapters in the same process.
|
||||
|
||||
---
|
||||
|
||||
## User Identity
|
||||
@@ -312,8 +249,8 @@ waiting for them to check in.
|
||||
Two modes:
|
||||
|
||||
- **Username** — provide a turnstone `username`. The gateway resolves
|
||||
it via the `channel_users` table and sends to every linked platform
|
||||
the user has (e.g. Discord + Slack).
|
||||
it via the `channel_users` table and sends to all linked channels
|
||||
(e.g. Discord + future Slack).
|
||||
- **Direct** — provide `channel_type` + `channel_id` to target a
|
||||
specific platform channel or user DM.
|
||||
|
||||
@@ -414,6 +351,10 @@ class ChannelAdapter(Protocol):
|
||||
async def stop(self) -> None: ...
|
||||
async def send(self, channel_id: str, content: str) -> str: ...
|
||||
async def send_notification(self, channel_id: str, content: str, ws_id: str) -> str: ...
|
||||
async def edit_message(self, channel_id: str, message_id: str, content: str) -> None: ...
|
||||
async def send_approval_request(self, channel_id: str, ws_id: str, correlation_id: str, items: list[dict]) -> None: ...
|
||||
async def send_plan_review(self, channel_id: str, ws_id: str, correlation_id: str, content: str) -> None: ...
|
||||
async def create_thread(self, parent_channel_id: str, name: str, message_id: str = "") -> str: ...
|
||||
```
|
||||
|
||||
`send_notification()` is like `send()` but associates the outgoing
|
||||
@@ -421,11 +362,6 @@ message with a `ws_id` so that user replies can be routed back to the
|
||||
originating workstream. Adapters must track the mapping from outgoing
|
||||
message ID to `(ws_id, target_user_id)` and handle DM replies.
|
||||
|
||||
Platform-specific concerns — approval prompts, message edits, thread
|
||||
creation — live inside the adapter implementation and are not part of
|
||||
the protocol surface. Each adapter drives those via its
|
||||
own `_on_ws_event` dispatcher using SDK-native APIs.
|
||||
|
||||
To add a new platform:
|
||||
|
||||
1. Create `turnstone/channels/<platform>/` package
|
||||
|
||||
+6
-16
@@ -334,7 +334,7 @@ The console reverse-proxies each node's server UI at `/node/{node_id}/`. This al
|
||||
|
||||
### URL Rewriting
|
||||
|
||||
The server UI uses root-relative URLs (`/v1/api/workstreams/{ws_id}/send`, `/static/app.js`, `/shared/base.css`, etc.). Since `<base>` tags cannot rewrite root-relative URLs, the console uses a JS shim approach:
|
||||
The server UI uses root-relative URLs (`/v1/api/send`, `/static/app.js`, `/shared/base.css`, etc.). Since `<base>` tags cannot rewrite root-relative URLs, the console uses a JS shim approach:
|
||||
|
||||
1. **HTML rewriting** — when serving `index.html`, replaces `href=` and `src=` references to both `/static/` and `/shared/` with the proxy prefix (`/node/{node_id}/static/` and `/node/{node_id}/shared/` respectively).
|
||||
|
||||
@@ -344,7 +344,7 @@ The server UI uses root-relative URLs (`/v1/api/workstreams/{ws_id}/send`, `/sta
|
||||
|
||||
### SSE Proxy
|
||||
|
||||
SSE streams (`/v1/api/workstreams/{ws_id}/events`, `/v1/api/events/global`) are proxied as raw byte passthrough — the console opens an `httpx.AsyncClient.stream()` to the upstream server (with `read=None` and `pool=None` timeouts since SSE connections are long-lived) and relays every byte via `StreamingResponse`. This preserves server-side ping comments, event framing, and keepalives verbatim without parsing or re-encoding.
|
||||
SSE streams (`/v1/api/events`, `/v1/api/events/global`) are proxied as raw byte passthrough — the console opens an `httpx.AsyncClient.stream()` to the upstream server (with `read=None` and `pool=None` timeouts since SSE connections are long-lived) and relays every byte via `StreamingResponse`. This preserves server-side ping comments, event framing, and keepalives verbatim without parsing or re-encoding.
|
||||
|
||||
### Authentication
|
||||
|
||||
@@ -379,7 +379,6 @@ Breadcrumb: `Cluster > Running` or `Cluster > db-west-04`. Server-side paginated
|
||||
Triggered by the "+ new" header button. A modal dialog with:
|
||||
|
||||
- **Node selector** — dropdown with three targeting modes: "Auto (best available)" picks the node with the most headroom, "General pool (any node)" picks a node with available capacity using round-robin, or a specific node from the list (showing capacity).
|
||||
- **Persona** — optional dropdown listing the enabled personas for the workstream kind. Sets the system-message composition and capability envelope at creation, snapshotted server-side; empty uses the kind's default. Picking one requires no `persona.*` permission.
|
||||
- **Profile** — optional dropdown listing enabled skills. Applies the skill's model, auto-approve policy, token budget, and other behavioral settings at creation time.
|
||||
- **Name** — optional text input. Auto-generated if left empty.
|
||||
- **Model** — optional text input for a model alias from the target node's registry.
|
||||
@@ -397,19 +396,10 @@ The browser maintains a local `clusterState` object that mirrors the cluster sna
|
||||
|
||||
Accessed via the "admin" button in the header (visible when authenticated
|
||||
with `approve` scope). Provides user, API token, channel link, MCP server,
|
||||
and skill management with tabs that include Users, API Tokens, Channels,
|
||||
Schedules, Watches, Personas, Roles, Policies, Prompts, Judge, Skills,
|
||||
MCP Servers, Usage, Audit, Memories, Models, Nodes, Settings, and TLS. See also
|
||||
[Governance](governance.md) for the Roles, Policies, Skills, Usage, and
|
||||
Audit tabs, and [Settings](settings.md) for the database-backed
|
||||
configuration editor.
|
||||
|
||||
The **Channels** tab links users to either a Discord or Slack account
|
||||
via a per-row channel-type selector. The **Models** tab is a CRUD
|
||||
editor for `model_definitions`, the **Nodes** tab edits per-node
|
||||
metadata, and the **TLS** tab manages CA and leaf certificates for the
|
||||
internal mTLS fabric. The **Settings** tab edits ConfigStore values
|
||||
live; edits apply without restart.
|
||||
and skill management with 13 tabs (see also
|
||||
[Governance](governance.md) for
|
||||
the Roles, Policies, Skills, Usage, and Audit tabs, and
|
||||
[Settings](settings.md) for the database-backed configuration editor):
|
||||
|
||||
**Users tab:**
|
||||
|
||||
|
||||
@@ -1,380 +0,0 @@
|
||||
# Coordinator API tour
|
||||
|
||||
Turnstone's **coordinator workstream** is a session hosted on the
|
||||
console whose job is to orchestrate other workstreams. It runs an LLM
|
||||
that can spawn child workstreams on any node, watch their progress,
|
||||
wait for them to finish, steer them mid-flight, and tear them down.
|
||||
This doc walks the full lifecycle — one request, one response, and the
|
||||
relevant SSE events at each step.
|
||||
|
||||
Aimed at integrators driving a coordinator from a custom UI or SDK
|
||||
without reverse-engineering the built-in console page. The shapes
|
||||
here match the live OpenAPI spec served at `/openapi.json` and
|
||||
rendered at `/docs` on every `turnstone-console` process. Every
|
||||
step references the operation id from that spec so doc updates track
|
||||
schema changes.
|
||||
|
||||
> **Auth throughout.** Every endpoint below sits behind bearer-token
|
||||
> auth and the `admin.coordinator` permission. A session-scoped JWT
|
||||
> is minted per login (see [docs/oidc.md](oidc.md) / [docs/security.md](security.md));
|
||||
> a service token may call the read paths but destructive governance
|
||||
> paths (`/restrict`, `/close_all_children`) require
|
||||
> the explicit `admin.coordinator` grant — a service-token owner
|
||||
> match isn't enough.
|
||||
|
||||
---
|
||||
|
||||
## The 9 steps
|
||||
|
||||
> **URL convergence (1.5.0).** Pre-1.5 coord-only endpoints lived
|
||||
> under `/v1/api/coordinator/...`. The Stage 2 verb-shape lift
|
||||
> consolidated coord and interactive onto the unified
|
||||
> `/v1/api/workstreams/{ws_id}/<verb>` tree; coord still distinguishes
|
||||
> itself via the `kind=coordinator` row classifier rather than a
|
||||
> separate URL space. The endpoints below reflect the post-lift
|
||||
> surface served by `turnstone-console`.
|
||||
|
||||
| # | Action | Operation |
|
||||
|---|------------------------------|-------------------------------------------------------------|
|
||||
| 1 | Create | `POST /v1/api/workstreams/new` |
|
||||
| 2 | Subscribe to events | `GET /v1/api/workstreams/{ws_id}/events` (SSE) |
|
||||
| 3 | Send a user message | `POST /v1/api/workstreams/{ws_id}/send` |
|
||||
| 4 | Inspect children | `GET /v1/api/workstreams/{ws_id}/children` |
|
||||
| 5 | Inspect one workstream | `GET /v1/api/cluster/ws/{ws_id}/detail` |
|
||||
| 6 | Wait for fan-out | model-side tool `wait_for_workstream` |
|
||||
| 7 | Govern | `POST /v1/api/workstreams/{ws_id}/trust` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/restrict` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/close_all_children` |
|
||||
| 8 | Approve / cancel | `POST /v1/api/workstreams/{ws_id}/approve` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/cancel` |
|
||||
| 9 | Close | `POST /v1/api/workstreams/{ws_id}/close` |
|
||||
|
||||
Refer to `/openapi.json` (Swagger UI at `/docs`) on any
|
||||
`turnstone-console` process for the authoritative operation ids and
|
||||
schemas. Coordinator-only verbs (`/children`, `/trust`, `/restrict`,
|
||||
`/close_all_children`) 404 against `kind=interactive`
|
||||
rows; the shared verbs (`/send`, `/approve`, `/cancel`, `/events`,
|
||||
`/history`, `/open`, `/close`, etc.) work on both kinds.
|
||||
|
||||
---
|
||||
|
||||
## 1. Create a coordinator
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/new
|
||||
Content-Type: application/json
|
||||
Authorization: Bearer <token>
|
||||
|
||||
{
|
||||
"name": "release-coord",
|
||||
"skill": "engineer-orchestrator",
|
||||
"initial_message": "audit /auth for CSRF handling across all active routes"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 201 Created
|
||||
Content-Type: application/json
|
||||
|
||||
{"ws_id": "a1b2c3d4e5f6...", "name": "release-coord"}
|
||||
```
|
||||
|
||||
All three body fields are optional — an empty body still creates a
|
||||
coordinator with an auto-generated name and no initial message.
|
||||
Returns **503** with a remediation message when the cluster isn't
|
||||
configured with a coordinator model; see
|
||||
[`coordinator.model_alias`](settings.md) to set one.
|
||||
|
||||
**SSE implication:** the `ws_created` event fires on the cluster-wide
|
||||
stream (`/v1/api/cluster/events`) once the row is committed. Per-ws
|
||||
subscribers (step 2) see the session warm up as token traffic starts.
|
||||
|
||||
---
|
||||
|
||||
## 2. Subscribe to the per-coordinator event stream
|
||||
|
||||
```http
|
||||
GET /v1/api/workstreams/{ws_id}/events HTTP/1.1
|
||||
Accept: text/event-stream
|
||||
Authorization: Bearer <token>
|
||||
```
|
||||
|
||||
One persistent SSE connection per browser tab / SDK caller — the
|
||||
console fans each event out to every listener queue (cap 500 events
|
||||
per queue, put_nowait drop on overflow). Events come in flat JSON
|
||||
with a `type` field. The recurring shapes a UI has to handle:
|
||||
|
||||
| `type` | Emitted when | Payload highlights |
|
||||
|---------------------|--------------------------------------------------------------------------------------------|--------------------|
|
||||
| `thinking_start` / `thinking_stop` | Model has entered / exited a reasoning block | — |
|
||||
| `reasoning` | Reasoning-token stream chunk (when the model exposes it) | `text` |
|
||||
| `content` | Assistant-content stream chunk | `text` |
|
||||
| `stream_end` | End of a single provider stream | — |
|
||||
| `tool_result` | A tool call completed (success or error) | `call_id`, `name`, `output`, `is_error?` |
|
||||
| `tool_output_chunk` | Streaming tool output (e.g. long bash command) | `call_id`, `chunk` |
|
||||
| `approve_request` | One or more tool calls need operator approval | `items: [{call_id, header, preview, func_name, approval_label, needs_approval}]` |
|
||||
| `approval_resolved` | Operator answered the approval prompt | `approved`, `feedback` |
|
||||
| `state_change` | Worker-thread state transition (also re-emitted with the current state on every fresh subscribe so refresh-mid-stream restores composer mode) | `state` ∈ `running`, `thinking`, `attention`, `idle`, `error` |
|
||||
| `in_progress_snapshot` | One-shot replay of the in-progress turn's content + reasoning when this client connects mid-stream | `content`, `reasoning` |
|
||||
| `status` | Token usage + context-window snapshot (fires on every streaming tick) | `prompt_tokens`, `completion_tokens`, `total_tokens`, `context_window`, `pct`, `effort`, `cache_creation_tokens`, `cache_read_tokens` |
|
||||
| `rename` | Session's display name changed | `name` |
|
||||
| `intent_verdict` | Intent judge produced a verdict on a pending tool call | `risk_level`, `recommendation`, `reasons` |
|
||||
| `output_warning` | Output guard flagged a tool result | `call_id`, `risk_level`, `flags` |
|
||||
| `child_ws_created` | A direct child of this coord was just created (fan-out from the cluster bus) | `child_ws_id`, `node_id`, `name`, `parent_ws_id` (`ws_id` in the envelope is always the coord's own id) |
|
||||
| `child_ws_state` | A direct child transitioned state | `child_ws_id`, `state` |
|
||||
| `child_ws_closed` | A direct child closed | `child_ws_id` |
|
||||
| `child_ws_rename` | A direct child's name changed | `child_ws_id`, `name` |
|
||||
| `wait_started` / `wait_progress` / `wait_ended` | `wait_for_workstream` tool lifecycle (see §6) | `call_id`, `ws_ids`, `elapsed`, `results`, `complete` |
|
||||
| `batch_started` / `batch_ended` | `spawn_batch` / `close_all_children` tool lifecycle | `call_id`, `op`, `total`/`succeeded`/`denied`/`closed`/`failed`/`skipped` |
|
||||
| `info` / `error` | Operational messages | `message` |
|
||||
|
||||
**Reconnection contract:** a freshly-opened SSE connection receives
|
||||
the current snapshot of any pending tool approval (`approve_request`
|
||||
is re-sent if unresolved), any in-flight `wait_*` / `batch_*`
|
||||
indicator, the worker's current `state_change`, and an
|
||||
`in_progress_snapshot` carrying any partial content / reasoning the
|
||||
model has produced for the in-progress turn — so a tab refresh
|
||||
mid-approval, mid-tool-execution, or mid-stream restores both the
|
||||
correct composer mode and the partial assistant text without waiting
|
||||
for the response to complete.
|
||||
|
||||
---
|
||||
|
||||
## 3. Send the first user message
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/send
|
||||
Content-Type: application/json
|
||||
|
||||
{"message": "audit /auth for CSRF handling across all active routes"}
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 200 OK
|
||||
{"status": "ok"}
|
||||
```
|
||||
|
||||
The message is queued for the worker thread at its next tool-result
|
||||
seam (so you can send follow-ups mid-conversation without corrupting
|
||||
the in-progress turn). On the SSE stream you'll see `state_change`
|
||||
→ `thinking_start` → streaming `reasoning` / `content` / `tool_result`
|
||||
events, finishing with `state_change → idle` or an
|
||||
`approve_request` when the model invokes a gated tool.
|
||||
|
||||
---
|
||||
|
||||
## 4. Inspect direct children
|
||||
|
||||
```http
|
||||
GET /v1/api/workstreams/{ws_id}/children HTTP/1.1
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{"ws_id": "d4e5f6...", "name": "csrf-audit", "state": "running", "node_id": "gpu-3"},
|
||||
{"ws_id": "e1f2a3...", "name": "xss-audit", "state": "idle", "node_id": "gpu-1"}
|
||||
],
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
The response key is `items`, not `children` — the endpoint shape
|
||||
follows the cluster-wide workstream-list idiom rather than the
|
||||
coordinator `list_workstreams` tool's (which uses `children`).
|
||||
Rows include every state stored for the parent (`running`, `idle`,
|
||||
`closed`, ...); the endpoint does not accept a state query param,
|
||||
so clients should inspect each row's `state` field and filter
|
||||
locally if they want to hide closed/deleted children. Nested
|
||||
coordinator rows are dropped server-side so only interactive
|
||||
descendants appear.
|
||||
|
||||
---
|
||||
|
||||
## 5. Inspect one workstream (storage + live block + tail)
|
||||
|
||||
```http
|
||||
GET /v1/api/cluster/ws/{ws_id}/detail?message_limit=20 HTTP/1.1
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"persisted": { "ws_id": "...", "state": "running", "parent_ws_id": "...", "kind": "interactive", ... },
|
||||
"live": { "state": "thinking", "tokens": 12843, "activity": "...", "pending_approval": null },
|
||||
"tail": [ {"role": "assistant", "content": "...", "tokens": 128}, ... ]
|
||||
}
|
||||
```
|
||||
|
||||
Works for any workstream the caller has `admin.cluster.inspect` on,
|
||||
not just children of a single coordinator — useful for a cluster
|
||||
admin panel watching multiple coordinators at once. `live` is
|
||||
`null` when the owning node is unreachable or has dropped the row
|
||||
from its dashboard cache; callers should degrade gracefully, not
|
||||
treat it as an error.
|
||||
|
||||
For fan-out views, prefer
|
||||
[`GET /v1/api/cluster/ws/live?ids=a,b,c`](bulk-endpoints.md) — it
|
||||
collapses N per-row round-trips into one, returning the live block
|
||||
for every id in a `{results, denied, truncated}` envelope.
|
||||
|
||||
---
|
||||
|
||||
## 6. Wait for fan-out (`wait_for_workstream`)
|
||||
|
||||
`wait_for_workstream` is a **model-side tool**, not an HTTP endpoint
|
||||
— the coordinator's LLM invokes it with a list of child ws_ids, the
|
||||
session's worker thread blocks inside the tool, and a sequence of
|
||||
`wait_started` / `wait_progress` / `wait_ended` SSE events is emitted
|
||||
for the UI to drive a "waiting on N children" indicator.
|
||||
|
||||

|
||||
|
||||
Key properties:
|
||||
|
||||
- **Caps** — up to 32 ws_ids per call, up to 600 seconds per call.
|
||||
A coordinator that needs to wait on more children re-invokes the
|
||||
tool with a fresh timeout.
|
||||
- **Modes** — `mode="any"` returns as soon as one child reaches a
|
||||
real terminal state (`idle` / `error` / `closed` / `deleted`);
|
||||
`mode="all"` waits for every polled child to reach a real
|
||||
terminal state.
|
||||
- **Progress throttling** — the poll loop runs every 500 ms but the
|
||||
SSE emission is diff-on-state-change plus a 5-second heartbeat. A
|
||||
600 s wait generates O(dozens) of progress events, not 1200.
|
||||
- **Unresolvable ids** — ws_ids are validated up front (exactly
|
||||
32 hex chars; copy them verbatim): a malformed id fails the call
|
||||
immediately with did-you-mean suggestions and a roster of the
|
||||
coord's children. An id the caller doesn't own, a missing row, or
|
||||
a child hard-deleted mid-wait is reported as `state="not_found"`
|
||||
and aborts the wait on the tick that observes it (top-level
|
||||
`error` / `not_found` / `children` fields, `complete=false`) — the
|
||||
LLM should fix the id and re-issue, not conclude the child died.
|
||||
Foreign and missing collapse into one shape, so the wait can't be
|
||||
used as an existence oracle.
|
||||
|
||||
Prefer `wait_for_workstream` over polling `inspect_workstream` in a
|
||||
loop — a wait consumes one assistant turn regardless of how long the
|
||||
children take, whereas each `inspect_workstream` poll costs a full
|
||||
turn (plus judge, plus tokens). On a fan-out of 3+ children this
|
||||
rounds to a 10× token-efficiency win.
|
||||
|
||||
---
|
||||
|
||||
## 7. Governance — trust, restrict, close_all_children
|
||||
|
||||
These three endpoints let an operator steer a live coordinator session
|
||||
mid-flight. All three emit an audit event tagged
|
||||
`coordinator.<action>` via the dedicated audit executor so a cascade
|
||||
burst can't starve audit writes.
|
||||
|
||||
### `POST /trust` — auto-approve own-subtree sends
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/trust
|
||||
{"send": true}
|
||||
```
|
||||
|
||||
Flips `trust_send=true` on the live session. Subsequent
|
||||
`send_to_workstream` calls that target a ws_id in the coordinator's
|
||||
own subtree skip the approval prompt; foreign ws_ids and other tool
|
||||
calls still go through the normal flow. Requires both
|
||||
`admin.coordinator` AND `coordinator.trust.send` permissions (the
|
||||
second grants a service token the opt-in it otherwise wouldn't get).
|
||||
|
||||
### `POST /restrict` — revoke tool access mid-session
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/restrict
|
||||
{"revoke": ["spawn_workstream", "delete_workstream"]}
|
||||
```
|
||||
|
||||
Unions the names into the session's revoked-tools set. Additive and
|
||||
idempotent — calling twice with overlapping lists converges to the
|
||||
union. Revocations don't survive a session close/reopen; operators
|
||||
opt in per session. Cap 256 tool names per request, 128 chars each.
|
||||
|
||||
### `POST /close_all_children` — soft-close the direct fan-out
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/close_all_children
|
||||
{"reason": "audit round complete"}
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{"status": "ok", "closed": ["c-1", "c-2"], "failed": [], "skipped": []}
|
||||
```
|
||||
|
||||
Soft-close cascade bounded by a concurrency semaphore. The `reason`
|
||||
(up to 512 chars) propagates into each closed child's audit +
|
||||
`workstream_config` for postmortem. The model-facing tool that
|
||||
pairs with this endpoint asks for a bounded teardown of the
|
||||
coordinator's own fan-out. This *soft-closes*; to *cancel* the
|
||||
fan-out instead, cancel the coordinator (§8) — a coordinator cancel
|
||||
auto-cascades to its direct children.
|
||||
|
||||
See [bulk-endpoints.md](bulk-endpoints.md) for why `close_all_children`
|
||||
uses the cascade-mutation shape and how it differs from the
|
||||
`spawn_batch` / `cluster/ws/live` shape.
|
||||
|
||||
---
|
||||
|
||||
## 8. Approve / cancel
|
||||
|
||||
The `approve` endpoint is what resolves an `approve_request` SSE
|
||||
event. The coordinator's worker thread is blocked inside
|
||||
`ui.approve_tools` waiting for this POST.
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/approve
|
||||
{"approved": true, "feedback": null, "always": false}
|
||||
{"approved": false, "feedback": "spawn count looks too high — try 3 not 10"}
|
||||
{"approved": true, "feedback": null, "always": true} // always-approve this tool name
|
||||
```
|
||||
|
||||
`cancel` drops the coordinator's in-flight generation and, for a
|
||||
coordinator, auto-cascades the cancel to its direct children:
|
||||
`cancel_workstream` is dispatched through the routing proxy for
|
||||
every direct child in the registry. The coordinator itself is left
|
||||
idle and open for a fresh `send`:
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/cancel
|
||||
{}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Close
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/close
|
||||
{}
|
||||
```
|
||||
|
||||
Soft-closes the session — state persists, children keep running
|
||||
(wind them down first with `close_all_children`, or by cancelling
|
||||
the coordinator, which cascades the cancel to its direct children),
|
||||
the worker thread exits, SSE streams send a final `stream_end` and
|
||||
disconnect. The row is reopenable via
|
||||
`POST /v1/api/workstreams/{ws_id}/open` so long as it hasn't been
|
||||
deleted.
|
||||
|
||||
---
|
||||
|
||||
## Further reading
|
||||
|
||||
- [coordinator-skills.md](coordinator-skills.md) — writing a skill
|
||||
that runs on a coordinator session (orchestrator framing,
|
||||
workflow patterns, `SkillKind` classifier).
|
||||
- [bulk-endpoints.md](bulk-endpoints.md) — the two bulk-shape
|
||||
idioms (`{results, denied, truncated}` vs
|
||||
`{<bucket>, failed, skipped}`) used by `cluster/ws/live`,
|
||||
`spawn_batch`, and `close_all_children`.
|
||||
- [architecture.md](architecture.md) — cluster-wide architecture
|
||||
including how coordinator sessions fit next to node-hosted
|
||||
interactive workstreams.
|
||||
- The live OpenAPI spec (`/openapi.json` on any console process)
|
||||
and Swagger UI (`/docs`) — authoritative schemas for every
|
||||
endpoint above.
|
||||
@@ -1,358 +0,0 @@
|
||||
# Writing a coordinator-specific skill
|
||||
|
||||
A skill is prompt-level framing that steers a Turnstone session
|
||||
toward a narrow task. Most skills target **interactive** sessions —
|
||||
the single-workstream "do this thing" surface where the model wields
|
||||
`bash`, `edit_file`, `web_fetch`, and the rest of the maker toolset.
|
||||
|
||||
A **coordinator skill** is different. It runs on a session whose job
|
||||
is to orchestrate other sessions. The toolset is smaller and
|
||||
narrower, the role is an orchestrator instead of a maker, and the
|
||||
success metric is "did the plan resolve" instead of "did the code
|
||||
compile". This doc covers the differences a skill author has to
|
||||
care about.
|
||||
|
||||
---
|
||||
|
||||
## `kind` — authored audience metadata
|
||||
|
||||
A row in `prompt_templates` carries a `kind` column (see
|
||||
[`turnstone/core/skill_kind.py`](../turnstone/core/skill_kind.py);
|
||||
migration 044 added the column). Three values:
|
||||
|
||||
| `SkillKind` enum | Stored as | Meaning |
|
||||
|-------------------------|-----------------|----------------------------------------------------------------------------|
|
||||
| `SkillKind.INTERACTIVE` | `"interactive"` | Authored for the interactive maker role (single-workstream "do this"). |
|
||||
| `SkillKind.COORDINATOR` | `"coordinator"` | Authored for the orchestrator role (delegate, monitor, synthesise). |
|
||||
| `SkillKind.ANY` | `"any"` | Either surface (or audience-neutral). Default on create. |
|
||||
|
||||
The `kind` field is a `StrEnum` — drop-in `str` compatible — so DB
|
||||
rows, JSON payloads, and `==` comparisons all work without translation
|
||||
at the edge.
|
||||
|
||||
**`kind` is metadata, not an enforcement boundary.** The model can
|
||||
`skills(action='find')` across every kind from any session, `get` any
|
||||
row by name, and `load` any visible skill regardless of session kind.
|
||||
Actual runtime capability is gated by `allowed_tools` + `auto_approve`
|
||||
on the skill and the operator's approval card on every `load` /
|
||||
`spawn_workstream(skill=...)` decision — `kind` doesn't add or remove
|
||||
any of that. It's a sorting / grouping / search-narrowing hint.
|
||||
|
||||
The opt-in filter is on `skills(action='find', kind='coordinator')`
|
||||
(or `'interactive'`) — pass it when you want to narrow a catalog
|
||||
browse to a specific authored audience. Omitting it (or passing
|
||||
`kind='any'`) returns the full catalog. When supplied, the storage
|
||||
filter widens to `[<kind>, 'any']` so audience-neutral rows remain
|
||||
visible inside the narrowed view.
|
||||
|
||||
**Tagging a new skill as coordinator-targeted** — set `kind` to
|
||||
`SkillKind.COORDINATOR` (or the literal `"coordinator"`) when you
|
||||
`skills(action='create', kind='coordinator', ...)` or POST to
|
||||
`/v1/api/admin/skills`. Use this to signal intent to other skill
|
||||
authors and to make the orchestrator-targeted catalog easy to
|
||||
browse — not to hide the skill from interactive sessions. Existing
|
||||
rows default to `SkillKind.ANY`; bump them to `COORDINATOR` if
|
||||
you've rewritten the prompt around the orchestrator toolset and
|
||||
want the kind filter to surface them as such.
|
||||
|
||||
---
|
||||
|
||||
## Tool surface differences
|
||||
|
||||
Coordinator sessions receive a **fixed** tool set, defined in
|
||||
`turnstone/core/tools.py` as `COORDINATOR_TOOLS`. Nothing a skill
|
||||
or MCP config can do adds to it. Current members:
|
||||
|
||||
| Tool | Category | Notes |
|
||||
|---------------------------|-----------------|---------------------------------------------------------------------|
|
||||
| `spawn_workstream` | delegate | Create one child. Requires approval. |
|
||||
| `spawn_batch` | delegate | Create up to 10 children in one approval. Partial-success shape. |
|
||||
| `inspect_workstream` | observe | Read state + tail of one child. Auto-approved (no mutation). |
|
||||
| `list_workstreams` | observe | List the direct children (same shape as `/children` endpoint). |
|
||||
| `wait_for_workstream` | block | Block until one/all listed children hit a terminal state. |
|
||||
| `send_to_workstream` | steer | Queue a follow-up message to a running child. |
|
||||
| `close_workstream` | wind-down | Soft-close one child. Requires approval. |
|
||||
| `close_all_children` | wind-down | Soft-close every direct child in one approval. Partial-success shape. |
|
||||
| `cancel_workstream` | wind-down | Drop the in-flight generation; leaves child idle for a fresh send. |
|
||||
| `delete_workstream` | wind-down | Hard-delete one child. Requires approval. |
|
||||
| `list_nodes` | discover | Enumerate live cluster nodes + capabilities. |
|
||||
| `skills` (action=find) | discover | Browse the skill catalog; opt-in `kind` filter narrows by audience. |
|
||||
| `memory` | persist | Durable orchestration memory (`coordinator` scope, per-user — survives across coordinator sessions). |
|
||||
| `notify` | broadcast | Post a status update to a human channel at a narrative beat. |
|
||||
| `tasks` | plan | Orchestrator-only scratchpad. Children don't see it. |
|
||||
|
||||
Explicitly **not** in the coordinator set:
|
||||
|
||||
- `bash` / `edit_file` / `write_file` / `append_file` / `diff_file` — no local FS.
|
||||
- `read_file` / `search` — no local FS reads.
|
||||
- `web_fetch` / `web_search` — no direct web access.
|
||||
- `task_agent` — sub-agent tool is zeroed on coord sessions.
|
||||
- `recall` / `watch` / `read_resource` / `use_prompt` — UX / persistence tools that belong to interactive sessions. The dual-kind `memory` / `skills` / `notify` tools are available on both kinds (see the table above).
|
||||
|
||||
If your skill needs a coordinator to "run a command" or "read a
|
||||
file", write the delegate pattern instead: spawn a child with an
|
||||
appropriate skill, `wait_for_workstream`, then `inspect_workstream`
|
||||
for the output. The coordinator stays the orchestrator.
|
||||
|
||||
---
|
||||
|
||||
## Framing differences
|
||||
|
||||
Interactive skills compose on top of `base_interactive.md` — a
|
||||
"maker" framing: get the work done, use the tools, edit the code,
|
||||
close the loop.
|
||||
|
||||
Coordinator skills compose on top of
|
||||
[`personas/orchestrator.md`](../turnstone/prompts/personas/orchestrator.md) —
|
||||
an "orchestrator" framing: decompose, delegate, monitor, synthesise.
|
||||
The base text is short but sets the tone every coordinator skill
|
||||
inherits:
|
||||
|
||||
> You are a coordinator. Your role is to orchestrate work across
|
||||
> the cluster... You do
|
||||
> not edit files, run shell commands, browse the web, or manipulate
|
||||
> the codebase directly. Children do that.
|
||||
|
||||
Write your skill's system prompt to *add* task-specific orchestration
|
||||
hints on top — don't re-explain the role, don't paste tool JSON,
|
||||
don't try to override the "no direct action" contract. Keep the
|
||||
additions to: (a) the specific kind of work this skill delegates;
|
||||
(b) the preferred skill tags for children; (c) the synthesis shape
|
||||
the skill should end on.
|
||||
|
||||
---
|
||||
|
||||
## `tasks` integration
|
||||
|
||||
`tasks` is the coordinator's scratchpad — a persisted, ordered
|
||||
list of rows with fields `{id, title, status, child_ws_id, created,
|
||||
updated}` that only this coordinator sees. Children don't see it;
|
||||
the user does via the sidebar. Five actions: `add`, `update`,
|
||||
`remove`, `reorder`, `list` (only `list` is auto-approved; the
|
||||
mutators go through the approval flow).
|
||||
|
||||
The input schema refers to rows by `task_id`; the persisted row
|
||||
object exposes the same id as `id`. The `child_ws_id` field is a
|
||||
free-form label the skill sets to link a task to a spawned
|
||||
workstream — it is NOT validated against the workstreams table, so
|
||||
a skill can set it to a placeholder before `spawn_workstream`
|
||||
returns or keep it pointing at a closed child for later audit.
|
||||
|
||||
A skill's initial prompt can seed the task list by calling
|
||||
`tasks(action="add", title=...)` as its very first tool calls —
|
||||
the user gets a visible plan before any child is spawned, and the
|
||||
coordinator's future self has something concrete to iterate on.
|
||||
Status transitions (`pending` → `in_progress` → `done` / `blocked`)
|
||||
are the skill's main feedback loop: mutate the task when the child
|
||||
covering it finishes, not when the child starts. Use
|
||||
`tasks(action="update", task_id=..., child_ws_id=<ws_id>)` to
|
||||
link a task to the child that owns it once spawn returns.
|
||||
|
||||
A final gotcha: parallel tool dispatch does NOT serialise reads
|
||||
after writes in the same batch. If a skill issues an `update` and
|
||||
a `list` in one parallel tool batch, the `list` response may reflect
|
||||
the pre-update state. Dispatch mutate and list serially (one
|
||||
tool_use turn each) when the list must observe the mutation.
|
||||
|
||||
Keep the tasks coarse-grained — one per child, roughly. A 20-task
|
||||
list for a 3-child fan-out is noise; a 1-task list for a 5-child
|
||||
fan-out loses the plan. The sidebar renders tasks as the operator's
|
||||
mental model of "what the coord thinks it's doing".
|
||||
|
||||
---
|
||||
|
||||
## Referencing children by `ws_id`
|
||||
|
||||
Every ws_id returned by `spawn_workstream` / `spawn_batch` is a
|
||||
**full 32-char hex string**. The skill's system prompt must not
|
||||
invent ws_ids — a model that hallucinates `"child-1"` or `"ws-abc"`
|
||||
hits the tenant guard in `CoordinatorClient._is_own_subtree`, which
|
||||
validates ws_id against `parent_ws_id=coord_ws_id` AND
|
||||
`user_id=owner` in storage. The rejection shape is uniform and
|
||||
recovery-oriented:
|
||||
|
||||
- **Mutating ops** (`send_to_workstream`, `close_workstream`,
|
||||
`cancel_workstream`, `delete_workstream`) and
|
||||
**`inspect_workstream`** return
|
||||
`{"error": "no workstream matching '<ref>' among your children; …",
|
||||
"status": 404, "ws_id": "<ref>", "did_you_mean": [...],
|
||||
"children": [...], "children_truncated": bool}` — a did-you-mean
|
||||
(edit distance ≤ 3 against the coord's own children, which catches
|
||||
the garbled-hex incident class: a 32-char id whose `aaa` run
|
||||
collapsed to `a`) plus a roster of the coord's children. A ref
|
||||
that matches a child's display NAME is called out explicitly with
|
||||
the right id (names are mutable labels, not addresses). Foreign
|
||||
and nonexistent ids produce the same payload (no existence
|
||||
oracle), every hint references only the coord's own children, and
|
||||
near-miss ids are never auto-resolved — the skill should fix the
|
||||
id and re-issue, not treat the child as dead.
|
||||
- **`wait_for_workstream`** validates ids before waiting: a
|
||||
malformed id fails the whole call immediately (`invalid_ws_ids`
|
||||
carries the per-id payloads above, `elapsed=0`); a well-formed id
|
||||
that is foreign, nonexistent, or hard-deleted mid-wait surfaces as
|
||||
`state="not_found"` and aborts the wait on that tick with
|
||||
top-level `error` / `not_found` / `children` fields.
|
||||
`complete=true` therefore means every polled lane really finished
|
||||
— an unobservable id can neither burn the timeout nor ride along
|
||||
to a "complete" result.
|
||||
|
||||
Pattern: capture each spawn result in the next tool call's input.
|
||||
The JSON tool-result carries `{"child_ws_id": "...", "name": "...",
|
||||
"node_id": "...", "routing_strategy": "..."}`; the model should
|
||||
extract the `child_ws_id` and pass it as `ws_id` (or in the `ws_ids`
|
||||
list) to `inspect_workstream` / `wait_for_workstream` /
|
||||
`send_to_workstream` / `close_workstream` verbatim. The asymmetry
|
||||
— spawn returns `child_ws_id` but the other tools accept `ws_id` /
|
||||
`ws_ids` — is intentional: it defuses a coordinator-LLM recency
|
||||
bias where seeing `ws_id` in a spawn return primed re-spawn loops
|
||||
instead of progression to the wait phase.
|
||||
|
||||
A UI that wants human-readable identifiers should render the `name`
|
||||
field and keep the workstream id as the click-through key — note
|
||||
that the id *value* is the same regardless of whether it arrived
|
||||
under the `child_ws_id` key (spawn return) or the `ws_id` key
|
||||
(every other tool's input/output); only the field name differs.
|
||||
|
||||
---
|
||||
|
||||
## `wait_for_workstream` vs `inspect_workstream`
|
||||
|
||||
Two distinct semantics, different cost profiles:
|
||||
|
||||
- **`wait_for_workstream(ws_ids=[...], timeout=60, mode="any")`** —
|
||||
blocks inside a single tool call until one (or all, for `mode="all"`)
|
||||
of the listed children reaches a terminal state (`idle`, `error`,
|
||||
`closed`, `deleted`). The worker thread blocks up to `timeout`
|
||||
seconds; the assistant turn remains a single round-trip regardless
|
||||
of how long the wait actually takes. Prefer this for "the plan
|
||||
needs child X to finish before the next step."
|
||||
- **`inspect_workstream(ws_id=...)`** — single read of the child's
|
||||
state + tail. Costs a full assistant turn (judge, tokens, stream).
|
||||
Prefer this for "what does the final message say?" after the child
|
||||
has already resolved (via `wait_for_workstream` or a known
|
||||
transition).
|
||||
|
||||
Rule of thumb: wait once for a fan-out, then inspect once per
|
||||
child for the content. A loop of inspect-every-few-seconds is a
|
||||
token-burning antipattern — on 3+ children it rounds to a 10×
|
||||
efficiency hit over a wait+inspect pair.
|
||||
|
||||
---
|
||||
|
||||
## Common coordinator patterns
|
||||
|
||||
Three patterns cover most coordinator skills. Pick the one that
|
||||
matches the task, or combine them deliberately.
|
||||
|
||||
### Pattern 1 — delegate-and-summarise
|
||||
|
||||
One specialist child, one focused brief, one synthesis message back
|
||||
to the user. Appropriate when the user's request is "run the thing
|
||||
and tell me what happened" and the work fits in one workstream.
|
||||
|
||||
```
|
||||
tasks(action='add', title='audit /auth for CSRF')
|
||||
spawn_workstream(skill='engineer', initial_message='audit /auth ...')
|
||||
wait_for_workstream(ws_ids=[<child>], timeout=300)
|
||||
inspect_workstream(ws_id=<child>)
|
||||
→ synthesise the final message into a user-facing response
|
||||
tasks(action='update', task_id='t_01', status='done')
|
||||
close_workstream(ws_id=<child>, reason='audit complete')
|
||||
```
|
||||
|
||||
### Pattern 2 — fan-out-and-synthesise
|
||||
|
||||
N children running in parallel, each with a distinct brief, all
|
||||
waited-on together, then synthesised. Appropriate when the user's
|
||||
request naturally decomposes into independent subtasks.
|
||||
|
||||
```
|
||||
tasks seeds:
|
||||
t_01 benchmark Anthropic 4.7 latency on summarisation
|
||||
t_02 benchmark OpenAI GPT-5.2 latency on summarisation
|
||||
t_03 benchmark Gemini 2.5 latency on summarisation
|
||||
spawn_batch(children=[...3 briefs...])
|
||||
wait_for_workstream(ws_ids=[c1, c2, c3], mode='all', timeout=600)
|
||||
inspect_workstream(ws_id=c1); ...(c2); ...(c3)
|
||||
→ synthesise head-to-head comparison
|
||||
tasks → all done
|
||||
close_all_children(reason='benchmark complete')
|
||||
```
|
||||
|
||||
Prefer `spawn_batch` over 3 individual `spawn_workstream` calls —
|
||||
one approval instead of three, one audit trail, deterministic
|
||||
sibling ordering. Pair with `wait_for_workstream(mode='all')` and
|
||||
`close_all_children(reason=...)` to wind the fan-out down in one
|
||||
approval each.
|
||||
|
||||
### Pattern 3 — plan-then-delegate
|
||||
|
||||
The coordinator first uses its own reasoning to carve the plan,
|
||||
records it in `tasks`, then spawns children that each own one
|
||||
task. Appropriate when the user's request is "figure out how to X"
|
||||
and the coordinator's planning step is itself valuable.
|
||||
|
||||
```
|
||||
→ coord reasons about the shape of the work
|
||||
tasks(action='add', title='...') × N # the plan, visible in the sidebar
|
||||
for task in tasks:
|
||||
spawn_workstream(skill=..., initial_message=task.brief)
|
||||
tasks(action='update', task_id=task.id, notes='ws=<child_ws_id>')
|
||||
wait_for_workstream(ws_ids=[...], mode='all', timeout=...)
|
||||
for child in children:
|
||||
inspect_workstream(ws_id=child)
|
||||
tasks(action='update', task_id=..., status='done', notes='result summary')
|
||||
→ synthesise
|
||||
```
|
||||
|
||||
The key distinction from Pattern 2: the plan is an artifact the user
|
||||
can see and interact with (via the sidebar). If the coordinator's
|
||||
reasoning-pass was wrong about the decomposition, the user can
|
||||
course-correct before any child runs.
|
||||
|
||||
---
|
||||
|
||||
## Testing a coordinator skill
|
||||
|
||||
Coordinator sessions are hosted on the console, not on a node.
|
||||
Integration tests that drive a real coord session live under
|
||||
`tests/test_coordinator_end_to_end.py` — they spin a console with
|
||||
an in-memory SQLite backend and a fake upstream node, then drive
|
||||
the session through its HTTP surface.
|
||||
|
||||
For a new coordinator skill:
|
||||
|
||||
1. Write the skill prompt as a string and pass it to the
|
||||
`coord_session` fixture's `skill=` kwarg (see
|
||||
`tests/test_coordinator_tools.py` for the pattern).
|
||||
2. Build a small fake cluster: one node + two children via
|
||||
the `_seed_children` helper in `tests/_coord_test_helpers.py`
|
||||
(``_seed_children(mgr._adapter, coord.id, ["child-1", "child-2"])``).
|
||||
3. Drive the session with seeded tool_call dicts matching the
|
||||
provider layer's shape. The unit-level tests in
|
||||
`tests/test_coordinator_tools.py` show the helper (`_tc(name,
|
||||
args, call_id)`).
|
||||
4. Assert the skill's decision shape — which tools fire in what
|
||||
order, what the tasks looks like at the end, which
|
||||
`_error` reasons appear on the denied-path.
|
||||
|
||||
A full end-to-end test isn't required for every skill; a
|
||||
prepare-step unit test that asserts "given this initial message, the
|
||||
first tool call is X with Y args" is usually sufficient to catch
|
||||
framing drift without a real LLM in the loop.
|
||||
|
||||
---
|
||||
|
||||
## Further reading
|
||||
|
||||
- [coordinator-api-tour.md](coordinator-api-tour.md) — the HTTP
|
||||
surface every coordinator skill indirectly drives.
|
||||
- [bulk-endpoints.md](bulk-endpoints.md) — the response shape
|
||||
`spawn_batch` and `close_all_children` use, so your skill can
|
||||
parse results / denied arrays correctly.
|
||||
- [governance.md](governance.md) — the broader governance surface
|
||||
(`/trust`, `/restrict`, role-based permissions)
|
||||
that wraps every coord session.
|
||||
- [settings.md](settings.md) — `coordinator.model_alias` and
|
||||
`coordinator.reasoning_effort` settings that gate which LLM runs
|
||||
the coordinator session at all.
|
||||
@@ -1,27 +1,34 @@
|
||||
# Consistent Hash Ring — Reference Design
|
||||
|
||||
**Status**: Reference — alternative routing strategy
|
||||
**Status**: Reference (not currently in the hot path)
|
||||
**Date**: 2026-03-30
|
||||
|
||||
Live routing uses **rendezvous (HRW) hashing** in
|
||||
`turnstone/core/rendezvous.py` and `turnstone/console/router.py`. This
|
||||
document captures a vnode-ring approach as a reference for future
|
||||
evaluation if the cluster outgrows rendezvous's O(N)-per-route
|
||||
characteristic.
|
||||
## Overview
|
||||
|
||||
The FNV-1a-32 hash function specified below is bit-identical to the
|
||||
hash used by the live rendezvous implementation; cross-language clients
|
||||
can rely on these test vectors.
|
||||
This document describes a consistent hash ring algorithm evaluated during
|
||||
the design of the direct HTTP transport routing system. The current
|
||||
implementation uses weight-proportional bucket assignment with a
|
||||
donor/recipient rebalancing algorithm (see `direct-http-transport.md`).
|
||||
The consistent hash ring is documented here as a reference for future
|
||||
scalability work — if the cluster grows beyond the point where the
|
||||
weight-proportional approach is sufficient, the ring provides a
|
||||
proven alternative with stronger stability guarantees.
|
||||
|
||||
## When the ring approach becomes interesting
|
||||
## When to consider the ring approach
|
||||
|
||||
The vnode ring becomes preferable to rendezvous hashing when:
|
||||
The current weight-proportional seeding + donor/recipient rebalancer works
|
||||
well when:
|
||||
- Cluster size is moderate (< 50 nodes)
|
||||
- Nodes join/leave infrequently
|
||||
- The rebalancer runs centrally (in the console)
|
||||
|
||||
- Cluster size grows large (50+ nodes) and the per-route O(N) hash
|
||||
computation becomes visible against downstream HTTP cost.
|
||||
- Decentralised routing is needed (each node computes the ring locally,
|
||||
no central console required).
|
||||
- A precomputed flat-array lookup is desired so the routing hot path
|
||||
avoids hashing entirely.
|
||||
The consistent hash ring becomes advantageous when:
|
||||
- Cluster size grows large (50+ nodes) and frequent membership changes
|
||||
cause the donor/recipient algorithm to churn
|
||||
- Decentralized routing is needed (each node computes the ring locally,
|
||||
no central console required)
|
||||
- Cross-language determinism is important (multiple implementations must
|
||||
agree on the same assignment without sharing state)
|
||||
|
||||
## Algorithm
|
||||
|
||||
@@ -126,17 +133,16 @@ class HashRing:
|
||||
# Precompute all 65536 bucket assignments
|
||||
```
|
||||
|
||||
## Comparison with rendezvous (HRW) hashing
|
||||
## Comparison with current approach
|
||||
|
||||
| Aspect | Rendezvous (live) | Consistent hash ring (this doc) |
|
||||
|--------|-------------------|---------------------------------|
|
||||
| Per-route cost | O(N) hash computes | O(log V) bisect against precomputed array |
|
||||
| Seeding | None — pure function | Build vnode array on every membership change |
|
||||
| Node addition | Pure function moves ~1/N keys | Ring moves ~1/N buckets |
|
||||
| Node removal | Surviving nodes' keys unchanged | Surviving nodes' buckets unchanged |
|
||||
| Decentralised | Yes — pure function over services | Yes — each node computes locally |
|
||||
| Persistent state | None | None on the hot path; precomputed array in memory |
|
||||
| Complexity | ~20 LOC | Virtual-node construction + bisect |
|
||||
| Aspect | Weight-proportional (current) | Consistent hash ring |
|
||||
|--------|------------------------------|---------------------|
|
||||
| Seeding | Exact weight split, deterministic | Hash-based, ~3% variance |
|
||||
| Node addition | Donor/recipient moves only excess | Ring moves ~1/N buckets |
|
||||
| Node removal | Dead buckets → most underloaded | Ring redistributes to clockwise neighbors |
|
||||
| Cross-node churn | Zero (only donor→recipient) | Zero (ring stability guarantee) |
|
||||
| Decentralized | No (needs central rebalancer) | Yes (each node computes locally) |
|
||||
| Complexity | Simple weight arithmetic | Virtual node construction + bisect |
|
||||
|
||||
## Test vectors
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ eval --> sqlite : SQLite
|
||||
|
||||
console --> server : HTTP proxy\n(hash-ring bucket lookup,\nproxy /node/{id}/* traffic)
|
||||
|
||||
channel --> server : HTTP + SSE\n(POST /v1/api/workstreams/{ws_id}/send,\nGET /v1/api/workstreams/{ws_id}/events)
|
||||
channel --> server : HTTP + SSE\n(POST /v1/api/send,\nGET /v1/api/events)
|
||||
|
||||
' Notes
|
||||
note right of console
|
||||
|
||||
@@ -19,8 +19,7 @@ package "Entry Points" <<Rectangle>> {
|
||||
component [cli.py\nturnstone] as cli <<entry>>
|
||||
component [server.py\nturnstone-server] as server <<entry>>
|
||||
component [eval.py\nturnstone-eval] as eval <<entry>>
|
||||
component [admin.py\nturnstone-admin] as admin <<entry>>
|
||||
component [bootstrap.py\nturnstone-bootstrap] as bootstrap <<entry>>
|
||||
component [chat.py\n(re-exports)] as chat <<entry>>
|
||||
}
|
||||
|
||||
' Core engine
|
||||
@@ -34,12 +33,13 @@ package "turnstone/core/" <<Rectangle>> {
|
||||
component [metrics.py\nPrometheus metrics] as metrics <<core>>
|
||||
component [config.py\nTOML config] as config <<core>>
|
||||
component [safety.py\nPath validation] as safety <<core>>
|
||||
component [sandbox.py\nCommand sandbox] as sandbox <<core>>
|
||||
component [edit.py\nFile editing] as edit <<core>>
|
||||
component [web.py\nWeb helpers] as web <<core>>
|
||||
component [auth.py\nAuthentication] as auth <<core>>
|
||||
component [healthcheck.py\nBackendHealthMonitor] as healthcheck <<core>>
|
||||
component [ratelimit.py\nRateLimiter] as ratelimit <<core>>
|
||||
component [mcp_client.py\nMCPClientManager\n(push + manual refresh)] as mcp <<core>>
|
||||
component [mcp_client.py\nMCPClientManager\n(push + periodic refresh)] as mcp <<core>>
|
||||
component [tool_search.py\nToolSearchManager, BM25] as toolsearch <<core>>
|
||||
component [model_registry.py\nModelRegistry] as registry <<core>>
|
||||
}
|
||||
@@ -48,8 +48,7 @@ package "turnstone/core/" <<Rectangle>> {
|
||||
package "turnstone/channels/" <<Rectangle>> {
|
||||
component [_routing.py\nChannelRouter] as router <<channel>>
|
||||
component [discord/bot.py\nDiscordBot] as discordbot <<channel>>
|
||||
component [slack/bot.py\nSlackBot (Socket Mode)] as slackbot <<channel>>
|
||||
component [cli.py\nturnstone-channel] as gateway <<channel>>
|
||||
component [gateway.py\nturnstone-channel] as gateway <<channel>>
|
||||
}
|
||||
|
||||
' Console
|
||||
@@ -113,8 +112,7 @@ eval --> memory
|
||||
eval --> config
|
||||
eval --> tools
|
||||
|
||||
admin --> auth
|
||||
bootstrap --> providers
|
||||
chat --> session
|
||||
|
||||
' Core internal deps
|
||||
session --> providers
|
||||
@@ -122,6 +120,7 @@ session --> tools
|
||||
session --> memory
|
||||
memory --> storage
|
||||
session --> safety
|
||||
session --> sandbox
|
||||
session --> edit
|
||||
session --> web
|
||||
session --> healthcheck
|
||||
@@ -136,10 +135,8 @@ tools --> schemas
|
||||
|
||||
' Channel dependencies
|
||||
gateway --> discordbot
|
||||
gateway --> slackbot
|
||||
gateway --> router
|
||||
discordbot --> sdkserver : HTTP + SSE
|
||||
slackbot --> sdkserver : HTTP + SSE
|
||||
router --> storage : channel_routes
|
||||
|
||||
' Console dependencies
|
||||
|
||||
@@ -15,6 +15,7 @@ interface "SessionUI" as SessionUI <<Protocol>> {
|
||||
+ on_tool_result(call_id: str, name: str, output: str, *, is_error: bool = False)
|
||||
+ on_tool_output_chunk(call_id: str, chunk: str)
|
||||
+ on_status(usage: dict, ctx_window: int, effort: str)
|
||||
+ on_plan_review(content: str) → str
|
||||
+ on_info(message: str)
|
||||
+ on_error(message: str)
|
||||
+ on_state_change(state: str)
|
||||
@@ -42,13 +43,15 @@ class "WorkstreamTerminalUI" as WsTermUI {
|
||||
class "WebUI" as WebUI {
|
||||
- _listeners: list[Queue]
|
||||
- _approval_event: Event
|
||||
- _plan_event: Event
|
||||
- _ws_prompt_tokens: int
|
||||
- _ws_tool_calls: dict
|
||||
+ resolve_approval(approved, feedback)
|
||||
+ resolve_plan(feedback)
|
||||
--
|
||||
Enqueues JSON events for SSE.
|
||||
Blocks on threading.Event for
|
||||
approval.
|
||||
approval/plan review.
|
||||
SSE handlers bridge Queue to
|
||||
async via run_in_executor().
|
||||
--
|
||||
@@ -66,10 +69,9 @@ class "NullUI" as NullUI {
|
||||
interface "LLMProvider" as LLMProvider <<Protocol>> {
|
||||
+ provider_name: str {property}
|
||||
+ get_capabilities(model) → ModelCapabilities
|
||||
+ create_streaming(client, model, messages, ..., replay_reasoning_to_model) → Iterator[StreamChunk]
|
||||
+ create_completion(client, model, messages, ..., replay_reasoning_to_model) → CompletionResult
|
||||
+ create_streaming(client, model, messages, ...) → Iterator[StreamChunk]
|
||||
+ create_completion(client, model, messages, ...) → CompletionResult
|
||||
+ convert_tools(tools) → list[dict]
|
||||
+ extract_reasoning_text(provider_blocks) → str
|
||||
+ retryable_error_names: frozenset[str] {property}
|
||||
--
|
||||
core/providers/_protocol.py
|
||||
@@ -124,7 +126,6 @@ class "ModelCapabilities" as ModelCaps <<frozen>> {
|
||||
+ supports_web_search: bool
|
||||
+ supports_tool_search: bool
|
||||
+ supports_vision: bool
|
||||
+ supports_reasoning_replay: bool
|
||||
}
|
||||
|
||||
' ChatSession
|
||||
@@ -142,6 +143,7 @@ class "ChatSession" as ChatSession {
|
||||
+ model_alias: str | None {property}
|
||||
- _tools: list[dict]
|
||||
- _task_tools: list[dict]
|
||||
- _agent_tools: list[dict]
|
||||
- _read_files: set[str]
|
||||
- system_messages: list[dict]
|
||||
--
|
||||
@@ -251,7 +253,7 @@ class "MCPClientManager" as MCPMgr {
|
||||
Background asyncio event loop
|
||||
bridges async MCP SDK to
|
||||
sync ChatSession dispatch.
|
||||
Push + manual refresh.
|
||||
Push + periodic + manual refresh.
|
||||
Resources + prompts discovered
|
||||
alongside tools at startup.
|
||||
--
|
||||
@@ -293,7 +295,7 @@ class "ModelRegistry" as ModelReg {
|
||||
--
|
||||
Thread-safe lazy client + provider
|
||||
creation. Loaded by load_model_registry()
|
||||
from DB + [models.*] config + CLI args.
|
||||
from CLI args + [models.*] config.
|
||||
--
|
||||
core/model_registry.py
|
||||
}
|
||||
@@ -304,9 +306,6 @@ class "ModelConfig" as ModelCfg <<frozen>> {
|
||||
+ base_url: str
|
||||
+ model: str
|
||||
+ context_window: int
|
||||
+ temperature: float | None
|
||||
+ max_tokens: int | None
|
||||
+ reasoning_effort: str | None
|
||||
}
|
||||
|
||||
' Circuit breaker state
|
||||
|
||||
@@ -24,14 +24,6 @@ CS -> DB : save_message(ws_id, "user", input)
|
||||
|
||||
group loop [while tool_calls present]
|
||||
|
||||
CS -> UI : on_turn_start()
|
||||
note right of UI
|
||||
SessionUIBase resets the per-turn inflight
|
||||
buffers (_ws_inflight_content / reasoning /
|
||||
seq) that fuel the SSE in_progress_snapshot
|
||||
event for mid-stream refresh resume.
|
||||
end note
|
||||
|
||||
CS -> UI : on_state_change("thinking")
|
||||
CS -> UI : on_thinking_start()
|
||||
|
||||
@@ -81,14 +73,6 @@ group loop [while tool_calls present]
|
||||
|
||||
CS -> CS : _update_token_table()\ncalibrate chars_per_token ratio
|
||||
CS -> CS : messages.append(assistant_msg)
|
||||
CS -> UI : on_turn_committed()
|
||||
note right of UI
|
||||
Drops the per-turn inflight buffers — the
|
||||
assistant message is now in the history
|
||||
list, so the in_progress_snapshot must
|
||||
not re-render it during the next tool-
|
||||
execution window or the next streaming turn.
|
||||
end note
|
||||
CS -> DB : save_message(ws_id, "assistant", content)
|
||||
CS -> DB : save_message(ws_id, "tool_call", ...) ×N
|
||||
|
||||
@@ -139,9 +123,10 @@ group loop [while tool_calls present]
|
||||
read_file → open().read() or base64 image
|
||||
search → grep subprocess
|
||||
edit_file → string replace
|
||||
task → _run_agent() sub-loop
|
||||
task/plan → _run_agent() sub-loop
|
||||
math → sandboxed subprocess
|
||||
web_fetch → httpx + LLM summarize
|
||||
web_search → provider-native or SearxNG fallback
|
||||
web_search → provider-native or Tavily fallback
|
||||
memory/recall → SQLite
|
||||
end note
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
:Dispatch to _prepare_{func_name}();
|
||||
|
||||
note right
|
||||
**Dispatch table (16 built-in + tool_search):**
|
||||
**Dispatch table (19 built-in + tool_search):**
|
||||
┌───────────────┬──────────────────┐
|
||||
│ Tool │ Needs Approval? │
|
||||
├───────────────┼──────────────────┤
|
||||
@@ -34,10 +34,13 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
│ edit_file │ ✓ Yes │
|
||||
│ search │ ✗ Auto-approve │
|
||||
│ diff_file │ ✗ Auto-approve │
|
||||
│ math │ ✗ Auto-approve │
|
||||
│ man │ ✗ Auto-approve │
|
||||
│ web_fetch │ ✗ Auto-approve │
|
||||
│ web_search │ ✗ Auto-approve │
|
||||
│ tool_search │ ✗ Auto-approve │
|
||||
│ task_agent │ ✓ Yes │
|
||||
│ plan_agent │ ✓ Yes │
|
||||
│ memory │ ✗ Auto-approve │
|
||||
│ recall │ ✗ Auto-approve │
|
||||
│ notify │ ✗ Auto-approve │
|
||||
@@ -107,10 +110,13 @@ partition "Phase 3: Execute" #E3F2FD {
|
||||
├─ _exec_write_file: makedirs + write
|
||||
├─ _exec_edit_file: find_occurrences + replace
|
||||
├─ _exec_search: grep subprocess
|
||||
├─ _exec_math: sandboxed subprocess
|
||||
├─ _exec_man: man/info subprocess
|
||||
├─ _exec_web_fetch: httpx.get + LLM summary
|
||||
├─ _exec_web_search: SearxNG JSON GET (fallback for local models)
|
||||
├─ _exec_web_search: Tavily API POST (fallback for local models)
|
||||
├─ _exec_tool_search: BM25 search + expand_visible()
|
||||
├─ _exec_task: _run_agent(TASK_AGENT_TOOLS)
|
||||
├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only)
|
||||
├─ _exec_notify: HTTP POST to channel gateway
|
||||
├─ _exec_memory: structured memory save/search/delete/list
|
||||
├─ _exec_recall: conversation history FTS5 search
|
||||
@@ -125,6 +131,11 @@ partition "Phase 3: Execute" #E3F2FD {
|
||||
|
||||
:bash: ui.on_tool_output_chunk(call_id, line) per stdout line;
|
||||
:ui.on_tool_result(call_id, name, output, is_error) for each;
|
||||
|
||||
if (plan tool was executed?) then (yes)
|
||||
:ui.on_plan_review(output);
|
||||
:Block for user review/feedback;
|
||||
endif
|
||||
}
|
||||
|
||||
:Return (results, user_feedback);
|
||||
|
||||
@@ -13,7 +13,7 @@ skinparam state {
|
||||
state "IDLE" as idle <<idle>> : Waiting for user input.\nNo active LLM call or tool execution.
|
||||
state "THINKING" as thinking <<thinking>> : LLM streaming response.\nTokens flowing (reasoning + content).
|
||||
state "RUNNING" as running <<running>> : Tools executing.\nThreadPoolExecutor active.
|
||||
state "ATTENTION" as attention <<attention>> : Blocked on user action.\nTool approval needed.
|
||||
state "ATTENTION" as attention <<attention>> : Blocked on user action.\nTool approval or plan review needed.
|
||||
state "ERROR" as error <<error>> : Exception occurred.\nRecoverable on next send().
|
||||
|
||||
[*] --> idle : Session created
|
||||
@@ -34,6 +34,8 @@ attention --> running : User denies\n(denial recorded)\n_emit_state("running")
|
||||
|
||||
running --> thinking : Tool results appended,\nnext LLM call\n_emit_state("thinking")
|
||||
|
||||
running --> attention : Plan tool complete,\non_plan_review()\n_emit_state("attention")
|
||||
|
||||
running --> error : Exception during\ntool execution
|
||||
|
||||
error --> thinking : New send() call\n_emit_state("thinking")
|
||||
@@ -42,7 +44,7 @@ thinking --> idle : cancel() called\nstream aborted\n_emit_state("idle")
|
||||
|
||||
running --> idle : cancel() called\n_emit_state("idle")
|
||||
|
||||
attention --> idle : cancel() unblocks\napproval wait\n_emit_state("idle")
|
||||
attention --> idle : cancel() unblocks\napproval/plan wait\n_emit_state("idle")
|
||||
|
||||
note left of idle
|
||||
**Cancel escalation:**
|
||||
|
||||
@@ -170,15 +170,15 @@ Server --> Browser : Shimmed app.js
|
||||
deactivate Server
|
||||
|
||||
note right of Browser
|
||||
All fetch("/v1/api/workstreams/{ws_id}/send") calls in the
|
||||
server UI now become fetch("/node/nodeA/v1/api/workstreams/{ws_id}/send"),
|
||||
All fetch("/v1/api/send") calls in the
|
||||
server UI now become fetch("/node/nodeA/v1/api/send"),
|
||||
routed through the console proxy.
|
||||
end note
|
||||
|
||||
Browser -> Server : GET /node/nodeA/v1/api/workstreams/ws789/events
|
||||
Browser -> Server : GET /node/nodeA/v1/api/events?ws_id=ws789
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/v1/api/workstreams/ws789/events\n(SSE stream via httpx.AsyncClient timeout=None)
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/v1/api/events?ws_id=ws789\n(SSE stream via httpx.AsyncClient timeout=None)
|
||||
activate NodeA
|
||||
|
||||
loop SSE streaming
|
||||
@@ -189,10 +189,10 @@ end
|
||||
deactivate NodeA
|
||||
deactivate Server
|
||||
|
||||
Browser -> Server : POST /node/nodeA/v1/api/workstreams/ws789/send\n{message:"hello"}
|
||||
Browser -> Server : POST /node/nodeA/v1/api/send\n{message:"hello", ws_id:"ws789"}
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : POST http://10.0.1.1:8080/v1/api/workstreams/ws789/send\n(body forwarded)
|
||||
Server -> NodeA : POST http://10.0.1.1:8080/v1/api/send\n(body forwarded)
|
||||
activate NodeA
|
||||
NodeA --> Server : {status:"ok"}
|
||||
deactivate NodeA
|
||||
|
||||
@@ -30,6 +30,7 @@ package "turnstone/sdk/ (Python)" {
|
||||
+ close_workstream()
|
||||
+ send(message, ws_id)
|
||||
+ approve()
|
||||
+ plan_feedback()
|
||||
+ command()
|
||||
+ cancel(ws_id)
|
||||
+ stream_events(ws_id)
|
||||
|
||||
@@ -23,7 +23,7 @@ interface "StorageBackend" as SB <<protocol>> {
|
||||
+resolve_workstream(alias_or_id) → str | None
|
||||
+delete_workstream(ws_id) → bool
|
||||
+prune_workstreams(retention_days) → (int, int)
|
||||
+list_workstreams(node_id, limit, *, parent_ws_id, kind, user_id) → list
|
||||
+list_workstreams(node_id, limit) → list
|
||||
+save_workstream_config(ws_id, config)
|
||||
+load_workstream_config(ws_id) → dict
|
||||
+kv_get(key) → str | None
|
||||
|
||||
@@ -79,7 +79,7 @@ class "Scope Hierarchy" as SH <<scope>> {
|
||||
--
|
||||
GET → read
|
||||
POST write paths → write
|
||||
POST /api/workstreams/{ws_id}/approve → approve
|
||||
POST /api/approve → approve
|
||||
/api/admin/* → approve
|
||||
}
|
||||
|
||||
|
||||
@@ -20,14 +20,11 @@ class "Discord" as Discord <<platform>> {
|
||||
asyncio event loop
|
||||
}
|
||||
|
||||
class "Slack" as Slack <<platform>> {
|
||||
Socket Mode WebSocket
|
||||
class "Slack (future)" as Slack <<platform>> {
|
||||
Socket Mode / Events API
|
||||
Block Kit messages
|
||||
Slash command (default /turnstone)
|
||||
DM + channel events
|
||||
--
|
||||
slack-bolt (Python)
|
||||
asyncio event loop
|
||||
Planned integration
|
||||
}
|
||||
|
||||
class "Teams (future)" as Teams <<platform>> {
|
||||
@@ -41,7 +38,7 @@ class "Teams (future)" as Teams <<platform>> {
|
||||
class "turnstone-channel" as ChannelService <<service>> {
|
||||
entry point: turnstone-channel
|
||||
--
|
||||
One process — hosts one or more adapters
|
||||
One process per platform
|
||||
asyncio event loop
|
||||
Structured logging (structlog)
|
||||
--log-level, --log-format
|
||||
@@ -50,19 +47,6 @@ class "turnstone-channel" as ChannelService <<service>> {
|
||||
GET /health
|
||||
}
|
||||
|
||||
class "SlackBot" as SlackBot <<service>> {
|
||||
+on_message(event)
|
||||
+on_action(action) (Block Kit buttons)
|
||||
+send(channel_id, content)
|
||||
+send_notification(channel_id, content, ws_id)
|
||||
+run(bot_token, app_token)
|
||||
--
|
||||
slack-bolt AsyncApp
|
||||
Socket Mode client
|
||||
Per-user channel sessions via slash command
|
||||
DM routing without slash command
|
||||
}
|
||||
|
||||
class "DiscordBot" as Bot <<service>> {
|
||||
+on_message(msg)
|
||||
+on_interaction(interaction)
|
||||
@@ -95,10 +79,10 @@ class "ChannelRouter" as Router <<service>> {
|
||||
|
||||
' -- Server --
|
||||
class "turnstone-server" as Server <<server>> {
|
||||
POST /v1/api/workstreams/{ws_id}/send
|
||||
POST /v1/api/workstreams/{ws_id}/approve
|
||||
POST /v1/api/send
|
||||
POST /v1/api/approve
|
||||
POST /v1/api/workstreams/new
|
||||
GET /v1/api/workstreams/{ws_id}/events
|
||||
GET /v1/api/events?ws_id=
|
||||
--
|
||||
LLM execution + tool use
|
||||
SSE event stream
|
||||
@@ -148,21 +132,16 @@ Bot --> Router : on_message\non_interaction
|
||||
Router --> CU : resolve identity
|
||||
Router --> CR : resolve / register route
|
||||
|
||||
Router --> Server : POST /v1/api/workstreams/{ws_id}/send\nPOST /v1/api/workstreams/{ws_id}/approve\nPOST /v1/api/workstreams/new
|
||||
Bot --> Server : GET /v1/api/workstreams/{ws_id}/events\n(SSE via httpx-sse)
|
||||
Router --> Server : POST /v1/api/send\nPOST /v1/api/approve\nPOST /v1/api/workstreams/new
|
||||
Bot --> Server : GET /v1/api/events?ws_id=\n(SSE via httpx-sse)
|
||||
Server --> Bot : SSE event stream
|
||||
|
||||
Bot --> Discord : reply / embed\nbutton callback
|
||||
|
||||
Slack --> SlackBot : socket-mode\nevents
|
||||
SlackBot --> Router : on_message / on_action
|
||||
SlackBot --> Server : POST /v1/api/workstreams/{ws_id}/send\nGET /v1/api/workstreams/{ws_id}/events
|
||||
SlackBot --> Slack : post / update\nBlock Kit button callbacks
|
||||
|
||||
Slack .[hidden]. Discord
|
||||
Teams .[hidden]. Slack
|
||||
|
||||
ChannelService --> Bot : creates + runs
|
||||
ChannelService --> SlackBot : creates + runs
|
||||
ChannelService --> Router : creates
|
||||
ChannelService --> SVC : register / heartbeat /\nderegister
|
||||
|
||||
@@ -179,7 +158,7 @@ note right of Bot
|
||||
(or creates new workstream)
|
||||
4. ChannelRouter resolves platform user -> user_id
|
||||
via channel_users table
|
||||
5. Router sends POST /v1/api/workstreams/{ws_id}/send to server
|
||||
5. Router sends POST /v1/api/send to server
|
||||
|
||||
**Workstream Resume (evicted workstreams)**
|
||||
1. Stale route detected (no active SSE listener)
|
||||
@@ -193,7 +172,7 @@ end note
|
||||
note right of Server
|
||||
**Outbound Flow**
|
||||
1. Server emits SSE events on
|
||||
GET /v1/api/workstreams/{ws_id}/events
|
||||
GET /v1/api/events?ws_id=
|
||||
2. Bot subscribes via httpx-sse
|
||||
3. Bot formats and sends to Discord thread
|
||||
end note
|
||||
@@ -204,7 +183,7 @@ note bottom of CR
|
||||
2. Bot renders Discord buttons (Approve / Deny)
|
||||
3. User clicks button -> on_interaction()
|
||||
4. Router builds ApproveMessage
|
||||
5. Router sends POST /v1/api/workstreams/{ws_id}/approve to server
|
||||
5. Router sends POST /v1/api/approve to server
|
||||
end note
|
||||
|
||||
note bottom of CU
|
||||
|
||||
@@ -190,25 +190,21 @@ group Push Notifications (debounced 5s per server)
|
||||
MCPMgr -> Storage : sync_prompts_to_storage()
|
||||
end
|
||||
|
||||
group Manual Refresh
|
||||
Session -> MCPMgr : refresh_sync()
|
||||
group Periodic Polling (default 4h)
|
||||
MCPMgr -> MCPMgr : _periodic_refresh()
|
||||
note right
|
||||
/mcp refresh [server] —
|
||||
re-fetches catalog and
|
||||
attempts reconnect for
|
||||
disconnected servers.
|
||||
Only polls capabilities
|
||||
without push support.
|
||||
Staggered per-server.
|
||||
Disconnected servers get
|
||||
reconnect attempts with
|
||||
exponential backoff (60s-1h).
|
||||
end note
|
||||
end
|
||||
|
||||
group Manual Reconnect
|
||||
Session -> MCPMgr : reconnect_sync(name)
|
||||
note right
|
||||
Operator-driven via the
|
||||
console admin panel —
|
||||
tears down session, clears
|
||||
circuit breaker, runs a
|
||||
fresh handshake.
|
||||
end note
|
||||
group Manual Refresh
|
||||
Session -> MCPMgr : refresh_sync()
|
||||
note right: /mcp refresh [server]
|
||||
end
|
||||
|
||||
== Policy Evaluation ==
|
||||
|
||||
@@ -202,7 +202,7 @@ note over Session, Judge
|
||||
Cross-model: separate provider/client from [judge] config.
|
||||
|
||||
**Sub-agent exemption:**
|
||||
Task sub-agents skip intent validation entirely.
|
||||
Plan agent and task agent skip intent validation entirely.
|
||||
|
||||
**Output guard:**
|
||||
Runs when judge_config.output_guard is true (default).
|
||||
@@ -211,7 +211,7 @@ note over Session, Judge
|
||||
**Storage:**
|
||||
intent_verdicts table (migration 012), output_assessments table
|
||||
(migration 022). Both queryable via admin API endpoints
|
||||
(requires admin.judge permission). Skills store risk_level,
|
||||
(requires admin.judge permission). Skills store scan_status,
|
||||
scan_report, scan_version for install-time risk assessment.
|
||||
end note
|
||||
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
@startuml
|
||||
title Turnstone - coordinator wait_for_workstream lifecycle
|
||||
|
||||
skinparam sequenceArrowThickness 1.5
|
||||
skinparam noteBackgroundColor #FDF6E3
|
||||
|
||||
participant "Coordinator\nLLM" as LLM
|
||||
participant "ChatSession\n(worker thread)" as CS
|
||||
participant "CoordinatorClient" as CC
|
||||
participant "SessionUI\n(SSE fanout)" as UI
|
||||
participant "Console routing\nproxy" as RP
|
||||
database "Storage\n(workstreams row)" as DB
|
||||
participant "Child\nnode" as NODE
|
||||
|
||||
== Spawn ==
|
||||
|
||||
LLM -> CS : tool_call spawn_workstream(...)
|
||||
activate CS
|
||||
CS -> CC : spawn(initial_message=...,\nparent_ws_id=coord, user_id=...)
|
||||
CC -> RP : POST /v1/api/route/workstreams/new
|
||||
RP -> NODE : dispatch (rendezvous)
|
||||
NODE -> DB : insert workstreams row\nstate='running'
|
||||
RP --> CC : {ws_id, node_id, name, status: 200}
|
||||
CC --> CS : {ws_id, ...}
|
||||
CS -> UI : on_tool_result\n("spawn_workstream", ws_id)
|
||||
deactivate CS
|
||||
|
||||
note right of LLM
|
||||
Model now knows the child ws_id.
|
||||
It can inspect / send / wait, and
|
||||
the parent registry tracks it.
|
||||
end note
|
||||
|
||||
== Wait (blocking) ==
|
||||
|
||||
LLM -> CS : tool_call wait_for_workstream\n(ws_ids=[child], mode="any", timeout=60)
|
||||
activate CS
|
||||
CS -> CS : _prepare_wait_for_workstream\n(validate ws_ids, timeout, mode)
|
||||
CS -> UI : emit wait_started\n{call_id, ws_ids, mode, timeout}
|
||||
|
||||
CS -> CC : wait_for_workstream(ws_ids, timeout,\nmode, progress_callback)
|
||||
activate CC
|
||||
|
||||
loop every 500ms up to timeout
|
||||
CC -> DB : read workstreams row(s)
|
||||
DB --> CC : {state, updated, tokens, ...}
|
||||
alt state in {idle, error, closed, deleted}
|
||||
note over CC
|
||||
real-terminal state ->
|
||||
completion condition met
|
||||
end note
|
||||
else still running / thinking / attention
|
||||
CC -> CS : progress_callback(snap)\n(diff-on-change or 5s heartbeat)
|
||||
CS -> UI : emit wait_progress\n{call_id, elapsed, results?}
|
||||
end
|
||||
end
|
||||
|
||||
CC --> CS : {complete, elapsed,\nresults: {ws_id: snap}}
|
||||
deactivate CC
|
||||
|
||||
CS -> UI : emit wait_ended\n{call_id, complete, elapsed, results}
|
||||
CS -> UI : on_tool_result\n("wait_for_workstream",\n"complete after Ns (R/N resolved)")
|
||||
CS --> LLM : tool_result (full results dict)
|
||||
deactivate CS
|
||||
|
||||
note left of UI
|
||||
Sidebar "waiting on N children" indicator
|
||||
keys on call_id - started / progress / ended
|
||||
scope to a single wait invocation so
|
||||
nested waits render independent badges.
|
||||
end note
|
||||
|
||||
== After wait: inspect + close ==
|
||||
|
||||
LLM -> CS : tool_call inspect_workstream(ws_id=child)
|
||||
CS -> CC : inspect(ws_id)
|
||||
CC -> DB : read row + tail
|
||||
CC --> CS : {state, messages, tokens, ...}
|
||||
CS --> LLM : tool_result (serialised)
|
||||
|
||||
LLM -> CS : tool_call close_workstream\n(ws_id=child, reason="...")
|
||||
CS -> CC : close_workstream(ws_id, reason)
|
||||
CC -> RP : POST /v1/api/route/workstreams/close
|
||||
RP -> NODE : dispatch
|
||||
NODE -> DB : state='closed',\nclose_reason='...'
|
||||
RP --> CC : {status: 200}
|
||||
CC --> CS : {closed: true, status: 200, reason: ...}
|
||||
CS --> LLM : tool_result
|
||||
@enduml
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:95dd5ebc899a1261d516686a5aa3319a7f45015d411302825fa28afbfc82e1ce
|
||||
size 326766
|
||||
oid sha256:75da80e6bd205e45b9fe48aa2f87197110c908f44e1d98c52eee69f956274711
|
||||
size 400402
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9857db23fe3c4316d492073aac69c7e7558b1abe3b95ad7756d4a5933bd0ece7
|
||||
size 620214
|
||||
oid sha256:6471e611beebf647f3a191eb16588571a404cc52a43067883a2b6f06dd936376
|
||||
size 594676
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d9c7769a600c38e6387390e6c42db8152e0f80c31d17b2218f7f636b71c7b868
|
||||
size 355459
|
||||
oid sha256:3aa8d972bba40d78152f9f0c762b9f5ec616d8052c45fa52b7dd1c679ed81d61
|
||||
size 325245
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:23ca090b5656baaf70820cbe4ab6c27f0a3a02e18b4db0695614cf9489c23980
|
||||
size 281440
|
||||
oid sha256:674712a0563f51837383184652efeb28b7bec13378be636e89d2959bfba39d1e
|
||||
size 281519
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ae4f79fb22600106f8cb0af4ba5586bb26ea5d57e27ef382fdc59b6549fdbd21
|
||||
size 415473
|
||||
oid sha256:8a5957b71822656098cbe779ca619b7e56e9b11b0f1c4eac4fc880ed62b71a2a
|
||||
size 358670
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d6aff446a062aa08f316985d00c2183148694f786d7f22172bc50b30046c728b
|
||||
size 379259
|
||||
oid sha256:7623df33be9baf7647ca1c2450640df57e1cd73e8be1f8168aae16e546ad683c
|
||||
size 459941
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:aa12d81dc578f7e65bf4df3152b3de1736289c422f83d0b0cd32107726722357
|
||||
size 172028
|
||||
+96
-230
@@ -1,284 +1,150 @@
|
||||
# Docker Deployment
|
||||
|
||||
Turnstone ships two Docker Compose stacks:
|
||||
Docker Compose stack for running the full turnstone platform.
|
||||
|
||||
| Stack | File | Use it for |
|
||||
|-------|------|------------|
|
||||
| **Dev cluster** | `compose.yaml` (repo root) | Clone-and-run. Builds locally, zero config, full 10-node cluster. |
|
||||
| **Production** | `turnstone/deploy/compose.yaml` | Pip/pipx installs. Pulls released images from ghcr.io, requires real secrets. |
|
||||
|
||||
## Quick start — local cluster
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/turnstonelabs/turnstone
|
||||
cd turnstone
|
||||
# Copy and edit environment config
|
||||
cp .env.example .env
|
||||
|
||||
# Full stack (needs an LLM API on the host)
|
||||
docker compose up
|
||||
```
|
||||
|
||||
That builds one image and brings up the whole stack: PostgreSQL, the console,
|
||||
Caddy, the channel gateway, and **10 server nodes** (`node-1`…`node-10`). No
|
||||
`.env` is required — it ships with insecure dev defaults so it just works.
|
||||
Console dashboard: http://localhost:8090
|
||||
|
||||
Open the dashboard at **https://localhost:8443**. It's served by Caddy with its
|
||||
own local CA, so trust the root certificate once (or click through the browser
|
||||
warning):
|
||||
> See also: [Deployment diagram](diagrams/png/12-deployment.png)
|
||||
|
||||
## Services
|
||||
|
||||
| Service | Port | Profile | Description |
|
||||
|---------|------|---------|-------------|
|
||||
| `server` | 8080 | default | Web UI + chat workstreams + LLM |
|
||||
| `console` | 8090 | default | Cluster dashboard |
|
||||
| `channel` | — | production | Channel gateway (Discord, Slack, etc.) |
|
||||
| `server-1`…`server-10` | — | cluster | 10-node server fleet (PostgreSQL required) |
|
||||
|
||||
## Profiles
|
||||
|
||||
**Default** (no flag) — starts `server` and `console`. Requires an OpenAI-compatible LLM API running on the host (default: `http://localhost:8000/v1`).
|
||||
|
||||
```bash
|
||||
docker compose exec caddy cat /data/caddy/pki/authorities/local/root.crt
|
||||
docker compose up
|
||||
```
|
||||
|
||||
Create your first admin user (any node works — they share one database):
|
||||
**Production** — adds PostgreSQL and the channel gateway. Requires `POSTGRES_PASSWORD` and (for Discord) `TURNSTONE_DISCORD_TOKEN`:
|
||||
|
||||
```bash
|
||||
docker compose exec node-1 turnstone-admin create-user --username admin --name "Admin"
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
### Bring your own LLM
|
||||
|
||||
Nodes boot **without** an LLM and appear in the console immediately. Add real
|
||||
model backends (OpenAI, Anthropic, or a local/vLLM endpoint) from the console
|
||||
UI's **Models** tab. To set a node's bootstrap default instead, point
|
||||
`LLM_BASE_URL` / `OPENAI_API_KEY` at an OpenAI-compatible endpoint in `.env`.
|
||||
|
||||
### Fewer nodes
|
||||
|
||||
Ten nodes is heavy on a laptop. Start a subset by naming the services (always
|
||||
include `postgres`, `console`, and `caddy`):
|
||||
**Cluster** — 10-node server fleet sharing PostgreSQL. Access all nodes via the console at `:8090`. Requires `POSTGRES_PASSWORD`:
|
||||
|
||||
```bash
|
||||
docker compose up postgres console caddy channel node-1 node-2 node-3
|
||||
docker compose --profile cluster up
|
||||
```
|
||||
|
||||
## Why HTTPS-only?
|
||||
|
||||
The console's plain-HTTP port (8090) is **not** published to the host. A plain
|
||||
HTTP/1.1 origin caps the browser at 6 connections, which starves the
|
||||
dashboard's per-pane SSE streams. Caddy serves the browser over HTTP/2
|
||||
(multiplexed) and proxies to `console:8090` on the internal network, so the cap
|
||||
is gone. Everything goes through `https://localhost:8443`.
|
||||
|
||||
## Join a bare-metal host
|
||||
|
||||
PostgreSQL, the console's ACME endpoint (`:8090`), and SearxNG (`:8081`) are
|
||||
published on `127.0.0.1`, so a `turnstone-server` running directly on the same
|
||||
machine — for example to use a local GPU — can join the same cluster (enrolling
|
||||
its mTLS cert and running `web_search`) and show up in the console alongside the
|
||||
containerized nodes.
|
||||
|
||||
Put the secret and connection settings in `~/.config/turnstone/config.toml`
|
||||
(secrets belong in this file, not the process environment — keep it `0600`,
|
||||
the loader warns otherwise):
|
||||
|
||||
```toml
|
||||
[auth]
|
||||
jwt_secret = "dev-only-insecure-jwt-secret-change-me-for-real-deployments"
|
||||
|
||||
[database]
|
||||
backend = "postgresql"
|
||||
url = "postgresql+psycopg://turnstone:turnstone@localhost:5432/turnstone"
|
||||
|
||||
[api]
|
||||
base_url = "http://localhost:8000/v1" # your local model endpoint
|
||||
api_key = "dummy"
|
||||
```
|
||||
|
||||
Then start the server. The node identity isn't a secret, so it stays on the
|
||||
command line:
|
||||
|
||||
```bash
|
||||
chmod 600 ~/.config/turnstone/config.toml
|
||||
TURNSTONE_NODE_ID=host-1 \
|
||||
TURNSTONE_ADVERTISE_URL=http://host.docker.internal:8080 \
|
||||
TURNSTONE_CONSOLE_URL=http://localhost:8090 \
|
||||
TURNSTONE_SEARXNG_URL=http://localhost:8081 \
|
||||
turnstone-server --host 0.0.0.0 --port 8080
|
||||
```
|
||||
|
||||
The host server registers itself in PostgreSQL; the console reaches it back via
|
||||
`host.docker.internal`. `TURNSTONE_CONSOLE_URL` points the node at the console's
|
||||
published ACME endpoint so it can enroll its mTLS certificate (needed only when
|
||||
the cluster runs mTLS; harmless otherwise), and `TURNSTONE_SEARXNG_URL` points
|
||||
`web_search` at the published SearxNG. The `jwt_secret` and DB credentials above
|
||||
are the dev-stack defaults — match whatever you set in `.env` if you changed them.
|
||||
|
||||
To let a server on a **different** machine join, start the stack with
|
||||
`TURNSTONE_HOST_IP=<this host's LAN IP>` — that binds PostgreSQL, the console
|
||||
ACME endpoint, and SearxNG to that interface. Then on the remote box set the
|
||||
three URLs above to that IP, and set `TURNSTONE_ADVERTISE_URL` to the **remote**
|
||||
box's own IP (the address the console dials back). **Set a strong
|
||||
`POSTGRES_PASSWORD` first** — `TURNSTONE_HOST_IP` exposes the database (and every
|
||||
user account + API-token hash in it), the console API, and the unauthenticated
|
||||
SearxNG to your network.
|
||||
|
||||
To run the bare-metal node as a hardened, persistent service instead of by hand,
|
||||
use the systemd units in [`deploy/systemd/`](../deploy/systemd/).
|
||||
|
||||
## Production stack
|
||||
|
||||
For a real deployment use the bundled stack, which pulls released images
|
||||
instead of building:
|
||||
|
||||
```bash
|
||||
docker compose -f turnstone/deploy/compose.yaml up
|
||||
```
|
||||
|
||||
It's the same shape as the dev stack — Caddy-fronted console, channel, and a
|
||||
PostgreSQL all share one database so the console discovers the node — but it
|
||||
pulls released images, runs a single server node, and has **no baked-in
|
||||
secrets**. Set these in `.env` first (generate with `openssl rand -hex 32`):
|
||||
|
||||
```bash
|
||||
TURNSTONE_JWT_SECRET=<python -c "import secrets; print(secrets.token_hex(32))">
|
||||
POSTGRES_PASSWORD=<a strong password>
|
||||
```
|
||||
|
||||
The dashboard is at **https://localhost:8443** (Caddy, same as the dev stack);
|
||||
the console's HTTP port isn't published. For a real domain and a publicly
|
||||
trusted cert, edit `turnstone/deploy/Caddyfile` to point Caddy at Let's Encrypt
|
||||
(see [tls.md](tls.md)). Pin the image with `TURNSTONE_IMAGE_TAG` (default:
|
||||
`latest`).
|
||||
|
||||
### mTLS
|
||||
|
||||
Layer the TLS overlay on the production stack to enable mutual TLS between
|
||||
services. A bootstrap container creates a CA and every service auto-provisions
|
||||
certs via the console's ACME endpoint:
|
||||
|
||||
```bash
|
||||
docker compose -f turnstone/deploy/compose.yaml -f deploy/docker-compose.tls.yml up
|
||||
```
|
||||
|
||||
See [tls.md](tls.md) for details.
|
||||
|
||||
## Configuration
|
||||
|
||||
Everything is configured with environment variables in `.env` (copy from
|
||||
[`.env.example`](../.env.example)). The dev stack needs none of them — they're
|
||||
overrides.
|
||||
All configuration is via environment variables in `.env` (copy from `.env.example`):
|
||||
|
||||
### LLM backend
|
||||
### LLM Backend
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `LLM_BASE_URL` | `http://host.docker.internal:8000/v1` | Bootstrap OpenAI-compatible API URL (real backends go in the UI) |
|
||||
| `LLM_BASE_URL` | `http://host.docker.internal:8000/v1` | OpenAI-compatible API URL |
|
||||
| `OPENAI_API_KEY` | `dummy` | API key (`dummy` for local servers) |
|
||||
| `TURNSTONE_SEARXNG_URL` | `http://searxng:8080` | SearxNG URL for the `web_search` tool (local/vLLM models only; Anthropic/OpenAI use native search). Defaults to the bundled `searxng` service; set to an external instance's URL. To turn web search off, clear `tools.searxng_url` in the admin Settings tab. |
|
||||
| `SEARXNG_IMAGE_TAG` | `latest` | Tag for the bundled `searxng/searxng` image |
|
||||
| `MODEL` | — | Override the default model alias |
|
||||
| `TAVILY_API_KEY` | — | Web search API key (only needed for local/vLLM models; Anthropic and OpenAI search models use native search) |
|
||||
|
||||
### Auth & database
|
||||
|
||||
| Variable | Default (dev / prod) | Description |
|
||||
|----------|----------------------|-------------|
|
||||
| `TURNSTONE_JWT_SECRET` | insecure default / **required** | JWT signing secret. Every service must share one value. |
|
||||
| `TURNSTONE_DB_BACKEND` | `postgresql` | `sqlite` or `postgresql`. Multi-node discovery requires `postgresql`. |
|
||||
| `TURNSTONE_DB_URL` | bundled Postgres | SQLAlchemy URL. Override to use an external database. |
|
||||
| `POSTGRES_USER` | `turnstone` | PostgreSQL username |
|
||||
| `POSTGRES_PASSWORD` | `turnstone` / **required** | PostgreSQL password |
|
||||
| `POSTGRES_MAX_CONNECTIONS` | `300` | `max_connections` for the bundled Postgres |
|
||||
|
||||
> **Discovery needs a shared database.** Each server registers and heartbeats
|
||||
> into a `services` table that the console polls. All services in these stacks
|
||||
> point at the same PostgreSQL by default; SQLite-per-container can't see other
|
||||
> containers.
|
||||
|
||||
> **Large clusters:** each process keeps a small pool (5 max). Beyond ~50 nodes,
|
||||
> put [PgBouncer](pgbouncer.md) (transaction pooling) between turnstone and
|
||||
> PostgreSQL.
|
||||
|
||||
### Ports
|
||||
|
||||
Both stacks publish Caddy (dashboard) and PostgreSQL; the dev stack additionally
|
||||
publishes the console's ACME endpoint and SearxNG on localhost so a bare-metal
|
||||
node can enroll its cert and run `web_search`. Everything else is reached through
|
||||
Caddy or proxied by the console:
|
||||
### Server
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `CONSOLE_HTTPS_PORT` | `8443` | Host port for Caddy (dashboard HTTPS) |
|
||||
| `SEARXNG_HTTPS_PORT` | `8444` | Host port for the SearxNG UI via Caddy (dev: localhost-only; prod: opt-in) |
|
||||
| `POSTGRES_PORT` | `5432` | Host port for PostgreSQL (for bare-metal joins) |
|
||||
| `SEARXNG_API_PORT` | `8081` | Host port for the SearxNG API a bare-metal node's `web_search` dials (dev stack) |
|
||||
| `TURNSTONE_HOST_IP` | `127.0.0.1` | Interface PostgreSQL, the console ACME endpoint, and SearxNG bind on (dev stack). Set to this host's LAN IP so a bare-metal node on **another machine** can reach them — set a strong `POSTGRES_PASSWORD` first (it also exposes the DB and the unauthenticated SearxNG to your network). |
|
||||
| `POSTGRES_BIND` | `127.0.0.1` | Production stack (`turnstone/deploy/compose.yaml`) only: interface PostgreSQL binds on; set to the host's LAN IP for remote joins. |
|
||||
| `SERVER_PORT` | `8080` | Host port mapping |
|
||||
| `SKIP_PERMISSIONS` | — | Set to any value to auto-approve all tools |
|
||||
|
||||
### Channel gateway
|
||||
### Console
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (enables the Discord adapter) |
|
||||
| `TURNSTONE_DISCORD_GUILD` | `0` | Restrict to one guild (0 = all) |
|
||||
| `TURNSTONE_SLACK_TOKEN` | — | Slack Bot User OAuth token `xoxb-…` |
|
||||
| `TURNSTONE_SLACK_APP_TOKEN` | — | Slack App-Level token `xapp-…` (with the Slack token) |
|
||||
| `CONSOLE_PORT` | `8090` | Host port mapping |
|
||||
|
||||
The channel runs HTTP-only with no adapters until a token is set, so it's safe
|
||||
to leave running. See [Channel Integrations](channels.md) for app setup.
|
||||
### Auth
|
||||
|
||||
### Web search (SearxNG)
|
||||
|
||||
The `web_search` tool for local/vLLM models is backed by a self-hosted
|
||||
[SearxNG](https://searxng.org) metasearch service, bundled into both stacks as the
|
||||
`searxng` service. The Turnstone nodes reach it over the internal docker network at
|
||||
`http://searxng:8080` — its API port is **not** published. Its config —
|
||||
[`turnstone/deploy/searxng/settings.yml`](../turnstone/deploy/searxng/settings.yml),
|
||||
mounted read-only — enables the JSON API and leaves the rate limiter off (the
|
||||
limiter would need a separate Valkey/Redis instance). A `searxng-cache` volume
|
||||
persists its favicon + internal cache across restarts. Commercial providers
|
||||
(Anthropic, OpenAI) use their own native search and never touch this service.
|
||||
|
||||
Point at an existing SearxNG instead of the bundled one with `TURNSTONE_SEARXNG_URL`,
|
||||
or narrow the engines via `tools.searxng_engines` in the admin Settings tab (e.g.
|
||||
`duckduckgo,wikipedia`).
|
||||
|
||||
**SearxNG web UI.** Caddy can also serve SearxNG's own search/Preferences UI on a
|
||||
dedicated port. The dev stack publishes it at **`https://localhost:8444`** bound to
|
||||
localhost only; the production stack does **not** publish it by default (uncomment
|
||||
the `8444` port on the `caddy` service to opt in). Change the port with
|
||||
`SEARXNG_HTTPS_PORT`. **SearxNG has no authentication** — never bind this to a public
|
||||
interface, or anyone who can reach it can search through your instance.
|
||||
|
||||
> **AGPL note for operators.** SearxNG is licensed AGPL-3.0. Kept on the internal
|
||||
> network (or bound to localhost), no external user interacts with it — so the AGPL
|
||||
> §13 (remote network interaction) source-offer obligation does not attach. If you
|
||||
> publish SearxNG to remote users (bind its port to a public interface, or front it
|
||||
> with your own reverse proxy) you become the operator of a network-reachable AGPL
|
||||
> service and must offer its corresponding source; that is trivially satisfied by
|
||||
> linking to upstream <https://github.com/searxng/searxng>. Turnstone's own license is
|
||||
> unaffected: it talks to SearxNG over HTTP as a separate process (mere aggregation),
|
||||
> not by linking.
|
||||
|
||||
### Other
|
||||
Auth is always enabled. `TURNSTONE_JWT_SECRET` is required.
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `WORKSPACE_MOUNT` | empty volume | Host directory bind-mounted at `/workspace` for the model to read/write |
|
||||
| `SKIP_PERMISSIONS` | — | Set to any value to auto-approve all tool calls (dev only) |
|
||||
| `MCP_CONFIG` | — | Path to an MCP server config file |
|
||||
| `TURNSTONE_IMAGE_TAG` | `latest` | ghcr.io image tag — production stack |
|
||||
| `TURNSTONE_JWT_SECRET` | — | Secret key for signing JWTs (required) |
|
||||
|
||||
## Building
|
||||
### Database
|
||||
|
||||
Both stacks install all entry points into a single image (`turnstone`,
|
||||
`turnstone-server`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`,
|
||||
`turnstone-eval`, `turnstone-optimizer`, `turnstone-doctor`):
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DB_BACKEND` | `sqlite` | Storage backend: `sqlite` or `postgresql` |
|
||||
| `TURNSTONE_DB_URL` | — | Database URL (e.g. `postgresql://user:pass@db:5432/turnstone`). For SQLite, defaults to `/data/.turnstone.db` |
|
||||
| `TURNSTONE_DB_POOL_SIZE` | `2` | PostgreSQL connection pool size per process (default: 2 base + 3 overflow = 5 max) |
|
||||
|
||||
The database stores workstream history, user accounts, and API tokens. When using JWT auth, a database backend is required for user storage.
|
||||
|
||||
> **Large clusters:** Each turnstone process maintains a small connection pool (5 max). At hundreds of nodes this adds up — use [PgBouncer](pgbouncer.md) in transaction pooling mode between turnstone and PostgreSQL.
|
||||
|
||||
> **First-time setup:** After deploying with auth enabled, create an initial admin user by running `turnstone-admin create-user` inside the container:
|
||||
>
|
||||
> ```bash
|
||||
> docker compose exec server turnstone-admin create-user --username admin --name "Admin"
|
||||
> ```
|
||||
>
|
||||
> You will be prompted to set a password. Use it to log in via the UI or SDK, then create additional users through the admin API. Pass `--token --scopes read,write,approve` to also generate an initial API token.
|
||||
|
||||
### Channel Gateway
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord adapter) |
|
||||
| `TURNSTONE_DISCORD_GUILD` | `0` | Restrict to a single Discord guild (0 = all guilds) |
|
||||
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages to the server via HTTP. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
## Scaling
|
||||
|
||||
For multi-node testing, use the `cluster` profile which provides 10 server instances with unique node IDs (`node-1` through `node-10`), resource limits, and shared PostgreSQL:
|
||||
|
||||
```bash
|
||||
docker compose build # build the dev image
|
||||
docker compose build --no-cache # rebuild from scratch
|
||||
POSTGRES_PASSWORD=secret docker compose --profile cluster up
|
||||
```
|
||||
|
||||
The default `server` also runs alongside the cluster nodes (11 total). All nodes are accessible via the console dashboard at `:8090`.
|
||||
|
||||
For production clusters beyond ~50 nodes, add PgBouncer between turnstone services and PostgreSQL. See [PgBouncer Connection Pooling](pgbouncer.md) for Docker Compose and Helm configuration.
|
||||
|
||||
## Volumes
|
||||
|
||||
| Volume | Purpose |
|
||||
|--------|---------|
|
||||
| `postgres-data` | PostgreSQL data directory |
|
||||
| `turnstone-data` | `/data` per node (SQLite fallback, local state) |
|
||||
| `workspace` | `/workspace` (unless `WORKSPACE_MOUNT` is set) |
|
||||
| `caddy-data` / `caddy-config` | Caddy's local CA and config (dev stack) |
|
||||
| Volume | Mount | Purpose |
|
||||
|--------|-------|---------|
|
||||
| `turnstone-data` | `/data` | SQLite database (`.turnstone.db`) |
|
||||
|
||||
## Building
|
||||
|
||||
The image uses a multi-stage Dockerfile:
|
||||
|
||||
```bash
|
||||
# Build all services
|
||||
docker compose build
|
||||
|
||||
# Rebuild without cache
|
||||
docker compose build --no-cache
|
||||
```
|
||||
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-eval`.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
docker compose down # stop and remove containers
|
||||
docker compose down -v # also remove volumes (database, certs)
|
||||
# Stop and remove containers
|
||||
docker compose down
|
||||
|
||||
# Stop, remove containers and volumes
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
+26
-56
@@ -1,19 +1,11 @@
|
||||
# Evaluation and Prompt Optimization (turnstone-eval, turnstone-optimizer)
|
||||
# Evaluation and Prompt Optimization (turnstone-eval)
|
||||
|
||||
Evaluation for turnstone is split into two commands:
|
||||
`turnstone-eval` is the evaluation and prompt optimization system for turnstone. It
|
||||
runs test cases against the LLM, scores tool call sequences against expected
|
||||
actions, and optionally uses a multi-agent pipeline to optimize the developer
|
||||
prompt and tool descriptions.
|
||||
|
||||
- **`turnstone-eval`** — the measurement substrate. Runs test cases against the LLM
|
||||
and scores tool call sequences against expected actions. A single measurement pass,
|
||||
no self-modification.
|
||||
- **`turnstone-optimizer`** — the prompt/tool optimizer. Loops over the measurement
|
||||
substrate, using a multi-agent pipeline (analyst, optimizer, observer, diversifier,
|
||||
tool optimizer) to edit the developer prompt and tool descriptions so more tests pass.
|
||||
|
||||
The dependency is strictly one-way: the optimizer consumes the eval substrate; the
|
||||
substrate never depends on the optimizer.
|
||||
|
||||
Source: `turnstone/eval/core.py` (measurement substrate), `turnstone/eval/cli.py`
|
||||
(the `turnstone-eval` CLI), `turnstone/optimizer.py` (the `turnstone-optimizer` CLI).
|
||||
Source: `turnstone/eval.py`
|
||||
|
||||
---
|
||||
|
||||
@@ -35,8 +27,8 @@ This approach (inspired by [Learning to Self-Evolve](https://arxiv.org/abs/2603.
|
||||
prevents irrecoverable collapse from bad edits — UCB naturally backtracks to
|
||||
high-scoring ancestors instead of following a linear chain.
|
||||
|
||||
The `turnstone-eval` command (or `turnstone-optimizer --no-optimize`) executes only
|
||||
steps 2-4: a single measurement pass over the root prompt, no optimization.
|
||||
When optimization is disabled (`--no-optimize`), only steps 2-4 execute
|
||||
(a single iteration evaluating the root node).
|
||||
|
||||
---
|
||||
|
||||
@@ -282,7 +274,8 @@ for iteration in 0..max_iterations:
|
||||
|
||||
### Phase 1: Analyst (`_run_analyst`)
|
||||
|
||||
A multi-turn agent with a `bash` tool for computing statistics. It receives per-case results with failure classifications and
|
||||
A multi-turn agent with `math` (Python) and `bash` tools for computing
|
||||
statistics. It receives per-case results with failure classifications and
|
||||
produces a structured diagnosis:
|
||||
|
||||
- **Failure patterns**: Shared root causes across failing cases
|
||||
@@ -460,46 +453,30 @@ structure is:
|
||||
|
||||
## CLI Usage
|
||||
|
||||
Two console scripts (installed as entry points), or the equivalent `python -m`
|
||||
invocations:
|
||||
|
||||
- `turnstone-eval` / `python -m turnstone.eval.cli` — measure only.
|
||||
- `turnstone-optimizer` / `python -m turnstone.optimizer` — optimize.
|
||||
|
||||
### Measure (`turnstone-eval`)
|
||||
The entry point is `turnstone-eval` (installed as a console script) or
|
||||
`python -m turnstone.eval`.
|
||||
|
||||
```
|
||||
turnstone-eval tests.json # one measurement pass, print scores
|
||||
turnstone-eval tests.json --prompt custom.txt # measure a custom prompt
|
||||
turnstone-eval tests.json --n-runs 5 # more runs per case
|
||||
turnstone-eval tests.json --parallel 4 # run cases across 4 workers
|
||||
turnstone-eval tests.json -v # verbose per-turn logging
|
||||
turnstone-eval tests.json # evaluate + optimize
|
||||
turnstone-eval tests.json --no-optimize # evaluate only (single iteration)
|
||||
turnstone-eval tests.json --n-runs 5 --max-iter 10 # more thorough evaluation
|
||||
turnstone-eval tests.json --prompt custom.txt # start from a custom prompt
|
||||
turnstone-eval tests.json --optimize-tools # optimize tool descriptions only
|
||||
turnstone-eval tests.json --diversify 10 # test with prompt variants
|
||||
turnstone-eval tests.json -v # verbose per-turn logging
|
||||
```
|
||||
|
||||
### Optimize (`turnstone-optimizer`)
|
||||
### Multi-model setup (local test model, cloud optimizer)
|
||||
|
||||
```
|
||||
turnstone-optimizer tests.json # evaluate + optimize
|
||||
turnstone-optimizer tests.json --no-optimize # single pass, no optimization
|
||||
turnstone-optimizer tests.json --n-runs 5 --max-iter 10 # more thorough optimization
|
||||
turnstone-optimizer tests.json --prompt custom.txt # start from a custom prompt
|
||||
turnstone-optimizer tests.json --optimize-tools # optimize tool descriptions only
|
||||
turnstone-optimizer tests.json --diversify 10 # test with prompt variants
|
||||
```
|
||||
|
||||
#### Multi-model setup (local test model, cloud optimizer)
|
||||
|
||||
```
|
||||
turnstone-optimizer tests.json \
|
||||
turnstone-eval tests.json \
|
||||
--base-url http://localhost:8000/v1 \
|
||||
--optimizer-base-url https://api.anthropic.com \
|
||||
--optimizer-model claude-sonnet-4-6 \
|
||||
--analyst-model claude-opus-4-6
|
||||
```
|
||||
|
||||
### Measurement Options
|
||||
|
||||
Accepted by **both** commands.
|
||||
### All Options
|
||||
|
||||
| Flag | Default | Description |
|
||||
|-------------------------|----------------------------|-------------|
|
||||
@@ -508,26 +485,19 @@ Accepted by **both** commands.
|
||||
| `--model` | auto-detect | Model name. Auto-detected from the API if not specified. |
|
||||
| `--prompt` | turnstone built-in prompt | Path to initial prompt text file. |
|
||||
| `--n-runs` | from tests.json or 3 | Number of runs per test case. |
|
||||
| `--max-iter` | 5 | Maximum optimization iterations. |
|
||||
| `--no-optimize` | false | Run evaluation only (sets max-iter to 1). |
|
||||
| `--temperature` | 0.7 | Sampling temperature. |
|
||||
| `--max-tokens` | 32768 | Max completion tokens. |
|
||||
| `--reasoning-effort` | `medium` | Reasoning effort: `low`, `medium`, or `high`. |
|
||||
| `--context-window` | 131072 | Context window size. |
|
||||
| `--output` | `eval_results.json` | Output results file path. |
|
||||
| `-v`, `--verbose` | false | Show detailed per-turn logging. |
|
||||
| `--explore-constant` | 1.414 (sqrt(2)) | UCB exploration constant C. |
|
||||
| `--test-timeout` | 300 | Per-test timeout in seconds. |
|
||||
| `--suite-timeout` | 0 (unlimited) | Total suite timeout in seconds. |
|
||||
| `--no-fast-fail` | false | Disable early termination on all-zero initial runs. |
|
||||
| `--parallel` | 1 (serial) | Parallel workers (0=auto, N=use N workers). |
|
||||
|
||||
### Optimizer Options
|
||||
|
||||
Accepted by **`turnstone-optimizer`** only.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|-------------------------|----------------------------|-------------|
|
||||
| `--max-iter` | 5 | Maximum optimization iterations. |
|
||||
| `--no-optimize` | false | Run a single measurement pass (sets max-iter to 1). |
|
||||
| `--explore-constant` | 1.414 (sqrt(2)) | UCB exploration constant C. |
|
||||
| `--suite-timeout` | 0 (unlimited) | Total suite timeout in seconds. |
|
||||
| `--optimizer-model` | same as `--model` | Model for prompt optimization. |
|
||||
| `--optimizer-base-url` | same as `--base-url` | Base URL for optimizer model. |
|
||||
| `--observer-model` | same as optimizer | Model for meta-optimization (observer). |
|
||||
|
||||
+12
-26
@@ -13,7 +13,7 @@ The permission model has two layers:
|
||||
|
||||
1. **Scopes** (legacy) — `read`, `write`, `approve`. Checked by `AuthMiddleware`
|
||||
on every request based on URL path classification.
|
||||
2. **Permissions** (granular) — named permission strings checked per-endpoint by
|
||||
2. **Permissions** (granular) — 15 permission strings checked per-endpoint by
|
||||
`require_permission()`.
|
||||
|
||||
**Built-in roles** (seeded by migration 008):
|
||||
@@ -24,11 +24,7 @@ The permission model has two layers:
|
||||
| operator | read, write, workstreams.create, workstreams.close |
|
||||
| viewer | read |
|
||||
|
||||
Custom roles can be created with any subset of the valid permissions.
|
||||
The `persona.create` / `persona.read` / `persona.write` family gates
|
||||
persona administration; migration `063` seeds all three onto
|
||||
`builtin-admin`, and any role can be granted them through the standard
|
||||
role and permission-override editors.
|
||||
Custom roles can be created with any subset of the 15 valid permissions.
|
||||
|
||||
**Auth flow:**
|
||||
1. User logs in (password or API token) → `_load_user_permissions()` aggregates
|
||||
@@ -66,14 +62,14 @@ etc.) since workstream templates were merged into the skills system in v0.8.0.
|
||||
- **Default skills**: All `is_default=true` skills auto-apply to new
|
||||
workstreams, concatenated in alphabetical order by name. Use name prefixes
|
||||
(e.g. `01-safety`, `02-style`) to control ordering.
|
||||
- **Explicit selection**: `--skill <name>` CLI flag, `skill` field on
|
||||
- **Explicit selection**: `--template <name>` CLI flag, `template` field on
|
||||
`POST /v1/api/workstreams/new`, console creation modal dropdown, scheduled task
|
||||
config, and channel adapter config. An explicit skill *replaces* defaults.
|
||||
- **Variables**: Three built-in placeholders resolved at load time:
|
||||
`{{model}}` (active model name), `{{ws_id}}` (workstream ID),
|
||||
`{{node_id}}` (server node ID). Unrecognized placeholders are kept as-is.
|
||||
- **Runtime switching**: `/skill <name>` to switch, `/skill clear` to revert
|
||||
to defaults, `/skill` to show current. Persisted across resume.
|
||||
- **Runtime switching**: `/template <name>` to switch, `/template clear` to revert
|
||||
to defaults, `/template` to show current. Persisted across resume.
|
||||
- **Model-driven loading**: The `skill` built-in tool lets the model
|
||||
discover and activate skills mid-conversation. `search` action finds skills
|
||||
by query (auto-approved); `load` action activates by name (requires user
|
||||
@@ -95,7 +91,7 @@ etc.) since workstream templates were merged into the skills system in v0.8.0.
|
||||
time. The scanner evaluates four risk axes: content risk (command execution,
|
||||
data exfiltration), supply chain risk (pipe-to-shell, transitive installs),
|
||||
vulnerability risk (prompt injection, insecure credentials), and declared
|
||||
capability risk (from `allowed-tools` in SKILL.md). Results populate the `risk_level`
|
||||
capability risk (from `allowed-tools` in SKILL.md). Results populate the `scan_status`
|
||||
(safe/low/medium/high/critical) and `scan_report` (JSON breakdown) columns.
|
||||
These fields are system-managed and cannot be overwritten via the admin API.
|
||||
- **Discovery**: External skills can be discovered and installed from registries:
|
||||
@@ -131,12 +127,9 @@ Per-LLM-request token and tool call metrics:
|
||||
LLM response with prompt/completion tokens, cache tokens, tool call count,
|
||||
model, ws_id
|
||||
- **Prompt caching**: Anthropic automatic caching (`cache_control: ephemeral`)
|
||||
and OpenAI caching are enabled by default. Pre-5.6 GPT-5 models request
|
||||
`prompt_cache_retention: 24h`; GPT-5.6 uses
|
||||
`prompt_cache_options: {"ttl": "30m"}`. GPT-5.6 cache writes use the
|
||||
provider's 1.25× input-token rate. `cache_creation_tokens` and
|
||||
`cache_read_tokens` are tracked per request in `usage_events` and surfaced
|
||||
in the Usage admin tab
|
||||
and OpenAI extended retention (`prompt_cache_retention: 24h` for GPT-5.x)
|
||||
are enabled by default. `cache_creation_tokens` and `cache_read_tokens` are
|
||||
tracked per request in `usage_events` and surfaced in the Usage admin tab
|
||||
- **Querying**: `GET /v1/api/admin/usage` with `group_by` (day/hour/model/user)
|
||||
and time range filtering — includes cache token aggregates
|
||||
- **Prometheus**: `turnstone_tokens_total{type="cache_creation|cache_read"}`
|
||||
@@ -184,7 +177,6 @@ All under `/v1/api/admin/` (requires `approve` scope + granular permission).
|
||||
| Orgs | 3 (list, get, update) | `admin.orgs` |
|
||||
| Tool Policies | 4 (CRUD) | `admin.policies` |
|
||||
| Skills | 4 (CRUD) | `admin.skills` |
|
||||
| Personas | 4 (list, create, get, edit/archive) | `persona.read` / `persona.create` / `persona.write` |
|
||||
| Schedules | 6 (CRUD + runs) | `admin.schedules` |
|
||||
| Watches | 3 (list, create, cancel) | `admin.watches` |
|
||||
| Usage | 1 (aggregated query) | `admin.usage` |
|
||||
@@ -194,21 +186,15 @@ Full OpenAPI spec at `/openapi.json` and Swagger UI at `/docs`.
|
||||
|
||||
## Admin Console UI
|
||||
|
||||
Governance-related tabs within the 18-tab admin panel:
|
||||
6 new tabs added to the admin panel (11 total):
|
||||
|
||||
- **Roles** — CRUD roles, permission checkbox grid, user role assignment modal
|
||||
- **Policies** — CRUD tool policies with colored action badges (green/red/amber)
|
||||
- **Prompts** — Prompt-policy editor (heuristics for admin guardrails)
|
||||
- **Skills** — CRUD skills with wide modal, textarea editor; Discover pill for
|
||||
installing from skills.sh / GitHub; per-row scan badges (safe/low/med/high/critical)
|
||||
- **Judge** — Intent validation configuration and verdict history
|
||||
- **Skills** — CRUD skills with wide modal, textarea editor
|
||||
- **Usage** — Summary readouts + CSS bar chart, time range + group-by selectors
|
||||
- **Audit** — Filterable log with relative timestamps, load-more pagination
|
||||
|
||||
Tabs are permission-gated: hidden if the user lacks the required permission.
|
||||
See [docs/console.md](console.md) for the full tab list and
|
||||
[docs/settings.md](settings.md) for the Settings tab that edits live
|
||||
ConfigStore values.
|
||||
|
||||
## SDK
|
||||
|
||||
@@ -230,7 +216,7 @@ Both Python and TypeScript console SDKs expose governance methods:
|
||||
- **Privilege escalation prevented**: `admin_assign_role` blocks self-assignment
|
||||
and requires caller to hold a superset of the target role's permissions
|
||||
- **Permission validation**: Role create/update validates permissions against
|
||||
the permission allowlist (`_VALID_PERMISSIONS`)
|
||||
a 15-item allowlist (`_VALID_PERMISSIONS`)
|
||||
- **Self-deletion blocked**: `admin_delete_user` rejects attempts to delete
|
||||
your own account (matching the self-assignment guard on role endpoints)
|
||||
- **Field allowlists**: Storage `update_*` methods filter fields against
|
||||
|
||||
+24
-134
@@ -37,31 +37,13 @@ model = "" # empty = same as session model
|
||||
provider = "" # empty = same as session provider
|
||||
base_url = ""
|
||||
api_key = ""
|
||||
smart_approvals = false # auto-approve high-confidence "approve" LLM verdicts (opt-in)
|
||||
confidence_threshold = 0.95 # Smart Approvals auto-approve bar (LLM recommendation=approve)
|
||||
confidence_threshold = 0.7 # reserved for v2 smart approvals (not used in v1)
|
||||
max_context_ratio = 0.5 # max % of judge context window for history
|
||||
timeout = 120.0 # seconds (generous for local models)
|
||||
timeout = 60.0 # seconds (generous for local models)
|
||||
read_only_tools = true # judge can use read_file/list_directory
|
||||
cancel_on_approval = false # stop judging remaining tool calls once user decides
|
||||
```
|
||||
|
||||
### Smart Approvals
|
||||
|
||||
With `smart_approvals = true` (off by default) a tool call is approved
|
||||
automatically — no operator prompt — when the intent judge's **LLM** verdict
|
||||
recommends `approve` with confidence at or above `confidence_threshold`. Every
|
||||
other outcome still reaches a human: `review` / `deny` recommendations,
|
||||
confidence below the threshold, judge errors or timeouts (`llm_fallback`), and
|
||||
any call the deterministic heuristic rules explicitly flagged `deny` or
|
||||
`critical`. That heuristic floor blocks only those explicit danger verdicts — it
|
||||
is **not** a general "never lower the heuristic" rule: the heuristic's default
|
||||
for an unmatched tool is `review`, and letting a confident LLM `approve` upgrade
|
||||
a `review` is exactly what Smart Approvals is for. Only `deny` / `critical`
|
||||
findings are off-limits to auto-approval. Requires the judge to be enabled;
|
||||
auto-approved calls are tagged `smart_approval` in the dashboard and audit trail.
|
||||
Smart Approvals applies to the web and coordinator surfaces, not the interactive
|
||||
CLI.
|
||||
|
||||
All fields are optional. The judge is enabled by default; use `enabled = false`
|
||||
(or `--no-judge` on the command line) to disable it.
|
||||
|
||||
@@ -71,13 +53,10 @@ All fields are optional. The judge is enabled by default; use `enabled = false`
|
||||
--judge / --no-judge Enable/disable (default: enabled)
|
||||
--judge-model MODEL Model for judge
|
||||
--judge-provider PROVIDER Provider for judge
|
||||
--judge-timeout SECONDS LLM judge timeout (default: 120)
|
||||
--judge-confidence FLOAT Confidence threshold, 0-1 (default: 0.95)
|
||||
--judge-timeout SECONDS LLM judge timeout (default: 60)
|
||||
--judge-confidence FLOAT Confidence threshold (default: 0.7)
|
||||
```
|
||||
|
||||
(Smart Approvals is configured via `[judge] smart_approvals` / the admin Judge
|
||||
settings, not a CLI flag — the interactive CLI prompts for approval directly.)
|
||||
|
||||
CLI flags override `config.toml` values.
|
||||
|
||||
---
|
||||
@@ -125,7 +104,7 @@ last) and returns the first matching rule. Each rule has:
|
||||
| Critical | 0.90 | deny | `rm -rf /`, `mkfs`, `dd if=`, pipe-to-shell, chmod 777 on root, write/edit to `/etc/` or `.ssh/`, download-then-execute chains (`curl -o file && chmod +x && bash`) |
|
||||
| High | 0.80 | review | `sudo`, `kill -9`, destructive git, DROP TABLE, write/edit secrets, HTTP mutations, `ssh`/`scp`, credential file access, browser automation + data export, transitive installs (`npx skills add`, `pip install git+`), control plane mutations (`crontab`, `systemctl enable/start/stop`) |
|
||||
| Medium | 0.70 | review | Content ingestion pipelines (`curl \| python3`), interpreter execution (`python3 script.py`, `node build.js`), cloud CLI mutations (`az/gcloud/aws/kubectl/terraform` with create/delete/destroy verbs), package installs, `write_file`, MCP tools, Docker operations |
|
||||
| Low | 0.85 | approve | `read_file`, `list_directory`, `search`, `recall`, `use_prompt`, `tool_search`, `read_resource`, `web_search`, read-only bash (`ls`, `cat`, `head`, `grep`, `find`, etc.) |
|
||||
| Low | 0.85 | approve | `read_file`, `list_directory`, `search`, `recall`, `man`, `use_prompt`, `tool_search`, `read_resource`, `web_search`, read-only bash (`ls`, `cat`, `head`, `grep`, `find`, etc.) |
|
||||
|
||||
When no rule matches, the heuristic returns a default verdict: medium risk,
|
||||
0.50 confidence, "review" recommendation.
|
||||
@@ -193,10 +172,9 @@ Security hardening blocks access to sensitive paths:
|
||||
|
||||
### Timeout
|
||||
|
||||
The `timeout` setting (default 120 seconds) applies **per turn**, not as a total
|
||||
budget across turns — each of the up to 5 turns gets a fresh budget, so a slow
|
||||
earlier turn doesn't starve later ones. If a turn's budget expires, the judge
|
||||
attempts to parse whatever partial response is available.
|
||||
The `timeout` setting (default 60 seconds) is a total budget across all judge
|
||||
turns. Time is decremented after each LLM call. If the budget expires mid-turn,
|
||||
the judge attempts to parse whatever partial response is available.
|
||||
|
||||
---
|
||||
|
||||
@@ -232,31 +210,8 @@ calls for approval, it calls `_evaluate_intent()` which:
|
||||
4. Attaches each heuristic verdict to its item as `_heuristic_verdict`
|
||||
5. The daemon thread runs the LLM judge and delivers results via `ui.on_intent_verdict()`
|
||||
|
||||
The daemon evaluates items sequentially, so a large parallel batch can outlive
|
||||
its approval gate. With `cancel_on_approval = false` (the default) the daemon
|
||||
runs every item to completion: verdicts that land after the operator decided
|
||||
still stream to the UI and persist, stamped with the decision. The daemon is
|
||||
aborted only when the next tool batch supersedes it or the session closes —
|
||||
then each unfinished item degrades to an `llm_fallback` verdict. With
|
||||
`cancel_on_approval = true` the abort additionally fires the moment the gate
|
||||
resolves, trading verdict completeness for inference savings — recommended
|
||||
when the judge shares a single local inference backend with the session model,
|
||||
where a large batch's remaining judge calls would otherwise compete with the
|
||||
next turn's completion.
|
||||
|
||||
Verdicts that arrive after a *newer batch* has replaced the judge generation
|
||||
are withheld from the live surfaces (a reused call_id must never ride a stale
|
||||
`approve` into Smart Approvals) but still persist with
|
||||
`user_decision = "superseded"` so the audit trail records the judge's answer.
|
||||
|
||||
Sub-agent (task agent) tool calls are judge-gated too. Each runs the same
|
||||
intent pipeline as its own `agent_gate` generation, grounded in that sub-agent's
|
||||
own trajectory -- its task prompt is the delegation contract the operator
|
||||
approved, so "does this call serve the task" is the right local question.
|
||||
Agent-gate generations never occupy the main loop's supersede slot (parallel
|
||||
siblings would otherwise make each other's verdicts look stale); per-cycle
|
||||
generation checks enforce staleness instead, and `judge.cancel_on_approval`
|
||||
fires per gate exactly like the main loop.
|
||||
Sub-agents (plan agent, task agent) are exempt from intent validation -- they
|
||||
always get full tool visibility without judge evaluation.
|
||||
|
||||
---
|
||||
|
||||
@@ -266,13 +221,7 @@ All verdicts are persisted to the `intent_verdicts` table (migration 012):
|
||||
|
||||
- Heuristic verdicts are stored when the `approve_request` event is emitted
|
||||
- LLM verdicts are stored when the `intent_verdict` event is delivered
|
||||
- The `user_decision` column is updated when the user approves or denies;
|
||||
auto-approved rows carry the bypass reason (`policy`, `blanket`,
|
||||
`auto_approve_tools`, `smart_approval`), and rows whose verdict landed only
|
||||
after a newer batch replaced the judge generation carry `superseded`
|
||||
- Every stored verdict — including the benign `risk_level = "none"` majority —
|
||||
is re-attached to its tool call on history replay, so a reloaded workstream
|
||||
shows the same verdict badges the live stream did
|
||||
- The `user_decision` column is updated when the user approves or denies
|
||||
|
||||
The console admin panel exposes verdict history via:
|
||||
|
||||
@@ -366,7 +315,7 @@ four independent risk axes:
|
||||
`Bash(*)` (unrestricted shell) is high risk. `Bash(git:*)` is low.
|
||||
Read-only tools are safe.
|
||||
|
||||
Results are stored in `risk_level` (tier: safe/low/medium/high/critical) and
|
||||
Results are stored in `scan_status` (tier: safe/low/medium/high/critical) and
|
||||
`scan_report` (JSON breakdown) on the `prompt_templates` table. These fields are
|
||||
system-managed and not editable via the admin API.
|
||||
|
||||
@@ -423,36 +372,10 @@ redact_secrets = true # auto-redact detected credentials (default)
|
||||
|
||||
Configurable at runtime via the admin Settings tab.
|
||||
|
||||
### Merge semantics (heuristic + LLM judge)
|
||||
|
||||
The chip is a **merge** of the two detectors (issue #560, "show, annotated"),
|
||||
not a winner-take-all:
|
||||
|
||||
- `risk_level` = **max**(heuristic, llm) and `flags` = **union**. A positive
|
||||
from either detector surfaces; a negative ("none") or failed/absent LLM
|
||||
**never lowers** a heuristic positive. The judge reads adversarial tool
|
||||
output, so it may raise the alarm but must not be able to hide a
|
||||
deterministic regex finding — defeating the judge can't erase the tripwire.
|
||||
- Credential **redaction** is a heuristic-only signal the LLM cannot override.
|
||||
- When the judge returned a verdict, its OWN verdict rides along as
|
||||
annotation (`judge_risk` / `confidence` / `reasoning` / `judge_model`) so
|
||||
the operator sees the judge's opinion even when it disagrees with the
|
||||
displayed (merged) risk.
|
||||
|
||||
The same merge runs live and on reconnect (both call
|
||||
`output_guard.merge_guard_display_payload`), so the chip can't drift between
|
||||
the two surfaces.
|
||||
|
||||
The MODEL on the other side of the conversation is shown the merged
|
||||
`risk_level` + `flags` (via the `GuardAdvisory` spliced into the tool-result
|
||||
envelope), but is **never** told the judge cleared a finding — a judge fooled
|
||||
into "none" must not get to talk the model out of caution. The judge's
|
||||
"benign" verdict is operator-facing only.
|
||||
|
||||
### SSE event: `output_warning`
|
||||
|
||||
When the merged finding is non-clean (or credentials were redacted), an
|
||||
`output_warning` SSE event is emitted to the frontend. A regex-only finding:
|
||||
When the output guard detects risk signals, an `output_warning` SSE event is
|
||||
emitted to the frontend:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -463,58 +386,25 @@ When the merged finding is non-clean (or credentials were redacted), an
|
||||
"flags": ["credential_leak"],
|
||||
"annotations": ["API key detected (sk-proj-...)"],
|
||||
"output_length": 1024,
|
||||
"redacted": true,
|
||||
"tier": "heuristic"
|
||||
"redacted": true
|
||||
}
|
||||
```
|
||||
|
||||
When the LLM judge returned a verdict, `tier` is `"llm"` and the event carries
|
||||
the judge's own verdict as annotation. Here the regex flagged MEDIUM but the
|
||||
judge assessed the output benign — the finding still surfaces (`risk_level`
|
||||
stays MEDIUM), annotated with the judge's dissent (`judge_risk: "none"`):
|
||||
The web UI renders this as an inline warning after the tool result. The CLI
|
||||
shows a colored terminal warning. The server forwards it as an
|
||||
`OutputWarningEvent` for console subscribers.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "output_warning",
|
||||
"call_id": "call_def456",
|
||||
"func_name": "web_fetch",
|
||||
"risk_level": "medium",
|
||||
"flags": ["camouflaged_injection"],
|
||||
"annotations": ["Authority-framed directive embedded in the document."],
|
||||
"output_length": 8192,
|
||||
"redacted": false,
|
||||
"tier": "llm",
|
||||
"judge_risk": "none",
|
||||
"confidence": 0.92,
|
||||
"reasoning": "Legitimate analyst commentary; no injection.",
|
||||
"judge_model": "gpt-5-mini"
|
||||
}
|
||||
```
|
||||
|
||||
`judge_risk` (the judge's OWN risk verdict, which may differ from the merged
|
||||
`risk_level`), `confidence` (0.0–1.0), `reasoning`, and `judge_model` are
|
||||
present only on the `"llm"` tier. The identical shape is projected onto
|
||||
history replay by `build_merged_output_assessment_payload`, so the inline chip
|
||||
renders the same live and on refresh.
|
||||
|
||||
The web UI renders this as an inline warning after the tool result — the
|
||||
`"llm"` tier adds a `⚖ LLM · NN%` badge (showing the judge's verdict when it
|
||||
differs from the displayed risk, e.g. `⚖ LLM: none · 92%`) and the judge's
|
||||
rationale. The CLI shows a colored terminal warning. The server forwards it as
|
||||
an `OutputWarningEvent` for console subscribers.
|
||||
|
||||
Assessments are persisted to the `output_assessments` table (one row per
|
||||
`(call_id, tier)`) for calibration. Raw tool output is never stored — only
|
||||
metadata: flags, risk level, annotations, output length, redaction status,
|
||||
and — for the LLM tier — confidence, reasoning, judge model, and latency.
|
||||
Assessments are persisted to the `output_assessments` table for v2
|
||||
calibration. Raw tool output is never stored — only metadata (flags, risk
|
||||
level, annotations, output length, redaction status).
|
||||
|
||||
### Session-level skill scan warning
|
||||
|
||||
When a skill with `risk_level` of `high` or `critical` is loaded into a
|
||||
When a skill with `scan_status` of `high` or `critical` is loaded into a
|
||||
session, a warning is emitted via `on_info`:
|
||||
|
||||
```
|
||||
⚠ Skill 'my-skill' has risk level: high.
|
||||
⚠ Skill 'my-skill' has scan status: high.
|
||||
Review scan report in admin panel before enabling in production.
|
||||
```
|
||||
|
||||
@@ -531,7 +421,7 @@ All three evaluation systems persist their assessments for future calibration:
|
||||
|-------|--------|-------------|
|
||||
| `intent_verdicts` | Intent judge (heuristic + LLM) | `func_name`, `risk_level`, `confidence`, `user_decision` |
|
||||
| `output_assessments` | Output guard | `func_name`, `risk_level`, `flags`, `redacted` |
|
||||
| `prompt_templates` | Skill scanner | `risk_level`, `scan_report`, `scan_version` |
|
||||
| `prompt_templates` | Skill scanner | `scan_status`, `scan_report`, `scan_version` |
|
||||
|
||||
Run v1 with all tools requiring manual approval to build a local dataset.
|
||||
In v2, calibration tooling will analyze this data to:
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
# MCP OAuth — per-user authorization for MCP servers
|
||||
|
||||
Turnstone supports **per-(user, MCP server) OAuth 2.1 + PKCE** delegation so each Turnstone user authorizes a remote MCP server with their own identity, rather than sharing a single bearer token across the deployment. This is the right shape for MCP servers that expose user-specific data (a personal CRM, an email inbox, a calendar) and for MCP servers that want per-user audit attribution.
|
||||
|
||||
Per-user OAuth is opt-in per `mcp_servers` row. Local-auth Turnstone installs with no `oauth_user` rows exercise zero new code paths — the entire feature is dark by default.
|
||||
|
||||
> **Note**: This is a separate authorization layer from Turnstone's own user authentication. A user who logs into Turnstone with a local username + password can still authorize a per-server OAuth MCP server. OIDC SSO and per-server OAuth are orthogonal.
|
||||
|
||||
---
|
||||
|
||||
## When to use which `auth_type`
|
||||
|
||||
The MCP server admin form exposes three authorization modes ("Multitenant Authorization"):
|
||||
|
||||
| `auth_type` | What it means | When to use |
|
||||
|---|---|---|
|
||||
| `none` | No headers attached. Open MCP server (or one gated by network policy only). | Internal MCP servers on a trusted network. |
|
||||
| `static` | One static bearer token, configured per server, sent on every request from every user. | Service-to-service MCP servers where per-user attribution doesn't matter, or single-tenant deployments. |
|
||||
| `oauth_user` *(recommended for user-data servers)* | Each user authorizes separately via OAuth 2.1 + PKCE; Turnstone stores per-user tokens encrypted at rest. | MCP servers that expose user-specific data or that want per-user audit attribution. |
|
||||
|
||||
Switching `auth_type` away from `oauth_user` orphans existing per-user tokens. Use the admin **bulk-revoke** affordance on the server row (Phase 9) to clear them, or let them expire naturally — they're inert without the matching `auth_type` value.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites for `auth_type=oauth_user`
|
||||
|
||||
1. **Encryption key**. Tokens are stored encrypted with Fernet. Set `[security] mcp_token_encryption_key` in `config.toml` (Turnstone won't start with an `oauth_user` row configured but no key installed). Rotate via `MultiFernet` — add the new key first, then later remove the old one once all rows have been re-encrypted.
|
||||
|
||||
2. **MCP server publishes RFC 9728 PRM and RFC 8414 AS metadata** *or* you configure the AS URL override on the server row. PKCE S256 is mandatory; Turnstone refuses to connect to authorization servers that don't advertise `code_challenge_methods_supported: ["S256"]`.
|
||||
|
||||
3. **OAuth client registration**. Two paths:
|
||||
- **Pre-registered** (most common): you create an OAuth client at the authorization server (manually, via admin console, or via Terraform), then paste the `client_id` / `client_secret` into the Turnstone admin form.
|
||||
- **Dynamic client registration** (RFC 7591): if the AS supports it and you select that mode in the admin form, Turnstone registers a client at first use and persists the `client_id` automatically.
|
||||
|
||||
4. **Redirect URI** registered at the authorization server: `https://your-turnstone-host/v1/api/mcp/oauth/callback`.
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### Per-server fields (admin UI)
|
||||
|
||||
| Field | Required | Description |
|
||||
|---|---|---|
|
||||
| Server URL | Yes | The MCP server's `streamable-http` base URL. |
|
||||
| Multitenant Authorization | Yes | `none` / `static` / `oauth_user` (recommended). |
|
||||
| Authorization Server URL | No | Override for RFC 9728 PRM discovery. Set when your AS endpoint differs from the MCP server URL (e.g., corporate AS protecting a third-party MCP). When unset, Turnstone falls back to PRM discovery against the MCP server itself. |
|
||||
| Client Registration | Yes (oauth_user) | `preregistered` or `dynamic`. |
|
||||
| Client ID | Yes (preregistered) | OAuth 2.0 client ID. Stored unencrypted. |
|
||||
| Client Secret | Optional (write-only) | OAuth 2.0 client secret (confidential client). Encrypted at rest. Written but never re-read by the API; field stays masked. |
|
||||
| Scopes | No | Space-separated default scope set requested at the authorize endpoint. Per-tool step-up may union additional scopes from a server's `insufficient_scope` response. |
|
||||
| Audience | No | RFC 8707 `resource=` parameter sent on every authorize and token request. Defaults to the MCP server URL when unset. Validate against the `aud` claim in returned JWT tokens. |
|
||||
|
||||
### Encryption key
|
||||
|
||||
```toml
|
||||
[security]
|
||||
mcp_token_encryption_key = "base64-fernet-key"
|
||||
# For rotation, list the keys in priority order — first is used for new
|
||||
# writes, all are tried for reads.
|
||||
# mcp_token_encryption_keys = ["new-key", "old-key"]
|
||||
```
|
||||
|
||||
Keep this in `config.toml` rather than environment variables. An in-process LLM with shell-tool access can read the server's environment via `env` / `os.environ` and exfiltrate any secret stored there; secrets in `config.toml` are only loaded into the server at startup and never re-read on a tool-driven path, so a prompt-injection attack against the agent cannot reach them.
|
||||
|
||||
---
|
||||
|
||||
## Lifecycle
|
||||
|
||||
1. **First tool call** for a user against an `oauth_user` MCP server: pool dispatch finds no stored token, returns `mcp_consent_required` to the agent. Dashboard renders an inline "Connect" action card.
|
||||
|
||||
2. **User clicks Connect**: opens `/v1/api/mcp/oauth/start?server=<name>` in a popup. Browser redirects through the AS authorize endpoint, user grants consent, AS redirects back to `/v1/api/mcp/oauth/callback`. Turnstone exchanges code → tokens via PKCE, validates audience, encrypts, persists in `mcp_user_tokens`, redirects user back to the originating URL.
|
||||
|
||||
3. **Subsequent tool calls** by the same user against the same server reuse the persisted token via the per-(user, server) session pool. Tokens auto-refresh via the refresh-token grant when expired; failed refresh emits `mcp_consent_required` to drive re-consent.
|
||||
|
||||
4. **Step-up scope**: when a tool call hits `403` with `WWW-Authenticate: error="insufficient_scope"`, Turnstone emits `mcp_insufficient_scope` with the parsed scope set; the dashboard offers a "Connect with additional scopes" affordance that opens `/v1/api/mcp/oauth/start?server=<name>&scopes=<extra>` so the union of original + new scopes flows into the AS authorize request.
|
||||
|
||||
5. **User revoke** (settings modal): `DELETE /v1/api/mcp/oauth/connections/{server_name}` runs the authoritative local delete + best-effort RFC 7009 upstream revoke (fire-and-forget, capped at 256 concurrent in-flight tasks).
|
||||
|
||||
6. **Admin bulk-revoke** (Phase 9): `POST /v1/api/admin/mcp-servers/{name}/bulk-revoke` drops every user's token for the server. Upstream RFC 7009 revoke is intentionally **not** attempted in bulk (avoids N upstream HTTP calls per admin click); tokens at the AS expire naturally. Use the per-user revoke endpoint if you need guaranteed upstream invalidation.
|
||||
|
||||
---
|
||||
|
||||
## Admin status indicators
|
||||
|
||||
The MCP Servers admin tab shows per-server status pills (Phase 9):
|
||||
|
||||
- **Consented users count** — distinct users with a non-expired token for this server. Surfaced as a `bulk-revoke (N)` button when ≥1; clicking it opens a confirmation dialog. Hidden when 0.
|
||||
- **Last refresh** — timestamp + outcome (`ok` / `error:ClassName`) of the most recent manual or auto-reconnect refresh. Per node. Absent until at least one refresh has occurred (renders as "never" in the admin UI).
|
||||
|
||||
Additional indicators (circuit-breaker state, encryption-key mismatch) are exposed via `get_server_status` on the API but do not yet have a dedicated admin pill — operators see them today via the per-server status text + error tooltip and in audit logs. A future phase may surface these as discrete pills.
|
||||
|
||||
---
|
||||
|
||||
## Auth-type transitions
|
||||
|
||||
| From | To | What happens |
|
||||
|---|---|---|
|
||||
| `none` / `static` → `oauth_user` | — | New code path activates for this server. Existing static headers (if any) are no longer sent. Users must authorize on first use. |
|
||||
| `oauth_user` → `none` / `static` | — | Existing `mcp_user_tokens` rows are **orphaned** — inert without a matching `auth_type`. Use admin bulk-revoke to drop them, or let them expire. Switching back to `oauth_user` later re-activates the orphaned rows if they haven't been deleted. |
|
||||
| OAuth `client_id` or `client_secret` rotated | — | Existing tokens may stop refreshing if the AS treats them as bound to the previous client. Bulk-revoke after rotation. |
|
||||
|
||||
The orphan-by-default behavior is chosen so switching back to `oauth_user` is non-destructive. Bulk-revoke is the explicit cleanup path.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Likely cause | Action |
|
||||
|---|---|---|
|
||||
| `mcp_consent_required` even after consenting | Token persistence failed, or refresh-token rejected by AS | Check audit log for `mcp_server.oauth.persist_failed` or `mcp_server.oauth.token_revoked`. Re-consent via settings modal. |
|
||||
| `mcp_token_undecryptable_key_unknown` | Encryption key rotated without keeping the previous key in the keyring | Add the previous key back to `mcp_token_encryption_keys` until all rows have been re-encrypted, then drop. |
|
||||
| `mcp_oauth_url_insecure` | MCP server URL is `http://` (not `https://`) on a non-loopback host | Use `https://`. Per-user bearers must not transit cleartext. |
|
||||
| Tools fail in scheduled / Discord / Slack runs | OAuth-MCP requires browser-based consent | Users must pre-consent via the web UI. Phase 9 dashboard badge surfaces deferred consents from these runs on next login. |
|
||||
| Circuit breaker open repeatedly | Transport-level errors on the MCP server (DNS, TLS, 5xx) | Check the per-server error pill; auth errors do not trip the breaker. |
|
||||
|
||||
See also: `docs/operations/mcp-oauth-headless.md` for the cron / channel-driven run caveat.
|
||||
@@ -146,7 +146,7 @@ with TurnstoneConsole("http://localhost:8081", token="...") as client:
|
||||
### TypeScript
|
||||
|
||||
```typescript
|
||||
import { TurnstoneConsole } from "@turnstone/sdk";
|
||||
import { TurnstoneConsole } from "@anthropic/turnstone-sdk";
|
||||
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://localhost:8081",
|
||||
|
||||
+5
-35
@@ -26,40 +26,15 @@ Each memory has three dimensions:
|
||||
|
||||
### Memory scopes
|
||||
|
||||
| Scope | Visibility |
|
||||
|---------------|-----------------------------------------------------------------|
|
||||
| `global` | Visible to all workstreams and users |
|
||||
| `workstream` | Visible only within the originating workstream |
|
||||
| `user` | Follows the authenticated user across workstreams |
|
||||
| `coordinator` | Coordinator sessions only; follows the user across coordinators |
|
||||
| Scope | Visibility |
|
||||
|--------------|-----------------------------------------------------------|
|
||||
| `global` | Visible to all workstreams and users |
|
||||
| `workstream` | Visible only within the originating workstream |
|
||||
| `user` | Follows the authenticated user across workstreams |
|
||||
|
||||
A memory's identity is the tuple `(name, scope, scope_id)`. Saving a memory
|
||||
with the same identity upserts -- updating content while preserving the ID.
|
||||
|
||||
### Coordinator scope
|
||||
|
||||
Coordinator sessions are isolated to a single scope: `coordinator`, keyed by
|
||||
the coordinator's creator `user_id`. It is durable -- every coordinator
|
||||
session the same user runs (including concurrent ones) shares one
|
||||
orchestration namespace, so procedures and lessons survive close/reopen.
|
||||
|
||||
Isolation is bidirectional and enforced by session kind, not by secrecy of
|
||||
the scope id:
|
||||
|
||||
- A coordinator session can read and write **only** `coordinator`-scope rows.
|
||||
It never sees `global`/`workstream`/`user` memories, so content written by
|
||||
interactive sessions (which routinely ingest untrusted MCP/attachment
|
||||
output) cannot reach a coordinator's system message.
|
||||
- Interactive sessions -- including a coordinator's own children, which share
|
||||
its `user_id` -- are rejected from the `coordinator` scope on every memory
|
||||
action. Children cannot plant rows the parent coordinator would read.
|
||||
- The REST memory API (`/v1/api/memories`) does not accept the `coordinator`
|
||||
scope at all; the scope is written exclusively through a coordinator
|
||||
session's own memory tool.
|
||||
|
||||
Coordinator sessions require an authenticated user identity -- an anonymous
|
||||
coordinator cannot be constructed, so the scope id is always a real user.
|
||||
|
||||
### BM25 relevance injection
|
||||
|
||||
On every conversation turn, the system:
|
||||
@@ -75,11 +50,6 @@ This means the model always has its most relevant memories available without
|
||||
explicit recall -- but can still use `memory(action='search')` for deeper
|
||||
lookup.
|
||||
|
||||
The persona memory lever gates this pathway: a workstream whose persona
|
||||
turns memory off receives no relevance injection at all -- the steps
|
||||
above run only when memory is enabled for the session. See
|
||||
[Personas](personas.md).
|
||||
|
||||
### Nudges
|
||||
|
||||
The metacognition layer can nudge the model to save memories at appropriate
|
||||
|
||||
+13
-119
@@ -39,20 +39,18 @@ are set.
|
||||
| `TURNSTONE_OIDC_ROLE_CLAIM` | No | — | ID token claim containing role/group values (see [Role Mapping](#role-mapping)) |
|
||||
| `TURNSTONE_OIDC_ROLE_MAP` | No | — | Mapping from claim values to Turnstone role IDs (see [Role Mapping](#role-mapping)) |
|
||||
| `TURNSTONE_OIDC_PASSWORD_ENABLED` | No | `true` | Set to `false` to hide the password form and block all username/password logins (including admin). API tokens continue to work. |
|
||||
| `TURNSTONE_OIDC_REDIRECT_BASE` | Yes | — | Externally-reachable origin for the OIDC redirect URI (e.g. `https://app.example.com`). Without this, OIDC will refuse to start. The previous Host-header fallback was unsafe under permissive reverse proxies. |
|
||||
| `TURNSTONE_OIDC_TRUSTED_ENDPOINT_HOSTS` | No | — | Comma-separated list of additional hostnames whose endpoints the IdP discovery document is allowed to reference. See [Cross-host endpoints](#cross-host-endpoints). |
|
||||
| `TURNSTONE_OIDC_ALLOW_PRIVATE_NETWORK` | No | `false` | Allow the issuer (and its discovered endpoints) to resolve to private/internal addresses — needed for a self-hosted IdP on an internal network. See [Self-hosted and internal IdPs](#self-hosted-and-internal-idps). |
|
||||
| `TURNSTONE_OIDC_REDIRECT_BASE` | No | — | Externally-reachable origin for the OIDC redirect URI (e.g. `https://app.example.com`). Recommended when running behind a reverse proxy. When unset, derived from the request Host header. |
|
||||
|
||||
All four required fields — issuer, client ID, client secret, and
|
||||
`TURNSTONE_OIDC_REDIRECT_BASE` — must be set. If any are missing OIDC
|
||||
is disabled at startup (an error is logged when only `redirect_base`
|
||||
is missing) and the login screen shows only the password form.
|
||||
OIDC is enabled when all three required fields (issuer, client ID, client
|
||||
secret) are non-empty. If any is missing, OIDC is silently disabled and
|
||||
the login screen shows only the password form.
|
||||
|
||||
### Redirect base (required)
|
||||
### Reverse Proxy / Load Balancer
|
||||
|
||||
`TURNSTONE_OIDC_REDIRECT_BASE` pins the redirect URI sent to the identity
|
||||
provider to a known externally-visible origin. Set it to the public origin
|
||||
of your Turnstone deployment:
|
||||
When Turnstone runs behind a reverse proxy, the internal `Host` header may
|
||||
not match the externally-reachable URL. Set `TURNSTONE_OIDC_REDIRECT_BASE`
|
||||
to the public origin so the redirect URI sent to the identity provider is
|
||||
correct:
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_REDIRECT_BASE=https://app.example.com
|
||||
@@ -62,78 +60,6 @@ The resulting callback URL will be
|
||||
`https://app.example.com/v1/api/auth/oidc/callback` — register this as the
|
||||
authorized redirect URI in your identity provider.
|
||||
|
||||
OIDC will refuse to start when this variable is unset. There is no
|
||||
Host-header fallback: a permissive reverse proxy or direct backend access
|
||||
would otherwise let an attacker spoof `Host` and steer the IdP redirect
|
||||
to a callback origin they control.
|
||||
|
||||
### Cross-host endpoints
|
||||
|
||||
By default, every endpoint in the IdP discovery document
|
||||
(`token_endpoint`, `jwks_uri`, `userinfo_endpoint`) must share the
|
||||
issuer's `(scheme, host, port)`. This prevents a hostile or compromised
|
||||
IdP from redirecting the token-exchange POST (which carries
|
||||
`client_secret`) to an arbitrary host, and prevents JWKS fetches from
|
||||
being aimed at internal services.
|
||||
|
||||
A few public IdPs legitimately split endpoints across hostnames. Google
|
||||
is the canonical example:
|
||||
|
||||
| Field | Hostname |
|
||||
|-------|----------|
|
||||
| issuer | `accounts.google.com` |
|
||||
| token_endpoint | `oauth2.googleapis.com` |
|
||||
| jwks_uri | `www.googleapis.com` |
|
||||
| userinfo_endpoint | `openidconnect.googleapis.com` |
|
||||
|
||||
Google's set is built in — operators using `https://accounts.google.com`
|
||||
need no extra configuration.
|
||||
|
||||
For other IdPs whose discovery document references a non-issuer host,
|
||||
extend the allow-list explicitly:
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_TRUSTED_ENDPOINT_HOSTS=token.example.com,keys.example.com
|
||||
```
|
||||
|
||||
The same scheme / no-userinfo / SSRF rules apply to allow-listed hosts —
|
||||
this knob only relaxes the same-origin check, not the security gates.
|
||||
Each entry is a hostname (no scheme, no path).
|
||||
|
||||
### Self-hosted and internal IdPs
|
||||
|
||||
By default Turnstone refuses an issuer whose hostname resolves to a
|
||||
private or internal address:
|
||||
|
||||
```
|
||||
OIDCError: endpoint URL resolves to non-public address (10.0.0.5): https://auth.example.site
|
||||
```
|
||||
|
||||
This is SSRF hardening, not a licensing or product restriction: the OIDC
|
||||
flow makes server-side HTTP requests (discovery, JWKS, token exchange),
|
||||
and refusing non-public destinations keeps a mistyped or maliciously
|
||||
steered issuer from aiming those fetches at internal services. For a
|
||||
self-hosted IdP (Keycloak, Authentik, Dex, …) on a private network,
|
||||
opt in explicitly in `config.toml`:
|
||||
|
||||
```toml
|
||||
[oidc]
|
||||
allow_private_network = true
|
||||
```
|
||||
|
||||
or via `TURNSTONE_OIDC_ALLOW_PRIVATE_NETWORK=true` (the env var wins
|
||||
when both are set).
|
||||
|
||||
The opt-in admits private-range (RFC 1918), unique-local, CGNAT
|
||||
(100.64/10 — tailnets), and loopback addresses. Link-local, multicast,
|
||||
and reserved ranges stay refused even with the opt-in — cloud metadata
|
||||
services (169.254.169.254) live there, and no legitimate IdP does. The
|
||||
HTTPS requirement and the same-origin endpoint checks are unaffected.
|
||||
|
||||
This knob only affects the login-flow IdP configured here. OAuth
|
||||
endpoints advertised by remote MCP servers are untrusted input and are
|
||||
always held to the strict public-address rule.
|
||||
|
||||
### config.toml alternative
|
||||
|
||||
```toml
|
||||
@@ -146,8 +72,6 @@ provider_name = "Google"
|
||||
role_claim = "groups"
|
||||
password_enabled = true
|
||||
redirect_base = "https://app.example.com"
|
||||
# Self-hosted IdP on an internal network (see "Self-hosted and internal IdPs")
|
||||
allow_private_network = false
|
||||
|
||||
[oidc.role_map]
|
||||
admin = "builtin-admin"
|
||||
@@ -274,19 +198,6 @@ TURNSTONE_OIDC_ROLE_MAP="admin:builtin-admin,engineering:builtin-operator,viewer
|
||||
the user authenticates via OIDC, so new group memberships are picked
|
||||
up on the next login.
|
||||
|
||||
### `assigned_by` markers
|
||||
|
||||
Role assignments record an `assigned_by` value that controls how the
|
||||
sync logic treats them. OIDC-driven flows use two distinct markers:
|
||||
|
||||
- `oidc` — set by claim-driven role mapping; revoked automatically on
|
||||
the next login when the corresponding claim value is no longer
|
||||
present.
|
||||
- `oidc-default` — applied to brand-new OIDC users who have no
|
||||
claim-mapped roles, as a safety net so they still get
|
||||
`builtin-viewer` access on first login. Survives subsequent logins
|
||||
regardless of claim contents and is never revoked by `apply_role_mapping`.
|
||||
|
||||
### Built-in Roles
|
||||
|
||||
| Role ID | Permissions |
|
||||
@@ -464,27 +375,10 @@ callback validation. Entries are automatically cleaned up after 5 minutes.
|
||||
|
||||
### "OIDC not configured"
|
||||
|
||||
All four required environment variables must be set:
|
||||
`TURNSTONE_OIDC_ISSUER`, `TURNSTONE_OIDC_CLIENT_ID`,
|
||||
`TURNSTONE_OIDC_CLIENT_SECRET`, and `TURNSTONE_OIDC_REDIRECT_BASE`.
|
||||
Check that none are empty or whitespace-only.
|
||||
|
||||
### "OIDC enabled but TURNSTONE_OIDC_REDIRECT_BASE is unset"
|
||||
|
||||
This error is logged when the three credential variables are set but
|
||||
`TURNSTONE_OIDC_REDIRECT_BASE` is missing. OIDC is disabled at startup
|
||||
to prevent Host-header-derived redirect URI spoofing. Set the variable
|
||||
to your service's externally-visible origin (e.g.
|
||||
`https://app.example.com`) and restart the server. See
|
||||
[Redirect base](#redirect-base-required) for the rationale.
|
||||
|
||||
### Discovery silently disables OIDC with "host does not match issuer"
|
||||
|
||||
The IdP discovery document points `token_endpoint`, `jwks_uri`, or
|
||||
`userinfo_endpoint` at a hostname that doesn't share the issuer's
|
||||
origin. If the IdP is legitimate, add the additional hostname(s) to
|
||||
`TURNSTONE_OIDC_TRUSTED_ENDPOINT_HOSTS`. Google is allow-listed
|
||||
automatically; see [Cross-host endpoints](#cross-host-endpoints).
|
||||
All three required environment variables must be set:
|
||||
`TURNSTONE_OIDC_ISSUER`, `TURNSTONE_OIDC_CLIENT_ID`, and
|
||||
`TURNSTONE_OIDC_CLIENT_SECRET`. Check that none are empty or
|
||||
whitespace-only.
|
||||
|
||||
### "Login session expired"
|
||||
|
||||
|
||||
+1
-1
@@ -94,7 +94,7 @@ cannot bypass the proxy.
|
||||
|--------|-------|---------|
|
||||
| `openai_api` | `api.openai.com` | OpenAI LLM API |
|
||||
| `anthropic_api` | `api.anthropic.com` | Anthropic LLM API |
|
||||
| `searxng` | `searxng:8080` (bundled service) | Web search backend |
|
||||
| `tavily_api` | `api.tavily.com` | Web search fallback |
|
||||
| `skills_registry` | `skills.sh` | Skill discovery |
|
||||
| `github_api` | `api.github.com` (read-only L7), `raw.githubusercontent.com` | Skill fetch, GitHub API |
|
||||
| `mcp_registry` | `registry.modelcontextprotocol.io` (read-only L7) | MCP server discovery |
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
# MCP OAuth in headless / scheduled / channel-driven runs
|
||||
|
||||
**Constraint**: OAuth-MCP servers (`auth_type=oauth_user`) require browser-based user consent. Users must pre-consent via the web UI before any run that cannot drive a browser redirect.
|
||||
|
||||
**Affected surfaces**:
|
||||
|
||||
- Scheduled workstreams (`turnstone-console` task scheduler).
|
||||
- Discord adapter runs.
|
||||
- Slack adapter runs.
|
||||
- Any future channel adapter without an interactive browser session.
|
||||
|
||||
**What happens when consent is missing**:
|
||||
|
||||
A tool call against an `oauth_user` server returns a structured `mcp_consent_required` error to the agent. The agent surfaces the deferred work in its output. Turnstone persists a record to `mcp_pending_consent` so the dashboard badge surfaces the deferred consent need to the user on next login.
|
||||
|
||||
**Recovery**:
|
||||
|
||||
The user opens the dashboard, sees the gear-icon badge counting pending consents, opens the settings modal, clicks Connect for each affected server, and completes the OAuth dance. The pending-consent record is cleared by the OAuth callback handler on success. Subsequent scheduled / channel runs use the freshly-stored token.
|
||||
|
||||
**Pre-consent recipe**:
|
||||
|
||||
Before scheduling a workstream that depends on an `oauth_user` MCP server, the user should:
|
||||
|
||||
1. Open the dashboard.
|
||||
2. Open the settings modal (gear icon).
|
||||
3. Click Connect on each MCP server the schedule will use.
|
||||
4. Confirm consent in the popup.
|
||||
|
||||
This stores tokens that the scheduled run will reuse. Refresh-token rotation is handled transparently on the run side; only the first consent requires browser interaction.
|
||||
@@ -1,173 +0,0 @@
|
||||
# Personas
|
||||
|
||||
A **persona** is a named, reusable bundle attached to a workstream **at
|
||||
creation** that controls how its system message is composed and what
|
||||
capability envelope it runs with. Personas answer a recurring operational
|
||||
complaint: the default composition primes every session for heavy tool use,
|
||||
and there was no per-workstream dial to launch a "just write prose" or
|
||||
"evidence-first research" session.
|
||||
|
||||
A persona is exactly four levers — no more:
|
||||
|
||||
| Lever | What it does |
|
||||
|---|---|
|
||||
| **Base prompt** | Replaces the BASE module of the composed system message. *Only* BASE: ENV, CONTEXT, TOOLS, and POLICIES keep composing, so mandatory [prompt policies](governance.md) ride on top of every persona. Built-in personas source their prose from a repo file; operator personas store it inline — see [Where persona prompts live](#where-persona-prompts-live). |
|
||||
| **Tool visibility** | Which tools the session advertises. Tri-state: *unrestricted* (tracks tool growth and MCP catalogs), *no tools* (the TOOLS prompt block self-suppresses and zero definitions go on the wire), or an *exact set* of names. Including `tool_search` in a set makes it **soft** — tools the model discovers through search join the visible set; omitting it makes the set **hard** (the search pathway is disabled entirely). On commercial providers a soft set costs one prompt-cache re-prime per `tool_search` expansion, since each expansion rewrites the wire tool set and recomposes the prompt. |
|
||||
| **MCP** | Whether the workstream talks to MCP at all. **Session-wide**: off means no MCP tools for the persona's own hands *or* for in-process task agents, no resource/prompt catalogs, and no listener registrations. This lever expresses infrastructure intent, not behavior shaping. |
|
||||
| **Memory** | Whether the persona's **own hands** get memory: recalled-memory injection into the prompt, memory-directed metacognitive nudges, and the `memory` tool. Task agents keep their own envelope, and compaction spill/markers are session mechanics that are never persona-gated. An exact tool set that hides `memory` also mutes those nudges, and the compaction-resume pointer follows `recall`'s visibility. |
|
||||
|
||||
Visibility is behavior shaping, **not** a security boundary: any tool call
|
||||
that does reach the wire still clears the same approval, judge, and policy
|
||||
machinery as always. RBAC and tool policies remain the enforcement layers.
|
||||
|
||||
## Snapshot semantics — resolve once, stamp forever
|
||||
|
||||
The persona is resolved **once**, at workstream creation, and stamped into
|
||||
`workstream_config` as five keys (`persona`, `persona_prompt`,
|
||||
`persona_tools`, `persona_mcp`, `persona_memory`). From then on the session
|
||||
reads only the stamp:
|
||||
|
||||
- **Editing or archiving a persona never changes an existing workstream.**
|
||||
Rehydrate, resume, and post-compaction resume all run from the stamp.
|
||||
A mid-session REPL `/resume` adopts the target workstream's stamp for
|
||||
prompt, tools, and memory; for the MCP lever it can only narrow in
|
||||
place — adopting an MCP-off stamp drops the live MCP surface, while
|
||||
adopting an MCP-on stamp into a session whose persona dropped MCP at
|
||||
construction is refused with an error telling you to reopen the
|
||||
workstream fresh.
|
||||
- A workstream outlives its persona — an archived persona keeps labelling
|
||||
the workstreams stamped with it.
|
||||
- A partial or unparseable stamp is treated as corruption: session
|
||||
construction fails loudly rather than silently falling back to a default
|
||||
envelope the operator never chose.
|
||||
- Workstreams created before personas existed carry no stamp and keep
|
||||
legacy behavior, byte-identical to the `engineer` / `orchestrator`
|
||||
defaults below — with one exception: pre-1.7 workstreams that had
|
||||
`creative_mode` set are converted by migration `063` into full
|
||||
`writer` stamps, so they resume as writing sessions rather than as
|
||||
legacy defaults.
|
||||
- Forking (`resume_ws` on create) resumes the source's stamped persona; the
|
||||
fork does not re-resolve.
|
||||
|
||||
## Seed personas
|
||||
|
||||
Migration `063` seeds six personas. The two per-kind **defaults** carry no
|
||||
overrides at all, so a zero-touch launch behaves exactly as it did before
|
||||
personas existed:
|
||||
|
||||
| Persona | Kind | Base prompt | Tools | MCP | Memory |
|
||||
|---|---|---|---|---|---|
|
||||
| `engineer` *(default)* | interactive | stock | unrestricted | on | on |
|
||||
| `orchestrator` *(default)* | coordinator | stock | unrestricted | on | on |
|
||||
| `scribe` | interactive | custom (faithful structuring of given material) | none | off | off |
|
||||
| `researcher` | interactive | custom (evidence-first) | `read_file`, `search`, `web_fetch`, `web_search`, `recall`, `memory`, `tool_search` (soft) | off | on |
|
||||
| `writer` | interactive | custom (creative writing partner — replaces the removed `/creative`) | none | off | on |
|
||||
| `executive` | coordinator | custom (delegate, interrogate plans, judge outcomes) | spawn/inspect/lifecycle tools plus `memory`: `spawn_workstream`, `spawn_batch`, `send_to_workstream`, `wait_for_workstream`, `inspect_workstream`, `list_workstreams`, `list_nodes`, `close_workstream`, `cancel_workstream`, `memory` (hard) | off | on |
|
||||
|
||||
Notes:
|
||||
|
||||
- `scribe` turns memory off deliberately: recalled memories would
|
||||
contaminate faithful summarization with unrelated context.
|
||||
- `researcher`'s set is soft (includes `tool_search`): it starts with
|
||||
read and evidence tools but can pull in others on demand — e.g. load
|
||||
`bash` to run a snippet and verify a calculation. It is evidence-first,
|
||||
not sandboxed; any escalated tool still hits the normal approval path.
|
||||
- Coordinator sessions do not merge MCP today, so the MCP lever on
|
||||
coordinator personas is forward-compatible bookkeeping; it bites on
|
||||
interactive workstreams.
|
||||
|
||||
## Where persona prompts live
|
||||
|
||||
Prompt source is explicit in the persona row — two nullable columns, never both empty:
|
||||
|
||||
| `base_prompt_file` | `base_prompt` | Meaning |
|
||||
|---|---|---|
|
||||
| set (e.g. `scribe.md`) | — | **built-in**: prose lives in `prompts/personas/<file>`, code-owned and PR-reviewed |
|
||||
| set | set | built-in with an **operator override** layered on top (the inline text wins) |
|
||||
| — | set | **operator** persona, inline prose |
|
||||
|
||||
A `CHECK` forbids the both-empty row, so resolution is a plain coalesce —
|
||||
`base_prompt ?? load(base_prompt_file)` — with no implicit "inherit the default"
|
||||
branch in application logic. `base_prompt_file` is set only by the migration/code
|
||||
(the admin API never exposes it): it marks a persona as built-in and blocks
|
||||
archive, so `engineer` and `orchestrator` can't be removed. To customise a
|
||||
built-in, set `base_prompt` on it (clear it to revert), or create your own persona.
|
||||
|
||||
The resolved prompt is **frozen into the workstream at creation** — later edits to
|
||||
a built-in's file or an operator's row never change a running workstream; only new
|
||||
ones pick up the change. "No persona" is not a state: every workstream is stamped,
|
||||
and an empty `persona=` resolves to the kind's `is_default` (`engineer` /
|
||||
`orchestrator`).
|
||||
|
||||
## Choosing a persona
|
||||
|
||||
Every creation surface takes an optional persona; empty always means the
|
||||
kind's default (or plain legacy behavior on a database with no personas
|
||||
seeded):
|
||||
|
||||
- **Web/console**: the persona select on the console launcher, the server
|
||||
webui's new-workstream dialog, and the dashboard composer. Selecting a
|
||||
persona requires **no** `persona.*` permission — the picker feed
|
||||
(`GET /v1/api/personas`) is authenticated-only and returns display fields.
|
||||
- **API/SDK**: `CreateWorkstreamRequest.persona` (Python:
|
||||
`create_workstream(persona=...)`; TypeScript: `{ persona: ... }`).
|
||||
- **CLI**: `turnstone --persona <name>`. Unknown or disabled names error at
|
||||
startup. `--resume` ignores `--persona` and adopts the resumed
|
||||
workstream's stamp.
|
||||
- **Coordinator spawn**: `spawn_workstream` / `spawn_batch` take a
|
||||
`persona` argument, validated when the coordinator prepares the spawn
|
||||
and re-checked by the node that creates the child (children are always
|
||||
interactive-kind). Omitted means the interactive **default** — a child
|
||||
never inherits its parent coordinator's persona.
|
||||
- **Sub-agents**: `task_agent` takes a `persona` argument setting the
|
||||
sub-agent's identity and capability envelope (resolved against
|
||||
interactive-kind personas, frozen into the task at prep). Omitted keeps
|
||||
the default autonomous task-agent identity — never the parent's persona.
|
||||
|
||||
## How agents discover personas
|
||||
|
||||
Agents are told, not expected to guess: the live persona list (enabled,
|
||||
interactive-kind — children and sub-agents are always interactive) is
|
||||
injected into the `persona` parameter description of `task_agent`,
|
||||
`spawn_workstream`, and `spawn_batch` whenever the session's tool surface
|
||||
is rendered — session start, MCP catalog change, model-registry reload.
|
||||
Each entry carries the name, the default marker, and the persona's
|
||||
one-line description so the model can pick by purpose (descriptions drop
|
||||
out past 25 personas; the name list always enumerates completely).
|
||||
|
||||
A persona created after that render is still reachable — pass its name.
|
||||
Every resolve failure enumerates the names currently valid for the kind,
|
||||
so a stale list (or a typo) self-corrects on the next attempt.
|
||||
|
||||
Resolution is forgiving on all surfaces (they share one rule):
|
||||
|
||||
- names match case-insensitively (`Writer` resolves `writer`);
|
||||
- an input that uniquely matches a persona's **display name**
|
||||
(case-insensitive, among the kind's enabled personas — display names are
|
||||
not unique, and a same-label persona of another kind neither blocks nor
|
||||
wins) resolves to that persona; an ambiguous match errors, listing the
|
||||
candidate slugs;
|
||||
- whatever variant matched, the stamped identity, approval chrome, and
|
||||
wire always carry the canonical `name` slug.
|
||||
|
||||
## Authoring (console)
|
||||
|
||||
Personas are managed in the console's **Manage → Governance → Personas**
|
||||
tab. The admin shelf exposes exactly the four levers plus the kind
|
||||
list, the default marker, and archive. Rules:
|
||||
|
||||
- `name` is an immutable lowercase slug — and the identifier agents and
|
||||
the CLI launch the persona by (`persona=` on the spawn tools,
|
||||
`--persona` on the CLI); the create shelf says so under **Name**.
|
||||
`display_name` is a list label, editable any time, and deliberately
|
||||
not an identifier (a unique display name happens to resolve, as a
|
||||
forgiveness fallback — don't design workflows around it).
|
||||
- Exactly one default per kind, storage-enforced: flipping the flag on a
|
||||
successor demotes the incumbent atomically, defaults are single-kind,
|
||||
and a default cannot be archived.
|
||||
- **Archive only** — there is no delete verb, so every stamped
|
||||
workstream's provenance stays explicable.
|
||||
|
||||
RBAC: `persona.create` / `persona.read` / `persona.write` gate the admin
|
||||
CRUD (`/v1/api/admin/personas`); all three are granted to `builtin-admin`
|
||||
by migration `063`, and other roles opt in via role permission overrides.
|
||||
+13
-39
@@ -40,20 +40,18 @@ Add PgBouncer between turnstone services and PostgreSQL:
|
||||
```yaml
|
||||
services:
|
||||
pgbouncer:
|
||||
image: edoburu/pgbouncer:latest
|
||||
image: bitnami/pgbouncer:latest
|
||||
environment:
|
||||
DB_HOST: postgres
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: ${POSTGRES_DB:-turnstone}
|
||||
DB_USER: ${POSTGRES_USER:-turnstone}
|
||||
DB_PASSWORD: ${POSTGRES_PASSWORD:?}
|
||||
LISTEN_PORT: "6432"
|
||||
AUTH_TYPE: ${POSTGRES_AUTH_TYPE:-scram-sha-256}
|
||||
POOL_MODE: transaction
|
||||
DEFAULT_POOL_SIZE: "40"
|
||||
MAX_CLIENT_CONN: "5000"
|
||||
MAX_DB_CONNECTIONS: "80"
|
||||
SERVER_IDLE_TIMEOUT: "300"
|
||||
POSTGRESQL_HOST: postgres
|
||||
POSTGRESQL_PORT: "5432"
|
||||
POSTGRESQL_DATABASE: turnstone
|
||||
POSTGRESQL_USERNAME: ${POSTGRES_USER:-turnstone}
|
||||
POSTGRESQL_PASSWORD: ${POSTGRES_PASSWORD:?}
|
||||
PGBOUNCER_POOL_MODE: transaction
|
||||
PGBOUNCER_DEFAULT_POOL_SIZE: "40"
|
||||
PGBOUNCER_MAX_CLIENT_CONN: "5000"
|
||||
PGBOUNCER_MAX_DB_CONNECTIONS: "80"
|
||||
PGBOUNCER_SERVER_IDLE_TIMEOUT: "300"
|
||||
ports:
|
||||
- "6432:6432"
|
||||
networks:
|
||||
@@ -69,7 +67,7 @@ services:
|
||||
```
|
||||
|
||||
Then point turnstone services at PgBouncer instead of PostgreSQL
|
||||
directly by changing `TURNSTONE_DB_URL`:
|
||||
directly by changing the `DATABASE_URL` (or `TURNSTONE_DB_URL`):
|
||||
|
||||
```bash
|
||||
# Before (direct)
|
||||
@@ -84,7 +82,7 @@ TURNSTONE_DB_URL=postgresql://turnstone:secret@pgbouncer:6432/turnstone
|
||||
## Helm / Kubernetes
|
||||
|
||||
Add a PgBouncer deployment or use a Helm chart like
|
||||
[edoburu/pgbouncer](https://github.com/edoburu/docker-pgbouncer/tree/master/examples/kubernetes).
|
||||
[bitnami/pgbouncer](https://github.com/bitnami/charts/tree/main/bitnami/pgbouncer).
|
||||
|
||||
In `values.yaml`, point the database at PgBouncer:
|
||||
|
||||
@@ -199,28 +197,4 @@ does not support prepared statements. Turnstone's SQLAlchemy layer does
|
||||
not use server-side prepared statements by default, so this is not an
|
||||
issue.
|
||||
|
||||
**LISTEN / NOTIFY not supported in transaction mode** — PgBouncer's
|
||||
transaction pooling assigns a real server connection only for the
|
||||
duration of each transaction, then returns it to the pool. PostgreSQL
|
||||
`LISTEN` is session state — a transaction-pooled client can't hold the
|
||||
multi-statement session a long-lived `LISTEN` needs. The console's
|
||||
`NotifyDispatcher` (reactive node discovery via the `services` channel)
|
||||
therefore opens a **dedicated, direct-to-Postgres** connection that
|
||||
bypasses PgBouncer.
|
||||
|
||||
Configure via `config.toml` `[database] listen_url` (preferred —
|
||||
co-located with the main `url`) or the `TURNSTONE_DB_LISTEN_URL` env var
|
||||
(config.toml wins when both are set). Defaults to the main DB URL when
|
||||
unset.
|
||||
|
||||
| Setting | Behaviour |
|
||||
|---|---|
|
||||
| unset | Listener uses `TURNSTONE_DB_URL` as-is. Fine when PgBouncer is in **session** mode, or when there's no pooler in front of Postgres. With transaction-mode PgBouncer the listener's `LISTEN` will fail and the dispatcher retries with exponential backoff (1 s → 30 s cap) without ever succeeding. Reactive NOTIFY-driven node discovery is silently lost; the cluster collector's 60 s `_discovery_loop` is the only remaining backstop. |
|
||||
| set to direct-to-PG URL (e.g. `postgresql://…/turnstone`) | Listener bypasses PgBouncer for its one dedicated connection. Reactive discovery latency drops from up-to-60 s to ~500 ms. The rest of the storage layer continues to go through PgBouncer in transaction mode. |
|
||||
|
||||
Set this whenever PgBouncer is in transaction mode (the recommended
|
||||
setting per this doc). The override only adds one long-lived PG
|
||||
connection per console process — sized into the cluster's
|
||||
`max_connections` budget alongside the pool.
|
||||
|
||||
See also: [Docker deployment](docker.md) · [Security](security.md)
|
||||
|
||||
+15
-25
@@ -1,25 +1,17 @@
|
||||
# Release Process
|
||||
|
||||
Turnstone ships several parallel release tracks from a single PyPI package.
|
||||
Turnstone uses two parallel release tracks published from a single PyPI package.
|
||||
|
||||
## Release Tracks
|
||||
|
||||
| Track | Versions | Branch | Docker tags | PyPI install |
|
||||
|-------|----------|--------|-------------|--------------|
|
||||
| **Stable 1.5** | `1.5.x` | `stable/1.5` | `:1.5.x`, `:1.5` | `pip install 'turnstone==1.5.*'` |
|
||||
| **Stable 1.6** | `1.6.x` | `stable/1.6` | `:1.6.x`, `:1.6`, `:stable`, `:latest` | `pip install turnstone` |
|
||||
| **Experimental** | `1.7.0aN` | `main` | `:1.7.0aN`, `:experimental` | `pip install turnstone --pre` |
|
||||
| **Stable** | `1.1.0`, `1.1.1` | `stable/1.1` | `:1.1.0`, `:1.1`, `:stable`, `:latest` | `pip install turnstone` |
|
||||
| **Experimental** | `1.2.0a1`, `1.2.0a2` | `main` | `:1.2.0a1`, `:experimental` | `pip install turnstone --pre` |
|
||||
|
||||
- **Stable** tracks receive bugfixes only. The most-recent stable minor
|
||||
owns the `:stable` / `:latest` Docker tags and the default PyPI
|
||||
install.
|
||||
- **Experimental** (always on `main`) receives new features. May be
|
||||
rough around the edges.
|
||||
- When experimental matures, it is promoted to a new stable minor via
|
||||
a `stable/X.Y` branch. One prior stable track is maintained alongside
|
||||
the current one; at each promotion the oldest track is retired — its
|
||||
branch is deleted, while its tags and released artifacts remain
|
||||
available.
|
||||
- **Stable** receives bugfixes only. Production-grade.
|
||||
- **Experimental** receives new features. May be rough around the edges.
|
||||
- When experimental matures, it is promoted to stable. The previous stable branch stops receiving patches.
|
||||
|
||||
## Version Scheme
|
||||
|
||||
@@ -34,17 +26,17 @@ Turnstone ships several parallel release tracks from a single PyPI package.
|
||||
## Releasing an Experimental Version (from main)
|
||||
|
||||
```bash
|
||||
scripts/release.sh 1.7.0a2 --push
|
||||
scripts/release.sh 1.1.0a2 --push
|
||||
```
|
||||
|
||||
This bumps `pyproject.toml` + `turnstone/__init__.py`, regenerates `uv.lock`, commits, tags `v1.7.0a2`, and pushes. CI runs, then publish + Docker workflows fire automatically.
|
||||
This bumps `pyproject.toml` + `turnstone/__init__.py`, regenerates `uv.lock`, commits, tags `v1.1.0a2`, and pushes. CI runs, then publish + Docker workflows fire automatically.
|
||||
|
||||
## Releasing a Stable Patch (from stable/X.Y)
|
||||
|
||||
```bash
|
||||
git checkout stable/1.6
|
||||
git checkout stable/1.0
|
||||
git cherry-pick <commit-hash> # bugfix from main
|
||||
scripts/release.sh 1.6.1 --push
|
||||
scripts/release.sh 1.0.2 --push
|
||||
```
|
||||
|
||||
## Promoting Experimental to Stable
|
||||
@@ -53,19 +45,17 @@ When `main` is ready for a stable release:
|
||||
|
||||
```bash
|
||||
# 1. Tag the stable release on main
|
||||
scripts/release.sh 1.6.0 --push
|
||||
scripts/release.sh 1.1.0 --push
|
||||
|
||||
# 2. Create the stable maintenance branch from that tag
|
||||
git branch stable/1.6 v1.6.0
|
||||
git push origin stable/1.6
|
||||
git branch stable/1.1 v1.1.0
|
||||
git push origin stable/1.1
|
||||
|
||||
# 3. Start the next experimental cycle on main
|
||||
scripts/release.sh 1.7.0a1 --push
|
||||
scripts/release.sh 1.2.0a1 --push
|
||||
```
|
||||
|
||||
The previous stable branch continues to receive security-only patches;
|
||||
the track before it is retired at each promotion (at 1.6.0:
|
||||
`stable/1.5` stays maintained, `stable/1.4` is retired).
|
||||
The previous `stable/1.0` branch stops receiving patches at this point.
|
||||
|
||||
## CI/CD Pipeline
|
||||
|
||||
|
||||
+5
-40
@@ -69,14 +69,11 @@ Both `TurnstoneServer` (sync) and `AsyncTurnstoneServer` (async) expose:
|
||||
|----------|--------|---------|
|
||||
| **Workstreams** | `list_workstreams()` | `ListWorkstreamsResponse` |
|
||||
| | `dashboard()` | `DashboardResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve, skill, persona, initial_message, attachments)` | `CreateWorkstreamResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve, skill)` | `CreateWorkstreamResponse` |
|
||||
| | `close_workstream(ws_id)` | `StatusResponse` |
|
||||
| **Attachments** | `upload_attachment(ws_id, filename, data, *, mime_type=...)` | `UploadAttachmentResponse` |
|
||||
| | `list_attachments(ws_id)` | `ListAttachmentsResponse` |
|
||||
| | `get_attachment_content(ws_id, attachment_id)` | `bytes` |
|
||||
| | `delete_attachment(ws_id, attachment_id)` | `StatusResponse` |
|
||||
| **Chat** | `send(message, ws_id)` | `SendResponse` |
|
||||
| | `approve(*, ws_id, approved, feedback, always)` | `StatusResponse` |
|
||||
| | `plan_feedback(*, ws_id, feedback)` | `StatusResponse` |
|
||||
| | `command(*, ws_id, command)` | `StatusResponse` |
|
||||
| | `cancel(ws_id, *, force=False)` | `StatusResponse` |
|
||||
| **Streaming** | `stream_events(ws_id)` | `Iterator[ServerEvent]` |
|
||||
@@ -100,7 +97,7 @@ Both `TurnstoneConsole` (sync) and `AsyncTurnstoneConsole` (async) expose:
|
||||
| | `workstreams(*, state, node, search, sort, page, per_page)` | `ClusterWorkstreamsResponse` |
|
||||
| | `node_detail(node_id)` | `NodeDetailResponse` |
|
||||
| | `snapshot()` | `ClusterSnapshotResponse` |
|
||||
| | `create_workstream(*, node_id, name, model, initial_message, skill, persona)` | `ConsoleCreateWsResponse` |
|
||||
| | `create_workstream(*, node_id, name, model, initial_message, skill)` | `ConsoleCreateWsResponse` |
|
||||
| **Schedules** | `list_schedules()` | `ListSchedulesResponse` |
|
||||
| | `create_schedule(*, name, schedule_type, initial_message, ...)` | `ScheduleInfo` |
|
||||
| | `get_schedule(task_id)` | `ScheduleInfo` |
|
||||
@@ -133,12 +130,10 @@ SSE events are deserialized into typed dataclasses. Use `event.type` to discrimi
|
||||
| `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output`, `is_error` |
|
||||
| `tool_output_chunk` | `ToolOutputChunkEvent` | `call_id`, `chunk` |
|
||||
| `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort`, `cache_creation_tokens`, `cache_read_tokens` |
|
||||
| `plan_review` | `PlanReviewEvent` | `content` |
|
||||
| `error` | `ErrorEvent` | `message` |
|
||||
| `info` | `InfoEvent` | `message` |
|
||||
| `stream_end` | `StreamEndEvent` | — |
|
||||
| `state_change` | `StateChangeEvent` | `state` ∈ `running`/`thinking`/`attention`/`idle`/`error` |
|
||||
| `in_progress_snapshot` | `InProgressSnapshotEvent` | `content`, `reasoning` (one-shot mid-stream refresh resume) |
|
||||
| `approval_resolved` | `ApprovalResolvedEvent` | `approved`, `feedback` |
|
||||
| `cancelled` | `CancelledEvent` | — |
|
||||
|
||||
**Global events** (from `stream_global_events()`):
|
||||
@@ -176,36 +171,6 @@ result.ok # True if no errors and not timed out
|
||||
result.timed_out # True if timeout expired
|
||||
```
|
||||
|
||||
### Attachments
|
||||
|
||||
Upload files to a workstream and attach them to the next user turn:
|
||||
|
||||
```python
|
||||
# Upload separately, then send a message — attachments auto-attach
|
||||
with open("screenshot.png", "rb") as f:
|
||||
att = client.upload_attachment(ws.ws_id, "screenshot.png",
|
||||
f.read(),
|
||||
mime_type="image/png")
|
||||
client.send("What's wrong in this screenshot?", ws.ws_id)
|
||||
|
||||
# Or attach at workstream-creation time (multipart upload)
|
||||
from turnstone.sdk import AttachmentUpload
|
||||
|
||||
with open("notes.txt", "rb") as f:
|
||||
ws = client.create_workstream(
|
||||
name="triage",
|
||||
initial_message="Summarize the notes",
|
||||
attachments=[AttachmentUpload(data=f.read(),
|
||||
filename="notes.txt",
|
||||
mime_type="text/plain")],
|
||||
)
|
||||
```
|
||||
|
||||
Limits: images ≤ 4 MiB (png/jpeg/gif/webp), text ≤ 512 KiB (UTF-8),
|
||||
10 pending per (workstream, user). The SDK auto-generates `ws_id` on the
|
||||
client so cluster-routed callers bind attachments to the owning node
|
||||
before the request lands.
|
||||
|
||||
### Error Handling
|
||||
|
||||
Non-2xx responses raise `TurnstoneAPIError`:
|
||||
@@ -319,7 +284,7 @@ turnstone/sdk/ Python SDK (sub-package)
|
||||
_base.py Shared httpx async client, auth, error handling
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
events.py 38 SSE event dataclasses with type registry
|
||||
events.py 27 SSE event dataclasses with type registry
|
||||
server.py AsyncTurnstoneServer + TurnstoneServer
|
||||
console.py AsyncTurnstoneConsole + TurnstoneConsole
|
||||
|
||||
|
||||
+8
-11
@@ -1,10 +1,8 @@
|
||||
# Security and Authentication
|
||||
|
||||
Turnstone uses a layered authentication system with two token types
|
||||
(database-backed API tokens + HMAC-SHA256 JWTs), hierarchical scopes,
|
||||
and a split architecture where the console manages credentials while
|
||||
individual server nodes validate JWTs locally. Inter-service traffic
|
||||
uses short-lived service JWTs minted by `ServiceTokenManager`.
|
||||
Turnstone uses a layered authentication system with three token types,
|
||||
hierarchical scopes, and a split architecture where the console manages
|
||||
credentials while individual server nodes validate JWTs locally.
|
||||
|
||||
---
|
||||
|
||||
@@ -39,7 +37,7 @@ Claims:
|
||||
|-------|-------------|
|
||||
| `sub` | User ID |
|
||||
| `scopes` | Comma-separated scope list (`read,write,approve`) |
|
||||
| `src` | Token source (`password`, `database`, `oidc`, or a service origin like `console`, `cli`, or `channel`) |
|
||||
| `src` | Token source (`password`, `api_token`, `config`, `oidc`) |
|
||||
| `iss` | Issuer — always `turnstone` |
|
||||
| `aud` | Audience — `turnstone-server` or `turnstone-console` |
|
||||
| `iat` | Issued-at timestamp |
|
||||
@@ -67,11 +65,10 @@ Scopes are hierarchical — higher scopes imply all lower ones.
|
||||
| Method | Path pattern | Required scope |
|
||||
|--------|-------------|----------------|
|
||||
| GET | Any protected path | `read` |
|
||||
| POST | `/api/command` | `write` |
|
||||
| POST | `/api/workstreams/new`, `/api/cluster/workstreams/new` | `write` |
|
||||
| POST | `/api/workstreams/{ws_id}/{send,cancel,close,delete,open,refresh-title,title,attachments}` | `write` |
|
||||
| DELETE | `/api/workstreams/{ws_id}/send` (dequeue), `/api/workstreams/{ws_id}/attachments/{attachment_id}` | `write` |
|
||||
| POST | `/api/workstreams/{ws_id}/approve` | `approve` |
|
||||
| POST | `/api/send`, `/api/plan`, `/api/command` | `write` |
|
||||
| POST | `/api/workstreams/new`, `/api/workstreams/close` | `write` |
|
||||
| POST | `/api/cluster/workstreams/new` | `write` |
|
||||
| POST | `/api/approve` | `approve` |
|
||||
| Any | `/api/admin/*` | `approve` |
|
||||
|
||||
Public paths bypass authentication entirely: `/`, `/health`, `/metrics`,
|
||||
|
||||
+4
-74
@@ -36,76 +36,6 @@ users to the admin Settings API.
|
||||
|
||||
---
|
||||
|
||||
## Per-Model Sampling Overrides
|
||||
|
||||
The global `model.temperature`, `model.max_tokens`, and `model.reasoning_effort`
|
||||
settings serve as cluster-wide defaults. Individual models can override these
|
||||
via per-model settings in the `model_definitions` table (admin Models tab).
|
||||
|
||||
Resolution order for sampling parameters:
|
||||
|
||||
| Priority | Source |
|
||||
|----------|--------|
|
||||
| 1 (highest) | Per-model override (set in Models tab) |
|
||||
| 2 | Global default (set in Settings tab) |
|
||||
| 3 | Registry default (code) |
|
||||
|
||||
When a per-model override is `NULL` (empty in the UI), the global default is
|
||||
used. Switching models via `/model <alias>` re-resolves sampling parameters
|
||||
from the new model's overrides or global defaults.
|
||||
|
||||
### Responses output controls (per-model)
|
||||
|
||||
Models whose capability table declares Responses output controls expose two
|
||||
additional fields in the Models create/edit shelf:
|
||||
|
||||
| Field | Stored capability | Values | Effect |
|
||||
|-------|-------------------|--------|--------|
|
||||
| Output verbosity | `verbosity` | `low`, `medium`, `high` | Controls answer length independently of reasoning effort. |
|
||||
| Reasoning mode | `reasoning_mode` | `standard`, `pro` | Selects standard or higher-compute Pro execution without changing the model ID. |
|
||||
|
||||
An empty selection means provider default and omits the capability key. Known
|
||||
GPT-5.6 models inherit support from the built-in table without persisting
|
||||
redundant support flags. An OpenAI-compatible model pinned to the Responses API
|
||||
can opt in with the `supports_verbosity` and `supports_pro_mode` capability
|
||||
tiles. Chat Completions and non-Responses providers do not surface or submit
|
||||
these controls.
|
||||
|
||||
**Removed settings:** `model.name` and `model.context_window` have been removed
|
||||
from ConfigStore. Model names and context windows are now configured per-model
|
||||
in the Models tab. A startup warning is logged if these keys appear in
|
||||
`config.toml`.
|
||||
|
||||
### Reasoning persistence (per-model)
|
||||
|
||||
Two boolean flags on `model_definitions` (migration 052) control how
|
||||
reasoning text round-trips per model:
|
||||
|
||||
| Flag | Default | Effect |
|
||||
|------|---------|--------|
|
||||
| `surface_persisted_reasoning` | `True` | Surface stored reasoning text on `/history` payloads so a page reload re-renders the reasoning bubble. **Storage of reasoning bytes is independent of this flag** — they ride in `provider_data` regardless. |
|
||||
| `replay_reasoning_to_model` | `False` | Send stored reasoning blocks back to the provider on subsequent turns. Capability-gated: only takes effect when the model's `ModelCapabilities.supports_reasoning_replay` is also `True`. Set on canonical OpenAI gpt-5*/o-series and Anthropic Claude entries; unknown / local-server models default to `False` so an operator who flips the flag on a model whose API doesn't understand reasoning replay silently no-ops rather than 400-ing. |
|
||||
|
||||
Edit both via the admin Models tab. See the architecture doc for the
|
||||
provider-side mechanics (Anthropic `thinking`, OpenAI Responses
|
||||
`reasoning` + `include=["reasoning.encrypted_content"]`, synthetic
|
||||
`reasoning_text` for Chat Completions / vLLM / llama.cpp / Gemini-compat).
|
||||
|
||||
### Task agent overrides
|
||||
|
||||
`task_agent` sub-sessions resolve independently from the conversation model
|
||||
so operators can pick a cheaper/faster model for autonomous loops:
|
||||
|
||||
| Setting | Purpose |
|
||||
|---------|---------|
|
||||
| `model.task_alias` | Alias used for `task_agent` sub-sessions. Falls back to `[model].agent_model` in config.toml, then the session's active model. |
|
||||
| `model.task_effort` | Reasoning effort for `task_agent`. Empty string means "inherit from the session". |
|
||||
|
||||
Both are live-editable from the Settings tab and take effect on the
|
||||
next sub-agent invocation — no restart required.
|
||||
|
||||
---
|
||||
|
||||
## Bootstrap vs ConfigStore
|
||||
|
||||
**Bootstrap settings** are required before storage is available (database
|
||||
@@ -119,17 +49,17 @@ connection, Redis, auth secrets, server bind address). These stay in
|
||||
| Auth | `[auth]` | config.toml / env |
|
||||
| Console bind | `[console]` | config.toml / env |
|
||||
|
||||
**ConfigStore settings** are loaded from the database after storage
|
||||
initialization:
|
||||
**ConfigStore settings** (51 settings) are loaded from the database after
|
||||
storage initialization:
|
||||
|
||||
| Section | Settings |
|
||||
|---------|----------|
|
||||
| `model` | default_alias, temperature, max_tokens, reasoning_effort, task_alias, task_effort |
|
||||
| `model` | name, temperature, max_tokens, reasoning_effort, context_window |
|
||||
| `session` | instructions, retention_days, compact_max_tokens, auto_compact_pct |
|
||||
| `tools` | timeout, truncation, agent_max_turns, skip_permissions, search, search_threshold, search_max_results |
|
||||
| `server` | workstream_idle_timeout, max_workstreams |
|
||||
| `cluster` | node_fan_out_limit, mcp_max_servers |
|
||||
| `mcp` | config_path, registry_url |
|
||||
| `mcp` | config_path, refresh_interval, registry_url |
|
||||
| `ratelimit` | enabled, requests_per_second, burst, trusted_proxies |
|
||||
| `health` | backend_probe_interval, backend_probe_timeout, circuit_breaker_threshold, circuit_breaker_cooldown |
|
||||
| `judge` | enabled, model, provider, base_url, api_key, confidence_threshold, max_context_ratio, timeout, read_only_tools, output_guard, redact_secrets, cancel_on_approval |
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
---
|
||||
name: import-conversation-history
|
||||
description: Use this skill when the user wants to import or migrate conversation history from another LLM chat or coding tool (e.g. ChatGPT, Claude.ai, Cursor, Copilot Chat, Aider, Gemini, a custom JSON export) into Turnstone. The skill teaches Turnstone's destination contracts — workstream identity, the OpenAI-shaped message rows, tool-call/result pairing, provider-fidelity blobs, attachments, and archive-vs-resumable choice — so the agent can map any source format onto them. Trigger phrases: "import my chats", "migrate this transcript into Turnstone", "bring my Claude.ai history over", "load this export as a workstream".
|
||||
version: 1.0.0
|
||||
---
|
||||
|
||||
# Importing Conversation History into Turnstone
|
||||
|
||||
## Overview
|
||||
|
||||
Source formats vary; the destination does not. Your job is to translate whatever the user hands you (JSON dump, ZIP export, scraped HTML, screenshot OCR, raw transcript) into Turnstone's internal shape: **one workstream row** plus an ordered sequence of **conversation rows** in OpenAI message format. This skill documents the destination so you can write a correct mapper for any source.
|
||||
|
||||
Two questions to settle with the user before writing anything:
|
||||
|
||||
1. **Archive or resumable?** An archive ("saved" workstream — `state="closed"`) is read-only history. A resumable workstream (`state="idle"`) lets the user continue the conversation; this only works cleanly when the source LLM matches a Turnstone-supported provider/model and tool definitions still resolve.
|
||||
2. **One workstream per source thread, or merge?** Default to one-to-one unless the user explicitly asks to merge.
|
||||
|
||||
Default to **archive** when in doubt — resuming a foreign transcript with mismatched tool schemas or stale provider signatures will fail at the next turn.
|
||||
|
||||
## Turnstone Data Model (the destination)
|
||||
|
||||
Two tables carry the conversation:
|
||||
|
||||
### `workstreams` (one row per imported thread)
|
||||
|
||||
| Column | Required | Notes |
|
||||
|---|---|---|
|
||||
| `ws_id` | yes | 32-char lowercase hex. Auto-generate with `secrets.token_hex(16)` if you don't already have one. **First 4 hex chars are the routing bucket** — see "Identity & Routing" below. |
|
||||
| `name` | yes | Short title. Pull from source thread title; fall back to first ~60 chars of first user message. |
|
||||
| `state` | yes | `"closed"` for archive, `"idle"` for resumable. Never set `"running"` on import. |
|
||||
| `kind` | yes | `"interactive"` for normal threads. Do NOT use `"coordinator"` for imports — that's reserved for cluster-spawned coordinator workstreams. |
|
||||
| `parent_ws_id` | no | Leave NULL. Only set if you're importing a coordinator-spawned subtree and re-parenting it; rare. |
|
||||
| `user_id` | yes | Owner. Must exist in `users`; importer must know which Turnstone user owns the imported history. |
|
||||
| `node_id` | yes (multi-node) | Denormalized cache of the node that owns this `ws_id`'s bucket. Single-node deployments can leave it NULL or set it to the only node. |
|
||||
| `alias` | no | Human-typeable short name. Optional; must be unique cluster-wide if set. |
|
||||
| `title` | no | Auto-titled later by the LLM; safe to leave NULL on import. |
|
||||
| `skill_id`, `skill_version` | yes | Default `""` and `0` unless the source thread was scoped to a Turnstone skill. |
|
||||
| `created`, `updated` | yes | ISO8601 strings. Use the source's first/last message timestamps when available. |
|
||||
|
||||
### `conversations` (many rows per thread, ordered by `id`/`timestamp`)
|
||||
|
||||
| Column | Notes |
|
||||
|---|---|
|
||||
| `ws_id` | The workstream this row belongs to. |
|
||||
| `timestamp` | ISO8601 string. Preserve source timestamps; fall back to monotonically increasing values if unknown. **Order is canonical via `id` (autoincrement), not `timestamp`** — but always insert in conversational order so both agree. |
|
||||
| `role` | One of `system`, `user`, `assistant`, `tool`, `developer`. See role mapping below. |
|
||||
| `content` | Text. May be NULL for assistant rows that are *only* tool calls. |
|
||||
| `tool_name` | Set on `role="tool"` rows (the tool whose result this is). NULL otherwise. |
|
||||
| `tool_call_id` | Set on `role="tool"` rows (matches the assistant row's `tool_calls[].id`). NULL otherwise. |
|
||||
| `tool_calls` | JSON-encoded list, on `role="assistant"` rows that issued tool calls. OpenAI shape — see "Tool Calls" below. |
|
||||
| `provider_data` | JSON blob preserving provider-native content blocks (Anthropic `signature`, Gemini `thought_signature`, etc.). Optional; only matters for **resumable** imports against the same provider. Skip for archives. |
|
||||
|
||||
The internal format is **OpenAI-shaped**, even when the source was Anthropic or Gemini. Providers translate at their own API boundary; storage stays uniform.
|
||||
|
||||
## Identity & Routing (`ws_id`)
|
||||
|
||||
- `ws_id` is **32-char lowercase hex** (i.e. `secrets.token_hex(16)`).
|
||||
- The **routing bucket** is `int(ws_id[:4], 16)` — the first 4 hex chars place this workstream on a specific node via the consistent hash ring.
|
||||
- For multi-node imports: either insert through the console's routing proxy (which forwards to the owning node), or generate `ws_id`s and write directly to each node's database in batches grouped by bucket.
|
||||
- For single-node imports: bucket math is irrelevant; any `ws_id` works.
|
||||
- **Do not reuse the source platform's IDs as `ws_id`** unless they happen to be 32-char hex. Generate fresh; if you need the old ID for traceability, store it in `workstream_config` under a key like `import.source_id`.
|
||||
|
||||
## Recommended Import Path
|
||||
|
||||
Three options, in order of preference:
|
||||
|
||||
### 1. Storage protocol (recommended for full history)
|
||||
|
||||
Use `turnstone.core.storage.Storage.save_messages_bulk(rows)`. This is the canonical bulk-insert primitive and bypasses the LLM round-trip entirely.
|
||||
|
||||
```python
|
||||
from turnstone.core.storage import get_storage # construct via the same path the server uses
|
||||
|
||||
storage = get_storage(...) # see turnstone.core.storage.__init__ for the project's wiring
|
||||
|
||||
storage.create_workstream( # or whatever the project's exposed creator is — check turnstone/core/storage/_protocol.py
|
||||
ws_id=ws_id,
|
||||
user_id=user_id,
|
||||
name=name,
|
||||
state="closed",
|
||||
kind="interactive",
|
||||
...
|
||||
)
|
||||
|
||||
storage.save_messages_bulk([
|
||||
{"ws_id": ws_id, "role": "user", "content": "Hello"},
|
||||
{"ws_id": ws_id, "role": "assistant", "content": "Hi! What can I help with?"},
|
||||
{"ws_id": ws_id, "role": "assistant", "content": None,
|
||||
"tool_calls": json.dumps([{"id": "call_1", "type": "function",
|
||||
"function": {"name": "search", "arguments": "{\"q\":\"x\"}"}}])},
|
||||
{"ws_id": ws_id, "role": "tool", "tool_name": "search", "tool_call_id": "call_1",
|
||||
"content": "result text"},
|
||||
# ...
|
||||
])
|
||||
```
|
||||
|
||||
`save_messages_bulk` handles `timestamp` and the workstream's `updated` column internally, so you don't need to compute them per row. **Verify the exact creator signature** by reading `turnstone/core/storage/_protocol.py` — table layout has shifted across migrations and the Storage protocol is the source of truth.
|
||||
|
||||
### 2. SDK `create_workstream(resume_ws=...)` (when the source is already a Turnstone workstream)
|
||||
|
||||
Only useful for *Turnstone → Turnstone* re-parenting. Not relevant for foreign sources.
|
||||
|
||||
### 3. SDK `create_workstream(initial_message=...)` + `send()` per turn (last resort)
|
||||
|
||||
Only fits archives where the source had **no tool calls** and you don't care about preserving assistant turns verbatim. Each `send()` triggers a real LLM round-trip, which is expensive and rewrites assistant content. Don't use this for full history.
|
||||
|
||||
## Role Mapping
|
||||
|
||||
Common source-role conventions and how they map to Turnstone:
|
||||
|
||||
| Source role | Turnstone `role` | Notes |
|
||||
|---|---|---|
|
||||
| `user`, `human` | `user` | Direct map. |
|
||||
| `assistant`, `ai`, `model`, `bot` | `assistant` | Direct map. |
|
||||
| `system` | `system` | Preserve only if it's content the user wrote (custom instructions). Drop boilerplate provider preambles — Turnstone composes its own system message. |
|
||||
| `developer` (OpenAI o-series) | `developer` | Preserve. |
|
||||
| `tool`, `function`, `tool_result` | `tool` | Must carry `tool_name` and `tool_call_id` matching the prior assistant row's `tool_calls[].id`. |
|
||||
| `tool_use` (Anthropic) | `assistant` with `tool_calls` | Anthropic emits tool calls *inside* an assistant message; flatten to OpenAI shape. |
|
||||
| `human_feedback`, `revision` | `user` | Treat as a follow-up user turn. |
|
||||
|
||||
## Tool Calls (the most error-prone part)
|
||||
|
||||
Turnstone stores tool calls in OpenAI's nested-function shape on the assistant row, and matches them with `role="tool"` result rows by `tool_call_id`.
|
||||
|
||||
### Assistant row with tool calls
|
||||
|
||||
```json
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_abc123",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_web",
|
||||
"arguments": "{\"query\":\"turnstone import\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`tool_calls[].function.arguments` is **a JSON-encoded string**, not an object. Source formats commonly get this wrong — Anthropic stores arguments as a parsed object, Gemini as a struct. Always re-serialize to a string.
|
||||
|
||||
### Tool result row
|
||||
|
||||
```json
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_name": "search_web",
|
||||
"tool_call_id": "call_abc123",
|
||||
"content": "..."
|
||||
}
|
||||
```
|
||||
|
||||
Pairing rules:
|
||||
- Every assistant `tool_calls[].id` MUST be followed by exactly one `role="tool"` row with the matching `tool_call_id`, before the next user/assistant turn.
|
||||
- If the source dropped the tool result (cut-off transcript), insert a synthetic `role="tool"` row with `content="[tool result missing in source]"` to keep the chain valid. An assistant row with an unanswered `tool_calls[].id` will break replay and any LLM round-trip.
|
||||
- Multi-tool assistant turns: one `role="tool"` row per call, in any order, all before the next non-tool row.
|
||||
|
||||
### Tool ID generation
|
||||
|
||||
If the source used opaque tool IDs that aren't unique within a thread (some platforms reuse them), regenerate with a stable scheme like `f"call_{i}"` where `i` is a per-thread counter. Update both the assistant and tool rows together.
|
||||
|
||||
## Provider Fidelity (`provider_data`)
|
||||
|
||||
Skip this entirely for **archive** imports.
|
||||
|
||||
For **resumable** imports against the same provider, populate `provider_data` to preserve provider-specific tool-call metadata that the next API round-trip will require:
|
||||
|
||||
- **Anthropic**: `signature` field on thinking blocks; required for round-tripping extended-thinking responses.
|
||||
- **Gemini**: `thought_signature` on tool calls; required for fidelity.
|
||||
- **OpenAI**: typically nothing to preserve.
|
||||
|
||||
The runtime-side dict key is `_provider_content` (a list of provider-native blocks); the persisted column is `provider_data` (the same list, JSON-encoded). If you don't have provider-native blocks from the source — and you usually won't, because a foreign export won't include them — leave `provider_data` NULL. The first new turn will succeed without it, but the previous assistant turn's reasoning won't replay back to the model.
|
||||
|
||||
## Attachments
|
||||
|
||||
If the source thread had image or file attachments:
|
||||
|
||||
- **Size limits**: images ≤ 4 MiB, text documents ≤ 512 KiB. Reject or downsample anything bigger.
|
||||
- **Allowed types**: server validates magic bytes for images and UTF-8-decodes for text. Binary blobs that aren't images won't pass.
|
||||
- **Lifecycle**: pending → reserved → consumed. For imports, the cleanest path is to upload as pending and immediately consume by attaching to the relevant `conversations.id`.
|
||||
|
||||
Two import paths:
|
||||
|
||||
1. **Bulk-insert + post-attach**: insert messages first, get back the assistant/user `conversations.id`, then write `workstream_attachments` rows linking the file to `message_id`.
|
||||
2. **SDK multipart create**: `create_workstream(attachments=[...], initial_message=...)` for the *first* turn only — the server reserves and consumes them onto that turn. Doesn't help for mid-thread attachments.
|
||||
|
||||
For full-history imports with multiple attachments at different turns, path (1) is the only option.
|
||||
|
||||
## Validation Checklist
|
||||
|
||||
Before declaring success, verify:
|
||||
|
||||
- [ ] `ws_id` is 32-char lowercase hex.
|
||||
- [ ] `workstreams` row exists with the right `user_id`, `state`, `kind`.
|
||||
- [ ] Conversation rows are inserted **in order** (autoincrement `id` will reflect insert order).
|
||||
- [ ] Every assistant `tool_calls[].id` has a matching `role="tool"` row with the same `tool_call_id`.
|
||||
- [ ] `tool_calls[].function.arguments` is a JSON-encoded **string**, not a parsed object.
|
||||
- [ ] First message is typically `role="user"` (not `system`) — Turnstone composes its own system prompt at runtime.
|
||||
- [ ] No empty assistant rows (`content=NULL` AND `tool_calls=NULL` is invalid).
|
||||
- [ ] If multi-node: the `ws_id`'s bucket maps to a node that exists; `workstreams.node_id` matches.
|
||||
- [ ] Round-trip test: run `Storage.load_messages(ws_id)` and confirm the reconstructed list matches what you inserted (modulo timestamps).
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- **Don't import the source provider's system prompt verbatim.** Provider boilerplate ("You are Claude...", "You are ChatGPT...") will conflict with Turnstone's composed system message and confuse the model on resume. Drop it; preserve only user-authored custom instructions.
|
||||
- **Don't preserve foreign tool definitions as Turnstone tools.** If the source had custom tools that don't exist in Turnstone, the assistant rows that called them are still valid history (archive), but the workstream is **not resumable** — mark `state="closed"`.
|
||||
- **Don't fabricate `tool_call_id`s without re-pairing.** Mismatched ids silently break the replay chain on the next turn.
|
||||
- **Don't skip the `tool_name` field on `role="tool"` rows.** Some load paths use it for display and audit; NULL there will render as "unknown tool".
|
||||
- **Don't write through the LLM (`send()` per turn) for full history.** It's expensive, rewrites assistant turns, and rate-limits will bite long imports.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Task | Path |
|
||||
|---|---|
|
||||
| Generate ws_id | `secrets.token_hex(16)` |
|
||||
| Bulk insert messages | `Storage.save_messages_bulk(rows)` |
|
||||
| Archive (read-only) | `state="closed"`, skip `provider_data` |
|
||||
| Resumable | `state="idle"`, populate `provider_data` if same provider |
|
||||
| Tool call id | OpenAI shape: `{"id": ..., "type": "function", "function": {"name": ..., "arguments": "<json string>"}}` |
|
||||
| Tool result row | `role="tool"`, `tool_name`, `tool_call_id`, `content` |
|
||||
| Source role → Turnstone role | See "Role Mapping" table |
|
||||
| Per-thread metadata | Store source IDs in `workstream_config` under `import.*` keys |
|
||||
|
||||
## Files to read before writing the importer
|
||||
|
||||
- `turnstone/core/storage/_schema.py` — authoritative table definitions.
|
||||
- `turnstone/core/storage/_protocol.py` — `save_message`, `save_messages_bulk`, `load_messages` signatures.
|
||||
- `turnstone/core/session.py` (around the message-save section) — how the runtime constructs in-memory message dicts; mirror this shape on import to round-trip cleanly.
|
||||
- `turnstone/api/server_schemas.py` — Pydantic shapes for the SDK paths if you go through HTTP.
|
||||
+10
-110
@@ -8,7 +8,7 @@ inter-service communication, powered by [lacme](https://pypi.org/project/lacme/)
|
||||
## Quick Start (Docker Compose)
|
||||
|
||||
```bash
|
||||
docker compose -f turnstone/deploy/compose.yaml -f deploy/docker-compose.tls.yml up
|
||||
docker compose -f compose.yaml -f deploy/docker-compose.tls.yml up
|
||||
```
|
||||
|
||||
This:
|
||||
@@ -19,53 +19,6 @@ This:
|
||||
|
||||
---
|
||||
|
||||
## Browser access (dashboard HTTPS)
|
||||
|
||||
The mTLS above secures **service-to-service** traffic (node↔node, collector and
|
||||
routing proxy → nodes). The **console dashboard itself serves plain HTTP** — and
|
||||
must, because it is the cluster's ACME bootstrap endpoint: new nodes fetch
|
||||
`/acme/ca.pem` and provision their first cert over HTTP, before they have the CA
|
||||
to verify TLS. So the console cannot be HTTPS-only on its port.
|
||||
|
||||
To put the **browser → console** hop on HTTPS, terminate TLS at a reverse proxy
|
||||
in front of the console. The dev stack (root `compose.yaml`) ships a `caddy`
|
||||
service that does exactly this — and it's the only published entry point, so the
|
||||
dashboard is HTTPS by default:
|
||||
|
||||
```bash
|
||||
docker compose up
|
||||
# dashboard: https://localhost:${CONSOLE_HTTPS_PORT:-8443}
|
||||
```
|
||||
|
||||
The production stack (`turnstone/deploy/compose.yaml`) bundles the same `caddy`
|
||||
service, so the dashboard is HTTPS there too. For a real domain and a publicly
|
||||
trusted cert, point Caddy at Let's Encrypt by editing `turnstone/deploy/Caddyfile`.
|
||||
|
||||
```
|
||||
browser --h2 / HTTPS--> caddy:443 --h1.1 / HTTP--> console:8090
|
||||
```
|
||||
|
||||
Caddy uses its **own local CA** (`tls internal`, see `turnstone/deploy/Caddyfile`), so the
|
||||
setup is self-contained with no dependency on the console's ACME path. Trust the
|
||||
local root once to silence the browser warning:
|
||||
|
||||
```bash
|
||||
docker compose exec caddy \
|
||||
cat /data/caddy/pki/authorities/local/root.crt # import into your OS/browser
|
||||
```
|
||||
|
||||
**Can Caddy get its cert from the console's internal CA instead?** Technically
|
||||
yes — the console exposes a real ACME directory (`/acme/directory`) with
|
||||
auto-approval, so Caddy's `tls { ca http://console:8090/acme/directory }` would
|
||||
mint a cert for any name. It's not recommended as the default: lacme's ACME
|
||||
responder is built for turnstone's own client (interop with Caddy's client is
|
||||
unverified), it couples Caddy startup to the console, and the browser must trust
|
||||
a private CA either way — so it buys nothing over `tls internal`. For a publicly
|
||||
trusted cert (no warning), point Caddy at Let's Encrypt with a real domain
|
||||
instead.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
@@ -88,32 +41,6 @@ Console (CA + ACME Server)
|
||||
- **Frontend cert** (HTTPS): From an external ACME CA (e.g. Let's Encrypt)
|
||||
if `tls.acme_directory` is set, otherwise self-issued from the internal CA.
|
||||
|
||||
### Boot, retry, and fallback
|
||||
|
||||
With `tls.enabled`, a node fetches the CA cert and requests its own cert
|
||||
during startup, retrying with exponential backoff (6 attempts, ~31 s total)
|
||||
— enough to absorb a whole-stack restart where every node races the console
|
||||
for its listener. If all attempts fail, the node **falls back to plain
|
||||
HTTP** (availability over confidentiality) and reports `"tls": "fallback"`
|
||||
in `GET /health`; a node serving HTTPS reports `"tls": "active"`, and the
|
||||
key is absent when TLS is disabled. Fallback persists until the next
|
||||
restart — it is not upgraded in place.
|
||||
|
||||
### Container healthcheck under mTLS
|
||||
|
||||
An mTLS listener rejects plain-HTTP probes at the socket, so
|
||||
`docker/healthcheck.py` falls back to HTTPS when the plain probe fails:
|
||||
it presents the node's own cert as the client cert and pins the cluster
|
||||
CA, using the PEM files the server writes at boot under
|
||||
`$TURNSTONE_TLS_PEM_DIR` (default `<tmpdir>/turnstone-tls`). The probe
|
||||
dials `localhost` for the TLS attempt — the internal CA issues DNS SANs
|
||||
only, so a literal-IP URL would fail verification. Cert renewal rewrites
|
||||
the PEM dir alongside the live listener swap, so the probe's client cert
|
||||
never outlives the served cert. With TLS disabled the plain probe succeeds
|
||||
and the PEM directory is never consulted. On bare metal with multiple
|
||||
nodes per host, set `TURNSTONE_TLS_PEM_DIR` per node (each boot clears
|
||||
stale `lacme-pem-*` dirs under its root).
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
@@ -244,20 +171,10 @@ const client = new TurnstoneServer({
|
||||
### Node Bootstrap Flow
|
||||
|
||||
1. Node starts, connects to shared database (plain connection)
|
||||
2. Discovers the console URL from the `services` table — or honors an explicit
|
||||
`TURNSTONE_CONSOLE_URL` (a bare-metal node outside the compose network can't
|
||||
resolve the in-cluster `console` name, so it points this at the console's
|
||||
published ACME endpoint)
|
||||
2. Discovers console URL from `services` table
|
||||
3. Fetches CA root cert from `http://console/acme/ca.pem` (plain HTTP, TOFU)
|
||||
4. Requests a service cert via ACME (plain HTTP, JWS-signed). The cert's
|
||||
primary domain / SAN is the node's **advertised host** (the host of
|
||||
`TURNSTONE_ADVERTISE_URL`, e.g. `node-1`) — the name peers actually dial,
|
||||
not the container hostname. This makes mTLS hostname verification succeed
|
||||
and keys the cert by a stable name that survives container recreation.
|
||||
5. Starts auto-renewal (24h interval, re-issues before expiry) **scoped to its
|
||||
own certificate**. Each node renews only its own cert; the shared store is
|
||||
never swept wholesale. Renewed certs are hot-swapped into the live HTTPS
|
||||
listener with no restart.
|
||||
4. Requests service cert via ACME protocol (plain HTTP, JWS-signed)
|
||||
5. Starts auto-renewal (24h interval, re-issues before expiry)
|
||||
6. All subsequent inter-service communication uses mTLS
|
||||
|
||||
### Console Startup Flow
|
||||
@@ -266,9 +183,7 @@ const client = new TurnstoneServer({
|
||||
2. Initialize CA (load from DB or generate new root key)
|
||||
3. Mount ACME responder at `/acme` (serves `/ca.pem` natively)
|
||||
4. Issue console certs (internal + optional frontend)
|
||||
5. Start CA-direct auto-renewal (no network, signs directly), scoped to the
|
||||
console's own cert, plus a periodic GC that reclaims cert rows for
|
||||
long-departed nodes
|
||||
5. Start CA-direct auto-renewal (no network, signs directly)
|
||||
6. Register console URL in services table with heartbeat
|
||||
|
||||
---
|
||||
@@ -280,32 +195,17 @@ const client = new TurnstoneServer({
|
||||
Certs are valid for 48 hours. If auto-renewal stopped (e.g. console was down),
|
||||
restart the service to re-request a cert.
|
||||
|
||||
### Collector/proxy can't reach a node (TLS hostname mismatch)
|
||||
|
||||
mTLS verifies a node's advertised host against the cert's SANs. Each node's
|
||||
cert is issued for the host in its `TURNSTONE_ADVERTISE_URL`, so that name is
|
||||
always a SAN automatically — you do **not** need to set `TURNSTONE_TLS_SANS`
|
||||
per node. Only set `TURNSTONE_TLS_SANS` to add *extra* names (e.g. a node
|
||||
fronted under a second hostname). Symptom if this is wrong: the console
|
||||
dashboard shows nodes as unreachable and `openssl s_client` reports the served
|
||||
cert's SANs don't include the dialed name.
|
||||
|
||||
### "No console service found"
|
||||
|
||||
The console registers itself in the `services` table on startup. If the console
|
||||
hasn't started or the registration expired (1 hour TTL), nodes can't discover
|
||||
it. Set `TURNSTONE_CONSOLE_URL` to a reachable console address (this is also how
|
||||
a bare-metal node that can't resolve the in-cluster `console` name enrolls).
|
||||
it. Use `--console-url` explicitly.
|
||||
|
||||
### Browser HTTPS to the console
|
||||
### Let's Encrypt for console frontend
|
||||
|
||||
The console serves plain HTTP (it's the ACME bootstrap endpoint — see
|
||||
[Browser access](#browser-access-dashboard-https)). Put browser traffic on
|
||||
HTTPS by terminating TLS at a reverse proxy; the `cluster` profile's `caddy`
|
||||
service does this with Caddy's local CA. For a publicly trusted cert, front the
|
||||
console with a proxy pointed at Let's Encrypt using a real domain. The
|
||||
`tls.acme_directory` setting only governs the console's internal/frontend cert
|
||||
material — it does **not** make the console listen on HTTPS itself.
|
||||
Set `tls.acme_directory` to `https://acme-v02.api.letsencrypt.org/directory`
|
||||
in the admin Settings tab. The console will request a publicly trusted cert
|
||||
for its HTTPS endpoint. Internal mTLS still uses the private CA.
|
||||
|
||||
### Verifying the cert chain
|
||||
|
||||
|
||||
+135
-145
@@ -1,6 +1,6 @@
|
||||
# Tools Reference
|
||||
|
||||
turnstone exposes 17 built-in tools plus any number of external MCP tools to the
|
||||
turnstone exposes 19 built-in tools plus any number of external MCP tools to the
|
||||
LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON
|
||||
files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`.
|
||||
MCP tools are discovered from configured MCP servers at startup by
|
||||
@@ -22,6 +22,7 @@ schema plus turnstone-specific metadata keys:
|
||||
"properties": { ... },
|
||||
"required": ["param1"]
|
||||
},
|
||||
"agent": true,
|
||||
"task_agent": true,
|
||||
"auto_approve": true,
|
||||
"primary_key": "param1"
|
||||
@@ -32,7 +33,8 @@ schema plus turnstone-specific metadata keys:
|
||||
|
||||
| Key | Type | Meaning |
|
||||
|----------------|------|---------|
|
||||
| `task_agent` | bool | Tool is available to task sub-agents. |
|
||||
| `agent` | bool | Tool is available to plan/task sub-agents (read-only subset). |
|
||||
| `task_agent` | bool | Tool is available to task sub-agents (broader subset). |
|
||||
| `auto_approve` | bool | Tool runs without user confirmation (read-only, safe operations). |
|
||||
| `primary_key` | str | When the model sends a bare string instead of JSON args, map it to this parameter name. |
|
||||
|
||||
@@ -44,10 +46,12 @@ schema plus turnstone-specific metadata keys:
|
||||
|
||||
| Name | Description |
|
||||
|---------------------|-------------|
|
||||
| `TOOLS` | All 29 loaded built-in tool definitions (interactive + coordinator union). Sessions send a kind-specific subset (`INTERACTIVE_TOOLS` or `COORDINATOR_TOOLS`). |
|
||||
| `TOOLS` | All 19 tool definitions (sent to the model). |
|
||||
| `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. |
|
||||
| `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. |
|
||||
| `TASK_AUTO_TOOLS` | Set of all tool names with `auto_approve: true` -- used by task-agent sub-sessions to skip confirmation for matching available tools. |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 29 built-in tool names (interactive + coordinator union). Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. |
|
||||
| `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 19 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. |
|
||||
|
||||
---
|
||||
@@ -65,7 +69,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools
|
||||
- Parses the JSON arguments (with fallback for malformed JSON).
|
||||
- If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string
|
||||
to the correct parameter.
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 19
|
||||
built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and
|
||||
the generic `_prepare_mcp_tool()` handler for MCP tools.
|
||||
- Validates arguments and builds a preview dict containing:
|
||||
@@ -107,6 +111,9 @@ Each item's `execute` callable is invoked:
|
||||
denials are tracked separately. This removes the need for text-prefix heuristics.
|
||||
Other tools deliver results atomically via
|
||||
`ui.on_tool_result(call_id, name, output, is_error=...)` only.
|
||||
- Special post-execution gate for `plan`: the plan output is shown to the user
|
||||
for review, and the user can reject or annotate it.
|
||||
|
||||
---
|
||||
|
||||
## Tool Approval Flow
|
||||
@@ -114,6 +121,7 @@ Each item's `execute` callable is invoked:
|
||||
**Auto-approved** (no user confirmation needed at runtime):
|
||||
- `read_file` -- reads files, no side effects
|
||||
- `search` -- grep-style search, no side effects
|
||||
- `man` -- reads man pages, no side effects
|
||||
- `memory` -- structured persistent memory (save/search/delete/list)
|
||||
- `recall` -- searches conversation history
|
||||
- `notify` -- sends notifications to linked channels (time-sensitive, auto-approved for urgency)
|
||||
@@ -122,17 +130,16 @@ Each item's `execute` callable is invoked:
|
||||
- `bash` -- arbitrary command execution
|
||||
- `write_file` -- creates or overwrites files
|
||||
- `edit_file` -- modifies file content
|
||||
- `math` -- sandboxed computation (confirmation required despite being sandboxed)
|
||||
- `web_fetch` -- fetches a URL (SSRF-protected, but makes network requests)
|
||||
- `web_search` -- web search via self-hosted SearxNG (makes network requests)
|
||||
- `task_agent` -- spawns an autonomous sub-agent
|
||||
- `open_preview` -- **URL targets only** (network access, gated like `web_fetch`);
|
||||
file-path and `attachment:` targets are local reads and run unprompted like
|
||||
`read_file`
|
||||
- `web_search` -- web search via Tavily API (makes network requests)
|
||||
- `task` -- spawns an autonomous sub-agent
|
||||
- `plan` -- spawns a planning sub-agent, plus post-execution review gate
|
||||
|
||||
Note: The JSON schema metadata key `auto_approve` controls membership in
|
||||
`TASK_AUTO_TOOLS` (used for task agent sub-sessions). The actual runtime
|
||||
approval behavior is determined by the `needs_approval` field set in each
|
||||
`_prepare_*` method on `ChatSession`. These two mechanisms can differ.
|
||||
`AGENT_AUTO_TOOLS`/`TASK_AUTO_TOOLS` (used for agent sub-sessions). The actual
|
||||
runtime approval behavior is determined by the `needs_approval` field set in
|
||||
each `_prepare_*` method on `ChatSession`. These two mechanisms can differ.
|
||||
|
||||
---
|
||||
|
||||
@@ -158,10 +165,12 @@ Every tool defines a `primary_key`. The mapping is:
|
||||
| `write_file` | `content` |
|
||||
| `edit_file` | `old_string`|
|
||||
| `search` | `query` |
|
||||
| `math` | `code` |
|
||||
| `man` | `page` |
|
||||
| `web_fetch` | `url` |
|
||||
| `web_search` | `query` |
|
||||
| `open_preview` | `target` |
|
||||
| `task_agent` | `prompt` |
|
||||
| `task` | `prompt` |
|
||||
| `plan` | `prompt` |
|
||||
| `memory` | `name` |
|
||||
| `recall` | `query` |
|
||||
| `notify` | `message` |
|
||||
@@ -185,7 +194,7 @@ Execute a bash command and return stdout + stderr.
|
||||
- **What it does**: Runs the command in a subprocess with a configurable timeout. Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`). Environment variables containing secrets are scrubbed (`*_KEY`, `*_SECRET`, `*_TOKEN`, etc.).
|
||||
- **Output format**: Stdout is returned directly. Stderr lines are prefixed with `[stderr]` so the model can distinguish them. When the command itself redirects stderr to stdout (`2>&1`), no prefix is added. Output exceeding 256KB is truncated (head + tail preserved, middle replaced with a truncation notice).
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only.
|
||||
- **Agent availability**: `task_agent` only (not available to plan sub-agents).
|
||||
|
||||
---
|
||||
|
||||
@@ -203,7 +212,7 @@ base64-encoded image data for supported image formats.
|
||||
- **What it does**: For text files, reads and returns content with line numbers. For image files (PNG, JPEG, GIF, WebP, BMP, TIFF, ICO), returns image data as multi-part content when the model supports vision, or a text description when it does not. SVG files are read as text. Images larger than 4 MB are rejected. Must be called before `edit_file` on the same path (the session tracks which files have been read).
|
||||
- **Vision support**: Controlled by `ModelCapabilities.supports_vision`. All commercial OpenAI and Anthropic models have vision enabled. Local models (vLLM, llama.cpp, NIM) default to off — enable via `[models.*.capabilities] supports_vision = true` in config.toml.
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
@@ -259,7 +268,7 @@ Show a unified diff between two files, or between a file and a provided string.
|
||||
|
||||
- **What it does**: Returns unified diff output using Python's `difflib`. Binary files (containing null bytes) are rejected with a clear error. Files read through `diff_file` satisfy `edit_file`'s read guard — you can diff then edit without a separate `read_file` call. Large diffs are streamed with early cutoff at the tool truncation limit.
|
||||
- **Auto-approve**: Yes (read-only).
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
@@ -274,12 +283,44 @@ Search file contents for a regex pattern.
|
||||
|
||||
- **What it does**: Recursively searches for the pattern using `grep -rn`. Returns matching lines with file paths and line numbers.
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
## Computation
|
||||
|
||||
### math
|
||||
|
||||
Execute Python code for math and computation in a sandbox.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `code` | string | yes | Python code to execute. Must use `print()` for output. |
|
||||
|
||||
- **What it does**: Runs Python code in a sandboxed environment with pre-imported libraries: `sympy`, `numpy`, `scipy`, `math`, `fractions`, `itertools`, `functools`, `collections`, `decimal`, `operator`, `random`, `re`, `string`. Common sympy names (`symbols`, `solve`, `simplify`, `sqrt`, `Matrix`, etc.) are pre-imported. `pytest` is also available for import.
|
||||
- **Installation**: `sympy`, `numpy`, `scipy`, and `pytest` require the `[sandbox]` extras group: `pip install turnstone[sandbox]` (included in `[all]`).
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
## Information
|
||||
|
||||
### man
|
||||
|
||||
Read a man page.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `page` | string | yes | The man page name (e.g. `grep`, `socket`, `printf`). |
|
||||
| `section` | string | no | Manual section (e.g. `1` commands, `2` syscalls, `3` library). |
|
||||
|
||||
- **What it does**: Returns the full formatted manual entry. Preferred over `bash('man ...')` or `web_search` for command/API documentation.
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
### web_fetch
|
||||
|
||||
Fetch a URL and extract specific information from it.
|
||||
@@ -289,9 +330,9 @@ Fetch a URL and extract specific information from it.
|
||||
| `url` | string | yes | The URL to fetch (must start with `http://` or `https://`). |
|
||||
| `question` | string | yes | What to extract or answer from the page content. |
|
||||
|
||||
- **What it does**: Fetches the URL, strips HTML to plain text, and uses the LLM to extract the answer to the question from the page content. Every redirect hop is SSRF-screened before it is requested. Private/internal addresses are refused by default; enable `tools.allow_private_network` (console Settings → Tools) to make them approvable for self-hosted setups whose services live on the local network — the approval prompt marks such requests, and a public site redirecting into private space is refused regardless.
|
||||
- **What it does**: Fetches the URL, strips HTML to plain text, and uses the LLM to extract the answer to the question from the page content. Protected against SSRF (blocks private/internal IPs).
|
||||
- **Auto-approve**: No -- requires user confirmation (makes network requests).
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
@@ -303,91 +344,20 @@ Search the web using a text query.
|
||||
|---------------|---------|----------|-------------|
|
||||
| `query` | string | yes | The search query. |
|
||||
| `max_results` | integer | no | Max results to return (default 5, max 20). |
|
||||
| `category` | string | no | Search category: `general` (default), `news`, `it` (code/tech), or `science`. Maps to SearxNG categories; the model picks per query. |
|
||||
| `topic` | string | no | Search topic: `general`, `news`, or `finance` (default `general`). |
|
||||
|
||||
- **What it does**: Searches the web and returns ranked results with titles, URLs, and content snippets. Uses provider-native search when available:
|
||||
- **Anthropic**: Replaced at the API boundary with Anthropic's `web_search_20250305` server-side tool. Claude decides when to search; the API executes it and returns results with citations inline. No backend needed.
|
||||
- **Anthropic**: Replaced at the API boundary with Anthropic's `web_search_20250305` server-side tool. Claude decides when to search; the API executes it and returns results with citations inline. No Tavily key needed.
|
||||
- **OpenAI search models** (`gpt-5-search-api`): Replaced with `web_search_options` parameter. The model always searches and returns `url_citation` annotations.
|
||||
- **Local/vLLM models**: Falls back to a self-hosted [SearxNG](https://searxng.org) instance. Set `searxng_url` in `config.toml` `[tools]` or `$TURNSTONE_SEARXNG_URL` (the docker-compose stack bundles a `searxng` service and points at it by default). Operators with a custom MCP search server can instead set `web_search_backend = "mcp:server:tool"`.
|
||||
- **Local/vLLM models**: Falls back to the Tavily API. Requires `tavily_key` in `config.toml` or `$TAVILY_API_KEY`.
|
||||
- **Auto-approve**: Yes (auto-approved for all tool dispatch paths).
|
||||
- **Agent availability**: `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
### Reranking (optional)
|
||||
|
||||
`web_search` can use an external **reranker** to re-order the backend's result pool by relevance to the query before returning the top hits. Turnstone runs no reranker model itself; it POSTs to a Cohere/Jina-compatible `/rerank` endpoint (self-hosted [vLLM](https://docs.vllm.ai) / [TEI](https://github.com/huggingface/text-embeddings-inference) / llama.cpp, or hosted Cohere/Jina/Voyage).
|
||||
|
||||
**Disabled by default.** In the console **Models** tab, add a model definition whose `base_url` is a Cohere/Jina-compatible `/rerank` endpoint and whose capabilities include `{"supports_rerank": true}`, then select it under **Models → Roles → Reranker**. It's managed like every other model (write-only key, enable/disable, calibration). The reranker is purely this per-model definition — there is no global `rerank_url`-style endpoint setting.
|
||||
|
||||
The `rerank_web_search` toggle defaults on once a reranker is selected. If the endpoint is unreachable or errors, web_search falls back silently to the backend's native result order — reranking never makes a search fail.
|
||||
|
||||
When `rerank_bm25` is enabled, the candidate text for memory, tool, and skill retrieval (memory name/description/content and tool/skill names + descriptions) is also sent to the rerank endpoint — a self-hosted endpoint (vLLM/TEI/llama.cpp) keeps it on your infrastructure, a hosted provider (Cohere/Jina/Voyage) sends it off-box.
|
||||
|
||||
**Serving a Qwen3-Reranker with vLLM.** The model is instruction-aware, so vLLM **must** apply its chat template — pass `--chat-template` explicitly. Without it the bare query produces near-random scores and reranking actively *hurts* retrieval (verified: an irrelevant passage outscored the correct one):
|
||||
|
||||
```bash
|
||||
vllm serve /models/Qwen3-Reranker-0.6B \
|
||||
--runner pooling \
|
||||
--hf-overrides '{"architectures":["Qwen3ForSequenceClassification"],"classifier_from_token":["no","yes"],"is_original_qwen3_reranker":true}' \
|
||||
--chat-template /models/Qwen3-Reranker-0.6B/chat_template.jinja \
|
||||
--served-model-name qwen3-reranker --port 8000
|
||||
```
|
||||
|
||||
Then add a reranker model in the **Models** tab with `base_url` `http://vllm:8000/rerank` (model name `qwen3-reranker`) and select it under **Models → Roles → Reranker**.
|
||||
|
||||
For an endpoint that does *not* apply the model's template, set `rerank_instruction` instead — Turnstone then wraps each query as `<Instruct>: {instruction}` / `<Query>: {query}` (Qwen3's own default is `Given a web search query, retrieve relevant passages that answer the query`). Use the chat template **or** the instruction, not both (they double-wrap).
|
||||
|
||||
**Picking `rerank_bm25_threshold`.** The relevance floor that gates proactive memory injection is a probability in `[0, 1]`, but the right value differs per model (a sharp 0.6B reranker may want ~0.95; a broader 4B ~0.33). Calibrate it against your endpoint:
|
||||
|
||||
```bash
|
||||
turnstone-admin rerank-calibrate # probe the endpoint, recommend a floor
|
||||
turnstone-admin rerank-calibrate --apply # ...and write tools.rerank_bm25_threshold
|
||||
```
|
||||
|
||||
It reports the score scale, whether the endpoint cleanly separates relevant from irrelevant probes (a **"no clean separation"** result flags a mis-served or weak reranker), and the suggested floor. Leave the threshold at `0` to rerank-without-filtering.
|
||||
|
||||
---
|
||||
|
||||
### open_preview
|
||||
|
||||
Show the user rich content in a preview pane beside the conversation.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `target` | string | yes | An http(s) URL, a file path, or `attachment:<id>` for a file attached to the conversation. |
|
||||
| `kind` | string | no | Rendering override: `web`, `pdf`, `image`, `table`, `text`, or `markdown`. Detected from the content when omitted. |
|
||||
| `title` | string | no | Pane header title. Defaults to the page title, filename, or URL. |
|
||||
|
||||
- **What it does**: Resolves the target to bytes (URLs fetch through the same
|
||||
SSRF-guarded path as `web_fetch`, screened per redirect hop, honoring the
|
||||
same `tools.allow_private_network` opt-in), classifies the
|
||||
content, stores it content-addressed against the workstream, and opens the
|
||||
frontend preview pane beside the conversation: web pages render in a fully
|
||||
sandboxed iframe (no scripts, opaque origin), PDFs in the browser viewer,
|
||||
images inline, CSV/TSV/JSON as a sortable table, text/markdown rendered. A
|
||||
previewed web page loads none of its remote images or styles by default, so
|
||||
opening it never reveals the viewer to the page's site; a toggle in the pane
|
||||
header turns remote content back on for that preview. The
|
||||
model receives only a one-line confirmation — to reason about content, use
|
||||
`web_fetch` / `read_file` instead. Preview content is size-capped per kind
|
||||
(pages 4 MB, PDFs 32 MB, images 4 MB, tables 2 MB, text 512 KB) and GC'd
|
||||
with the workstream.
|
||||
- **Auto-approve**: URL targets require confirmation (network access); file
|
||||
paths and `attachment:` targets run unprompted (local reads).
|
||||
- **Agent availability**: interactive sessions only (not `task_agent`, not
|
||||
coordinators).
|
||||
- **Surfaces**: the pane renders in the web UI (standalone and console). The
|
||||
CLI prints the confirmation line only — there is no terminal pane.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
## Agent
|
||||
|
||||
The tool name uses the `_agent` suffix — bare `task` collides with
|
||||
chat-template channel names on some local models.
|
||||
|
||||
### task_agent
|
||||
### task
|
||||
|
||||
Delegate a general-purpose task to an autonomous sub-agent.
|
||||
|
||||
@@ -395,9 +365,23 @@ Delegate a general-purpose task to an autonomous sub-agent.
|
||||
|-----------|--------|----------|-------------|
|
||||
| `prompt` | string | yes | Complete task description for the sub-agent. |
|
||||
|
||||
- **What it does**: Spawns a sub-agent that inherits the `TASK_AGENT_TOOLS` set (read, write, edit, search, bash, web tools, memory tools). The sub-agent runs autonomously to completion. Use for work that requires file modifications or command execution.
|
||||
- **What it does**: Spawns a sub-agent that inherits the `TASK_AGENT_TOOLS` set (read, write, edit, search, bash, math, man, web tools, memory tools). The sub-agent runs autonomously to completion. Use for work that requires file modifications or command execution.
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: Top-level only.
|
||||
- **Agent availability**: Not available to sub-agents (top-level only).
|
||||
|
||||
---
|
||||
|
||||
### plan
|
||||
|
||||
Plan before implementing -- an autonomous agent explores the codebase and writes a structured plan.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `prompt` | string | yes | What to plan -- the goal, constraints, and scope. |
|
||||
|
||||
- **What it does**: Spawns a planning sub-agent with `AGENT_TOOLS` (read-only tools: `read_file`, `search`, `math`, `man`, `web_fetch`, `web_search`). The agent explores the codebase and writes a structured plan to `.plan-<ws_id>.md` (unique per workstream, so concurrent workstreams never collide). If the `plan` tool has been called before in the same session, the prior plan is passed to the agent as context so it refines rather than restarts. After completion, the user is prompted to review and can accept, reject, or annotate the plan.
|
||||
- **Auto-approve**: No -- requires user confirmation, plus post-execution review gate.
|
||||
- **Agent availability**: Not available to sub-agents (top-level only).
|
||||
|
||||
---
|
||||
|
||||
@@ -458,7 +442,7 @@ Provide either `username` for user-based targeting or `channel_type` +
|
||||
|
||||
- **What it does**: Sends a notification via the channel gateway's HTTP endpoint (`POST /v1/api/notify`). The server queries the `services` table for healthy channel gateways, authenticates with a service JWT (`aud: turnstone-channel`), and delivers to the first healthy gateway. On failure, retries up to 2 additional times with backoff (1s, 3s). Rate-limited to 5 notifications per turn (counter only increments on success).
|
||||
- **Auto-approve**: Yes — notifications are time-sensitive and auto-approved so the model can alert users urgently.
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
> See [Channel Integrations: Notifications](channels.md#notifications)
|
||||
> for the full delivery flow, service registry details, and security
|
||||
@@ -534,7 +518,7 @@ data.get("mergedAt") is not None
|
||||
- Duplicate names rejected within the same workstream.
|
||||
|
||||
- **Auto-approve**: `create` requires approval; `list` and `cancel` are auto-approved.
|
||||
- **Agent availability**: Main session only — not available to task sub-agents.
|
||||
- **Agent availability**: Main session only — not available to plan/task sub-agents.
|
||||
|
||||
> See [Watch Architecture](diagrams/png/18-watch-architecture.png) for the
|
||||
> full poll → evaluate → dispatch flow.
|
||||
@@ -559,39 +543,41 @@ pre-configure skills at workstream creation.
|
||||
- `load` — Activate a skill by name. Calls `set_skill()` which handles content
|
||||
rendering with `{{model}}`/`{{ws_id}}`/`{{node_id}}` variables, system message
|
||||
reinitialization, and config persistence. Returns the skill name, description,
|
||||
and security risk level. Warns on high/critical risk level.
|
||||
and security scan tier. Warns on high/critical scan status.
|
||||
- `search` — Find available skills by query. Uses BM25 relevance ranking over
|
||||
name, description, tags, and category (same `BM25Index` used by memory
|
||||
relevance and tool search). Returns up to 10 results with name, description,
|
||||
category, risk level, and activation type.
|
||||
category, scan status, and activation type.
|
||||
|
||||
- **Auto-approve**: `load` requires approval (changes session behavior); `search`
|
||||
is auto-approved (read-only).
|
||||
- **Agent availability**: Main session only — not available to task sub-agents.
|
||||
- **Agent availability**: Main session only — not available to plan/task sub-agents.
|
||||
|
||||
---
|
||||
|
||||
## Summary Table
|
||||
|
||||
| Tool | Category | Auto-approve | task_agent | primary_key |
|
||||
|--------------|------------|--------------|------------|-------------|
|
||||
| `bash` | File Ops | No | Yes | `command` |
|
||||
| `read_file` | File Ops | Yes | Yes | `path` |
|
||||
| `write_file` | File Ops | No | Yes | `content` |
|
||||
| `edit_file` | File Ops | No | Yes | `old_string`|
|
||||
| `search` | File Ops | Yes | Yes | `query` |
|
||||
| `web_fetch` | Info | No | Yes | `url` |
|
||||
| `web_search` | Info | No | Yes | `query` |
|
||||
| `open_preview`| Info | URL: no; path/attachment: yes | No | `target` |
|
||||
| `task_agent` | Agent | No | No | `prompt` |
|
||||
| `memory` | Memory | Yes | No | `name` |
|
||||
| `recall` | Memory | Yes | No | `query` |
|
||||
| `notify` | Notify | Yes | Yes | `message` |
|
||||
| `watch` | Monitor | No (create) | No | `command` |
|
||||
| `read_resource`| MCP | No | Yes | `uri` |
|
||||
| `use_prompt` | MCP | No | Yes | `name` |
|
||||
| `skill` | Skills | No (load) | No | `name` |
|
||||
| `tool_search`| Search | Yes | No | `query` |
|
||||
| Tool | Category | Auto-approve | agent | task_agent | primary_key |
|
||||
|--------------|------------|--------------|-------|------------|-------------|
|
||||
| `bash` | File Ops | No | No | Yes | `command` |
|
||||
| `read_file` | File Ops | Yes | Yes | Yes | `path` |
|
||||
| `write_file` | File Ops | No | No | Yes | `content` |
|
||||
| `edit_file` | File Ops | No | No | Yes | `old_string`|
|
||||
| `search` | File Ops | Yes | Yes | Yes | `query` |
|
||||
| `math` | Compute | No | Yes | Yes | `code` |
|
||||
| `man` | Info | Yes | Yes | Yes | `page` |
|
||||
| `web_fetch` | Info | No | Yes | Yes | `url` |
|
||||
| `web_search` | Info | No | Yes | Yes | `query` |
|
||||
| `task` | Agent | No | No | No | `prompt` |
|
||||
| `plan` | Agent | No | No | No | `prompt` |
|
||||
| `memory` | Memory | Yes | No | No | `name` |
|
||||
| `recall` | Memory | Yes | No | No | `query` |
|
||||
| `notify` | Notify | Yes | Yes | Yes | `message` |
|
||||
| `watch` | Monitor | No (create) | No | No | `command` |
|
||||
| `read_resource`| MCP | No | Yes | Yes | `uri` |
|
||||
| `use_prompt` | MCP | No | Yes | Yes | `name` |
|
||||
| `skill` | Skills | No (load) | No | No | `name` |
|
||||
| `tool_search`| Search | Yes | No | No | `query` |
|
||||
|
||||
---
|
||||
|
||||
@@ -618,11 +604,6 @@ Tool search uses the best available mechanism for each provider:
|
||||
`_exec_tool_search()` runs a pure-Python BM25 index over tool names and
|
||||
descriptions, then expands the matched tools into the visible set.
|
||||
|
||||
A persona with a tool-visibility set overrides this selection: any exact
|
||||
set forces tool search into the client-side BM25 mechanism (tier 3)
|
||||
regardless of provider, and a **hard** set — one whose visible tools omit
|
||||
`tool_search` — disables tool search entirely.
|
||||
|
||||
### Configuration
|
||||
|
||||
Tool search is configured in `config.toml` under the `[tools]` section:
|
||||
@@ -648,9 +629,8 @@ CLI flags override the config file:
|
||||
directly.
|
||||
|
||||
2. **Partitioning**: When active, tools are split into two sets:
|
||||
- **Always-on** -- the built-in tools present in the current session
|
||||
(interactive sessions currently have 16; `BUILTIN_TOOL_NAMES` is the
|
||||
28-tool built-in union). These are always visible to the model.
|
||||
- **Always-on** -- the 19 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
These are always visible to the model.
|
||||
- **Deferred** -- all MCP tools. These are not sent in the tool list unless
|
||||
the model searches for them.
|
||||
|
||||
@@ -664,10 +644,10 @@ CLI flags override the config file:
|
||||
|
||||
### Agent exemption
|
||||
|
||||
Task sub-agents do not use tool search. They operate on the scoped tool set
|
||||
(`TASK_AGENT_TOOLS`) with MCP tools merged in. Tool search is only active for
|
||||
the top-level session, where the model can interactively search for tools it
|
||||
needs.
|
||||
Plan and task sub-agents do not use tool search. They operate on scoped tool
|
||||
sets (`AGENT_TOOLS` for plan agents, `TASK_AGENT_TOOLS` for task agents) with
|
||||
MCP tools merged in. Tool search is only active for the top-level session,
|
||||
where the model can interactively search for tools it needs.
|
||||
|
||||
---
|
||||
|
||||
@@ -692,7 +672,7 @@ MCP-compatible service.
|
||||
3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI
|
||||
function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`.
|
||||
|
||||
4. **Merging**: MCP tools are appended after the 17 built-in tools via
|
||||
4. **Merging**: MCP tools are appended after the 19 built-in tools via
|
||||
`merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority.
|
||||
When dynamic tool search is active, MCP tools are deferred rather than directly
|
||||
visible -- the model discovers them via search as needed (see
|
||||
@@ -718,6 +698,7 @@ gives per-tool-type granularity (e.g., all `use_prompt` calls).
|
||||
MCP tools are available to:
|
||||
- **Main session** — full access
|
||||
- **Task sub-agents** — via `self._task_tools` (merged list)
|
||||
- **Plan sub-agents** — via `self._agent_tools` (merged list)
|
||||
|
||||
### Naming convention
|
||||
|
||||
@@ -774,25 +755,34 @@ MCP tools (3):
|
||||
|
||||
### Dynamic tool refresh
|
||||
|
||||
MCP tool lists stay up-to-date without restart through two mechanisms:
|
||||
MCP tool lists stay up-to-date without restart through three mechanisms:
|
||||
|
||||
1. **Push notifications** -- MCP servers that declare `tools.listChanged: true` in
|
||||
their capabilities send `notifications/tools/list_changed` when their tool list
|
||||
changes. `MCPClientManager` registers a `message_handler` on each `ClientSession`
|
||||
that triggers an immediate refresh for that server.
|
||||
|
||||
2. **Manual** -- `/mcp refresh` re-fetches tools from all servers immediately.
|
||||
2. **Periodic timer** -- Servers that do *not* support push notifications are polled
|
||||
on a configurable interval (default 4 hours). The timer is staggered using a
|
||||
launch-time seed (`monotonic_ns ^ pid`) so cluster nodes don't all hit MCP
|
||||
servers simultaneously. Configure via `[mcp] refresh_interval` in `config.toml`
|
||||
or `--mcp-refresh-interval SECONDS` on the CLI. Set to `0` to disable.
|
||||
|
||||
3. **Manual** -- `/mcp refresh` re-fetches tools from all servers immediately.
|
||||
`/mcp refresh <server>` targets a single server. If a server has disconnected,
|
||||
manual refresh attempts reconnection. The console admin panel exposes the
|
||||
same controls (refresh / reconnect buttons per server) for cluster-wide
|
||||
fan-out.
|
||||
manual refresh attempts reconnection.
|
||||
|
||||
When tools change, `MCPClientManager` rebuilds its merged tool list using copy-on-write
|
||||
(new list/dict objects assigned atomically) and notifies all active `ChatSession`
|
||||
instances via registered listener callbacks. Each session rebuilds its `_tools`,
|
||||
`_task_tools`, and reconstructs its `ToolSearchManager` (if active),
|
||||
`_task_tools`, `_agent_tools`, and reconstructs its `ToolSearchManager` (if active),
|
||||
preserving the set of previously expanded (discovered) tools.
|
||||
|
||||
```toml
|
||||
[mcp]
|
||||
refresh_interval = 14400 # seconds (default 4h), 0 to disable
|
||||
```
|
||||
|
||||
```
|
||||
/mcp refresh
|
||||
MCP refresh complete:
|
||||
@@ -846,7 +836,7 @@ Use read_resource(uri='...') to access the resources listed above.
|
||||
|
||||
- **What it does**: Reads the resource from its MCP server via `MCPClientManager.read_resource_sync()`. Returns text content for text resources or base64-encoded data for binary resources. Output is truncated by the standard tool output limiter.
|
||||
- **Auto-approve**: No -- requires user confirmation (reads external data).
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
### Capability guards
|
||||
|
||||
@@ -888,7 +878,7 @@ the `initialize` handshake. Each prompt is stored with its prefixed name
|
||||
|
||||
- **What it does**: Invokes an MCP prompt by name via `MCPClientManager.get_prompt_sync()`, expanding it into messages. Returns the expanded prompt content formatted as `[role]: content` blocks joined with blank lines. The prompt catalog is listed in the system message so the model knows which prompts are available. Output is truncated by the standard tool output limiter.
|
||||
- **Auto-approve**: No -- requires user confirmation (invokes external prompt servers).
|
||||
- **Agent availability**: `task_agent`.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
### Invocation
|
||||
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"defaults": {
|
||||
"n_runs": 3
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"id": "search-first",
|
||||
"skill": {
|
||||
"name": "search-first",
|
||||
"content": "# Search First\n\nBefore answering ANY question about where something lives in the codebase, you MUST call the `search` tool first. Never answer from memory."
|
||||
},
|
||||
"user_prompt": "Where is JWT token validation implemented in this project?",
|
||||
"expected_actions": [{ "tool": "search" }],
|
||||
"match_mode": "ordered_subset",
|
||||
"max_turns": 4
|
||||
},
|
||||
{
|
||||
"id": "test-after-edit",
|
||||
"skill": {
|
||||
"name": "test-after-edit",
|
||||
"content": "# Test After Edit\n\nAfter editing or writing ANY file, you MUST run the test suite with `python -m pytest` via bash before you finish. Do not report done until tests have run."
|
||||
},
|
||||
"user_prompt": "Add a function `clamp(x, lo, hi)` that clamps x to [lo, hi] in utils.py.",
|
||||
"setup": {
|
||||
"files": {
|
||||
"utils.py": ""
|
||||
}
|
||||
},
|
||||
"expected_actions": [
|
||||
{ "tool": "write_file" },
|
||||
{ "tool": "bash", "args_pattern": { "command": "pytest" } }
|
||||
],
|
||||
"match_mode": "ordered_subset",
|
||||
"max_turns": 8
|
||||
},
|
||||
{
|
||||
"id": "changelog-update",
|
||||
"skill": {
|
||||
"name": "changelog-update",
|
||||
"content": "# Changelog Discipline\n\nWhenever you modify a file, you MUST also append a one-line entry to CHANGELOG.md describing the change in the same task."
|
||||
},
|
||||
"user_prompt": "Fix the off-by-one so pager.py shows the last page. Edit pager.py.",
|
||||
"setup": {
|
||||
"files": {
|
||||
"pager.py": "def last_page(total_items, per_page):\n # off-by-one: drops the final partial page\n return total_items // per_page\n",
|
||||
"CHANGELOG.md": "# Changelog\n"
|
||||
}
|
||||
},
|
||||
"expected_actions": [
|
||||
{ "tool": "edit_file", "args_pattern": { "path": "CHANGELOG.md" } }
|
||||
],
|
||||
"match_mode": "subset",
|
||||
"max_turns": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
__pycache__/
|
||||
.venv/
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
uv.lock
|
||||
@@ -1,282 +0,0 @@
|
||||
# Understone
|
||||
|
||||
A small, multiplayer, BBS-style **ANSI door game** served over the Model
|
||||
Context Protocol (MCP). It is a text RPG in the spirit of *Legend of the Red
|
||||
Dragon* — explore an overworld of box-drawing maps, fight wandering monsters,
|
||||
shop and rest in town, and descend a dungeon — except the "door" is an MCP
|
||||
server and the player drives it by talking to an AI assistant.
|
||||
|
||||
The server is the rules engine and the single source of truth. Players share
|
||||
**one persistent world**: your assistant calls tools, the server returns
|
||||
authoritative frames and facts, and the assistant narrates the story around
|
||||
them.
|
||||
|
||||
This is a self-contained reference example. It depends only on `mcp` — there
|
||||
is no dependency on Turnstone itself — so it runs against any MCP client.
|
||||
|
||||
## How to play
|
||||
|
||||
There is **no prompt to paste and no persona to configure**. The tool schema
|
||||
is the whole interface. Once the server is registered with your assistant:
|
||||
|
||||
1. Tell your assistant you'd like to play an ANSI door game / text dungeon
|
||||
RPG (it can discover the tools by name and description).
|
||||
2. The assistant calls `door_help` to learn how to run the world, then
|
||||
`door_join` with your adventurer's name.
|
||||
3. Play unfolds as a conversation: "head east", "fight it", "rest at the inn".
|
||||
|
||||
Everything the assistant needs to run the game well is returned by
|
||||
`door_help`.
|
||||
|
||||
## Gameplay
|
||||
|
||||
A run is a little RPG loop, played a bit each day:
|
||||
|
||||
- **Explore** the overworld of box-drawing maps. Walking is free, but the wild
|
||||
country has texture — a step may turn up a wandering monster, a purse of
|
||||
gold, a healing spring, a small trap (which can never kill you), or a scrap
|
||||
of old Vale lore. Only one such find happens per move, and the non-combat
|
||||
ones don't interrupt your walk.
|
||||
- **Fight, shop, and heal** in and around town. Fighting and descending one
|
||||
rung of the dungeon each spend one of your daily turns; resting, shopping and
|
||||
moving do not.
|
||||
- **Delve the deep, a rung at a time.** The dungeon is a ladder of guardians:
|
||||
each `descend` faces the next one past your deepest and either advances your
|
||||
depth or bounces you home (your depth persists either way). Carry a few
|
||||
**potions in your satchel** — `quaff` the strongest when you choose, and if a
|
||||
fight would kill you the satchel saves you automatically, the elixir burning
|
||||
down your throat at death's edge. Clearing a rung also yields **forge ore**,
|
||||
which rides the satchel (a won forest fight sometimes turns up a little, too).
|
||||
- **Forge an edge — with gold AND ore.** At the shop's **forge** you can add a
|
||||
+1 edge to your equipped weapon or armour, up to a cap, each step dearer than
|
||||
the last. A step costs gold *and* the ore you won in the deep — so the forge is
|
||||
fed by descending, not just by a fat purse. Watch, too, for the **rare beasts**
|
||||
that prowl the forest: felling one is Herald news and always drops a draught.
|
||||
- **Win the game** by slaying **the Wyrm Below**. Once your hero is seasoned
|
||||
enough AND has plumbed the deep to its floor, `challenge` it at the dungeon. A
|
||||
victory frees the Vale, carves your run into the **Hall of Legends**, and — in
|
||||
the tradition of the classic BBS door games — begins a new life: your
|
||||
character resets to first-day gear and stats but keeps a permanent ★ for every
|
||||
Wyrm slain, ready to do it all again.
|
||||
- **Read the news.** `door_log` is the **Understone Herald**, a shared
|
||||
broadsheet of notable deeds across the whole world — who joined, who rose a
|
||||
level, who was dragged home by a goblin, and who freed the Vale.
|
||||
- **Make it social.** It is a shared world, so you can touch other players.
|
||||
`ambush` a rival who has not yet acted today — a classic
|
||||
style player-kill that robs a sleeping foe of some gold, except the surest
|
||||
defence is simply to take your own turn (an active player is awake and can't
|
||||
be caught). Lose the ambush and *you* are the one who flees, shamed on the
|
||||
feed. `post` a private note another player reads on their next visit (it
|
||||
never reaches the public Herald). Or `gamble` a little gold at the inn's dice
|
||||
against the house. Ambush spends a turn; mail and dice do not.
|
||||
- **Bank your coin.** The inn keeps a strongbox: `deposit` gold into the
|
||||
**vault** and `withdraw` it later (no turn either way). Banked gold is **safe
|
||||
from ambush** — a sleeping-robber only ever lifts what you carry — and it is
|
||||
the one thing that **survives a Wyrm-win reset**, carrying wealth across runs.
|
||||
|
||||
## Installation
|
||||
|
||||
This example uses [`uv`](https://docs.astral.sh/uv/). From the example
|
||||
directory:
|
||||
|
||||
```bash
|
||||
cd examples/door-game
|
||||
uv venv
|
||||
uv pip install -e .
|
||||
```
|
||||
|
||||
That installs the `understone` entry point into the environment.
|
||||
|
||||
To run the tests and quality gates:
|
||||
|
||||
```bash
|
||||
uv pip install -e ".[test,dev]"
|
||||
uv run pytest
|
||||
uv run ruff check .
|
||||
uv run ruff format --check .
|
||||
uv run mypy understone/
|
||||
```
|
||||
|
||||
## Running the server
|
||||
|
||||
By default the server speaks the **stdio** transport, which is how MCP clients
|
||||
launch a per-session subprocess:
|
||||
|
||||
```bash
|
||||
understone
|
||||
```
|
||||
|
||||
To host one shared world over HTTP for several clients, run the
|
||||
**streamable-http** transport as a single long-lived process:
|
||||
|
||||
```bash
|
||||
UNDERSTONE_TRANSPORT=streamable-http understone
|
||||
```
|
||||
|
||||
### Environment variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `UNDERSTONE_DB` | `./understone.db` | SQLite database file for the world's state. |
|
||||
| `UNDERSTONE_WORLD` | _(packaged pack)_ | Directory of a content pack to load instead of the bundled Vale of Understone. |
|
||||
| `UNDERSTONE_TRANSPORT` | `stdio` | `stdio` or `streamable-http`. |
|
||||
| `UNDERSTONE_HOST` | `127.0.0.1` | Bind host (streamable-http only). |
|
||||
| `UNDERSTONE_PORT` | `8077` | Bind port (streamable-http only). |
|
||||
| `UNDERSTONE_PATH` | `/mcp` | HTTP path for the MCP endpoint (streamable-http only). |
|
||||
|
||||
## The Watch — a live spectator view
|
||||
|
||||
When the server runs under the **streamable-http** transport, it also serves a
|
||||
read-only **Watch** page: the lobby TV of the Vale. Point a browser at
|
||||
|
||||
```
|
||||
http://127.0.0.1:8077/watch
|
||||
```
|
||||
|
||||
(the host and port follow `UNDERSTONE_HOST` / `UNDERSTONE_PORT`). It is a
|
||||
period **CRT spectator console** — a green-and-amber phosphor map of the whole
|
||||
world with every adventurer's `☻` marker, a live **Understone Herald** feed, the
|
||||
**Hall of Legends**, and a roster of who is currently abroad. It refreshes every
|
||||
couple of seconds; if it loses contact it dims and reads `SIGNAL LOST` until the
|
||||
server returns. The console's palette follows the pack: a world may pick its own
|
||||
CRT colour with `settings.watch_theme` (`phosphor` green, `amber` gold, `ice`
|
||||
blue, `ember` red), defaulting to the Vale's green if it says nothing.
|
||||
|
||||
The Watch is **strictly read-only**. Input never flows through it — there are no
|
||||
controls, no forms, nothing that can change the world. It reads the same shared
|
||||
state the tools do and paints it; that is all. There is no authentication, in
|
||||
keeping with the rest of this easter-egg server (see the safety note below), so
|
||||
treat the page as you would the MCP endpoint itself.
|
||||
|
||||
> _Screenshot: the Watch console — a phosphor-green overworld map with amber
|
||||
> `☻` markers, the Herald feed and Hall of Legends down the right-hand rail.
|
||||
> (Image placeholder; run the server and open the URL to see it live.)_
|
||||
|
||||
When the Watch is up, the `door_join` welcome and the `door_help` manual both
|
||||
print its URL so players (and the assistant narrating for them) know it exists.
|
||||
If you bind to `0.0.0.0` to share the world across a network, advertise a host
|
||||
that browsers can actually reach (your machine's LAN address or hostname) rather
|
||||
than `0.0.0.0` itself — the link is composed from `UNDERSTONE_HOST`.
|
||||
|
||||
## Authoring worlds
|
||||
|
||||
The Vale of Understone is just the *bundled* world. The whole game — its map,
|
||||
monsters, economy, and endgame — is a **content pack**: a directory of six JSON
|
||||
files the server loads at start. Nothing about the Vale is privileged; point
|
||||
the server at another pack and it runs that world instead. This is the seam
|
||||
where the game becomes its own authoring target: a pack is plain data, so a
|
||||
person *or an LLM* can write one, and the same zero-setup philosophy that makes
|
||||
the game playable with no prompt makes it **authorable with no code**.
|
||||
|
||||
The loop has these commands:
|
||||
|
||||
```bash
|
||||
understone newpack mypack # scaffold a pack (copies the Vale as a template)
|
||||
# ...edit or LLM-generate the JSON in mypack/ to describe your world...
|
||||
understone validate mypack # check it; prints a report or names what's wrong
|
||||
understone simulate mypack # play a greedy bot through it and measure the balance
|
||||
UNDERSTONE_WORLD=mypack understone # serve your world
|
||||
understone worlds # list the bundled worlds and whether each is sound
|
||||
```
|
||||
|
||||
`newpack` writes a starting template plus an `AUTHORING.md` manual — the
|
||||
file-by-file schema, the enforced limits, and design guidance — written to be
|
||||
followed cold by a model. `validate` loads the pack through exactly the same
|
||||
hardened loader the server uses and either prints a summary ending **"This pack
|
||||
is sound. The door stands open."** or fails with one precise line naming the
|
||||
file, the row, and the field at fault.
|
||||
|
||||
`simulate` is the **balance instrument**: it drives a deliberately simple,
|
||||
greedy bot through the *real* game — the same `join`/`move`/`action` calls the
|
||||
tools make — over a seeded RNG and an injected clock, then prints a report
|
||||
(final level, gold earned, fights fought, rungs cleared, whether and when the
|
||||
Wyrm fell). It is a tuning probe, not a player to admire: it answers "is this
|
||||
world *shaped* right, and is it *winnable*?". Pass `--days N`, `--seed S`, or
|
||||
`--seeds K` for a multi-seed sweep with means and spreads. `worlds` lists every
|
||||
bundled world — the default Vale plus any alternate packs shipped under
|
||||
`understone/world/packs/` — loading each so it can report it as sound or flawed.
|
||||
|
||||
**A second bundled world: The Cinder Wastes.** Understone ships a second world
|
||||
alongside the Vale, in `understone/world/packs/cinder-wastes/` — a volcanic
|
||||
ash-and-slag map whose Watch page glows ember-red instead of the Vale's green
|
||||
phosphor. It is the pipeline's own dogfood: it was authored **by an LLM working
|
||||
only from `AUTHORING.md` and the `validate` loop**, with no engine code touched,
|
||||
then bundled verbatim. `understone worlds` lists it as sound, and
|
||||
`understone simulate understone/world/packs/cinder-wastes --days 50 --seeds 3`
|
||||
shows the greedy bot taking its Magma Wyrm — the end-to-end proof that a world
|
||||
described purely as data, from the manual alone, is genuinely playable to
|
||||
victory. Serve it with
|
||||
`UNDERSTONE_WORLD=understone/world/packs/cinder-wastes understone`.
|
||||
|
||||
Packs are validated **hard** at load: every map glyph must render as exactly
|
||||
one terminal column (no fullwidth runes, no emoji, no combining marks — the
|
||||
frames are box-drawing rectangles) and may not collide with the frame's
|
||||
box-drawing lines or the player markers, dimensions and counts are bounded,
|
||||
display names are length-checked, and every cross-reference (a legend
|
||||
character, a starting item, the boss monster, a dungeon tier) must resolve. The
|
||||
loader also pins the rules that keep the endgame coherent: a world has exactly
|
||||
one boss, and a dungeon tier's lead monster (its fixed rung guardian) may not be
|
||||
a rare. Because packs are now routinely untrusted, generated output, those error
|
||||
messages are not a nuisance — they are the **feedback loop**. Iterate against
|
||||
them until the door stands open.
|
||||
|
||||
## Registering with Turnstone
|
||||
|
||||
Understone is an ordinary MCP server, so it plugs into Turnstone's MCP client
|
||||
config two ways.
|
||||
|
||||
**Stdio (per-session subprocess).** Turnstone launches the `understone`
|
||||
command for each session. Each session gets its own subprocess, so for a
|
||||
truly shared world prefer the HTTP form below; stdio is simplest for solo
|
||||
play.
|
||||
|
||||
```toml
|
||||
[mcp.servers.understone]
|
||||
command = "understone"
|
||||
|
||||
[mcp.servers.understone.env]
|
||||
UNDERSTONE_DB = "/var/lib/understone/world.db"
|
||||
```
|
||||
|
||||
**Streamable-HTTP (one shared world).** Run a single Understone process with
|
||||
`UNDERSTONE_TRANSPORT=streamable-http` and point every client at its URL. This
|
||||
is the right setup for multiplayer: one process, one database, one world that
|
||||
all adventurers share.
|
||||
|
||||
```toml
|
||||
[mcp.servers.understone]
|
||||
url = "http://localhost:8077/mcp"
|
||||
```
|
||||
|
||||
> **Operator note.** For multiplayer, start exactly one shared process —
|
||||
> `UNDERSTONE_TRANSPORT=streamable-http understone` — and have all clients use
|
||||
> the url form. The world lives in a single SQLite file written by that one
|
||||
> process.
|
||||
|
||||
## The tools
|
||||
|
||||
| Tool | What it does |
|
||||
|------|--------------|
|
||||
| `door_help` | The game-master manual. Start here. |
|
||||
| `door_join` | Create or resume an adventurer; returns the opening map. |
|
||||
| `door_status` | The character sheet (read-only). |
|
||||
| `door_look` | Redraw the current view — overworld map or location menu. |
|
||||
| `door_move` | Walk the overworld (free; no daily turn spent). |
|
||||
| `door_action` | Context verbs: fight, flee, ambush (a rival), rest, deposit/withdraw (the inn vault), buy, sell, forge (a +1 edge, gold + ore), heal, gamble (inn dice), descend (one rung), challenge (the Wyrm), post (mail another player), quaff (a carried potion), leave. |
|
||||
| `door_log` | The Understone Herald — the shared feed of notable deeds. |
|
||||
| `door_rank` | The leaderboard, plus the Hall of Legends (★ marks Wyrm kills). |
|
||||
| `door_bestow` | Game-master grant of a little gold/healing for a story beat. |
|
||||
|
||||
## A note on identity and safety
|
||||
|
||||
This example is an **easter egg**, not a hardened service. Identity is
|
||||
**self-asserted**: a "player" is just a name passed to the tools, and there is
|
||||
**no authentication** — anyone who can reach the server can act as any name.
|
||||
That is fine for a shared toy world among people who trust each other, and
|
||||
deliberately out of scope for a game. Do not store anything sensitive in it,
|
||||
and if you expose the HTTP transport beyond localhost, put it behind whatever
|
||||
access control your environment already provides.
|
||||
|
||||
The game master's `door_bestow` channel can only grant small, capped amounts
|
||||
of in-game gold and healing — never items, never turns — and every grant is
|
||||
written to the public in-world log, so its reach is bounded by design.
|
||||
@@ -1,55 +0,0 @@
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.29"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "understone"
|
||||
version = "0.10.0"
|
||||
description = "Understone — a BBS-style ANSI door game served over MCP."
|
||||
requires-python = ">=3.11"
|
||||
license = "Apache-2.0"
|
||||
dependencies = [
|
||||
"mcp>=1.27,<2",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
understone = "understone.server:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest>=9.0"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14"]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["understone"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "SIM", "TCH"]
|
||||
ignore = ["E501"]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.11"
|
||||
strict = true
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
check_untyped_defs = true
|
||||
no_implicit_optional = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["mcp", "mcp.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
disallow_untyped_defs = false
|
||||
@@ -1,256 +0,0 @@
|
||||
"""Shared test fixtures and builders.
|
||||
|
||||
These builders construct engine objects directly (no JSON loader) so the
|
||||
engine tests stay independent of the content pack. Later chunks add
|
||||
fixtures that load the shipped world and build the game façade.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from understone.engine.models import (
|
||||
Item,
|
||||
LocationDef,
|
||||
Mode,
|
||||
Monster,
|
||||
Player,
|
||||
Settings,
|
||||
Slot,
|
||||
TerrainDef,
|
||||
WorldEvent,
|
||||
Zone,
|
||||
)
|
||||
from understone.engine.world import World
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from understone.game import Game
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Terrain kinds for synthetic test worlds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GRASS = TerrainDef(key="grass", glyph=".", walkable=True, encounter_rate=0.0, color="floor")
|
||||
WALL = TerrainDef(key="wall", glyph="█", walkable=False, encounter_rate=0.0, color="wall")
|
||||
WATER = TerrainDef(key="water", glyph="~", walkable=False, encounter_rate=0.0, color="water")
|
||||
FOREST = TerrainDef(key="forest", glyph="↑", walkable=True, encounter_rate=1.0, color="tree")
|
||||
SAFE_FOREST = TerrainDef(key="forest", glyph="↑", walkable=True, encounter_rate=0.0, color="tree")
|
||||
|
||||
|
||||
DEFAULT_SETTINGS = Settings(
|
||||
daily_turns=10,
|
||||
rest_cost=15,
|
||||
heal_cost_per_hp=2,
|
||||
starting_gold=20,
|
||||
starting_weapon="rusty_dagger",
|
||||
starting_armor="cloth_tunic",
|
||||
start_hp=20,
|
||||
start_atk=3,
|
||||
start_def=0,
|
||||
xp_base=100,
|
||||
growth_max_hp=6,
|
||||
growth_atk=2,
|
||||
growth_def=1,
|
||||
bestow_daily_budget=25,
|
||||
dungeon_tiers=(4, 5),
|
||||
boss_monster="wyrm_below",
|
||||
wyrm_min_level=6,
|
||||
ambush_min_level=3,
|
||||
ambush_level_band=2,
|
||||
ambush_gold_pct=25,
|
||||
post_daily_cap=5,
|
||||
gamble_max_bet=50,
|
||||
gamble_daily_cap=5,
|
||||
satchel_max=3,
|
||||
forge_base_cost=60,
|
||||
forge_max_plus=3,
|
||||
rare_drop_item="minor_potion",
|
||||
forge_ore_item="iron_ore",
|
||||
forge_ore_per_plus=1,
|
||||
ore_dungeon_drop=2,
|
||||
ore_forest_chance=0.2,
|
||||
watch_theme="phosphor",
|
||||
)
|
||||
|
||||
|
||||
def make_settings(**overrides: object) -> Settings:
|
||||
"""Return DEFAULT_SETTINGS with field overrides for band testing."""
|
||||
base = {
|
||||
"daily_turns": DEFAULT_SETTINGS.daily_turns,
|
||||
"rest_cost": DEFAULT_SETTINGS.rest_cost,
|
||||
"heal_cost_per_hp": DEFAULT_SETTINGS.heal_cost_per_hp,
|
||||
"starting_gold": DEFAULT_SETTINGS.starting_gold,
|
||||
"starting_weapon": DEFAULT_SETTINGS.starting_weapon,
|
||||
"starting_armor": DEFAULT_SETTINGS.starting_armor,
|
||||
"start_hp": DEFAULT_SETTINGS.start_hp,
|
||||
"start_atk": DEFAULT_SETTINGS.start_atk,
|
||||
"start_def": DEFAULT_SETTINGS.start_def,
|
||||
"xp_base": DEFAULT_SETTINGS.xp_base,
|
||||
"growth_max_hp": DEFAULT_SETTINGS.growth_max_hp,
|
||||
"growth_atk": DEFAULT_SETTINGS.growth_atk,
|
||||
"growth_def": DEFAULT_SETTINGS.growth_def,
|
||||
"bestow_daily_budget": DEFAULT_SETTINGS.bestow_daily_budget,
|
||||
"dungeon_tiers": DEFAULT_SETTINGS.dungeon_tiers,
|
||||
"boss_monster": DEFAULT_SETTINGS.boss_monster,
|
||||
"wyrm_min_level": DEFAULT_SETTINGS.wyrm_min_level,
|
||||
"ambush_min_level": DEFAULT_SETTINGS.ambush_min_level,
|
||||
"ambush_level_band": DEFAULT_SETTINGS.ambush_level_band,
|
||||
"ambush_gold_pct": DEFAULT_SETTINGS.ambush_gold_pct,
|
||||
"post_daily_cap": DEFAULT_SETTINGS.post_daily_cap,
|
||||
"gamble_max_bet": DEFAULT_SETTINGS.gamble_max_bet,
|
||||
"gamble_daily_cap": DEFAULT_SETTINGS.gamble_daily_cap,
|
||||
"satchel_max": DEFAULT_SETTINGS.satchel_max,
|
||||
"forge_base_cost": DEFAULT_SETTINGS.forge_base_cost,
|
||||
"forge_max_plus": DEFAULT_SETTINGS.forge_max_plus,
|
||||
"rare_drop_item": DEFAULT_SETTINGS.rare_drop_item,
|
||||
"forge_ore_item": DEFAULT_SETTINGS.forge_ore_item,
|
||||
"forge_ore_per_plus": DEFAULT_SETTINGS.forge_ore_per_plus,
|
||||
"ore_dungeon_drop": DEFAULT_SETTINGS.ore_dungeon_drop,
|
||||
"ore_forest_chance": DEFAULT_SETTINGS.ore_forest_chance,
|
||||
"watch_theme": DEFAULT_SETTINGS.watch_theme,
|
||||
}
|
||||
base.update(overrides)
|
||||
return Settings(**base) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_player(**overrides: object) -> Player:
|
||||
"""Build a Player at sane defaults; override any field by keyword."""
|
||||
fields = {
|
||||
"name": "Tester",
|
||||
"x": 5,
|
||||
"y": 5,
|
||||
"hp": 20,
|
||||
"max_hp": 20,
|
||||
"level": 1,
|
||||
"xp": 0,
|
||||
"gold": 50,
|
||||
"atk": 5,
|
||||
"def_": 1,
|
||||
"weapon_id": "rusty_dagger",
|
||||
"armor_id": "cloth_tunic",
|
||||
"turns_left": 10,
|
||||
"turn_day": 0,
|
||||
"mode": Mode.TILE,
|
||||
"at_location": "",
|
||||
"created_at": "2026-01-01T00:00:00+00:00",
|
||||
"last_seen": "2026-01-01T00:00:00+00:00",
|
||||
"log_cursor": 0,
|
||||
"bestow_spent": 0,
|
||||
"bestow_day": 0,
|
||||
"wins": 0,
|
||||
"posts_sent": 0,
|
||||
"post_day": 0,
|
||||
"gambles": 0,
|
||||
"gamble_day": 0,
|
||||
}
|
||||
fields.update(overrides)
|
||||
return Player(**fields) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_monster(**overrides: object) -> Monster:
|
||||
"""Build a Monster at tier-1 defaults."""
|
||||
fields = {
|
||||
"tier": 1,
|
||||
"name": "Field Rat",
|
||||
"hp": 6,
|
||||
"atk": 3,
|
||||
"def_": 0,
|
||||
"xp": 8,
|
||||
"gold": 3,
|
||||
"monster_id": "",
|
||||
"boss": False,
|
||||
}
|
||||
fields.update(overrides)
|
||||
return Monster(**fields) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_world(
|
||||
*,
|
||||
grid: list[list[TerrainDef]] | None = None,
|
||||
width: int = 11,
|
||||
height: int = 11,
|
||||
spawn: tuple[int, int] = (5, 5),
|
||||
locations: list[LocationDef] | None = None,
|
||||
zones: list[Zone] | None = None,
|
||||
monsters: list[Monster] | None = None,
|
||||
items: list[Item] | None = None,
|
||||
settings: Settings | None = None,
|
||||
events: list[WorldEvent] | None = None,
|
||||
) -> World:
|
||||
"""Build a small synthetic World (all-grass by default)."""
|
||||
if grid is None:
|
||||
grid = [[GRASS for _ in range(width)] for _ in range(height)]
|
||||
return World(
|
||||
name="Test Vale",
|
||||
width=width,
|
||||
height=height,
|
||||
spawn=spawn,
|
||||
terrain=grid,
|
||||
locations=locations or [],
|
||||
zones=zones or [],
|
||||
monsters=monsters or [make_monster()],
|
||||
items=items or _default_items(),
|
||||
settings=settings or DEFAULT_SETTINGS,
|
||||
events=events,
|
||||
)
|
||||
|
||||
|
||||
def _default_items() -> list[Item]:
|
||||
return [
|
||||
Item("rusty_dagger", "Rusty Dagger", Slot.WEAPON, 2, 0, 0, 0),
|
||||
Item("short_sword", "Short Sword", Slot.WEAPON, 5, 0, 0, 40),
|
||||
Item("cloth_tunic", "Cloth Tunic", Slot.ARMOR, 0, 1, 0, 0),
|
||||
Item("leather_armor", "Leather Armor", Slot.ARMOR, 0, 3, 0, 50),
|
||||
Item("minor_potion", "Minor Potion", Slot.CONSUMABLE, 0, 0, 15, 12),
|
||||
Item("iron_ore", "Iron Ore", Slot.MATERIAL, 0, 0, 0, 0),
|
||||
]
|
||||
|
||||
|
||||
def fixed_clock(moment: datetime) -> Callable[[], datetime]:
|
||||
"""Return a clock callable that always reports *moment*."""
|
||||
|
||||
def _clock() -> datetime:
|
||||
return moment
|
||||
|
||||
return _clock
|
||||
|
||||
|
||||
def utc(year: int, month: int, day: int, hour: int = 0, minute: int = 0) -> datetime:
|
||||
"""Construct a tz-aware UTC datetime."""
|
||||
return datetime(year, month, day, hour, minute, tzinfo=UTC)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Satchel test helpers (the v0.10 stack encoding)
|
||||
# ---------------------------------------------------------------------------
|
||||
# The satchel is stack-based ("id:qty"); these wrap the game façade's stack
|
||||
# helpers so a test can seed/read a bag as a flat id list (duplicate ids
|
||||
# collapse to one stack), keeping the assertions readable. Shared by the
|
||||
# descend and Wyrm suites.
|
||||
|
||||
|
||||
def set_satchel(game: Game, player: object, ids: list[str]) -> None:
|
||||
"""Seed *player*'s satchel from a flat id list (duplicates -> one stack qty)."""
|
||||
counts = Counter(ids)
|
||||
stacks = [(item_id, counts[item_id]) for item_id in dict.fromkeys(ids)]
|
||||
game._satchel_set_stacks(player, stacks) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def satchel_ids(game: Game, player: object) -> list[str]:
|
||||
"""Return the satchel as a flat id list, each stack expanded by its qty."""
|
||||
out: list[str] = []
|
||||
for item_id, qty in game._satchel_stacks(player): # type: ignore[arg-type]
|
||||
out.extend([item_id] * qty)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def small_world() -> World:
|
||||
"""An 11x11 all-grass world with the default content tables."""
|
||||
return make_world()
|
||||
@@ -1,7 +0,0 @@
|
||||
┌── The Sleeping Drake ───┐
|
||||
│ A warm hearth crackles. │
|
||||
│ A bed costs 15 gold. │
|
||||
│ │
|
||||
│ (R)est (L)eave │
|
||||
└─────────────────────────┘
|
||||
[ status ]
|
||||
@@ -1,8 +0,0 @@
|
||||
┌─ Vale ──┐
|
||||
│@........│
|
||||
│.........│
|
||||
│.........│
|
||||
│.........│
|
||||
│.........│
|
||||
└─────────┘
|
||||
[ status ]
|
||||
@@ -1,8 +0,0 @@
|
||||
┌─ Vale ──┐
|
||||
│.........│
|
||||
│.........│
|
||||
│....@....│
|
||||
│.........│
|
||||
│.........│
|
||||
└─────────┘
|
||||
[ status ]
|
||||
@@ -1,374 +0,0 @@
|
||||
"""Tests for the pack-authoring command surface.
|
||||
|
||||
Covers the validate/newpack functions directly (sound and broken packs, the
|
||||
scaffold round-trip, AUTHORING.md generation from the live loader bands, and
|
||||
the refuse-non-empty guard), the ``server.main`` argv dispatch (validate routes
|
||||
through and bare invocation still reaches serve without binding a port), and
|
||||
one end-to-end subprocess smoke of ``python -m understone validate``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from understone import cli, server
|
||||
from understone.world import loader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
EXAMPLE_DIR = Path(__file__).resolve().parents[1]
|
||||
SHIPPED = EXAMPLE_DIR / "understone" / "world" / "data"
|
||||
|
||||
# The six content files a scaffolded pack must carry, plus the manual.
|
||||
_PACK_JSONS = {
|
||||
"terrain.json",
|
||||
"monsters.json",
|
||||
"items.json",
|
||||
"locations.json",
|
||||
"events.json",
|
||||
"world.json",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cli_validate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cli_validate_sound_pack_reports_and_returns_zero() -> None:
|
||||
out, err = StringIO(), StringIO()
|
||||
rc = cli.cli_validate(SHIPPED, out=out, err=err)
|
||||
|
||||
assert rc == 0
|
||||
report = out.getvalue()
|
||||
assert "This pack is sound. The door stands open." in report
|
||||
# The report surfaces the headline facts the brief calls for.
|
||||
assert "The Vale of Understone" in report
|
||||
assert "96x48" in report
|
||||
assert "1 boss" in report
|
||||
assert "% fight" in report
|
||||
assert err.getvalue() == ""
|
||||
|
||||
|
||||
def test_cli_validate_broken_pack_names_field_and_returns_two(tmp_path: Path) -> None:
|
||||
# A pack whose daily_turns is out of band: the loader names the field.
|
||||
pack = _clone_shipped(tmp_path)
|
||||
_patch_world(pack, _break_daily_turns)
|
||||
|
||||
out, err = StringIO(), StringIO()
|
||||
rc = cli.cli_validate(pack, out=out, err=err)
|
||||
|
||||
assert rc == 2
|
||||
message = err.getvalue()
|
||||
assert message.startswith("The pack is flawed:")
|
||||
assert "daily_turns" in message # the offending field is named
|
||||
assert out.getvalue() == ""
|
||||
|
||||
|
||||
def test_cli_validate_missing_directory_returns_two(tmp_path: Path) -> None:
|
||||
out, err = StringIO(), StringIO()
|
||||
rc = cli.cli_validate(tmp_path / "nope", out=out, err=err)
|
||||
assert rc == 2
|
||||
assert "The pack is flawed:" in err.getvalue()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cli_newpack
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cli_newpack_writes_template_and_manual(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "mypack"
|
||||
out, err = StringIO(), StringIO()
|
||||
rc = cli.cli_newpack(dest, out=out, err=err)
|
||||
|
||||
assert rc == 0
|
||||
present = {p.name for p in dest.iterdir()}
|
||||
assert present >= _PACK_JSONS # the six content files are all there
|
||||
assert "AUTHORING.md" in present
|
||||
# Next-steps guidance points the author at the validate verb.
|
||||
assert "understone validate" in out.getvalue()
|
||||
|
||||
|
||||
def test_cli_newpack_scaffold_validates(tmp_path: Path) -> None:
|
||||
"""The load-bearing test: a freshly scaffolded pack loads cleanly.
|
||||
|
||||
newpack -> load_world round-trip. If the template the scaffolder copies
|
||||
ever drifts out of the loader's bands, this fails immediately.
|
||||
"""
|
||||
dest = tmp_path / "mypack"
|
||||
assert cli.cli_newpack(dest, out=StringIO(), err=StringIO()) == 0
|
||||
|
||||
world = loader.load_world(dest)
|
||||
assert world.name == "The Vale of Understone"
|
||||
assert world.width == 96
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_renders_live_band(tmp_path: Path) -> None:
|
||||
"""AUTHORING.md's bands are generated from the loader, not hand-copied.
|
||||
|
||||
The daily_turns band is read straight from the live loader table and must
|
||||
appear verbatim in the scaffolded manual — proving generation from source.
|
||||
"""
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
lo, hi = loader.SETTINGS_BANDS["daily_turns"]
|
||||
assert lo is not None and hi is not None
|
||||
assert f"`{lo}..{hi}`" in manual
|
||||
assert "daily_turns" in manual
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_has_width_rule_and_live_palette(tmp_path: Path) -> None:
|
||||
"""AUTHORING.md documents the one-column rule and renders the live palette.
|
||||
|
||||
The width section states the Western-monospace assumption, and the safe
|
||||
palette is generated from ``textwidth.SAFE_PALETTE`` (same can't-drift
|
||||
pattern as the bands table) — every glyph appears, in a backticked cell.
|
||||
"""
|
||||
from understone.engine.textwidth import SAFE_PALETTE
|
||||
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
|
||||
assert "## Glyph width" in manual
|
||||
assert "exactly one terminal column" in manual
|
||||
assert "Western monospace" in manual # the stated assumption
|
||||
assert "Safe glyph palette" in manual
|
||||
for glyph in SAFE_PALETTE:
|
||||
assert f"`{glyph}`" in manual, f"palette glyph {glyph!r} missing from manual"
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_documents_action_sets(tmp_path: Path) -> None:
|
||||
"""AUTHORING.md documents each building's real verb menu.
|
||||
|
||||
The per-building menus are an explicit table: the inn's `gamble` (v0.8) and
|
||||
the v0.10 vault verbs `deposit`/`withdraw`, the shop's `forge`, and so on.
|
||||
This pins the table rows and the "quaff anywhere" note so a doc regression
|
||||
trips.
|
||||
"""
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
|
||||
assert "| `inn` | `rest`, `deposit`, `withdraw`, `gamble`, `leave` |" in manual
|
||||
assert "| `shop` | `buy`, `sell`, `forge`, `leave` |" in manual
|
||||
assert "| `healer` | `heal`, `leave` |" in manual
|
||||
assert "| `dungeon` | `descend`, `challenge`, `leave` |" in manual
|
||||
assert "`quaff`" in manual and "legal **anywhere**" in manual
|
||||
# The vault is described where its verbs are listed.
|
||||
assert "VAULT" in manual and "SAFE from ambush" in manual
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_documents_ore_forge(tmp_path: Path) -> None:
|
||||
"""AUTHORING.md documents the v0.10 ore-gated forge: material slot + settings.
|
||||
|
||||
The forge ore is a `material` item earned in combat; the four ore settings
|
||||
(item, per-plus, dungeon drop, forest chance) are documented, and the band
|
||||
figures are generated from the live loader so they cannot drift.
|
||||
"""
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
|
||||
assert "`material`" in manual # the new slot
|
||||
assert "forge_ore_item" in manual
|
||||
assert "ore_forest_chance" in manual # the float setting (prose, not the band table)
|
||||
# The two banded ore settings carry their LIVE bands.
|
||||
lo, hi = loader.SETTINGS_BANDS["ore_dungeon_drop"]
|
||||
assert f"`{lo}..{hi}`" in manual
|
||||
assert "earns in combat" in manual or "earned in combat" in manual
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_states_color_advisory_and_spawn_walkable(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""AUTHORING.md states color is advisory (loader does not validate it) and
|
||||
that spawn must be on walkable terrain — both v0.8 honesty fixes."""
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
|
||||
# color is documented as advisory / not validated (it matches loader behaviour).
|
||||
assert "advisory and not validated" in manual
|
||||
# spawn's walkability requirement is now stated where spawn is introduced.
|
||||
assert "must be on walkable terrain" in manual
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_color_roles_generated_from_enum(tmp_path: Path) -> None:
|
||||
"""AUTHORING.md's colour-role vocabulary is generated from the Color enum.
|
||||
|
||||
The v0.9 fix: the assignable roles were hand-listed (and went stale — road
|
||||
and the per-building roles were missing). They are now generated from
|
||||
``Color.assignable()`` — the single source for the overlay-vs-assignable
|
||||
split — so the manual lists exactly what the Watch can paint and cannot
|
||||
drift. This asserts the NEW roles appear, that every assignable enum role
|
||||
appears, and that the non-assignable roles (overlays + DEFAULT) are NOT
|
||||
offered as author-assignable.
|
||||
"""
|
||||
from understone.screen.palette import Color
|
||||
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
|
||||
# A sampling of the new v0.9 roles is offered in the manual, backticked.
|
||||
for role in ("road", "forest", "lava", "barren", "inn", "shop", "healer"):
|
||||
assert f"`{role}`" in manual, f"new colour role {role!r} missing from manual"
|
||||
|
||||
# EVERY assignable enum role appears (generated, so the full set is present).
|
||||
color_section = manual[manual.index("`color` — a palette role string") :].split("###", 1)[0]
|
||||
for role in Color.assignable():
|
||||
assert f"`{role.value}`" in manual, f"assignable role {role.value!r} missing from manual"
|
||||
|
||||
# The non-assignable roles (runtime overlays + the DEFAULT fallback) are NOT
|
||||
# offered as terrain/location colours.
|
||||
non_assignable = {c for c in Color} - set(Color.assignable())
|
||||
assert Color.DEFAULT in non_assignable # the fallback is not author-pickable
|
||||
for role in non_assignable:
|
||||
assert f"`{role.value}`" not in color_section, (
|
||||
f"non-assignable role {role.value!r} wrongly offered as author-assignable"
|
||||
)
|
||||
|
||||
|
||||
def test_cli_newpack_authoring_md_has_validate_coverage_split(tmp_path: Path) -> None:
|
||||
"""AUTHORING.md honestly separates machine-enforced rules from eyeball-only.
|
||||
|
||||
The v0.8 subsection lists what `validate` DOES catch (including the two new
|
||||
enforcements — rare-as-guardian and single-boss) and what it does NOT (chief
|
||||
among them: location menu `actions` contents are unvalidated).
|
||||
"""
|
||||
dest = tmp_path / "mypack"
|
||||
cli.cli_newpack(dest, out=StringIO(), err=StringIO())
|
||||
manual = (dest / "AUTHORING.md").read_text(encoding="utf-8")
|
||||
|
||||
assert "What `validate` checks, and what it cannot" in manual
|
||||
# The newly-enforced rules are named in the DOES-catch list.
|
||||
assert "Exactly one boss" in manual
|
||||
assert "fixed rung guardian) must" in manual # rare-as-guardian enforcement
|
||||
# The eyeball-only short list names the actions gap and the flavour caveat.
|
||||
assert "Location menu `actions` contents" in manual
|
||||
assert "Flavour and narration quality" in manual
|
||||
|
||||
|
||||
def test_cli_newpack_refuses_non_empty_dir(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "occupied"
|
||||
dest.mkdir()
|
||||
(dest / "keep.txt").write_text("mine", encoding="utf-8")
|
||||
|
||||
out, err = StringIO(), StringIO()
|
||||
rc = cli.cli_newpack(dest, out=out, err=err)
|
||||
|
||||
assert rc == 2
|
||||
assert "non-empty" in err.getvalue()
|
||||
# The pre-existing file is untouched (nothing was scaffolded over it).
|
||||
assert (dest / "keep.txt").read_text(encoding="utf-8") == "mine"
|
||||
assert not (dest / "AUTHORING.md").exists()
|
||||
|
||||
|
||||
def test_cli_newpack_into_empty_existing_dir_succeeds(tmp_path: Path) -> None:
|
||||
"""An existing but empty directory is a fine scaffold target."""
|
||||
dest = tmp_path / "empty"
|
||||
dest.mkdir()
|
||||
assert cli.cli_newpack(dest, out=StringIO(), err=StringIO()) == 0
|
||||
assert (dest / "AUTHORING.md").exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# server.main argv dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_main_validate_dispatch_returns_status(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture
|
||||
) -> None:
|
||||
# A broken pack routed through main exits 2; a sound one exits 0.
|
||||
pack = _clone_shipped(tmp_path)
|
||||
_patch_world(pack, _break_daily_turns)
|
||||
|
||||
with pytest.raises(SystemExit) as broken:
|
||||
server.main(["validate", str(pack)])
|
||||
assert broken.value.code == 2
|
||||
|
||||
with pytest.raises(SystemExit) as sound:
|
||||
server.main(["validate", str(SHIPPED)])
|
||||
assert sound.value.code == 0
|
||||
assert "The door stands open." in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_newpack_dispatch(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "viamain"
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
server.main(["newpack", str(dest)])
|
||||
assert exc.value.code == 0
|
||||
assert (dest / "AUTHORING.md").exists()
|
||||
|
||||
|
||||
def test_main_worlds_dispatch(capsys: pytest.CaptureFixture) -> None:
|
||||
"""`understone worlds` routes through main, exits 0, and lists the Vale."""
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
server.main(["worlds"])
|
||||
assert exc.value.code == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "vale" in out
|
||||
assert "The Vale of Understone" in out
|
||||
assert "UNDERSTONE_WORLD=" in out
|
||||
|
||||
|
||||
def test_bare_invocation_resolves_to_serve_without_side_effects() -> None:
|
||||
"""Parsing no argv yields the serve path, and parsing has no side effects.
|
||||
|
||||
The transport launch (_serve) is reachable, but argument parsing neither
|
||||
loads a world nor binds a port — so this asserts the resolved command
|
||||
without ever calling _serve.
|
||||
"""
|
||||
args = server._build_parser().parse_args([])
|
||||
assert args.cmd is None # None => the serve branch in main()
|
||||
assert callable(server._serve)
|
||||
|
||||
|
||||
def test_subprocess_validate_packaged_world_exits_zero() -> None:
|
||||
"""End-to-end smoke: `python -m understone validate <packaged dir>` exits 0."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "understone", "validate", str(SHIPPED)],
|
||||
cwd=EXAMPLE_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "The door stands open." in result.stdout
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _clone_shipped(tmp_path: Path) -> Path:
|
||||
dest = tmp_path / "pack"
|
||||
shutil.copytree(SHIPPED, dest)
|
||||
return dest
|
||||
|
||||
|
||||
def _patch_world(pack: Path, mutate: Callable[[dict[str, Any]], None]) -> None:
|
||||
path = pack / "world.json"
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
mutate(data)
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
|
||||
def _break_daily_turns(data: dict[str, Any]) -> None:
|
||||
"""Set daily_turns out of its 1..100 band so the pack fails to load."""
|
||||
data["settings"]["daily_turns"] = 0
|
||||
@@ -1,123 +0,0 @@
|
||||
"""Combat resolution tests.
|
||||
|
||||
Pins determinism (a fixed seed yields identical results twice, log and
|
||||
deltas), each outcome (win/lose/flee), xp/gold crediting on victory, and
|
||||
the defeat contract: the result flags a spawn bounce with no xp/gold and a
|
||||
zero hp delta (the façade applies hp=1 and the move).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.conftest import make_monster, make_player
|
||||
from understone.engine.combat import Outcome, resolve_fight, resolve_flee
|
||||
from understone.engine.rng import GameRNG
|
||||
|
||||
# A strong adventurer vs a Field Rat wins on every probed seed.
|
||||
_WIN_SEED = 1
|
||||
# A fragile adventurer vs a Stone Wyrm loses on every probed seed.
|
||||
_LOSE_SEED = 0
|
||||
# Flee outcomes (probed): seed 1 escapes clean, seed 0 is caught.
|
||||
_FLEE_CLEAN_SEED = 1
|
||||
_FLEE_CAUGHT_SEED = 0
|
||||
|
||||
|
||||
def _strong_player() -> object:
|
||||
return make_player(hp=20, max_hp=20, atk=5, def_=1, xp=0, gold=50)
|
||||
|
||||
|
||||
def _wyrm() -> object:
|
||||
return make_monster(tier=5, name="Stone Wyrm", hp=60, atk=18, def_=6, xp=140, gold=60)
|
||||
|
||||
|
||||
def test_fight_is_deterministic_under_fixed_seed() -> None:
|
||||
r1 = resolve_fight(GameRNG(seed=7), make_player(), make_monster())
|
||||
r2 = resolve_fight(GameRNG(seed=7), make_player(), make_monster())
|
||||
assert r1.log == r2.log
|
||||
assert (r1.outcome, r1.xp_delta, r1.gold_delta, r1.hp_delta) == (
|
||||
r2.outcome,
|
||||
r2.xp_delta,
|
||||
r2.gold_delta,
|
||||
r2.hp_delta,
|
||||
)
|
||||
|
||||
|
||||
def test_win_credits_xp_and_gold() -> None:
|
||||
player = make_player(hp=20, max_hp=20, atk=5, def_=1)
|
||||
monster = make_monster(hp=6, atk=3, def_=0, xp=8, gold=3)
|
||||
result = resolve_fight(GameRNG(seed=_WIN_SEED), player, monster)
|
||||
assert result.outcome is Outcome.WIN
|
||||
assert result.xp_delta == 8
|
||||
assert result.gold_delta == 3
|
||||
# hp_delta is non-positive (you may take a scratch) and never fatal here.
|
||||
assert result.hp_delta <= 0
|
||||
assert not result.bounce_to_spawn
|
||||
|
||||
|
||||
def test_win_deltas_are_exact_for_pinned_seed() -> None:
|
||||
player = make_player(hp=20, max_hp=20, atk=5, def_=1)
|
||||
monster = make_monster(hp=6, atk=3, def_=0, xp=8, gold=3)
|
||||
result = resolve_fight(GameRNG(seed=_WIN_SEED), player, monster)
|
||||
# Pinned from a determinism probe; guards against silent damage drift.
|
||||
assert result.hp_delta == -1
|
||||
# The engine no longer emits a "falls + reward" line — that sentence is
|
||||
# composed by the game façade where the xp/gold are actually banked — so
|
||||
# the WIN log is one line shorter than before and ends on the kill blow.
|
||||
assert len(result.log) == 4
|
||||
assert result.log[-1] == "You strike for 6. (Field Rat: 0 HP)"
|
||||
|
||||
|
||||
def test_win_log_does_not_claim_rewards() -> None:
|
||||
"""The engine narrates the kill blow only; it never claims xp/gold itself.
|
||||
|
||||
Reward ownership lives in the façade (so the Wyrm-win legacy reset, which
|
||||
keeps no xp/gold, narrates no reward). The deltas are still carried on the
|
||||
result for the caller to apply.
|
||||
"""
|
||||
player = make_player(hp=20, max_hp=20, atk=5, def_=1)
|
||||
monster = make_monster(hp=6, atk=3, def_=0, xp=8, gold=3)
|
||||
result = resolve_fight(GameRNG(seed=_WIN_SEED), player, monster)
|
||||
assert result.outcome is Outcome.WIN
|
||||
assert result.xp_delta == 8 and result.gold_delta == 3 # deltas still set
|
||||
joined = "\n".join(result.log)
|
||||
assert "falls" not in joined # no kill/reward sentence in the engine log
|
||||
assert "XP" not in joined and "gold" not in joined
|
||||
|
||||
|
||||
def test_loss_flags_bounce_without_rewards() -> None:
|
||||
result = resolve_fight(GameRNG(seed=_LOSE_SEED), _strong_player_loses(), _wyrm())
|
||||
assert result.outcome is Outcome.LOSE
|
||||
assert result.bounce_to_spawn is True
|
||||
assert result.xp_delta == 0
|
||||
assert result.gold_delta == 0
|
||||
# Combat does not set hp to 1 itself — that is the façade's job.
|
||||
assert result.hp_delta == 0
|
||||
|
||||
|
||||
def _strong_player_loses() -> object:
|
||||
return make_player(hp=12, max_hp=12, atk=4, def_=0)
|
||||
|
||||
|
||||
def test_flee_can_escape_clean() -> None:
|
||||
player = make_player(hp=20, max_hp=20, def_=1)
|
||||
monster = make_monster(atk=8, def_=2)
|
||||
result = resolve_flee(GameRNG(seed=_FLEE_CLEAN_SEED), player, monster)
|
||||
assert result.outcome is Outcome.FLED
|
||||
assert result.hp_delta == 0
|
||||
|
||||
|
||||
def test_flee_caught_costs_hp_but_never_kills() -> None:
|
||||
player = make_player(hp=20, max_hp=20, def_=1)
|
||||
monster = make_monster(atk=8, def_=2)
|
||||
result = resolve_flee(GameRNG(seed=_FLEE_CAUGHT_SEED), player, monster)
|
||||
assert result.outcome is Outcome.FLED
|
||||
assert result.hp_delta < 0
|
||||
# A caught flight cannot drop the player to or below zero.
|
||||
assert player.hp + result.hp_delta >= 1
|
||||
|
||||
|
||||
def test_flee_caught_never_kills_at_low_hp() -> None:
|
||||
player = make_player(hp=1, max_hp=20, def_=0)
|
||||
monster = make_monster(atk=40, def_=0)
|
||||
result = resolve_flee(GameRNG(seed=_FLEE_CAUGHT_SEED), player, monster)
|
||||
# At 1 HP the most a failed flee can cost is 0 (cannot go below 1).
|
||||
assert result.hp_delta == 0
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,858 +0,0 @@
|
||||
"""Game façade integration tests over the shipped world.
|
||||
|
||||
Drives a full session against a temp store, a frozen clock, and a seeded
|
||||
RNG: join -> status -> look -> move -> action(buy/rest/fight) -> log ->
|
||||
rank -> bestow. Persistence is exercised by reopening the store.
|
||||
|
||||
Negative-test discipline (turn guard and bestow cap):
|
||||
Two guards are pinned by assertions here. To confirm each assertion has
|
||||
teeth, the implementer temporarily reverted the guard line and observed
|
||||
the matching test FAIL, then restored it:
|
||||
|
||||
* Turn guard (engine/turns.py spend_turn): replacing
|
||||
``if player.turns_left <= 0: return False`` with ``return True``
|
||||
let fighting continue past the daily budget — ``test_turn_budget_blocks``
|
||||
then failed on the "spent for today" assertion. Restored.
|
||||
* Bestow cap (game.py bestow): removing the ``if cost > remaining``
|
||||
refusal let an over-budget bestowal through — ``test_bestow_cap_refuses``
|
||||
then failed on the unchanged-gold assertion. Restored.
|
||||
* Sanitizer control-char guard (game.py _sanitize): disabling the
|
||||
``not cleaned.isprintable()`` clause let a newline-injected name create a
|
||||
player row and a public event — ``test_join_rejects_control_char_name``
|
||||
then failed. Restored. (See the comment block above the hygiene tests.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import fixed_clock, utc
|
||||
from understone.engine.models import Mode
|
||||
from understone.engine.rng import GameRNG
|
||||
from understone.game import Game
|
||||
from understone.persistence import Store
|
||||
from understone.world.loader import load_world
|
||||
|
||||
PACK = Path(__file__).resolve().parents[1] / "understone" / "world" / "data"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock() -> object:
|
||||
return fixed_clock(utc(2026, 6, 12, 10, 0))
|
||||
|
||||
|
||||
def _game(tmp_path: Path, clock: object, seed: int = 7) -> Game:
|
||||
world = load_world(PACK)
|
||||
store = Store(tmp_path / "game.db")
|
||||
return Game(world, store, clock=clock, rng=GameRNG(seed=seed)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Join / status / look
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_join_creates_player_at_spawn(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
assert (player.x, player.y) == game.world.spawn
|
||||
assert player.gold == game.world.settings.starting_gold
|
||||
assert "@" in out
|
||||
assert game.world.name in out
|
||||
|
||||
|
||||
def test_join_resumes_existing(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
game.players["Brandr"].gold = 123
|
||||
out = game.join("Brandr")
|
||||
assert "Welcome back" in out
|
||||
assert game.players["Brandr"].gold == 123
|
||||
|
||||
|
||||
def test_status_unknown_player_is_friendly(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.status("Nobody")
|
||||
assert "has signed the ledger" in out
|
||||
assert "door_join" in out
|
||||
|
||||
|
||||
def test_look_overworld_has_frame(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
out = game.look("Brandr")
|
||||
assert "@" in out
|
||||
assert "┌" in out and "┐" in out
|
||||
assert len(out) < 2048
|
||||
|
||||
|
||||
def test_overworld_frame_textured_borders_intact(tmp_path: Path, clock: object) -> None:
|
||||
"""The textured overworld frame keeps square borders and a single player marker.
|
||||
|
||||
Structural discipline for the v0.6 texture: variants change the GLYPHS but
|
||||
must never change the geometry. The box rows are uniform width, exactly one
|
||||
'@' is painted, and the grass field shows more than one variant in a row
|
||||
(the deterministic stipple, not a flat sheet of '.').
|
||||
"""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
frame = game.look("Brandr")
|
||||
lines = frame.split("\n")
|
||||
# Box rows: top border + VIEW_H grid rows + bottom border, all equal width.
|
||||
box = [ln for ln in lines if ln and ln[0] in "┌│└"]
|
||||
widths = {len(ln) for ln in box}
|
||||
assert len(widths) == 1, f"textured frame rows ragged: {widths}"
|
||||
# Exactly one player marker, regardless of the surrounding texture.
|
||||
assert frame.count("@") == 1
|
||||
# The grass texture varies: a body row carries at least two of . , '
|
||||
body = [ln for ln in lines if ln.startswith("│")]
|
||||
assert any(len({ch for ch in ln if ch in ".,'"}) >= 2 for ln in body)
|
||||
|
||||
|
||||
def test_look_in_menu_shows_location(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
# Shop is two cells east of spawn along the road.
|
||||
game.move("Brandr", "", "east", 2)
|
||||
assert game.players["Brandr"].mode is Mode.MENU
|
||||
out = game.look("Brandr")
|
||||
assert "(B)uy" in out and "(L)eave" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Move
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_move_blocked_in_menu(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
game.move("Brandr", "", "east", 2) # into the shop menu
|
||||
out = game.move("Brandr", "", "east", 2)
|
||||
assert "inside" in out.lower()
|
||||
|
||||
|
||||
def test_move_enters_location(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
out = game.move("Brandr", "", "west", 2) # inn is two cells west
|
||||
assert game.players["Brandr"].at_location == "inn"
|
||||
assert "step inside" in out.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Actions: rest, fight, turn budget
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rest_heals_and_charges(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
player.hp = 5
|
||||
game.move("Brandr", "", "west", 2) # inn
|
||||
out = game.action("Brandr", "rest", "", "")
|
||||
assert player.hp == player.max_hp
|
||||
assert player.gold == game.world.settings.starting_gold - game.world.settings.rest_cost
|
||||
assert "full health" in out.lower()
|
||||
|
||||
|
||||
def test_rest_when_spent_restores_a_fresh_days_turns(tmp_path: Path, clock: object) -> None:
|
||||
"""Sleeping at the inn with no turns left rolls into a fresh day's allowance."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
daily = game.world.settings.daily_turns
|
||||
player.turns_left = 0 # spent for the day
|
||||
player.hp = 5
|
||||
game.move("Brandr", "", "west", 2) # step into the inn
|
||||
out = game.action("Brandr", "rest", "", "")
|
||||
assert player.turns_left == daily # a fresh day's turns restored
|
||||
assert player.hp == player.max_hp # and fully mended
|
||||
assert f"/{daily} ]" in out # footer reflects the refreshed budget
|
||||
|
||||
|
||||
def test_rest_with_turns_in_hand_never_inflates_the_budget(tmp_path: Path, clock: object) -> None:
|
||||
"""Resting mid-day mends but adds no turns — the top-up only fires at zero."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
daily = game.world.settings.daily_turns
|
||||
player.turns_left = daily - 3 # turns still in hand
|
||||
player.hp = 5
|
||||
game.move("Brandr", "", "west", 2) # step into the inn
|
||||
game.action("Brandr", "rest", "", "")
|
||||
assert player.turns_left == daily - 3 # unchanged: no farming past the cap
|
||||
assert player.hp == player.max_hp # but the heal still lands
|
||||
|
||||
|
||||
def test_fight_spends_a_turn_and_credits(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
# Drop into the forest_near zone so an encounter is available.
|
||||
player.x, player.y = 35, 25
|
||||
before_turns = player.turns_left
|
||||
out = game.action("Brandr", "fight", "", "")
|
||||
assert player.turns_left == before_turns - 1
|
||||
assert player.xp > 0
|
||||
assert "XP" in out
|
||||
|
||||
|
||||
def test_turn_budget_blocks(tmp_path: Path, clock: object) -> None:
|
||||
"""Pins the spend_turn guard: at 0 turns, fighting is refused.
|
||||
|
||||
See the module docstring for the revert-and-observe-failure check that
|
||||
proves this assertion has teeth.
|
||||
"""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
player.x, player.y = 35, 25
|
||||
player.turns_left = 0
|
||||
out = game.action("Brandr", "fight", "", "")
|
||||
assert "spent for today" in out.lower()
|
||||
# No turn was consumed past zero, and no XP was gained.
|
||||
assert player.turns_left == 0
|
||||
assert player.xp == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Log / rank
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_log_reports_then_advances(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
# A second player acting creates a public event Brandr has not yet seen.
|
||||
game.join("Sigrun")
|
||||
first = game.log("Brandr")
|
||||
assert "Sigrun" in first or "Brandr" in first
|
||||
assert "The Understone Herald" in first # dressed as the broadsheet
|
||||
# The cursor advanced; a second read with no new events is quiet.
|
||||
second = game.log("Brandr")
|
||||
assert "The Understone Herald" in second # the masthead still prints
|
||||
assert "still" in second.lower() # the herald-flavoured "all quiet" line
|
||||
|
||||
|
||||
def test_rank_marks_caller(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
game.join("Sigrun")
|
||||
game.players["Sigrun"].level = 5
|
||||
out = game.rank("Brandr")
|
||||
assert "Brandr" in out and "Sigrun" in out
|
||||
assert "*" in out # the caller's row is marked
|
||||
assert "┌" in out # box-drawing table
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rank ★ column: stars live in their own column, so a long name keeps them
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_win_stars_column_formats() -> None:
|
||||
"""Zero is blank, 1..5 render as ★ runs, and >5 collapses to ★xN."""
|
||||
from understone.game import _win_stars
|
||||
|
||||
assert _win_stars(0) == ""
|
||||
assert _win_stars(1) == "★"
|
||||
assert _win_stars(5) == "★★★★★"
|
||||
assert _win_stars(7) == "★x7"
|
||||
|
||||
|
||||
def test_long_name_with_one_win_keeps_its_star() -> None:
|
||||
"""A full 24-char name no longer eats its own ★ (the v0.1 truncation bug).
|
||||
|
||||
The name occupied the whole 20-wide field before, clipping the star away;
|
||||
with a separate stars column the ★ survives beside a maximal name.
|
||||
"""
|
||||
from understone.engine.rank import RankEntry
|
||||
from understone.game import _render_rank_table
|
||||
|
||||
name = "X" * 24
|
||||
rows = _render_rank_table([RankEntry(name=name, level=5, xp=100, gold=50, wins=1)], caller="")
|
||||
body = "\n".join(rows)
|
||||
assert name in body # the full name is present
|
||||
assert "★" in body # and so is its star
|
||||
|
||||
|
||||
def test_high_win_count_renders_compact_marker() -> None:
|
||||
"""Seven wins render as the compact ``★x7`` rather than seven glyphs."""
|
||||
from understone.engine.rank import RankEntry
|
||||
from understone.game import _render_rank_table
|
||||
|
||||
rows = _render_rank_table([RankEntry(name="Champ", level=9, xp=9, gold=9, wins=7)], caller="")
|
||||
body = "\n".join(rows)
|
||||
assert "★x7" in body
|
||||
assert "★★★★★★★" not in body # not seven literal stars
|
||||
|
||||
|
||||
def test_shared_world_other_player_marker(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
game.join("Sigrun")
|
||||
# Stand Sigrun one cell east of Brandr's spawn so she lands in the view.
|
||||
sig = game.players["Sigrun"]
|
||||
brandr = game.players["Brandr"]
|
||||
sig.x, sig.y = brandr.x + 1, brandr.y
|
||||
out = game.look("Brandr")
|
||||
assert "☻" in out # the other player shows as '☻'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bestow (+ cap negative test)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_bestow_grants_gold(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
before = player.gold
|
||||
out = game.bestow("Brandr", "a daring rescue", 10, 0)
|
||||
assert player.gold == before + 10
|
||||
assert player.bestow_spent == 10
|
||||
assert "bestowal" in out.lower()
|
||||
|
||||
|
||||
def test_bestow_heal_charges_only_applied(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
player.hp = player.max_hp - 3 # only 3 missing
|
||||
game.bestow("Brandr", "mercy after a hard fight", 0, 10)
|
||||
assert player.hp == player.max_hp
|
||||
# Charged for 3 HP at heal_cost_per_hp, not the requested 10.
|
||||
assert player.bestow_spent == 3 * game.world.settings.heal_cost_per_hp
|
||||
|
||||
|
||||
def test_bestow_requires_reason(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
out = game.bestow("Brandr", " ", 10, 0)
|
||||
assert "reason" in out.lower()
|
||||
assert game.players["Brandr"].gold == game.world.settings.starting_gold
|
||||
|
||||
|
||||
def test_bestow_requires_nonzero(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
out = game.bestow("Brandr", "nothing at all", 0, 0)
|
||||
assert "at least" in out.lower()
|
||||
|
||||
|
||||
def test_bestow_cap_refuses(tmp_path: Path, clock: object) -> None:
|
||||
"""Pins the bestow cap: an over-budget grant is refused without mutation.
|
||||
|
||||
See the module docstring for the revert-and-observe-failure check that
|
||||
proves this assertion has teeth.
|
||||
"""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
budget = game.world.settings.bestow_daily_budget
|
||||
before_gold = player.gold
|
||||
out = game.bestow("Brandr", "an absurd windfall", budget + 100, 0)
|
||||
assert "the fates allow" in out.lower()
|
||||
# Refused cleanly: no gold moved and no pool spent.
|
||||
assert player.gold == before_gold
|
||||
assert player.bestow_spent == 0
|
||||
|
||||
|
||||
def test_bestow_pool_resets_next_day(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
game.bestow("Brandr", "first blessing", 20, 0)
|
||||
assert player.bestow_spent == 20
|
||||
# Advance the clock past UTC midnight; the next bestow sees a fresh pool.
|
||||
game.clock = fixed_clock(utc(2026, 6, 13, 0, 5)) # type: ignore[assignment]
|
||||
game.bestow("Brandr", "a new day's fortune", 20, 0)
|
||||
assert player.bestow_spent == 20 # reset to 0 then +20, not 40
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persistence round-trip through the façade
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_state_survives_store_reopen(tmp_path: Path, clock: object) -> None:
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
game.players["Brandr"].x, game.players["Brandr"].y = 35, 25
|
||||
game.action("Brandr", "fight", "", "")
|
||||
xp_after = game.players["Brandr"].xp
|
||||
gold_after = game.players["Brandr"].gold
|
||||
game.store.close()
|
||||
|
||||
world = load_world(PACK)
|
||||
reopened = Store(tmp_path / "game.db")
|
||||
revived = Game(world, reopened, clock=clock) # type: ignore[arg-type]
|
||||
assert revived.players["Brandr"].xp == xp_after
|
||||
assert revived.players["Brandr"].gold == gold_after
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Day rollover applies to fight/descend, not just join/bestow
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _MutableClock:
|
||||
"""A clock whose reported moment can be advanced between calls."""
|
||||
|
||||
def __init__(self, moment: object) -> None:
|
||||
self.moment = moment
|
||||
|
||||
def __call__(self) -> object:
|
||||
return self.moment
|
||||
|
||||
|
||||
def test_fight_refreshes_budget_across_midnight(tmp_path: Path) -> None:
|
||||
"""A fight on a new UTC day must reset the budget without re-joining.
|
||||
|
||||
Before the fix, _resolve_encounter spent a turn without calling
|
||||
_ensure_day, so an exhausted player who returned the next day was still
|
||||
blocked until they happened to re-join.
|
||||
"""
|
||||
clk = _MutableClock(utc(2026, 6, 12, 23, 0))
|
||||
world = load_world(PACK)
|
||||
store = Store(tmp_path / "game.db")
|
||||
game = Game(world, store, clock=clk, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
player.x, player.y = 35, 25 # forest_near zone: an encounter is available
|
||||
player.turns_left = 0 # spent for the day
|
||||
daily = game.world.settings.daily_turns
|
||||
|
||||
clk.moment = utc(2026, 6, 13, 0, 5) # cross UTC midnight, no re-join
|
||||
out = game.action("Brandr", "fight", "", "")
|
||||
|
||||
assert "spent for today" not in out.lower() # the fresh day let the fight run
|
||||
assert player.turns_left == daily - 1 # reset to full, then one spent
|
||||
assert player.xp > 0
|
||||
assert f"/{daily} ]" in out # footer shows the refreshed budget
|
||||
|
||||
|
||||
def test_descend_refreshes_budget_across_midnight(tmp_path: Path) -> None:
|
||||
"""Descending on a new UTC day resets the budget without re-joining."""
|
||||
clk = _MutableClock(utc(2026, 6, 12, 23, 0))
|
||||
world = load_world(PACK)
|
||||
store = Store(tmp_path / "game.db")
|
||||
game = Game(world, store, clock=clk, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
game.join("Hero")
|
||||
player = game.players["Hero"]
|
||||
# Overwhelming stats so the gauntlet itself never bounces the player.
|
||||
player.level, player.atk, player.def_ = 20, 200, 100
|
||||
player.hp = player.max_hp = 500
|
||||
player.mode = Mode.MENU
|
||||
player.at_location = "dungeon"
|
||||
player.turns_left = 0
|
||||
daily = game.world.settings.daily_turns
|
||||
|
||||
clk.moment = utc(2026, 6, 13, 0, 5)
|
||||
out = game.action("Hero", "descend", "", "")
|
||||
|
||||
assert "too weary" not in out.lower()
|
||||
assert player.turns_left == daily - 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input hygiene chokepoint (the _sanitize helper)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Negative-test discipline (security invariant): to prove the control-char
|
||||
# rejection in Game._sanitize has teeth, the implementer temporarily replaced
|
||||
# its ``not cleaned.isprintable()`` clause with ``False`` (disabling the
|
||||
# check) and confirmed test_join_rejects_control_char_name FAILED — the
|
||||
# injected name created a player row and a public event. The clause was then
|
||||
# restored. The newline-injection test below is the standing regression for
|
||||
# that invariant.
|
||||
|
||||
|
||||
def test_join_rejects_control_char_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A bell/control character in a name is refused with the runes line."""
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("Bra\x07ndr")
|
||||
assert "strange runes" in out
|
||||
assert game.players == {} # no row created
|
||||
assert game.events == [] # nothing persisted
|
||||
|
||||
|
||||
def test_join_rejects_newline_name_no_persist(tmp_path: Path, clock: object) -> None:
|
||||
"""An embedded newline (log-injection vector) is refused, nothing written.
|
||||
|
||||
The name is kept short so it is the control-char clause — not the length
|
||||
clause — that rejects it; this is the standing regression for the
|
||||
isprintable security invariant documented in the module docstring.
|
||||
"""
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("Bra\nndr") # 7 chars: well under the 24 limit
|
||||
assert "strange runes" in out # the runes (bad-character) refusal, not length
|
||||
# The security invariant: no player row and no event row escaped the guard.
|
||||
assert game.players == {}
|
||||
assert game.events == []
|
||||
|
||||
|
||||
def test_join_rejects_overlong_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A 25-character name is refused with the narrow-ledger line."""
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("X" * 25)
|
||||
assert "ledger is narrow" in out
|
||||
assert game.players == {}
|
||||
|
||||
|
||||
def test_join_accepts_max_length_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A 24-character name is exactly at the limit and accepted."""
|
||||
game = _game(tmp_path, clock)
|
||||
name = "X" * 24
|
||||
game.join(name)
|
||||
assert name in game.players
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Narrow-ledger width rule (the _sanitize one-column clause, v0.6)
|
||||
#
|
||||
# Names/reasons/mail render inside fixed-width frames and tables, so a glyph
|
||||
# that does not fit a single column would shove a column out of true. The
|
||||
# sanitizer rejects wide runes and combining marks; a printable-but-wide name
|
||||
# gets the dedicated narrow-ledger refusal, not the control-char "runes" line.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_join_rejects_wide_cjk_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A CJK ideograph name is refused with the narrow-ledger line; nothing written."""
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("龍")
|
||||
assert "columns are narrow" in out
|
||||
assert game.players == {}
|
||||
assert game.events == []
|
||||
|
||||
|
||||
def test_join_rejects_emoji_name(tmp_path: Path, clock: object) -> None:
|
||||
"""An emoji in a name (🌲x) is wide and refused with the narrow-ledger line."""
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("🌲x")
|
||||
assert "columns are narrow" in out
|
||||
assert game.players == {}
|
||||
|
||||
|
||||
def test_join_rejects_fullwidth_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A fullwidth Latin letter (A) is two columns and refused."""
|
||||
game = _game(tmp_path, clock)
|
||||
out = game.join("A")
|
||||
assert "columns are narrow" in out
|
||||
assert game.players == {}
|
||||
|
||||
|
||||
def test_join_rejects_combining_mark_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A name with a combining mark (decomposed accent) is refused as wide.
|
||||
|
||||
The name is normalised to NFD so the 'o' carries a separate U+0308
|
||||
combining diaeresis — a zero-width code point that desynchronises the
|
||||
column count. Built explicitly so the source encoding cannot mask it.
|
||||
"""
|
||||
game = _game(tmp_path, clock)
|
||||
decomposed = unicodedata.normalize("NFD", "Bj\u00f6rn")
|
||||
assert any(unicodedata.combining(ch) for ch in decomposed) # genuinely NFD
|
||||
out = game.join(decomposed)
|
||||
assert "columns are narrow" in out
|
||||
assert game.players == {}
|
||||
|
||||
|
||||
def test_join_accepts_composed_latin_name(tmp_path: Path, clock: object) -> None:
|
||||
"""A precomposed Latin accent (NFC name) is all single-column and accepted."""
|
||||
game = _game(tmp_path, clock)
|
||||
composed = unicodedata.normalize("NFC", "Bj\u00f6rn")
|
||||
game.join(composed)
|
||||
assert composed in game.players
|
||||
|
||||
|
||||
def _seed_wide_named_player(db: Path, clock: object, wide_name: str) -> None:
|
||||
"""Write a stored adventurer whose name is a now-illegal wide rune.
|
||||
|
||||
Bypasses ``join`` (which would refuse a wide name at creation) by upserting
|
||||
a Player row straight through the Store, so the fixture stands in for a save
|
||||
that predates the narrow-ledger rule. Built by renaming a legitimately-
|
||||
created hero so every other field stays valid.
|
||||
"""
|
||||
from dataclasses import replace
|
||||
|
||||
world = load_world(PACK)
|
||||
seed = Store(db)
|
||||
game = Game(world, seed, clock=clock, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
game.join("Brandr")
|
||||
base = game.players["Brandr"]
|
||||
seed.upsert_player(replace(base, name=wide_name))
|
||||
seed.commit()
|
||||
seed.close()
|
||||
|
||||
|
||||
def test_join_resumes_stored_wide_name(tmp_path: Path, clock: object) -> None:
|
||||
"""An existing adventurer with a wide-rune name resumes \u2014 identity is never re-gated.
|
||||
|
||||
Resume keys off the exact stored name BEFORE the sanitizer, so a character
|
||||
whose name predates the narrow-ledger rule is welcomed back rather than
|
||||
locked out. This is the resume-by-exact-name invariant.
|
||||
"""
|
||||
db = tmp_path / "game.db"
|
||||
wide = "\u9f8d"
|
||||
_seed_wide_named_player(db, clock, wide)
|
||||
|
||||
world = load_world(PACK)
|
||||
game = Game(world, Store(db), clock=clock, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
out = game.join(wide)
|
||||
assert "Welcome back" in out # resumed, not refused
|
||||
assert "columns are narrow" not in out
|
||||
assert wide in game.players
|
||||
|
||||
|
||||
def test_join_still_refuses_new_wide_name(tmp_path: Path, clock: object) -> None:
|
||||
"""Creation is still gated: a NEW wide name with no stored row is refused.
|
||||
|
||||
The resume bypass is exact-name only; a wide name that matches no stored
|
||||
adventurer falls through to the creation gate and gets the narrow-ledger
|
||||
refusal, with nothing written.
|
||||
"""
|
||||
db = tmp_path / "game.db"
|
||||
# Seed one wide-named save, then try to CREATE a different wide name.
|
||||
_seed_wide_named_player(db, clock, "\u9f8d")
|
||||
|
||||
world = load_world(PACK)
|
||||
game = Game(world, Store(db), clock=clock, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
out = game.join("\u7363") # a different wide rune \u2014 no stored row for it
|
||||
assert "columns are narrow" in out
|
||||
assert "\u7363" not in game.players
|
||||
|
||||
|
||||
def test_bestow_rejects_newline_reason_no_persist(tmp_path: Path, clock: object) -> None:
|
||||
"""A newline-embedded bestow reason is refused; no event, pool unchanged."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
events_before = len(game.events)
|
||||
out = game.bestow("Brandr", "heroics\nand a forged log line", 10, 0)
|
||||
assert "plainly-spoken" in out
|
||||
assert len(game.events) == events_before # no bestow event appended
|
||||
assert player.bestow_spent == 0 # pool untouched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bestow: heal-only at full HP grants nothing (no empty grant persisted)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_bestow_heal_only_at_full_hp_refused(tmp_path: Path, clock: object) -> None:
|
||||
"""A heal-only bestow at full HP applies nothing and must not persist."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
assert player.hp == player.max_hp # join starts at full health
|
||||
events_before = len(game.events)
|
||||
out = game.bestow("Brandr", "a quiet blessing", 0, 10)
|
||||
assert "already hale" in out
|
||||
assert len(game.events) == events_before # no "Fortune favours" line written
|
||||
assert player.bestow_spent == 0 # nothing charged
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Descend the deep: one rung per descent (see test_descend.py for the ladder)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_descend_fights_one_rung_and_advances(tmp_path: Path, clock: object) -> None:
|
||||
"""A strong player clears the next rung: one foe fought, rewards banked, depth +1."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Hero")
|
||||
player = game.players["Hero"]
|
||||
player.level, player.atk, player.def_ = 20, 200, 100
|
||||
player.hp = player.max_hp = 500
|
||||
player.mode = Mode.MENU
|
||||
player.at_location = "dungeon"
|
||||
before_turns, before_gold, before_xp = player.turns_left, player.gold, player.xp
|
||||
|
||||
out = game.action("Hero", "descend", "", "")
|
||||
|
||||
# The first rung is the tier-3 guardian (Forest Wolf); deeper rungs do NOT
|
||||
# appear in one descent — the deep is fought a rung at a time now.
|
||||
assert "Forest Wolf" in out
|
||||
assert "Cave Troll" not in out
|
||||
assert player.deepest_rung == 1
|
||||
assert player.turns_left == before_turns - 1
|
||||
assert player.gold > before_gold
|
||||
assert player.xp > before_xp
|
||||
|
||||
|
||||
def test_descend_bounces_weak_player_to_spawn(tmp_path: Path, clock: object) -> None:
|
||||
"""A fresh weak player falls on the first rung and wakes at the spawn.
|
||||
|
||||
Depth is NOT advanced by a loss, but it persists at whatever it was (here 0).
|
||||
"""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Weakling")
|
||||
player = game.players["Weakling"]
|
||||
player.mode = Mode.MENU
|
||||
player.at_location = "dungeon"
|
||||
|
||||
out = game.action("Weakling", "descend", "", "")
|
||||
|
||||
assert player.hp == 1
|
||||
assert player.mode is Mode.TILE
|
||||
assert player.at_location == ""
|
||||
assert (player.x, player.y) == game.world.spawn
|
||||
assert player.deepest_rung == 0 # a loss never advances the deep
|
||||
# Felled by the first rung (the tier-3 Forest Wolf).
|
||||
assert "Forest Wolf" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shop façade: buy / upgrade / sell / heal stat arithmetic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_shop_buy_upgrade_sell_heal_cycle(tmp_path: Path, clock: object) -> None:
|
||||
"""Equip deltas apply once on buy/upgrade and unwind cleanly on sell."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
player.gold = 1000
|
||||
player.mode = Mode.MENU
|
||||
player.at_location = "shop"
|
||||
|
||||
short_sword = game.world.item_by_id("short_sword")
|
||||
war_axe = game.world.item_by_id("war_axe")
|
||||
starter = game.world.item_by_id(game.world.settings.starting_weapon)
|
||||
assert short_sword is not None and war_axe is not None and starter is not None
|
||||
|
||||
starter_atk = player.atk # 3 base + rusty dagger bonus
|
||||
|
||||
# Buy the short sword: gold falls by its price, atk rises by the delta.
|
||||
gold0 = player.gold
|
||||
game.action("Brandr", "buy", "", "short_sword")
|
||||
assert player.gold == gold0 - short_sword.price
|
||||
assert player.atk == starter_atk + (short_sword.atk - starter.atk)
|
||||
atk_with_sword = player.atk
|
||||
|
||||
# Upgrade to the war axe: atk reflects the difference, not a double-add.
|
||||
gold1 = player.gold
|
||||
game.action("Brandr", "buy", "", "war_axe")
|
||||
assert player.gold == gold1 - war_axe.price
|
||||
assert player.atk == atk_with_sword + (war_axe.atk - short_sword.atk)
|
||||
|
||||
# Sell the war axe: half-price refund, atk falls back to the starter bonus.
|
||||
gold2 = player.gold
|
||||
game.action("Brandr", "sell", "", "")
|
||||
assert player.gold == gold2 + war_axe.price // 2
|
||||
assert player.atk == starter_atk
|
||||
|
||||
# Heal at the shrine: HP restored, gold debited per missing point.
|
||||
player.mode = Mode.MENU
|
||||
player.at_location = "healer"
|
||||
player.hp = player.max_hp - 5
|
||||
per_hp = game.world.settings.heal_cost_per_hp
|
||||
gold3 = player.gold
|
||||
game.action("Brandr", "heal", "", "")
|
||||
assert player.hp == player.max_hp
|
||||
assert player.gold == gold3 - 5 * per_hp
|
||||
|
||||
|
||||
def test_sell_starter_weapon_refused(tmp_path: Path, clock: object) -> None:
|
||||
"""The starter blade is unsellable regardless of price (no free-gold loop)."""
|
||||
game = _game(tmp_path, clock)
|
||||
game.join("Brandr")
|
||||
player = game.players["Brandr"]
|
||||
assert player.weapon_id == game.world.settings.starting_weapon
|
||||
player.mode = Mode.MENU
|
||||
player.at_location = "shop"
|
||||
gold_before = player.gold
|
||||
out = game.action("Brandr", "sell", "", "")
|
||||
assert "nothing worth selling" in out.lower()
|
||||
assert player.gold == gold_before
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bounded in-memory event tail (full history stays in SQLite)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_event_tail_is_capped_but_log_still_works(tmp_path: Path, clock: object) -> None:
|
||||
"""Loading caps the resident tail; door_log still serves recent events."""
|
||||
from understone.engine.log import since
|
||||
from understone.game import EVENT_TAIL_KEEP
|
||||
|
||||
db = tmp_path / "game.db"
|
||||
seed_store = Store(db)
|
||||
last_id = 0
|
||||
for i in range(EVENT_TAIL_KEEP + 50):
|
||||
last_id = seed_store.insert_event("t", "sys", "note", f"event {i}")
|
||||
seed_store.commit()
|
||||
seed_store.close()
|
||||
|
||||
world = load_world(PACK)
|
||||
game = Game(world, Store(db), clock=clock, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
# Only the most recent EVENT_TAIL_KEEP events are resident in memory.
|
||||
assert len(game.events) == EVENT_TAIL_KEEP
|
||||
assert game.events[-1].event_id == last_id
|
||||
|
||||
# door_log still reports events after a recent cursor.
|
||||
recent_cursor = game.events[-3].event_id
|
||||
game.join("Brandr")
|
||||
game.players["Brandr"].log_cursor = recent_cursor
|
||||
out = game.log("Brandr")
|
||||
assert "The Understone Herald" in out # broadsheet masthead
|
||||
assert "since your last visit" in out
|
||||
fresh, new_cursor = since(game.events, recent_cursor)
|
||||
assert fresh # there are events past the cursor
|
||||
assert new_cursor == game.events[-1].event_id
|
||||
|
||||
|
||||
def test_private_mail_survives_tail_eviction(tmp_path: Path, clock: object) -> None:
|
||||
"""A private note older than the resident tail is still delivered (durable mail).
|
||||
|
||||
Public history that falls off the in-memory tail is gone by design (the
|
||||
broadsheet does not keep), but mail must not be: a note left while the
|
||||
recipient was away has to surface however many public events have since
|
||||
pushed it out of the tail. A third player — whose cursor also predates the
|
||||
note — must still never see it, because it was never theirs.
|
||||
"""
|
||||
from understone.persistence import EVENT_TAIL_KEEP
|
||||
|
||||
db = tmp_path / "game.db"
|
||||
store = Store(db)
|
||||
game = Game(load_world(PACK), store, clock=clock, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
game.join("Scribe")
|
||||
game.join("Reader")
|
||||
game.join("Bystander")
|
||||
# Scribe leaves Reader a private note; neither Reader nor Bystander reads it.
|
||||
secret = "the cellar key is under the third barrel"
|
||||
game.action("Scribe", "post", "Reader", "", secret)
|
||||
|
||||
# Flood the feed past the tail bound so the note is evicted from memory.
|
||||
for i in range(EVENT_TAIL_KEEP + 20):
|
||||
store.insert_event("t", "sys", "note", f"broadsheet filler {i}")
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
# Reopen: only the newest tail is resident, so the note now lives in the gap.
|
||||
reopened = Store(db)
|
||||
revived = Game(load_world(PACK), reopened, clock=clock, rng=GameRNG(seed=7)) # type: ignore[arg-type]
|
||||
note_id = next(
|
||||
e.event_id
|
||||
for e in reopened.targeted_events_since("Reader", 0) # note: from SQLite, not the tail
|
||||
if secret in e.text
|
||||
)
|
||||
assert note_id < revived.events[0].event_id # the note really is past the tail
|
||||
|
||||
# The recipient still sees the note, backfilled from SQLite...
|
||||
reader_log = revived.log("Reader")
|
||||
assert secret in reader_log
|
||||
assert "While you were away" in reader_log
|
||||
# ...but a third player never does, even though their cursor predates it too.
|
||||
third_log = revived.log("Bystander")
|
||||
assert secret not in third_log
|
||||
reopened.close()
|
||||
@@ -1,118 +0,0 @@
|
||||
"""XP curve, level-up, and restorative-maths tests.
|
||||
|
||||
Pins the threshold edges (at / just below / just above), a multi-level
|
||||
jump from a single award, the exact growth table, the inn's flat-rate
|
||||
full heal with affordability gating, and the healer's per-HP cost maths.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.conftest import DEFAULT_SETTINGS, make_player, make_settings
|
||||
from understone.engine.leveling import apply_xp, heal, rest, xp_for_level
|
||||
|
||||
# Default curve is 100 * (n-1)*n/2 cumulative:
|
||||
# L2 = 100, L3 = 300, L4 = 600, L5 = 1000.
|
||||
|
||||
|
||||
def test_xp_curve_thresholds() -> None:
|
||||
assert xp_for_level(1, DEFAULT_SETTINGS) == 0
|
||||
assert xp_for_level(2, DEFAULT_SETTINGS) == 100
|
||||
assert xp_for_level(3, DEFAULT_SETTINGS) == 300
|
||||
assert xp_for_level(4, DEFAULT_SETTINGS) == 600
|
||||
assert xp_for_level(5, DEFAULT_SETTINGS) == 1000
|
||||
|
||||
|
||||
def test_just_below_threshold_does_not_level() -> None:
|
||||
player = make_player(level=1, xp=0, hp=20, max_hp=20)
|
||||
gains = apply_xp(player, 99, DEFAULT_SETTINGS)
|
||||
assert gains == []
|
||||
assert player.level == 1
|
||||
|
||||
|
||||
def test_exact_threshold_levels_once() -> None:
|
||||
player = make_player(level=1, xp=0, hp=10, max_hp=20, atk=5, def_=1)
|
||||
gains = apply_xp(player, 100, DEFAULT_SETTINGS)
|
||||
assert len(gains) == 1
|
||||
assert player.level == 2
|
||||
# Growth table applied and a full heal granted on level-up.
|
||||
assert player.max_hp == 26
|
||||
assert player.atk == 7
|
||||
assert player.def_ == 2
|
||||
assert player.hp == player.max_hp
|
||||
|
||||
|
||||
def test_just_above_threshold_levels_once() -> None:
|
||||
player = make_player(level=1, xp=0)
|
||||
gains = apply_xp(player, 101, DEFAULT_SETTINGS)
|
||||
assert len(gains) == 1
|
||||
assert player.level == 2
|
||||
assert player.xp == 101
|
||||
|
||||
|
||||
def test_single_award_can_jump_multiple_levels() -> None:
|
||||
player = make_player(level=1, xp=0, max_hp=20, atk=5, def_=1)
|
||||
gains = apply_xp(player, 600, DEFAULT_SETTINGS)
|
||||
# 600 cumulative reaches level 4 (L2=100, L3=300, L4=600).
|
||||
assert player.level == 4
|
||||
assert [g.new_level for g in gains] == [2, 3, 4]
|
||||
# Three levels of growth stacked.
|
||||
assert player.max_hp == 20 + 3 * 6
|
||||
assert player.atk == 5 + 3 * 2
|
||||
assert player.def_ == 1 + 3 * 1
|
||||
|
||||
|
||||
def test_growth_table_respects_settings() -> None:
|
||||
settings = make_settings(growth_max_hp=10, growth_atk=3, growth_def=2, xp_base=50)
|
||||
player = make_player(level=1, xp=0, max_hp=20, atk=5, def_=1)
|
||||
apply_xp(player, 50, settings) # L2 at 50 with xp_base=50
|
||||
assert player.level == 2
|
||||
assert player.max_hp == 30
|
||||
assert player.atk == 8
|
||||
assert player.def_ == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# rest (inn) and heal (healer)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rest_full_heals_and_charges() -> None:
|
||||
player = make_player(hp=5, max_hp=20, gold=50)
|
||||
assert rest(player, cost=15) is True
|
||||
assert player.hp == 20
|
||||
assert player.gold == 35
|
||||
|
||||
|
||||
def test_rest_refused_when_unaffordable() -> None:
|
||||
player = make_player(hp=5, max_hp=20, gold=10)
|
||||
assert rest(player, cost=15) is False
|
||||
assert player.hp == 5
|
||||
assert player.gold == 10
|
||||
|
||||
|
||||
def test_heal_charges_only_for_hp_restored() -> None:
|
||||
player = make_player(hp=15, max_hp=20, gold=100)
|
||||
result = heal(player, amount=10, cost_per_hp=2)
|
||||
# Only 5 HP were missing.
|
||||
assert result.healed == 5
|
||||
assert result.cost == 10
|
||||
assert player.hp == 20
|
||||
assert player.gold == 90
|
||||
|
||||
|
||||
def test_heal_bounded_by_affordability() -> None:
|
||||
player = make_player(hp=2, max_hp=20, gold=7)
|
||||
result = heal(player, amount=10, cost_per_hp=2)
|
||||
# 7 gold buys 3 HP at 2/hp.
|
||||
assert result.healed == 3
|
||||
assert result.cost == 6
|
||||
assert player.hp == 5
|
||||
assert player.gold == 1
|
||||
|
||||
|
||||
def test_heal_noop_when_full() -> None:
|
||||
player = make_player(hp=20, max_hp=20, gold=100)
|
||||
result = heal(player, amount=10, cost_per_hp=2)
|
||||
assert result.healed == 0
|
||||
assert result.cost == 0
|
||||
assert player.gold == 100
|
||||
@@ -1,288 +0,0 @@
|
||||
"""End-to-end MCP integration test — the only test that touches the network.
|
||||
|
||||
Boots the real Understone FastMCP app (backed by a temp DB) in a uvicorn
|
||||
thread, then drives it over the real streamable-HTTP wire with the real MCP
|
||||
client: initialize, list_tools (all nine door_* names), join, look. A second
|
||||
client session joins a second adventurer in the SAME process and world, and
|
||||
the first player's view then shows the '&' other-player marker — proving the
|
||||
shared-world, single-process contract over a real wire.
|
||||
|
||||
A second test drives the read-only Watch routes that ride inside the same app:
|
||||
GET /watch (the HTML page), /watch/world.json (the static map), and
|
||||
/watch/state.json (the live snapshot) — confirming the spectator endpoints
|
||||
serve real world data alongside a working /mcp without breaking either.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import uvicorn
|
||||
from mcp import ClientSession
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
|
||||
from understone import server as understone_server
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
PACK = str(understone_server.PACKAGED_WORLD_DIR)
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
port = s.getsockname()[1]
|
||||
s.close()
|
||||
return int(port)
|
||||
|
||||
|
||||
def _build_server(port: int, db_path: str) -> uvicorn.Server:
|
||||
app = understone_server.create_app(db_path, PACK)
|
||||
config = uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning", access_log=False)
|
||||
return uvicorn.Server(config)
|
||||
|
||||
|
||||
def _wait_ready(port: int, timeout: float = 5.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=0.5):
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.05)
|
||||
raise TimeoutError(f"understone server at 127.0.0.1:{port} not ready after {timeout}s")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_server(tmp_path: Path) -> Any:
|
||||
"""Boot the real Understone app in a background uvicorn thread."""
|
||||
port = _find_free_port()
|
||||
db_path = str(tmp_path / "wire.db")
|
||||
server = _build_server(port, db_path)
|
||||
|
||||
def _run() -> None:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(server.serve())
|
||||
|
||||
thread = threading.Thread(target=_run, daemon=True, name="understone-itest")
|
||||
thread.start()
|
||||
try:
|
||||
_wait_ready(port)
|
||||
yield f"http://127.0.0.1:{port}/mcp"
|
||||
finally:
|
||||
server.should_exit = True
|
||||
thread.join(timeout=5)
|
||||
# create_app installed a module-level game whose Store holds an open
|
||||
# SQLite connection; close it and clear the singleton so the next test
|
||||
# builds its own rather than inheriting this temp DB.
|
||||
if understone_server._GAME is not None:
|
||||
understone_server._GAME.store.close()
|
||||
understone_server._GAME = None
|
||||
# FastMCP caches a StreamableHTTPSessionManager on the module-level mcp
|
||||
# singleton and refuses a second lifespan .run() on the same instance.
|
||||
# Reset it so each fixture instance boots a fresh session manager (the
|
||||
# production server only ever runs one). Without this, a second
|
||||
# fixture-using test fails on "run() can only be called once".
|
||||
understone_server.mcp._session_manager = None
|
||||
|
||||
|
||||
async def _call_text(session: ClientSession, name: str, arguments: dict[str, Any]) -> str:
|
||||
result = await session.call_tool(name, arguments)
|
||||
chunks = [block.text for block in result.content if getattr(block, "type", None) == "text"]
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
async def _drive(url: str) -> dict[str, Any]:
|
||||
"""Run the full client conversation and return observations."""
|
||||
observations: dict[str, Any] = {}
|
||||
async with (
|
||||
streamable_http_client(url) as (read, write, _get_session_id),
|
||||
ClientSession(read, write) as session,
|
||||
):
|
||||
await session.initialize()
|
||||
|
||||
tools = await session.list_tools()
|
||||
observations["tool_names"] = sorted(t.name for t in tools.tools)
|
||||
|
||||
observations["join_one"] = await _call_text(session, "door_join", {"player": "Brandr"})
|
||||
observations["look_one_before"] = await _call_text(
|
||||
session, "door_look", {"player": "Brandr"}
|
||||
)
|
||||
|
||||
# A SECOND, independent session joins a second adventurer in the same world.
|
||||
async with (
|
||||
streamable_http_client(url) as (read, write, _get_session_id),
|
||||
ClientSession(read, write) as session,
|
||||
):
|
||||
await session.initialize()
|
||||
# Place player two adjacent to player one so they share the view.
|
||||
await _call_text(session, "door_join", {"player": "Sigrun"})
|
||||
await _call_text(
|
||||
session, "door_move", {"player": "Sigrun", "heading": "east", "distance": 1}
|
||||
)
|
||||
|
||||
# Back as player one: the shared world now shows the other adventurer.
|
||||
async with (
|
||||
streamable_http_client(url) as (read, write, _get_session_id),
|
||||
ClientSession(read, write) as session,
|
||||
):
|
||||
await session.initialize()
|
||||
observations["look_one_after"] = await _call_text(
|
||||
session, "door_look", {"player": "Brandr"}
|
||||
)
|
||||
observations["rank"] = await _call_text(session, "door_rank", {"player": "Brandr"})
|
||||
|
||||
return observations
|
||||
|
||||
|
||||
def test_mcp_end_to_end(live_server: str) -> None:
|
||||
obs = asyncio.run(_drive(live_server))
|
||||
|
||||
# All nine tools are advertised over the wire.
|
||||
expected = {
|
||||
"door_help",
|
||||
"door_join",
|
||||
"door_status",
|
||||
"door_look",
|
||||
"door_move",
|
||||
"door_action",
|
||||
"door_log",
|
||||
"door_rank",
|
||||
"door_bestow",
|
||||
}
|
||||
assert set(obs["tool_names"]) == expected
|
||||
|
||||
# The join + look frames are real ASCII map frames.
|
||||
assert "@" in obs["join_one"]
|
||||
look_before = obs["look_one_before"]
|
||||
assert "@" in look_before
|
||||
assert "┌" in look_before and "┐" in look_before
|
||||
|
||||
# Shared-world proof: after player two joins next door, player one sees '☻'.
|
||||
assert "☻" in obs["look_one_after"]
|
||||
# And the leaderboard lists both adventurers (one process, one world).
|
||||
assert "Brandr" in obs["rank"]
|
||||
assert "Sigrun" in obs["rank"]
|
||||
|
||||
|
||||
def _watch_base(mcp_url: str) -> str:
|
||||
"""Derive the app root (where /watch lives) from the /mcp endpoint URL."""
|
||||
return mcp_url[: -len("/mcp")] if mcp_url.endswith("/mcp") else mcp_url
|
||||
|
||||
|
||||
async def _join_over_mcp(mcp_url: str, name: str) -> None:
|
||||
"""Sign one adventurer in over the real MCP wire (so state.json sees them)."""
|
||||
async with (
|
||||
streamable_http_client(mcp_url) as (read, write, _get_session_id),
|
||||
ClientSession(read, write) as session,
|
||||
):
|
||||
await session.initialize()
|
||||
await _call_text(session, "door_join", {"player": name})
|
||||
|
||||
|
||||
def test_watch_routes_serve_world_state(live_server: str) -> None:
|
||||
base = _watch_base(live_server)
|
||||
|
||||
# The MCP join writes the player into the shared world the routes read.
|
||||
asyncio.run(_join_over_mcp(live_server, "Watcher"))
|
||||
|
||||
with httpx.Client(timeout=5.0) as client:
|
||||
page = client.get(f"{base}/watch")
|
||||
world = client.get(f"{base}/watch/world.json")
|
||||
state = client.get(f"{base}/watch/state.json")
|
||||
|
||||
# The page is real HTML carrying the static masthead.
|
||||
assert page.status_code == 200
|
||||
assert page.headers["content-type"].startswith("text/html")
|
||||
assert "Understone — Live Watch" in page.text
|
||||
|
||||
# The static world payload matches the loaded world.
|
||||
assert world.status_code == 200
|
||||
world_body = world.json()
|
||||
assert world_body["width"] == 96
|
||||
assert world_body["height"] == 48
|
||||
assert len(world_body["glyph_rows"]) == world_body["height"]
|
||||
assert all(len(row) == world_body["width"] for row in world_body["glyph_rows"])
|
||||
|
||||
# The live snapshot lists the adventurer who joined over MCP.
|
||||
assert state.status_code == 200
|
||||
state_body = state.json()
|
||||
names = {p["name"] for p in state_body["players"]}
|
||||
assert "Watcher" in names
|
||||
|
||||
|
||||
def test_watch_routes_coexist_with_mcp(live_server: str) -> None:
|
||||
"""The custom routes don't shadow /mcp: tool calls still work alongside them."""
|
||||
base = _watch_base(live_server)
|
||||
|
||||
async def _drive_both() -> tuple[str, int]:
|
||||
async with (
|
||||
streamable_http_client(live_server) as (read, write, _get_session_id),
|
||||
ClientSession(read, write) as session,
|
||||
):
|
||||
await session.initialize()
|
||||
joined = await _call_text(session, "door_join", {"player": "Coexist"})
|
||||
with httpx.Client(timeout=5.0) as client:
|
||||
status = client.get(f"{base}/watch/state.json").status_code
|
||||
return joined, status
|
||||
|
||||
joined, watch_status = asyncio.run(_drive_both())
|
||||
assert "@" in joined # the MCP tool still returns a real frame
|
||||
assert watch_status == 200 # and the watch route still answers
|
||||
|
||||
|
||||
def test_streamable_http_host_gate_off_localhost() -> None:
|
||||
"""A non-localhost bind must accept remote `Host` headers on /mcp.
|
||||
|
||||
REGRESSION: FastMCP freezes DNS-rebinding protection (a localhost-only Host
|
||||
allowlist) at CONSTRUCTION, and ``server`` builds its FastMCP at import with
|
||||
the default 127.0.0.1 host. A 0.0.0.0/LAN bind therefore answered TCP and
|
||||
`/watch` but 421'd `/mcp` for every remote node ("Invalid Host header").
|
||||
``_serve`` drops the allowlist when bound off localhost; this pins the
|
||||
mechanism — a default instance rejects a foreign Host, a protection-disabled
|
||||
one accepts it (a 421 in the second case is the bug returning).
|
||||
|
||||
Uses fresh FastMCP instances (not the module singleton) so there is no
|
||||
shared-state or app-cache coupling with the live-server tests above.
|
||||
"""
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
foreign = {
|
||||
"Host": "192.168.0.239:8077",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
init = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2025-03-26",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "probe", "version": "0"},
|
||||
},
|
||||
}
|
||||
|
||||
# Default (localhost-baked allowlist) — a remote Host is refused.
|
||||
locked = FastMCP("hostgate-locked")
|
||||
with TestClient(locked.streamable_http_app()) as client:
|
||||
assert client.post("/mcp", headers=foreign, json=init).status_code == 421
|
||||
|
||||
# Protection disabled (what _serve does off localhost) — remote Host accepted.
|
||||
opened = FastMCP("hostgate-open")
|
||||
opened.settings.transport_security = TransportSecuritySettings(
|
||||
enable_dns_rebinding_protection=False
|
||||
)
|
||||
with TestClient(opened.streamable_http_app()) as client:
|
||||
resp = client.post("/mcp", headers=foreign, json=init)
|
||||
assert resp.status_code != 421, f"remote Host still rejected: {resp.status_code} {resp.text}"
|
||||
@@ -1,336 +0,0 @@
|
||||
"""Movement resolution tests.
|
||||
|
||||
Covers edge clipping on all four sides, blocking terrain, the two input
|
||||
forms (``"NNEE"`` vs heading+distance) and their equivalence, location
|
||||
entry flipping to MENU, the MAX_STEPS cap, and a stubbed always-encounter
|
||||
RNG interrupting a walk with a pending fight.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.conftest import (
|
||||
FOREST,
|
||||
GRASS,
|
||||
WALL,
|
||||
WATER,
|
||||
LocationDef,
|
||||
Zone,
|
||||
make_player,
|
||||
make_world,
|
||||
)
|
||||
from understone.engine.models import Mode, WorldEvent
|
||||
from understone.engine.movement import MAX_STEPS, parse_directions, resolve_move
|
||||
from understone.engine.rng import GameRNG
|
||||
|
||||
|
||||
class _NeverRNG(GameRNG):
|
||||
"""An RNG whose chance() never fires (no wandering encounters)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(seed=0)
|
||||
|
||||
def chance(self, probability: float) -> bool: # noqa: ARG002
|
||||
return False
|
||||
|
||||
|
||||
class _AlwaysRNG(GameRNG):
|
||||
"""An RNG whose chance() always fires (forces an encounter).
|
||||
|
||||
The seed still drives ``weighted_index``/``randint``, so different seeds
|
||||
select different event rows while every encounter roll fires.
|
||||
"""
|
||||
|
||||
def __init__(self, seed: int = 0) -> None:
|
||||
super().__init__(seed=seed)
|
||||
|
||||
def chance(self, probability: float) -> bool: # noqa: ARG002
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse_directions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_steps_string() -> None:
|
||||
assert parse_directions("NNEE", "", 1) == ["N", "N", "E", "E"]
|
||||
|
||||
|
||||
def test_parse_heading_distance() -> None:
|
||||
assert parse_directions("", "east", 3) == ["E", "E", "E"]
|
||||
|
||||
|
||||
def test_parse_clamps_to_max_steps() -> None:
|
||||
assert parse_directions("NNNNNNNNNNNN", "", 1) == ["N"] * MAX_STEPS
|
||||
assert parse_directions("", "north", 99) == ["N"] * MAX_STEPS
|
||||
|
||||
|
||||
def test_parse_rejects_unknown_direction() -> None:
|
||||
try:
|
||||
parse_directions("NQ", "", 1)
|
||||
except ValueError as exc:
|
||||
assert "Q" in str(exc)
|
||||
else: # pragma: no cover - failure path
|
||||
raise AssertionError("expected ValueError")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge clipping (all four sides)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_clip_north_edge() -> None:
|
||||
world = make_world()
|
||||
player = make_player(x=5, y=0)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="north", distance=3)
|
||||
assert player.y == 0
|
||||
assert result.steps_taken == 0
|
||||
assert result.blocked
|
||||
|
||||
|
||||
def test_clip_south_edge() -> None:
|
||||
world = make_world()
|
||||
player = make_player(x=5, y=10)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="south", distance=3)
|
||||
assert player.y == 10
|
||||
assert result.blocked
|
||||
|
||||
|
||||
def test_clip_west_edge() -> None:
|
||||
world = make_world()
|
||||
player = make_player(x=0, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="west", distance=3)
|
||||
assert player.x == 0
|
||||
assert result.blocked
|
||||
|
||||
|
||||
def test_clip_east_edge() -> None:
|
||||
world = make_world()
|
||||
player = make_player(x=10, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="east", distance=3)
|
||||
assert player.x == 10
|
||||
assert result.blocked
|
||||
|
||||
|
||||
def test_partial_move_then_clip() -> None:
|
||||
world = make_world()
|
||||
player = make_player(x=8, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="east", distance=5)
|
||||
# 8 -> 9 -> 10, then edge.
|
||||
assert player.x == 10
|
||||
assert result.steps_taken == 2
|
||||
assert result.blocked
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Blocking terrain
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_blocked_by_wall() -> None:
|
||||
grid = [[GRASS for _ in range(11)] for _ in range(11)]
|
||||
grid[5][6] = WALL
|
||||
world = make_world(grid=grid)
|
||||
player = make_player(x=5, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="east", distance=2)
|
||||
assert player.x == 5
|
||||
assert result.blocked
|
||||
assert "wall" in result.blocked_reason
|
||||
|
||||
|
||||
def test_blocked_by_water() -> None:
|
||||
grid = [[GRASS for _ in range(11)] for _ in range(11)]
|
||||
grid[4][5] = WATER
|
||||
world = make_world(grid=grid)
|
||||
player = make_player(x=5, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="north", distance=2)
|
||||
assert player.y == 5
|
||||
assert result.blocked
|
||||
assert "water" in result.blocked_reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input-form equivalence and direction correctness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_nnee_lands_at_expected_cell() -> None:
|
||||
world = make_world()
|
||||
player = make_player(x=5, y=5)
|
||||
resolve_move(world, player, _NeverRNG(), steps="NNEE")
|
||||
# Two north (y-2), two east (x+2).
|
||||
assert (player.x, player.y) == (7, 3)
|
||||
|
||||
|
||||
def test_heading_equivalent_to_steps() -> None:
|
||||
world_a = make_world()
|
||||
player_a = make_player(x=5, y=5)
|
||||
resolve_move(world_a, player_a, _NeverRNG(), steps="EEE")
|
||||
|
||||
world_b = make_world()
|
||||
player_b = make_player(x=5, y=5)
|
||||
resolve_move(world_b, player_b, _NeverRNG(), heading="east", distance=3)
|
||||
|
||||
assert (player_a.x, player_a.y) == (player_b.x, player_b.y)
|
||||
|
||||
|
||||
def test_max_steps_truncates_long_walk() -> None:
|
||||
world = make_world(width=40, height=11)
|
||||
player = make_player(x=0, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="east", distance=99)
|
||||
assert result.steps_taken == MAX_STEPS
|
||||
assert player.x == MAX_STEPS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Location entry flips to MENU
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_entering_location_flips_menu_mode() -> None:
|
||||
loc = LocationDef(
|
||||
key="inn",
|
||||
kind="inn",
|
||||
name="The Sleeping Drake",
|
||||
x=7,
|
||||
y=5,
|
||||
glyph="I",
|
||||
color="town",
|
||||
actions=("rest", "leave"),
|
||||
)
|
||||
world = make_world(locations=[loc])
|
||||
player = make_player(x=5, y=5)
|
||||
result = resolve_move(world, player, _NeverRNG(), heading="east", distance=4)
|
||||
assert player.mode is Mode.MENU
|
||||
assert player.at_location == "inn"
|
||||
assert result.entered_location == "inn"
|
||||
# Stopped on the door at x=7 even though distance asked for 4.
|
||||
assert (player.x, player.y) == (7, 5)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Encounter interrupt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_always_encounter_stops_with_pending_fight() -> None:
|
||||
grid = [[FOREST for _ in range(11)] for _ in range(11)]
|
||||
zone = Zone(key="wood", x0=0, y0=0, x1=10, y1=10, tier_lo=1, tier_hi=2)
|
||||
world = make_world(grid=grid, zones=[zone])
|
||||
player = make_player(x=5, y=5)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=5)
|
||||
assert result.pending_fight == (1, 2)
|
||||
# The encounter fires on the first entered cell.
|
||||
assert result.steps_taken == 1
|
||||
assert player.x == 6
|
||||
|
||||
|
||||
def test_no_zone_means_no_encounter() -> None:
|
||||
grid = [[FOREST for _ in range(11)] for _ in range(11)]
|
||||
world = make_world(grid=grid, zones=[])
|
||||
player = make_player(x=5, y=5)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=3)
|
||||
assert result.pending_fight is None
|
||||
assert result.steps_taken == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Weighted non-combat overworld events (v0.2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _event_world(*events: WorldEvent) -> object:
|
||||
"""An all-forest, fully-zoned world carrying a crafted event table."""
|
||||
grid = [[FOREST for _ in range(11)] for _ in range(11)]
|
||||
zone = Zone(key="wood", x0=0, y0=0, x1=10, y1=10, tier_lo=1, tier_hi=2)
|
||||
return make_world(grid=grid, zones=[zone], events=list(events))
|
||||
|
||||
|
||||
def test_event_fight_stops_the_walk() -> None:
|
||||
"""A fight-kind event sets pending_fight and halts the walk like v0.1."""
|
||||
world = _event_world(WorldEvent("fight", 1, "", 0, 0))
|
||||
player = make_player(x=5, y=5)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=5)
|
||||
assert result.pending_fight == (1, 2)
|
||||
assert result.event is None
|
||||
assert result.steps_taken == 1 # stopped on the first triggering cell
|
||||
|
||||
|
||||
def test_event_gold_credits_and_continues() -> None:
|
||||
"""A gold event credits the rolled amount and does NOT stop the walk."""
|
||||
world = _event_world(WorldEvent("gold", 1, "a coin-purse", 5, 5))
|
||||
player = make_player(x=5, y=5, gold=10)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=3)
|
||||
assert result.event is not None
|
||||
assert result.event.kind == "gold"
|
||||
assert result.event.amount == 5 # min == max == 5, so deterministic
|
||||
assert player.gold == 15
|
||||
assert result.pending_fight is None
|
||||
assert result.steps_taken == 3 # the walk ran to completion
|
||||
|
||||
|
||||
def test_event_heal_caps_at_max_hp() -> None:
|
||||
"""A heal event never overfills: hp is clamped to max_hp."""
|
||||
world = _event_world(WorldEvent("heal", 1, "a spring", 50, 50))
|
||||
player = make_player(x=5, y=5, hp=18, max_hp=20)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=1)
|
||||
assert player.hp == 20 # +50 requested, capped at the 2 missing
|
||||
assert result.event is not None and result.event.amount == 2
|
||||
|
||||
|
||||
def test_event_trap_floors_hp_at_one_and_spares_gold() -> None:
|
||||
"""A trap event never kills (floors at 1 HP) and never touches gold."""
|
||||
world = _event_world(WorldEvent("trap", 1, "old briars", 500, 500))
|
||||
player = make_player(x=5, y=5, hp=10, max_hp=20, gold=42)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=1)
|
||||
assert player.hp == 1 # huge trap, but floored
|
||||
assert player.gold == 42 # gold untouched
|
||||
assert result.event is not None and result.event.amount == 9 # only 9 could be taken
|
||||
|
||||
|
||||
def test_event_lore_mutates_nothing() -> None:
|
||||
"""A lore event changes no state and reports a zero amount."""
|
||||
world = _event_world(WorldEvent("lore", 1, "an old waystone", 0, 0))
|
||||
player = make_player(x=5, y=5, hp=15, max_hp=20, gold=7)
|
||||
before = (player.hp, player.gold)
|
||||
result = resolve_move(world, player, _AlwaysRNG(), heading="east", distance=2)
|
||||
assert (player.hp, player.gold) == before
|
||||
assert result.event is not None and result.event.kind == "lore"
|
||||
assert result.event.amount == 0
|
||||
assert result.steps_taken == 2
|
||||
|
||||
|
||||
def test_at_most_one_event_per_walk() -> None:
|
||||
"""Once any event fires, no further cells roll for the rest of the walk.
|
||||
|
||||
Two distinct gold rolls would credit 2 gold (1 each); a single fired event
|
||||
credits exactly 1, proving the walk stops rolling after the first trigger.
|
||||
"""
|
||||
world = _event_world(WorldEvent("gold", 1, "a coin", 1, 1))
|
||||
player = make_player(x=5, y=5, gold=0)
|
||||
resolve_move(world, player, _AlwaysRNG(), heading="east", distance=5)
|
||||
assert player.gold == 1 # exactly one event, not five
|
||||
|
||||
|
||||
def test_each_event_kind_reachable_with_crafted_table() -> None:
|
||||
"""Equal weights make every kind in a crafted table reachable from movement."""
|
||||
table = [
|
||||
WorldEvent("fight", 1, "", 0, 0),
|
||||
WorldEvent("gold", 1, "g", 1, 1),
|
||||
WorldEvent("heal", 1, "h", 1, 1),
|
||||
WorldEvent("trap", 1, "t", 1, 1),
|
||||
WorldEvent("lore", 1, "l", 0, 0),
|
||||
]
|
||||
zone = Zone(key="wood", x0=0, y0=0, x1=0, y1=0, tier_lo=1, tier_hi=2)
|
||||
grid = [[FOREST for _ in range(11)] for _ in range(11)]
|
||||
world = make_world(grid=grid, zones=[zone], events=table)
|
||||
|
||||
seen: set[str] = set()
|
||||
for seed in range(60):
|
||||
player = make_player(x=0, y=1, hp=10, max_hp=20) # one step north into the zone cell
|
||||
result = resolve_move(world, player, _AlwaysRNG(seed), steps="N")
|
||||
if result.pending_fight is not None:
|
||||
seen.add("fight")
|
||||
elif result.event is not None:
|
||||
seen.add(result.event.kind)
|
||||
assert seen == {"fight", "gold", "heal", "trap", "lore"}
|
||||
@@ -1,9 +0,0 @@
|
||||
"""Smoke test for the packaging skeleton."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import understone
|
||||
|
||||
|
||||
def test_version_present() -> None:
|
||||
assert understone.__version__ == "0.10.0"
|
||||
@@ -1,269 +0,0 @@
|
||||
"""SQLite persistence tests.
|
||||
|
||||
Covers idempotent schema init, a full player round-trip through every
|
||||
column (including ``def_``, ``turn_day``, ``log_cursor`` and the bestow
|
||||
fields), event append with cursor-based catch-up, leaderboard tie-breaks,
|
||||
and that WAL journaling is active.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tests.conftest import make_player
|
||||
from understone.engine.log import since
|
||||
from understone.engine.models import Mode
|
||||
from understone.persistence import Store
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "understone.db")
|
||||
|
||||
|
||||
def test_schema_init_is_idempotent(tmp_path: Path) -> None:
|
||||
db = tmp_path / "understone.db"
|
||||
Store(db).close()
|
||||
# Re-opening the same file must not error or duplicate schema.
|
||||
second = Store(db)
|
||||
assert second.get_meta("schema_version") == "1"
|
||||
second.close()
|
||||
|
||||
|
||||
def test_wal_mode_active(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
assert store.journal_mode().lower() == "wal"
|
||||
store.close()
|
||||
|
||||
|
||||
def test_player_round_trip_all_columns(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
player = make_player(
|
||||
name="Brandr",
|
||||
x=12,
|
||||
y=7,
|
||||
hp=18,
|
||||
max_hp=26,
|
||||
level=3,
|
||||
xp=305,
|
||||
gold=88,
|
||||
atk=9,
|
||||
def_=4,
|
||||
weapon_id="short_sword",
|
||||
armor_id="leather_armor",
|
||||
turns_left=6,
|
||||
turn_day=739_400,
|
||||
mode=Mode.MENU,
|
||||
at_location="inn",
|
||||
log_cursor=42,
|
||||
bestow_spent=15,
|
||||
bestow_day=739_400,
|
||||
posts_sent=3,
|
||||
post_day=739_400,
|
||||
gambles=2,
|
||||
gamble_day=739_400,
|
||||
banked=420,
|
||||
)
|
||||
store.upsert_player(player)
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
reopened = _store(tmp_path)
|
||||
players, _ = reopened.load_all()
|
||||
loaded = players["Brandr"]
|
||||
assert loaded == player
|
||||
assert loaded.banked == 420
|
||||
# Spot-check the fields most prone to silent drop.
|
||||
assert loaded.def_ == 4
|
||||
assert loaded.turn_day == 739_400
|
||||
assert loaded.log_cursor == 42
|
||||
assert loaded.bestow_spent == 15
|
||||
assert loaded.bestow_day == 739_400
|
||||
assert loaded.mode is Mode.MENU
|
||||
# The v0.5 social columns survive the round-trip too.
|
||||
assert loaded.posts_sent == 3
|
||||
assert loaded.post_day == 739_400
|
||||
assert loaded.gambles == 2
|
||||
assert loaded.gamble_day == 739_400
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_event_target_round_trips(tmp_path: Path) -> None:
|
||||
"""A targeted (private) event keeps its target across a reopen; public is ''."""
|
||||
store = _store(tmp_path)
|
||||
pub = store.insert_event("t1", "Brandr", "join", "set out")
|
||||
priv = store.insert_event("t2", "Sigrun", "ambushed", "robbed in your sleep", "Brandr")
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
reopened = _store(tmp_path)
|
||||
_, events = reopened.load_all()
|
||||
by_id = {e.event_id: e for e in events}
|
||||
assert by_id[pub].target == "" # public stays empty
|
||||
assert by_id[priv].target == "Brandr" # private keeps its recipient
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_ambush_table_per_day_uniqueness(tmp_path: Path) -> None:
|
||||
"""The ambushes PK is (attacker, target, day): one row per pair per day."""
|
||||
store = _store(tmp_path)
|
||||
day = 739_400
|
||||
assert store.has_ambushed("Brandr", "Sigrun", day) is False
|
||||
store.record_ambush("Brandr", "Sigrun", day)
|
||||
store.commit()
|
||||
assert store.has_ambushed("Brandr", "Sigrun", day) is True
|
||||
# A second record for the same pair/day is a no-op (INSERT OR IGNORE):
|
||||
# the duplicate must not raise and must not add a row.
|
||||
store.record_ambush("Brandr", "Sigrun", day)
|
||||
store.commit()
|
||||
rows = store._conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM ambushes WHERE attacker=? AND target=? AND day=?",
|
||||
("Brandr", "Sigrun", day),
|
||||
).fetchone()
|
||||
assert rows["n"] == 1
|
||||
# A new day is a fresh attempt; the old day stays recorded.
|
||||
assert store.has_ambushed("Brandr", "Sigrun", day + 1) is False
|
||||
store.record_ambush("Brandr", "Sigrun", day + 1)
|
||||
store.commit()
|
||||
assert store.has_ambushed("Brandr", "Sigrun", day) is True
|
||||
assert store.has_ambushed("Brandr", "Sigrun", day + 1) is True
|
||||
store.close()
|
||||
|
||||
|
||||
def test_upsert_updates_existing_row(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
player = make_player(name="Sigrun", gold=10)
|
||||
store.upsert_player(player)
|
||||
store.commit()
|
||||
player.gold = 999
|
||||
store.upsert_player(player)
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
reopened = _store(tmp_path)
|
||||
players, _ = reopened.load_all()
|
||||
assert players["Sigrun"].gold == 999
|
||||
assert len(players) == 1
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_event_append_and_since_cursor(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
id1 = store.insert_event("t1", "Brandr", "fight", "slew a rat")
|
||||
id2 = store.insert_event("t2", "Sigrun", "bestow", "blessed with gold")
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
reopened = _store(tmp_path)
|
||||
_, events = reopened.load_all()
|
||||
assert [e.event_id for e in events] == [id1, id2]
|
||||
|
||||
# Catch up from a cursor before both, then advance past the first.
|
||||
fresh, cursor = since(events, 0)
|
||||
assert len(fresh) == 2
|
||||
assert cursor == id2
|
||||
|
||||
after_first, cursor2 = since(events, id1)
|
||||
assert [e.event_id for e in after_first] == [id2]
|
||||
assert cursor2 == id2
|
||||
|
||||
nothing, cursor3 = since(events, id2)
|
||||
assert nothing == []
|
||||
assert cursor3 == id2
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_top_ranks_tie_breaks(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
# Same level: higher XP ranks first; equal XP breaks by name ascending.
|
||||
store.upsert_player(make_player(name="Carol", level=5, xp=1200, gold=10))
|
||||
store.upsert_player(make_player(name="Alice", level=5, xp=1500, gold=10))
|
||||
store.upsert_player(make_player(name="Bob", level=5, xp=1500, gold=10))
|
||||
store.upsert_player(make_player(name="Dave", level=4, xp=9999, gold=10))
|
||||
store.commit()
|
||||
|
||||
ranks = store.top_ranks(limit=10)
|
||||
assert [r.name for r in ranks] == ["Alice", "Bob", "Carol", "Dave"]
|
||||
store.close()
|
||||
|
||||
|
||||
def test_top_ranks_honours_limit(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
for i in range(15):
|
||||
store.upsert_player(make_player(name=f"P{i:02d}", level=i, xp=i * 10))
|
||||
store.commit()
|
||||
ranks = store.top_ranks(limit=10)
|
||||
assert len(ranks) == 10
|
||||
# Highest level first.
|
||||
assert ranks[0].name == "P14"
|
||||
store.close()
|
||||
|
||||
|
||||
def test_meta_round_trip(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
store.set_meta("world_name", "The Vale of Understone")
|
||||
assert store.get_meta("world_name") == "The Vale of Understone"
|
||||
assert store.get_meta("missing") is None
|
||||
store.close()
|
||||
|
||||
|
||||
def test_retention_columns_round_trip(tmp_path: Path) -> None:
|
||||
"""The retention columns survive a reopen: depth, the v0.10 stack-encoded
|
||||
satchel, the two forged plusses, and the v0.10 banked vault gold."""
|
||||
store = _store(tmp_path)
|
||||
player = make_player(
|
||||
name="Delver",
|
||||
deepest_rung=2,
|
||||
satchel="minor_potion:3,iron_ore:5", # v0.10 "id:qty" stack encoding
|
||||
weapon_plus=2,
|
||||
armor_plus=1,
|
||||
banked=300,
|
||||
)
|
||||
store.upsert_player(player)
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
reopened = _store(tmp_path)
|
||||
players, _ = reopened.load_all()
|
||||
loaded = players["Delver"]
|
||||
assert loaded == player # full equality across every column
|
||||
assert loaded.deepest_rung == 2
|
||||
assert loaded.satchel == "minor_potion:3,iron_ore:5"
|
||||
assert loaded.weapon_plus == 2
|
||||
assert loaded.armor_plus == 1
|
||||
assert loaded.banked == 300
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_v0_7_depth_columns_default_for_legacy_rows(tmp_path: Path) -> None:
|
||||
"""A row written without the new columns loads them at their defaults.
|
||||
|
||||
The schema mutates in place (no migration, stamp stays 1), so the new
|
||||
columns carry DB-side defaults: a pre-v0.7 player row (inserted with the
|
||||
legacy column set) must read back deepest_rung 0, an empty satchel, and
|
||||
zero plusses rather than erroring.
|
||||
"""
|
||||
store = _store(tmp_path)
|
||||
store._conn.execute(
|
||||
"INSERT INTO players "
|
||||
"(name, x, y, hp, max_hp, level, xp, gold, atk, def_, weapon_id, armor_id, "
|
||||
" turns_left, turn_day, mode, at_location, created_at, last_seen, log_cursor, "
|
||||
" bestow_spent, bestow_day) "
|
||||
"VALUES ('Old', 5, 5, 20, 20, 1, 0, 20, 5, 1, 'rusty_dagger', 'cloth_tunic', "
|
||||
" 10, 0, 'tile', '', 't0', 't0', 0, 0, 0)",
|
||||
)
|
||||
store.commit()
|
||||
store.close()
|
||||
|
||||
reopened = _store(tmp_path)
|
||||
players, _ = reopened.load_all()
|
||||
old = players["Old"]
|
||||
assert old.deepest_rung == 0
|
||||
assert old.satchel == ""
|
||||
assert old.weapon_plus == 0
|
||||
assert old.armor_plus == 0
|
||||
assert old.banked == 0 # the v0.10 vault column defaults too
|
||||
assert reopened.get_meta("schema_version") == "1" # stamp unchanged
|
||||
reopened.close()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user