mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 07:22:24 -06:00
Compare commits
269 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4d402fea6b | |||
| 46d14ddd86 | |||
| 7c4157f78d | |||
| 3856d80709 | |||
| 8142d2f1ad | |||
| c234d66ebf | |||
| b180770eff | |||
| 9d2e11f2be | |||
| 57080f4615 | |||
| 45f27fb2a7 | |||
| ebc8e75285 | |||
| 485af92f7f | |||
| 664d44c109 | |||
| 3cf9485169 | |||
| d43b9d1647 | |||
| ea8d9d1798 | |||
| d9aa50dca9 | |||
| 6f89d0cc13 | |||
| 62d2a0fe6a | |||
| 5df37f83a7 | |||
| 651c4d98cd | |||
| e901e859c7 | |||
| 200dcfeac5 | |||
| 8c414feba2 | |||
| d7cea053b6 | |||
| c45e98462b | |||
| e17cbe35a5 | |||
| fd47c23177 | |||
| 9fe988b1be | |||
| 3ce66960bc | |||
| 8a852a12e3 | |||
| 9a518657a3 | |||
| c424176c73 | |||
| 2c32e89de3 | |||
| 06310c74ee | |||
| dfad58a3d2 | |||
| e31197d64a | |||
| 5f9200f6a0 | |||
| 84b0d5615c | |||
| d41621877f | |||
| a4f3d205d1 | |||
| 6078a88533 | |||
| 843fa04e65 | |||
| 473298199d | |||
| c251e2dac8 | |||
| ac47476d0a | |||
| 055bd5a88f | |||
| a7d9461735 | |||
| 9de77c3ee3 | |||
| 0cfe521ce7 | |||
| c2750de7a4 | |||
| bd782f804e | |||
| 87b69a318b | |||
| a315cabe71 | |||
| be17d8c5d0 | |||
| 62ce450b06 | |||
| d19dad05bd | |||
| 262a6a9918 | |||
| 2bb55590bf | |||
| 0e02d1b52c | |||
| 9b29453e9b | |||
| c4ff1caf09 | |||
| 22245145db | |||
| 8eacc4d632 | |||
| 23fed785c4 | |||
| 688c27e68a | |||
| 405baf7cb2 | |||
| 1027c22333 | |||
| 381651049b | |||
| 322b7dabc4 | |||
| d5e86c8493 | |||
| c154ea3966 | |||
| 755ab51802 | |||
| 9df8ab836f | |||
| 8d88e6a7eb | |||
| cce292f793 | |||
| 6adc577d30 | |||
| 02c50b81c1 | |||
| 753cd04b4e | |||
| c3217748dc | |||
| 8cbff49694 | |||
| 4f26d63c14 | |||
| 74347fb29f | |||
| 2ace8cccc8 | |||
| e95b8f5ca1 | |||
| 7a32c51a1c | |||
| dce663105b | |||
| cfef3616e6 | |||
| c22d39a798 | |||
| 63921450b1 | |||
| 929fad63be | |||
| 976e9df3b6 | |||
| 7cb21b84f1 | |||
| 7263edd48d | |||
| 6742c7e405 | |||
| 1aa6982868 | |||
| 42d1abbd04 | |||
| f543ed714a | |||
| 2c6abb0fde | |||
| 491fc6748a | |||
| 9a996f0067 | |||
| a465ac6383 | |||
| a4539923e4 | |||
| 42e99d6990 | |||
| a0ff22e137 | |||
| de4b3b3909 | |||
| 120d229b5f | |||
| c6f4c11870 | |||
| 979fab37a9 | |||
| da5bf90a4b | |||
| 70c18467cb | |||
| 801774bc4a | |||
| 497984b452 | |||
| bdc1eba34c | |||
| 028c77cae5 | |||
| 76d007d83f | |||
| 3f432b8a42 | |||
| f74aa2264e | |||
| d00aae2429 | |||
| 5e09940745 | |||
| 72bd62d3d8 | |||
| d31f89b2e3 | |||
| 9bae8f1a10 | |||
| a012561195 | |||
| 4198b59a0f | |||
| 4f6ef13ce9 | |||
| 52716ed611 | |||
| 6c9a7d7351 | |||
| 3518f7953c | |||
| 48769e5a97 | |||
| adb4ff6399 | |||
| 131a1ec943 | |||
| 1cded9b430 | |||
| 62c741eb8a | |||
| 3362917e1e | |||
| 698cbbf988 | |||
| e47a08b7bc | |||
| aaa427debd | |||
| 611af76971 | |||
| bbe28ecab3 | |||
| 93a9fd3c28 | |||
| 62ff3217d0 | |||
| b086390558 | |||
| d08a57dfc2 | |||
| 9fdf51ff3d | |||
| 45471894da | |||
| c0b5952573 | |||
| b9d5b5b671 | |||
| 274c97135e | |||
| e87f8e19c2 | |||
| bb221f4dab | |||
| 361876b17a | |||
| 5d26cd6593 | |||
| 2d4420e00d | |||
| b20548583d | |||
| f63b2915cc | |||
| bf85bbea94 | |||
| 803d8ee8f9 | |||
| 17b5961a70 | |||
| 037308f3b1 | |||
| d7a9895855 | |||
| 0ba49b8bb7 | |||
| 51b5b3ee74 | |||
| c3b0ddeba7 | |||
| a533e1c783 | |||
| d8bc78556f | |||
| 4f5854e768 | |||
| bf2dc04cb3 | |||
| bb894d073b | |||
| b3934a2d14 | |||
| 3d02cf66b4 | |||
| 7631b88792 | |||
| a07172b0c0 | |||
| f3d33bf44a | |||
| fdb1a189e8 | |||
| 58e2d9348f | |||
| 771d03b8e6 | |||
| d147aaea36 | |||
| 8b747178e0 | |||
| d57280d807 | |||
| b9870f279c | |||
| 71ee340bc6 | |||
| c5d5d0b7cd | |||
| 1f9d03c3e0 | |||
| 24e082df05 | |||
| 4a78d20eea | |||
| e0d17e0f99 | |||
| cd6c49dd01 | |||
| 8454e961ba | |||
| 4ae38bc2ae | |||
| ab1a71c86c | |||
| ce57df6888 | |||
| 275f40eebb | |||
| 2c510f8617 | |||
| 2afb9c7f72 | |||
| 5b8ab94446 | |||
| 30828e9f9c | |||
| 6d0dc6df94 | |||
| 7e680ee883 | |||
| e950219246 | |||
| b3764a8035 | |||
| 756c4d8929 | |||
| 04c50568e9 | |||
| 3bf220c503 | |||
| 4b853e329e | |||
| 29ffdc36d0 | |||
| ada8b80509 | |||
| 1f47ca62de | |||
| 83d9233304 | |||
| bf06102d37 | |||
| e015b4512d | |||
| 0c1afff7fc | |||
| a94051a995 | |||
| 2f906ea1f9 | |||
| b61bfd1aa6 | |||
| 19c3a48b10 | |||
| 414eb52d67 | |||
| 86b404177b | |||
| 10165bb8a1 | |||
| 5d478573cc | |||
| 1b24e4717f | |||
| 9a2db63c07 | |||
| e159837b74 | |||
| ec3454ee2e | |||
| 5cbb832162 | |||
| 4e4ae2a91d | |||
| c7d0bac638 | |||
| e86305c143 | |||
| d0fc42195a | |||
| 760321f7ee | |||
| 693e51f782 | |||
| ba07409724 | |||
| c76a61841e | |||
| 341d2f604f | |||
| 3f7f8495d6 | |||
| dc464ac313 | |||
| 52d59cf7b7 | |||
| 2dc885ab4d | |||
| 14488f43e0 | |||
| 90f2070146 | |||
| 1e551830ea | |||
| 84cc212ecd | |||
| da4025d338 | |||
| 88085c29ff | |||
| 3152667a0c | |||
| 4b44d88401 | |||
| 8957b9ce0e | |||
| 28a6b0dd33 | |||
| 7bc17cc072 | |||
| 10a1800492 | |||
| 1010f163f0 | |||
| c28bfc1e58 | |||
| e71ea38953 | |||
| 5378b33641 | |||
| 9b605f81a3 | |||
| f05e6bddad | |||
| 75eda9a096 | |||
| 4c00d71150 | |||
| 80e1924d7f | |||
| 471bf89c8b | |||
| 35785d3a0e | |||
| 2ff0cd8240 | |||
| f9f0ff0b53 | |||
| df8a36ced4 | |||
| ef6cac6428 | |||
| f4c3b3a9c4 | |||
| d06db3feee | |||
| 50544c0d1b | |||
| d1c484737f |
+34
-14
@@ -1,29 +1,49 @@
|
||||
# =============================================================================
|
||||
# Turnstone Environment Variables
|
||||
# Copy to .env and adjust values for your deployment
|
||||
# Copy to .env and adjust values for your deployment.
|
||||
#
|
||||
# Usage:
|
||||
# Single node: docker compose --profile production up
|
||||
# 10-node cluster: docker compose --profile cluster up
|
||||
# =============================================================================
|
||||
|
||||
# -- LLM Backend --------------------------------------------------------------
|
||||
LLM_BASE_URL=http://host.docker.internal:8000/v1
|
||||
OPENAI_API_KEY=sk-...
|
||||
# ANTHROPIC_API_KEY=sk-ant-... # Set instead for Anthropic provider
|
||||
# TAVILY_API_KEY=tvly-... # For web search fallback (local models only)
|
||||
OPENAI_API_KEY=dummy
|
||||
# ANTHROPIC_API_KEY=sk-ant-...# Set instead of OPENAI_API_KEY for Anthropic
|
||||
# TAVILY_API_KEY=tvly-... # Web search fallback (local models only)
|
||||
# MODEL=# Override default model alias
|
||||
|
||||
# -- Database (production profile) --------------------------------------------
|
||||
# -- Authentication (required) ------------------------------------------------
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
TURNSTONE_JWT_SECRET=changeme-to-32-bytes-of-hex
|
||||
|
||||
# -- Database ------------------------------------------------------------------
|
||||
# Single-node default is SQLite (zero config). Set these for PostgreSQL:
|
||||
# DB_BACKEND=postgresql
|
||||
# POSTGRES_USER=turnstone
|
||||
# POSTGRES_PASSWORD=changeme
|
||||
# DATABASE_URL=postgresql+psycopg://turnstone:changeme@postgres:5432/turnstone
|
||||
|
||||
# -- Redis ---------------------------------------------------------------------
|
||||
# REDIS_PASSWORD=
|
||||
# REDIS_PORT=6379
|
||||
|
||||
# -- Authentication ------------------------------------------------------------
|
||||
# TURNSTONE_AUTH_ENABLED=true
|
||||
# TURNSTONE_AUTH_TOKEN=your-secret-token
|
||||
# TURNSTONE_JWT_SECRET=python -c "import secrets; print(secrets.token_hex(32))"
|
||||
|
||||
# -- Ports ---------------------------------------------------------------------
|
||||
# SERVER_PORT=8080
|
||||
# CONSOLE_PORT=8090
|
||||
|
||||
# -- Workspace -----------------------------------------------------------------
|
||||
# Bind-mount a host directory into the container at /workspace.
|
||||
# The model can read/write files here. Default: empty Docker volume.
|
||||
# WORKSPACE_MOUNT=/path/to/your/project
|
||||
|
||||
# -- Agent behavior ------------------------------------------------------------
|
||||
# SKIP_PERMISSIONS=true # Auto-approve all tool calls (dev only)
|
||||
# MCP_CONFIG=/workspace/mcp.json# MCP server configuration file
|
||||
|
||||
# -- Discord channel gateway ---------------------------------------------------
|
||||
# TURNSTONE_DISCORD_TOKEN=
|
||||
# TURNSTONE_DISCORD_GUILD=0
|
||||
|
||||
# -- Cluster (profile: cluster) -----------------------------------------------
|
||||
# These are set per-node in compose.yaml; only override for custom topologies.
|
||||
# TURNSTONE_NODE_ID=node-1
|
||||
# TURNSTONE_ADVERTISE_URL=http://server-1:8080
|
||||
|
||||
|
||||
+34
-108
@@ -5,55 +5,40 @@
|
||||
"helpers:pinGitHubActionDigests",
|
||||
":separateMajorReleases"
|
||||
],
|
||||
"labels": [
|
||||
"dependencies"
|
||||
"gitIgnoredAuthors": [
|
||||
"41898282+github-actions[bot]@users.noreply.github.com"
|
||||
],
|
||||
"labels": ["dependencies"],
|
||||
"prConcurrentLimit": 5,
|
||||
"prHourlyLimit": 2,
|
||||
"schedule": [
|
||||
"before 9am on Monday"
|
||||
],
|
||||
"schedule": ["before 9am on Monday"],
|
||||
"timezone": "America/New_York",
|
||||
"lockFileMaintenance": {
|
||||
"enabled": true,
|
||||
"schedule": [
|
||||
"before 9am on Monday"
|
||||
]
|
||||
"schedule": ["before 9am on Monday"]
|
||||
},
|
||||
"customManagers": [
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "Track vendored KaTeX version",
|
||||
"managerFilePatterns": [
|
||||
"/pyproject\\.toml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"katex-(?<currentValue>[\\d.]+)/"
|
||||
],
|
||||
"managerFilePatterns": ["/pyproject\\.toml$/"],
|
||||
"matchStrings": ["katex-(?<currentValue>[\\d.]+)/"],
|
||||
"depNameTemplate": "katex",
|
||||
"datasourceTemplate": "npm"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "Track vendored Highlight.js version",
|
||||
"managerFilePatterns": [
|
||||
"/pyproject\\.toml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"hljs-(?<currentValue>[\\d.]+)/"
|
||||
],
|
||||
"managerFilePatterns": ["/pyproject\\.toml$/"],
|
||||
"matchStrings": ["hljs-(?<currentValue>[\\d.]+)/"],
|
||||
"depNameTemplate": "highlight.js",
|
||||
"datasourceTemplate": "npm"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "Track vendored Mermaid version",
|
||||
"managerFilePatterns": [
|
||||
"/pyproject\\.toml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"mermaid-(?<currentValue>[\\d.]+)/"
|
||||
],
|
||||
"managerFilePatterns": ["/pyproject\\.toml$/"],
|
||||
"matchStrings": ["mermaid-(?<currentValue>[\\d.]+)/"],
|
||||
"depNameTemplate": "mermaid",
|
||||
"datasourceTemplate": "npm"
|
||||
}
|
||||
@@ -62,14 +47,8 @@
|
||||
{
|
||||
"description": "LLM SDKs — always review manually",
|
||||
"groupName": "LLM SDKs",
|
||||
"matchPackageNames": [
|
||||
"openai",
|
||||
"anthropic",
|
||||
"mcp"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on Monday"
|
||||
],
|
||||
"matchPackageNames": ["openai", "anthropic", "mcp"],
|
||||
"schedule": ["before 9am on Monday"],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
@@ -83,73 +62,38 @@
|
||||
"httpx-sse",
|
||||
"pydantic"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on Wednesday"
|
||||
],
|
||||
"schedule": ["before 9am on Wednesday"],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
"matchUpdateTypes": ["patch"]
|
||||
},
|
||||
{
|
||||
"description": "Database layer",
|
||||
"groupName": "Database",
|
||||
"matchPackageNames": [
|
||||
"sqlalchemy",
|
||||
"alembic",
|
||||
"psycopg"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on Wednesday"
|
||||
],
|
||||
"matchPackageNames": ["sqlalchemy", "alembic", "psycopg"],
|
||||
"schedule": ["before 9am on Wednesday"],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
"matchUpdateTypes": ["patch"]
|
||||
},
|
||||
{
|
||||
"description": "Security-critical — always review manually",
|
||||
"groupName": "Security",
|
||||
"matchPackageNames": [
|
||||
"PyJWT",
|
||||
"pyjwt",
|
||||
"bcrypt"
|
||||
],
|
||||
"matchPackageNames": ["PyJWT", "pyjwt", "bcrypt"],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
"description": "Infrastructure dependencies",
|
||||
"groupName": "Infrastructure",
|
||||
"matchPackageNames": [
|
||||
"structlog",
|
||||
"redis",
|
||||
"croniter",
|
||||
"discord.py"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"matchPackageNames": ["structlog", "croniter", "discord.py"],
|
||||
"schedule": ["before 9am on the first day of the month"],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
"matchUpdateTypes": ["patch"]
|
||||
},
|
||||
{
|
||||
"description": "Vendored JS — requires manual file download after merge",
|
||||
"description": "Vendored JS — CI workflow downloads files automatically",
|
||||
"groupName": "Vendored JS",
|
||||
"matchPackageNames": [
|
||||
"katex",
|
||||
"highlight.js",
|
||||
"mermaid"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"automerge": false,
|
||||
"prBodyNotes": [
|
||||
"This PR updates version references only.",
|
||||
"After merging, run `scripts/update-vendored-js.sh <lib> <version>` to download the actual files."
|
||||
]
|
||||
"matchPackageNames": ["katex", "highlight.js", "mermaid"],
|
||||
"schedule": ["before 9am on the first day of the month"],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
"description": "Dev/test tooling",
|
||||
@@ -157,51 +101,33 @@
|
||||
"matchPackageNames": [
|
||||
"ruff",
|
||||
"mypy",
|
||||
"types-redis",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pre-commit"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"schedule": ["before 9am on the first day of the month"],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
"matchUpdateTypes": ["patch"]
|
||||
},
|
||||
{
|
||||
"description": "Docker base images",
|
||||
"groupName": "Docker Images",
|
||||
"matchManagers": [
|
||||
"dockerfile",
|
||||
"docker-compose"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"matchManagers": ["dockerfile", "docker-compose"],
|
||||
"schedule": ["before 9am on the first day of the month"],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
"description": "TypeScript SDK dev dependencies",
|
||||
"groupName": "TypeScript SDK",
|
||||
"matchFileNames": [
|
||||
"sdk/typescript/**"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"matchFileNames": ["sdk/typescript/**"],
|
||||
"schedule": ["before 9am on the first day of the month"],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
"matchUpdateTypes": ["patch"]
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions — group all action updates",
|
||||
"groupName": "GitHub Actions",
|
||||
"matchManagers": [
|
||||
"github-actions"
|
||||
],
|
||||
"matchManagers": ["github-actions"],
|
||||
"automerge": false
|
||||
}
|
||||
]
|
||||
|
||||
@@ -2,9 +2,10 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, "stable/*"]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [main, "stable/*"]
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
@@ -25,8 +26,8 @@ jobs:
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install mypy types-redis
|
||||
- run: pip install -e ".[mq]"
|
||||
- run: pip install mypy
|
||||
- run: pip install -e ".[all]"
|
||||
- run: mypy turnstone/
|
||||
|
||||
test:
|
||||
@@ -39,7 +40,7 @@ jobs:
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- run: pip install -e ".[test,mq]"
|
||||
- run: pip install -e ".[test]"
|
||||
- run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -q
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
@@ -47,11 +48,37 @@ jobs:
|
||||
name: coverage-${{ matrix.python-version }}
|
||||
path: coverage.xml
|
||||
|
||||
test-postgres:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: turnstone_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd="pg_isready -U postgres"
|
||||
--health-interval=10s
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install -e ".[test,postgres]"
|
||||
- run: pytest tests/ -m "not live" --storage-backend=postgresql -q
|
||||
env:
|
||||
TURNSTONE_TEST_PG_URL: postgresql+psycopg://postgres:postgres@localhost:5432/turnstone_test
|
||||
|
||||
lock-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@e06108dd0aef18192324c70427afc47652e63a82 # v7
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- run: uv lock --check
|
||||
@@ -60,7 +87,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@e06108dd0aef18192324c70427afc47652e63a82 # v7
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
@@ -68,7 +95,8 @@ jobs:
|
||||
python-version: "3.14"
|
||||
- run: uv sync --frozen --all-extras
|
||||
- run: uv pip install pip-audit
|
||||
- run: uv run pip-audit --strict --desc
|
||||
- name: Security audit (dependencies)
|
||||
run: uv export --no-emit-project --frozen | uv run pip-audit --strict --desc -r /dev/stdin
|
||||
|
||||
security-ts:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
name: Publish Docker Image
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["CI"]
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
group: docker-${{ github.event.workflow_run.head_sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
if: github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve release tag
|
||||
id: tag
|
||||
run: |
|
||||
TAG=$(git tag --points-at HEAD | grep '^v' | head -1)
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "No v* tag at HEAD — skipping publish"
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Compute Docker tags
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
id: tags
|
||||
env:
|
||||
REF: ${{ steps.tag.outputs.tag }}
|
||||
run: |
|
||||
VERSION="${REF#v}"
|
||||
FULL="${REGISTRY}/${IMAGE_NAME}"
|
||||
FULL="${FULL,,}"
|
||||
|
||||
if echo "$VERSION" | grep -qE '(a|b|rc)[0-9]+$'; then
|
||||
TAGS="${FULL}:${VERSION},${FULL}:experimental"
|
||||
else
|
||||
MINOR="${VERSION%.*}"
|
||||
TAGS="${FULL}:${VERSION},${FULL}:${MINOR},${FULL}:stable,${FULL}:latest"
|
||||
fi
|
||||
echo "tags=${TAGS}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
|
||||
- name: Build and push
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.tags.outputs.tags }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -2,7 +2,7 @@ name: Docker Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, "stable/*"]
|
||||
schedule:
|
||||
- cron: "0 6 * * 1" # Weekly Monday 06:00 UTC
|
||||
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
name: Publish to PyPI
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
workflow_run:
|
||||
workflows: ["CI"]
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
group: publish-${{ github.event.workflow_run.head_sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -10,20 +15,43 @@ permissions:
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve release tag
|
||||
id: tag
|
||||
run: |
|
||||
TAG=$(git tag --points-at HEAD | grep '^v' | head -1)
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "No v* tag at HEAD — skipping publish"
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install build
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
- run: python -m build
|
||||
- uses: pypa/gh-action-pypi-publish@release/v1
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
- uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@b25b93d384199fc0fc8c2e126b2d937a0cbeb2ae # v2
|
||||
if: steps.tag.outputs.skip == 'false'
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
tag_name: ${{ steps.tag.outputs.tag }}
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: ${{ contains(github.ref, '-') }}
|
||||
prerelease: ${{ contains(steps.tag.outputs.tag, 'a') || contains(steps.tag.outputs.tag, 'b') || contains(steps.tag.outputs.tag, 'rc') }}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
name: Complete Vendored JS Updates
|
||||
|
||||
# When Renovate bumps a vendored JS version in pyproject.toml, this
|
||||
# workflow downloads the actual files and commits them to the PR branch
|
||||
# so the PR is merge-ready without manual intervention.
|
||||
#
|
||||
# Note: the commit is made with GITHUB_TOKEN, so it won't re-trigger CI
|
||||
# automatically. The reviewer should re-run CI once this workflow passes,
|
||||
# or Renovate's next rebase will trigger it.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- pyproject.toml
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: "PR number to update"
|
||||
required: true
|
||||
type: number
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
vendor-js:
|
||||
if: github.actor == 'renovate[bot]' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Resolve PR head ref
|
||||
id: ref
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
ref=$(gh pr view "${{ inputs.pr_number }}" --repo "${{ github.repository }}" --json headRefName -q .headRefName)
|
||||
else
|
||||
ref="${{ github.head_ref }}"
|
||||
fi
|
||||
echo "head_ref=${ref}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ steps.ref.outputs.head_ref }}
|
||||
|
||||
- name: Detect vendored JS changes
|
||||
id: detect
|
||||
run: |
|
||||
updates=()
|
||||
for lib in katex hljs mermaid; do
|
||||
version=$(grep -oE "${lib}-[0-9.]+" pyproject.toml | head -1 | sed "s/${lib}-//")
|
||||
[[ -z "$version" ]] && continue
|
||||
[[ -d "turnstone/shared_static/${lib}-${version}" ]] && continue
|
||||
updates+=("${lib}:${version}")
|
||||
done
|
||||
if [[ ${#updates[@]} -eq 0 ]]; then
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "found=true" >> "$GITHUB_OUTPUT"
|
||||
printf '%s\n' "${updates[@]}" > /tmp/updates.txt
|
||||
echo "Libs to update:"
|
||||
cat /tmp/updates.txt
|
||||
fi
|
||||
|
||||
- name: Download vendored files
|
||||
if: steps.detect.outputs.found == 'true'
|
||||
run: |
|
||||
while IFS=: read -r lib version; do
|
||||
echo "::group::Updating ${lib} to ${version}"
|
||||
bash scripts/update-vendored-js.sh "$lib" "$version"
|
||||
echo "::endgroup::"
|
||||
done < /tmp/updates.txt
|
||||
|
||||
- name: Commit and push
|
||||
if: steps.detect.outputs.found == 'true'
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add -A
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "chore: download vendored JS files"
|
||||
git push
|
||||
@@ -19,3 +19,5 @@ venv/
|
||||
.hypothesis/
|
||||
PROGRESS.md
|
||||
.coverage
|
||||
tools/skill_audit_analysis/data/
|
||||
tools/skill_audit_analysis/output/
|
||||
|
||||
@@ -10,7 +10,7 @@ repos:
|
||||
rev: v1.19.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
additional_dependencies: [types-redis>=4.6, redis>=7.2]
|
||||
additional_dependencies: []
|
||||
args: [--config-file=pyproject.toml]
|
||||
pass_filenames: false
|
||||
entry: mypy turnstone/
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# libexpat integer overflow — no fix available in Debian repos yet
|
||||
# https://avd.aquasec.com/nvd/cve-2026-25210
|
||||
# Review: remove this entry once a patched libexpat1 is published
|
||||
CVE-2026-25210
|
||||
|
||||
# ncurses buffer overflow — no fix in Debian 13 repos yet
|
||||
# Affects libncursesw6, libtinfo6, ncurses-base, ncurses-bin
|
||||
# https://avd.aquasec.com/nvd/cve-2025-69720
|
||||
CVE-2025-69720
|
||||
|
||||
# nghttp2 DoS via malformed HTTP/2 frames — no fix in Debian 13 repos yet
|
||||
# Affects libnghttp2-14
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27135
|
||||
CVE-2026-27135
|
||||
|
||||
# systemd arbitrary code execution via spurious IPC — no fix in Debian 13 repos yet
|
||||
# Affects libsystemd0, libudev1
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29111
|
||||
CVE-2026-29111
|
||||
|
||||
# glibc iconv() DoS — fix_deferred, no patched libc in Debian 13 yet
|
||||
# Affects libc-bin, libc6
|
||||
# https://avd.aquasec.com/nvd/cve-2026-4046
|
||||
CVE-2026-4046
|
||||
|
||||
# minimatch ReDoS — transitive npm dep (MCP server), no direct exposure
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27903
|
||||
CVE-2026-27903
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27904
|
||||
CVE-2026-27904
|
||||
|
||||
# picomatch ReDoS — transitive npm dep, no direct exposure
|
||||
# https://avd.aquasec.com/nvd/cve-2026-33671
|
||||
CVE-2026-33671
|
||||
|
||||
# node-tar path traversal — transitive npm dep, not used to extract untrusted archives
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29786
|
||||
CVE-2026-29786
|
||||
# https://avd.aquasec.com/nvd/cve-2026-31802
|
||||
CVE-2026-31802
|
||||
+22
-9
@@ -1,6 +1,6 @@
|
||||
# =============================================================================
|
||||
# Turnstone — Docker build with uv for reproducible, locked installs
|
||||
# Single image for all services: server, bridge, console, sim, eval
|
||||
# Single image for all services: server, console, channel, eval
|
||||
# =============================================================================
|
||||
|
||||
FROM python:3.14-slim
|
||||
@@ -8,29 +8,39 @@ FROM python:3.14-slim
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.10.10 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.3 /uv /usr/local/bin/uv
|
||||
|
||||
# System dependencies for psycopg (PostgreSQL client library)
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libpq5 \
|
||||
# Remove the slim image's man page exclusion so man-db has actual content
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/docker
|
||||
|
||||
# System dependencies: psycopg (libpq5), developer tooling for agent workflows
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
|
||||
libpq5 git curl jq man-db manpages procps file \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Node.js LTS (for npx-based MCP servers like @modelcontextprotocol/server-github)
|
||||
COPY --from=node:24-slim /usr/local/bin/node /usr/local/bin/node
|
||||
COPY --from=node:24-slim /usr/local/lib/node_modules /usr/local/lib/node_modules
|
||||
RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
|
||||
&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx
|
||||
|
||||
# Non-root user
|
||||
RUN useradd --create-home --shell /bin/bash turnstone
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Compile bytecode for faster startup
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
|
||||
# Install dependencies first (cached layer — only re-runs when deps change)
|
||||
COPY pyproject.toml uv.lock README.md LICENSE ./
|
||||
RUN uv sync --frozen --no-install-project --no-dev \
|
||||
--extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic
|
||||
--no-compile --extra all
|
||||
|
||||
# Install the project itself
|
||||
COPY turnstone/ turnstone/
|
||||
RUN uv sync --frozen --no-dev \
|
||||
--extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic
|
||||
--no-compile --extra all
|
||||
|
||||
# Compile bytecode in a separate step (avoids fd exhaustion during install)
|
||||
RUN python -m compileall -q .venv turnstone/
|
||||
|
||||
# Add venv to PATH so entry points are found
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
@@ -45,6 +55,9 @@ COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
WORKDIR /data
|
||||
RUN chown turnstone:turnstone /data
|
||||
|
||||
# Workspace mount point — bind-mount a host directory here
|
||||
RUN mkdir -p /workspace && chown turnstone:turnstone /workspace
|
||||
|
||||
USER turnstone
|
||||
|
||||
ENTRYPOINT ["entrypoint.sh"]
|
||||
|
||||
+2
-2
@@ -45,9 +45,9 @@ That's it — no flags, no arguments. The wizard prompts for everything.
|
||||
The wizard supports two deployment modes:
|
||||
|
||||
- **Single-node production** (`docker compose --profile production up`) —
|
||||
1 server + bridge + console + PostgreSQL + Redis. Good for most use cases.
|
||||
1 server + console + PostgreSQL. Good for most use cases.
|
||||
- **Multi-node cluster** (`docker compose --profile cluster up`) —
|
||||
10-node server/bridge fleet + PostgreSQL + Redis. For high-throughput or
|
||||
10-node server fleet + console + PostgreSQL. For high-throughput or
|
||||
HA deployments.
|
||||
|
||||
## Example Session
|
||||
|
||||
@@ -5,409 +5,136 @@
|
||||
[](https://pypi.org/project/turnstone/)
|
||||
[](LICENSE)
|
||||
|
||||
Multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers, driven by message queues or interactive interfaces.
|
||||
Multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers with direct HTTP routing, interactive interfaces, and enterprise governance.
|
||||
|
||||
Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) — a bird that flips rocks to expose what's hiding underneath.
|
||||
<p align="center">
|
||||
<img src="docs/assets/hero.png" alt="Turnstone console — multi-workstream AI orchestration with mermaid diagrams" width="960"/>
|
||||
</p>
|
||||
|
||||
Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) (*Arenaria interpres*) — a shorebird that flips stones to discover what's hiding underneath.
|
||||
|
||||
### Release Tracks
|
||||
|
||||
| Track | Install | Docker | Description |
|
||||
|-------|---------|--------|-------------|
|
||||
| **Stable** | `pip install turnstone` | `ghcr.io/turnstonelabs/turnstone:stable` | Production-grade. Bugfixes only. |
|
||||
| **Experimental** | `pip install turnstone --pre` | `ghcr.io/turnstonelabs/turnstone:experimental` | New features. May have rough edges. |
|
||||
|
||||
See [docs/releasing.md](docs/releasing.md) for the full release process.
|
||||
|
||||
## What it does
|
||||
|
||||
Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports. It runs as:
|
||||
Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports.
|
||||
|
||||
- **Interactive sessions** — terminal CLI or browser UI with parallel workstreams
|
||||
- **Queue-driven agents** — trigger workstreams via message queue, stream progress, approve or auto-approve tool use
|
||||
- **Multi-node clusters** — generic work load-balances across nodes, directed work routes to a specific server
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams, reverse proxy for server UIs
|
||||
- **Intent validation** — an LLM judge evaluates every tool call before approval, presenting risk assessments and evidence-based recommendations so users can make informed decisions instead of blindly approving raw tool calls
|
||||
- **Governance & compliance** — RBAC, OIDC SSO (Okta, Azure AD, Google, Keycloak), tool policies, prompt templates, workstream templates, usage tracking, and append-only audit logs
|
||||
- **Cluster simulator** — test the stack at scale (up to 1000 nodes) without an LLM backend
|
||||
|
||||
Works with any OpenAI-compatible API (vLLM, llama.cpp, NVIDIA NIM) or Anthropic's native Messages API. Supports [MCP](https://modelcontextprotocol.io/) for external tool servers with native deferred tool loading on Anthropic and OpenAI APIs (BM25 fallback for local models).
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams with console routing proxy
|
||||
- **Intent validation** — LLM judge evaluates every tool call with risk assessments and evidence
|
||||
- **Governance** — RBAC, OIDC SSO, tool policies, skills, usage tracking, audit logs
|
||||
- **Multi-provider** — OpenAI-compatible APIs (vLLM, llama.cpp, NIM) and Anthropic Messages API
|
||||
- **MCP support** — external tool servers with native deferred loading (Anthropic/OpenAI) or BM25 fallback
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/diagrams/architecture-overview.svg" alt="Turnstone system architecture — data flow from clients through gateways, Redis MQ, cluster nodes, to LLM providers" width="960"/>
|
||||
<img src="docs/diagrams/architecture-overview.svg" alt="Turnstone system architecture" width="960"/>
|
||||
</p>
|
||||
|
||||
## Quickstart
|
||||
|
||||
### Interactive (terminal)
|
||||
|
||||
```bash
|
||||
pip install turnstone
|
||||
|
||||
# Terminal REPL
|
||||
turnstone --base-url http://localhost:8000/v1
|
||||
```
|
||||
|
||||
### Interactive (browser)
|
||||
|
||||
```bash
|
||||
# Browser UI
|
||||
turnstone-server --port 8080 --base-url http://localhost:8000/v1
|
||||
```
|
||||
|
||||
### Queue-driven (programmatic)
|
||||
|
||||
```bash
|
||||
pip install turnstone[mq]
|
||||
turnstone-bridge --server-url http://localhost:8080 --redis-host localhost
|
||||
```
|
||||
|
||||
```python
|
||||
from turnstone.mq import TurnstoneClient
|
||||
|
||||
with TurnstoneClient() as client:
|
||||
# Generic — any available node picks it up
|
||||
result = client.send_and_wait("Analyze the error logs", auto_approve=True)
|
||||
print(result.content)
|
||||
|
||||
# Directed — must run on a specific server
|
||||
result = client.send_and_wait(
|
||||
"Check disk I/O on this server",
|
||||
target_node="server-12",
|
||||
auto_approve=True,
|
||||
)
|
||||
```
|
||||
|
||||
### Cluster dashboard
|
||||
|
||||
```bash
|
||||
# Cluster dashboard
|
||||
pip install turnstone[console]
|
||||
turnstone-console --redis-host localhost --port 8090
|
||||
turnstone-console --port 8090
|
||||
```
|
||||
|
||||
Then open `http://localhost:8090` for the cluster-wide dashboard. Create workstreams from the console and interact with any node's server UI through the built-in reverse proxy — no direct server port access required.
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
cp .env.example .env # edit LLM_BASE_URL, OPENAI_API_KEY, etc.
|
||||
docker compose up # starts redis + server + bridge + console (SQLite)
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
For production with PostgreSQL:
|
||||
See [QUICKSTART.md](QUICKSTART.md) for the bootstrap wizard and [docs/docker.md](docs/docker.md) for Docker configuration and profiles.
|
||||
|
||||
```bash
|
||||
# Requires POSTGRES_PASSWORD and DB_BACKEND=postgresql in .env (or exported)
|
||||
docker compose --profile production up # adds PostgreSQL, uses it as database
|
||||
### Programmatic (SDK)
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
ws = client.create_workstream(name="demo")
|
||||
result = client.send_and_wait("Analyze the error logs", ws.ws_id, auto_approve=True)
|
||||
print(result.content)
|
||||
```
|
||||
|
||||
Console dashboard at http://localhost:8090. See [docs/docker.md](docs/docker.md) for configuration, scaling, and profiles.
|
||||
|
||||
### Simulator
|
||||
|
||||
Test the multi-node stack at scale without an LLM backend:
|
||||
|
||||
```bash
|
||||
docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
Or standalone:
|
||||
|
||||
```bash
|
||||
pip install turnstone[sim]
|
||||
turnstone-sim --nodes 100 --scenario steady --duration 60 --mps 10
|
||||
```
|
||||
|
||||
See [docs/simulator.md](docs/simulator.md) for scenarios, CLI reference, and metrics.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Diagrams
|
||||
|
||||
Detailed UML diagrams are available in [`docs/diagrams/`](docs/diagrams/):
|
||||
|
||||
| Diagram | Description |
|
||||
|---------|-------------|
|
||||
| [System Context](docs/diagrams/png/01-system-context.png) | Top-level components and external dependencies |
|
||||
| [Package Structure](docs/diagrams/png/02-package-structure.png) | Python modules and dependency graph |
|
||||
| [Core Engine Classes](docs/diagrams/png/03-core-engine-classes.png) | SessionUI protocol, ChatSession, LLMProvider, WorkstreamManager |
|
||||
| [Conversation Turn](docs/diagrams/png/04-conversation-turn.png) | Full message lifecycle through the engine (provider-agnostic) |
|
||||
| [Tool Pipeline](docs/diagrams/png/05-tool-pipeline.png) | Three-phase prepare/approve/execute |
|
||||
| [MQ Protocol](docs/diagrams/png/06-mq-protocol.png) | 9 inbound + 19 outbound message types |
|
||||
| [Message Routing](docs/diagrams/png/07-message-routing.png) | Multi-node routing scenarios |
|
||||
| [Redis Key Schema](docs/diagrams/png/08-redis-key-schema.png) | All Redis keys, types, and TTLs |
|
||||
| [Workstream States](docs/diagrams/png/09-workstream-states.png) | State machine transitions |
|
||||
| [Simulator](docs/diagrams/png/10-simulator-architecture.png) | SimCluster, dispatchers, scenarios |
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection threads |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose service topology |
|
||||
| [SDK Architecture](docs/diagrams/png/13-sdk-architecture.png) | Python + TypeScript client libraries |
|
||||
| [Storage Architecture](docs/diagrams/png/14-storage-architecture.png) | Pluggable database backends (SQLite + PostgreSQL) |
|
||||
| [Auth Architecture](docs/diagrams/png/15-auth-architecture.png) | JWT, scopes, token types, login flows |
|
||||
| [Channel Architecture](docs/diagrams/png/16-channel-architecture.png) | Discord/Slack adapter protocol and routing |
|
||||
| [Notify Flow](docs/diagrams/png/17-notify-flow.png) | Channel notification dispatch |
|
||||
| [Watch Architecture](docs/diagrams/png/18-watch-architecture.png) | Periodic command polling daemon |
|
||||
| [Governance Architecture](docs/diagrams/png/19-governance-architecture.png) | RBAC, policies, audit, usage enforcement flow |
|
||||
| [WS Template Architecture](docs/diagrams/png/21-ws-template-architecture.png) | Workstream template application and lifecycle |
|
||||
| [Judge Architecture](docs/diagrams/png/22-judge-architecture.png) | Intent validation two-tier evaluation pipeline |
|
||||
| [OIDC Architecture](docs/diagrams/png/25-oidc-architecture.png) | OIDC SSO authorization code flow with PKCE |
|
||||
|
||||
### Governance
|
||||
|
||||
Turnstone includes a built-in governance layer for enterprise deployments — manage who can do what, which tools run unattended, and where every token goes.
|
||||
|
||||
- **RBAC** — 15 granular permissions, 3 built-in roles (admin / operator / viewer), custom roles, privilege escalation prevention
|
||||
- **OIDC SSO** — single sign-on via any OpenID Connect provider (Okta, Azure AD, Google, Keycloak); Authorization Code Flow with PKCE, auto-provisioning, claim-based role mapping with demotion propagation; see [docs/oidc.md](docs/oidc.md)
|
||||
- **Tool policies** — glob-pattern rules (`allow` / `deny` / `ask`) with priority ordering; automate approvals or lock down dangerous tools
|
||||
- **Prompt templates** — reusable system messages with `{{variable}}` substitution and categories
|
||||
- **Usage tracking** — per-request token and tool metrics, aggregation by day / model / user, automatic 90-day pruning
|
||||
- **Audit logging** — append-only event trail for all admin mutations, IP-aware, 365-day retention
|
||||
|
||||
All governance features are managed through the console admin panel (13 tabs) and the full REST API. Runtime settings (model, tools, rate limiting, health, judge, memory) are configurable via the admin Settings tab — no config file edits or restarts needed for most changes. See [docs/governance.md](docs/governance.md) for setup and [docs/settings.md](docs/settings.md) for the settings reference.
|
||||
|
||||
### Intent Validation (LLM Judge)
|
||||
|
||||
Every tool call that requires human approval is evaluated by an intent validation judge that provides a structured risk assessment alongside the approval prompt — so instead of "approve this bash command?", users see a verdict with risk level, confidence, recommendation, and reasoning.
|
||||
|
||||
The system uses a two-tier evaluation pipeline:
|
||||
|
||||
1. **Heuristic tier** (instant, free) — 23 pattern-based rules classify tool calls by severity. Catches destructive commands (`rm -rf /`, `DROP TABLE`), privilege escalation (`sudo`), credential access, and more. Results appear immediately.
|
||||
2. **LLM judge tier** (async) — A full LLM evaluation runs in the background with access to `read_file` and `list_directory` for evidence gathering. The judge can inspect files that a write would overwrite, check directory contents before a delete, and cite specific evidence in its reasoning. Results update the UI progressively when ready.
|
||||
|
||||
The judge defaults to the same model as the session (self-consistency) but can be configured to use a separate model — useful when running a small local model for tasks but wanting a commercial model for safety evaluation.
|
||||
|
||||
```toml
|
||||
[judge]
|
||||
enabled = true # on by default
|
||||
model = "" # empty = same as session model
|
||||
provider = "" # empty = same as session provider
|
||||
timeout = 60.0 # generous for local models
|
||||
```
|
||||
|
||||
Verdicts are persisted for audit and exposed via Prometheus metrics (`turnstone_judge_verdicts_total`, `turnstone_judge_llm_latency_seconds`). See [docs/judge.md](docs/judge.md) for the full guide.
|
||||
|
||||
## Multi-node routing
|
||||
|
||||
Each Turnstone server runs a bridge process. Bridges share a Redis instance for coordination:
|
||||
|
||||
| Redis Key | Purpose |
|
||||
|-----------|---------|
|
||||
| `turnstone:inbound` | Shared work queue — generic tasks, any node |
|
||||
| `turnstone:inbound:{node_id}` | Per-node queue — directed tasks |
|
||||
| `turnstone:ws:{ws_id}` | Workstream ownership — auto-routes follow-ups |
|
||||
| `turnstone:node:{node_id}` | Node heartbeat + metadata for discovery |
|
||||
| `turnstone:events:{ws_id}` | Per-workstream event pub/sub |
|
||||
| `turnstone:events:global` | Global event pub/sub |
|
||||
| `turnstone:events:cluster` | Cluster-wide state changes (for turnstone-console) |
|
||||
|
||||
**Routing rules:**
|
||||
1. Message has `target_node` → routes to that node's queue
|
||||
2. Message has `ws_id` → looks up owner, routes to owning node
|
||||
3. Neither → shared queue, next available bridge picks it up
|
||||
|
||||
Bridges BLPOP from their per-node queue (priority) then the shared queue. Directed work always takes precedence.
|
||||
|
||||
## Tools
|
||||
|
||||
15 built-in tools, 2 agent tools, plus external tools via MCP:
|
||||
Built-in tools for shell, files, search, web, memory, notifications, and autonomous sub-agents — plus external tools via [MCP](https://modelcontextprotocol.io/) with native deferred loading. See [docs/tools.md](docs/tools.md) for the full reference and [docs/mcp.md](docs/mcp.md) for MCP configuration.
|
||||
|
||||
| Tool | Description | Auto-approved |
|
||||
|------|-------------|:---:|
|
||||
| `bash` | Execute shell commands | |
|
||||
| `read_file` | Read file contents (text or images with vision models) | yes |
|
||||
| `write_file` | Write/create files | |
|
||||
| `edit_file` | Fuzzy-match file editing | |
|
||||
| `search` | Search files by name/content | yes |
|
||||
| `math` | Sandboxed Python evaluation | |
|
||||
| `man` | Read man pages | yes |
|
||||
| `web_fetch` | Fetch URL content | |
|
||||
| `web_search` | Web search (provider-native or Tavily) | |
|
||||
| `memory` | Structured persistent memory (save/search/delete/list) | yes |
|
||||
| `recall` | Search conversation history | yes |
|
||||
| `notify` | Send notifications to linked channels | yes |
|
||||
| `watch` | Periodic command polling with conditions | |
|
||||
| `task` | Spawn autonomous sub-agent | |
|
||||
| `plan` | Explore codebase, write .plan.md | |
|
||||
| `mcp__*` | External tools from MCP servers | |
|
||||
## Architecture
|
||||
|
||||
When the total tool count exceeds a configurable threshold (default 20), MCP tools are automatically deferred using native `defer_loading` on Anthropic and OpenAI APIs, or a transparent client-side BM25 search for local models. The LLM discovers deferred tools on demand via a `tool_search` capability — no configuration needed beyond `--tool-search auto` (the default).
|
||||
**Single-node**: Client → Server (direct HTTP + SSE). No external dependencies beyond the database.
|
||||
|
||||
### MCP Tool Servers
|
||||
**Multi-node**: Client → Console (hash ring routing proxy) → Server nodes. The console maintains a 65536-entry bucket cache for O(1) workstream routing. A rebalancer daemon redistributes buckets when nodes join or leave.
|
||||
|
||||
Turnstone supports the [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) for connecting external tool servers. MCP tools are discovered at startup, converted to OpenAI function-calling format, and merged with built-in tools. Each MCP tool is prefixed with `mcp__{server}__{tool}` to avoid name collisions. Tool lists stay fresh via push notifications (`tools.listChanged`), periodic polling for servers without push, and manual `/mcp refresh`.
|
||||
| Component | Purpose |
|
||||
|-----------|---------|
|
||||
| `turnstone` | Terminal CLI (REPL) |
|
||||
| `turnstone-server` | Web UI + REST API + SSE events |
|
||||
| `turnstone-console` | Cluster dashboard + routing proxy + admin panel |
|
||||
| `turnstone-channel` | Channel gateway (Discord, with adapters for Slack/Teams planned) |
|
||||
| `turnstone-admin` | User/token management CLI |
|
||||
| `turnstone-eval` | Eval harness for prompt/tool optimization |
|
||||
| `turnstone-bootstrap` | LLM-guided setup wizard |
|
||||
|
||||
Configure via `config.toml` or `--mcp-config`:
|
||||
### Diagrams
|
||||
|
||||
```toml
|
||||
[mcp.servers.github]
|
||||
command = "npx"
|
||||
args = ["-y", "@modelcontextprotocol/server-github"]
|
||||
UML diagrams in [`docs/diagrams/`](docs/diagrams/):
|
||||
|
||||
[mcp.servers.github.env]
|
||||
GITHUB_TOKEN = "ghp_..."
|
||||
```
|
||||
| Diagram | Description |
|
||||
|---------|-------------|
|
||||
| [System Context](docs/diagrams/png/01-system-context.png) | Components and external dependencies |
|
||||
| [Package Structure](docs/diagrams/png/02-package-structure.png) | Python modules and dependency graph |
|
||||
| [Core Engine](docs/diagrams/png/03-core-engine-classes.png) | SessionUI, ChatSession, LLMProvider |
|
||||
| [Conversation Turn](docs/diagrams/png/04-conversation-turn.png) | Message lifecycle through the engine |
|
||||
| [Tool Pipeline](docs/diagrams/png/05-tool-pipeline.png) | Prepare / approve / execute |
|
||||
| [Workstream States](docs/diagrams/png/09-workstream-states.png) | State machine transitions |
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose topology |
|
||||
| [Auth](docs/diagrams/png/15-auth-architecture.png) | JWT, scopes, login flows |
|
||||
| [Channels](docs/diagrams/png/16-channel-architecture.png) | Discord adapter + routing |
|
||||
| [Judge](docs/diagrams/png/22-judge-architecture.png) | Intent validation pipeline |
|
||||
| [OIDC](docs/diagrams/png/25-oidc-architecture.png) | SSO authorization code flow |
|
||||
|
||||
Or use a standard MCP JSON config file:
|
||||
## Documentation
|
||||
|
||||
```bash
|
||||
turnstone --mcp-config ~/.config/turnstone/mcp.json
|
||||
turnstone-server --mcp-config ~/.config/turnstone/mcp.json
|
||||
```
|
||||
|
||||
Use `/mcp` in the REPL to list connected tools, `/mcp refresh` to re-fetch tool lists from servers. MCP tools require user approval by default (overridden by `--skip-permissions` or UI auto-approve).
|
||||
|
||||
### Multi-Model and Multi-Provider Support
|
||||
|
||||
Turnstone supports multiple model backends per server instance, including different LLM providers. `ChatSession` delegates all API communication to pluggable `LLMProvider` adapters — the internal message format stays OpenAI-like, and each provider translates at the API boundary. Define named models in `config.toml` and select per-workstream or switch mid-session with `/model <alias>`.
|
||||
|
||||
```toml
|
||||
[models.local]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
model = "qwen3-32b"
|
||||
# provider defaults to "openai" (works with vLLM, llama.cpp, etc.)
|
||||
|
||||
[models.claude]
|
||||
provider = "anthropic"
|
||||
api_key = "sk-ant-..."
|
||||
model = "claude-opus-4-6"
|
||||
context_window = 200000
|
||||
|
||||
[models.openai]
|
||||
base_url = "https://api.openai.com/v1"
|
||||
api_key = "sk-..."
|
||||
model = "gpt-5"
|
||||
context_window = 400000
|
||||
|
||||
[model]
|
||||
default = "local" # which model to use by default
|
||||
fallback = ["claude", "openai"] # try these if the primary is unreachable
|
||||
agent_model = "claude" # optional: separate model for plan/task sub-agents
|
||||
```
|
||||
|
||||
Supported providers: `"openai"` (default -- OpenAI, vLLM, llama.cpp, any OpenAI-compatible API) and `"anthropic"` (Anthropic Messages API, requires `pip install turnstone[anthropic]`).
|
||||
|
||||
Use `/model` to show available models, `/model claude` to switch. Workstreams created via the API accept an optional `model` parameter.
|
||||
|
||||
## Configuration
|
||||
|
||||
All entry points read `~/.config/turnstone/config.toml`. CLI flags override config values.
|
||||
|
||||
```toml
|
||||
[api]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
api_key = ""
|
||||
tavily_key = "" # only needed for local/vLLM models without native search
|
||||
|
||||
[model]
|
||||
name = "" # empty = auto-detect
|
||||
temperature = 0.5
|
||||
reasoning_effort = "medium"
|
||||
default = "default" # model alias for new workstreams
|
||||
fallback = [] # ordered list of fallback model aliases
|
||||
agent_model = "" # model alias for plan/task sub-agents
|
||||
|
||||
[tools]
|
||||
timeout = 30
|
||||
skip_permissions = false
|
||||
search = "auto" # "auto" (enable when >threshold tools), "on", "off"
|
||||
search_threshold = 20 # min tools before tool search activates
|
||||
search_max_results = 5 # max tools returned per search query
|
||||
|
||||
[server]
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
max_workstreams = 10 # auto-evicts oldest idle when full
|
||||
|
||||
[redis]
|
||||
host = "localhost"
|
||||
port = 6379
|
||||
password = ""
|
||||
|
||||
[bridge]
|
||||
server_url = "http://localhost:8080"
|
||||
node_id = "" # empty = hostname_xxxx
|
||||
|
||||
[console]
|
||||
host = "0.0.0.0"
|
||||
port = 8090
|
||||
url = "http://localhost:8090" # used by CLI /cluster commands
|
||||
poll_interval = 10
|
||||
|
||||
[health]
|
||||
backend_probe_interval = 30
|
||||
backend_probe_timeout = 5
|
||||
circuit_breaker_threshold = 5
|
||||
circuit_breaker_cooldown = 60
|
||||
|
||||
[ratelimit]
|
||||
enabled = true
|
||||
requests_per_second = 10.0
|
||||
burst = 20
|
||||
|
||||
[database]
|
||||
backend = "sqlite" # "sqlite" (default) or "postgresql"
|
||||
path = ".turnstone.db" # SQLite file path (relative to working directory)
|
||||
# url = "postgresql+psycopg://user:pass@host:5432/turnstone" # PostgreSQL
|
||||
# pool_size = 5 # PostgreSQL connection pool size
|
||||
|
||||
[judge]
|
||||
enabled = true # intent validation for tool approvals (--no-judge to disable)
|
||||
model = "" # empty = same as session model (self-consistency)
|
||||
provider = "" # empty = same as session provider
|
||||
timeout = 60.0 # LLM judge timeout in seconds
|
||||
confidence_threshold = 0.7
|
||||
|
||||
[mcp]
|
||||
config_path = "" # path to MCP JSON config file (alternative to TOML sections)
|
||||
refresh_interval = 14400 # periodic refresh for servers without push notifications (seconds, 0 to disable)
|
||||
|
||||
[mcp.servers.example] # one section per MCP server
|
||||
command = "npx"
|
||||
args = ["-y", "@modelcontextprotocol/server-example"]
|
||||
# type = "stdio" # "stdio" (default) or "http"
|
||||
# url = "" # for HTTP transport
|
||||
```
|
||||
|
||||
Precedence: CLI args > environment variables > config.toml > defaults.
|
||||
|
||||
## Workstreams
|
||||
|
||||
Parallel independent conversations, each with its own session and state:
|
||||
|
||||
| Symbol | State | Meaning |
|
||||
|--------|-------|---------|
|
||||
| `·` | idle | Waiting for input |
|
||||
| `◌` | thinking | Model is generating |
|
||||
| `▸` | running | Tool execution in progress |
|
||||
| `◆` | attention | Waiting for approval |
|
||||
| `✖` | error | Something went wrong |
|
||||
|
||||
Idle workstreams are automatically cleaned up after 2 hours (configurable). In multi-node deployments, workstream ownership is tracked in Redis — follow-up messages auto-route to the owning node.
|
||||
|
||||
## Monitoring
|
||||
|
||||
`/metrics` endpoint exposes Prometheus-format metrics:
|
||||
|
||||
- `turnstone_tokens_total{direction}` — prompt/completion token counters
|
||||
- `turnstone_tool_calls_total{tool}` — per-tool invocation counts
|
||||
- `turnstone_workstream_context_ratio{ws_id}` — per-workstream context utilization
|
||||
- `turnstone_http_request_duration_seconds` — request latency histogram
|
||||
- `turnstone_workstreams_by_state{state}` — workstream state gauges
|
||||
- `turnstone_sse_connections_active` — current open SSE connections
|
||||
- `turnstone_ratelimit_rejected_total` — requests rejected by rate limiter
|
||||
- `turnstone_backend_up` — LLM backend reachability (0/1)
|
||||
- `turnstone_circuit_state` — circuit breaker state (0=closed, 1=open, 2=half_open)
|
||||
- `turnstone_workstreams_evicted_total` — workstreams auto-evicted at capacity
|
||||
- `turnstone_judge_verdicts_total{tier,risk_level}` — intent validation verdicts by tier and risk
|
||||
- `turnstone_judge_llm_latency_seconds` — LLM judge evaluation latency histogram
|
||||
- `turnstone_judge_enabled` — whether the intent validation judge is active (0/1)
|
||||
|
||||
Per-workstream metrics are labeled by `ws_id` (bounded to 10 max workstreams).
|
||||
|
||||
### Health & Rate Limiting
|
||||
|
||||
**Health degradation.** A background `BackendHealthMonitor` probes the LLM backend every `backend_probe_interval` seconds. When the backend is unreachable, `/health` reports `"status": "degraded"` (HTTP 200) and the `turnstone_backend_up` gauge drops to 0.
|
||||
|
||||
**Circuit breaker.** After `circuit_breaker_threshold` consecutive probe failures the circuit opens (CLOSED -> OPEN). While open, `ChatSession._create_stream_with_retry` skips the backend entirely and returns an error. After `circuit_breaker_cooldown` seconds the circuit enters HALF_OPEN, allowing a single probe. A successful probe closes the circuit; a failure re-opens it.
|
||||
|
||||
**Per-IP rate limiting.** When `[ratelimit].enabled` is true, each client IP is tracked with a token-bucket limiter (`requests_per_second` / `burst`). Rate limiting is applied in `do_GET`/`do_POST` after authentication but before route dispatch. `/health` and `/metrics` are exempt. Requests that exceed the limit receive HTTP 429 with a `Retry-After` header.
|
||||
|
||||
**Workstream eviction.** When `WorkstreamManager.create()` would exceed `max_workstreams`, the oldest IDLE workstream is automatically evicted and the `turnstone_workstreams_evicted_total` counter is incremented. Configure via `[server].max_workstreams` (default 10).
|
||||
| Topic | Link |
|
||||
|-------|------|
|
||||
| Configuration reference | [docs/settings.md](docs/settings.md) |
|
||||
| API reference | [docs/api-reference.md](docs/api-reference.md) |
|
||||
| Docker deployment | [docs/docker.md](docs/docker.md) |
|
||||
| Intent validation (judge) | [docs/judge.md](docs/judge.md) |
|
||||
| Governance & RBAC | [docs/governance.md](docs/governance.md) |
|
||||
| OIDC SSO | [docs/oidc.md](docs/oidc.md) |
|
||||
| TLS / mTLS | [docs/tls.md](docs/tls.md) |
|
||||
| Channel integrations | [docs/channels.md](docs/channels.md) |
|
||||
| Console dashboard | [docs/console.md](docs/console.md) |
|
||||
| Eval harness | [docs/eval.md](docs/eval.md) |
|
||||
| Tools reference | [docs/tools.md](docs/tools.md) |
|
||||
| MCP integration | [docs/mcp.md](docs/mcp.md) |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.11+
|
||||
- An OpenAI-compatible API endpoint ([vLLM](https://github.com/vllm-project/vllm), [NVIDIA NIM](https://build.nvidia.com/), [llama.cpp](https://github.com/ggml-org/llama.cpp), etc.) or an Anthropic API key
|
||||
- Redis (for message queue bridge — `pip install turnstone[mq]`)
|
||||
- Anthropic provider (optional — `pip install turnstone[anthropic]`)
|
||||
- PostgreSQL (optional, for production — `pip install turnstone[postgres]`)
|
||||
- [Git LFS](https://git-lfs.com/) (for cloning — diagram PNGs are stored in LFS)
|
||||
- An OpenAI-compatible API endpoint or Anthropic API key
|
||||
- Optional: PostgreSQL (`pip install turnstone[postgres]`), Anthropic (`pip install turnstone[anthropic]`)
|
||||
- [Git LFS](https://git-lfs.com/) for cloning (diagram PNGs)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+41
-323
@@ -6,8 +6,6 @@
|
||||
# Single node: docker compose --profile production up
|
||||
# Production (PG): DB_BACKEND=postgresql docker compose --profile production up
|
||||
# 10-node cluster: docker compose --profile cluster up
|
||||
# Cluster + DDG: docker compose --profile ddgCluster up
|
||||
# With simulator: docker compose --profile sim up
|
||||
# =============================================================================
|
||||
|
||||
name: turnstone
|
||||
@@ -17,8 +15,8 @@ networks:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
redis-data:
|
||||
turnstone-data:
|
||||
workspace:
|
||||
postgres-data:
|
||||
|
||||
services:
|
||||
@@ -30,7 +28,12 @@ services:
|
||||
profiles:
|
||||
- production
|
||||
- cluster
|
||||
- ddgCluster
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- max_connections=${POSTGRES_MAX_CONNECTIONS:-300}
|
||||
- -c
|
||||
- shared_buffers=128MB
|
||||
environment:
|
||||
POSTGRES_DB: turnstone
|
||||
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
|
||||
@@ -49,41 +52,10 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
memory: 1G
|
||||
cpus: '1.0'
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Redis — message broker, pub/sub, node registry
|
||||
# -------------------------------------------------------------------
|
||||
redis:
|
||||
image: redis:8.6-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
redis-server
|
||||
--save 60 1
|
||||
--loglevel warning
|
||||
$${REDIS_PASSWORD:+--requirepass $$REDIS_PASSWORD}
|
||||
ports:
|
||||
- "${REDIS_PORT:-6379}:6379"
|
||||
environment:
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
networks:
|
||||
- turnstone-net
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- redis-cli $${REDIS_PASSWORD:+-a $$REDIS_PASSWORD} ping | grep -q PONG
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-server — Web UI + chat workstreams + LLM interaction
|
||||
# -------------------------------------------------------------------
|
||||
@@ -109,69 +81,34 @@ services:
|
||||
- "${SERVER_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- turnstone-data:/data
|
||||
- ./docker/mcp-ddg.json:/etc/turnstone/mcp-ddg.json:ro
|
||||
- ${WORKSPACE_MOUNT:-workspace}:/workspace
|
||||
environment:
|
||||
- LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-dummy}
|
||||
- TAVILY_API_KEY=${TAVILY_API_KEY:-}
|
||||
- SKIP_PERMISSIONS=${SKIP_PERMISSIONS:-}
|
||||
- TURNSTONE_AUTH_ENABLED=${TURNSTONE_AUTH_ENABLED:-}
|
||||
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- MODEL=${MODEL:-}
|
||||
- MCP_CONFIG=${MCP_CONFIG:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-}
|
||||
- TURNSTONE_ADVERTISE_URL=${TURNSTONE_ADVERTISE_URL:-http://server:8080}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
ddg-search:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-bridge — Redis <-> HTTP bridge for multi-node routing
|
||||
# Node ID auto-generated from container hostname (no --node-id needed)
|
||||
# -------------------------------------------------------------------
|
||||
bridge:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
environment:
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
server:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
@@ -185,23 +122,16 @@ services:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port=8090
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --poll-interval=${CONSOLE_POLL_INTERVAL:-10}
|
||||
ports:
|
||||
- "${CONSOLE_PORT:-8090}:8090"
|
||||
environment:
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- TURNSTONE_AUTH_ENABLED=${TURNSTONE_AUTH_ENABLED:-}
|
||||
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_CONSOLE_URL=http://console:8090
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8090/health"]
|
||||
interval: 10s
|
||||
@@ -221,112 +151,34 @@ services:
|
||||
profiles:
|
||||
- production
|
||||
- cluster
|
||||
- ddgCluster
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-channel
|
||||
--redis-host=redis
|
||||
--redis-port=6379
|
||||
--http-host=0.0.0.0
|
||||
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
|
||||
environment:
|
||||
- TURNSTONE_DISCORD_TOKEN=${TURNSTONE_DISCORD_TOKEN:-}
|
||||
- TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0}
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
|
||||
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ddg-search — DuckDuckGo Search MCP server (HTTP transport)
|
||||
# Provides web search + content fetch tools to turnstone via MCP.
|
||||
# No API key required.
|
||||
#
|
||||
# Start with: MCP_CONFIG=/etc/turnstone/mcp-ddg.json \
|
||||
# docker compose --profile ddgCluster up
|
||||
# -------------------------------------------------------------------
|
||||
ddg-search:
|
||||
image: python:3.14-slim
|
||||
profiles:
|
||||
- ddgCluster
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
pip install --no-cache-dir duckduckgo-mcp-server &&
|
||||
python -c "from mcp.server.transport_security import TransportSecuritySettings; import duckduckgo_mcp_server.server as s; s.safe_search=s.SafeSearchMode.OFF; s.mcp.settings.host='0.0.0.0'; s.mcp.settings.port=3000; s.mcp.settings.transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False); s.mcp.run(transport='streamable-http')"
|
||||
networks:
|
||||
- turnstone-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import socket; s=socket.create_connection(('0.0.0.0',3000),2); s.close()\""]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
cpus: '0.25'
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-sim — Multi-node cluster simulator (no LLM needed)
|
||||
# Start with: docker compose --profile sim up
|
||||
# -------------------------------------------------------------------
|
||||
sim:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
profiles:
|
||||
- sim
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-sim
|
||||
--nodes "$${SIM_NODES}"
|
||||
--scenario "$${SIM_SCENARIO}"
|
||||
--duration "$${SIM_DURATION}"
|
||||
--mps "$${SIM_MPS}"
|
||||
--redis-host redis
|
||||
--redis-port 6379
|
||||
--log-level "$${SIM_LOG_LEVEL}"
|
||||
$${SIM_SEED:+--seed $$SIM_SEED}
|
||||
$${SIM_METRICS_FILE:+--metrics-file $$SIM_METRICS_FILE}
|
||||
environment:
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- SIM_NODES=${SIM_NODES:-100}
|
||||
- SIM_SCENARIO=${SIM_SCENARIO:-steady}
|
||||
- SIM_DURATION=${SIM_DURATION:-60}
|
||||
- SIM_MPS=${SIM_MPS:-5.0}
|
||||
- SIM_LOG_LEVEL=${SIM_LOG_LEVEL:-INFO}
|
||||
- SIM_SEED=${SIM_SEED:-}
|
||||
- SIM_METRICS_FILE=${SIM_METRICS_FILE:-}
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
# ===================================================================
|
||||
# 10-node cluster (profile: cluster)
|
||||
#
|
||||
# Each node is a server + bridge pair. All share the same PostgreSQL
|
||||
# and Redis instances. Access via console at :8090.
|
||||
# All nodes share the same PostgreSQL instance.
|
||||
# Access via console at :8090.
|
||||
#
|
||||
# Start: docker compose --profile cluster up
|
||||
# ===================================================================
|
||||
@@ -334,9 +186,10 @@ services:
|
||||
# -- cluster servers ------------------------------------------------
|
||||
|
||||
server-1: &cluster-server
|
||||
image: turnstone:local
|
||||
build: { context: ., dockerfile: Dockerfile }
|
||||
profiles: [cluster, ddgCluster]
|
||||
command: &cluster-server-cmd
|
||||
profiles: [cluster]
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
@@ -350,32 +203,30 @@ services:
|
||||
$${MCP_CONFIG:+--mcp-config $$MCP_CONFIG}
|
||||
volumes:
|
||||
- turnstone-data:/data
|
||||
- ./docker/mcp-ddg.json:/etc/turnstone/mcp-ddg.json:ro
|
||||
- ${WORKSPACE_MOUNT:-workspace}:/workspace
|
||||
environment: &cluster-server-env
|
||||
LLM_BASE_URL: ${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
|
||||
OPENAI_API_KEY: ${OPENAI_API_KEY:-dummy}
|
||||
TAVILY_API_KEY: ${TAVILY_API_KEY:-}
|
||||
SKIP_PERMISSIONS: ${SKIP_PERMISSIONS:-}
|
||||
TURNSTONE_AUTH_ENABLED: ${TURNSTONE_AUTH_ENABLED:-}
|
||||
TURNSTONE_AUTH_TOKEN: ${TURNSTONE_AUTH_TOKEN:-}
|
||||
TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
MODEL: ${MODEL:-}
|
||||
MCP_CONFIG: ${MCP_CONFIG:-}
|
||||
TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql}
|
||||
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
|
||||
TURNSTONE_NODE_ID: node-1
|
||||
TURNSTONE_ADVERTISE_URL: http://server-1:8080
|
||||
extra_hosts: ["host.docker.internal:host-gateway"]
|
||||
networks: [turnstone-net]
|
||||
depends_on:
|
||||
redis: { condition: service_healthy }
|
||||
postgres: { condition: service_healthy }
|
||||
ddg-search: { condition: service_healthy, required: false }
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 384M, cpus: '0.5' }
|
||||
@@ -383,162 +234,29 @@ services:
|
||||
|
||||
server-2:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-2 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-2, TURNSTONE_ADVERTISE_URL: "http://server-2:8080" }
|
||||
server-3:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-3 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-3, TURNSTONE_ADVERTISE_URL: "http://server-3:8080" }
|
||||
server-4:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-4 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-4, TURNSTONE_ADVERTISE_URL: "http://server-4:8080" }
|
||||
server-5:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-5 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-5, TURNSTONE_ADVERTISE_URL: "http://server-5:8080" }
|
||||
server-6:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-6 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-6, TURNSTONE_ADVERTISE_URL: "http://server-6:8080" }
|
||||
server-7:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-7 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-7, TURNSTONE_ADVERTISE_URL: "http://server-7:8080" }
|
||||
server-8:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-8 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-8, TURNSTONE_ADVERTISE_URL: "http://server-8:8080" }
|
||||
server-9:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-9 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-9, TURNSTONE_ADVERTISE_URL: "http://server-9:8080" }
|
||||
server-10:
|
||||
<<: *cluster-server
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-10 }
|
||||
environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: node-10, TURNSTONE_ADVERTISE_URL: "http://server-10:8080" }
|
||||
|
||||
# -- cluster bridges ------------------------------------------------
|
||||
|
||||
bridge-1: &cluster-bridge
|
||||
build: { context: ., dockerfile: Dockerfile }
|
||||
profiles: [cluster, ddgCluster]
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-1:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
environment: &cluster-bridge-env
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
|
||||
TURNSTONE_AUTH_TOKEN: ${TURNSTONE_AUTH_TOKEN:-}
|
||||
TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:-}
|
||||
networks: [turnstone-net]
|
||||
depends_on:
|
||||
server-1: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 256M, cpus: '0.25' }
|
||||
restart: unless-stopped
|
||||
|
||||
bridge-2:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-2:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-2: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-3:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-3:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-3: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-4:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-4:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-4: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-5:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-5:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-5: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-6:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-6:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-6: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-7:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-7:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-7: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-8:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-8:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-8: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-9:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-9:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-9: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
bridge-10:
|
||||
<<: *cluster-bridge
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server-10:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
depends_on:
|
||||
server-10: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# TLS overlay — enables mTLS across the turnstone cluster.
|
||||
#
|
||||
# Usage (requires base compose.yaml with production profile):
|
||||
# docker compose -f compose.yaml -f deploy/docker-compose.tls.yml --profile production up
|
||||
#
|
||||
# The tls-init service bootstraps a CA and issues certs.
|
||||
# All turnstone services auto-provision their own certs via the
|
||||
# console's ACME endpoint.
|
||||
|
||||
services:
|
||||
# Bootstrap: create CA before anything starts.
|
||||
# Runs as root to create directories in the volume, then chowns
|
||||
# to turnstone:turnstone with restrictive perms (keys 0600).
|
||||
tls-init:
|
||||
build: .
|
||||
user: root
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
turnstone-admin tls-bootstrap --out /certs
|
||||
chown -R turnstone:turnstone /certs
|
||||
find /certs -type d -exec chmod 750 {} +
|
||||
find /certs -type f -name '*key.pem' -exec chmod 600 {} +
|
||||
find /certs -type f ! -name '*key.pem' -exec chmod 640 {} +
|
||||
volumes:
|
||||
- tls-certs:/certs
|
||||
networks:
|
||||
- turnstone-net
|
||||
restart: "no"
|
||||
|
||||
# Console: runs the internal CA + ACME server
|
||||
console:
|
||||
depends_on:
|
||||
tls-init:
|
||||
condition: service_completed_successfully
|
||||
volumes:
|
||||
- tls-certs:/certs:ro
|
||||
environment:
|
||||
TURNSTONE_TLS_ENABLED: "true"
|
||||
TURNSTONE_TLS_SANS: "console"
|
||||
TURNSTONE_CONSOLE_URL: "http://console:8090"
|
||||
command:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port=8090
|
||||
- --poll-interval=${CONSOLE_POLL_INTERVAL:-10}
|
||||
|
||||
# Server: auto-provisions certs via console ACME, serves HTTPS
|
||||
server:
|
||||
depends_on:
|
||||
console:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- tls-certs:/certs:ro
|
||||
environment:
|
||||
TURNSTONE_TLS_ENABLED: "true"
|
||||
TURNSTONE_TLS_SANS: "server"
|
||||
# Disable healthcheck — server serves HTTPS with mTLS which the
|
||||
# stdlib healthcheck script can't satisfy. The base compose
|
||||
# healthcheck uses plain HTTP which won't work on an HTTPS listener.
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
# Channel: TLS
|
||||
channel:
|
||||
depends_on:
|
||||
console:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- tls-certs:/certs:ro
|
||||
environment:
|
||||
TURNSTONE_TLS_ENABLED: "true"
|
||||
TURNSTONE_TLS_SANS: "channel"
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-channel
|
||||
--http-host=0.0.0.0
|
||||
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
|
||||
|
||||
volumes:
|
||||
tls-certs:
|
||||
@@ -10,7 +10,3 @@ dependencies:
|
||||
version: ~18.5.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
- name: redis
|
||||
version: ~25.3.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: redis.enabled
|
||||
|
||||
@@ -25,14 +25,10 @@ Then open: http://localhost:{{ .Values.console.service.port }}
|
||||
|
||||
Components deployed:
|
||||
- Server: {{ include "turnstone.fullname" . }}-server ({{ .Values.server.replicas }} replica(s))
|
||||
- Bridge: {{ include "turnstone.fullname" . }}-bridge ({{ .Values.bridge.replicas }} replica(s))
|
||||
- Console: {{ include "turnstone.fullname" . }}-console ({{ .Values.console.replicas }} replica(s))
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
- PostgreSQL (bitnami subchart)
|
||||
{{- end }}
|
||||
{{- if .Values.redis.enabled }}
|
||||
- Redis (bitnami subchart)
|
||||
{{- end }}
|
||||
|
||||
{{- if not .Values.llm.apiKey }}
|
||||
{{- if not .Values.llm.existingSecret }}
|
||||
|
||||
@@ -110,28 +110,6 @@ Determine the PostgreSQL username.
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the Redis host.
|
||||
*/}}
|
||||
{{- define "turnstone.redis.host" -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- printf "%s-redis-master" .Release.Name }}
|
||||
{{- else }}
|
||||
{{- .Values.redis.external.host }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the Redis port.
|
||||
*/}}
|
||||
{{- define "turnstone.redis.port" -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- printf "6379" }}
|
||||
{{- else }}
|
||||
{{- .Values.redis.external.port | toString }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the secret name for LLM API keys.
|
||||
*/}}
|
||||
|
||||
@@ -14,8 +14,6 @@ data:
|
||||
TURNSTONE_SERVER_PORT: {{ .Values.server.service.port | quote }}
|
||||
TURNSTONE_CONSOLE_HOST: "0.0.0.0"
|
||||
TURNSTONE_CONSOLE_PORT: {{ .Values.console.service.port | quote }}
|
||||
TURNSTONE_REDIS_HOST: {{ include "turnstone.redis.host" . | quote }}
|
||||
TURNSTONE_REDIS_PORT: {{ include "turnstone.redis.port" . | quote }}
|
||||
TURNSTONE_POLL_INTERVAL: "5"
|
||||
{{- if .Values.llm.baseUrl }}
|
||||
TURNSTONE_LLM_BASE_URL: {{ .Values.llm.baseUrl | quote }}
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-bridge
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
spec:
|
||||
replicas: {{ .Values.bridge.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
spec:
|
||||
serviceAccountName: {{ include "turnstone.serviceAccountName" . }}
|
||||
containers:
|
||||
- name: bridge
|
||||
image: {{ include "turnstone.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url={{ printf "http://%s-server:%s" (include "turnstone.fullname" .) (.Values.server.service.port | toString) }}
|
||||
- --redis-host={{ include "turnstone.redis.host" . }}
|
||||
- --redis-port={{ include "turnstone.redis.port" . }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
env:
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.bridge.resources | nindent 12 }}
|
||||
@@ -26,8 +26,6 @@ spec:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port={{ .Values.console.service.port }}
|
||||
- --redis-host={{ include "turnstone.redis.host" . }}
|
||||
- --redis-port={{ include "turnstone.redis.port" . }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.console.service.port }}
|
||||
@@ -38,13 +36,13 @@ spec:
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
{{- if or .Values.auth.existingSecret .Values.auth.jwtSecret }}
|
||||
env:
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
- name: TURNSTONE_JWT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
name: {{ include "turnstone.auth.secretName" . }}
|
||||
key: TURNSTONE_JWT_SECRET
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
|
||||
@@ -41,12 +41,12 @@ spec:
|
||||
env:
|
||||
- name: TURNSTONE_DB_URL
|
||||
value: "postgresql+psycopg://$(TURNSTONE_DB_USER):$(POSTGRES_PASSWORD)@$(TURNSTONE_DB_HOST):$(TURNSTONE_DB_PORT)/$(TURNSTONE_DB_NAME)"
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
{{- if or .Values.auth.existingSecret .Values.auth.jwtSecret }}
|
||||
- name: TURNSTONE_JWT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
name: {{ include "turnstone.auth.secretName" . }}
|
||||
key: TURNSTONE_JWT_SECRET
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
|
||||
@@ -15,14 +15,7 @@ data:
|
||||
{{- else if and (not .Values.postgresql.enabled) .Values.database.external.password }}
|
||||
POSTGRES_PASSWORD: {{ .Values.database.external.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.auth.enabled .Values.auth.token (not .Values.auth.existingSecret) }}
|
||||
TURNSTONE_AUTH_TOKEN: {{ .Values.auth.token | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.redis.enabled .Values.redis.auth }}
|
||||
{{- if .Values.redis.auth.password }}
|
||||
REDIS_PASSWORD: {{ .Values.redis.auth.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- else if and (not .Values.redis.enabled) .Values.redis.external.password }}
|
||||
REDIS_PASSWORD: {{ .Values.redis.external.password | b64enc | quote }}
|
||||
{{- if and .Values.auth.jwtSecret (not .Values.auth.existingSecret) }}
|
||||
TURNSTONE_JWT_SECRET: {{ .Values.auth.jwtSecret | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -24,16 +24,6 @@ postgresql:
|
||||
database: turnstone
|
||||
username: turnstone
|
||||
|
||||
# -- Redis configuration
|
||||
redis:
|
||||
enabled: true
|
||||
architecture: standalone
|
||||
# External Redis settings (used when redis.enabled is false)
|
||||
external:
|
||||
host: ""
|
||||
port: 6379
|
||||
existingSecret: ""
|
||||
|
||||
# -- Turnstone server (main API + web UI)
|
||||
server:
|
||||
replicas: 1
|
||||
@@ -48,17 +38,6 @@ server:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
# -- Turnstone bridge (Redis MQ connector)
|
||||
bridge:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
|
||||
# -- Turnstone console (cluster dashboard)
|
||||
console:
|
||||
replicas: 1
|
||||
@@ -80,10 +59,9 @@ llm:
|
||||
apiKey: ""
|
||||
existingSecret: ""
|
||||
|
||||
# -- Authentication
|
||||
# -- Authentication (always enabled, JWT secret required)
|
||||
auth:
|
||||
enabled: false
|
||||
token: ""
|
||||
jwtSecret: ""
|
||||
existingSecret: ""
|
||||
|
||||
# -- Ingress configuration
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# OpenShell inference routing for Turnstone.
|
||||
#
|
||||
# When using inference routing, the sandbox process connects to
|
||||
# https://inference.local instead of the real LLM API. The OpenShell
|
||||
# proxy intercepts, rewrites credentials, and forwards to the backend.
|
||||
#
|
||||
# This keeps real API keys out of the sandbox entirely — the process
|
||||
# only sees opaque placeholder tokens in its environment.
|
||||
#
|
||||
# Usage:
|
||||
# openshell sandbox run \
|
||||
# --inference-routes deploy/openshell/routes.yaml \
|
||||
# ...
|
||||
#
|
||||
# Then start turnstone with:
|
||||
# python3 -m turnstone.server --base-url https://inference.local
|
||||
#
|
||||
# CUSTOMIZE: uncomment one of the provider blocks below.
|
||||
|
||||
routes:
|
||||
|
||||
# --- OpenAI ---
|
||||
# - name: inference.local
|
||||
# endpoint: https://api.openai.com/v1
|
||||
# model: gpt-5
|
||||
# provider_type: openai
|
||||
# protocols:
|
||||
# - openai_chat_completions
|
||||
# - model_discovery
|
||||
# api_key_env: OPENAI_API_KEY
|
||||
|
||||
# --- Anthropic ---
|
||||
# - name: inference.local
|
||||
# endpoint: https://api.anthropic.com
|
||||
# model: claude-sonnet-4-6
|
||||
# provider_type: anthropic
|
||||
# protocols:
|
||||
# - anthropic_messages
|
||||
# api_key_env: ANTHROPIC_API_KEY
|
||||
|
||||
# --- Local model server (vLLM / llama.cpp) ---
|
||||
# No secret resolution needed — local servers typically have no auth.
|
||||
# Omit both api_key and api_key_env to skip credential injection.
|
||||
# - name: inference.local
|
||||
# endpoint: http://localhost:8000/v1
|
||||
# model: meta-llama/Llama-3.1-70B-Instruct
|
||||
# protocols:
|
||||
# - openai_chat_completions
|
||||
# - model_discovery
|
||||
@@ -0,0 +1,318 @@
|
||||
# OpenShell sandbox policy for Turnstone AI orchestration platform.
|
||||
#
|
||||
# This policy wraps a turnstone-server process (the primary sandbox target).
|
||||
# The console and channel gateway are separate processes that would each
|
||||
# need their own sandbox with a tailored policy variant.
|
||||
#
|
||||
# Usage:
|
||||
# openshell sandbox run \
|
||||
# --policy deploy/openshell/turnstone-policy.yaml \
|
||||
# --workdir /project \
|
||||
# -- python3 -m turnstone.server --host 0.0.0.0 --port 8080
|
||||
#
|
||||
# For inference routing (keeps real API keys out of the sandbox):
|
||||
# openshell sandbox run \
|
||||
# --policy deploy/openshell/turnstone-policy.yaml \
|
||||
# --inference-routes deploy/openshell/routes.yaml \
|
||||
# --workdir /project \
|
||||
# -- python3 -m turnstone.server --host 0.0.0.0 --port 8080 \
|
||||
# --base-url https://inference.local
|
||||
#
|
||||
# Note: inference.local is intercepted by the OpenShell proxy before
|
||||
# network policy evaluation — no network_policies entry is needed for it.
|
||||
#
|
||||
# Customization points (search for "CUSTOMIZE"):
|
||||
# - OIDC issuer endpoint
|
||||
# - MCP HTTP server endpoints
|
||||
# - Additional tool binaries
|
||||
# - web_fetch domain allowlist
|
||||
|
||||
version: 1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Filesystem: Landlock kernel enforcement
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static — cannot be changed after sandbox creation.
|
||||
# include_workdir adds the --workdir path to read_write automatically.
|
||||
|
||||
filesystem_policy:
|
||||
include_workdir: true
|
||||
|
||||
read_only:
|
||||
# Python runtime + installed packages (includes turnstone package)
|
||||
- /usr
|
||||
- /lib
|
||||
- /lib64
|
||||
# System essentials
|
||||
- /etc
|
||||
- /proc
|
||||
- /dev/urandom
|
||||
# Turnstone config (read-only — writes go to database)
|
||||
# CUSTOMIZE: adjust if config lives elsewhere
|
||||
- /home/sandbox/.config/turnstone
|
||||
|
||||
read_write:
|
||||
# Working directory is added via include_workdir
|
||||
# Temp files (bash tool scripts, eval workdirs)
|
||||
- /tmp
|
||||
# Shell redirections (2>/dev/null)
|
||||
- /dev/null
|
||||
# SQLite database (default location is workdir, covered by include_workdir)
|
||||
# Logs
|
||||
- /var/log
|
||||
|
||||
landlock:
|
||||
# best_effort: degrade gracefully on kernels without Landlock (< 5.13)
|
||||
# Change to hard_requirement for production hardened deployments
|
||||
compatibility: best_effort
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Process: privilege separation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
process:
|
||||
run_as_user: sandbox
|
||||
run_as_group: sandbox
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Network: per-endpoint, per-binary allowlisting
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default-deny. Only listed host:port pairs are reachable.
|
||||
# Child processes (MCP servers, bash subcommands) inherit the network
|
||||
# namespace — they cannot bypass the proxy.
|
||||
|
||||
network_policies:
|
||||
|
||||
# --- LLM API providers ---
|
||||
|
||||
openai_api:
|
||||
name: openai-api
|
||||
endpoints:
|
||||
- host: api.openai.com
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
anthropic_api:
|
||||
name: anthropic-api
|
||||
endpoints:
|
||||
- host: api.anthropic.com
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- Web search fallback (Tavily) ---
|
||||
|
||||
tavily_api:
|
||||
name: tavily-search
|
||||
endpoints:
|
||||
- host: api.tavily.com
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- Skill discovery ---
|
||||
|
||||
skills_registry:
|
||||
name: skills-registry
|
||||
endpoints:
|
||||
- host: skills.sh
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
github_api:
|
||||
name: github-api
|
||||
endpoints:
|
||||
- host: api.github.com
|
||||
port: 443
|
||||
protocol: rest
|
||||
tls: terminate
|
||||
enforcement: enforce
|
||||
access: read-only
|
||||
- host: raw.githubusercontent.com
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
mcp_registry:
|
||||
name: mcp-registry
|
||||
endpoints:
|
||||
- host: registry.modelcontextprotocol.io
|
||||
port: 443
|
||||
protocol: rest
|
||||
tls: terminate
|
||||
enforcement: enforce
|
||||
access: read-only
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- OIDC SSO ---
|
||||
# CUSTOMIZE: replace with your identity provider's hostname
|
||||
|
||||
# oidc_provider:
|
||||
# name: oidc-provider
|
||||
# endpoints:
|
||||
# - host: login.example.com
|
||||
# port: 443
|
||||
# binaries:
|
||||
# - path: /usr/bin/python3*
|
||||
# - path: /usr/local/bin/python3*
|
||||
|
||||
# --- Discord (channel integration) ---
|
||||
# Uncomment if using turnstone-channel with Discord adapter.
|
||||
|
||||
# discord:
|
||||
# name: discord
|
||||
# endpoints:
|
||||
# - host: discord.com
|
||||
# port: 443
|
||||
# - host: gateway.discord.gg
|
||||
# port: 443
|
||||
# - host: cdn.discordapp.com
|
||||
# port: 443
|
||||
# binaries:
|
||||
# - path: /usr/bin/python3*
|
||||
# - path: /usr/local/bin/python3*
|
||||
|
||||
# --- web_fetch tool: curated domain allowlist ---
|
||||
#
|
||||
# This is the hard tradeoff. Turnstone's web_fetch tool lets the LLM
|
||||
# fetch arbitrary public URLs. OpenShell cannot allow "all HTTPS" —
|
||||
# every domain must be enumerated.
|
||||
#
|
||||
# Strategy: allowlist the domains your workloads actually need.
|
||||
# The web_fetch tool will return a connection error for unlisted domains,
|
||||
# which the LLM handles gracefully (it tells the user it can't reach
|
||||
# that site).
|
||||
#
|
||||
# CUSTOMIZE: add domains your workstreams need to fetch from.
|
||||
|
||||
web_fetch_common:
|
||||
name: web-fetch-common
|
||||
endpoints:
|
||||
# Documentation sites
|
||||
- host: "**.readthedocs.io"
|
||||
port: 443
|
||||
- host: docs.python.org
|
||||
port: 443
|
||||
- host: "**.github.io"
|
||||
port: 443
|
||||
# Package registries (metadata lookups)
|
||||
- host: pypi.org
|
||||
port: 443
|
||||
- host: www.npmjs.com
|
||||
port: 443
|
||||
# Stack Overflow / reference
|
||||
- host: stackoverflow.com
|
||||
port: 443
|
||||
- host: "**.stackexchange.com"
|
||||
port: 443
|
||||
# Wikipedia
|
||||
- host: "**.wikipedia.org"
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- MCP HTTP servers ---
|
||||
# CUSTOMIZE: add endpoints for any MCP servers using streamable-http
|
||||
# transport. stdio-transport MCP servers need no network entry (they
|
||||
# communicate via stdin/stdout pipes within the sandbox).
|
||||
|
||||
# mcp_http_servers:
|
||||
# name: mcp-http
|
||||
# endpoints:
|
||||
# - host: mcp.internal.example.com
|
||||
# port: 443
|
||||
# binaries:
|
||||
# - path: /usr/bin/python3*
|
||||
# - path: /usr/local/bin/python3*
|
||||
|
||||
# --- Bash tool: curl/wget ---
|
||||
# The bash tool can run curl/wget. These inherit the network namespace
|
||||
# so they can only reach allowed endpoints. But they need binary entries
|
||||
# to pass the proxy's identity check.
|
||||
|
||||
bash_network_tools:
|
||||
name: bash-network-tools
|
||||
endpoints:
|
||||
# Mirrors web_fetch_common — curl/wget should have the same reach.
|
||||
- host: "**.readthedocs.io"
|
||||
port: 443
|
||||
- host: docs.python.org
|
||||
port: 443
|
||||
- host: "**.github.io"
|
||||
port: 443
|
||||
- host: pypi.org
|
||||
port: 443
|
||||
- host: www.npmjs.com
|
||||
port: 443
|
||||
- host: stackoverflow.com
|
||||
port: 443
|
||||
- host: "**.stackexchange.com"
|
||||
port: 443
|
||||
- host: "**.wikipedia.org"
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/curl
|
||||
- path: /usr/bin/wget
|
||||
|
||||
# --- Package installation ---
|
||||
# pip install / uv add from the bash tool.
|
||||
|
||||
package_registries:
|
||||
name: package-install
|
||||
endpoints:
|
||||
- host: pypi.org
|
||||
port: 443
|
||||
- host: files.pythonhosted.org
|
||||
port: 443
|
||||
- host: "**.pypi.org"
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/pip*
|
||||
- path: /usr/local/bin/pip*
|
||||
- path: /usr/bin/uv
|
||||
- path: /usr/local/bin/uv
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- Git operations ---
|
||||
# read-only: clone, fetch, pull. No push (L7 enforcement).
|
||||
|
||||
git_operations:
|
||||
name: git-read-only
|
||||
endpoints:
|
||||
- host: github.com
|
||||
port: 443
|
||||
protocol: rest
|
||||
tls: terminate
|
||||
enforcement: enforce
|
||||
rules:
|
||||
- allow:
|
||||
method: GET
|
||||
path: "/**/info/refs*"
|
||||
- allow:
|
||||
method: POST
|
||||
path: "/**/git-upload-pack"
|
||||
- host: gitlab.com
|
||||
port: 443
|
||||
protocol: rest
|
||||
tls: terminate
|
||||
enforcement: enforce
|
||||
rules:
|
||||
- allow:
|
||||
method: GET
|
||||
path: "/**/info/refs*"
|
||||
- allow:
|
||||
method: POST
|
||||
path: "/**/git-upload-pack"
|
||||
binaries:
|
||||
- path: /usr/bin/git
|
||||
@@ -22,8 +22,3 @@ output "rds_endpoint" {
|
||||
description = "RDS PostgreSQL endpoint."
|
||||
value = module.turnstone.rds_endpoint
|
||||
}
|
||||
|
||||
output "redis_endpoint" {
|
||||
description = "ElastiCache Redis endpoint."
|
||||
value = module.turnstone.redis_endpoint
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ variable "vpc_id" {
|
||||
}
|
||||
|
||||
variable "private_subnet_ids" {
|
||||
description = "List of private subnet IDs for ECS tasks, RDS, and ElastiCache."
|
||||
description = "List of private subnet IDs for ECS tasks and RDS."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# ---------- ElastiCache Subnet Group ----------
|
||||
|
||||
resource "aws_elasticache_subnet_group" "this" {
|
||||
name = "${var.name_prefix}-${var.environment}"
|
||||
subnet_ids = var.private_subnet_ids
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- ElastiCache Redis Replication Group ----------
|
||||
|
||||
resource "aws_elasticache_replication_group" "this" {
|
||||
replication_group_id = "${var.name_prefix}-${var.environment}"
|
||||
description = "Turnstone Redis for MQ and session state"
|
||||
|
||||
engine = "redis"
|
||||
engine_version = "7.1"
|
||||
node_type = var.redis_node_type
|
||||
num_cache_clusters = 1
|
||||
port = 6379
|
||||
|
||||
subnet_group_name = aws_elasticache_subnet_group.this.name
|
||||
security_group_ids = [aws_security_group.redis.id]
|
||||
|
||||
at_rest_encryption_enabled = true
|
||||
transit_encryption_enabled = true
|
||||
|
||||
automatic_failover_enabled = false
|
||||
|
||||
tags = local.common_tags
|
||||
}
|
||||
@@ -40,8 +40,8 @@ resource "aws_iam_role_policy" "ecs_execution_secrets" {
|
||||
[
|
||||
aws_secretsmanager_secret.openai_api_key.arn,
|
||||
aws_secretsmanager_secret.db_password.arn,
|
||||
aws_secretsmanager_secret.jwt_secret.arn,
|
||||
],
|
||||
var.auth_token != "" ? [aws_secretsmanager_secret.auth_token[0].arn] : [],
|
||||
)
|
||||
},
|
||||
]
|
||||
|
||||
@@ -27,7 +27,6 @@ locals {
|
||||
{ name = "TURNSTONE_ENV", value = var.environment },
|
||||
{ name = "TURNSTONE_DB_BACKEND", value = "postgresql" },
|
||||
{ name = "TURNSTONE_LLM_BASE_URL", value = var.llm_base_url },
|
||||
{ name = "TURNSTONE_REDIS_URL", value = "redis://${aws_elasticache_replication_group.this.primary_endpoint_address}:6379/0" },
|
||||
]
|
||||
|
||||
# Secrets pulled from Secrets Manager at container start.
|
||||
@@ -42,20 +41,26 @@ locals {
|
||||
},
|
||||
]
|
||||
|
||||
auth_env = var.auth_token != "" ? [
|
||||
{ name = "TURNSTONE_AUTH_ENABLED", value = "true" },
|
||||
] : []
|
||||
|
||||
auth_secrets = var.auth_token != "" ? [
|
||||
auth_secrets = [
|
||||
{
|
||||
name = "TURNSTONE_AUTH_TOKEN"
|
||||
valueFrom = aws_secretsmanager_secret_version.auth_token[0].arn
|
||||
name = "TURNSTONE_JWT_SECRET"
|
||||
valueFrom = aws_secretsmanager_secret_version.jwt_secret.arn
|
||||
},
|
||||
] : []
|
||||
]
|
||||
}
|
||||
|
||||
# ---------- Secrets Manager ----------
|
||||
|
||||
resource "aws_secretsmanager_secret" "jwt_secret" {
|
||||
name = "${var.name_prefix}-${var.environment}-jwt-secret"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "jwt_secret" {
|
||||
secret_id = aws_secretsmanager_secret.jwt_secret.id
|
||||
secret_string = var.jwt_secret
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "openai_api_key" {
|
||||
name = "${var.name_prefix}-${var.environment}-openai-api-key"
|
||||
tags = local.common_tags
|
||||
@@ -66,17 +71,7 @@ resource "aws_secretsmanager_secret_version" "openai_api_key" {
|
||||
secret_string = var.openai_api_key
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "auth_token" {
|
||||
count = var.auth_token != "" ? 1 : 0
|
||||
name = "${var.name_prefix}-${var.environment}-auth-token"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "auth_token" {
|
||||
count = var.auth_token != "" ? 1 : 0
|
||||
secret_id = aws_secretsmanager_secret.auth_token[0].id
|
||||
secret_string = var.auth_token
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "db_password" {
|
||||
name = "${var.name_prefix}-${var.environment}-db-password"
|
||||
@@ -141,7 +136,7 @@ resource "aws_ecs_task_definition" "server" {
|
||||
{ containerPort = 8080, protocol = "tcp" },
|
||||
]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
environment = local.common_env
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
@@ -187,57 +182,6 @@ resource "aws_ecs_service" "server" {
|
||||
depends_on = [aws_lb_target_group.server]
|
||||
}
|
||||
|
||||
# ---------- Bridge Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "bridge" {
|
||||
family = "${var.name_prefix}-bridge"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = var.bridge_cpu
|
||||
memory = var.bridge_memory
|
||||
execution_role_arn = aws_iam_role.ecs_execution.arn
|
||||
task_role_arn = aws_iam_role.ecs_task.arn
|
||||
tags = local.common_tags
|
||||
|
||||
container_definitions = jsonencode([
|
||||
{
|
||||
name = "bridge"
|
||||
image = local.full_image
|
||||
essential = true
|
||||
command = ["turnstone-bridge"]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
"awslogs-group" = aws_cloudwatch_log_group.this.name
|
||||
"awslogs-region" = data.aws_region.current.name
|
||||
"awslogs-stream-prefix" = "bridge"
|
||||
}
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
resource "aws_ecs_service" "bridge" {
|
||||
name = "${var.name_prefix}-bridge"
|
||||
cluster = aws_ecs_cluster.this.id
|
||||
task_definition = aws_ecs_task_definition.bridge.arn
|
||||
desired_count = 1
|
||||
launch_type = "FARGATE"
|
||||
tags = local.common_tags
|
||||
|
||||
network_configuration {
|
||||
subnets = var.private_subnet_ids
|
||||
security_groups = [aws_security_group.ecs_tasks.id]
|
||||
assign_public_ip = false
|
||||
}
|
||||
|
||||
depends_on = [aws_ecs_service.server]
|
||||
}
|
||||
|
||||
# ---------- Console Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "console" {
|
||||
@@ -261,7 +205,7 @@ resource "aws_ecs_task_definition" "console" {
|
||||
{ containerPort = 8090, protocol = "tcp" },
|
||||
]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
environment = local.common_env
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
|
||||
@@ -22,8 +22,3 @@ output "rds_endpoint" {
|
||||
description = "Endpoint of the RDS PostgreSQL instance (host:port)."
|
||||
value = aws_db_instance.this.endpoint
|
||||
}
|
||||
|
||||
output "redis_endpoint" {
|
||||
description = "Primary endpoint of the ElastiCache Redis replication group."
|
||||
value = aws_elasticache_replication_group.this.primary_endpoint_address
|
||||
}
|
||||
|
||||
@@ -112,22 +112,3 @@ resource "aws_vpc_security_group_ingress_rule" "rds_from_ecs" {
|
||||
referenced_security_group_id = aws_security_group.ecs_tasks.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- Redis Security Group ----------
|
||||
|
||||
resource "aws_security_group" "redis" {
|
||||
name = "${var.name_prefix}-redis-${var.environment}"
|
||||
description = "Allow Redis access from ECS tasks"
|
||||
vpc_id = var.vpc_id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "redis_from_ecs" {
|
||||
security_group_id = aws_security_group.redis.id
|
||||
description = "Redis from ECS tasks"
|
||||
from_port = 6379
|
||||
to_port = 6379
|
||||
ip_protocol = "tcp"
|
||||
referenced_security_group_id = aws_security_group.ecs_tasks.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ variable "vpc_id" {
|
||||
}
|
||||
|
||||
variable "private_subnet_ids" {
|
||||
description = "List of private subnet IDs for ECS tasks, RDS, and ElastiCache."
|
||||
description = "List of private subnet IDs for ECS tasks and RDS."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
@@ -50,14 +50,6 @@ variable "db_instance_class" {
|
||||
default = "db.t4g.micro"
|
||||
}
|
||||
|
||||
# --- ElastiCache ---
|
||||
|
||||
variable "redis_node_type" {
|
||||
description = "ElastiCache node type for Redis."
|
||||
type = string
|
||||
default = "cache.t4g.micro"
|
||||
}
|
||||
|
||||
# --- ECS Task Sizing ---
|
||||
|
||||
variable "server_cpu" {
|
||||
@@ -72,18 +64,6 @@ variable "server_memory" {
|
||||
default = 1024
|
||||
}
|
||||
|
||||
variable "bridge_cpu" {
|
||||
description = "CPU units for the bridge task."
|
||||
type = number
|
||||
default = 256
|
||||
}
|
||||
|
||||
variable "bridge_memory" {
|
||||
description = "Memory (MiB) for the bridge task."
|
||||
type = number
|
||||
default = 512
|
||||
}
|
||||
|
||||
variable "console_cpu" {
|
||||
description = "CPU units for the console task."
|
||||
type = number
|
||||
@@ -110,11 +90,10 @@ variable "name_prefix" {
|
||||
default = "turnstone"
|
||||
}
|
||||
|
||||
variable "auth_token" {
|
||||
description = "Optional authentication token for the Turnstone API. Empty string disables auth."
|
||||
variable "jwt_secret" {
|
||||
description = "JWT signing secret for Turnstone auth (required, min 32 characters)."
|
||||
type = string
|
||||
sensitive = true
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "certificate_arn" {
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"ddg": {
|
||||
"url": "http://ddg-search:3000/mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
+321
-70
@@ -2,8 +2,6 @@
|
||||
|
||||
## Overview
|
||||
|
||||
> See also: [MQ Protocol diagram](diagrams/png/06-mq-protocol.png) | [Message Routing diagram](diagrams/png/07-message-routing.png) | [Redis Key Schema diagram](diagrams/png/08-redis-key-schema.png)
|
||||
|
||||
`turnstone-server` exposes a browser-based chat UI backed by a
|
||||
**Starlette** ASGI application served by **uvicorn**. The server uses
|
||||
**Server-Sent Events (SSE)** via `sse-starlette` for real-time streaming
|
||||
@@ -58,7 +56,7 @@ console.log(result.content);
|
||||
|
||||
## Authentication
|
||||
|
||||
When auth is enabled (`[auth].enabled = true` or `TURNSTONE_AUTH_ENABLED=1`), all API endpoints except public paths require a valid token.
|
||||
Auth is always enabled. All API endpoints except public paths require a valid token.
|
||||
|
||||
### Sending Credentials
|
||||
|
||||
@@ -67,15 +65,14 @@ Include a token in one of two ways:
|
||||
- **Bearer header**: `Authorization: Bearer <token>`
|
||||
- **Cookie**: `turnstone_auth=<token>` (set automatically by the login endpoint)
|
||||
|
||||
The server accepts three token types:
|
||||
The server accepts two token types:
|
||||
|
||||
| Type | Format | Example |
|
||||
|------|--------|---------|
|
||||
| JWT | Base64 segments separated by dots | `eyJhbG...` |
|
||||
| API token | `ts_` prefix + 64 hex chars | `ts_a1b2c3d4...` |
|
||||
| Config token | Arbitrary string from `config.toml` | `my-secret-token` |
|
||||
|
||||
JWTs are the recommended credential for browser sessions. API tokens are suitable for programmatic access and CI/CD. Config tokens are a simple option for single-node deployments.
|
||||
JWTs are the recommended credential for browser sessions. API tokens are suitable for programmatic access and CI/CD.
|
||||
|
||||
### `POST /v1/api/auth/login`
|
||||
|
||||
@@ -386,10 +383,10 @@ Each item in `items` (shared by `tool_info` and `approve_request`):
|
||||
{"type": "tool_output_chunk", "call_id": "call_abc123", "chunk": "Building project...\n"}
|
||||
```
|
||||
|
||||
**`tool_result`** -- final output from a completed tool execution. The `call_id` matches the corresponding `tool_info`/`approve_request` item and any preceding `tool_output_chunk` events. For bash tools, this arrives after all streaming chunks and includes both stdout and stderr.
|
||||
**`tool_result`** -- final output from a completed tool execution. The `call_id` matches the corresponding `tool_info`/`approve_request` item and any preceding `tool_output_chunk` events. For bash tools, this arrives after all streaming chunks and includes both stdout and stderr. The `is_error` field is `true` when the tool execution failed (e.g. bash exit code >= 2 or signal, file not found, timeout). Exit code 1 is ambiguous (e.g. `grep` no-match) and is not flagged. User denials are tracked separately via a `denied` flag. Clients should use `is_error` instead of text-prefix heuristics.
|
||||
|
||||
```json
|
||||
{"type": "tool_result", "call_id": "call_abc123", "name": "bash", "output": "file1.py\nfile2.py\n"}
|
||||
{"type": "tool_result", "call_id": "call_abc123", "name": "bash", "output": "file1.py\nfile2.py\n", "is_error": false}
|
||||
```
|
||||
|
||||
**`status`** -- token usage statistics, sent after each model turn.
|
||||
@@ -402,18 +399,22 @@ Each item in `items` (shared by `tool_info` and `approve_request`):
|
||||
"total_tokens": 1280,
|
||||
"context_window": 131072,
|
||||
"pct": 1.0,
|
||||
"effort": "medium"
|
||||
"effort": "medium",
|
||||
"cache_creation_tokens": 800,
|
||||
"cache_read_tokens": 200
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------------------|--------|----------------------------------------------|
|
||||
| `prompt_tokens` | int | Tokens in the prompt |
|
||||
| `completion_tokens` | int | Tokens generated by the model |
|
||||
| `total_tokens` | int | `prompt_tokens + completion_tokens` |
|
||||
| `context_window` | int | Total context window size in tokens |
|
||||
| `pct` | float | Percentage of context window used |
|
||||
| `effort` | string | Reasoning effort level (`low`/`medium`/`high`) |
|
||||
| Field | Type | Description |
|
||||
|--------------------------|--------|------------------------------------------------------|
|
||||
| `prompt_tokens` | int | Tokens in the prompt |
|
||||
| `completion_tokens` | int | Tokens generated by the model |
|
||||
| `total_tokens` | int | `prompt_tokens + completion_tokens` |
|
||||
| `context_window` | int | Total context window size in tokens |
|
||||
| `pct` | float | Percentage of context window used |
|
||||
| `effort` | string | Reasoning effort level (`low`/`medium`/`high`) |
|
||||
| `cache_creation_tokens` | int | Tokens written to prompt cache (Anthropic) |
|
||||
| `cache_read_tokens` | int | Tokens served from prompt cache (Anthropic + OpenAI) |
|
||||
|
||||
**`plan_review`** -- the model is proposing a plan and wants feedback. The
|
||||
client must respond via `POST /v1/api/plan`.
|
||||
@@ -448,9 +449,12 @@ after `/clear` or `/new` commands).
|
||||
{"type": "clear_ui"}
|
||||
```
|
||||
|
||||
**`cancelled`** -- the generation was cancelled by the user (via the Stop
|
||||
button or `POST /v1/api/cancel`). The client should finalize any in-progress
|
||||
assistant message with whatever partial content was streamed.
|
||||
**`cancelled`** -- a cancel request was acknowledged (via the Stop button or
|
||||
`POST /v1/api/cancel`). This signals that cancellation is in progress, not
|
||||
that it is complete. The worker thread may still be finishing — wait for
|
||||
`stream_end` before transitioning to a ready state. The client should clear
|
||||
any in-progress assistant rendering but not re-enable the send button until
|
||||
`stream_end` arrives.
|
||||
|
||||
```json
|
||||
{"type": "cancelled"}
|
||||
@@ -516,7 +520,7 @@ inactivity.
|
||||
|
||||
Each SSE connection to a workstream receives its own delivery queue. Events
|
||||
produced by the worker thread are fanned out to all registered listener queues,
|
||||
so multiple consumers (browser, bridge, console proxy, SDK) can connect
|
||||
so multiple consumers (browser, console proxy, SDK) can connect
|
||||
simultaneously and each receives every event. On reconnect the client receives
|
||||
a full history replay, so no catch-up mechanism is needed.
|
||||
|
||||
@@ -550,7 +554,7 @@ Possible `state` values:
|
||||
| `error` | An error occurred |
|
||||
|
||||
**Fan-out pattern:** Each connected client receives its own bounded queue
|
||||
(`maxsize=500`). A dedicated fan-out thread reads from the shared global queue
|
||||
(`maxsize=1000`). A dedicated fan-out thread reads from the shared global queue
|
||||
and copies each event to every client queue. If a client queue is full, the
|
||||
event is silently dropped for that client.
|
||||
|
||||
@@ -618,64 +622,35 @@ Each saved workstream object:
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/templates`
|
||||
### `GET /v1/api/skills`
|
||||
|
||||
Returns a summary list of all available prompt templates. This is a read-only
|
||||
endpoint (requires `read` scope) that exposes template names and categories
|
||||
without revealing template content. Useful for populating template selectors
|
||||
in UIs or discovering available templates before creating a workstream.
|
||||
Returns a summary list of all available skills. This is a read-only
|
||||
endpoint (requires `read` scope) that exposes skill names and categories
|
||||
without revealing skill content. Useful for populating skill selectors
|
||||
in UIs or discovering available skills before creating a workstream.
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"templates": [
|
||||
"skills": [
|
||||
{"name": "safety-guidelines", "category": "safety", "is_default": true, "origin": "manual"},
|
||||
{"name": "mcp__server__code", "category": "", "is_default": false, "origin": "mcp"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each template summary:
|
||||
Each skill summary:
|
||||
|
||||
| Field | Type | Description |
|
||||
|--------------|--------|------------------------------------------------------|
|
||||
| `name` | string | Template name (used in `template` field on creation) |
|
||||
| `category` | string | Template category |
|
||||
| `is_default` | bool | Whether template is auto-applied to all sessions |
|
||||
| `origin` | string | Template origin: `manual` or `mcp` |
|
||||
| `name` | string | Skill name (used in `skill` field on workstream creation) |
|
||||
| `category` | string | Skill category |
|
||||
| `is_default` | bool | Whether skill is auto-applied to all sessions |
|
||||
| `origin` | string | Skill origin: `manual` or `mcp` |
|
||||
|
||||
> **Note:** For full template management (create, update, delete, view content),
|
||||
> use the admin endpoints at `GET /v1/api/admin/templates` (requires `admin.templates` permission).
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/ws-templates`
|
||||
|
||||
Returns a summary list of enabled workstream templates. This is a read-only
|
||||
endpoint (requires `read` scope) for populating template selectors in UIs.
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"ws_templates": [
|
||||
{"name": "code-review", "description": "Code review profile", "model": "gpt-5"},
|
||||
{"name": "ops-triage", "description": "On-call triage", "model": ""}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each workstream template summary:
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------------|--------|-------------------------------------------------|
|
||||
| `name` | string | Template name (used in `ws_template` on creation)|
|
||||
| `description` | string | Human-readable description |
|
||||
| `model` | string | Model alias override (empty = use default) |
|
||||
|
||||
> **Note:** For full workstream template management, use the admin endpoints at
|
||||
> `GET /v1/api/admin/ws-templates` (requires `admin.templates` permission).
|
||||
> **Note:** For full skill management (create, update, delete, view content),
|
||||
> use the admin endpoints at `GET /v1/api/admin/skills` (requires `admin.skills` permission).
|
||||
|
||||
---
|
||||
|
||||
@@ -818,23 +793,35 @@ containing the resumed session's messages.
|
||||
|
||||
Cancels the active generation in a workstream. Sets a cooperative cancellation
|
||||
flag that is checked at multiple points in the generation loop (per streaming
|
||||
chunk, before tool execution, inside bash commands). The session transitions to
|
||||
`idle` state and preserves any partial content already streamed.
|
||||
chunk, before tool execution, inside bash commands). Also closes the underlying
|
||||
HTTP stream to the LLM provider, unblocking any pending read immediately.
|
||||
The session transitions to `idle` state and preserves any partial content
|
||||
already streamed.
|
||||
|
||||
If the workstream is waiting for tool approval or plan review, the pending
|
||||
prompt is automatically denied/rejected to unblock the worker thread.
|
||||
|
||||
Calling this endpoint when the workstream is already idle is a harmless no-op.
|
||||
|
||||
**Force cancel:** When `force` is `true`, the server abandons the stuck worker
|
||||
thread immediately and transitions the workstream to `idle`. The abandoned
|
||||
thread continues to wind down in the background (killing any running
|
||||
subprocesses and exiting at the next cancellation checkpoint). During this
|
||||
wind-down it may emit a final `stream_end` event which the server suppresses
|
||||
for the orphaned thread. Use force cancel when cooperative cancel has not
|
||||
resolved within a few seconds — the web UI offers this as a "Force Stop"
|
||||
button automatically.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"ws_id": "abc123"}
|
||||
{"ws_id": "abc123", "force": false}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|--------|--------|----------|----------------------|
|
||||
| `ws_id`| string | yes | Target workstream ID |
|
||||
| `force`| bool | no | Abandon stuck worker immediately (default: `false`) |
|
||||
|
||||
**Response:**
|
||||
|
||||
@@ -869,10 +856,9 @@ All fields are optional. The body can be empty or an empty JSON object.
|
||||
| `model` | string | default | Model alias from the registry (`[models.*]`) |
|
||||
| `auto_approve` | bool | false | Auto-approve all tool calls for this workstream |
|
||||
| `resume_ws` | string | "" | Workstream ID to resume atomically during creation (empty = fresh)|
|
||||
| `template` | string | "" | Prompt template name (replaces default templates; 400 if not found)|
|
||||
| `ws_template` | string | "" | Workstream template name. Applies model, temperature, reasoning effort, max tokens, auto-approve policy, and token budget. Returns 400 if not found or disabled. |
|
||||
| `skill` | string | "" | Skill name. Applies content (system prompt), model, temperature, reasoning effort, max tokens, auto-approve policy, token budget, and other session config from the skill. Returns 400 if not found or disabled. Ignored when `resume_ws` is set (resumed sessions restore their own skill). |
|
||||
|
||||
> **Template precedence:** When `ws_template` is specified, its model override takes effect before workstream creation. Both `template` (prompt template) and `ws_template` (workstream template) can be used together — `ws_template` controls the behavioral profile while `template` sets the system message text. If `ws_template` defines its own system prompt or prompt template reference, that takes precedence over the `template` parameter.
|
||||
> **Skill behavior:** When `skill` is specified, the skill's content is injected as a system message and its session config fields (model, temperature, auto-approve, token budget, etc.) override system defaults for the new workstream.
|
||||
|
||||
**Response (success):**
|
||||
|
||||
@@ -1311,6 +1297,139 @@ is on the **console** server and requires the `admin.judge` permission.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/output-assessments` (Console)
|
||||
|
||||
List output guard assessments from the `output_assessments` table. This endpoint
|
||||
is on the **console** server and requires the `admin.judge` permission.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|--------------|--------|----------|----------------------------------------------------|
|
||||
| `ws_id` | string | no | Filter by workstream ID |
|
||||
| `risk_level` | string | no | Filter by risk level (`low`/`medium`/`high`) |
|
||||
| `since` | string | no | ISO timestamp lower bound |
|
||||
| `until` | string | no | ISO timestamp upper bound |
|
||||
| `limit` | int | no | Max results (default 100, max 500) |
|
||||
| `offset` | int | no | Pagination offset (default 0) |
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"assessments": [
|
||||
{
|
||||
"assessment_id": "a1b2c3d4e5f6",
|
||||
"ws_id": "ws-1",
|
||||
"call_id": "call_abc123",
|
||||
"func_name": "bash",
|
||||
"flags": "[\"credential_leak\"]",
|
||||
"risk_level": "high",
|
||||
"annotations": "[\"API key detected (sk-proj-...)\"]",
|
||||
"output_length": 1024,
|
||||
"redacted": 1,
|
||||
"created": "2026-03-16T10:00:00"
|
||||
}
|
||||
],
|
||||
"total": 7
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/admin/skills/{skill_id}/rescan` (Console)
|
||||
|
||||
Re-scan a skill's content for security signals using the current scanner
|
||||
version. Requires the `admin.skills` permission.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|------------|--------|-------------|
|
||||
| `skill_id` | string | Skill (prompt template) ID |
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"scan_status": "medium",
|
||||
"scan_report": "{\"composite\": 1.75, \"details\": {...}}",
|
||||
"scan_version": "1"
|
||||
}
|
||||
```
|
||||
|
||||
**Error:** `404` if skill not found.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/skills/discover` (Console)
|
||||
|
||||
Search external skill registries for available skills. Requires the
|
||||
`admin.skills` permission.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|--------|---------|-------------|
|
||||
| `q` | string | `""` | Search query |
|
||||
| `limit` | int | `20` | Max results (1–100) |
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"skills": [
|
||||
{
|
||||
"id": "owner/repo/skill-name",
|
||||
"name": "skill-name",
|
||||
"description": "A skill description",
|
||||
"author": "Author Name",
|
||||
"source": "skills.sh",
|
||||
"source_url": "https://github.com/owner/repo",
|
||||
"install_count": 42,
|
||||
"tags": ["coding", "review"],
|
||||
"installed": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Error:** `502` if the registry is unreachable.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/admin/skills/install` (Console)
|
||||
|
||||
Install a skill from an external source (skills.sh registry or GitHub).
|
||||
Requires the `admin.skills` permission.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "github",
|
||||
"url": "https://github.com/owner/skill-repo"
|
||||
}
|
||||
```
|
||||
|
||||
Or for skills.sh:
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "skills.sh",
|
||||
"skill_id": "owner/skill-name"
|
||||
}
|
||||
```
|
||||
|
||||
**Response:** Same as `GET /v1/api/admin/skills/{skill_id}` — the created
|
||||
skill object.
|
||||
|
||||
**Errors:** `400` invalid source or missing fields, `404` SKILL.md not found,
|
||||
`409` skill already installed (duplicate source_url or name), `502` source
|
||||
unreachable.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/settings` (Console)
|
||||
|
||||
List all settings with their effective values, defaults, and metadata. Requires
|
||||
@@ -1470,6 +1589,87 @@ Secrets (`env`, `headers` fields) are masked with `***` by default. Use `?reveal
|
||||
|
||||
---
|
||||
|
||||
### MCP Registry
|
||||
|
||||
#### Search Registry
|
||||
|
||||
`GET /v1/api/admin/mcp-registry/search`
|
||||
|
||||
Search the official MCP Registry for available servers. Permission: `admin.mcp`.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `search` | string | `""` | Search query. Empty returns a browsable listing. |
|
||||
| `limit` | integer | `20` | Results per page (max 100). |
|
||||
| `cursor` | string | — | Opaque cursor for pagination. |
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": [
|
||||
{
|
||||
"name": "io.example/mcp-server",
|
||||
"description": "...",
|
||||
"title": "Example Server",
|
||||
"version": "1.0.0",
|
||||
"website_url": "https://example.com",
|
||||
"repository": {"url": "...", "source": "github"},
|
||||
"icons": [],
|
||||
"remotes": [{"type": "streamable-http", "url": "...", "headers": [...], "variables": {...}}],
|
||||
"packages": [{"registry_type": "npm", "identifier": "@example/server", "version": "1.0.0", "transport_type": "stdio", "environment_variables": [...]}],
|
||||
"meta": {"status": "active", "is_latest": true},
|
||||
"installed": false,
|
||||
"installed_server_id": "",
|
||||
"installed_version": "",
|
||||
"update_available": false
|
||||
}
|
||||
],
|
||||
"total": 100,
|
||||
"next_cursor": "abc123"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:** `502` (registry unreachable).
|
||||
|
||||
#### Install from Registry
|
||||
|
||||
`POST /v1/api/admin/mcp-registry/install`
|
||||
|
||||
Install an MCP server from the registry. Auto-reloads all cluster nodes. Permission: `admin.mcp`.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"registry_name": "io.example/mcp-server",
|
||||
"source": "remote",
|
||||
"index": 0,
|
||||
"name": "",
|
||||
"variables": {},
|
||||
"env": {"API_KEY": "sk-..."},
|
||||
"headers": {"Authorization": "Bearer ..."}
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `registry_name` | string | yes | Server name from registry search results. |
|
||||
| `source` | string | yes | `"remote"` (streamable-http) or `"package"` (npm/pypi). |
|
||||
| `index` | integer | no (default `0`) | Which remote or package entry to use. |
|
||||
| `name` | string | no | Custom server name. Auto-derived from registry name if empty. |
|
||||
| `variables` | object | no | Values for URL template `{var}` placeholders. |
|
||||
| `env` | object | no | Environment variable values for package servers. |
|
||||
| `headers` | object | no | Header values for remote servers. |
|
||||
|
||||
**Response:** Same as `POST /v1/api/admin/mcp-servers` (McpServerDetail).
|
||||
|
||||
**Errors:** `400` (validation), `404` (not in registry), `409` (already installed or name collision), `502` (registry unreachable).
|
||||
|
||||
---
|
||||
|
||||
### `OPTIONS` (any path)
|
||||
|
||||
Handles CORS preflight requests.
|
||||
@@ -1656,3 +1856,54 @@ turnstone_tokens_total{type="completion"} 12150
|
||||
turnstone_tool_calls_total{tool="bash"} 7
|
||||
turnstone_tool_calls_total{tool="read_file"} 3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Console Routing Proxy Endpoints
|
||||
|
||||
These endpoints are served by the console (`turnstone-console`) and proxy
|
||||
requests to the correct server node via the hash ring bucket cache. In
|
||||
multi-node deployments, clients (SDK, channel gateway) talk to the console
|
||||
instead of individual server nodes.
|
||||
|
||||
### `POST /v1/api/route/workstreams/new`
|
||||
|
||||
Create a workstream via hash-ring routing. The console generates the `ws_id`,
|
||||
routes to the assigned node, and includes `node_url` in the response for
|
||||
direct SSE connections.
|
||||
|
||||
### `POST /v1/api/route/send`
|
||||
|
||||
Proxy a message to the workstream's assigned server node.
|
||||
|
||||
### `POST /v1/api/route/approve`
|
||||
|
||||
Proxy an approval response to the workstream's assigned server node.
|
||||
|
||||
### `POST /v1/api/route/cancel`
|
||||
|
||||
Cancel generation on a workstream.
|
||||
|
||||
### `POST /v1/api/route/command`
|
||||
|
||||
Send a slash command to a workstream.
|
||||
|
||||
### `POST /v1/api/route/plan`
|
||||
|
||||
Send plan review feedback to a workstream.
|
||||
|
||||
### `POST /v1/api/route/workstreams/close`
|
||||
|
||||
Close a workstream.
|
||||
|
||||
### `GET /v1/api/route?ws_id=X`
|
||||
|
||||
Look up which server node owns a workstream. Returns `{"node_url": "...", "node_id": "..."}`.
|
||||
Used by channel adapters to open direct SSE connections to the correct server node.
|
||||
|
||||
### `GET /metrics` (Console)
|
||||
|
||||
Prometheus metrics for the console routing layer. Includes:
|
||||
`turnstone_router_requests_total`, `turnstone_router_request_duration_seconds`,
|
||||
`turnstone_ring_membership_size`, `turnstone_ring_version`,
|
||||
`turnstone_ring_rebalance_total`, `turnstone_ring_migrations_total`.
|
||||
|
||||
+105
-140
@@ -3,7 +3,7 @@
|
||||
Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent
|
||||
memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or
|
||||
Anthropic's native Messages API via pluggable provider adapters, and gives the
|
||||
model 17 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
model 19 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
reading, writing, searching, planning, and executing code.
|
||||
|
||||
The core design principle is a **UI-agnostic engine with pluggable frontends**.
|
||||
@@ -18,10 +18,9 @@ plugs in.
|
||||
|---------|--------|----------|---------|
|
||||
| `turnstone` | `turnstone.cli` | `TerminalUI` | Interactive terminal REPL |
|
||||
| `turnstone-server` | `turnstone.server` | `WebUI` | Browser-based chat (HTTP + SSE) |
|
||||
| `turnstone-bridge` | `turnstone.mq.bridge` | Bridge | Message queue ↔ HTTP API bridge |
|
||||
| `turnstone-console` | `turnstone.console.server` | ClusterCollector | Cluster dashboard (aggregates all nodes) |
|
||||
| `turnstone-eval` | `turnstone.eval` | `NullUI` | Headless evaluation and prompt optimization |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) via Redis MQ |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) |
|
||||
| `turnstone-admin` | `turnstone.core.admin_cli` | — | Offline user and API token management |
|
||||
|
||||
---
|
||||
@@ -42,7 +41,7 @@ turnstone/
|
||||
__init__.py create_provider() + create_client() factory functions
|
||||
workstream.py Parallel workstream manager (WorkstreamState, Workstream, WorkstreamManager)
|
||||
tools.py Tool schema loader (JSON -> OpenAI function-calling format)
|
||||
mcp_client.py MCPClientManager — MCP server connections, tool discovery, dynamic refresh, async-sync bridge
|
||||
mcp_client.py MCPClientManager — MCP server connections, tool discovery, dynamic refresh
|
||||
tool_search.py Dynamic tool search — BM25 index, session-scoped tool visibility
|
||||
watch.py WatchRunner daemon — periodic command polling, condition DSL, result dispatch
|
||||
judge.py Intent validation — heuristic rules + LLM judge, advisory verdicts
|
||||
@@ -74,24 +73,19 @@ turnstone/
|
||||
_base.py Shared httpx async client, auth, error handling
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
mq/
|
||||
protocol.py Inbound/outbound message dataclasses (JSON serialization)
|
||||
broker.py Abstract MessageBroker protocol + RedisBroker
|
||||
bridge.py Bridge service (queue ↔ turnstone-server HTTP API)
|
||||
client.py TurnstoneClient library + TurnResult for MQ-based access
|
||||
console/
|
||||
collector.py ClusterCollector — aggregates state from all nodes via Redis + HTTP
|
||||
scheduler.py TaskScheduler — background cron/at scheduler, dispatches via MQ
|
||||
collector.py ClusterCollector — aggregates state from all nodes via SSE
|
||||
scheduler.py TaskScheduler — background cron/at scheduler, dispatches via HTTP
|
||||
server.py Cluster dashboard HTTP server + SSE + CLI entry point
|
||||
static/ Cluster dashboard web UI (page-specific HTML, CSS, JS)
|
||||
channels/
|
||||
cli.py Unified channel gateway entry point (turnstone-channel)
|
||||
_protocol.py ChannelAdapter protocol, ChannelEvent dataclass
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via MQ
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via HTTP
|
||||
_config.py Base ChannelConfig dataclass
|
||||
discord/ Discord adapter (bot, cog, views, streaming, config)
|
||||
shared_static/ Shared design system (base.css, auth.js, theme.js, toast.js, utils.js, kb.js)
|
||||
katex-0.16.38/ Vendored KaTeX math rendering library (MIT, woff2 fonts)
|
||||
katex-0.16.44/ Vendored KaTeX math rendering library (MIT, woff2 fonts)
|
||||
ui/
|
||||
colors.py ANSI color constants with NO_COLOR support
|
||||
markdown.py Streaming terminal markdown renderer (line-buffered)
|
||||
@@ -100,7 +94,7 @@ turnstone/
|
||||
index.html Single-page app shell (links to CSS and JS)
|
||||
style.css Page-specific UI styles (dashboard, markdown elements, approval blocks)
|
||||
renderer.js Markdown + LaTeX renderer (tables, nested lists, blockquotes, KaTeX math)
|
||||
app.js Page-specific client-side JavaScript (SSE, workstreams, tool approval)
|
||||
app.js Split-pane UI (Pane class, binary layout tree, SSE, tool approval)
|
||||
tools/
|
||||
*.json 15 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
```
|
||||
@@ -242,7 +236,7 @@ class SessionUI(Protocol):
|
||||
def on_content_token(self, text: str) -> None: ...
|
||||
def on_stream_end(self) -> None: ...
|
||||
def approve_tools(self, items: list[dict]) -> tuple[bool, str | None]: ...
|
||||
def on_tool_result(self, call_id: str, name: str, output: str) -> None: ...
|
||||
def on_tool_result(self, call_id: str, name: str, output: str, *, is_error: bool = False) -> None: ...
|
||||
def on_tool_output_chunk(self, call_id: str, chunk: str) -> None: ...
|
||||
def on_status(self, usage: dict, context_window: int, effort: str) -> None: ...
|
||||
def on_plan_review(self, content: str) -> str: ...
|
||||
@@ -259,7 +253,7 @@ class SessionUI(Protocol):
|
||||
| Class | Module | Notes |
|
||||
|-------|--------|-------|
|
||||
| `TerminalUI` | `turnstone.cli` | ANSI colors, `MarkdownRenderer`, `Spinner`, readline-based `input()` for approval |
|
||||
| `WebUI` | `turnstone.server` | SSE event queue per workstream, `threading.Event` for blocking on approval/plan |
|
||||
| `WebUI` | `turnstone.server` | SSE event queue per workstream + global broadcast, `threading.Event` for blocking on approval/plan. `on_state_change` sends to both per-workstream and global SSE (the browser UI uses per-workstream `state_change` events to manage busy/idle transitions; `stream_end` only finalizes markdown rendering). |
|
||||
| `NullUI` | `turnstone.eval` | Discards all output; `approve_tools` always returns `(True, None)` |
|
||||
|
||||
### WorkstreamTerminalUI
|
||||
@@ -353,7 +347,7 @@ remove the tab immediately. Controlled by `--workstream-idle-timeout` (default:
|
||||
|
||||
**Workstream eviction at capacity:** When `WorkstreamManager.create()` would
|
||||
exceed `max_workstreams` (configurable via `[server].max_workstreams`, default
|
||||
10), the oldest IDLE workstream is automatically evicted to make room. The
|
||||
50), the oldest IDLE workstream is automatically evicted to make room. The
|
||||
`turnstone_workstreams_evicted_total` counter is incremented on each eviction.
|
||||
If no IDLE workstream is available the create request fails as before.
|
||||
|
||||
@@ -375,12 +369,19 @@ non-idle background workstreams above the input prompt.
|
||||
### Web Workstreams
|
||||
|
||||
- **Tab bar**: Each workstream renders as a tab with a colored state indicator
|
||||
(CSS `@keyframes pulse` animation per state).
|
||||
- **Per-tab SSE**: `connectContentSSE(wsId)` opens
|
||||
`/v1/api/events?ws_id=<id>` for the active tab's event stream.
|
||||
(CSS `@keyframes pulse` animation per state). Clicking a tab switches the
|
||||
focused pane's workstream (or focuses an existing pane showing that ws).
|
||||
- **Split panes**: The UI supports tiling multiple workstreams side-by-side or
|
||||
stacked via a binary layout tree. Each `Pane` instance encapsulates its own
|
||||
SSE connection, message area, input, and state (busy, approval, streaming).
|
||||
Split via right-click context menu, pane header buttons, or keyboard
|
||||
(`Ctrl+\`, `Ctrl+Shift+\`). Max 6 panes; no duplicate workstreams across panes.
|
||||
Layout persisted to `localStorage`.
|
||||
- **Per-pane SSE**: `Pane.connectSSE(wsId)` opens
|
||||
`/v1/api/events?ws_id=<id>` for each pane's event stream independently.
|
||||
- **Global SSE**: `connectGlobalSSE()` opens `/v1/api/events/global` which
|
||||
receives `ws_state` broadcasts from all workstreams, used to update tab
|
||||
indicators without switching.
|
||||
indicators and pane headers without switching.
|
||||
- **New tab / close**: POST `/v1/api/workstreams/new`, POST `/v1/api/workstreams/close`.
|
||||
|
||||
### Thread Safety
|
||||
@@ -550,6 +551,17 @@ at connection time (server names with `__` are rejected).
|
||||
servers are unaffected. Tool execution errors return error strings to the LLM
|
||||
rather than crashing the session.
|
||||
|
||||
**Registry discovery:** The console admin panel provides a registry discovery
|
||||
surface backed by the official MCP Registry (registry.modelcontextprotocol.io).
|
||||
`MCPRegistryClient` (`turnstone/core/mcp_registry.py`) is a standalone httpx
|
||||
async client that queries the registry's v0.1 API for server discovery. Search
|
||||
results are annotated with installed status by cross-referencing the
|
||||
`mcp_servers` table. Installation creates a DB row with `registry_name`,
|
||||
`registry_version`, and `registry_meta` columns (migration 019), then triggers
|
||||
cluster-wide node reload via `_notify_nodes_mcp_reload()`. The registry URL is
|
||||
configurable via the `mcp.registry_url` setting for enterprise/private
|
||||
registries.
|
||||
|
||||
### Provider Adapter Layer
|
||||
|
||||
> See also: [Core Engine Classes diagram](diagrams/png/03-core-engine-classes.png)
|
||||
@@ -585,7 +597,7 @@ LLMProvider (protocol)
|
||||
| `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason` |
|
||||
| `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage` |
|
||||
| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search`, `supports_tool_search`, `supports_vision` |
|
||||
| `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens` |
|
||||
| `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens`, `cache_creation_tokens`, `cache_read_tokens` |
|
||||
|
||||
**OpenAIProvider** (`_openai.py`): passes messages through unchanged (they are
|
||||
already in OpenAI format), including multi-part content blocks (text + images)
|
||||
@@ -593,7 +605,10 @@ in tool results. Model capability lookup table covers GPT-5/5.1/5.2/5.3/5.4,
|
||||
O-series, and search models (`gpt-5-search-api`) — all with `supports_vision`.
|
||||
For search models, injects `web_search_options` and removes the `web_search`
|
||||
function tool (the model always searches). Citations from `url_citation`
|
||||
annotations are formatted as footnotes. Unknown models (local servers) get
|
||||
annotations are formatted as footnotes. Extended prompt cache retention
|
||||
(`prompt_cache_retention: "24h"`) is enabled for GPT-5.x models at no
|
||||
additional cost. Cached token counts are extracted from
|
||||
`usage.prompt_tokens_details.cached_tokens`. Unknown models (local servers) get
|
||||
permissive defaults with `supports_vision=False` and use Tavily for web search.
|
||||
|
||||
**AnthropicProvider** (`_anthropic.py`): converts OpenAI-format messages to
|
||||
@@ -607,8 +622,14 @@ Sonnet 4.6. Replaces the `web_search` function tool with Anthropic's native
|
||||
`web_search_20250305` server-side tool — Claude decides when to search, the
|
||||
API executes it, and results stream back as `server_tool_use` /
|
||||
`web_search_tool_result` content blocks (emitted as `info_delta` for UI
|
||||
display). The `anthropic` SDK is imported lazily so it remains an optional
|
||||
dependency (`pip install turnstone[anthropic]`).
|
||||
display). Automatic prompt caching is enabled via top-level `cache_control:
|
||||
{"type": "ephemeral"}` — the API places the cache breakpoint on the last
|
||||
cacheable block and advances it as conversations grow (90% input cost
|
||||
reduction on cache hits, 1.25x write on first turn). Cache metrics
|
||||
(`cache_creation_input_tokens`, `cache_read_input_tokens`) are extracted from
|
||||
both streaming and non-streaming responses. The `anthropic` SDK is imported
|
||||
lazily so it remains an optional dependency (`pip install
|
||||
turnstone[anthropic]`).
|
||||
|
||||
**Factory functions** (`__init__.py`): `create_provider(name)` returns a
|
||||
singleton provider instance (thread-safe). `create_client(name, base_url,
|
||||
@@ -677,8 +698,7 @@ supports_vision = true
|
||||
sub-agents, allowing a cheaper model for autonomous loops
|
||||
|
||||
**Per-workstream selection:** `POST /v1/api/workstreams/new` accepts an optional
|
||||
`"model"` field. The bridge `CreateWorkstreamMessage` carries the same field
|
||||
through the MQ protocol, along with `ws_template` (workstream template name)
|
||||
`"model"` field, along with `skill` (skill name)
|
||||
which can override the model before workstream creation.
|
||||
|
||||
### Tool Output Truncation
|
||||
@@ -808,10 +828,16 @@ and are the single source of truth for both backends and Alembic migrations.
|
||||
backend = "sqlite" # "sqlite" | "postgresql"
|
||||
path = ".turnstone.db" # SQLite file path
|
||||
url = "" # PostgreSQL connection URL
|
||||
pool_size = 5 # PostgreSQL connection pool size
|
||||
pool_size = 2 # PostgreSQL connection pool size (per process)
|
||||
```
|
||||
|
||||
Environment variables: `TURNSTONE_DB_BACKEND`, `TURNSTONE_DB_URL`, `TURNSTONE_DB_PATH`.
|
||||
Environment variables: `TURNSTONE_DB_BACKEND`, `TURNSTONE_DB_URL`, `TURNSTONE_DB_PATH`,
|
||||
`TURNSTONE_DB_POOL_SIZE`.
|
||||
|
||||
The default pool is intentionally small (2 base + 3 overflow = 5 per process)
|
||||
because all database operations are short-burst queries that hold connections for
|
||||
milliseconds. For clusters with many nodes sharing a PostgreSQL instance, use
|
||||
[PgBouncer](pgbouncer.md) in transaction pooling mode.
|
||||
|
||||
### Persistence and Resume
|
||||
|
||||
@@ -933,6 +959,9 @@ warns if the summary was truncated.
|
||||
seeded with `history.replaceState({turnstone: 'dashboard'})` on load. The
|
||||
`popstate` listener restores the correct tab or shows the dashboard,
|
||||
guarded by `_historyNavigation = true` to prevent re-entrant pushState.
|
||||
- **Pane focus**: `mousedown` and `focusin` events on pane containers update
|
||||
`focusedPaneId`. Approval shortcuts (y/n/a) apply to the focused pane.
|
||||
`Ctrl+Alt+Arrow` cycles focus between panes.
|
||||
|
||||
### Eval Resilience
|
||||
|
||||
@@ -987,13 +1016,10 @@ limits using a token-bucket algorithm. Each IP gets a `TokenBucket` with
|
||||
Turnstone supports three authentication mechanisms, unified behind an
|
||||
`AuthResult` dataclass that carries `user_id`, `scopes`, and `token_source`:
|
||||
|
||||
1. **Config-file tokens** — static secrets in `config.toml` `[[auth.tokens]]`
|
||||
or the `TURNSTONE_AUTH_TOKEN` env var. Validated in-memory via
|
||||
`hmac.compare_digest`. Map to scopes through their role (`read` or `full`).
|
||||
2. **API tokens** — database-backed, prefixed `ts_`, stored as SHA-256 hashes
|
||||
1. **API tokens** — database-backed, prefixed `ts_`, stored as SHA-256 hashes
|
||||
in the `api_tokens` table. Can be exchanged for JWTs via
|
||||
`POST /v1/api/auth/login`.
|
||||
3. **JWTs** — short-lived HMAC-SHA256 session tokens (default 24h) issued after
|
||||
2. **JWTs** — short-lived HMAC-SHA256 session tokens (default 24h) issued after
|
||||
successful credential validation. Contain `sub` (user_id), `scopes`, and
|
||||
`src` (origin) in claims.
|
||||
|
||||
@@ -1017,9 +1043,8 @@ Three hierarchical scopes control endpoint access:
|
||||
2. **Token extraction** — `Authorization: Bearer <token>` header first, then
|
||||
`turnstone_auth` cookie as fallback.
|
||||
3. **Token type detection** — dots in the token indicate JWT; `ts_` prefix
|
||||
indicates API token; otherwise config-file token.
|
||||
4. **Validation** — JWT signature check, API token hash lookup in storage, or
|
||||
config-token hmac comparison.
|
||||
indicates API token.
|
||||
4. **Validation** — JWT signature check or API token hash lookup in storage.
|
||||
5. **Scope check** — `required_scope(method, path)` determines the minimum
|
||||
scope; the request is rejected with 403 if the token lacks it.
|
||||
6. **Context propagation** — on success, `ctx_user_id` is set so structured
|
||||
@@ -1169,97 +1194,41 @@ calls `_fg_event.wait()`, which blocks the worker thread until the user
|
||||
switches to that workstream. The `_bg_attention_notify` callback writes a
|
||||
bell + status line to stderr to alert the user.
|
||||
|
||||
### Message Queue Bridge
|
||||
|
||||
```
|
||||
Main thread Global SSE thread Per-WS SSE threads (×N)
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| Inbound loop | | GET /events/glob | | GET /events?ws_id |
|
||||
| BLPOP on Redis | | Parse SSE via | | Parse SSE via |
|
||||
| | | httpx-sse | | httpx-sse |
|
||||
| Dispatch to | | Forward state | | Forward content, |
|
||||
| handler | | changes | | tool results |
|
||||
| POST to server | | Detect turn | | Handle approval |
|
||||
| Publish ACK | | completion | | forwarding |
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| | |
|
||||
+-- Redis inbound queue +-- Redis pub/sub +-- Redis pub/sub
|
||||
(RPUSH/BLPOP) (PUBLISH) (PUBLISH)
|
||||
+ response queue
|
||||
(BLPOP on
|
||||
approval)
|
||||
```
|
||||
|
||||
**Approval flow:** When a per-WS SSE thread receives an `approve_request`, it checks
|
||||
the workstream's `auto_approve_tools` set. If all requested tools are in the set, the
|
||||
bridge auto-approves via `POST /v1/api/approve`. Otherwise, it publishes an
|
||||
`ApprovalRequestEvent` to the outbound channel with a `request_id`, then blocks on
|
||||
`BLPOP` of a Redis response queue (`turnstone:resp:{request_id}`) until the client pushes
|
||||
a response or the approval timeout (default 3600s / 1 hour) expires.
|
||||
|
||||
**Cancellation:** The `CancelMessage` (type `"cancel"`) is a routed inbound message.
|
||||
The bridge dispatches it to `POST /v1/api/cancel` on the server owning the workstream,
|
||||
which sets the cooperative cancel flag and unblocks any pending approval/plan waits.
|
||||
|
||||
**Completion detection:** The bridge tracks which `correlation_id` maps to which
|
||||
`ws_id` for active sends. The server accumulates content tokens in the WebUI and
|
||||
piggybacks the full response text onto the `ws_state → idle` global SSE event.
|
||||
When the bridge receives this event, it emits a synthetic `TurnCompleteEvent`
|
||||
carrying the correlation ID and the server-provided `content`. This lets downstream
|
||||
consumers (e.g. the Discord bot) recover the full response when individual
|
||||
`ContentEvent`s were missed, and serves as the primary delivery path for
|
||||
bidirectional notification DM forwarding.
|
||||
|
||||
**Multi-node routing:** Each bridge retrieves its `node_id` from the server's
|
||||
`/health` endpoint on startup (with exponential backoff retry). The server
|
||||
generates the `node_id` (`{hostname}_{4hex}`) and is the sole authority for
|
||||
node identity. The bridge BLPOPs
|
||||
from both `turnstone:inbound:{node_id}` (directed, priority) and `turnstone:inbound` (shared).
|
||||
Messages with `target_node` set are pushed to the target's per-node queue. Messages
|
||||
for existing workstreams are auto-routed via `turnstone:ws:{ws_id}` ownership keys in Redis.
|
||||
If a bridge picks up a shared-queue message for a workstream owned by another node, it
|
||||
re-routes to that node's queue (1 extra hop). Bridges publish heartbeats to
|
||||
`turnstone:node:{node_id}` with configurable TTL for node discovery.
|
||||
On startup, `_recover_workstreams` re-registers ownership of existing
|
||||
workstreams and publishes `WorkstreamCreatedEvent` to the cluster channel
|
||||
so the console collector picks them up immediately.
|
||||
|
||||
### Cluster Console
|
||||
|
||||
```
|
||||
Monitoring (3 daemon threads) Control + Proxy (async Starlette)
|
||||
Monitoring (2 daemon threads) Control + Proxy (async Starlette)
|
||||
+------------------+ +----------------------------+
|
||||
| Event subscriber | | POST /v1/api/cluster/ |
|
||||
| SUBSCRIBE on | | workstreams/new |
|
||||
| events:cluster | | → LPUSH to Redis |
|
||||
+------------------+ | inbound:{node_id} |
|
||||
| Node discovery | +----------------------------+
|
||||
| SCAN node:* keys | | GET /node/{node_id}/ |
|
||||
| every 15 seconds | | → httpx.AsyncClient |
|
||||
+------------------+ | proxy to server_url |
|
||||
| Poll loop | | GET /node/{id}/v1/api/events |
|
||||
| GET /v1/api/dash | | → SSE stream proxy |
|
||||
| GET /health | | POST /node/{id}/v1/api/send |
|
||||
| ThreadPoolExec | | → forwarded to server |
|
||||
| Node discovery | | POST /v1/api/cluster/ |
|
||||
| Service registry | | workstreams/new |
|
||||
| every 60 seconds | | → POST to target server |
|
||||
+------------------+ +----------------------------+
|
||||
| SSE manager | | GET /node/{node_id}/ |
|
||||
| asyncio loop | | → httpx.AsyncClient |
|
||||
| 1 task per node | | proxy to server_url |
|
||||
| /events/global | | GET /node/{id}/v1/api/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/send |
|
||||
| → forwarded to server |
|
||||
+----------------------------+
|
||||
```
|
||||
|
||||
The console HTTP layer is a Starlette/ASGI app served by uvicorn. The SSE
|
||||
endpoint uses `EventSourceResponse` with the same listener queue pattern as
|
||||
the main server. `ClusterCollector`'s background threads (event subscriber,
|
||||
node discovery, poll loop) use sync Redis clients and `ThreadPoolExecutor`
|
||||
for parallel HTTP polling. The poll loop diffs workstream IDs between poll
|
||||
cycles and fans out synthetic `ws_created`/`ws_closed` SSE events for any
|
||||
changes, ensuring browser clients stay in sync even when real-time cluster
|
||||
events are missed (e.g. bridge startup recovery).
|
||||
the main server. `ClusterCollector` runs two daemon threads: a discovery loop
|
||||
that queries the service registry every 60 seconds, and an SSE manager that
|
||||
runs a single asyncio event loop multiplexing persistent SSE connections to
|
||||
all nodes via `GET /v1/api/events/global`. Each node delivers a full snapshot
|
||||
on connect followed by real-time delta events — state changes, health
|
||||
transitions, and aggregate metrics arrive sub-second instead of on a 15-second
|
||||
poll cycle.
|
||||
|
||||
The console has two write-path capabilities:
|
||||
|
||||
1. **Workstream creation** — pushes `CreateWorkstreamMessage` to Redis inbound
|
||||
queues targeting specific nodes. The bridge on each node picks up the message
|
||||
and creates the workstream on the local server. Auto-selects the node with
|
||||
the most available capacity if no target is specified. When a `ws_template`
|
||||
field is present, the server resolves the template BEFORE `mgr.create()`
|
||||
1. **Workstream creation** — sends HTTP requests to target server nodes
|
||||
to create workstreams. Auto-selects the node with
|
||||
the most available capacity if no target is specified. When a `skill`
|
||||
field is present, the server resolves the skill BEFORE `mgr.create()`
|
||||
(applying the model override to the creation request) and snapshot-applies
|
||||
remaining settings (auto-approve, token budget, temperature, etc.) to the
|
||||
workstream config AFTER creation.
|
||||
@@ -1324,7 +1293,7 @@ event loop on a daemon thread.
|
||||
|
||||
**Event types**: 27 standalone dataclasses in `events.py` with a type-registry
|
||||
pattern matching `OutboundEvent.from_json()` from `mq/protocol.py`. Events are
|
||||
decoupled from the MQ package so SDK consumers don't need the `redis` dependency.
|
||||
decoupled from server internals.
|
||||
|
||||
**TypeScript SDK**: `sdk/typescript/` — separate npm package with the same API
|
||||
surface. Zero browser dependencies, SSE via `fetch` + `ReadableStream` parsing.
|
||||
@@ -1345,20 +1314,19 @@ with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
|
||||
> See also: [Channel Integrations guide](channels.md)
|
||||
|
||||
The `turnstone-channel` gateway bridges external messaging platforms
|
||||
(Discord, Slack, Teams) to the turnstone cluster via Redis MQ. Each
|
||||
The `turnstone-channel` gateway connects external messaging platforms
|
||||
(Discord, Slack, Teams) to the turnstone cluster via HTTP. Each
|
||||
platform adapter implements the `ChannelAdapter` protocol and translates
|
||||
between platform-native events and turnstone MQ messages.
|
||||
between platform-native events and turnstone server API calls.
|
||||
|
||||
The `ChannelRouter` manages bidirectional routing: it maps platform
|
||||
channel/thread IDs to turnstone workstream IDs, handles workstream
|
||||
creation and stale-route recovery, and resolves platform users to
|
||||
turnstone identities via the `channel_users` table. When an evicted
|
||||
workstream is reactivated, the router uses atomic resume via the
|
||||
`resume_ws` field on `CreateWorkstreamMessage` — the server resumes
|
||||
`resume_ws` field on the workstream creation request — the server resumes
|
||||
the old workstream's conversation during creation in a single HTTP
|
||||
request, eliminating ordering fragility. The bridge emits a
|
||||
`WorkstreamResumedEvent` to confirm success.
|
||||
request, eliminating ordering fragility.
|
||||
|
||||
Discord ships as the first adapter. See [channels.md](channels.md) for
|
||||
setup instructions, configuration reference, and the adapter development
|
||||
@@ -1367,7 +1335,7 @@ guide.
|
||||
### Notification Subsystem
|
||||
|
||||
The `notify` tool enables the LLM to send notifications to users or
|
||||
channels without going through MQ. The server calls the channel gateway
|
||||
channels directly. The server calls the channel gateway
|
||||
directly over HTTP for lower latency: `_exec_notify()` queries the
|
||||
`services` database table for healthy channel gateways (heartbeat within
|
||||
120 seconds), authenticates with a service JWT (`aud: turnstone-channel`),
|
||||
@@ -1397,7 +1365,7 @@ at 100 (FIFO eviction) and cleaned up on workstream close.
|
||||
> See also: [Governance documentation](governance.md) | [Governance Architecture diagram](diagrams/19-governance-architecture.puml)
|
||||
|
||||
Turnstone governance extends the Phase 1 auth system with role-based access
|
||||
control (RBAC), tool execution policies, prompt templates, usage tracking,
|
||||
control (RBAC), tool execution policies, skills, usage tracking,
|
||||
and audit logging. The permission model has two layers: legacy scopes
|
||||
(`read`, `write`, `approve`) checked by `AuthMiddleware`, and 15 granular
|
||||
permissions checked per-endpoint by `require_permission()`. Three built-in
|
||||
@@ -1407,25 +1375,22 @@ can be created with any permission subset. JWTs carry both `scopes` and
|
||||
|
||||
Tool policies use glob pattern matching (`fnmatch`) with priority-ordered
|
||||
first-match-wins evaluation to control tool execution (allow/deny/ask).
|
||||
Prompt templates provide reusable system messages with `{{variable}}`
|
||||
substitution. Usage events are recorded per-LLM-request for token
|
||||
accounting. An append-only audit log captures all admin mutations.
|
||||
Skills provide reusable system messages with `{{variable}}` substitution
|
||||
plus session configuration (model, temperature, auto-approve, token budget,
|
||||
etc.). Usage events are recorded per-LLM-request for token accounting.
|
||||
An append-only audit log captures all admin mutations.
|
||||
|
||||
Workstream templates build on top of prompt templates as complete behavioral
|
||||
profiles applied at workstream creation. While prompt templates inject system
|
||||
message text, workstream templates define model, temperature, reasoning effort,
|
||||
max tokens, auto-approve policy, token budget, and agent max turns. Templates
|
||||
are snapshot-applied once at creation — not a live binding. The
|
||||
`workstream_templates` table (migration 011) supports auto-versioning, and
|
||||
workstreams record which template and version spawned them. Token budget
|
||||
Skills are snapshot-applied once at workstream creation — not a live binding.
|
||||
The `prompt_templates` table (which stores skills) supports auto-versioning,
|
||||
and workstreams record which skill and version spawned them. Token budget
|
||||
enforcement tracks consumption in `session.send()` with 80% warning and
|
||||
100% approval gate via the `__budget_override__` synthetic tool name.
|
||||
|
||||
The console admin panel adds 6 governance tabs (Roles, Policies, Templates,
|
||||
WS Templates, Usage, Audit), a Memories tab, a Settings tab (form-based
|
||||
editor for all ConfigStore settings), and an MCP Servers tab (database-backed
|
||||
server definitions with live connection status and cluster-wide reload) for a
|
||||
total of 14 tabs, all permission-gated.
|
||||
The console admin panel adds 5 governance tabs (Roles, Policies, Skills,
|
||||
Usage, Audit), a Memories tab, a Settings tab (form-based editor for all
|
||||
ConfigStore settings), and an MCP Servers tab (database-backed server
|
||||
definitions with live connection status and cluster-wide reload) for a
|
||||
total of 13 tabs, all permission-gated.
|
||||
Both Python and TypeScript SDKs expose governance methods on the console
|
||||
client.
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:75c1832b6079e8628f4bbf4ce98d37880c4de133636b7555e3869990b046ddc6
|
||||
size 567704
|
||||
+23
-31
@@ -1,9 +1,10 @@
|
||||
# Channel Integrations
|
||||
|
||||
The `turnstone-channel` gateway connects external messaging platforms to
|
||||
turnstone workstreams via Redis MQ. Each platform adapter translates
|
||||
turnstone workstreams via direct HTTP to the server (single-node) or the
|
||||
console routing proxy (multi-node). Each platform adapter translates
|
||||
platform-native events (messages, button clicks, slash commands) into
|
||||
turnstone MQ messages, and renders workstream output back into the
|
||||
turnstone API calls, and renders workstream output back into the
|
||||
platform's UI.
|
||||
|
||||
Discord ships as the first adapter. The adapter protocol is designed for
|
||||
@@ -20,10 +21,9 @@ Discord Gateway
|
||||
turnstone-channel (Discord adapter)
|
||||
|
|
||||
v
|
||||
Redis MQ
|
||||
|
|
||||
v
|
||||
turnstone-bridge ──> turnstone-server
|
||||
turnstone-server (direct HTTP)
|
||||
or
|
||||
turnstone-console (routing proxy, multi-node)
|
||||
```
|
||||
|
||||
Key components:
|
||||
@@ -34,10 +34,7 @@ Key components:
|
||||
`send_approval_request()`, `send_plan_review()`, and `create_thread()`.
|
||||
- **ChannelRouter** (`turnstone/channels/_routing.py`) — maps
|
||||
channel/thread IDs to turnstone workstream IDs. Handles workstream
|
||||
creation via MQ, stale route detection, and user identity resolution.
|
||||
- **AsyncRedisBroker** (`turnstone/mq/async_broker.py`) — async Redis
|
||||
client compatible with discord.py's event loop. Used by the router for
|
||||
pub/sub and queue operations.
|
||||
creation via HTTP, stale route detection, and user identity resolution.
|
||||
- **channel_users table** — maps `(channel_type, channel_user_id)` to a
|
||||
turnstone `user_id`. Messages from unlinked users are silently dropped.
|
||||
- **channel_routes table** — persistent channel-to-workstream mappings.
|
||||
@@ -84,8 +81,7 @@ TURNSTONE_DISCORD_GUILD=123456789 # optional, restrict to one guild
|
||||
turnstone-channel \
|
||||
--discord-token "your-bot-token" \
|
||||
--discord-guild 123456789 \
|
||||
--redis-host localhost \
|
||||
--redis-port 6379
|
||||
--server-url http://localhost:8080
|
||||
```
|
||||
|
||||
**Docker Compose** (production profile):
|
||||
@@ -138,7 +134,7 @@ An admin can also force-link or unlink users via the console admin panel
|
||||
thread auto-creates a new workstream and atomically resumes the
|
||||
previous workstream via the `resume_ws` field on
|
||||
`CreateWorkstreamMessage`. The server resumes the workstream during
|
||||
creation (same HTTP request), and the bridge emits a
|
||||
creation (same HTTP request), and the server emits a
|
||||
`WorkstreamResumedEvent` back to the channel. The thread receives a
|
||||
*"Resumed: {name} ({count} messages restored)"* confirmation.
|
||||
|
||||
@@ -160,8 +156,7 @@ an orange embed with:
|
||||
- Tool name and argument preview
|
||||
- **Approve** (green), **Reject** (red), **Always Approve** (gray) buttons
|
||||
- Only linked users can interact with approval buttons
|
||||
- The approval decision is forwarded through MQ to the bridge, which
|
||||
relays it to the server
|
||||
- The approval decision is forwarded to the server via HTTP
|
||||
|
||||
Buttons use static `custom_id` values so they survive bot restarts.
|
||||
Correlation data (`ws_id`, `correlation_id`) is stored in the embed footer.
|
||||
@@ -181,7 +176,7 @@ Plan review requests are displayed as a blue embed with:
|
||||
- **Approve Plan** (green) button — approves the plan with empty feedback
|
||||
- **Request Changes** (gray) button — opens a modal for feedback text
|
||||
(up to 2000 characters)
|
||||
- Feedback is forwarded through MQ as a `PlanFeedbackMessage`
|
||||
- Feedback is forwarded to the server via HTTP
|
||||
|
||||
---
|
||||
|
||||
@@ -192,15 +187,12 @@ Plan review requests are displayed as a blue embed with:
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Bot token (required to enable Discord) |
|
||||
| `--discord-guild` | — | `0` (all guilds) | Restrict to a single Discord guild |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated channel IDs to allow |
|
||||
| `--redis-host` | `REDIS_HOST` | `localhost` | Redis host |
|
||||
| `--redis-port` | — | `6379` | Redis port |
|
||||
| `--redis-password` | `REDIS_PASSWORD` | — | Redis password |
|
||||
| `--redis-db` | — | `0` | Redis DB number |
|
||||
| `--server-url` | `TURNSTONE_SERVER_URL` | `http://localhost:8080` | Server URL (single-node) |
|
||||
| `--console-url` | `TURNSTONE_CONSOLE_URL` | — | Console URL (multi-node routing proxy) |
|
||||
| `--model` | — | server default | Default model for new workstreams |
|
||||
| `--auto-approve` | — | `false` | Auto-approve ALL tool calls (skips approval buttons entirely) |
|
||||
| `--http-host` | — | `127.0.0.1` | HTTP server bind address for notify endpoint |
|
||||
| `--http-port` | `TURNSTONE_CHANNEL_PORT` | `8091` | HTTP server port |
|
||||
| `--auth-token` | `TURNSTONE_CHANNEL_AUTH_TOKEN` | — | Static auth token for `/v1/api/notify` (alternative to JWT) |
|
||||
| `--log-level` | `TURNSTONE_LOG_LEVEL` | `INFO` | Log level |
|
||||
| `--log-format` | `TURNSTONE_LOG_FORMAT` | `auto` | Log format (`auto`/`json`/`text`) |
|
||||
|
||||
@@ -232,13 +224,13 @@ See [Security: Database Schema](security.md#database-schema) for the
|
||||
3. **Eviction** — the server evicts an idle workstream for capacity. The
|
||||
route is preserved and the thread stays open.
|
||||
4. **Reactivation** — the next message in the thread detects the stale
|
||||
route (no MQ owner) and creates a new workstream with the old `ws_id`
|
||||
as `resume_ws` on the `CreateWorkstreamMessage`. The server resumes
|
||||
route and creates a new workstream with the old `ws_id`
|
||||
as `resume_ws` on the creation request. The server resumes
|
||||
the workstream during creation (no separate command or reverse lookup
|
||||
needed). The bridge emits a `WorkstreamResumedEvent` to the channel, and
|
||||
needed). The channel receives a `WorkstreamResumedEvent`, and
|
||||
the thread displays *"Resumed: {name} ({count} messages restored)"*.
|
||||
If the old workstream was pruned, a fresh one starts with no error.
|
||||
5. **Close** — `/close` command closes the workstream via MQ, deletes the
|
||||
5. **Close** — `/close` command closes the workstream via HTTP, deletes the
|
||||
route, unsubscribes from events, and archives the Discord thread.
|
||||
|
||||
---
|
||||
@@ -264,7 +256,7 @@ Two modes:
|
||||
|
||||
### Delivery Flow
|
||||
|
||||
Notifications bypass MQ for lower latency. The server calls the channel
|
||||
Notifications use direct HTTP for low latency. The server calls the channel
|
||||
gateway directly over HTTP:
|
||||
|
||||
1. The LLM calls the `notify` tool with a message and target
|
||||
@@ -328,11 +320,11 @@ The `services` table schema:
|
||||
### Security
|
||||
|
||||
- **Authentication** — the gateway's `POST /v1/api/notify` endpoint
|
||||
requires authentication. Configure either `TURNSTONE_JWT_SECRET`
|
||||
(the server mints JWTs with `aud: turnstone-channel` automatically)
|
||||
or a static token via `--auth-token`. If neither is set, the
|
||||
gateway fails closed and rejects all requests with 401. Server JWTs
|
||||
(`aud: turnstone-server`) are rejected.
|
||||
requires authentication. Configure `TURNSTONE_JWT_SECRET` so the
|
||||
server can mint JWTs with `aud: turnstone-channel` automatically.
|
||||
If the secret is not set, the gateway fails closed and rejects all
|
||||
requests with 401. Server JWTs (`aud: turnstone-server`) are
|
||||
rejected.
|
||||
- **Rate limit** — maximum 5 notifications per turn. The counter only
|
||||
increments on successful delivery, so failures don't consume the
|
||||
budget.
|
||||
|
||||
+50
-79
@@ -1,16 +1,16 @@
|
||||
# Cluster Dashboard (turnstone-console)
|
||||
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It connects to the shared Redis broker, discovers nodes via heartbeat keys, polls each node's HTTP API for workstream data, and subscribes to a cluster event channel for real-time state changes.
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It discovers nodes via the `services` database table and subscribes to each node's SSE event stream for real-time workstream, health, and metric updates.
|
||||
|
||||
The console also supports **workstream creation** (dispatched via MQ to target nodes) and a **reverse proxy** that serves each node's server UI through the console port — so users only need network access to the console, not to individual server nodes.
|
||||
The console also supports **workstream creation** (dispatched via HTTP proxy to target nodes) and a **reverse proxy** that serves each node's server UI through the console port — so users only need network access to the console, not to individual server nodes.
|
||||
|
||||
## Architecture
|
||||
|
||||
> See also: [Console Data Flow diagram](diagrams/png/11-console-data-flow.png)
|
||||
|
||||
```
|
||||
┌── Redis ←── turnstone-bridge ←── turnstone-server
|
||||
│ (MQ) (per node) (per node)
|
||||
┌── services table ── turnstone-server
|
||||
│ (node registry) (per node)
|
||||
turnstone-console ──────┤
|
||||
(one instance) │
|
||||
└── turnstone-server (direct HTTP proxy)
|
||||
@@ -21,45 +21,28 @@ turnstone-console ──────┤
|
||||
|
||||
Data flows in two directions:
|
||||
|
||||
- **Inbound (monitoring):** Bridges publish state changes to `{prefix}:events:cluster` on Redis pub/sub. The console subscribes for real-time updates and periodically polls each node's `GET /v1/api/dashboard` for full workstream snapshots.
|
||||
- **Outbound (control):** The console pushes `CreateWorkstreamMessage` to Redis inbound queues targeting specific nodes. Bridges pick up these messages and create workstreams on their local servers.
|
||||
- **Inbound (monitoring):** The console discovers nodes via the `services` database table (nodes register on startup and send periodic heartbeats). It opens a persistent SSE connection to each node's `GET /v1/api/events/global` endpoint, receiving a full snapshot on connect followed by real-time delta events (state changes, health transitions, aggregate metrics).
|
||||
- **Outbound (control):** The console proxies workstream creation requests to target nodes via HTTP.
|
||||
- **Proxy (pass-through):** The console reverse-proxies each node's server UI at `/node/{node_id}/`, forwarding HTTP and SSE traffic so the browser never contacts server nodes directly.
|
||||
|
||||
### Data Sources
|
||||
|
||||
| Source | Method | Direction | Data |
|
||||
|--------|--------|-----------|------|
|
||||
| Redis heartbeats | `SCAN turnstone:node:*` | Read | Node discovery (node_id, server_url, started) |
|
||||
| Redis pub/sub | `SUBSCRIBE turnstone:events:cluster` | Read | State changes, creates, closes, renames |
|
||||
| Node HTTP API | `GET {server_url}/v1/api/dashboard` | Read | Full workstream list with tokens, context, activity |
|
||||
| Node HTTP API | `GET {server_url}/health` | Read | Node health status |
|
||||
| Redis inbound queue | `RPUSH turnstone:inbound:{node_id}` | Write | Workstream creation commands |
|
||||
| `services` table | Database query | Read | Node discovery (node_id, server_url, started) |
|
||||
| Node SSE | `GET {server_url}/v1/api/events/global` | Stream | Snapshot on connect, then real-time delta events (state, health, aggregate) |
|
||||
| Node HTTP API | `POST {server_url}/v1/api/workstreams/new` | Write | Workstream creation |
|
||||
| Node HTTP API | `GET/POST {server_url}/*` | Proxy | Server UI, API requests, SSE streams |
|
||||
|
||||
### Redis Key: Cluster Event Channel
|
||||
|
||||
Bridges publish to `{prefix}:events:cluster` whenever a workstream state change, creation, closure, or rename occurs. Events include `node_id` so the console can attribute them to the correct node.
|
||||
|
||||
Event types on the cluster channel:
|
||||
|
||||
| Event | Fields | Trigger |
|
||||
|-------|--------|---------|
|
||||
| `cluster_state` | ws_id, state, node_id, tokens, context_ratio, activity | Workstream state transition |
|
||||
| `ws_created` | ws_id, name, node_id | New workstream created |
|
||||
| `ws_closed` | ws_id | Workstream closed |
|
||||
| `ws_rename` | ws_id, name | Workstream renamed |
|
||||
|
||||
---
|
||||
|
||||
## ClusterCollector
|
||||
|
||||
The collector (`turnstone/console/collector.py`) maintains an in-memory snapshot of all nodes and workstreams. Three daemon threads handle data acquisition:
|
||||
The collector (`turnstone/console/collector.py`) maintains an in-memory snapshot of all nodes and workstreams. Two daemon threads handle data acquisition:
|
||||
|
||||
1. **Event subscriber** — subscribes to `{prefix}:events:cluster` via `RedisBroker.subscribe_cluster()`. Applies state changes, creates, closes, and renames to the in-memory model immediately.
|
||||
1. **Node discovery** — queries the `services` database table every 60 seconds. Adds newly discovered nodes, removes expired ones (stale heartbeats), emits `node_joined` / `node_lost` events to SSE listeners, and spawns/cancels SSE tasks for new/lost nodes.
|
||||
|
||||
2. **Node discovery** — scans heartbeat keys every 15 seconds via `broker.list_nodes()`. Adds newly discovered nodes, removes expired ones, emits `node_joined` / `node_lost` events to SSE listeners.
|
||||
|
||||
3. **Poll loop** — fetches `GET /v1/api/dashboard` and `GET /health` from each known node every 10 seconds. Uses `ThreadPoolExecutor(max_workers=50)` for parallelism. Each poll replaces the node's workstream list with the authoritative server data.
|
||||
2. **SSE manager** — a single asyncio event loop on one thread multiplexes persistent SSE connections to all discovered nodes via `GET /v1/api/events/global`. Each connection receives a `node_snapshot` on connect (workstreams, health, aggregate) followed by real-time delta events (`ws_state`, `ws_created`, `ws_closed`, `ws_rename`, `health_changed`, `aggregate`). On disconnect, the node is marked unreachable and the connection is retried with exponential backoff (1s–30s). An `?expected_node_id=` query parameter provides identity verification against IP reuse (server returns 409 on mismatch).
|
||||
|
||||
A `get_snapshot()` method builds the full cluster state under a single lock acquisition — overview aggregates and per-node workstream lists in one atomic read. This is served both as a REST endpoint and as the initial SSE event on client connect.
|
||||
|
||||
@@ -69,10 +52,11 @@ All reads and writes to the node/workstream map are protected by a single `threa
|
||||
|
||||
### Scale Considerations
|
||||
|
||||
- **10,000 workstreams** at ~500 bytes each = ~5 MB in memory
|
||||
- **1,000 nodes** polled in parallel with 50 threads at ~100ms each = ~2 second poll cycle
|
||||
- **50,000 workstreams** (1,000 nodes × 50 per node) at ~500 bytes each = ~25 MB in memory
|
||||
- **1,000 nodes** connected via persistent SSE — a single asyncio event loop multiplexes all connections with negligible overhead. Ensure `ulimit -n` >= 4096 for fd headroom
|
||||
- **Filtering and pagination** run in-memory on the full workstream list — sub-millisecond at this scale
|
||||
- **SSE fan-out** uses the same per-client queue pattern as the per-node server — backed-up clients get events dropped, not blocking
|
||||
- **SSE fan-out** uses per-client queues (2,000 events) — backed-up clients get events dropped, not blocking
|
||||
- **Database** — for clusters sharing PostgreSQL, use [PgBouncer](pgbouncer.md) in transaction pooling mode
|
||||
|
||||
---
|
||||
|
||||
@@ -182,7 +166,7 @@ Full cluster state in a single response — all nodes with their workstreams plu
|
||||
|
||||
### `POST /v1/api/cluster/workstreams/new`
|
||||
|
||||
Create a new workstream on a target node. Dispatches a `CreateWorkstreamMessage` through the Redis MQ pipeline — the bridge on the target node picks it up and creates the workstream on the server. Requires `write` scope.
|
||||
Create a new workstream on a target node. The console proxies the creation request to the target node's HTTP API. Requires `write` scope.
|
||||
|
||||
Request:
|
||||
|
||||
@@ -196,9 +180,9 @@ Request:
|
||||
|
||||
All fields are optional:
|
||||
- `node_id` — targeting mode:
|
||||
- **omitted or `"auto"`** — console picks the reachable node with the most available capacity (max_ws - ws_total) and pushes to its directed queue.
|
||||
- **`"pool"`** — pushes to the shared inbound queue; the next available bridge picks it up (true general-pool dispatch).
|
||||
- **specific node ID** — pushes to that node's directed queue.
|
||||
- **omitted or `"auto"`** — console picks the reachable node with the most available capacity (max_ws - ws_total) and proxies the request to it.
|
||||
- **`"pool"`** — console picks a reachable node with available capacity using round-robin selection.
|
||||
- **specific node ID** — proxies the request to that node directly.
|
||||
- `name` — workstream display name. Auto-generated if omitted.
|
||||
- `model` — model alias from the target node's registry. Uses the node's default model if omitted.
|
||||
|
||||
@@ -212,7 +196,7 @@ Response:
|
||||
}
|
||||
```
|
||||
|
||||
Creation is asynchronous — the response confirms the MQ message was dispatched. A `ws_created` event on the cluster SSE stream confirms the workstream was actually created.
|
||||
The response confirms the workstream creation request was proxied to the target node. A `ws_created` event on the cluster SSE stream confirms the workstream was actually created.
|
||||
|
||||
### `GET /v1/api/cluster/events`
|
||||
|
||||
@@ -306,18 +290,6 @@ Revoke a specific API token.
|
||||
|
||||
These endpoints manage the `channel_users` table mappings that connect external platform identities (e.g. Discord user IDs) to turnstone users. See [Channel Integrations](channels.md) for details on the linking flow.
|
||||
|
||||
### Workstream Templates
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/v1/api/admin/ws-templates` | List all workstream templates |
|
||||
| POST | `/v1/api/admin/ws-templates` | Create a workstream template |
|
||||
| GET | `/v1/api/admin/ws-templates/{id}` | Get a single workstream template |
|
||||
| PUT | `/v1/api/admin/ws-templates/{id}` | Update (auto-versions, audit logged) |
|
||||
| DELETE | `/v1/api/admin/ws-templates/{id}` | Delete + cascade versions (audit logged) |
|
||||
| GET | `/v1/api/admin/ws-templates/{id}/versions` | Version history |
|
||||
| GET | `/v1/api/ws-templates` | Enabled templates summary (name, description, model) — requires write scope, not admin |
|
||||
|
||||
#### `GET /v1/api/auth/status`
|
||||
|
||||
Public endpoint for login UI state detection. Returns auth configuration, not
|
||||
@@ -376,7 +348,7 @@ SSE streams (`/v1/api/events`, `/v1/api/events/global`) are proxied as raw byte
|
||||
|
||||
### Authentication
|
||||
|
||||
The proxy forwards the user's JWT to upstream server nodes — it extracts the token from the incoming request's cookie (or `Authorization` header) and adds it as a `Bearer` header on the proxied request. Since all services share the same `TURNSTONE_JWT_SECRET`, the user's JWT is valid on every node without re-authentication. The console's own auth middleware also checks proxy routes — `POST` requests to proxy write endpoints (`/v1/api/send`, `/v1/api/approve`, etc.) require `write` scope, preventing read-only tokens from escalating via proxy. The static `--auth-token` / `proxy_auth_token` is used as a fallback when no user JWT is present.
|
||||
The proxy mints a short-lived (5-minute) JWT per request carrying the real user's `user_id`, `scopes`, and `permissions` with `aud: turnstone-server`. The user's console JWT (`aud: turnstone-console`) cannot be forwarded directly — it would be rejected by the server's audience validation — so the console re-signs a new server-audience JWT from the validated `AuthResult`. This preserves audit attribution (the upstream server sees the real user, not a service identity) and enforces scope narrowing as defense in depth (a read-only console user's proxied request carries only `read` scope). The JWT `src` claim is set to `"console-proxy"` for audit traceability. When no user context is available (auth disabled), the proxy falls back to a `ServiceTokenManager` with service identity `console-proxy`. The static `--auth-token` / `proxy_auth_token` is used as a final fallback.
|
||||
|
||||
---
|
||||
|
||||
@@ -406,8 +378,8 @@ Breadcrumb: `Cluster > Running` or `Cluster > db-west-04`. Server-side paginated
|
||||
|
||||
Triggered by the "+ new" header button. A modal dialog with:
|
||||
|
||||
- **Node selector** — dropdown with three targeting modes: "Auto (best available)" picks the node with the most headroom, "General pool (any node)" pushes to the shared queue for any bridge to pick up, or a specific node from the list (showing capacity).
|
||||
- **Profile** — optional dropdown listing enabled workstream templates. Applies the template's model, auto-approve policy, token budget, and other behavioral settings at creation time.
|
||||
- **Node selector** — dropdown with three targeting modes: "Auto (best available)" picks the node with the most headroom, "General pool (any node)" picks a node with available capacity using round-robin, or a specific node from the list (showing capacity).
|
||||
- **Profile** — optional dropdown listing enabled skills. Applies the skill's model, auto-approve policy, token budget, and other behavioral settings at creation time.
|
||||
- **Name** — optional text input. Auto-generated if left empty.
|
||||
- **Model** — optional text input for a model alias from the target node's registry.
|
||||
|
||||
@@ -420,9 +392,10 @@ The browser maintains a local `clusterState` object that mirrors the cluster sna
|
||||
### 5. Admin Panel
|
||||
|
||||
Accessed via the "admin" button in the header (visible when authenticated
|
||||
with `approve` scope). Provides user, API token, channel link, and workstream
|
||||
template management with 13 tabs (see also [Governance](governance.md) for
|
||||
the Roles, Policies, Templates, WS Templates, Usage, and Audit tabs, and
|
||||
with `approve` scope). Provides user, API token, channel link, MCP server,
|
||||
and skill management with 13 tabs (see also
|
||||
[Governance](governance.md) for
|
||||
the Roles, Policies, Skills, Usage, and Audit tabs, and
|
||||
[Settings](settings.md) for the database-backed configuration editor):
|
||||
|
||||
**Users tab:**
|
||||
@@ -459,6 +432,22 @@ the Roles, Policies, Templates, WS Templates, Usage, and Audit tabs, and
|
||||
- Admins can force-link users who have not self-linked via `/link` in
|
||||
Discord
|
||||
|
||||
**MCP Servers tab:**
|
||||
|
||||
The tab has two views toggled via a pill control: **Servers** and
|
||||
**Registry**.
|
||||
|
||||
- **Servers view** -- lists all installed MCP servers with source badges
|
||||
(CONFIG, MANUAL, REGISTRY), transport badges, tool/resource/prompt
|
||||
counts, per-node connection status, and CRUD actions for DB-managed
|
||||
servers
|
||||
- **Registry view** -- search the official MCP Registry to discover and
|
||||
install servers. Results show server name, description, version, source
|
||||
type badges (remote/npm/pypi), and Install/Installed/Update buttons.
|
||||
Remote servers without required configuration are installed with one
|
||||
click; servers needing env vars, headers, or URL variables open an
|
||||
install modal for configuration
|
||||
|
||||
**Accessibility:**
|
||||
|
||||
- Full keyboard navigation: focus traps in modals, Escape to close, arrow
|
||||
@@ -476,17 +465,17 @@ to create the initial admin user and receive a JWT in one step. See
|
||||
|
||||
## Scheduled Tasks
|
||||
|
||||
The console includes a background **TaskScheduler** daemon that creates workstreams on a timed basis via the MQ broker. It supports cron-based recurring schedules and one-shot `at` schedules.
|
||||
The console includes a background **TaskScheduler** daemon that creates workstreams on a timed basis via HTTP proxy to target nodes. It supports cron-based recurring schedules and one-shot `at` schedules.
|
||||
|
||||
### Architecture
|
||||
|
||||
The scheduler runs as a daemon thread inside the console process. Every `check_interval` seconds (default 15) it:
|
||||
|
||||
1. Acquires a distributed lock via Redis `SET NX EX` (prevents duplicate dispatch in multi-console deployments)
|
||||
1. Acquires a distributed lock via the `system_settings` table (prevents duplicate dispatch in multi-console deployments)
|
||||
2. Queries the storage backend for tasks whose `next_run <= now` and `enabled = true`
|
||||
3. Dispatches each due task as one or more `CreateWorkstreamMessage` via MQ
|
||||
3. Dispatches each due task as one or more workstream creation requests via HTTP proxy
|
||||
4. Updates `last_run` and computes the next `next_run` (or disables one-shot `at` tasks)
|
||||
5. Releases the lock via Lua script (safe conditional delete)
|
||||
5. Releases the lock
|
||||
|
||||
Run history is automatically pruned (runs older than 90 days) approximately once per hour.
|
||||
|
||||
@@ -502,7 +491,7 @@ Run history is automatically pruned (runs older than 90 days) approximately once
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `auto` | Picks the reachable node with the most available capacity |
|
||||
| `pool` | Pushes to the shared inbound queue (any bridge picks it up) |
|
||||
| `pool` | Picks a reachable node with available capacity using round-robin |
|
||||
| `all` | Fan-out to all reachable nodes (capped at `max_fan_out`, default 20) |
|
||||
| `<node_id>` | Targets a specific node by ID |
|
||||
|
||||
@@ -639,12 +628,6 @@ CLI flags for `turnstone-console`:
|
||||
|------|---------|-------------|
|
||||
| `--host` | `0.0.0.0` | Bind host |
|
||||
| `--port` | `8090` | HTTP port |
|
||||
| `--redis-host` | `localhost` | Redis host |
|
||||
| `--redis-port` | `6379` | Redis port |
|
||||
| `--redis-password` | `$REDIS_PASSWORD` | Redis password |
|
||||
| `--redis-db` | `0` | Redis DB |
|
||||
| `--poll-interval` | `10` | Node polling interval (seconds) |
|
||||
| `--auth-token` | `$TURNSTONE_AUTH_TOKEN` | Bearer token for server node communication and proxy |
|
||||
| `--log-level` | `INFO` | Log level |
|
||||
|
||||
Config file (`~/.config/turnstone/config.toml`):
|
||||
@@ -654,12 +637,6 @@ Config file (`~/.config/turnstone/config.toml`):
|
||||
host = "0.0.0.0"
|
||||
port = 8090
|
||||
url = "http://localhost:8090" # used by CLI /cluster commands
|
||||
poll_interval = 10
|
||||
|
||||
[redis]
|
||||
host = "localhost"
|
||||
port = 6379
|
||||
password = "my-redis-password"
|
||||
```
|
||||
|
||||
---
|
||||
@@ -667,17 +644,11 @@ password = "my-redis-password"
|
||||
## Deployment
|
||||
|
||||
```bash
|
||||
# Start Redis
|
||||
redis-server
|
||||
|
||||
# Start turnstone servers (one per node)
|
||||
turnstone-server --port 8080
|
||||
|
||||
# Start bridges (one per server)
|
||||
turnstone-bridge --server-url http://localhost:8080 --node-id node-a
|
||||
|
||||
# Start cluster console (one instance)
|
||||
turnstone-console --redis-host localhost --port 8090 --auth-token "$TURNSTONE_AUTH_TOKEN"
|
||||
turnstone-console --port 8090
|
||||
```
|
||||
|
||||
Open `http://localhost:8090` for the cluster dashboard. Create workstreams via the "+ new" button. Click any workstream to open the proxied server UI — no direct access to server ports required.
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# Consistent Hash Ring — Reference Design
|
||||
|
||||
**Status**: Reference (not currently in the hot path)
|
||||
**Date**: 2026-03-30
|
||||
|
||||
## Overview
|
||||
|
||||
This document describes a consistent hash ring algorithm evaluated during
|
||||
the design of the direct HTTP transport routing system. The current
|
||||
implementation uses weight-proportional bucket assignment with a
|
||||
donor/recipient rebalancing algorithm (see `direct-http-transport.md`).
|
||||
The consistent hash ring is documented here as a reference for future
|
||||
scalability work — if the cluster grows beyond the point where the
|
||||
weight-proportional approach is sufficient, the ring provides a
|
||||
proven alternative with stronger stability guarantees.
|
||||
|
||||
## When to consider the ring approach
|
||||
|
||||
The current weight-proportional seeding + donor/recipient rebalancer works
|
||||
well when:
|
||||
- Cluster size is moderate (< 50 nodes)
|
||||
- Nodes join/leave infrequently
|
||||
- The rebalancer runs centrally (in the console)
|
||||
|
||||
The consistent hash ring becomes advantageous when:
|
||||
- Cluster size grows large (50+ nodes) and frequent membership changes
|
||||
cause the donor/recipient algorithm to churn
|
||||
- Decentralized routing is needed (each node computes the ring locally,
|
||||
no central console required)
|
||||
- Cross-language determinism is important (multiple implementations must
|
||||
agree on the same assignment without sharing state)
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Hash function: FNV-1a (32-bit)
|
||||
|
||||
```python
|
||||
def fnv1a_32(data: bytes) -> int:
|
||||
"""FNV-1a 32-bit hash.
|
||||
|
||||
Basis: 0x811C9DC5, Prime: 0x01000193.
|
||||
XOR each byte, then multiply by prime (masked to 32 bits).
|
||||
"""
|
||||
h = 0x811C9DC5
|
||||
for b in data:
|
||||
h ^= b
|
||||
h = (h * 0x01000193) & 0xFFFFFFFF
|
||||
return h
|
||||
```
|
||||
|
||||
Known test vectors:
|
||||
- `fnv1a_32(b"")` = `0x811C9DC5` (basis value)
|
||||
- `fnv1a_32(b"foobar")` = `0xBF9CF968`
|
||||
|
||||
Cross-language implementations:
|
||||
- **Python**: loop above (no dependencies)
|
||||
- **Go**: same algorithm with `uint32` arithmetic
|
||||
- **TypeScript**: same algorithm with `>>> 0` for unsigned 32-bit
|
||||
|
||||
### Virtual nodes
|
||||
|
||||
Each physical node with weight `w` gets `w * 150` virtual positions on a
|
||||
16-bit ring (65536 positions). Virtual node `i` of physical node `N` is
|
||||
placed at:
|
||||
|
||||
```
|
||||
position = fnv1a_32(f"{N.node_id}:{i}".encode()) % 65536
|
||||
```
|
||||
|
||||
With 150 vnodes per unit weight:
|
||||
- 2 equal-weight nodes: ~50/50 split (measured: 38-62% range due to
|
||||
hash variance, stddev ~3% with large vnode counts)
|
||||
- 3 nodes at weights 2:1:1: ~50/25/25 (within 10% tolerance)
|
||||
|
||||
### Lookup
|
||||
|
||||
```python
|
||||
def owner(bucket: int) -> str:
|
||||
"""O(log n) bisect-right walk to find the next virtual node clockwise."""
|
||||
idx = bisect_right(positions, bucket)
|
||||
if idx >= len(positions):
|
||||
idx = 0 # wrap around
|
||||
return vnode_map[positions[idx]]
|
||||
```
|
||||
|
||||
### Stability properties
|
||||
|
||||
The consistent hash ring guarantees:
|
||||
- **Node addition**: adding a node moves at most `1/N` of buckets (where N
|
||||
is the new node count). Other nodes' buckets are unaffected.
|
||||
- **Node removal**: only the removed node's buckets are reassigned. Buckets
|
||||
owned by surviving nodes don't move.
|
||||
- **Determinism**: same membership list always produces the same ring.
|
||||
No coordination needed between processes.
|
||||
|
||||
### Full assignment precomputation
|
||||
|
||||
```python
|
||||
def assignments() -> list[tuple[int, str]]:
|
||||
"""Compute all 65536 bucket-to-node mappings."""
|
||||
return [(b, owner(b)) for b in range(65536)]
|
||||
```
|
||||
|
||||
This produces a complete assignment table that can be loaded into a flat
|
||||
array for O(1) request-time lookup. The ring itself is never consulted
|
||||
on the hot path.
|
||||
|
||||
## Data structures
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RingNode:
|
||||
node_id: str
|
||||
url: str
|
||||
weight: int = 1
|
||||
|
||||
class HashRing:
|
||||
"""Immutable consistent hash ring. Thread-safe (no mutable state)."""
|
||||
|
||||
def __init__(self, nodes: Sequence[RingNode], vnodes_per_unit: int = 150):
|
||||
# Validate no duplicate node_ids
|
||||
# Build sorted array of (position, node_id) tuples
|
||||
# positions[i] = fnv1a_32(f"{node_id}:{i}".encode()) % RING_SIZE
|
||||
|
||||
def owner(self, bucket: int) -> RingNode | None:
|
||||
# bisect_right + wrap
|
||||
|
||||
@property
|
||||
def version(self) -> int:
|
||||
# Deterministic hash of membership: fnv1a_32 of sorted node_id:weight pairs
|
||||
|
||||
def assignments(self) -> list[tuple[int, str]]:
|
||||
# Precompute all 65536 bucket assignments
|
||||
```
|
||||
|
||||
## Comparison with current approach
|
||||
|
||||
| Aspect | Weight-proportional (current) | Consistent hash ring |
|
||||
|--------|------------------------------|---------------------|
|
||||
| Seeding | Exact weight split, deterministic | Hash-based, ~3% variance |
|
||||
| Node addition | Donor/recipient moves only excess | Ring moves ~1/N buckets |
|
||||
| Node removal | Dead buckets → most underloaded | Ring redistributes to clockwise neighbors |
|
||||
| Cross-node churn | Zero (only donor→recipient) | Zero (ring stability guarantee) |
|
||||
| Decentralized | No (needs central rebalancer) | Yes (each node computes locally) |
|
||||
| Complexity | Simple weight arithmetic | Virtual node construction + bisect |
|
||||
|
||||
## Test vectors
|
||||
|
||||
For cross-language implementation validation:
|
||||
|
||||
```json
|
||||
{
|
||||
"fnv1a_32": [
|
||||
{"input": "", "output": 2166136261},
|
||||
{"input": "foobar", "output": 3215766888}
|
||||
],
|
||||
"bucket_of": [
|
||||
{"ws_id": "a3f100000000000000000000000000000", "bucket": 41969},
|
||||
{"ws_id": "00000000000000000000000000000000", "bucket": 0},
|
||||
{"ws_id": "ffff0000000000000000000000000000", "bucket": 65535}
|
||||
],
|
||||
"ring_single_node": {
|
||||
"nodes": [{"node_id": "n1", "weight": 1}],
|
||||
"vnodes_per_unit": 150,
|
||||
"expected_n1_buckets": 65536
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -13,24 +13,22 @@ cloud "LLM Providers" as llm {
|
||||
component [OpenAI-compatible API\n(OpenAI, vLLM, llama.cpp)] as llm_openai
|
||||
component [Anthropic Messages API] as llm_anthropic
|
||||
}
|
||||
database "Redis" as redis
|
||||
database "SQLite\n(.turnstone.db)" as sqlite
|
||||
|
||||
' Turnstone System Boundary
|
||||
package "Turnstone Platform" {
|
||||
component [turnstone\n(CLI)] as cli <<entry point>>
|
||||
component [turnstone-server\n(HTTP + SSE)] as server <<entry point>>
|
||||
component [turnstone-bridge\n(Queue ↔ HTTP)] as bridge <<service>>
|
||||
component [turnstone-console\n(Dashboard)] as console <<service>>
|
||||
component [turnstone-console\n(Dashboard + Router)] as console <<service>>
|
||||
component [turnstone-eval\n(Headless)] as eval <<entry point>>
|
||||
component [turnstone-sim\n(Simulator)] as sim <<service>>
|
||||
component [turnstone-channel\n(Channel Gateway)] as channel <<service>>
|
||||
}
|
||||
|
||||
' User connections
|
||||
cli_user --> cli : stdin / stdout
|
||||
browser_user --> server : HTTP + SSE\n(port 8080)
|
||||
browser_user --> console : HTTP + SSE\n(port 8090)
|
||||
ext_client --> redis : Redis LIST\n(push commands)
|
||||
ext_client --> server : HTTP + SSE\n(SDK / API)
|
||||
eval_user --> eval : Python API
|
||||
|
||||
' Internal connections
|
||||
@@ -43,26 +41,16 @@ server --> sqlite : SQLite
|
||||
eval --> llm : LLM Provider API\n(non-streaming)
|
||||
eval --> sqlite : SQLite
|
||||
|
||||
bridge --> server : HTTP REST\n(POST /v1/api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /v1/api/events)
|
||||
bridge --> redis : Redis LIST + PUBSUB\n+ STRING (routing, heartbeats)
|
||||
console --> server : HTTP proxy\n(hash-ring bucket lookup,\nproxy /node/{id}/* traffic)
|
||||
|
||||
console --> redis : Redis PUBSUB + STRING + LIST\n(cluster events, heartbeats,\nworkstream creation commands)
|
||||
console --> server : HTTP polling + reverse proxy\n(GET /v1/api/dashboard,\nproxy /node/{id}/* traffic)
|
||||
|
||||
sim --> redis : Redis LIST + PUBSUB\n+ STRING (heartbeats)
|
||||
channel --> server : HTTP + SSE\n(POST /v1/api/send,\nGET /v1/api/events)
|
||||
|
||||
' Notes
|
||||
note right of sim
|
||||
Simulator replaces Server+Bridge
|
||||
with lightweight SimNodes that
|
||||
publish to the same Redis channels.
|
||||
end note
|
||||
|
||||
note right of redis
|
||||
Shared message broker:
|
||||
- LIST: command queues
|
||||
- STRING: heartbeats, routing
|
||||
- PUBSUB: event broadcast
|
||||
note right of console
|
||||
Multi-node router:
|
||||
- Hash-ring bucket lookup
|
||||
- Proxies create/send/approve
|
||||
- Direct SSE from client to node
|
||||
- HTTP polling for dashboard
|
||||
end note
|
||||
@enduml
|
||||
|
||||
@@ -6,13 +6,12 @@ title Turnstone — Package & Module Structure
|
||||
skinparam component {
|
||||
BackgroundColor<<entry>> #B8D4E3
|
||||
BackgroundColor<<core>> #C8E6C9
|
||||
BackgroundColor<<mq>> #FFE0B2
|
||||
BackgroundColor<<sim>> #E1BEE7
|
||||
BackgroundColor<<console>> #B2EBF2
|
||||
BackgroundColor<<ui>> #F0F4C3
|
||||
BackgroundColor<<artifact>> #ECEFF1
|
||||
BackgroundColor<<sdk>> #FFCDD2
|
||||
BackgroundColor<<api>> #D1C4E9
|
||||
BackgroundColor<<channel>> #FFE0B2
|
||||
}
|
||||
|
||||
' Entry points
|
||||
@@ -45,23 +44,11 @@ package "turnstone/core/" <<Rectangle>> {
|
||||
component [model_registry.py\nModelRegistry] as registry <<core>>
|
||||
}
|
||||
|
||||
' MQ subsystem
|
||||
package "turnstone/mq/" <<Rectangle>> {
|
||||
component [protocol.py\n28 message types] as protocol <<mq>>
|
||||
component [broker.py\nMessageBroker, RedisBroker] as broker <<mq>>
|
||||
component [bridge.py\nturnstone-bridge] as bridge <<mq>>
|
||||
component [client.py\nTurnstoneClient] as client <<mq>>
|
||||
}
|
||||
|
||||
' Simulator
|
||||
package "turnstone/sim/" <<Rectangle>> {
|
||||
component [cluster.py\nSimCluster] as simcluster <<sim>>
|
||||
component [node.py\nSimNode, SimWorkstream] as simnode <<sim>>
|
||||
component [engine.py\nSimEngine] as simengine <<sim>>
|
||||
component [scenario.py\n5 scenarios] as scenario <<sim>>
|
||||
component [sim/config.py\nSimConfig] as simconfig <<sim>>
|
||||
component [sim/metrics.py\nSim metrics] as simmetrics <<sim>>
|
||||
component [sim/cli.py\nturnstone-sim] as simcli <<sim>>
|
||||
' Channels
|
||||
package "turnstone/channels/" <<Rectangle>> {
|
||||
component [_routing.py\nChannelRouter] as router <<channel>>
|
||||
component [discord/bot.py\nDiscordBot] as discordbot <<channel>>
|
||||
component [gateway.py\nturnstone-channel] as gateway <<channel>>
|
||||
}
|
||||
|
||||
' Console
|
||||
@@ -97,7 +84,7 @@ package "turnstone/sdk/" <<Rectangle>> {
|
||||
|
||||
' Tool schemas
|
||||
package "turnstone/tools/" <<Rectangle>> {
|
||||
component [*.json\n18 tool schemas] as schemas <<artifact>>
|
||||
component [*.json\n19 tool schemas] as schemas <<artifact>>
|
||||
}
|
||||
|
||||
' Entry point dependencies
|
||||
@@ -146,35 +133,17 @@ mcp --> config
|
||||
registry --> config
|
||||
tools --> schemas
|
||||
|
||||
' MQ dependencies
|
||||
bridge --> protocol
|
||||
bridge --> broker
|
||||
bridge --> config
|
||||
client --> protocol
|
||||
client --> broker
|
||||
|
||||
' Sim dependencies
|
||||
simcli --> simcluster
|
||||
simcli --> simconfig
|
||||
simcli --> scenario
|
||||
simcluster --> simnode
|
||||
simcluster --> broker
|
||||
simcluster --> simmetrics
|
||||
simcluster --> simconfig
|
||||
simnode --> simengine
|
||||
simnode --> protocol
|
||||
simnode --> simconfig
|
||||
simnode --> simmetrics
|
||||
scenario --> broker
|
||||
scenario --> protocol
|
||||
scenario --> simconfig
|
||||
scenario --> simmetrics
|
||||
' Channel dependencies
|
||||
gateway --> discordbot
|
||||
gateway --> router
|
||||
discordbot --> sdkserver : HTTP + SSE
|
||||
router --> storage : channel_routes
|
||||
|
||||
' Console dependencies
|
||||
consoleserver --> collector
|
||||
consoleserver --> config
|
||||
consoleserver --> auth
|
||||
collector --> broker
|
||||
collector --> server : HTTP polling
|
||||
|
||||
' API dependencies
|
||||
serverspec --> openapi
|
||||
|
||||
@@ -12,7 +12,7 @@ interface "SessionUI" as SessionUI <<Protocol>> {
|
||||
+ on_content_token(text: str)
|
||||
+ on_stream_end()
|
||||
+ approve_tools(items: list) → (bool, str|None)
|
||||
+ on_tool_result(call_id: str, name: str, output: str)
|
||||
+ on_tool_result(call_id: str, name: str, output: str, *, is_error: bool = False)
|
||||
+ on_tool_output_chunk(call_id: str, chunk: str)
|
||||
+ on_status(usage: dict, ctx_window: int, effort: str)
|
||||
+ on_plan_review(content: str) → str
|
||||
@@ -84,6 +84,8 @@ class "OpenAIProvider" as OpenAIProv {
|
||||
in OpenAI format.
|
||||
Search models: web_search_options
|
||||
+ url_citation annotations.
|
||||
Extended cache: 24h retention
|
||||
for GPT-5.x (free).
|
||||
--
|
||||
core/providers/_openai.py
|
||||
}
|
||||
@@ -94,6 +96,8 @@ class "AnthropicProvider" as AnthropicProv {
|
||||
Adaptive + manual thinking.
|
||||
Native web search via
|
||||
web_search_20250305 server tool.
|
||||
Auto prompt caching via
|
||||
cache_control: ephemeral.
|
||||
Lazy anthropic SDK import.
|
||||
--
|
||||
core/providers/_anthropic.py
|
||||
@@ -246,13 +250,11 @@ class "MCPClientManager" as MCPMgr {
|
||||
|
||||
' ToolSearchManager
|
||||
class "ToolSearchManager" as ToolSearchMgr {
|
||||
- _all_tools: list[dict]
|
||||
- _always_on: list[dict]
|
||||
- _deferred: list[dict]
|
||||
- _expanded: dict[str, None]
|
||||
- _index: BM25Index
|
||||
--
|
||||
+ should_activate() → bool
|
||||
+ get_visible_tools() → list[dict]
|
||||
+ get_deferred_tools() → list[dict]
|
||||
+ get_expanded_names() → list[str]
|
||||
|
||||
@@ -133,7 +133,8 @@ group loop [while tool_calls present]
|
||||
note right of TP
|
||||
bash: on_tool_output_chunk(call_id, line)
|
||||
called per stdout line,
|
||||
then on_tool_result(call_id, name, output).
|
||||
then on_tool_result(call_id, name, output, is_error).
|
||||
is_error=True when execution failed.
|
||||
call_id routes chunks/results to correct
|
||||
tool div during parallel execution.
|
||||
Other tools: on_tool_result() only.
|
||||
|
||||
@@ -24,7 +24,7 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
:Dispatch to _prepare_{func_name}();
|
||||
|
||||
note right
|
||||
**Dispatch table (17 tools):**
|
||||
**Dispatch table (19 built-in + tool_search):**
|
||||
┌───────────────┬──────────────────┐
|
||||
│ Tool │ Needs Approval? │
|
||||
├───────────────┼──────────────────┤
|
||||
@@ -33,16 +33,19 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
│ write_file │ ✓ Yes │
|
||||
│ edit_file │ ✓ Yes │
|
||||
│ search │ ✗ Auto-approve │
|
||||
│ diff_file │ ✗ Auto-approve │
|
||||
│ math │ ✗ Auto-approve │
|
||||
│ man │ ✗ Auto-approve │
|
||||
│ web_fetch │ ✗ Auto-approve │
|
||||
│ web_search │ ✗ Auto-approve │
|
||||
│ tool_search │ ✗ Auto-approve │
|
||||
│ task │ ✓ Yes │
|
||||
│ plan │ ✓ Yes │
|
||||
│ task_agent │ ✓ Yes │
|
||||
│ plan_agent │ ✓ Yes │
|
||||
│ memory │ ✗ Auto-approve │
|
||||
│ recall │ ✗ Auto-approve │
|
||||
│ notify │ ✗ Auto-approve │
|
||||
│ watch │ ✓ create only │
|
||||
│ skill │ ✓ load only │
|
||||
│ read_resource │ ✓ Yes │
|
||||
│ use_prompt │ ✓ Yes │
|
||||
├───────────────┼──────────────────┤
|
||||
@@ -127,7 +130,7 @@ partition "Phase 3: Execute" #E3F2FD {
|
||||
:_truncate_output() on each result\n(max context_window × chars_per_token × 0.5 chars\ndefault: ~context_window × 2 chars);
|
||||
|
||||
:bash: ui.on_tool_output_chunk(call_id, line) per stdout line;
|
||||
:ui.on_tool_result(call_id, name, output) for each;
|
||||
:ui.on_tool_result(call_id, name, output, is_error) for each;
|
||||
|
||||
if (plan tool was executed?) then (yes)
|
||||
:ui.on_plan_review(output);
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Message Queue Protocol Types
|
||||
skinparam classAttributeIconSize 0
|
||||
skinparam packageStyle rectangle
|
||||
|
||||
package "Inbound Messages (Client → Bridge)" #FFF3E0 {
|
||||
|
||||
abstract class "InboundMessage" as IM {
|
||||
+ type: str
|
||||
+ correlation_id: str {auto: uuid4().hex[:12]}
|
||||
+ timestamp: float {auto: time.time()}
|
||||
--
|
||||
+ to_json() → str
|
||||
+ {static} from_json(raw) → InboundMessage
|
||||
}
|
||||
|
||||
class SendMessage {
|
||||
type = "send"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ message: str
|
||||
+ auto_approve: bool = False
|
||||
+ auto_approve_tools: list[str] = []
|
||||
+ name: str = ""
|
||||
+ target_node: str = ""
|
||||
}
|
||||
|
||||
class ApproveMessage {
|
||||
type = "approve"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ request_id: str
|
||||
+ approved: bool = True
|
||||
+ feedback: str | None
|
||||
+ always: bool = False
|
||||
}
|
||||
|
||||
class PlanFeedbackMessage {
|
||||
type = "plan_feedback"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ request_id: str
|
||||
+ feedback: str
|
||||
}
|
||||
|
||||
class CommandMessage {
|
||||
type = "command"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ command: str
|
||||
}
|
||||
|
||||
class CreateWorkstreamMessage {
|
||||
type = "create_workstream"
|
||||
--
|
||||
+ name: str = ""
|
||||
+ auto_approve: bool = False
|
||||
+ auto_approve_tools: list[str] = []
|
||||
+ target_node: str = ""
|
||||
+ initial_message: str = ""
|
||||
+ template: str = ""
|
||||
+ ws_template: str = ""
|
||||
}
|
||||
|
||||
class CloseWorkstreamMessage {
|
||||
type = "close_workstream"
|
||||
--
|
||||
+ ws_id: str
|
||||
}
|
||||
|
||||
class ListWorkstreamsMessage {
|
||||
type = "list_workstreams"
|
||||
}
|
||||
|
||||
class HealthMessage {
|
||||
type = "health"
|
||||
}
|
||||
|
||||
class ListNodesMessage {
|
||||
type = "list_nodes"
|
||||
}
|
||||
|
||||
class CancelMessage {
|
||||
type = "cancel"
|
||||
--
|
||||
+ ws_id: str
|
||||
}
|
||||
|
||||
IM <|-- SendMessage
|
||||
IM <|-- ApproveMessage
|
||||
IM <|-- PlanFeedbackMessage
|
||||
IM <|-- CommandMessage
|
||||
IM <|-- CreateWorkstreamMessage
|
||||
IM <|-- CloseWorkstreamMessage
|
||||
IM <|-- ListWorkstreamsMessage
|
||||
IM <|-- HealthMessage
|
||||
IM <|-- ListNodesMessage
|
||||
IM <|-- CancelMessage
|
||||
}
|
||||
|
||||
package "Outbound Events (Bridge → Client)" #E3F2FD {
|
||||
|
||||
abstract class "OutboundEvent" as OE {
|
||||
+ type: str
|
||||
+ ws_id: str
|
||||
+ correlation_id: str
|
||||
+ timestamp: float
|
||||
--
|
||||
+ to_json() → str
|
||||
+ {static} from_json(raw) → OutboundEvent
|
||||
}
|
||||
|
||||
package "Streaming" #BBDEFB {
|
||||
class ContentEvent {
|
||||
type = "content"
|
||||
+ text: str
|
||||
}
|
||||
class ReasoningEvent {
|
||||
type = "reasoning"
|
||||
+ text: str
|
||||
}
|
||||
class StreamEndEvent {
|
||||
type = "stream_end"
|
||||
}
|
||||
}
|
||||
|
||||
package "Tools" #C8E6C9 {
|
||||
class ToolInfoEvent {
|
||||
type = "tool_info"
|
||||
+ items: list
|
||||
}
|
||||
class ApprovalRequestEvent {
|
||||
type = "approval_request"
|
||||
+ items: list
|
||||
..
|
||||
correlation_id = request_id
|
||||
}
|
||||
class ToolOutputChunkEvent {
|
||||
type = "tool_output_chunk"
|
||||
+ call_id: str
|
||||
+ chunk: str
|
||||
}
|
||||
class ToolResultEvent {
|
||||
type = "tool_result"
|
||||
+ call_id: str
|
||||
+ name: str
|
||||
+ output: str
|
||||
}
|
||||
class PlanReviewEvent {
|
||||
type = "plan_review"
|
||||
+ content: str
|
||||
}
|
||||
}
|
||||
|
||||
package "Status" #FFF9C4 {
|
||||
class AckEvent {
|
||||
type = "ack"
|
||||
+ status: str
|
||||
+ detail: str
|
||||
}
|
||||
class StatusEvent {
|
||||
type = "status"
|
||||
+ prompt_tokens: int
|
||||
+ completion_tokens: int
|
||||
+ total_tokens: int
|
||||
+ context_window: int
|
||||
+ pct: float
|
||||
+ effort: str
|
||||
}
|
||||
class StateChangeEvent {
|
||||
type = "state_change"
|
||||
+ state: str
|
||||
}
|
||||
class TurnCompleteEvent {
|
||||
type = "turn_complete"
|
||||
+ content: str
|
||||
}
|
||||
}
|
||||
|
||||
package "Lifecycle" #F8BBD0 {
|
||||
class WorkstreamCreatedEvent {
|
||||
type = "ws_created"
|
||||
+ name: str
|
||||
}
|
||||
class WorkstreamClosedEvent {
|
||||
type = "ws_closed"
|
||||
}
|
||||
class WorkstreamListEvent {
|
||||
type = "ws_list"
|
||||
+ workstreams: list
|
||||
}
|
||||
class WorkstreamRenameEvent {
|
||||
type = "ws_rename"
|
||||
+ name: str
|
||||
}
|
||||
}
|
||||
|
||||
package "System" #E0E0E0 {
|
||||
class HealthResponseEvent {
|
||||
type = "health_response"
|
||||
+ data: dict
|
||||
}
|
||||
class ErrorEvent {
|
||||
type = "error"
|
||||
+ message: str
|
||||
}
|
||||
class InfoEvent {
|
||||
type = "info"
|
||||
+ message: str
|
||||
}
|
||||
class NodeListEvent {
|
||||
type = "node_list"
|
||||
+ nodes: list
|
||||
}
|
||||
class ClusterStateEvent {
|
||||
type = "cluster_state"
|
||||
+ state: str
|
||||
+ node_id: str
|
||||
+ tokens: int
|
||||
+ context_ratio: float
|
||||
+ activity: str
|
||||
+ activity_state: str
|
||||
}
|
||||
}
|
||||
|
||||
OE <|-- ContentEvent
|
||||
OE <|-- ReasoningEvent
|
||||
OE <|-- StreamEndEvent
|
||||
OE <|-- ToolInfoEvent
|
||||
OE <|-- ApprovalRequestEvent
|
||||
OE <|-- ToolResultEvent
|
||||
OE <|-- PlanReviewEvent
|
||||
OE <|-- AckEvent
|
||||
OE <|-- StatusEvent
|
||||
OE <|-- StateChangeEvent
|
||||
OE <|-- TurnCompleteEvent
|
||||
OE <|-- WorkstreamCreatedEvent
|
||||
OE <|-- WorkstreamClosedEvent
|
||||
OE <|-- WorkstreamListEvent
|
||||
OE <|-- WorkstreamRenameEvent
|
||||
OE <|-- HealthResponseEvent
|
||||
OE <|-- ErrorEvent
|
||||
OE <|-- InfoEvent
|
||||
OE <|-- NodeListEvent
|
||||
OE <|-- ClusterStateEvent
|
||||
}
|
||||
|
||||
note bottom of IM
|
||||
**Deserialization**: Strict type-dispatch via _INBOUND_REGISTRY.
|
||||
Unknown type raises ValueError.
|
||||
end note
|
||||
|
||||
note bottom of OE
|
||||
**Deserialization**: Lenient type-dispatch via _OUTBOUND_REGISTRY.
|
||||
Unknown type falls back to base OutboundEvent.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -1,105 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Multi-Node Message Routing
|
||||
|
||||
skinparam sequenceArrowThickness 1.5
|
||||
|
||||
participant "TurnstoneClient" as Client
|
||||
collections "Redis" as Redis
|
||||
participant "Bridge-A\n(node_id: nodeA)" as BridgeA
|
||||
participant "Bridge-B\n(node_id: nodeB)" as BridgeB
|
||||
participant "Server-A" as ServerA
|
||||
|
||||
== Scenario A: New Message — No Workstream Affinity ==
|
||||
|
||||
Client -> Redis : RPUSH turnstone:inbound\n{type:"send", message:"...", ws_id:""}
|
||||
note right of Redis : Shared queue — any bridge can pick up
|
||||
|
||||
BridgeA -> Redis : BLPOP [turnstone:inbound:nodeA,\n turnstone:inbound]
|
||||
Redis --> BridgeA : SendMessage (from shared queue)
|
||||
|
||||
BridgeA -> ServerA : POST /v1/api/workstreams/new\n{name:"", auto_approve:false}
|
||||
ServerA --> BridgeA : {ws_id:"abc12345", name:"ws-abc1"}
|
||||
|
||||
BridgeA -> Redis : SET turnstone:ws:abc12345 "nodeA"
|
||||
note right : Register workstream ownership
|
||||
|
||||
BridgeA -> ServerA : GET /v1/api/events?ws_id=abc12345
|
||||
note right : Start per-WS SSE thread
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nWorkstreamCreatedEvent
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:cluster\nClusterStateEvent(ws_id, state:"idle", node_id:"nodeA")
|
||||
|
||||
BridgeA -> ServerA : POST /v1/api/send\n{message:"...", ws_id:"abc12345"}
|
||||
ServerA --> BridgeA : {status:"ok"}
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nAckEvent(status:"ok")
|
||||
|
||||
... SSE events flow: content, tool_output_chunk, tool_result, status, state_change ...
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nContentEvent, ToolResultEvent, ...
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nStateChangeEvent(state:"idle", content:"...")
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nTurnCompleteEvent(content:"...")
|
||||
|
||||
== Scenario B: Directed Message to Specific Node ==
|
||||
|
||||
Client -> Redis : RPUSH turnstone:inbound:nodeB\n{type:"send", target_node:"nodeB", ...}
|
||||
note right : Per-node queue — only nodeB picks up
|
||||
|
||||
BridgeB -> Redis : BLPOP [turnstone:inbound:nodeB,\n turnstone:inbound]
|
||||
Redis --> BridgeB : SendMessage (from per-node queue, priority)
|
||||
|
||||
note right of BridgeB : Process locally on nodeB
|
||||
|
||||
== Scenario C: Re-routing (Lands on Wrong Node) ==
|
||||
|
||||
Client -> Redis : RPUSH turnstone:inbound\n{type:"send", ws_id:"abc12345"}
|
||||
|
||||
BridgeB -> Redis : BLPOP [..., turnstone:inbound]
|
||||
Redis --> BridgeB : SendMessage (ws_id: abc12345)
|
||||
|
||||
BridgeB -> Redis : GET turnstone:ws:abc12345
|
||||
Redis --> BridgeB : "nodeA"
|
||||
|
||||
note right of BridgeB : Owner is nodeA, not me — re-route
|
||||
|
||||
BridgeB -> Redis : RPUSH turnstone:inbound:nodeA\n(re-routed message)
|
||||
|
||||
BridgeA -> Redis : BLPOP [turnstone:inbound:nodeA, ...]
|
||||
Redis --> BridgeA : SendMessage (from per-node queue)
|
||||
note right of BridgeA : Process locally — I own this workstream
|
||||
|
||||
== Scenario D: Approval via Response Queue ==
|
||||
|
||||
BridgeA <- ServerA : SSE: {type:"approve_request", items:[...]}
|
||||
|
||||
note right of BridgeA
|
||||
Bridge checks auto-approve:
|
||||
1. _ws_auto_approve[ws_id]? → auto
|
||||
2. All tools in safe set? → auto
|
||||
(read_file, search, man,
|
||||
memory, recall)
|
||||
3. Otherwise → manual approval
|
||||
end note
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nApprovalRequestEvent(correlation_id: req_xyz)
|
||||
|
||||
Client <- Redis : (subscribed) ApprovalRequestEvent
|
||||
|
||||
Client -> Redis : RPUSH turnstone:resp:req_xyz\nApproveMessage(approved:true)
|
||||
note right : Response queue — bypasses inbound queue
|
||||
|
||||
BridgeA -> Redis : BLPOP turnstone:resp:req_xyz\n(spawned approval thread, timeout 300s)
|
||||
Redis --> BridgeA : ApproveMessage
|
||||
|
||||
BridgeA -> ServerA : POST /v1/api/approve\n{approved:true, ws_id:"abc12345"}
|
||||
|
||||
== Heartbeat (continuous) ==
|
||||
|
||||
BridgeA -> Redis : SET turnstone:node:nodeA\n{server_url, started} EX 60
|
||||
note right : Every 30s — TTL 60s
|
||||
|
||||
BridgeB -> Redis : SET turnstone:node:nodeB\n{server_url, started} EX 60
|
||||
|
||||
@enduml
|
||||
@@ -1,98 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Redis Key Schema
|
||||
|
||||
skinparam component {
|
||||
BackgroundColor<<LIST>> #BBDEFB
|
||||
BackgroundColor<<STRING>> #C8E6C9
|
||||
BackgroundColor<<PUBSUB>> #FFE0B2
|
||||
}
|
||||
|
||||
skinparam note {
|
||||
BackgroundColor #FAFAFA
|
||||
}
|
||||
|
||||
package "Queues (Redis LIST)" #E3F2FD {
|
||||
component [**turnstone:inbound**\n\nShared command queue.\nAny bridge can consume.\n\nOps: RPUSH (write), BLPOP (read)] as inbound <<LIST>>
|
||||
|
||||
component [**turnstone:inbound:{node_id}**\n\nPer-node directed queue.\nPriority over shared queue.\n\nOps: RPUSH (write), BLPOP (read)] as inbound_node <<LIST>>
|
||||
|
||||
component [**turnstone:resp:{request_id}**\n\nPer-request response queue.\nFor approval / plan feedback.\nTTL: 600s\n\nOps: RPUSH + EXPIRE (write), BLPOP (read)] as resp <<LIST>>
|
||||
}
|
||||
|
||||
package "Routing (Redis STRING)" #E8F5E9 {
|
||||
component [**turnstone:ws:{ws_id}**\n\nWorkstream → node ownership.\nValue: node_id string.\nNo TTL.\n\nOps: SET, GET, DEL] as ws_owner <<STRING>>
|
||||
|
||||
component [**turnstone:node:{node_id}**\n\nNode heartbeat + metadata.\nValue: JSON {server_url, started, ...}\nTTL: 60s (refreshed every 30s)\n\nOps: SET with EX, GET, SCAN] as node_hb <<STRING>>
|
||||
}
|
||||
|
||||
package "Event Channels (Redis PUBSUB)" #FFF3E0 {
|
||||
component [**turnstone:events:global**\n\nGlobal event broadcast.\nAll state changes, ws lifecycle.\n\nOps: PUBLISH, SUBSCRIBE] as evt_global <<PUBSUB>>
|
||||
|
||||
component [**turnstone:events:{ws_id}**\n\nPer-workstream events.\nContent, tools, status.\n\nOps: PUBLISH, SUBSCRIBE] as evt_ws <<PUBSUB>>
|
||||
|
||||
component [**turnstone:events:cluster**\n\nCluster-wide state changes.\nUsed by Console dashboard.\n\nOps: PUBLISH, SUBSCRIBE] as evt_cluster <<PUBSUB>>
|
||||
}
|
||||
|
||||
' Readers / Writers
|
||||
|
||||
actor "TurnstoneClient" as client
|
||||
actor "Bridge" as bridge
|
||||
actor "SimNode" as sim
|
||||
actor "Console\nCollector" as console
|
||||
actor "Scenario\n(injector)" as scenario
|
||||
|
||||
' Queue interactions
|
||||
client --> inbound : RPUSH\n(send commands)
|
||||
client --> inbound_node : RPUSH\n(directed)
|
||||
scenario --> inbound : RPUSH\n(inject load)
|
||||
scenario --> inbound_node : RPUSH\n(directed scenario)
|
||||
bridge --> inbound : BLPOP\n(consume)
|
||||
bridge --> inbound_node : BLPOP\n(priority)
|
||||
bridge --> inbound_node : RPUSH\n(re-route)
|
||||
sim --> inbound_node : BLPOP\n(via dispatcher)
|
||||
|
||||
client --> resp : RPUSH\n(approval response)
|
||||
bridge --> resp : BLPOP\n(wait for approval)
|
||||
|
||||
' Routing interactions
|
||||
bridge --> ws_owner : SET / GET / DEL
|
||||
client --> ws_owner : GET\n(route lookup)
|
||||
sim --> ws_owner : SET / DEL
|
||||
|
||||
bridge --> node_hb : SET with EX\n(heartbeat)
|
||||
sim --> node_hb : SET with EX\n(heartbeat)
|
||||
console --> node_hb : SCAN + GET\n(discovery)
|
||||
client --> node_hb : SCAN + GET\n(list_nodes)
|
||||
|
||||
' Pub/sub interactions
|
||||
bridge --> evt_global : PUBLISH
|
||||
bridge --> evt_ws : PUBLISH
|
||||
bridge --> evt_cluster : PUBLISH
|
||||
client --> evt_global : SUBSCRIBE
|
||||
client --> evt_ws : SUBSCRIBE
|
||||
sim --> evt_global : PUBLISH
|
||||
sim --> evt_ws : PUBLISH
|
||||
sim --> evt_cluster : PUBLISH
|
||||
console --> evt_cluster : SUBSCRIBE
|
||||
|
||||
note bottom of inbound
|
||||
**BLPOP priority**: Bridges call
|
||||
BLPOP [per-node, shared] so the
|
||||
per-node queue is always checked first.
|
||||
end note
|
||||
|
||||
note bottom of resp
|
||||
**Bypasses inbound queue**: Approval
|
||||
responses go directly to the response
|
||||
queue, not through the inbound queue.
|
||||
Auto-cleaned after 600s TTL.
|
||||
end note
|
||||
|
||||
note bottom of evt_cluster
|
||||
**ClusterStateEvent** includes node_id,
|
||||
tokens, and context_ratio — enriched
|
||||
data not available on the global channel.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -40,12 +40,23 @@ running --> error : Exception during\ntool execution
|
||||
|
||||
error --> thinking : New send() call\n_emit_state("thinking")
|
||||
|
||||
thinking --> idle : cancel() called\n_emit_state("idle")
|
||||
thinking --> idle : cancel() called\nstream aborted\n_emit_state("idle")
|
||||
|
||||
running --> idle : cancel() called\n_emit_state("idle")
|
||||
|
||||
attention --> idle : cancel() unblocks\napproval/plan wait\n_emit_state("idle")
|
||||
|
||||
note left of idle
|
||||
**Cancel escalation:**
|
||||
1. **Cooperative**: cancel() sets event + closes
|
||||
SDK stream → worker exits at next checkpoint
|
||||
2. **Force**: force=true abandons the worker
|
||||
thread, emits stream_end immediately.
|
||||
Orphaned thread still kills subprocesses
|
||||
but skips message mutations (generation
|
||||
counter prevents stale writes).
|
||||
end note
|
||||
|
||||
note right of thinking
|
||||
**Emitted via:**
|
||||
session._emit_state(state)
|
||||
@@ -53,7 +64,7 @@ note right of thinking
|
||||
|
||||
**Propagation:**
|
||||
• WebUI → global SSE queue (ws_state)
|
||||
• Bridge → PUBLISH to global + cluster channels
|
||||
• Console → HTTP polling picks up state
|
||||
• CLI → WorkstreamManager.set_state()
|
||||
end note
|
||||
|
||||
@@ -61,27 +72,8 @@ note left of attention
|
||||
**Blocking mechanisms:**
|
||||
• TerminalUI: input() prompt
|
||||
• WebUI: threading.Event.wait()
|
||||
• Bridge: BLPOP on response queue
|
||||
• ChannelBot: SSE event + Discord button
|
||||
• NullUI: auto-approve (never reaches)
|
||||
end note
|
||||
|
||||
state "SimWorkstream (simplified)" as sim_group {
|
||||
state "sim_idle" as si <<idle>>
|
||||
state "sim_thinking" as st <<thinking>>
|
||||
state "sim_running" as sr <<running>>
|
||||
state "sim_error" as se <<error>>
|
||||
|
||||
[*] --> si
|
||||
si --> st : process_turn() called
|
||||
st --> sr : Tool calls generated
|
||||
sr --> st : More rounds
|
||||
st --> si : No tools / max rounds
|
||||
st --> se : Uncaught exception
|
||||
}
|
||||
|
||||
note right of sim_group
|
||||
SimWorkstream has no ATTENTION state —
|
||||
tool approval is not simulated.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Simulator Architecture
|
||||
|
||||
skinparam component {
|
||||
BackgroundColor<<cluster>> #E1BEE7
|
||||
BackgroundColor<<node>> #CE93D8
|
||||
BackgroundColor<<engine>> #F3E5F5
|
||||
BackgroundColor<<scenario>> #FFF3E0
|
||||
BackgroundColor<<metrics>> #E8F5E9
|
||||
BackgroundColor<<redis>> #FFCDD2
|
||||
}
|
||||
|
||||
package "SimCluster" as cluster <<cluster>> {
|
||||
|
||||
component [**ThreadPoolExecutor**\nmax_workers=64\n(blocking Redis ops)] as executor <<cluster>>
|
||||
|
||||
component [**redis.ConnectionPool**\nmax_connections=64\ndecode_responses=True\n(shared across all nodes)] as pool <<redis>>
|
||||
|
||||
package "InboundDispatchers" {
|
||||
component [**Dispatcher 0**\nnodes 0-49] as d0
|
||||
component [**Dispatcher 1**\nnodes 50-99] as d1
|
||||
component [**...**\n(ceil(N/50) total)] as dn
|
||||
|
||||
note bottom of d0
|
||||
Each dispatcher calls BLPOP on a single Redis
|
||||
connection for up to 50 node queues + shared queue.
|
||||
Keys: [prefix:inbound:sim-0000, ..., prefix:inbound]
|
||||
Per-node keys have BLPOP priority over shared.
|
||||
end note
|
||||
}
|
||||
|
||||
package "SimNodes (N instances)" {
|
||||
component [**SimNode sim-0000**] as n0 <<node>>
|
||||
component [**SimNode sim-0001**] as n1 <<node>>
|
||||
component [**...**] as nn <<node>>
|
||||
|
||||
component [**SimEngine**\n(per node, seeded RNG)\n\nLLM simulation:\n gaussian(μ=2s, σ=0.5s) latency\n gaussian(μ=200, σ=50) tokens\n random word content\n P(tool_calls) = 0.6/0.3\n\nTool simulation:\n gaussian(μ=0.5s, σ=0.2s) latency\n P(failure) = 0.02] as engine <<engine>>
|
||||
|
||||
component [**SimWorkstream**\n(0..max_ws per node)\n\nState: idle→thinking→running→idle\nToken accounting: word_count × 3\nContent: 8-chunk streaming] as ws <<node>>
|
||||
}
|
||||
|
||||
component [**MetricsCollector**\n(thread-safe, shared)\n\nTracks: turn latencies,\nthroughput, utilization,\nerrors, node kills] as metrics <<metrics>>
|
||||
}
|
||||
|
||||
package "Scenarios (5 workload patterns)" <<scenario>> {
|
||||
component [**SteadyState**\nConstant rate:\n1/mps interval\nfor duration secs] as steady <<scenario>>
|
||||
|
||||
component [**Burst**\nburst_size messages\nas fast as possible\nthen wait] as burst <<scenario>>
|
||||
|
||||
component [**NodeFailure**\nSteadyState + periodic\nnode kills (up to N/2)] as failure <<scenario>>
|
||||
|
||||
component [**Directed**\nMessages targeted to\nspecific nodes via\ntarget_node field] as directed <<scenario>>
|
||||
|
||||
component [**Lifecycle**\n3 phases:\n1. Create workstreams\n2. Send messages\n3. Close half] as lifecycle <<scenario>>
|
||||
}
|
||||
|
||||
database "Redis" as redis <<redis>>
|
||||
|
||||
' Scenario -> Redis
|
||||
steady --> redis : RPUSH prefix:inbound\n(SendMessage)
|
||||
burst --> redis : RPUSH prefix:inbound\n(burst)
|
||||
failure --> redis : RPUSH prefix:inbound
|
||||
directed --> redis : RPUSH prefix:inbound:{node}\n(directed)
|
||||
lifecycle --> redis : RPUSH prefix:inbound\n(Create/Send/Close)
|
||||
|
||||
' Dispatchers -> Redis -> Nodes
|
||||
d0 --> redis : BLPOP [per-node..., shared]
|
||||
d1 --> redis : BLPOP [per-node..., shared]
|
||||
d0 --> n0 : handle_message(raw)
|
||||
d0 --> n1 : handle_message(raw)
|
||||
|
||||
' Nodes internal
|
||||
n0 --> engine : simulate_llm_response()\nsimulate_tool_execution()
|
||||
n0 --> ws : process_turn()
|
||||
|
||||
' Nodes -> Redis (events)
|
||||
n0 --> redis : PUBLISH prefix:events:global\n(StateChangeEvent)
|
||||
n0 --> redis : PUBLISH prefix:events:{ws_id}\n(ContentEvent, ToolResultEvent, ...)
|
||||
n0 --> redis : PUBLISH prefix:events:cluster\n(ClusterStateEvent)
|
||||
n0 --> redis : SET prefix:node:sim-0000\nEX 60 (heartbeat)
|
||||
n0 --> redis : SET prefix:ws:{ws_id}\n(ownership)
|
||||
|
||||
' Shared pool
|
||||
n0 ..> pool : PooledBroker\n(shared connection)
|
||||
n1 ..> pool : PooledBroker
|
||||
d0 ..> pool
|
||||
d0 ..> executor : asyncio.to_thread()
|
||||
|
||||
' Metrics
|
||||
ws --> metrics : record_turn(ws_id, node_id, latency)
|
||||
steady --> metrics : record_inject()
|
||||
burst --> metrics : record_inject()
|
||||
directed --> metrics : record_inject()
|
||||
lifecycle --> metrics : record_inject()
|
||||
cluster --> metrics : record_node_kill(node_id)
|
||||
cluster --> metrics : snapshot_utilization()\n(every metrics_interval)
|
||||
|
||||
note bottom of cluster
|
||||
**SimConfig** controls all simulation parameters:
|
||||
num_nodes, max_ws_per_node, redis settings,
|
||||
llm_latency_mean/stddev, tool_failure_rate,
|
||||
scenario, duration, messages_per_second, seed
|
||||
end note
|
||||
|
||||
note right of redis
|
||||
Simulator uses **real Redis** —
|
||||
not a mock. Console dashboard
|
||||
can monitor a running simulation
|
||||
via the same cluster channel.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -7,110 +7,79 @@ skinparam sequenceArrowThickness 1.5
|
||||
participant "Browser" as Browser
|
||||
participant "Console\nStarlette App" as Server
|
||||
participant "ClusterCollector" as CC
|
||||
collections "Redis" as Redis
|
||||
participant "Node-A Bridge" as BridgeA
|
||||
participant "Node-A\n(real server)" as NodeA
|
||||
participant "Node-B\n(sim node)" as NodeB
|
||||
participant "Node-A\n(server)" as NodeA
|
||||
participant "Node-B\n(server)" as NodeB
|
||||
|
||||
== Thread 1: Cluster Event Subscriber (real-time) ==
|
||||
== Thread 1: Node Discovery (every 60s) ==
|
||||
|
||||
CC -> Redis : SUBSCRIBE turnstone:events:cluster
|
||||
activate CC #E1BEE7
|
||||
|
||||
Redis --> CC : ClusterStateEvent\n{ws_id, state:"thinking",\nnode_id:"nodeA", tokens:500,\ncontext_ratio:0.05}
|
||||
CC -> CC : Update NodeSnapshot["nodeA"]\n.workstreams["ws123"].state = "thinking"
|
||||
CC -> CC : _fanout(event) → all SSE listeners
|
||||
|
||||
Redis --> CC : {"type":"ws_created",\nws_id:"ws456", name:"task-1",\nnode_id:"sim-0003"}
|
||||
CC -> CC : Add workstream to\nNodeSnapshot["sim-0003"]
|
||||
CC -> CC : _fanout(event)
|
||||
|
||||
Redis --> CC : ClusterStateEvent\n{ws_id:"ws456", state:"idle"}
|
||||
CC -> CC : Update workstream state
|
||||
|
||||
note right of CC
|
||||
Handles: cluster_state,
|
||||
ws_created, ws_closed, ws_rename
|
||||
|
||||
Thread runs continuously.
|
||||
All updates are thread-safe
|
||||
via threading.Lock.
|
||||
end note
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 2: Node Discovery (every 15s) ==
|
||||
|
||||
CC -> Redis : SCAN 0 MATCH turnstone:node:*
|
||||
activate CC #B2EBF2
|
||||
Redis --> CC : [turnstone:node:nodeA, turnstone:node:sim-0003, ...]
|
||||
|
||||
loop for each discovered key
|
||||
CC -> Redis : GET turnstone:node:{id}
|
||||
Redis --> CC : JSON: {server_url, started, max_ws, sim:true/false}
|
||||
end
|
||||
|
||||
CC -> CC : Create new NodeSnapshot\nfor newly discovered nodes
|
||||
CC -> CC : Remove NodeSnapshot\nfor disappeared nodes
|
||||
|
||||
CC -> CC : _fanout({type: "node_joined", ...})\n_fanout({type: "node_lost", ...})
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 3: HTTP Polling (every 10s, real nodes only) ==
|
||||
|
||||
CC -> CC : Filter nodes where\nserver_url.startswith("http")
|
||||
CC -> CC : list_services("server",\nmax_age_seconds=120)
|
||||
activate CC #C8E6C9
|
||||
|
||||
note right of CC
|
||||
sim:// nodes are SKIPPED.
|
||||
Their data comes exclusively
|
||||
from the cluster event channel.
|
||||
end note
|
||||
|
||||
CC -> NodeA : GET /v1/api/dashboard
|
||||
activate NodeA
|
||||
NodeA --> CC : {workstreams: [...],\naggregate: {total_tokens, ...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> NodeA : GET /health
|
||||
activate NodeA
|
||||
NodeA --> CC : {status:"ok", version:"0.3.0",\nmodel:"...", workstreams:{...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> CC : Diff old vs new workstream IDs
|
||||
CC -> CC : Replace NodeSnapshot["nodeA"]\n.workstreams, .health, .aggregate
|
||||
CC -> CC : _fanout(ws_created) for\nnewly appeared workstreams
|
||||
CC -> CC : _fanout(ws_closed) for\nremoved workstreams
|
||||
|
||||
note right of CC
|
||||
Poll-diff fanout ensures
|
||||
browser SSE clients learn
|
||||
about workstreams that
|
||||
appeared without a real-time
|
||||
cluster event (e.g. bridge
|
||||
startup recovery).
|
||||
end note
|
||||
|
||||
CC -x NodeB : (SKIPPED: sim:// URL)
|
||||
CC -> CC : New node? → spawn SSE task\nLost node? → cancel SSE task
|
||||
CC -> CC : _fanout(node_joined)\n_fanout(node_lost)
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 2: SSE Manager (asyncio event loop) ==
|
||||
|
||||
note over CC
|
||||
Single asyncio event loop multiplexes
|
||||
one persistent SSE connection per node.
|
||||
Scales to 1000+ nodes.
|
||||
end note
|
||||
|
||||
CC -> NodeA : GET /v1/api/events/global\n?expected_node_id=nodeA
|
||||
activate NodeA
|
||||
activate CC #BBDEFB
|
||||
|
||||
NodeA --> CC : data: {"type":"node_snapshot",\n"node_id":"nodeA",\n"workstreams":[...],\n"health":{...},\n"aggregate":{...}}
|
||||
|
||||
note right of CC
|
||||
Snapshot populates NodeSnapshot
|
||||
in-memory state. Reconciles
|
||||
against stale data (emits
|
||||
ws_created/ws_closed diffs).
|
||||
end note
|
||||
|
||||
loop real-time delta events
|
||||
NodeA --> CC : data: {"type":"ws_state",\n"ws_id":"ws1","state":"running"}
|
||||
CC -> CC : Update NodeSnapshot\n_fanout(cluster_state)
|
||||
end
|
||||
|
||||
alt health transition
|
||||
NodeA --> CC : data: {"type":"health_changed",\n"circuit_state":"open"}
|
||||
CC -> CC : Update node.health
|
||||
end
|
||||
|
||||
alt periodic aggregate (every 10s)
|
||||
NodeA --> CC : data: {"type":"aggregate",\n"total_tokens":50000}
|
||||
CC -> CC : Update node.aggregate
|
||||
end
|
||||
|
||||
deactivate CC
|
||||
deactivate NodeA
|
||||
|
||||
alt SSE disconnect
|
||||
CC -> CC : Mark node unreachable\nReconnect with backoff\n(1s → 30s cap)
|
||||
end
|
||||
|
||||
alt identity mismatch (409 or snapshot node_id differs)
|
||||
CC -> CC : Mark node unreachable\nStop reconnecting to this URL
|
||||
end
|
||||
|
||||
== Browser SSE Stream ==
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/events
|
||||
activate Server
|
||||
|
||||
Server -> CC : get_snapshot()
|
||||
Server -> CC : get_snapshot_and_register(queue)
|
||||
note right : Atomic: snapshot + listener\nregistration under both locks\n→ no event gap
|
||||
CC --> Server : ClusterSnapshot\n(full current state)
|
||||
|
||||
Server -> CC : register_listener(queue)
|
||||
note right : Per-client queue.Queue(maxsize=500)\nSSE via EventSourceResponse + run_in_executor()
|
||||
|
||||
Server -> Browser : data: {"type":"snapshot",...}\n(full state as first SSE event)
|
||||
|
||||
loop continuous (incremental updates)
|
||||
CC -> Server : event via listener queue\n(from any of the 3 threads)
|
||||
CC -> Server : event via listener queue\n(from SSE manager thread)
|
||||
Server -> Browser : data: {"type":"cluster_state",...}\n\n
|
||||
end
|
||||
|
||||
@@ -133,20 +102,20 @@ Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/overview
|
||||
Server -> CC : get_overview()
|
||||
CC --> Server : {nodes: 10, workstreams: 47,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.3.0"]}
|
||||
CC --> Server : {nodes: 2, workstreams: 12,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.9.7"]}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/nodes?sort=activity
|
||||
Server -> CC : get_nodes(sort_by="activity")
|
||||
CC --> Server : {nodes: [...], total: 10}
|
||||
CC --> Server : {nodes: [...], total: 2}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/workstreams\n?state=running&node=sim-0003
|
||||
Server -> CC : get_workstreams(state="running",\nnode="sim-0003")
|
||||
Browser -> Server : GET /v1/api/cluster/workstreams\n?state=running&node=nodeA
|
||||
Server -> CC : get_workstreams(state="running",\nnode="nodeA")
|
||||
CC --> Server : {workstreams: [...], total: 5,\npage: 1, per_page: 50, pages: 1}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
== Workstream Creation (via MQ) ==
|
||||
== Workstream Creation (via Console proxy) ==
|
||||
|
||||
Browser -> Server : POST /v1/api/cluster/workstreams/new\n{node_id:"nodeA", name:"new-task"}
|
||||
activate Server #FFECB3
|
||||
@@ -154,33 +123,22 @@ activate Server #FFECB3
|
||||
Server -> CC : _pick_best_node() or\nget_node_detail(node_id)
|
||||
CC --> Server : node validated
|
||||
|
||||
Server -> Server : Build CreateWorkstreamMessage\n{target_node:"nodeA", name:"new-task"}
|
||||
Server -> NodeA : POST http://nodeA:8080/v1/api/workstreams/new\n{name:"new-task", user_id: from auth_result}
|
||||
activate NodeA
|
||||
NodeA --> Server : {ws_id:"ws789", name:"new-task",\nnode_url:"http://nodeA:8080"}
|
||||
deactivate NodeA
|
||||
|
||||
Server -> Redis : RPUSH turnstone:inbound:nodeA\n(directed queue)
|
||||
Server --> Browser : {status:"ok", correlation_id:"abc",\ntarget_node:"nodeA"}
|
||||
Server --> Browser : {status:"ok", ws_id:"ws789",\nnode_url:"http://nodeA:8080"}
|
||||
deactivate Server
|
||||
|
||||
note right of Redis
|
||||
Bridge on Node-A picks up the
|
||||
message from its directed queue,
|
||||
POSTs to /v1/api/workstreams/new,
|
||||
registers ownership, publishes
|
||||
ws_created to cluster channel.
|
||||
note right of Server
|
||||
Console proxies the create request
|
||||
directly to the target node via HTTP.
|
||||
The response includes node_url so the
|
||||
client can establish a direct SSE
|
||||
connection for the data plane.
|
||||
end note
|
||||
|
||||
Redis --> BridgeA : BLPOP turnstone:inbound:nodeA
|
||||
activate BridgeA
|
||||
BridgeA -> NodeA : POST /v1/api/workstreams/new\n{name:"new-task"}
|
||||
NodeA --> BridgeA : {ws_id:"ws789", name:"new-task"}
|
||||
BridgeA -> Redis : SET turnstone:ws:ws789 = nodeA
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:cluster\n{type:"ws_created", ws_id:"ws789",\nnode_id:"nodeA", name:"new-task"}
|
||||
deactivate BridgeA
|
||||
|
||||
Redis --> CC : ws_created event
|
||||
CC -> CC : Add workstream to\nNodeSnapshot["nodeA"]
|
||||
CC -> CC : _fanout(event)
|
||||
Server -> Browser : SSE: data: {"type":"ws_created",...}
|
||||
|
||||
== Reverse Proxy (server UI through console port) ==
|
||||
|
||||
Browser -> Server : GET /node/nodeA/
|
||||
|
||||
@@ -14,98 +14,83 @@ node "Docker Host" as host {
|
||||
|
||||
frame "turnstone-net (bridge network)" as net {
|
||||
|
||||
node "redis" <<redis:7.4-alpine>> as redis_node {
|
||||
component [Redis Server\nport 6379] as redis
|
||||
note bottom of redis
|
||||
Healthcheck: redis-cli ping
|
||||
Volume: redis-data
|
||||
end note
|
||||
}
|
||||
|
||||
node "server" <<turnstone image>> as server_node {
|
||||
component [turnstone-server\nport 8080] as server
|
||||
note bottom of server
|
||||
Command: turnstone-server
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
Depends: redis (healthy)
|
||||
Volume: turnstone-data
|
||||
(/data)
|
||||
end note
|
||||
}
|
||||
|
||||
node "bridge ×N" <<turnstone image>> as bridge_node {
|
||||
component [turnstone-bridge] as bridge
|
||||
note bottom of bridge
|
||||
Command: turnstone-bridge
|
||||
--server-url http://server:8080
|
||||
--redis-host redis
|
||||
Depends: server + redis
|
||||
Scalable: --scale bridge=N
|
||||
node_id: auto from hostname
|
||||
end note
|
||||
}
|
||||
|
||||
node "console" <<turnstone image>> as console_node {
|
||||
component [turnstone-console\nport 8090] as console
|
||||
note bottom of console
|
||||
Command: turnstone-console
|
||||
--redis-host redis
|
||||
--port 8090
|
||||
Depends: redis
|
||||
Depends: server
|
||||
Hash-ring router for
|
||||
multi-node clusters
|
||||
end note
|
||||
}
|
||||
|
||||
node "sim (profile: sim)" <<turnstone image>> as sim_node {
|
||||
component [turnstone-sim] as sim
|
||||
note bottom of sim
|
||||
Command: turnstone-sim
|
||||
--redis-host redis
|
||||
--nodes 100
|
||||
--scenario steady
|
||||
Depends: redis
|
||||
Optional: only with
|
||||
--profile sim
|
||||
node "postgres (profile: production)" <<pgautoupgrade>> as pg_node {
|
||||
component [PostgreSQL\nport 5432] as postgres
|
||||
note bottom of postgres
|
||||
Healthcheck: pg_isready
|
||||
Volume: postgres-data
|
||||
Required for cluster
|
||||
and production profiles
|
||||
end note
|
||||
}
|
||||
|
||||
node "pgbouncer (optional)" <<bitnami/pgbouncer>> as pgb_node {
|
||||
component [PgBouncer\nport 6432] as pgbouncer
|
||||
note bottom of pgbouncer
|
||||
pool_mode: transaction
|
||||
Recommended for clusters
|
||||
> 50 nodes
|
||||
See docs/pgbouncer.md
|
||||
end note
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
actor "Browser\nUser" as browser
|
||||
actor "MQ Client" as mqclient
|
||||
actor "SDK /\nAPI Client" as apiclient
|
||||
|
||||
' External connections
|
||||
browser --> server : HTTP + SSE\nport 8080
|
||||
browser --> console : HTTP + SSE\nport 8090
|
||||
mqclient --> redis : Redis protocol\nport 6379
|
||||
apiclient --> server : HTTP + SSE\nport 8080
|
||||
|
||||
' Internal connections
|
||||
server --> redis : Redis protocol\n(6379)
|
||||
server --> llm_api : OpenAI API\n(HTTPS/HTTP)
|
||||
|
||||
bridge --> server : HTTP REST\n(POST /v1/api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /v1/api/events)
|
||||
bridge --> redis : Redis protocol\n(queues + pubsub)
|
||||
console --> server : HTTP proxy\n(hash-ring lookup,\nproxy /node/{id}/*)
|
||||
|
||||
console --> redis : Redis PUBSUB + LIST\n(cluster events,\nws creation commands)
|
||||
console --> server : HTTP polling + proxy\n(GET /v1/api/dashboard,\nproxy /node/{id}/*)
|
||||
|
||||
sim --> redis : Redis protocol\n(queues + pubsub + keys)
|
||||
' Database connections (production/cluster profiles)
|
||||
server ..> pgbouncer : PostgreSQL\n(pool_size=2)
|
||||
console ..> pgbouncer : PostgreSQL\n(auth/admin)
|
||||
pgbouncer --> postgres : transaction\npooling
|
||||
|
||||
' Environment variables
|
||||
note right of host
|
||||
**Environment Variables:**
|
||||
• LLM_BASE_URL — LLM endpoint
|
||||
• OPENAI_API_KEY — API key
|
||||
• REDIS_PASSWORD — Redis auth
|
||||
• TURNSTONE_AUTH_TOKEN — API auth
|
||||
* LLM_BASE_URL -- LLM endpoint
|
||||
* OPENAI_API_KEY -- API key
|
||||
* TURNSTONE_AUTH_TOKEN -- API auth
|
||||
* TURNSTONE_DB_URL -- PostgreSQL URL
|
||||
* POSTGRES_PASSWORD -- DB password
|
||||
end note
|
||||
|
||||
' Volumes
|
||||
database "redis-data" as rv
|
||||
database "turnstone-data" as tv
|
||||
database "postgres-data" as pv
|
||||
|
||||
redis_node --> rv
|
||||
server_node --> tv
|
||||
pg_node --> pv
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -53,10 +53,10 @@ class "SQLiteBackend" as SQLite <<sqlite>> {
|
||||
|
||||
class "PostgreSQLBackend" as PG <<postgres>> {
|
||||
-_engine: sa.Engine
|
||||
+__init__(url: str, pool_size: int)
|
||||
+__init__(url: str, pool_size: int = 2,\n max_overflow: int = 3)
|
||||
--
|
||||
tsvector + ILIKE search
|
||||
Connection pooling
|
||||
Connection pooling (5 max per process)
|
||||
}
|
||||
|
||||
' -- Schema --
|
||||
@@ -64,14 +64,12 @@ class "_schema.py" as Schema <<schema>> {
|
||||
+metadata: MetaData
|
||||
+memories: Table
|
||||
+conversations: Table
|
||||
+workstreams: Table (node_id, alias, title,\n state, ws_template_id, ws_template_version)
|
||||
+workstreams: Table (node_id, alias, title,\n state, skill_id)
|
||||
+workstream_config: Table
|
||||
+users: Table (username, password_hash)
|
||||
+api_tokens: Table (token_hash, scopes)
|
||||
+channel_users: Table (channel_type)
|
||||
+workstream_templates: Table (name, model,\n system_prompt, token_budget, version)
|
||||
+workstream_template_versions: Table\n (template_id, version, snapshot)
|
||||
+scheduled_tasks: Table (..., ws_template)
|
||||
+scheduled_tasks: Table (..., skill)
|
||||
--
|
||||
SQLAlchemy Core
|
||||
Single source of truth
|
||||
@@ -153,7 +151,7 @@ note right of Registry
|
||||
backend = "sqlite" | "postgresql"
|
||||
url = "postgresql+psycopg://..."
|
||||
path = ".turnstone.db"
|
||||
pool_size = 5
|
||||
pool_size = 2 (+ 3 overflow)
|
||||
end note
|
||||
|
||||
note bottom of SQLite
|
||||
@@ -164,8 +162,9 @@ end note
|
||||
|
||||
note bottom of PG
|
||||
Production backend.
|
||||
Multi-node / Docker
|
||||
default.
|
||||
Multi-node / Docker default.
|
||||
Use PgBouncer (transaction mode)
|
||||
for clusters > 50 nodes.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -176,4 +176,15 @@ note bottom of SH
|
||||
Both share JWT signing secret
|
||||
end note
|
||||
|
||||
note left of JWT
|
||||
**Console Proxy Token Minting**
|
||||
When proxying requests to server nodes:
|
||||
1. Console AuthMiddleware validates user JWT (aud: turnstone-console)
|
||||
2. Proxy mints new JWT (aud: turnstone-server)
|
||||
with real user_id, scopes, permissions
|
||||
3. src: "console-proxy" for audit traceability
|
||||
4. 5-minute expiry (fresh per request)
|
||||
5. Fallback: ServiceTokenManager if no user context
|
||||
end note
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -5,8 +5,6 @@ title Turnstone — Channel Integration Architecture
|
||||
skinparam class {
|
||||
BackgroundColor<<platform>> #E1BEE7
|
||||
BackgroundColor<<service>> #E8EAF6
|
||||
BackgroundColor<<mq>> #FFCDD2
|
||||
BackgroundColor<<bridge>> #C8E6C9
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
}
|
||||
@@ -63,53 +61,23 @@ class "DiscordBot" as Bot <<service>> {
|
||||
Renders approval buttons
|
||||
escape_mentions() on send
|
||||
--
|
||||
_notify_ws_map: msg_id → (ws_id, user_id)
|
||||
_notify_reply_channels: ws_id → (dm, user_id)
|
||||
_notify_ws_map: msg_id -> (ws_id, user_id)
|
||||
_notify_reply_channels: ws_id -> (dm, user_id)
|
||||
}
|
||||
|
||||
class "ChannelRouter" as Router <<service>> {
|
||||
+resolve_route(platform, channel_id)
|
||||
→ ws_id | None
|
||||
-> ws_id | None
|
||||
+register_route(channel_id, ws_id)
|
||||
+resolve_identity(platform, platform_user_id)
|
||||
→ user_id | None
|
||||
-> user_id | None
|
||||
--
|
||||
Maps channels → workstreams
|
||||
Maps platform users → turnstone users
|
||||
Maps channels -> workstreams
|
||||
Maps platform users -> turnstone users
|
||||
Caches routes in memory
|
||||
}
|
||||
|
||||
class "AsyncRedisBroker" as Broker <<service>> {
|
||||
+push_inbound(msg)
|
||||
+subscribe(ws_id) → AsyncIterator
|
||||
+subscribe_global() → AsyncIterator
|
||||
+push_response(correlation_id, msg)
|
||||
--
|
||||
redis.asyncio client
|
||||
Pub/sub + queue operations
|
||||
}
|
||||
|
||||
' -- Redis MQ --
|
||||
class "Redis MQ" as Redis <<mq>> {
|
||||
turnstone:inbound (LIST)
|
||||
turnstone:events:{ws_id} (PUBSUB)
|
||||
turnstone:events:global (PUBSUB)
|
||||
turnstone:resp:{corr_id} (LIST)
|
||||
--
|
||||
Shared message bus
|
||||
Same queues as bridge protocol
|
||||
}
|
||||
|
||||
' -- Bridge + Server --
|
||||
class "turnstone-bridge" as Bridge <<bridge>> {
|
||||
BLPOP turnstone:inbound
|
||||
Drive server via HTTP
|
||||
Relay SSE → Redis pub/sub
|
||||
--
|
||||
Owns workstream lifecycle
|
||||
Auto-approve / manual approve
|
||||
}
|
||||
|
||||
' -- Server --
|
||||
class "turnstone-server" as Server <<server>> {
|
||||
POST /v1/api/send
|
||||
POST /v1/api/approve
|
||||
@@ -128,7 +96,7 @@ class "channel_users" as CU <<storage>> {
|
||||
channel_user_id (PK)
|
||||
platform: "discord" | "slack"
|
||||
platform_user_id
|
||||
user_id → users
|
||||
user_id -> users
|
||||
linked_at
|
||||
--
|
||||
/link command creates row
|
||||
@@ -161,18 +129,13 @@ class "services" as SVC <<storage>> {
|
||||
' -- Relationships --
|
||||
Discord --> Bot : gateway\nevents
|
||||
Bot --> Router : on_message\non_interaction
|
||||
Router --> Broker : SendMessage\nApproveMessage
|
||||
Router --> CU : resolve identity
|
||||
Router --> CR : resolve / register route
|
||||
Broker --> Redis : RPUSH inbound\nRPUSH resp:{id}
|
||||
|
||||
Redis --> Bridge : BLPOP inbound
|
||||
Bridge --> Server : HTTP API
|
||||
Server --> Bridge : SSE events
|
||||
Bridge --> Redis : PUBLISH events:{ws_id}\nPUBLISH events:global
|
||||
Router --> Server : POST /v1/api/send\nPOST /v1/api/approve\nPOST /v1/api/workstreams/new
|
||||
Bot --> Server : GET /v1/api/events?ws_id=\n(SSE via httpx-sse)
|
||||
Server --> Bot : SSE event stream
|
||||
|
||||
Redis --> Broker : SUBSCRIBE events:{ws_id}
|
||||
Broker --> Bot : event stream
|
||||
Bot --> Discord : reply / embed\nbutton callback
|
||||
|
||||
Slack .[hidden]. Discord
|
||||
@@ -180,10 +143,9 @@ Teams .[hidden]. Slack
|
||||
|
||||
ChannelService --> Bot : creates + runs
|
||||
ChannelService --> Router : creates
|
||||
ChannelService --> Broker : creates
|
||||
ChannelService --> SVC : register / heartbeat /\nderegister
|
||||
|
||||
' -- Notification path (direct HTTP, bypasses MQ) --
|
||||
' -- Notification path (direct HTTP) --
|
||||
Server --> ChannelService : POST /v1/api/notify\n(JWT: aud=turnstone-channel)
|
||||
Server --> SVC : list_services("channel",\nmax_age_seconds=120)
|
||||
|
||||
@@ -192,38 +154,36 @@ note right of Bot
|
||||
**Inbound Flow**
|
||||
1. Discord message arrives via gateway
|
||||
2. Bot.on_message() fires
|
||||
3. ChannelRouter resolves channel → ws_id
|
||||
3. ChannelRouter resolves channel -> ws_id
|
||||
(or creates new workstream)
|
||||
4. ChannelRouter resolves platform user → user_id
|
||||
4. ChannelRouter resolves platform user -> user_id
|
||||
via channel_users table
|
||||
5. Broker.push_inbound(SendMessage)
|
||||
6. Bridge pops from Redis, drives server
|
||||
5. Router sends POST /v1/api/send to server
|
||||
|
||||
**Workstream Resume (evicted workstreams)**
|
||||
1. Stale route detected (no MQ owner)
|
||||
1. Stale route detected (no active SSE listener)
|
||||
2. Existing ws_id reused directly from route
|
||||
3. CreateWorkstreamMessage sent with
|
||||
3. POST /v1/api/workstreams/new with
|
||||
resume_ws=<ws_id>
|
||||
4. Server resumes atomically during creation
|
||||
5. Bridge emits WorkstreamResumedEvent → thread
|
||||
5. SSE emits WorkstreamResumedEvent -> thread
|
||||
end note
|
||||
|
||||
note right of Broker
|
||||
note right of Server
|
||||
**Outbound Flow**
|
||||
1. Server emits SSE events
|
||||
2. Bridge relays to Redis events:{ws_id}
|
||||
3. Broker.subscribe(ws_id) yields events
|
||||
4. Bot formats and sends to Discord thread
|
||||
1. Server emits SSE events on
|
||||
GET /v1/api/events?ws_id=
|
||||
2. Bot subscribes via httpx-sse
|
||||
3. Bot formats and sends to Discord thread
|
||||
end note
|
||||
|
||||
note bottom of CR
|
||||
**Approval Flow**
|
||||
1. ApprovalRequestEvent arrives via events:{ws_id}
|
||||
1. ApprovalRequestEvent arrives via SSE
|
||||
2. Bot renders Discord buttons (Approve / Deny)
|
||||
3. User clicks button → on_interaction()
|
||||
3. User clicks button -> on_interaction()
|
||||
4. Router builds ApproveMessage
|
||||
5. Broker.push_response(correlation_id, msg)
|
||||
6. Bridge pops from resp:{id}, calls POST /api/approve
|
||||
5. Router sends POST /v1/api/approve to server
|
||||
end note
|
||||
|
||||
note bottom of CU
|
||||
@@ -238,17 +198,17 @@ note bottom of CU
|
||||
end note
|
||||
|
||||
note bottom of SVC
|
||||
**Notification Flow** (direct HTTP, bypasses MQ)
|
||||
1. LLM calls notify tool → _prepare_notify()
|
||||
**Notification Flow** (direct HTTP)
|
||||
1. LLM calls notify tool -> _prepare_notify()
|
||||
2. _exec_notify() checks rate limit (5/turn)
|
||||
3. Queries services table for healthy gateways
|
||||
4. Mints JWT (aud: turnstone-channel) via
|
||||
ServiceTokenManager
|
||||
5. POSTs to first healthy gateway (incl. ws_id)
|
||||
6. Gateway validates JWT, resolves target
|
||||
7. adapter.send_notification() → Discord API
|
||||
(tracks msg_id → ws_id for reply routing)
|
||||
8. On failure: retry up to 3× (1s, 3s backoff)
|
||||
7. adapter.send_notification() -> Discord API
|
||||
(tracks msg_id -> ws_id for reply routing)
|
||||
8. On failure: retry up to 3x (1s, 3s backoff)
|
||||
9. SSRF: only http(s) URLs allowed
|
||||
|
||||
**Bidirectional DM Replies**
|
||||
|
||||
@@ -23,17 +23,16 @@ package "Governance Storage" {
|
||||
database "user_roles" as ur_db
|
||||
database "orgs" as orgs_db
|
||||
database "tool_policies" as tp_db
|
||||
database "prompt_templates" as pt_db
|
||||
database "prompt_templates\n(skills)" as pt_db
|
||||
database "usage_events" as ue_db
|
||||
database "audit_events" as ae_db
|
||||
database "workstream_templates" as wt_db
|
||||
database "workstream_template_versions" as wtv_db
|
||||
database "skills" as wt_db
|
||||
}
|
||||
|
||||
package "Runtime Enforcement" {
|
||||
[evaluate_tool_policies_batch()] as eval
|
||||
[WebUI.approve_tools()] as approve
|
||||
[record_usage_event()] as usage
|
||||
[record_usage_event()\n+cache_creation/read_tokens] as usage
|
||||
[record_audit()] as audit
|
||||
}
|
||||
|
||||
@@ -44,10 +43,9 @@ package "Template Runtime" {
|
||||
[set_template() / /template] as tset
|
||||
}
|
||||
|
||||
package "WS Template Runtime" {
|
||||
[resolve_ws_template()] as wtr
|
||||
package "Skill Runtime" {
|
||||
[resolve_skill()] as wtr
|
||||
[apply settings\n(model, budget, prompt)] as wta
|
||||
[drift detection\n(prompt_template_hash)] as wtd
|
||||
[budget gate\n(session.send)] as wtb
|
||||
}
|
||||
|
||||
@@ -76,7 +74,7 @@ audit --> ae_db : admin handlers
|
||||
|
||||
govjs --> roles_db : /v1/api/admin/roles
|
||||
govjs --> tp_db : /v1/api/admin/policies
|
||||
govjs --> pt_db : /v1/api/admin/templates
|
||||
govjs --> pt_db : /v1/api/admin/skills
|
||||
govjs --> ue_db : /v1/api/admin/usage
|
||||
govjs --> ae_db : /v1/api/admin/audit
|
||||
|
||||
@@ -87,17 +85,15 @@ tset --> tload : name or None
|
||||
|
||||
note right of pt_db
|
||||
Read-only listing:
|
||||
GET /v1/api/templates
|
||||
GET /v1/api/skills
|
||||
(read scope, summary only)
|
||||
end note
|
||||
|
||||
govjs --> wt_db : /v1/api/admin/ws-templates
|
||||
wtr --> wt_db : get_ws_template_by_name()
|
||||
wtr --> wta : template settings
|
||||
wta --> pt_db : prompt_template lookup
|
||||
wtd --> wt_db : compare hash
|
||||
govjs --> wt_db : /v1/api/admin/skills
|
||||
wtr --> wt_db : get_skill_by_name()
|
||||
wtr --> wta : skill settings
|
||||
wta --> pt_db : skill lookup
|
||||
wtb --> approve : __budget_override__
|
||||
wtv_db <.. wt_db : version snapshots
|
||||
|
||||
auth -[hidden]-> mw
|
||||
mw -[hidden]-> approve
|
||||
|
||||
@@ -8,6 +8,7 @@ skinparam participant {
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<ui>> #E8EAF6
|
||||
BackgroundColor<<registry>> #F8BBD0
|
||||
}
|
||||
|
||||
participant "MCP Server\n(external)" as MCPSrv <<mcp>>
|
||||
@@ -19,6 +20,9 @@ participant "Server / Console\n(health + UI)" as UI <<server>>
|
||||
participant "Console Admin UI\n(admin panel)" as Admin <<ui>>
|
||||
participant "Database\n(mcp_servers table)" as DB <<storage>>
|
||||
|
||||
participant "MCPRegistryClient\n(mcp_registry.py)" as RegClient <<mcp>>
|
||||
participant "MCP Registry\n(registry.modelcontextprotocol.io)" as Registry <<registry>>
|
||||
|
||||
== Admin-Driven Configuration ==
|
||||
|
||||
Admin -> DB : CRUD MCP server definitions\n(POST/PUT/DELETE /v1/api/admin/mcp-servers)
|
||||
@@ -33,6 +37,28 @@ note right
|
||||
- Changed entries → reconnect
|
||||
end note
|
||||
|
||||
== Registry Discovery & Install ==
|
||||
|
||||
Admin -> UI : GET /v1/api/admin/mcp-registry/search?search=...
|
||||
UI -> RegClient : search(q, limit, cursor)
|
||||
RegClient -> Registry : GET /v0.1/servers?search=...&latest=true
|
||||
Registry --> RegClient : Server entries\n(remotes, packages, meta)
|
||||
RegClient --> UI : RegistrySearchResult\n(annotated with installed status)
|
||||
UI --> Admin : Search results\n(Install / Installed badges)
|
||||
|
||||
Admin -> UI : POST /v1/api/admin/mcp-registry/install
|
||||
UI -> DB : create_mcp_server()\n(registry_name, version, meta)
|
||||
UI -> MCPMgr : POST /_internal/mcp-reload\n(fan-out to nodes)
|
||||
MCPMgr -> MCPMgr : reconcile_sync()
|
||||
MCPMgr -> MCPSrv : connect to new server
|
||||
|
||||
note over RegClient, Registry
|
||||
MCPRegistryClient is an async httpx client
|
||||
targeting registry.modelcontextprotocol.io/v0.1.
|
||||
resolve_install_config() translates registry
|
||||
remotes/packages into mcp_servers rows.
|
||||
end note
|
||||
|
||||
== Startup: Connection & Discovery ==
|
||||
|
||||
MCPMgr -> DB : load_mcp_config(storage=)\n(merge config file + DB)
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Workstream Template Architecture
|
||||
|
||||
skinparam participant {
|
||||
BackgroundColor<<admin>> #E8EAF6
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<session>> #C8E6C9
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<integration>> #F3E5F5
|
||||
}
|
||||
|
||||
participant "Admin / Console UI\n(governance.js)" as Admin <<admin>>
|
||||
participant "Server\n(server.py)" as Server <<server>>
|
||||
participant "ChatSession\n(session.py)" as Session <<session>>
|
||||
participant "StorageBackend\n(SQLite)" as Storage <<storage>>
|
||||
participant "Integration Points\n(scheduler, channel,\nbridge, MQ)" as Integrations <<integration>>
|
||||
|
||||
== Admin CRUD ==
|
||||
|
||||
Admin -> Server : POST /v1/api/admin/ws-templates
|
||||
note right
|
||||
**Payload:**
|
||||
name, model, system_prompt,
|
||||
temperature, reasoning_effort,
|
||||
max_tokens, agent_max_turns,
|
||||
auto_approve, auto_approve_tools,
|
||||
token_budget, prompt_template,
|
||||
prompt_template_hash, notify_on_complete
|
||||
end note
|
||||
|
||||
Server -> Storage : create_ws_template()
|
||||
Storage --> Server : ws_template_id
|
||||
|
||||
Admin -> Server : PUT /v1/api/admin/ws-templates/{id}
|
||||
Server -> Storage : get_ws_template(id)\n(snapshot pre-update state)
|
||||
Storage --> Server : existing template
|
||||
Server -> Storage : create_ws_template_version()\n(version snapshot)
|
||||
Server -> Storage : update_ws_template(id, ...)
|
||||
note right
|
||||
**Versioning:**
|
||||
Each update snapshots
|
||||
pre-update state into
|
||||
workstream_template_versions.
|
||||
version counter increments.
|
||||
end note
|
||||
|
||||
Admin -> Server : GET /v1/api/admin/ws-templates
|
||||
Server -> Storage : list_ws_templates()
|
||||
|
||||
Server <-- Server : GET /v1/api/ws-templates\n(read scope, summary only)
|
||||
note right
|
||||
**Read-only listing:**
|
||||
name, description, model.
|
||||
Used by creation UI dropdowns.
|
||||
Available on both server + console.
|
||||
end note
|
||||
|
||||
Admin -> Server : DELETE /v1/api/admin/ws-templates/{id}
|
||||
Server -> Storage : delete_ws_template(id)
|
||||
|
||||
== Workstream Creation Flow ==
|
||||
|
||||
Integrations -> Server : CreateWorkstreamMessage\n(ws_template="production-agent")
|
||||
note right
|
||||
**Sources:**
|
||||
- Console UI (Profile dropdown)
|
||||
- Scheduler (ws_template field)
|
||||
- Channel Router (ws_template)
|
||||
- Bridge (ws_template forwarding)
|
||||
- MQ Client (ws_template)
|
||||
end note
|
||||
|
||||
Server -> Storage : get_ws_template_by_name("production-agent")
|
||||
Storage --> Server : template dict
|
||||
|
||||
Server -> Server : resolve_ws_template()\napply model override
|
||||
note right
|
||||
**Settings applied:**
|
||||
- model (overrides default)
|
||||
- system_prompt
|
||||
- temperature
|
||||
- reasoning_effort
|
||||
- max_tokens
|
||||
- agent_max_turns
|
||||
- auto_approve / auto_approve_tools
|
||||
- token_budget
|
||||
- tool_search config
|
||||
end note
|
||||
|
||||
Server -> Session : mgr.create(model=template.model, ...)
|
||||
Session -> Session : _init_system_messages()
|
||||
|
||||
alt template has prompt_template
|
||||
Session -> Storage : get_prompt_template_by_name()
|
||||
Session -> Session : _render_template()\n{{model}}, {{ws_id}}, {{node_id}}
|
||||
end
|
||||
|
||||
Session -> Storage : _save_config()\n+ ws_template_id, ws_template_version
|
||||
|
||||
== Drift Detection ==
|
||||
|
||||
Server -> Server : compute prompt_template_hash\n(at creation time)
|
||||
note right
|
||||
**Hash stored:**
|
||||
SHA-256 of prompt_template
|
||||
content at ws creation time.
|
||||
Compared at next creation
|
||||
to detect upstream changes.
|
||||
end note
|
||||
|
||||
Server -> Storage : update_workstream()\n(store prompt_template_hash)
|
||||
|
||||
... later, new workstream created ...
|
||||
|
||||
Server -> Storage : get_ws_template()
|
||||
Server -> Server : compare hash vs\ncurrent prompt_template content
|
||||
alt hash mismatch
|
||||
Server -> Server : log.warning(\n"prompt template drift detected")
|
||||
end
|
||||
|
||||
== Token Budget Enforcement ==
|
||||
|
||||
Session -> Session : send(message)
|
||||
Session -> Session : _check_budget_gate()
|
||||
note right
|
||||
**Budget gate:**
|
||||
if token_budget set:
|
||||
total = prompt_tokens + completion_tokens
|
||||
if total >= token_budget:
|
||||
block further sends
|
||||
end note
|
||||
|
||||
alt budget exceeded
|
||||
Session -> Session : approve_tools(\n__budget_override__)
|
||||
note right
|
||||
Model can request
|
||||
budget override via
|
||||
special approval label.
|
||||
User must approve.
|
||||
end note
|
||||
else within budget
|
||||
Session -> Session : continue normal flow
|
||||
end
|
||||
|
||||
== Storage Schema ==
|
||||
|
||||
note over Storage
|
||||
**workstream_templates**
|
||||
id, name (unique), model, system_prompt,
|
||||
temperature, reasoning_effort, max_tokens,
|
||||
agent_max_turns, auto_approve, auto_approve_tools,
|
||||
token_budget, prompt_template, prompt_template_hash,
|
||||
tool_search, tool_search_threshold, tool_search_max_results,
|
||||
version, created_at, updated_at
|
||||
|
||||
**workstream_template_versions**
|
||||
id, template_id (FK), version, snapshot (JSON),
|
||||
created_at
|
||||
|
||||
**workstreams** (updated columns)
|
||||
+ ws_template_id: str | None
|
||||
+ ws_template_version: int | None
|
||||
|
||||
**scheduled_tasks** (updated column)
|
||||
+ ws_template: str | None
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -35,18 +35,24 @@ Session -> Judge : evaluate(items, messages, callback)
|
||||
|
||||
Judge -> Judge : evaluate_heuristic()\nfor each item
|
||||
note right
|
||||
**Rule table (first match wins):**
|
||||
**36 rules (first match wins):**
|
||||
Critical (0.90, deny): rm /, mkfs,
|
||||
dd, pipe-to-shell, chmod 777 /,
|
||||
write/edit /etc/ .ssh/
|
||||
write/edit /etc/ .ssh/,
|
||||
download-then-execute chains
|
||||
High (0.80, review): sudo, kill -9,
|
||||
destructive git, DROP TABLE,
|
||||
secrets, HTTP mutations, ssh/scp
|
||||
Medium (0.70, review): pip/npm install,
|
||||
secrets, HTTP mutations, ssh/scp,
|
||||
browser+data-export, transitive
|
||||
install, control-plane mutation
|
||||
Medium (0.70, review): content
|
||||
ingestion, interpreter exec,
|
||||
cloud CLI mutations, pkg install,
|
||||
write_file, MCP tools, docker ops
|
||||
Low (0.85, approve): read_file,
|
||||
list_directory, search, recall,
|
||||
read-only bash (ls, cat, grep...)
|
||||
tool_search, read_resource,
|
||||
web_search, read-only bash
|
||||
Default: medium, 0.50, review
|
||||
end note
|
||||
|
||||
@@ -141,6 +147,52 @@ note right
|
||||
with daemon judge thread.
|
||||
end note
|
||||
|
||||
== Tool Execution ==
|
||||
|
||||
Session -> Session : _execute_tools()
|
||||
note right
|
||||
Tools execute with
|
||||
user approval.
|
||||
end note
|
||||
|
||||
== Output Guard (synchronous, time-budgeted) ==
|
||||
|
||||
Session -> Session : _evaluate_output()\nfor each tool result
|
||||
note right
|
||||
**Priority-ordered checks (5s budget):**
|
||||
P1: Prompt injection (role injection,
|
||||
override phrases, instruction tags)
|
||||
P2: Credential leakage (API keys,
|
||||
PEM blocks, connection strings)
|
||||
P3: Encoded payloads (data URIs,
|
||||
hex shellcode)
|
||||
P4: Adversarial URLs (cloud metadata,
|
||||
credential query params)
|
||||
P5: System info disclosure (private
|
||||
IPs, sensitive paths)
|
||||
|
||||
Annotates + optionally redacts.
|
||||
Does NOT gate.
|
||||
end note
|
||||
|
||||
alt output_warning flags detected
|
||||
Session -> UI : SSE: output_warning\n{call_id, risk_level, flags,\nfunc_name, redacted}
|
||||
note right
|
||||
Credential values replaced
|
||||
with [REDACTED:<type>] before
|
||||
output enters conversation.
|
||||
sanitized text excluded from
|
||||
SSE payload (defense in depth).
|
||||
end note
|
||||
UI -> Storage : record_output_assessment()\nfire-and-forget persistence
|
||||
note right
|
||||
Stored: flags, risk_level,
|
||||
annotations, output_length,
|
||||
redacted (bool). Raw tool
|
||||
output is never stored.
|
||||
end note
|
||||
end
|
||||
|
||||
== Lifecycle ==
|
||||
|
||||
note over Session, Judge
|
||||
@@ -152,9 +204,15 @@ note over Session, Judge
|
||||
**Sub-agent exemption:**
|
||||
Plan agent and task agent skip intent validation entirely.
|
||||
|
||||
**Output guard:**
|
||||
Runs when judge_config.output_guard is true (default).
|
||||
Credential redaction when judge_config.redact_secrets is true.
|
||||
|
||||
**Storage:**
|
||||
intent_verdicts table (migration 012). Verdicts queryable via
|
||||
GET /v1/api/admin/verdicts (requires admin.judge permission).
|
||||
intent_verdicts table (migration 012), output_assessments table
|
||||
(migration 022). Both queryable via admin API endpoints
|
||||
(requires admin.judge permission). Skills store scan_status,
|
||||
scan_report, scan_version for install-time risk assessment.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -61,7 +61,7 @@ end
|
||||
|
||||
Session -> Session : _init_system_messages()\nevery conversation turn
|
||||
|
||||
Session -> Session : _get_visible_memories(\nlimit=fetch_limit)
|
||||
Session -> Session : _list_visible_memories(\nlimit=fetch_limit)
|
||||
note right
|
||||
**Scope resolution:**
|
||||
1. global scope (always)
|
||||
|
||||
@@ -144,7 +144,7 @@ note over Server, Registry
|
||||
**CLI entry point:**
|
||||
CLI flag > config.toml > argparse default
|
||||
|
||||
**Bootstrap settings** (database, Redis, auth, server bind):
|
||||
**Bootstrap settings** (database, auth, server bind):
|
||||
Always from config.toml / env vars — never in ConfigStore.
|
||||
end note
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
@startuml
|
||||
!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml
|
||||
LAYOUT_LEFT_RIGHT()
|
||||
|
||||
title Skills Discovery & Runtime Architecture
|
||||
|
||||
skinparam backgroundColor #1e1e2e
|
||||
skinparam defaultFontColor #cdd6f4
|
||||
skinparam defaultFontName "JetBrains Mono"
|
||||
skinparam arrowColor #89b4fa
|
||||
skinparam rectangleBorderColor #585b70
|
||||
skinparam rectangleBackgroundColor #313244
|
||||
skinparam noteBorderColor #585b70
|
||||
skinparam noteBackgroundColor #45475a
|
||||
skinparam packageBorderColor #585b70
|
||||
|
||||
package "External Sources" as ext #181825 {
|
||||
rectangle "skills.sh\nRegistry" as skillssh
|
||||
rectangle "GitHub\nRepositories" as github
|
||||
}
|
||||
|
||||
package "Console Server" as console #181825 {
|
||||
rectangle "admin_skill_discover\nGET /v1/api/admin/skills/discover" as discover
|
||||
rectangle "admin_skill_install\nPOST /v1/api/admin/skills/install" as install
|
||||
rectangle "_get_discovery_url\nsettings fallback" as settings
|
||||
}
|
||||
|
||||
package "Core Modules" as core #181825 {
|
||||
rectangle "SkillsShClient\nskill_sources.py" as client
|
||||
rectangle "fetch_skill_from_github\nskill_sources.py" as fetcher
|
||||
rectangle "parse_skill_md\nskill_parser.py" as parser
|
||||
rectangle "scan_skill_content\nstorage/_utils.py" as scanner
|
||||
}
|
||||
|
||||
package "Session Runtime" as runtime #181825 {
|
||||
rectangle "skill tool\nsession.py" as loadtool
|
||||
rectangle "set_skill()\nsession.py" as setskill
|
||||
rectangle "_load_skills()\nsession.py" as loadskills
|
||||
}
|
||||
|
||||
package "Storage" as storage #181825 {
|
||||
rectangle "prompt_templates\n(skills)" as skills_table
|
||||
rectangle "skill_resources\n(bundled files)" as resources_table
|
||||
rectangle "system_settings\n(discovery_url)" as settings_table
|
||||
}
|
||||
|
||||
package "Admin UI" as ui #181825 {
|
||||
rectangle "Skills Tab\nInstalled / Discover pill" as pill
|
||||
rectangle "Discovery View\nsearch + cards" as discoverui
|
||||
rectangle "GitHub Import\nmodal" as importui
|
||||
}
|
||||
|
||||
' External discovery flow
|
||||
discover --> settings : resolve URL
|
||||
settings --> settings_table : DB -> config -> default
|
||||
discover --> client : search(query)
|
||||
client --> skillssh : GET /api/search
|
||||
|
||||
install --> client : resolve_github_url()
|
||||
client --> skillssh : GET /api/skills/{id}
|
||||
install --> fetcher : fetch SKILL.md + resources
|
||||
fetcher --> github : raw.githubusercontent.com
|
||||
fetcher --> github : api.github.com/git/trees
|
||||
fetcher --> parser : parse frontmatter
|
||||
install --> scanner : auto-scan on create
|
||||
install --> skills_table : create_prompt_template
|
||||
install --> resources_table : create_skill_resource
|
||||
|
||||
' Runtime skill loading flow
|
||||
loadtool --> skills_table : search (BM25 ranking)
|
||||
loadtool --> setskill : load (name)
|
||||
setskill --> loadskills : reload + reinit system messages
|
||||
loadskills --> skills_table : get_skill_by_name
|
||||
|
||||
' UI flow
|
||||
pill --> discoverui : switch view
|
||||
discoverui --> discover : authFetch()
|
||||
importui --> install : POST (github source)
|
||||
|
||||
' Annotations
|
||||
note right of parser
|
||||
YAML frontmatter -> ParsedSkill
|
||||
allowed-tools (standard) -> allowed_tools (internal)
|
||||
Anthropic + Hermes tag formats
|
||||
Name validation (lowercase+hyphens)
|
||||
end note
|
||||
|
||||
note right of loadtool
|
||||
search: auto-approved (read-only)
|
||||
load: requires user approval
|
||||
Main session only (no sub-agents)
|
||||
end note
|
||||
|
||||
note right of scanner
|
||||
4 risk axes (content, supply chain,
|
||||
vulnerability, capability)
|
||||
Auto-triggers on create/update
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -34,253 +34,214 @@
|
||||
<text x="600" y="54" text-anchor="middle" fill="#8b949e" font-size="11" letter-spacing="1">SYSTEM ARCHITECTURE</text>
|
||||
|
||||
<!-- ==================== COLUMN HEADERS ==================== -->
|
||||
<text x="90" y="86" text-anchor="middle" fill="#58a6ff" font-size="9" font-weight="600" letter-spacing="2">CLIENTS</text>
|
||||
<text x="276" y="86" text-anchor="middle" fill="#3fb950" font-size="9" font-weight="600" letter-spacing="2">GATEWAYS</text>
|
||||
<text x="480" y="86" text-anchor="middle" fill="#f0883e" font-size="9" font-weight="600" letter-spacing="2">MESSAGE QUEUE</text>
|
||||
<text x="700" y="86" text-anchor="middle" fill="#f47067" font-size="9" font-weight="600" letter-spacing="2">CLUSTER NODES</text>
|
||||
<text x="940" y="86" text-anchor="middle" fill="#f778ba" font-size="9" font-weight="600" letter-spacing="2">LLM PROVIDERS</text>
|
||||
<text x="110" y="86" text-anchor="middle" fill="#58a6ff" font-size="9" font-weight="600" letter-spacing="2">CLIENTS</text>
|
||||
<text x="380" y="86" text-anchor="middle" fill="#3fb950" font-size="9" font-weight="600" letter-spacing="2">CONSOLE ROUTER</text>
|
||||
<text x="700" y="86" text-anchor="middle" fill="#f47067" font-size="9" font-weight="600" letter-spacing="2">SERVER NODES</text>
|
||||
<text x="1010" y="86" text-anchor="middle" fill="#f778ba" font-size="9" font-weight="600" letter-spacing="2">LLM PROVIDERS</text>
|
||||
|
||||
<!-- ==================== CLIENT BOXES ==================== -->
|
||||
<!-- CLI -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="108" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="108" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="108" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="111" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="130" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">CLI</text>
|
||||
<text x="90" y="145" text-anchor="middle" fill="#8b949e" font-size="9">terminal REPL</text>
|
||||
<rect x="40" y="108" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="108" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="108" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="111" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="130" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">CLI</text>
|
||||
<text x="110" y="145" text-anchor="middle" fill="#8b949e" font-size="9">terminal REPL</text>
|
||||
</g>
|
||||
|
||||
<!-- Browser UI -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="174" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="174" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="174" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="177" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="196" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Browser UI</text>
|
||||
<text x="90" y="211" text-anchor="middle" fill="#8b949e" font-size="9">HTTP + SSE</text>
|
||||
<rect x="40" y="174" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="174" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="174" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="177" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="196" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Browser UI</text>
|
||||
<text x="110" y="211" text-anchor="middle" fill="#8b949e" font-size="9">HTTP + SSE</text>
|
||||
</g>
|
||||
|
||||
<!-- SDK / API -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="244" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="244" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="244" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="247" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="266" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">SDK / API</text>
|
||||
<text x="90" y="281" text-anchor="middle" fill="#8b949e" font-size="9">programmatic</text>
|
||||
</g>
|
||||
|
||||
<!-- Discord / Slack -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="314" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="314" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="314" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="317" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="336" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Discord / Slack</text>
|
||||
<text x="90" y="351" text-anchor="middle" fill="#8b949e" font-size="9">chat platforms</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== GATEWAY BOXES ==================== -->
|
||||
<!-- Console -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="216" y="118" width="120" height="58" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="216" y="118" width="120" height="5" rx="5" fill="#3fb950"/>
|
||||
<rect x="216" y="118" width="120" height="5" fill="#3fb950"/>
|
||||
<rect x="216" y="121" width="120" height="2" fill="#161b22"/>
|
||||
<text x="276" y="142" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Console</text>
|
||||
<text x="276" y="157" text-anchor="middle" fill="#8b949e" font-size="9">dashboard + proxy</text>
|
||||
<text x="276" y="169" text-anchor="middle" fill="#8b949e" font-size="9">cluster management</text>
|
||||
<rect x="40" y="244" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="244" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="244" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="247" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="266" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">SDK / API</text>
|
||||
<text x="110" y="281" text-anchor="middle" fill="#8b949e" font-size="9">programmatic</text>
|
||||
</g>
|
||||
|
||||
<!-- Channel Gateway -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="216" y="292" width="120" height="58" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="216" y="292" width="120" height="5" rx="5" fill="#3fb950"/>
|
||||
<rect x="216" y="292" width="120" height="5" fill="#3fb950"/>
|
||||
<rect x="216" y="295" width="120" height="2" fill="#161b22"/>
|
||||
<text x="276" y="316" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Channel Gateway</text>
|
||||
<text x="276" y="331" text-anchor="middle" fill="#8b949e" font-size="9">platform adapter</text>
|
||||
<text x="276" y="343" text-anchor="middle" fill="#8b949e" font-size="9">Discord, Slack, ...</text>
|
||||
<rect x="40" y="314" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="314" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="314" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="317" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="336" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Channel Gateway</text>
|
||||
<text x="110" y="351" text-anchor="middle" fill="#8b949e" font-size="9">Discord, Slack, ...</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== REDIS MQ ==================== -->
|
||||
<!-- ==================== CONSOLE ROUTER ==================== -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="420" y="168" width="120" height="132" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="420" y="168" width="120" height="5" rx="5" fill="#f0883e"/>
|
||||
<rect x="420" y="168" width="120" height="5" fill="#f0883e"/>
|
||||
<rect x="420" y="171" width="120" height="2" fill="#161b22"/>
|
||||
<text x="480" y="198" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Redis MQ</text>
|
||||
<line x1="438" y1="210" x2="522" y2="210" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="480" y="228" text-anchor="middle" fill="#8b949e" font-size="9">inbound queues</text>
|
||||
<text x="480" y="243" text-anchor="middle" fill="#8b949e" font-size="9">event pub/sub</text>
|
||||
<text x="480" y="258" text-anchor="middle" fill="#8b949e" font-size="9">node heartbeats</text>
|
||||
<text x="480" y="273" text-anchor="middle" fill="#8b949e" font-size="9">workstream routing</text>
|
||||
<text x="480" y="288" text-anchor="middle" fill="#8b949e" font-size="9">cluster state</text>
|
||||
<rect x="300" y="148" width="160" height="170" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="300" y="148" width="160" height="5" rx="5" fill="#3fb950"/>
|
||||
<rect x="300" y="148" width="160" height="5" fill="#3fb950"/>
|
||||
<rect x="300" y="151" width="160" height="2" fill="#161b22"/>
|
||||
<text x="380" y="178" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Console</text>
|
||||
<line x1="318" y1="190" x2="442" y2="190" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="380" y="208" text-anchor="middle" fill="#8b949e" font-size="9">hash-ring router</text>
|
||||
<text x="380" y="223" text-anchor="middle" fill="#8b949e" font-size="9">cluster dashboard</text>
|
||||
<text x="380" y="238" text-anchor="middle" fill="#8b949e" font-size="9">reverse proxy</text>
|
||||
<line x1="318" y1="250" x2="442" y2="250" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="380" y="268" text-anchor="middle" fill="#484f58" font-size="8">control plane:</text>
|
||||
<text x="380" y="282" text-anchor="middle" fill="#484f58" font-size="8">create / send / approve</text>
|
||||
<text x="380" y="296" text-anchor="middle" fill="#484f58" font-size="8">cancel / command / close</text>
|
||||
<text x="380" y="310" text-anchor="middle" fill="#484f58" font-size="8">port 8090</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== CLUSTER NODES ==================== -->
|
||||
<!-- ==================== SERVER NODES ==================== -->
|
||||
<!-- Cluster outline -->
|
||||
<rect x="598" y="100" width="204" height="310" rx="8" fill="none" stroke="#30363d" stroke-width="1" stroke-dasharray="4,3"/>
|
||||
<rect x="570" y="100" width="260" height="310" rx="8" fill="none" stroke="#30363d" stroke-width="1" stroke-dasharray="4,3"/>
|
||||
<text x="700" y="422" text-anchor="middle" fill="#30363d" font-size="9" letter-spacing="1">CLUSTER</text>
|
||||
|
||||
<!-- Node A -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="614" y="118" width="170" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="614" y="118" width="170" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="614" y="118" width="170" height="5" fill="#f47067"/>
|
||||
<rect x="614" y="121" width="170" height="2" fill="#161b22"/>
|
||||
<text x="699" y="142" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Node A</text>
|
||||
<line x1="632" y1="152" x2="766" y2="152" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Bridge -->
|
||||
<rect x="626" y="162" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="661" y="180" text-anchor="middle" fill="#8b949e" font-size="9">bridge</text>
|
||||
<!-- Server -->
|
||||
<rect x="704" y="162" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="739" y="180" text-anchor="middle" fill="#8b949e" font-size="9">server</text>
|
||||
<!-- Arrow bridge to server -->
|
||||
<line x1="696" y1="176" x2="702" y2="176" stroke="#484f58" stroke-width="1" marker-end="url(#arrow)"/>
|
||||
<rect x="590" y="118" width="220" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="590" y="118" width="220" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="590" y="118" width="220" height="5" fill="#f47067"/>
|
||||
<rect x="590" y="121" width="220" height="2" fill="#161b22"/>
|
||||
<text x="700" y="142" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Server Node A</text>
|
||||
<line x1="608" y1="152" x2="792" y2="152" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Server process -->
|
||||
<rect x="608" y="162" width="184" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="700" y="180" text-anchor="middle" fill="#8b949e" font-size="9">turnstone-server :8080</text>
|
||||
<!-- Tools label -->
|
||||
<text x="699" y="206" text-anchor="middle" fill="#484f58" font-size="8">14 tools + MCP</text>
|
||||
<text x="700" y="206" text-anchor="middle" fill="#484f58" font-size="8">19 tools + MCP</text>
|
||||
</g>
|
||||
|
||||
<!-- Node B -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="614" y="238" width="170" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="614" y="238" width="170" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="614" y="238" width="170" height="5" fill="#f47067"/>
|
||||
<rect x="614" y="241" width="170" height="2" fill="#161b22"/>
|
||||
<text x="699" y="262" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Node B</text>
|
||||
<line x1="632" y1="272" x2="766" y2="272" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Bridge -->
|
||||
<rect x="626" y="282" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="661" y="300" text-anchor="middle" fill="#8b949e" font-size="9">bridge</text>
|
||||
<!-- Server -->
|
||||
<rect x="704" y="282" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="739" y="300" text-anchor="middle" fill="#8b949e" font-size="9">server</text>
|
||||
<!-- Arrow bridge to server -->
|
||||
<line x1="696" y1="296" x2="702" y2="296" stroke="#484f58" stroke-width="1" marker-end="url(#arrow)"/>
|
||||
<rect x="590" y="238" width="220" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="590" y="238" width="220" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="590" y="238" width="220" height="5" fill="#f47067"/>
|
||||
<rect x="590" y="241" width="220" height="2" fill="#161b22"/>
|
||||
<text x="700" y="262" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Server Node B</text>
|
||||
<line x1="608" y1="272" x2="792" y2="272" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Server process -->
|
||||
<rect x="608" y="282" width="184" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="700" y="300" text-anchor="middle" fill="#8b949e" font-size="9">turnstone-server :8080</text>
|
||||
<!-- Tools label -->
|
||||
<text x="699" y="326" text-anchor="middle" fill="#484f58" font-size="8">14 tools + MCP</text>
|
||||
<text x="700" y="326" text-anchor="middle" fill="#484f58" font-size="8">19 tools + MCP</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== LLM PROVIDERS ==================== -->
|
||||
<!-- OpenAI -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="870" y="130" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="870" y="130" width="140" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="870" y="130" width="140" height="5" fill="#f778ba"/>
|
||||
<rect x="870" y="133" width="140" height="2" fill="#161b22"/>
|
||||
<text x="940" y="153" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">OpenAI</text>
|
||||
<text x="940" y="167" text-anchor="middle" fill="#8b949e" font-size="9">GPT-5, o-series</text>
|
||||
<rect x="930" y="130" width="160" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="930" y="130" width="160" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="930" y="130" width="160" height="5" fill="#f778ba"/>
|
||||
<rect x="930" y="133" width="160" height="2" fill="#161b22"/>
|
||||
<text x="1010" y="153" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">OpenAI</text>
|
||||
<text x="1010" y="167" text-anchor="middle" fill="#8b949e" font-size="9">GPT-5, o-series</text>
|
||||
</g>
|
||||
|
||||
<!-- Anthropic -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="870" y="196" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="870" y="196" width="140" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="870" y="196" width="140" height="5" fill="#f778ba"/>
|
||||
<rect x="870" y="199" width="140" height="2" fill="#161b22"/>
|
||||
<text x="940" y="219" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Anthropic</text>
|
||||
<text x="940" y="233" text-anchor="middle" fill="#8b949e" font-size="9">Claude 4.5 / 4.6</text>
|
||||
<rect x="930" y="196" width="160" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="930" y="196" width="160" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="930" y="196" width="160" height="5" fill="#f778ba"/>
|
||||
<rect x="930" y="199" width="160" height="2" fill="#161b22"/>
|
||||
<text x="1010" y="219" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Anthropic</text>
|
||||
<text x="1010" y="233" text-anchor="middle" fill="#8b949e" font-size="9">Claude 4.5 / 4.6</text>
|
||||
</g>
|
||||
|
||||
<!-- Local / vLLM -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="870" y="262" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="870" y="262" width="140" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="870" y="262" width="140" height="5" fill="#f778ba"/>
|
||||
<rect x="870" y="265" width="140" height="2" fill="#161b22"/>
|
||||
<text x="940" y="285" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Local / vLLM</text>
|
||||
<text x="940" y="299" text-anchor="middle" fill="#8b949e" font-size="9">llama.cpp, NIM</text>
|
||||
<rect x="930" y="262" width="160" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="930" y="262" width="160" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="930" y="262" width="160" height="5" fill="#f778ba"/>
|
||||
<rect x="930" y="265" width="160" height="2" fill="#161b22"/>
|
||||
<text x="1010" y="285" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Local / vLLM</text>
|
||||
<text x="1010" y="299" text-anchor="middle" fill="#8b949e" font-size="9">llama.cpp, NIM</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== STORAGE ==================== -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="614" y="450" width="170" height="52" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="614" y="450" width="170" height="5" rx="5" fill="#bc8cff"/>
|
||||
<rect x="614" y="450" width="170" height="5" fill="#bc8cff"/>
|
||||
<rect x="614" y="453" width="170" height="2" fill="#161b22"/>
|
||||
<text x="699" y="476" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">PostgreSQL / SQLite</text>
|
||||
<text x="699" y="492" text-anchor="middle" fill="#8b949e" font-size="9">conversations, memory, auth</text>
|
||||
<rect x="590" y="450" width="220" height="52" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="590" y="450" width="220" height="5" rx="5" fill="#bc8cff"/>
|
||||
<rect x="590" y="450" width="220" height="5" fill="#bc8cff"/>
|
||||
<rect x="590" y="453" width="220" height="2" fill="#161b22"/>
|
||||
<text x="700" y="476" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">PostgreSQL / SQLite</text>
|
||||
<text x="700" y="492" text-anchor="middle" fill="#8b949e" font-size="9">conversations, memory, auth</text>
|
||||
</g>
|
||||
<text x="699" y="444" text-anchor="middle" fill="#bc8cff" font-size="9" font-weight="600" letter-spacing="2">STORAGE</text>
|
||||
<text x="700" y="444" text-anchor="middle" fill="#bc8cff" font-size="9" font-weight="600" letter-spacing="2">STORAGE</text>
|
||||
|
||||
<!-- ==================== CONNECTION LINES ==================== -->
|
||||
|
||||
<!-- CLIENT -> GATEWAY connections -->
|
||||
<!-- CLIENT -> CONSOLE connections (control plane) -->
|
||||
<!-- Browser -> Console -->
|
||||
<line x1="150" y1="197" x2="214" y2="155" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- Discord -> Channel -->
|
||||
<line x1="150" y1="337" x2="214" y2="325" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<line x1="180" y1="197" x2="298" y2="210" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- Channel -> Console -->
|
||||
<line x1="180" y1="337" x2="298" y2="290" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- SDK -> Console -->
|
||||
<line x1="180" y1="267" x2="298" y2="248" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<text x="240" y="238" fill="#484f58" font-size="8" text-anchor="middle">HTTP</text>
|
||||
|
||||
<!-- CLI -> direct to Node A server (top path, curved) -->
|
||||
<path d="M 150 131 C 200 131, 200 100, 400 100 L 400 100 C 500 100, 570 140, 612 168" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.5" fill="none" stroke-dasharray="6,3" marker-end="url(#arrow-blue)"/>
|
||||
<text x="370" y="96" fill="#484f58" font-size="8" text-anchor="middle">direct</text>
|
||||
<!-- CLI -> direct to Node A (single-node mode, above everything) -->
|
||||
<path d="M 180 120 L 588 120" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.4" fill="none" stroke-dasharray="6,3" marker-end="url(#arrow-blue)"/>
|
||||
<text x="390" y="114" fill="#484f58" font-size="8" text-anchor="middle">direct (single-node)</text>
|
||||
|
||||
<!-- SDK -> Redis (direct push) -->
|
||||
<line x1="150" y1="267" x2="418" y2="240" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- CONSOLE -> NODE connections (proxy) -->
|
||||
<!-- Console -> Node A -->
|
||||
<line x1="460" y1="200" x2="588" y2="176" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
<!-- Console -> Node B -->
|
||||
<line x1="460" y1="260" x2="588" y2="296" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
<text x="520" y="222" fill="#484f58" font-size="8" text-anchor="middle">proxy</text>
|
||||
|
||||
<!-- GATEWAY -> REDIS connections -->
|
||||
<!-- Console -> Redis -->
|
||||
<line x1="336" y1="160" x2="418" y2="200" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
<!-- Channel -> Redis -->
|
||||
<line x1="336" y1="318" x2="418" y2="272" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
|
||||
<!-- REDIS -> NODE connections -->
|
||||
<!-- Redis -> Node A bridge -->
|
||||
<line x1="540" y1="210" x2="624" y2="176" stroke="#f0883e" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-orange)"/>
|
||||
<!-- Redis -> Node B bridge -->
|
||||
<line x1="540" y1="260" x2="624" y2="296" stroke="#f0883e" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-orange)"/>
|
||||
|
||||
<!-- Console -> Node (proxy, dashed) -->
|
||||
<path d="M 336 147 C 380 130, 500 108, 612 145" stroke="#3fb950" stroke-width="1" stroke-opacity="0.4" fill="none" stroke-dasharray="4,3" marker-end="url(#arrow-green)"/>
|
||||
<text x="468" y="120" fill="#484f58" font-size="8" text-anchor="middle">proxy</text>
|
||||
<!-- CLIENT -> NODE direct SSE (data plane, below console) -->
|
||||
<!-- Browser -> Node A SSE (arc below console) -->
|
||||
<path d="M 180 205 C 240 370, 450 380, 588 330" stroke="#58a6ff" stroke-width="1" stroke-opacity="0.3" fill="none" stroke-dasharray="4,3" marker-end="url(#arrow-blue)"/>
|
||||
<text x="340" y="378" fill="#484f58" font-size="8" text-anchor="middle">SSE (data plane)</text>
|
||||
|
||||
<!-- NODE -> LLM connections -->
|
||||
<!-- Node A -> LLM providers -->
|
||||
<line x1="784" y1="168" x2="868" y2="155" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
<line x1="784" y1="176" x2="868" y2="219" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="784" y1="180" x2="868" y2="282" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
<line x1="810" y1="168" x2="928" y2="155" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
<line x1="810" y1="176" x2="928" y2="219" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="810" y1="180" x2="928" y2="282" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
|
||||
<!-- Node B -> LLM providers -->
|
||||
<line x1="784" y1="288" x2="868" y2="163" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
<line x1="784" y1="296" x2="868" y2="222" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="784" y1="300" x2="868" y2="288" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
<line x1="810" y1="288" x2="928" y2="163" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
<line x1="810" y1="296" x2="928" y2="222" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="810" y1="300" x2="928" y2="288" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
|
||||
<!-- NODE -> STORAGE connections -->
|
||||
<line x1="680" y1="338" x2="680" y2="448" stroke="#bc8cff" stroke-width="1.2" stroke-opacity="0.4" stroke-dasharray="4,3" marker-end="url(#arrow-muted)"/>
|
||||
<line x1="718" y1="218" x2="718" y2="236" stroke="#484f58" stroke-width="1" stroke-opacity="0.3" stroke-dasharray="2,2"/>
|
||||
|
||||
<!-- Extensibility hint -->
|
||||
<text x="699" y="392" text-anchor="middle" fill="#30363d" font-size="10">...</text>
|
||||
|
||||
<!-- Event flow: Bridges -> Redis (dashed, bidirectional feel) -->
|
||||
<line x1="624" y1="186" x2="542" y2="220" stroke="#f0883e" stroke-width="1" stroke-opacity="0.3" stroke-dasharray="3,3"/>
|
||||
<line x1="624" y1="286" x2="542" y2="250" stroke="#f0883e" stroke-width="1" stroke-opacity="0.3" stroke-dasharray="3,3"/>
|
||||
<text x="574" y="242" fill="#484f58" font-size="7" text-anchor="middle">events</text>
|
||||
<text x="700" y="392" text-anchor="middle" fill="#30363d" font-size="10">...</text>
|
||||
|
||||
<!-- ==================== FLOW LABELS ==================== -->
|
||||
<!-- Interactive flow label -->
|
||||
<!-- Direct / single-node flow label -->
|
||||
<rect x="30" y="395" width="10" height="10" rx="2" fill="none" stroke="#58a6ff" stroke-width="1.5" stroke-dasharray="3,2"/>
|
||||
<text x="46" y="404" fill="#8b949e" font-size="9">interactive (direct)</text>
|
||||
<text x="46" y="404" fill="#8b949e" font-size="9">direct (single-node / SSE)</text>
|
||||
|
||||
<!-- Queue flow label -->
|
||||
<rect x="160" y="395" width="10" height="10" rx="2" fill="none" stroke="#f0883e" stroke-width="1.5"/>
|
||||
<text x="176" y="404" fill="#8b949e" font-size="9">queue-driven</text>
|
||||
<!-- Control plane label -->
|
||||
<rect x="200" y="395" width="10" height="10" rx="2" fill="none" stroke="#58a6ff" stroke-width="1.5"/>
|
||||
<text x="216" y="404" fill="#8b949e" font-size="9">control plane (HTTP)</text>
|
||||
|
||||
<!-- Proxy/event label -->
|
||||
<rect x="275" y="395" width="10" height="10" rx="2" fill="none" stroke="#3fb950" stroke-width="1.5" stroke-dasharray="3,2"/>
|
||||
<text x="291" y="404" fill="#8b949e" font-size="9">proxy / events</text>
|
||||
<!-- Proxy label -->
|
||||
<rect x="340" y="395" width="10" height="10" rx="2" fill="none" stroke="#3fb950" stroke-width="1.5"/>
|
||||
<text x="356" y="404" fill="#8b949e" font-size="9">console proxy</text>
|
||||
|
||||
<!-- ==================== BOTTOM DETAILS ==================== -->
|
||||
<line x1="30" y1="430" x2="1170" y2="430" stroke="#21262d" stroke-width="1"/>
|
||||
|
||||
<!-- Routing rules at bottom, left-aligned -->
|
||||
<text x="44" y="456" fill="#30363d" font-size="9" font-weight="600" letter-spacing="1">ROUTING</text>
|
||||
<circle cx="44" cy="474" r="3" fill="#f47067" opacity="0.6"/>
|
||||
<text x="54" y="477" fill="#484f58" font-size="9">target_node set → route to specific node queue</text>
|
||||
<circle cx="44" cy="494" r="3" fill="#f0883e" opacity="0.6"/>
|
||||
<text x="54" y="497" fill="#484f58" font-size="9">ws_id set → route to owning node</text>
|
||||
<circle cx="44" cy="514" r="3" fill="#58a6ff" opacity="0.6"/>
|
||||
<text x="54" y="517" fill="#484f58" font-size="9">neither → shared queue, any node picks up</text></svg>
|
||||
<circle cx="44" cy="474" r="3" fill="#3fb950" opacity="0.6"/>
|
||||
<text x="54" y="477" fill="#484f58" font-size="9">control plane: client → console → server node (hash-ring bucket lookup)</text>
|
||||
<circle cx="44" cy="494" r="3" fill="#58a6ff" opacity="0.6"/>
|
||||
<text x="54" y="497" fill="#484f58" font-size="9">data plane: client → server node (direct SSE, node_url from create response)</text>
|
||||
<circle cx="44" cy="514" r="3" fill="#f47067" opacity="0.6"/>
|
||||
<text x="54" y="517" fill="#484f58" font-size="9">single-node: client → server (direct HTTP + SSE, no console needed)</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 16 KiB |
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d8ce6d2a43a991655c3f64a20b6e810fdb2f78eb767acc3d3d1b8d2c9f443181
|
||||
size 165011
|
||||
oid sha256:881a8b9bce67b5af9a52d5e50deaa72351cd99c76f18aad5caeb2b61131ca1af
|
||||
size 119798
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:efcc7cbe8161a54b5ec24bdfd47e8a142f70029e6e66c707e811b99369f85ebf
|
||||
size 310079
|
||||
oid sha256:75da80e6bd205e45b9fe48aa2f87197110c908f44e1d98c52eee69f956274711
|
||||
size 400402
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:01fbb3338df6426cefc2811541a865f268673b4febf32f524c264d120bc068fa
|
||||
size 589546
|
||||
oid sha256:6471e611beebf647f3a191eb16588571a404cc52a43067883a2b6f06dd936376
|
||||
size 594676
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:da9d32000e3d92d92ce621661ced60f276f9b5be652f5ed6123b400505415f4a
|
||||
size 319702
|
||||
oid sha256:3aa8d972bba40d78152f9f0c762b9f5ec616d8052c45fa52b7dd1c679ed81d61
|
||||
size 325245
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:43844b07d36beb04db871f6795a3f3be17852a6a484fdc0ea207403bd7f512a6
|
||||
size 274286
|
||||
oid sha256:674712a0563f51837383184652efeb28b7bec13378be636e89d2959bfba39d1e
|
||||
size 281519
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6e94a10f039a7f69517e84d0946e0c649035c15b38ebc2314e7b9cd501eb244d
|
||||
size 192559
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:09535722ba975e47cf0557a40b6c481f125ff2022c396f79715c3bba9f715871
|
||||
size 222032
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ed457b10b534b5fc2a5e190b281d7ded4dd1615da2229d67a373cf5dddccd059
|
||||
size 201601
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7896c6e041b6dbb89d034468fa980c8fe645df5eb969d45ef966ccc6399edac2
|
||||
size 200083
|
||||
oid sha256:04d2069a9b5155ad1e7d842147fd78535ad9106d6856520439c33a9868a47499
|
||||
size 156694
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:35cf3a6942f62dabcbbe012ac2f9e6f155332c894692981b076de5a25c1f3330
|
||||
size 374055
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a74b4b8b5dbfb1a51a01100b731477968942b01218bad9451a3d5a9cb3003294
|
||||
size 411665
|
||||
oid sha256:040f7d9ec7d676da40b9487e0825caf2c1574cbdd9f16d0998d90e0c2e4f8861
|
||||
size 360309
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:84524f4bc900708ac8adf081591d336f862830188eb8505e71a0f071b339d923
|
||||
size 252599
|
||||
oid sha256:a872556d111185f4531d1b68ee892b4ce5042d7ccf277e2cad08beb6932c9803
|
||||
size 191144
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fb5e7c221f6b1ee1082b37da32e65c45b5e468014cf6881a210e5b4d8a8dca8b
|
||||
size 255736
|
||||
oid sha256:b047cdc318c505f0f0895a65e14c5cc7552716053055cca57fa0a77db150e618
|
||||
size 255458
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6fc99bb8d84d6e9f3dac9d5c12ac7f569a041b29431c57612c24b50f332982ed
|
||||
size 462992
|
||||
oid sha256:8a5957b71822656098cbe779ca619b7e56e9b11b0f1c4eac4fc880ed62b71a2a
|
||||
size 358670
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3aaca1ae4c6c255dc9569f59e3ccc24f8b3bab0ac2a9b08c85e2af72d6a400c7
|
||||
size 218575
|
||||
oid sha256:98ba80fa1dab4d37299e61be079a6fbc8740fc3ab92196f828a765f74caf4556
|
||||
size 200720
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e4593873599342b2830fedd5d783e9a28eab0bb0d6589798ef6ef2649eeee80f
|
||||
size 324518
|
||||
oid sha256:a6b7769aa7e732ffbeb1eb7f5b65273a135fb3a78d9802ec36d3b92801c34f6b
|
||||
size 427745
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fadf5b07f8230ecf97805a86b308eaa9eb30516dd26900e5c9f70e6fb7562bab
|
||||
size 296339
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:feb31b9d05ea56544053ad00457c389acba977c07ecc08870960e6e0ca64aa11
|
||||
size 279971
|
||||
oid sha256:79a690c466a5d6f6d4292d78a27b9474e9e9c1373fa80e17dfe37700238c8af8
|
||||
size 382508
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:83c0e6aad3eb19f6bc475a30a77215e801da3da5930f0462417fe7eb6eda6be2
|
||||
size 347144
|
||||
oid sha256:06fe076f0835a891e00afc804fd1805196ebde9fc0d34998c7873e87287f982b
|
||||
size 346887
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ba14003062fec7eb9eaca7a3de945767e40bdd821468e19e7d1edcfa7ce1eb41
|
||||
size 193581
|
||||
+15
-48
@@ -1,6 +1,6 @@
|
||||
# Docker Deployment
|
||||
|
||||
Docker Compose stack for running the full turnstone platform or the simulator.
|
||||
Docker Compose stack for running the full turnstone platform.
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -10,9 +10,6 @@ cp .env.example .env
|
||||
|
||||
# Full stack (needs an LLM API on the host)
|
||||
docker compose up
|
||||
|
||||
# Simulator only (no LLM needed)
|
||||
docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
Console dashboard: http://localhost:8090
|
||||
@@ -23,18 +20,14 @@ Console dashboard: http://localhost:8090
|
||||
|
||||
| Service | Port | Profile | Description |
|
||||
|---------|------|---------|-------------|
|
||||
| `redis` | 6379 | default | Message broker, pub/sub, node registry |
|
||||
| `server` | 8080 | default | Web UI + chat workstreams + LLM |
|
||||
| `bridge` | — | default | Redis-to-HTTP bridge (multi-node routing) |
|
||||
| `console` | 8090 | default | Cluster dashboard |
|
||||
| `channel` | — | production | Channel gateway (Discord, Slack, etc.) |
|
||||
| `server-1`…`server-10` | — | cluster | 10-node server fleet (PostgreSQL required) |
|
||||
| `bridge-1`…`bridge-10` | — | cluster | Matching bridge fleet |
|
||||
| `sim` | — | sim | Multi-node cluster simulator |
|
||||
|
||||
## Profiles
|
||||
|
||||
**Default** (no flag) — starts `redis`, `server`, `bridge`, `console`. Requires an OpenAI-compatible LLM API running on the host (default: `http://localhost:8000/v1`).
|
||||
**Default** (no flag) — starts `server` and `console`. Requires an OpenAI-compatible LLM API running on the host (default: `http://localhost:8000/v1`).
|
||||
|
||||
```bash
|
||||
docker compose up
|
||||
@@ -46,22 +39,12 @@ docker compose up
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
**Cluster** — 10-node server/bridge fleet sharing PostgreSQL and Redis. Access all nodes via the console at `:8090`. Requires `POSTGRES_PASSWORD`:
|
||||
**Cluster** — 10-node server fleet sharing PostgreSQL. Access all nodes via the console at `:8090`. Requires `POSTGRES_PASSWORD`:
|
||||
|
||||
```bash
|
||||
docker compose --profile cluster up
|
||||
```
|
||||
|
||||
**Sim** — adds the simulator. Can run alongside the full stack or standalone with just Redis and the console:
|
||||
|
||||
```bash
|
||||
# Sim + console (no LLM needed)
|
||||
docker compose --profile sim up redis console sim
|
||||
|
||||
# Everything including sim
|
||||
docker compose --profile sim up
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration is via environment variables in `.env` (copy from `.env.example`):
|
||||
@@ -74,13 +57,6 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
| `OPENAI_API_KEY` | `dummy` | API key (`dummy` for local servers) |
|
||||
| `TAVILY_API_KEY` | — | Web search API key (only needed for local/vLLM models; Anthropic and OpenAI search models use native search) |
|
||||
|
||||
### Redis
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `REDIS_PASSWORD` | — | Redis auth password (empty = no auth) |
|
||||
| `REDIS_PORT` | `6379` | Host port mapping |
|
||||
|
||||
### Server
|
||||
|
||||
| Variable | Default | Description |
|
||||
@@ -93,15 +69,14 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `CONSOLE_PORT` | `8090` | Host port mapping |
|
||||
| `CONSOLE_POLL_INTERVAL` | `10` | Node polling interval (seconds) |
|
||||
|
||||
### Auth
|
||||
|
||||
Auth is always enabled. `TURNSTONE_JWT_SECRET` is required.
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_AUTH_ENABLED` | — | Set to `1` to require authentication |
|
||||
| `TURNSTONE_AUTH_TOKEN` | — | Config-file token for server/bridge/console (backward compat, works alongside JWT) |
|
||||
| `TURNSTONE_JWT_SECRET` | — | Secret key for signing JWTs (required when using user identity / JWT auth) |
|
||||
| `TURNSTONE_JWT_SECRET` | — | Secret key for signing JWTs (required) |
|
||||
|
||||
### Database
|
||||
|
||||
@@ -109,9 +84,12 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DB_BACKEND` | `sqlite` | Storage backend: `sqlite` or `postgresql` |
|
||||
| `TURNSTONE_DB_URL` | — | Database URL (e.g. `postgresql://user:pass@db:5432/turnstone`). For SQLite, defaults to `/data/.turnstone.db` |
|
||||
| `TURNSTONE_DB_POOL_SIZE` | `2` | PostgreSQL connection pool size per process (default: 2 base + 3 overflow = 5 max) |
|
||||
|
||||
The database stores workstream history, user accounts, and API tokens. When using JWT auth, a database backend is required for user storage.
|
||||
|
||||
> **Large clusters:** Each turnstone process maintains a small connection pool (5 max). At hundreds of nodes this adds up — use [PgBouncer](pgbouncer.md) in transaction pooling mode between turnstone and PostgreSQL.
|
||||
|
||||
> **First-time setup:** After deploying with auth enabled, create an initial admin user by running `turnstone-admin create-user` inside the container:
|
||||
>
|
||||
> ```bash
|
||||
@@ -127,35 +105,24 @@ The database stores workstream history, user accounts, and API tokens. When usin
|
||||
| `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord adapter) |
|
||||
| `TURNSTONE_DISCORD_GUILD` | `0` | Restrict to a single Discord guild (0 = all guilds) |
|
||||
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages through Redis MQ to the bridge and server. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
### Simulator
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `SIM_NODES` | `100` | Number of simulated nodes |
|
||||
| `SIM_SCENARIO` | `steady` | Scenario: `steady`, `burst`, `node_failure`, `directed`, `lifecycle` |
|
||||
| `SIM_DURATION` | `60` | Duration in seconds |
|
||||
| `SIM_MPS` | `5.0` | Messages per second (steady scenario) |
|
||||
| `SIM_LOG_LEVEL` | `INFO` | Log verbosity |
|
||||
| `SIM_SEED` | — | Random seed for reproducibility |
|
||||
| `SIM_METRICS_FILE` | — | Write JSON report to file |
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages to the server via HTTP. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
## Scaling
|
||||
|
||||
For multi-node testing, use the `cluster` profile which provides 10 dedicated server+bridge pairs with unique node IDs (`node-1` through `node-10`), resource limits, and shared PostgreSQL:
|
||||
For multi-node testing, use the `cluster` profile which provides 10 server instances with unique node IDs (`node-1` through `node-10`), resource limits, and shared PostgreSQL:
|
||||
|
||||
```bash
|
||||
POSTGRES_PASSWORD=secret docker compose --profile cluster up
|
||||
```
|
||||
|
||||
The default `server` and `bridge` also run alongside the cluster nodes (11 total). All nodes are accessible via the console dashboard at `:8090`.
|
||||
The default `server` also runs alongside the cluster nodes (11 total). All nodes are accessible via the console dashboard at `:8090`.
|
||||
|
||||
For production clusters beyond ~50 nodes, add PgBouncer between turnstone services and PostgreSQL. See [PgBouncer Connection Pooling](pgbouncer.md) for Docker Compose and Helm configuration.
|
||||
|
||||
## Volumes
|
||||
|
||||
| Volume | Mount | Purpose |
|
||||
|--------|-------|---------|
|
||||
| `redis-data` | `/data` | Redis persistence |
|
||||
| `turnstone-data` | `/data` | SQLite database (`.turnstone.db`) |
|
||||
|
||||
## Building
|
||||
@@ -170,7 +137,7 @@ docker compose build
|
||||
docker compose build --no-cache
|
||||
```
|
||||
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-bridge`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-sim`, `turnstone-eval`.
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-eval`.
|
||||
|
||||
## Cleanup
|
||||
|
||||
|
||||
+254
-68
@@ -2,7 +2,8 @@
|
||||
|
||||
`turnstone-eval` is the evaluation and prompt optimization system for turnstone. It
|
||||
runs test cases against the LLM, scores tool call sequences against expected
|
||||
actions, and optionally uses the model to self-optimize the developer prompt.
|
||||
actions, and optionally uses a multi-agent pipeline to optimize the developer
|
||||
prompt and tool descriptions.
|
||||
|
||||
Source: `turnstone/eval.py`
|
||||
|
||||
@@ -10,15 +11,24 @@ Source: `turnstone/eval.py`
|
||||
|
||||
## Overview
|
||||
|
||||
The system works in an iterative loop:
|
||||
The system uses UCB tree search to explore prompt variants:
|
||||
|
||||
1. Run each test case N times against the current developer prompt.
|
||||
2. Score each run by comparing the actual tool call sequence to expected actions.
|
||||
3. If not all tests pass, use the model to rewrite the prompt based on failures.
|
||||
4. Repeat until all tests pass or max iterations are reached.
|
||||
1. Maintain an **evolution tree** of prompt variants, starting from the initial prompt.
|
||||
2. Each iteration, **UCB1 selects** the most promising node to evaluate.
|
||||
3. Run each test case N times against the selected prompt.
|
||||
4. Score each run by comparing the actual tool call sequence to expected actions.
|
||||
5. If not all tests pass, run a **three-phase optimization pipeline**:
|
||||
- Phase 1: Analyst diagnoses semantic failure patterns
|
||||
- Phase 2: Tool optimizer adjusts tool descriptions (when `--optimize-tools`)
|
||||
- Phase 3: Prompt optimizer proposes a child variant (when not `--optimize-tools`)
|
||||
6. Add the child to the tree and repeat until all tests pass or max iterations reached.
|
||||
|
||||
When optimization is disabled (`--no-optimize`), only step 1 and 2 execute
|
||||
(a single iteration).
|
||||
This approach (inspired by [Learning to Self-Evolve](https://arxiv.org/abs/2603.18620))
|
||||
prevents irrecoverable collapse from bad edits — UCB naturally backtracks to
|
||||
high-scoring ancestors instead of following a linear chain.
|
||||
|
||||
When optimization is disabled (`--no-optimize`), only steps 2-4 execute
|
||||
(a single iteration evaluating the root node).
|
||||
|
||||
---
|
||||
|
||||
@@ -65,6 +75,7 @@ Test suites are JSON files with this structure:
|
||||
| `match_mode` | no | `"ordered_subset"` | How to match actual vs expected actions (see Scoring). |
|
||||
| `max_turns` | no | `10` | Maximum conversation turns before stopping. |
|
||||
| `n_runs` | no | suite default or 3 | Per-case override for number of runs. |
|
||||
| `holdout` | no | `false` | If `true`, this case is evaluated but excluded from optimizer feedback. Used to measure progress without overfitting. |
|
||||
|
||||
### Expected Action Specs
|
||||
|
||||
@@ -135,6 +146,7 @@ deterministic, non-interactive execution suitable for automated testing.
|
||||
| Stdout | Normal | Suppressed during execution |
|
||||
| Tool logging | Display only | Structured `tool_call_log` |
|
||||
| System prompt | Built-in developer prompt | Overridable via constructor |
|
||||
| Cancellation | N/A | `_cancelled` event for timeout cleanup |
|
||||
|
||||
### NullUI
|
||||
|
||||
@@ -156,20 +168,34 @@ def send_headless(
|
||||
Runs a complete multi-turn conversation:
|
||||
|
||||
1. Appends the user message.
|
||||
2. Calls the model API (non-streaming).
|
||||
3. If tool calls are returned, executes them (with stdout suppressed) and
|
||||
2. Checks `_cancelled` event — stops if set (timeout cleanup).
|
||||
3. Calls the model API (non-streaming).
|
||||
4. If tool calls are returned, executes them (with stdout suppressed) and
|
||||
logs each call to `self.tool_call_log`.
|
||||
4. Repeats up to `max_turns` or until the model responds without tool calls.
|
||||
5. Returns the tool call log: list of dicts with keys `tool`, `args`,
|
||||
5. Repeats up to `max_turns` or until the model responds without tool calls.
|
||||
6. Returns the tool call log: list of dicts with keys `tool`, `args`,
|
||||
`result` (truncated to 500 chars), and `turn`.
|
||||
|
||||
Parallel tool calls are capped at 10 per turn to prevent degenerate repetition.
|
||||
|
||||
### Timeout and Cancellation
|
||||
|
||||
Each test runs in a `ThreadPoolExecutor(max_workers=1)` with a per-test
|
||||
timeout (`--test-timeout`). Each attempt gets its own `OpenAI` client with
|
||||
a matching httpx read timeout. On timeout, three layers of defense prevent
|
||||
zombie connections:
|
||||
|
||||
1. **httpx timeout**: Per-request read timeout aborts the HTTP call and
|
||||
releases the server slot.
|
||||
2. **`_cancelled` event**: Prevents the orphan thread from starting new turns.
|
||||
3. **`run_client.close()`**: Closes the connection pool to abort any
|
||||
in-flight request.
|
||||
|
||||
### Retry Logic
|
||||
|
||||
`send_headless()` is called inside `_run_single_test()` with retry logic:
|
||||
3 attempts with exponential backoff (sleep `2^attempt` seconds) on any
|
||||
exception. This prevents transient API errors from poisoning eval scores.
|
||||
exception. `TimeoutError` is re-raised immediately (no retry).
|
||||
|
||||
---
|
||||
|
||||
@@ -180,68 +206,175 @@ Each test case runs in isolation:
|
||||
1. A fresh temp directory is created.
|
||||
2. Setup files are written to the temp directory.
|
||||
3. The working directory is changed to the temp directory.
|
||||
4. A new `HeadlessSession` is created with the current developer prompt.
|
||||
5. `send_headless()` runs the user prompt through the conversation loop.
|
||||
6. The tool log is scored against expected actions.
|
||||
7. The temp directory is cleaned up.
|
||||
4. A per-attempt `OpenAI` client is created with httpx timeout matching `--test-timeout`.
|
||||
5. A new `HeadlessSession` is created with the current developer prompt.
|
||||
6. `send_headless()` runs the user prompt through the conversation loop.
|
||||
7. The tool log is scored against expected actions.
|
||||
8. The temp directory is cleaned up.
|
||||
|
||||
The memory database is also isolated per test (an ephemeral SQLite database
|
||||
in the temp directory) so tests do not pollute each other or the user's
|
||||
real memory store.
|
||||
|
||||
### Parallel Execution
|
||||
|
||||
With `--parallel N` (N > 1), tests run in a `ProcessPoolExecutor` with N
|
||||
workers. Each subprocess creates its own `OpenAI` client. This is suitable
|
||||
for remote API endpoints but will overwhelm local inference servers. The
|
||||
default (`--parallel 1`) runs tests serially.
|
||||
|
||||
---
|
||||
|
||||
## Optimization Loop
|
||||
## Model Roles
|
||||
|
||||
`run_optimization()` is the main entry point for iterative prompt optimization.
|
||||
The eval pipeline uses up to five separate model roles, each independently
|
||||
configurable. All roles inherit from the test model by default, with a
|
||||
cascade chain:
|
||||
|
||||
```
|
||||
test model (--base-url, --model)
|
||||
└─ optimizer (--optimizer-*)
|
||||
├─ observer (--observer-*)
|
||||
├─ analyst (--analyst-*)
|
||||
├─ diversifier (--diversifier-*)
|
||||
└─ tool optimizer (--tool-optimizer-*)
|
||||
```
|
||||
|
||||
| Role | Purpose | When it runs |
|
||||
|------|---------|--------------|
|
||||
| **Test** | The model being evaluated | Every iteration |
|
||||
| **Analyst** | Diagnoses semantic failure patterns with tool use | When pass rate < 100% |
|
||||
| **Optimizer** | Rewrites the developer prompt | Every iteration (unless `--optimize-tools`) |
|
||||
| **Tool optimizer** | Rewrites tool descriptions | When `--optimize-tools` is set |
|
||||
| **Observer** | Tunes the optimizer's strategy | Every 3 iterations |
|
||||
| **Diversifier** | Generates prompt paraphrases | Once before the loop (when `--diversify N`) |
|
||||
|
||||
Typical setup: local model for test, Opus for analyst, Sonnet for
|
||||
optimizer/observer/diversifier.
|
||||
|
||||
---
|
||||
|
||||
## Optimization Pipeline
|
||||
|
||||
### Flow
|
||||
|
||||
```
|
||||
for iteration in 0..max_iterations:
|
||||
1. Run all test cases n_runs times with current prompt
|
||||
2. Score and aggregate results
|
||||
3. Save intermediate results to JSON
|
||||
4. If all tests pass -> stop
|
||||
5. Every 3 iterations (at iteration 2, 5, 8, ...):
|
||||
-> Observer reviews optimizer strategy
|
||||
-> Reset prompt to best-performing iteration
|
||||
6. Propose new prompt via optimizer model call
|
||||
7. If prompt unchanged -> stop
|
||||
8. Continue with new prompt
|
||||
1. UCB select → pick the most promising tree node
|
||||
2. Run all test cases n_runs times with selected node's prompt
|
||||
3. Update node score (rolling mean) and visit count
|
||||
4. Save intermediate results + tree state to JSON
|
||||
5. If all tests pass → stop
|
||||
6. Phase 1: Analyst diagnoses semantic failure patterns
|
||||
7. Phase 2 (--optimize-tools only): Tool optimizer adjusts descriptions
|
||||
8. Phase 3 (default only): Prompt optimizer proposes new prompt
|
||||
9. Every 3 iterations: Observer tunes the optimizer's strategy
|
||||
10. Add child node to tree (if prompt or tools changed)
|
||||
```
|
||||
|
||||
### Prompt Proposal (`_propose_prompt_modification`)
|
||||
### Phase 1: Analyst (`_run_analyst`)
|
||||
|
||||
Uses the model to rewrite the developer prompt based on test results:
|
||||
A multi-turn agent with `math` (Python) and `bash` tools for computing
|
||||
statistics. It receives per-case results with failure classifications and
|
||||
produces a structured diagnosis:
|
||||
|
||||
- **Input**: Current prompt, test case definitions, per-case results with
|
||||
actual vs expected tool sequences, and a history of the last 3 iterations.
|
||||
- **Optimizer system prompt** (`OPTIMIZER_SYSTEM`): Instructs the model to
|
||||
act as a text rewriter. Key guidance includes:
|
||||
- Address critical failure modes (text-only responses, write_file vs edit_file,
|
||||
unnecessary search before create, missing plan calls).
|
||||
- Preserve phrasing that drives 100% pass rate on passing tests.
|
||||
- Use direct imperative style with concrete tool call examples.
|
||||
- Stay within 130% of original prompt length.
|
||||
- **Output**: The rewritten prompt text (stripped of reasoning tags and code fences).
|
||||
- **Failure patterns**: Shared root causes across failing cases
|
||||
- **Success/failure contrast**: What distinguishes passing from failing cases
|
||||
- **Consistency signals**: Systematic (0%), flaky (1-79%), marginal (80-99%)
|
||||
- **Recommended fixes**: Priority-ordered patterns/examples to add or adjust
|
||||
|
||||
### Observer System (`_observe_and_update_optimizer`)
|
||||
The analyst is instructed to frame fixes as patterns and examples, not
|
||||
imperative rules — this feeds cleaner signal to the optimizer.
|
||||
|
||||
Every 3 iterations, a meta-level "observer" reviews the optimizer's strategy:
|
||||
In `--optimize-tools` mode, the analyst receives the current tool descriptions
|
||||
(with any overrides applied) and focuses on tool confusion and description
|
||||
issues rather than system prompt patterns.
|
||||
|
||||
- Analyzes the iteration history: score trends, regressions, prompt length changes,
|
||||
### Phase 2: Tool Optimizer (`_propose_tool_overrides`)
|
||||
|
||||
Runs when `--optimize-tools` is set. Receives the current tool descriptions,
|
||||
confusion failures (where the model picked the wrong tool), and the analyst's
|
||||
diagnosis. Returns a JSON override dict that modifies tool descriptions.
|
||||
|
||||
Overrides are validated against known tool names — only `description` and
|
||||
`parameters` changes are accepted (no tool renaming at eval time).
|
||||
|
||||
After each iteration, changed descriptions are logged as old → new diffs
|
||||
for easy visual inspection.
|
||||
|
||||
### Phase 3: Prompt Optimizer (`_propose_prompt_modification`)
|
||||
|
||||
Skipped in `--optimize-tools` mode. Receives the current prompt, test
|
||||
results with per-case pass rates and deltas from the parent node, and the
|
||||
analyst's diagnosis. Returns a rewritten prompt.
|
||||
|
||||
The optimizer is instructed to prefer patterns over rules — concrete tool
|
||||
chain examples teach better than imperative directives like "ALWAYS" or
|
||||
"NEVER." If the current prompt contains rule-heavy language, the optimizer
|
||||
is guided to replace it with examples.
|
||||
|
||||
### Two Optimization Surfaces
|
||||
|
||||
The system supports alternating between two optimization surfaces:
|
||||
|
||||
1. **System prompt optimization** (default): Freeze tool descriptions,
|
||||
optimize the developer prompt. Run until scores plateau.
|
||||
2. **Tool description optimization** (`--optimize-tools`): Freeze the system
|
||||
prompt, optimize tool descriptions only. Run until scores plateau.
|
||||
|
||||
Each surface lifts the floor for the other — tool description improvements
|
||||
may unlock system prompt gains that weren't reachable before, and vice versa.
|
||||
|
||||
### Observer (`_observe_and_update_optimizer`)
|
||||
|
||||
Every 3 iterations, a meta-level observer reviews the optimizer's strategy:
|
||||
|
||||
- Analyzes iteration history: score trends, regressions, prompt length changes,
|
||||
and diffs between iterations.
|
||||
- Summarizes the optimizer's behavioral patterns (list style, header usage, length).
|
||||
- Uses `OBSERVER_SYSTEM` to rewrite the optimizer's own system prompt.
|
||||
- Detects whether the optimizer is producing rule-heavy or pattern-based output.
|
||||
- Rewrites the optimizer's own system prompt to correct course.
|
||||
- Rejects degenerate outputs (over 200% of input length).
|
||||
- After updating the optimizer prompt, resets the developer prompt to the
|
||||
best-performing iteration so far.
|
||||
|
||||
This two-level optimization (optimizer + observer) helps the system escape
|
||||
local minima and adjust its rewriting strategy.
|
||||
### Prompt Diversification
|
||||
|
||||
### Result Persistence
|
||||
When `--diversify N` is set, the diversifier generates N paraphrased variants
|
||||
of each test case's user prompt before the optimization loop. Each run cycles
|
||||
through variants (round-robin), testing robustness across phrasings.
|
||||
|
||||
Variants can be cached back to the test suite JSON with `--save-variants`,
|
||||
and auto-loaded on subsequent runs even without `--diversify`.
|
||||
|
||||
---
|
||||
|
||||
## Evolution Tree
|
||||
|
||||
The optimization maintains a tree of prompt variants (`EvolutionNode`), where
|
||||
each node stores its prompt text, tool overrides, aggregated score, and visit
|
||||
count. The root node (ID 0) contains the initial prompt.
|
||||
|
||||
**UCB1 selection**: Each iteration picks the node with the highest Upper
|
||||
Confidence Bound score: `R_bar + C * sqrt(ln(N) / v)`, where `R_bar` is the
|
||||
node's mean score, `N` is total visits across all nodes, `v` is the node's
|
||||
visit count, and `C` is the exploration constant (`--explore-constant`,
|
||||
default sqrt(2)). Unvisited nodes are always selected first.
|
||||
|
||||
### Holdout Cases
|
||||
|
||||
Test cases with `"holdout": true` are evaluated every iteration but excluded
|
||||
from the optimizer's feedback. This prevents the optimizer from overfitting
|
||||
to specific test cases. Node scores are computed from holdout cases only
|
||||
(when present). If fewer than 2 non-holdout cases remain, holdout is disabled.
|
||||
|
||||
### Improvement-Based Feedback
|
||||
|
||||
The optimizer sees delta scores (`delta=+20%`) alongside absolute pass rates,
|
||||
showing how each case improved relative to the parent node's evaluation. This
|
||||
provides a cleaner signal than absolute scores alone — the optimizer can
|
||||
distinguish beneficial edits from harmful ones regardless of starting point.
|
||||
|
||||
---
|
||||
|
||||
## Result Persistence
|
||||
|
||||
After each iteration, results are written to the output JSON file. The
|
||||
structure is:
|
||||
@@ -251,9 +384,15 @@ structure is:
|
||||
"meta": {
|
||||
"model": "model-name",
|
||||
"base_url": "http://localhost:8000/v1",
|
||||
"optimizer_model": "claude-opus-4-6",
|
||||
"observer_model": "claude-opus-4-6",
|
||||
"started": "2025-01-01T00:00:00",
|
||||
"test_suite": "tests.json",
|
||||
"n_runs_default": 3
|
||||
"n_runs_default": 3,
|
||||
"explore_constant": 1.414,
|
||||
"holdout_ids": [],
|
||||
"diversify": 10,
|
||||
"prompt_variants": {"case_id": ["variant1", "variant2"]}
|
||||
},
|
||||
"iterations": [
|
||||
{
|
||||
@@ -261,7 +400,11 @@ structure is:
|
||||
"prompt": "the developer prompt used",
|
||||
"prompt_diff": null,
|
||||
"optimizer_system": "the optimizer system prompt",
|
||||
"analyst": "analyst diagnosis output",
|
||||
"tool_overrides": {"bash": {"description": "..."}},
|
||||
"timestamp": "2025-01-01T00:01:00",
|
||||
"tree_node_id": 0,
|
||||
"tree_child_id": 1,
|
||||
"cases": {
|
||||
"test_name": {
|
||||
"runs": [
|
||||
@@ -287,9 +430,21 @@ structure is:
|
||||
"overall_pass_rate": 0.8,
|
||||
"overall_avg_score": 0.87,
|
||||
"json_dumps": 0,
|
||||
"per_case_pass_rates": {"test_name": 1.0, ...}
|
||||
"per_case_pass_rates": {"test_name": 1.0}
|
||||
}
|
||||
}
|
||||
],
|
||||
"tree": [
|
||||
{
|
||||
"node_id": 0,
|
||||
"parent_id": null,
|
||||
"prompt": "initial prompt",
|
||||
"tool_overrides": {},
|
||||
"score": 0.85,
|
||||
"visit_count": 3,
|
||||
"children": [1, 2],
|
||||
"iteration": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -306,26 +461,57 @@ turnstone-eval tests.json # evaluate + optimize
|
||||
turnstone-eval tests.json --no-optimize # evaluate only (single iteration)
|
||||
turnstone-eval tests.json --n-runs 5 --max-iter 10 # more thorough evaluation
|
||||
turnstone-eval tests.json --prompt custom.txt # start from a custom prompt
|
||||
turnstone-eval tests.json --optimize-tools # optimize tool descriptions only
|
||||
turnstone-eval tests.json --diversify 10 # test with prompt variants
|
||||
turnstone-eval tests.json -v # verbose per-turn logging
|
||||
```
|
||||
|
||||
### Multi-model setup (local test model, cloud optimizer)
|
||||
|
||||
```
|
||||
turnstone-eval tests.json \
|
||||
--base-url http://localhost:8000/v1 \
|
||||
--optimizer-base-url https://api.anthropic.com \
|
||||
--optimizer-model claude-sonnet-4-6 \
|
||||
--analyst-model claude-opus-4-6
|
||||
```
|
||||
|
||||
### All Options
|
||||
|
||||
| Flag | Default | Description |
|
||||
|---------------------|-------------------------------|-------------|
|
||||
| `test_file` | (positional, required) | Path to test cases JSON file. |
|
||||
| `--base-url` | `http://localhost:8000/v1` | API base URL. |
|
||||
| `--model` | auto-detect | Model name. Auto-detected from the API if not specified. |
|
||||
| `--prompt` | turnstone built-in prompt | Path to initial prompt text file. |
|
||||
| `--n-runs` | from tests.json or 3 | Number of runs per test case. |
|
||||
| `--max-iter` | 5 | Maximum optimization iterations. |
|
||||
| `--no-optimize` | false | Run evaluation only (sets max-iter to 1). |
|
||||
| `--temperature` | 0.7 | Sampling temperature. |
|
||||
| `--max-tokens` | 32768 | Max completion tokens. |
|
||||
| `--reasoning-effort` | `medium` | Reasoning effort: `low`, `medium`, or `high`. |
|
||||
| `--context-window` | 131072 | Context window size. |
|
||||
| `--output` | `eval_results.json` | Output results file path. |
|
||||
| `-v`, `--verbose` | false | Show detailed per-turn logging (API calls, tool args, results). |
|
||||
| Flag | Default | Description |
|
||||
|-------------------------|----------------------------|-------------|
|
||||
| `test_file` | (positional, required) | Path to test cases JSON file. |
|
||||
| `--base-url` | `http://localhost:8000/v1` | API base URL for the test model. |
|
||||
| `--model` | auto-detect | Model name. Auto-detected from the API if not specified. |
|
||||
| `--prompt` | turnstone built-in prompt | Path to initial prompt text file. |
|
||||
| `--n-runs` | from tests.json or 3 | Number of runs per test case. |
|
||||
| `--max-iter` | 5 | Maximum optimization iterations. |
|
||||
| `--no-optimize` | false | Run evaluation only (sets max-iter to 1). |
|
||||
| `--temperature` | 0.7 | Sampling temperature. |
|
||||
| `--max-tokens` | 32768 | Max completion tokens. |
|
||||
| `--reasoning-effort` | `medium` | Reasoning effort: `low`, `medium`, or `high`. |
|
||||
| `--context-window` | 131072 | Context window size. |
|
||||
| `--output` | `eval_results.json` | Output results file path. |
|
||||
| `-v`, `--verbose` | false | Show detailed per-turn logging. |
|
||||
| `--explore-constant` | 1.414 (sqrt(2)) | UCB exploration constant C. |
|
||||
| `--test-timeout` | 300 | Per-test timeout in seconds. |
|
||||
| `--suite-timeout` | 0 (unlimited) | Total suite timeout in seconds. |
|
||||
| `--no-fast-fail` | false | Disable early termination on all-zero initial runs. |
|
||||
| `--parallel` | 1 (serial) | Parallel workers (0=auto, N=use N workers). |
|
||||
| `--optimizer-model` | same as `--model` | Model for prompt optimization. |
|
||||
| `--optimizer-base-url` | same as `--base-url` | Base URL for optimizer model. |
|
||||
| `--observer-model` | same as optimizer | Model for meta-optimization (observer). |
|
||||
| `--observer-base-url` | same as optimizer | Base URL for observer model. |
|
||||
| `--analyst-model` | same as optimizer | Model for failure analysis. |
|
||||
| `--analyst-base-url` | same as optimizer | Base URL for analyst model. |
|
||||
| `--diversify` | 0 (disabled) | Generate N prompt variants per test case. |
|
||||
| `--diversifier-model` | same as optimizer | Model for prompt diversification. |
|
||||
| `--diversifier-base-url`| same as optimizer | Base URL for diversifier model. |
|
||||
| `--save-variants` | false | Save generated variants back to test suite JSON. |
|
||||
| `--optimize-tools` | false | Optimize tool descriptions only (freeze system prompt). |
|
||||
| `--tool-optimizer-model` | same as optimizer | Model for tool description optimization. |
|
||||
| `--tool-optimizer-base-url` | same as optimizer | Base URL for tool optimizer model. |
|
||||
| `--save-tools` | false | Write optimized tool descriptions back to `turnstone/tools/*.json`. |
|
||||
|
||||
### Precedence for n_runs
|
||||
|
||||
|
||||
+69
-51
@@ -1,8 +1,7 @@
|
||||
# Governance
|
||||
|
||||
Turnstone governance provides role-based access control (RBAC), tool execution
|
||||
policies, prompt templates, usage tracking, and audit logging for the admin
|
||||
console.
|
||||
policies, skills, usage tracking, and audit logging for the admin console.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -21,7 +20,7 @@ The permission model has two layers:
|
||||
|
||||
| Role | Permissions |
|
||||
|------|-------------|
|
||||
| admin | read, write, approve, admin.users, admin.roles, admin.orgs, admin.policies, admin.templates, admin.audit, admin.usage, admin.schedules, admin.watches, tools.approve, workstreams.create, workstreams.close |
|
||||
| admin | read, write, approve, admin.users, admin.roles, admin.orgs, admin.policies, admin.skills, admin.audit, admin.usage, admin.schedules, admin.watches, tools.approve, workstreams.create, workstreams.close |
|
||||
| operator | read, write, workstreams.create, workstreams.close |
|
||||
| viewer | read |
|
||||
|
||||
@@ -51,68 +50,90 @@ Admin-defined rules that control tool execution:
|
||||
`mcp__*` to require approval for all)
|
||||
- Built-in tools continue to use `func_name` for backward compatibility
|
||||
|
||||
### Prompt Templates
|
||||
### Skills
|
||||
|
||||
Admin-curated system message templates injected at workstream startup:
|
||||
Admin-curated system message skills injected at workstream startup. Skills also
|
||||
include session configuration (model, temperature, auto-approve, token budget,
|
||||
etc.) since workstream templates were merged into the skills system in v0.8.0.
|
||||
|
||||
- **Runtime behavior**: Templates are loaded once at session creation and injected
|
||||
into the system message *before* user `instructions`. Templates set the baseline;
|
||||
- **Runtime behavior**: Skills are loaded once at session creation and injected
|
||||
into the system message *before* user `instructions`. Skills set the baseline;
|
||||
instructions customize per-workstream behavior.
|
||||
- **Default templates**: All `is_default=true` templates auto-apply to new
|
||||
- **Default skills**: All `is_default=true` skills auto-apply to new
|
||||
workstreams, concatenated in alphabetical order by name. Use name prefixes
|
||||
(e.g. `01-safety`, `02-style`) to control ordering.
|
||||
- **Explicit selection**: `--template <name>` CLI flag, `template` field on
|
||||
`POST /v1/api/workstreams/new`, console creation modal dropdown, scheduled task
|
||||
config, and channel adapter config. An explicit template *replaces* defaults.
|
||||
config, and channel adapter config. An explicit skill *replaces* defaults.
|
||||
- **Variables**: Three built-in placeholders resolved at load time:
|
||||
`{{model}}` (active model name), `{{ws_id}}` (workstream ID),
|
||||
`{{node_id}}` (server node ID). Unrecognized placeholders are kept as-is.
|
||||
- **Runtime switching**: `/template <name>` to switch, `/template clear` to revert
|
||||
to defaults, `/template` to show current. Persisted across resume.
|
||||
- **Model-driven loading**: The `skill` built-in tool lets the model
|
||||
discover and activate skills mid-conversation. `search` action finds skills
|
||||
by query (auto-approved); `load` action activates by name (requires user
|
||||
approval since it changes session behavior). Main session only.
|
||||
- **Categories**: general, engineering, support, custom, mcp
|
||||
- **Content limit**: 32 KB per template (enforced on create/update)
|
||||
- **Storage**: `prompt_templates` table with JSON `variables` array. Migration 010
|
||||
adds `template` column to `scheduled_tasks`.
|
||||
- **MCP sync**: MCP server prompts auto-sync into prompt_templates with
|
||||
`origin="mcp"`, `mcp_server` set, and `readonly=True`. Manual templates take
|
||||
- **Content limit**: 32 KB per skill (enforced on create/update)
|
||||
- **Storage**: `prompt_templates` table (stores skills) with JSON `variables`
|
||||
array. Migration 010 adds `template` column to `scheduled_tasks`.
|
||||
- **MCP sync**: MCP server prompts auto-sync into the `prompt_templates` table
|
||||
with `origin="mcp"`, `mcp_server` set, and `readonly=True`. Manual skills take
|
||||
precedence on name collision. MCP-synced content updates reset `is_default` to
|
||||
prevent compromised servers from injecting defaults. Admin UI shows origin badge
|
||||
and disables edit/delete for MCP-sourced templates.
|
||||
|
||||
### Workstream Templates
|
||||
|
||||
Workstream templates are behavioral profiles applied at workstream creation — the next level beyond prompt templates. While prompt templates inject system message text, workstream templates define the complete workstream configuration.
|
||||
|
||||
**What they define:**
|
||||
- System prompt (inline text OR reference to a prompt template by name)
|
||||
- Model override (empty = server default)
|
||||
- Temperature, reasoning effort, max tokens, agent max turns
|
||||
- Auto-approve policy (blanket and/or per-tool list)
|
||||
- Token budget (0 = unlimited; warns at 80%, requires approval at 100%)
|
||||
- Completion notification config (stored for v2 dispatch)
|
||||
|
||||
**Storage:** `workstream_templates` table (migration 011) with auto-versioning. Edits snapshot the pre-update state into `workstream_template_versions`. Workstreams record which template and version spawned them via `ws_template_id` + `ws_template_version` columns.
|
||||
|
||||
**Applied once at creation:** Template settings are snapshot-applied to the workstream's config. Not a live binding — template updates don't affect running workstreams.
|
||||
|
||||
**Prompt template drift detection:** When a workstream template references a prompt template, a SHA-256 hash of the prompt content is stored at ws_template create/update time. At workstream creation, the server compares the stored hash against current content and logs a warning on mismatch.
|
||||
|
||||
**Admin API:** 7 endpoints under `/v1/api/admin/ws-templates` (list, create, get, update, delete, version history) plus a read-only summary at `/v1/api/ws-templates`. Permission: `admin.ws_templates`.
|
||||
|
||||
**Console UI:** "WS Templates" tab with CRUD table, create/edit modals (name, description, system prompt source toggle, model, auto-approve, per-tool auto-approve, temperature, reasoning effort, max tokens, agent max turns, token budget, enabled), and version history modal. "Profile" dropdown on workstream creation modal. "WS Template" dropdown on scheduler create/edit modals.
|
||||
|
||||
**Token budget enforcement:** Tracked in `session.send()`. At 80% consumption, emits an info message. At 100%, the next turn requires explicit approval via the `__budget_override__` synthetic tool name (reuses existing approval UI — inline in browser, Discord buttons, bridge auto-approve). The synthetic name can be targeted by tool policies (e.g. `__budget_override__` → `allow` for admins).
|
||||
|
||||
**SDK:** Python (`list_ws_templates`, `create_ws_template`, `get_ws_template`, `update_ws_template`, `delete_ws_template`, `list_ws_template_versions`) and TypeScript (`listWsTemplates`, `createWsTemplate`, etc.) on both sync and async console clients. `ws_template` parameter on `create_workstream()` for both server and console SDKs.
|
||||
and disables edit/delete for MCP-sourced skills.
|
||||
- **Spec fields**: Skills support the full Agent Skills standard frontmatter:
|
||||
`name`, `description`, `license`, `compatibility`, `metadata` (author, version),
|
||||
`allowed-tools`. The `license` and `compatibility` fields are preserved on import
|
||||
and editable in the admin UI. See https://agentskills.io/specification.
|
||||
- **Security scanning**: Skills are automatically scanned at creation and update
|
||||
time. The scanner evaluates four risk axes: content risk (command execution,
|
||||
data exfiltration), supply chain risk (pipe-to-shell, transitive installs),
|
||||
vulnerability risk (prompt injection, insecure credentials), and declared
|
||||
capability risk (from `allowed-tools` in SKILL.md). Results populate the `scan_status`
|
||||
(safe/low/medium/high/critical) and `scan_report` (JSON breakdown) columns.
|
||||
These fields are system-managed and cannot be overwritten via the admin API.
|
||||
- **Discovery**: External skills can be discovered and installed from registries:
|
||||
- `GET /v1/api/admin/skills/discover?q=...` — search the skills.sh registry
|
||||
(or a custom registry via `skills.discovery_url` setting)
|
||||
- `POST /v1/api/admin/skills/install` — install from skills.sh or GitHub.
|
||||
Fetches the `SKILL.md` file, parses YAML frontmatter, creates a skill with
|
||||
`origin="source"` and `readonly=True`, stores bundled resources.
|
||||
- Admin UI: Skills tab has "Installed" / "Discover" pill toggle.
|
||||
Discovery view has search bar, result cards, and "Import from GitHub" modal.
|
||||
- SDK: `discover_skills(q)` and `install_skill(source, skill_id=..., url=...)`
|
||||
on both Python and TypeScript console clients.
|
||||
- **Runtime config on installed skills**: Installed (readonly) skills can have
|
||||
their runtime configuration edited — model, temperature, reasoning effort,
|
||||
token budget, max tokens, agent max turns, auto-approve, allowed tools,
|
||||
and enabled flag. The server restricts updates to these fields only via
|
||||
`_SKILL_RUNTIME_CONFIG_FIELDS` filtering; spec/content fields (name,
|
||||
description, tags, license, compatibility, content, activation) remain
|
||||
immutable. The admin UI shows "Save Config" instead of "Save" for these
|
||||
skills. Audit action: `skill.update.config`.
|
||||
- **Admin UI**: Create/Edit skill modals use a two-column spec manifest layout
|
||||
(left: Identity / Manifest / Deployment; right: Skill Content editor with
|
||||
monospace font). Runtime Config is a collapsible 3-column grid below.
|
||||
License uses an SPDX identifier dropdown (MIT, Apache-2.0, GPL-3.0, etc.).
|
||||
Installed skills show a cyan origin badge with source URL, spec fields are
|
||||
disabled, and all collapsible sections auto-expand in view mode.
|
||||
|
||||
### Usage Tracking
|
||||
|
||||
Per-LLM-request token and tool call metrics:
|
||||
|
||||
- **Recording**: `on_status()` in `WebUI` records a `usage_event` after each
|
||||
LLM response with prompt/completion tokens, tool call count, model, ws_id
|
||||
LLM response with prompt/completion tokens, cache tokens, tool call count,
|
||||
model, ws_id
|
||||
- **Prompt caching**: Anthropic automatic caching (`cache_control: ephemeral`)
|
||||
and OpenAI extended retention (`prompt_cache_retention: 24h` for GPT-5.x)
|
||||
are enabled by default. `cache_creation_tokens` and `cache_read_tokens` are
|
||||
tracked per request in `usage_events` and surfaced in the Usage admin tab
|
||||
- **Querying**: `GET /v1/api/admin/usage` with `group_by` (day/hour/model/user)
|
||||
and time range filtering
|
||||
and time range filtering — includes cache token aggregates
|
||||
- **Prometheus**: `turnstone_tokens_total{type="cache_creation|cache_read"}`
|
||||
counters on `/metrics`
|
||||
- **Pruning**: `prune_usage_events(retention_days=90)` and
|
||||
`prune_audit_events(retention_days=365)` run automatically via the
|
||||
console scheduler's periodic cleanup cycle
|
||||
@@ -126,7 +147,7 @@ Append-only trail of admin actions:
|
||||
channel.link, channel.unlink, role.create, role.update, role.delete,
|
||||
role.assign, role.unassign, policy.create, policy.update, policy.delete,
|
||||
template.create, template.update, template.delete,
|
||||
ws_template.create, ws_template.update, ws_template.delete, org.update
|
||||
skill.create, skill.update, skill.delete, org.update
|
||||
- **Querying**: `GET /v1/api/admin/audit` with action/user/time filters + pagination
|
||||
|
||||
## Database Schema
|
||||
@@ -139,8 +160,8 @@ Migration 008 adds 7 tables:
|
||||
| `roles` | Named permission bundles (3 builtin + custom) |
|
||||
| `user_roles` | User-to-role assignments (composite PK) |
|
||||
| `tool_policies` | Per-tool approve/deny/ask rules |
|
||||
| `prompt_templates` | Reusable system message templates |
|
||||
| `usage_events` | Per-request token/tool metrics |
|
||||
| `prompt_templates` | Reusable system message skills |
|
||||
| `usage_events` | Per-request token/tool/cache metrics |
|
||||
| `audit_events` | Admin action log |
|
||||
|
||||
Also adds `org_id` column to `users` table.
|
||||
@@ -155,9 +176,8 @@ All under `/v1/api/admin/` (requires `approve` scope + granular permission).
|
||||
| Roles | 7 (CRUD + assignment) | `admin.roles` / `admin.users` |
|
||||
| Orgs | 3 (list, get, update) | `admin.orgs` |
|
||||
| Tool Policies | 4 (CRUD) | `admin.policies` |
|
||||
| Prompt Templates | 4 (CRUD) | `admin.templates` |
|
||||
| Skills | 4 (CRUD) | `admin.skills` |
|
||||
| Schedules | 6 (CRUD + runs) | `admin.schedules` |
|
||||
| WS Templates | 7 (CRUD + versions + summary) | `admin.ws_templates` |
|
||||
| Watches | 3 (list, create, cancel) | `admin.watches` |
|
||||
| Usage | 1 (aggregated query) | `admin.usage` |
|
||||
| Audit | 1 (paginated, filtered) | `admin.audit` |
|
||||
@@ -170,8 +190,7 @@ Full OpenAPI spec at `/openapi.json` and Swagger UI at `/docs`.
|
||||
|
||||
- **Roles** — CRUD roles, permission checkbox grid, user role assignment modal
|
||||
- **Policies** — CRUD tool policies with colored action badges (green/red/amber)
|
||||
- **Templates** — CRUD prompt templates with wide modal, textarea editor
|
||||
- **WS Templates** — CRUD workstream templates with create/edit modals, version history
|
||||
- **Skills** — CRUD skills with wide modal, textarea editor
|
||||
- **Usage** — Summary readouts + CSS bar chart, time range + group-by selectors
|
||||
- **Audit** — Filterable log with relative timestamps, load-more pagination
|
||||
|
||||
@@ -187,7 +206,6 @@ Both Python and TypeScript console SDKs expose governance methods:
|
||||
- `list_orgs()`, `get_org()`, `update_org()`
|
||||
- `list_policies()`, `create_policy()`, `update_policy()`, `delete_policy()`
|
||||
- `list_templates()`, `create_template()`, `update_template()`, `delete_template()`
|
||||
- `list_ws_templates()`, `create_ws_template()`, `get_ws_template()`, `update_ws_template()`, `delete_ws_template()`, `list_ws_template_versions()`
|
||||
- `get_usage(since, group_by=...)`, `get_audit(action=..., limit=...)`
|
||||
|
||||
**TypeScript** (`TurnstoneConsole`):
|
||||
|
||||
+161
-11
@@ -85,14 +85,14 @@ last) and returns the first matching rule. Each rule has:
|
||||
argument text (command string for bash, path for file tools, JSON for others)
|
||||
- **Risk level, confidence, and recommendation**: Pre-assigned per rule
|
||||
|
||||
### Rule tiers
|
||||
### Rule tiers (36 rules)
|
||||
|
||||
| Tier | Confidence | Recommendation | Examples |
|
||||
|----------|-----------|----------------|----------|
|
||||
| Critical | 0.90 | deny | `rm -rf /`, `mkfs`, `dd if=`, pipe-to-shell, chmod 777 on root, write/edit to `/etc/`, `.ssh/` |
|
||||
| High | 0.80 | review | `sudo`, `kill -9`, destructive git (`reset --hard`, `push --force`, `clean -f`), DROP TABLE, write/edit secrets (`.env`, `.pem`, `.key`), HTTP mutations, `ssh`/`scp` |
|
||||
| Medium | 0.70 | review | Package installs (`pip`, `npm`, `apt`, `brew`, `cargo`), `write_file` (default), MCP tool calls, Docker operations |
|
||||
| Low | 0.85 | approve | `read_file`, `list_directory`, `search`, `recall`, `man`, `use_prompt`, read-only bash commands (`ls`, `cat`, `head`, `grep`, `find`, etc.) |
|
||||
| Critical | 0.90 | deny | `rm -rf /`, `mkfs`, `dd if=`, pipe-to-shell, chmod 777 on root, write/edit to `/etc/` or `.ssh/`, download-then-execute chains (`curl -o file && chmod +x && bash`) |
|
||||
| High | 0.80 | review | `sudo`, `kill -9`, destructive git, DROP TABLE, write/edit secrets, HTTP mutations, `ssh`/`scp`, credential file access, browser automation + data export, transitive installs (`npx skills add`, `pip install git+`), control plane mutations (`crontab`, `systemctl enable/start/stop`) |
|
||||
| Medium | 0.70 | review | Content ingestion pipelines (`curl \| python3`), interpreter execution (`python3 script.py`, `node build.js`), cloud CLI mutations (`az/gcloud/aws/kubectl/terraform` with create/delete/destroy verbs), package installs, `write_file`, MCP tools, Docker operations |
|
||||
| Low | 0.85 | approve | `read_file`, `list_directory`, `search`, `recall`, `man`, `use_prompt`, `tool_search`, `read_resource`, `web_search`, read-only bash (`ls`, `cat`, `head`, `grep`, `find`, etc.) |
|
||||
|
||||
When no rule matches, the heuristic returns a default verdict: medium risk,
|
||||
0.50 confidence, "review" recommendation.
|
||||
@@ -100,6 +100,29 @@ When no rule matches, the heuristic returns a default verdict: medium risk,
|
||||
The bash "read-only" rule handles simple pipelines and command chains by
|
||||
splitting on `|`, `&&`, `||`, and `;`, then checking each segment individually.
|
||||
|
||||
### Rules derived from audit data
|
||||
|
||||
Several rules were calibrated using analysis of 25K public agent skill
|
||||
security audits across three independent auditors:
|
||||
|
||||
- **`download-exec`**: Two-step download-then-execute chains that bypass the
|
||||
existing `pipe-to-shell` rule. 8% of critical-tier skills use this pattern.
|
||||
- **`transitive-install`**: Installing packages from URLs or git repos rather
|
||||
than vetted registries. Socket flags this as supply-chain critical in 36%
|
||||
of dangerous skills.
|
||||
- **`browser-data-export`**: Browser automation combined with cookie/session/
|
||||
profile export. OpenClaw treats browser profile access as operator-level
|
||||
capability.
|
||||
- **`control-plane-mutation`**: Persistent system changes (crontab, systemd)
|
||||
that outlive the session. OpenClaw denies control-plane tools by default.
|
||||
- **`content-ingestion`**: Fetch-and-process pipelines where remote content
|
||||
feeds into an interpreter (Snyk W011 pattern — indirect prompt injection
|
||||
surface).
|
||||
- **`interpreter-exec`**: Running a script file whose content hasn't been
|
||||
inspected. Opaque to command-level heuristics.
|
||||
- **`cloud-infra-mutation`**: Distinguishes destructive cloud CLI verbs
|
||||
(`create`, `delete`, `destroy`) from read-only ones (`show`, `list`, `get`).
|
||||
|
||||
---
|
||||
|
||||
## LLM Judge
|
||||
@@ -263,17 +286,144 @@ heuristic verdict badge with the LLM verdict:
|
||||
|
||||
---
|
||||
|
||||
## v2 Calibration Path
|
||||
## Skill Scanner
|
||||
|
||||
Run v1 with all tools requiring manual approval to build a local verdict
|
||||
dataset. The `intent_verdicts` table accumulates `(tool_call, verdict,
|
||||
user_decision)` triples over time. In v2, calibration tooling will analyze
|
||||
this dataset to:
|
||||
Skills are evaluated by a content scanner at creation and update time. The
|
||||
scanner runs the same class of pattern analysis as the heuristic rules but
|
||||
operates on SKILL.md content rather than individual tool calls. It evaluates
|
||||
four independent risk axes:
|
||||
|
||||
1. **Content risk** — command execution scope, external downloads, credential
|
||||
handling, eval/exec, sudo, data exfiltration, browser automation
|
||||
2. **Supply chain risk** — pipe-to-shell, transitive installs (`npx skills add`),
|
||||
obfuscation, download-execute chains, executable URLs from untrusted domains
|
||||
3. **Vulnerability risk** — prompt injection patterns, insecure credential
|
||||
handling, third-party content exposure (indirect prompt injection surface)
|
||||
4. **Declared capability risk** — parsed from `allowed-tools` in the skill's SKILL.md.
|
||||
`Bash(*)` (unrestricted shell) is high risk. `Bash(git:*)` is low.
|
||||
Read-only tools are safe.
|
||||
|
||||
Results are stored in `scan_status` (tier: safe/low/medium/high/critical) and
|
||||
`scan_report` (JSON breakdown) on the `prompt_templates` table. These fields are
|
||||
system-managed and not editable via the admin API.
|
||||
|
||||
The scanner is a pure function (~2ms) with no I/O. It runs synchronously in
|
||||
the storage layer. Scanner failures are silently caught to never block skill
|
||||
creation.
|
||||
|
||||
See [docs/governance.md](governance.md) for the skill governance model.
|
||||
|
||||
---
|
||||
|
||||
## Output Guard
|
||||
|
||||
The output guard evaluates tool execution results *after* execution but *before*
|
||||
they enter the conversation context. It catches content-level threats that the
|
||||
input heuristic (which evaluates commands) cannot see — prompt injection
|
||||
payloads in fetched web pages, credential leakage in command output, encoded
|
||||
payloads, and adversarial URLs.
|
||||
|
||||
The guard runs as a synchronous heuristic on the tool result text with a
|
||||
configurable time budget (default 5 seconds). Pattern checks run in priority
|
||||
order: prompt injection first, then credentials, then encoded payloads, then
|
||||
lower-priority checks. If the budget is exhausted mid-evaluation, whatever
|
||||
flags have been found so far are returned.
|
||||
|
||||
The guard **annotates but does not gate** — it surfaces warnings via the
|
||||
`on_output_warning` SSE event and optionally redacts detected credentials
|
||||
from the output before it enters the conversation.
|
||||
|
||||
### Detection priorities
|
||||
|
||||
| Priority | Category | Risk | Examples |
|
||||
|----------|----------|------|----------|
|
||||
| 1 | Prompt injection | high | Override phrases, role injection (`{"role":"system"}`), instruction override markers |
|
||||
| 2 | Credential leakage | high | API keys, private key blocks, connection strings, `.env` format secrets, JSON secrets (`"api_key": "..."`, `"password": "..."`, etc.) |
|
||||
| 3 | Encoded payloads | medium | Script data URIs, hex shellcode sequences |
|
||||
| 4 | Adversarial URLs | medium | Cloud metadata endpoints, credential-bearing query parameters |
|
||||
| 5 | System info disclosure | low | Private IP addresses, sensitive file paths |
|
||||
|
||||
### Credential redaction
|
||||
|
||||
When `redact_secrets` is enabled (default), detected credentials in tool output
|
||||
are replaced with `[REDACTED:<type>]` markers before the output enters the
|
||||
conversation. The original unredacted output is never shown to the model.
|
||||
Redaction types: `api_key`, `private_key`, `password`, `secret`.
|
||||
|
||||
### Configuration
|
||||
|
||||
```toml
|
||||
[judge]
|
||||
output_guard = true # enable output evaluation (default)
|
||||
redact_secrets = true # auto-redact detected credentials (default)
|
||||
```
|
||||
|
||||
Configurable at runtime via the admin Settings tab.
|
||||
|
||||
### SSE event: `output_warning`
|
||||
|
||||
When the output guard detects risk signals, an `output_warning` SSE event is
|
||||
emitted to the frontend:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "output_warning",
|
||||
"call_id": "call_abc123",
|
||||
"func_name": "bash",
|
||||
"risk_level": "high",
|
||||
"flags": ["credential_leak"],
|
||||
"annotations": ["API key detected (sk-proj-...)"],
|
||||
"output_length": 1024,
|
||||
"redacted": true
|
||||
}
|
||||
```
|
||||
|
||||
The web UI renders this as an inline warning after the tool result. The CLI
|
||||
shows a colored terminal warning. The server forwards it as an
|
||||
`OutputWarningEvent` for console subscribers.
|
||||
|
||||
Assessments are persisted to the `output_assessments` table for v2
|
||||
calibration. Raw tool output is never stored — only metadata (flags, risk
|
||||
level, annotations, output length, redaction status).
|
||||
|
||||
### Session-level skill scan warning
|
||||
|
||||
When a skill with `scan_status` of `high` or `critical` is loaded into a
|
||||
session, a warning is emitted via `on_info`:
|
||||
|
||||
```
|
||||
⚠ Skill 'my-skill' has scan status: high.
|
||||
Review scan report in admin panel before enabling in production.
|
||||
```
|
||||
|
||||
This ensures operators see a warning even if they missed the scan badge in
|
||||
the admin skills tab.
|
||||
|
||||
---
|
||||
|
||||
## Data Collection for v2 Calibration
|
||||
|
||||
All three evaluation systems persist their assessments for future calibration:
|
||||
|
||||
| Table | Source | Key columns |
|
||||
|-------|--------|-------------|
|
||||
| `intent_verdicts` | Intent judge (heuristic + LLM) | `func_name`, `risk_level`, `confidence`, `user_decision` |
|
||||
| `output_assessments` | Output guard | `func_name`, `risk_level`, `flags`, `redacted` |
|
||||
| `prompt_templates` | Skill scanner | `scan_status`, `scan_report`, `scan_version` |
|
||||
|
||||
Run v1 with all tools requiring manual approval to build a local dataset.
|
||||
In v2, calibration tooling will analyze this data to:
|
||||
|
||||
- Identify tools that are always approved (candidates for auto-approve policies)
|
||||
- Detect false positives in heuristic rules
|
||||
- Detect false positives in heuristic rules (intent + output guard)
|
||||
- Measure LLM judge accuracy against human decisions
|
||||
- Recommend policy changes to reduce approval fatigue
|
||||
- Tune output guard sensitivity per tool (e.g., `bash` output needs more
|
||||
scrutiny than `read_file`)
|
||||
|
||||
Output assessments are queryable via `GET /v1/api/admin/output-assessments`
|
||||
(requires `admin.judge` permission). Skills can be re-scanned via
|
||||
`POST /v1/api/admin/skills/{id}/rescan` when the scanner is updated.
|
||||
|
||||
This data-driven approach means v1 is both useful on its own and a foundation
|
||||
for automated policy tuning.
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
# MCP Registry Integration
|
||||
|
||||
Turnstone integrates with the [official MCP Registry](https://registry.modelcontextprotocol.io) to let administrators discover and install MCP servers directly from the console admin panel.
|
||||
|
||||
## Overview
|
||||
|
||||
The MCP Registry is maintained by the [Agentic AI Foundation](https://www.linuxfoundation.org/press/linux-foundation-announces-the-formation-of-the-agentic-ai-foundation) (Linux Foundation) and serves as the canonical discovery layer for MCP servers. Turnstone queries its REST API (v0.1) for server metadata and provides a one-click install flow.
|
||||
|
||||
Three sources of MCP servers coexist in Turnstone:
|
||||
|
||||
| Source | Badge | Description |
|
||||
|--------|-------|-------------|
|
||||
| **Config** | `CONFIG` (magenta) | Imported from `config.toml` or JSON file. Read-only in admin UI. |
|
||||
| **Manual** | `MANUAL` (cyan) | Added through the admin UI or API. Full CRUD. |
|
||||
| **Registry** | `REGISTRY` (green) | Installed from the MCP Registry. Tracked by `registry_name`. |
|
||||
|
||||
## Admin UI
|
||||
|
||||
The MCP admin tab has two views, toggled by a pill selector:
|
||||
|
||||
### Servers View
|
||||
|
||||
Lists all installed MCP servers regardless of source. Each server shows:
|
||||
|
||||
- **Source badge** — CONFIG, MANUAL, or REGISTRY
|
||||
- **Transport badge** — stdio or streamable-http
|
||||
- **Tool/resource/prompt counts** — aggregated across cluster nodes
|
||||
- **Per-node connection status** — connected (magenta dot), error (red), disabled (gray)
|
||||
- **Actions** — Edit / Delete (DB-managed servers only)
|
||||
|
||||
Clicking a server name opens the detail modal. For registry-installed servers, the detail modal includes a **Registry** section showing the registry name, installed version, description, and website link.
|
||||
|
||||
### Registry View
|
||||
|
||||
Search and browse the MCP Registry. Switching to this view auto-loads a listing. Type a query and press Enter or click Search to filter.
|
||||
|
||||
Each result card shows:
|
||||
|
||||
- **Server name and description**
|
||||
- **Source type badges** — remote (streamable-http), npm, pypi
|
||||
- **Version number**
|
||||
- **Install / Installed / Update button**
|
||||
|
||||
#### Install flow
|
||||
|
||||
- **One-click**: Remote servers with no required headers or URL variables install immediately — no modal, no form. The server is added to the database, all cluster nodes are notified, and a toast confirms success.
|
||||
|
||||
- **Modal**: Servers that require configuration (API keys, headers, URL template variables) or offer multiple install sources (both remote and package) open an install modal with:
|
||||
- Source selector (radio group) — only shown when both remote and package are available
|
||||
- Dynamic form fields for required/optional configuration
|
||||
- Secret fields rendered as password inputs
|
||||
|
||||
## Configuration
|
||||
|
||||
### Registry URL
|
||||
|
||||
By default, Turnstone queries `https://registry.modelcontextprotocol.io`. Override this for enterprise or private registries:
|
||||
|
||||
**Via admin Settings tab:**
|
||||
|
||||
Set `mcp.registry_url` to your registry's base URL.
|
||||
|
||||
**Via config.toml:**
|
||||
|
||||
```toml
|
||||
[mcp]
|
||||
registry_url = "https://registry.internal.example.com"
|
||||
```
|
||||
|
||||
The resolution order is: database setting > config.toml > default.
|
||||
|
||||
## API Endpoints
|
||||
|
||||
Both endpoints require `admin.mcp` permission.
|
||||
|
||||
### Search
|
||||
|
||||
```
|
||||
GET /v1/api/admin/mcp-registry/search?search=github&limit=20&cursor=...
|
||||
```
|
||||
|
||||
Query parameters:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `search` | string | `""` | Search query. Empty returns a browsable listing. |
|
||||
| `limit` | integer | `20` | Results per page (max 100). |
|
||||
| `cursor` | string | — | Opaque cursor from `next_cursor` for pagination. |
|
||||
|
||||
The response annotates each server with `installed`, `installed_server_id`, `installed_version`, and `update_available` by cross-referencing the `mcp_servers` table.
|
||||
|
||||
### Install
|
||||
|
||||
```
|
||||
POST /v1/api/admin/mcp-registry/install
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"registry_name": "io.example/mcp-server",
|
||||
"source": "remote",
|
||||
"index": 0,
|
||||
"name": "",
|
||||
"variables": {},
|
||||
"env": {"API_KEY": "sk-..."},
|
||||
"headers": {"Authorization": "Bearer ..."}
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `registry_name` | Yes | Server name from registry search results. |
|
||||
| `source` | Yes | `"remote"` (streamable-http) or `"package"` (npm/pypi). |
|
||||
| `index` | No | Which remote or package entry to use (default `0`). |
|
||||
| `name` | No | Custom server name. Auto-derived from registry name if empty. |
|
||||
| `variables` | No | Values for URL template `{var}` placeholders. |
|
||||
| `env` | No | Environment variable values for package servers. |
|
||||
| `headers` | No | Header values for remote servers. |
|
||||
|
||||
On success, the server is created in the database and all cluster nodes are automatically reloaded. Returns the created `McpServerDetail`.
|
||||
|
||||
Errors: `400` (validation), `404` (not found in registry), `409` (already installed or name collision), `502` (registry unreachable).
|
||||
|
||||
## SDK
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from turnstone.sdk.console import TurnstoneConsole
|
||||
|
||||
with TurnstoneConsole("http://localhost:8081", token="...") as client:
|
||||
# Search
|
||||
results = client.search_mcp_registry(q="github", limit=10)
|
||||
for srv in results.servers:
|
||||
print(f"{srv.name} v{srv.version} - {srv.description}")
|
||||
|
||||
# Install a remote server
|
||||
detail = client.install_from_registry(
|
||||
"io.example/mcp-server",
|
||||
"remote",
|
||||
headers={"Authorization": "Bearer sk-..."},
|
||||
)
|
||||
print(f"Installed: {detail.name}")
|
||||
```
|
||||
|
||||
### TypeScript
|
||||
|
||||
```typescript
|
||||
import { TurnstoneConsole } from "@anthropic/turnstone-sdk";
|
||||
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://localhost:8081",
|
||||
token: "...",
|
||||
});
|
||||
|
||||
// Search
|
||||
const results = await client.searchMcpRegistry({ q: "github", limit: 10 });
|
||||
for (const srv of results.servers) {
|
||||
console.log(`${srv.name} v${srv.version} - ${srv.description}`);
|
||||
}
|
||||
|
||||
// Install
|
||||
const detail = await client.installFromRegistry({
|
||||
registry_name: "io.example/mcp-server",
|
||||
source: "remote",
|
||||
headers: { Authorization: "Bearer sk-..." },
|
||||
});
|
||||
```
|
||||
|
||||
## Storage
|
||||
|
||||
Registry-installed servers are stored in the existing `mcp_servers` table with three additional columns (migration 019):
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `registry_name` | TEXT (nullable, unique) | Reverse-DNS name from the registry (e.g. `io.example/mcp-server`). |
|
||||
| `registry_version` | TEXT | Version at time of install. |
|
||||
| `registry_meta` | TEXT (JSON) | Snapshot of description, title, website, icons for display. |
|
||||
|
||||
The partial unique index on `registry_name` prevents duplicate installs while allowing multiple non-registry servers with `NULL` registry_name.
|
||||
|
||||
## Package Type Support
|
||||
|
||||
| Registry Type | Transport | Command | Status |
|
||||
|--------------|-----------|---------|--------|
|
||||
| Remote (streamable-http) | `streamable-http` | — (URL-based) | Supported |
|
||||
| `npm` | `stdio` | `npx -y @scope/package@version` | Supported |
|
||||
| `pypi` | `stdio` | `uvx package==version` | Supported |
|
||||
| `oci` | — | — | Not supported (no runtime available) |
|
||||
| `nuget` | — | — | Not supported |
|
||||
| `mcpb` | — | — | Not supported |
|
||||
|
||||
For `npm` and `pypi` packages, the corresponding runtime (`node`/`npx` or `python`/`uvx`) must be available on the cluster nodes. Connection failures due to missing runtimes appear in the per-node MCP status display.
|
||||
+5
-5
@@ -38,7 +38,7 @@ are set.
|
||||
| `TURNSTONE_OIDC_PROVIDER_NAME` | No | `SSO` | Display name for the login button (e.g. "Google", "Okta") |
|
||||
| `TURNSTONE_OIDC_ROLE_CLAIM` | No | — | ID token claim containing role/group values (see [Role Mapping](#role-mapping)) |
|
||||
| `TURNSTONE_OIDC_ROLE_MAP` | No | — | Mapping from claim values to Turnstone role IDs (see [Role Mapping](#role-mapping)) |
|
||||
| `TURNSTONE_OIDC_PASSWORD_ENABLED` | No | `true` | Set to `false` to hide the password form and block all username/password logins (including admin). API tokens and config-file tokens still work. |
|
||||
| `TURNSTONE_OIDC_PASSWORD_ENABLED` | No | `true` | Set to `false` to hide the password form and block all username/password logins (including admin). API tokens continue to work. |
|
||||
| `TURNSTONE_OIDC_REDIRECT_BASE` | No | — | Externally-reachable origin for the OIDC redirect URI (e.g. `https://app.example.com`). Recommended when running behind a reverse proxy. When unset, derived from the request Host header. |
|
||||
|
||||
OIDC is enabled when all three required fields (issuer, client ID, client
|
||||
@@ -246,10 +246,10 @@ password) before OIDC is enabled. The setup wizard always works
|
||||
regardless of this setting because it is only available when zero users
|
||||
exist in the database.
|
||||
|
||||
API token login (`POST /v1/api/auth/login` with a `ts_` token) and
|
||||
config-file tokens (`Authorization: Bearer tok_xxx`) continue to work
|
||||
regardless of this setting. OIDC-only mode affects password-based
|
||||
authentication only.
|
||||
API token login (`POST /v1/api/auth/login` with a `ts_` token)
|
||||
continues to work regardless of this setting. JWTs and API tokens are
|
||||
the supported authentication methods. OIDC-only mode affects
|
||||
password-based authentication only.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
# OpenShell Sandbox Integration
|
||||
|
||||
Turnstone can run inside an [OpenShell](https://github.com/NVIDIA/OpenShell)
|
||||
sandbox for kernel-enforced security boundaries around tool execution. OpenShell
|
||||
provides four layers of defense that Turnstone's application-level safety model
|
||||
does not cover:
|
||||
|
||||
| Layer | Mechanism | What it prevents |
|
||||
|-------|-----------|------------------|
|
||||
| Filesystem | Landlock | Writes to `/etc`, `~/.ssh`, system paths |
|
||||
| Network | Network namespace + seccomp + HTTP CONNECT proxy | Connections to unlisted hosts |
|
||||
| Process | `setuid` drop + verification | Privilege escalation to root |
|
||||
| Credentials | Proxy-level secret resolution | API keys in sandbox memory |
|
||||
|
||||
Turnstone's own safety layers (human approval, intent judge, tool policies,
|
||||
output guard) remain active inside the sandbox and handle threats at the semantic
|
||||
level -- what the LLM *means* to do with its legitimate access.
|
||||
|
||||
> See also: [Security and Authentication](security.md),
|
||||
> [Intent Validation](judge.md), [Governance](governance.md)
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Run turnstone-server in an OpenShell sandbox
|
||||
openshell sandbox run \
|
||||
--policy deploy/openshell/turnstone-policy.yaml \
|
||||
--workdir /path/to/project \
|
||||
-- python3 -m turnstone.server --host 0.0.0.0 --port 8080
|
||||
```
|
||||
|
||||
With inference routing (API keys never enter the sandbox):
|
||||
|
||||
```bash
|
||||
openshell sandbox run \
|
||||
--policy deploy/openshell/turnstone-policy.yaml \
|
||||
--inference-routes deploy/openshell/routes.yaml \
|
||||
--workdir /path/to/project \
|
||||
-- python3 -m turnstone.server --host 0.0.0.0 --port 8080 \
|
||||
--base-url https://inference.local
|
||||
```
|
||||
|
||||
The `inference.local` hostname is intercepted by the OpenShell proxy before
|
||||
network policy evaluation -- no network policy entry is needed for it.
|
||||
|
||||
---
|
||||
|
||||
## Policy Files
|
||||
|
||||
### `deploy/openshell/turnstone-policy.yaml`
|
||||
|
||||
The main sandbox policy. Covers filesystem, process, and network rules.
|
||||
|
||||
### `deploy/openshell/routes.yaml`
|
||||
|
||||
Inference routing configuration. Maps `inference.local` to real LLM API
|
||||
backends. Uncomment and configure the provider(s) you use.
|
||||
|
||||
---
|
||||
|
||||
## Filesystem Policy
|
||||
|
||||
The policy uses Landlock (Linux 5.13+) for kernel-enforced filesystem access
|
||||
control. Paths are locked at sandbox creation and cannot be changed at runtime.
|
||||
|
||||
| Path | Access | Purpose |
|
||||
|------|--------|---------|
|
||||
| `--workdir` | read-write | Project files (auto-added via `include_workdir`) |
|
||||
| `/tmp` | read-write | Bash tool temp scripts, eval workdirs |
|
||||
| `/dev/null` | read-write | Shell redirections (`2>/dev/null`) |
|
||||
| `/var/log` | read-write | Log files |
|
||||
| `/usr`, `/lib`, `/lib64` | read-only | Python runtime, installed packages |
|
||||
| `/etc` | read-only | System config, SSL certificates |
|
||||
| `/proc`, `/dev/urandom` | read-only | Process info, entropy |
|
||||
| `~/.config/turnstone` | read-only | Config file (writes go to database) |
|
||||
|
||||
Landlock runs in `best_effort` mode by default -- degrades gracefully on kernels
|
||||
without Landlock support. Set `compatibility: hard_requirement` for production
|
||||
hardened deployments.
|
||||
|
||||
---
|
||||
|
||||
## Network Policy
|
||||
|
||||
Default-deny. Only explicitly listed host:port pairs are reachable. All child
|
||||
processes (MCP servers, bash commands, grep) inherit the network namespace and
|
||||
cannot bypass the proxy.
|
||||
|
||||
### Included endpoints
|
||||
|
||||
| Policy | Hosts | Purpose |
|
||||
|--------|-------|---------|
|
||||
| `openai_api` | `api.openai.com` | OpenAI LLM API |
|
||||
| `anthropic_api` | `api.anthropic.com` | Anthropic LLM API |
|
||||
| `tavily_api` | `api.tavily.com` | Web search fallback |
|
||||
| `skills_registry` | `skills.sh` | Skill discovery |
|
||||
| `github_api` | `api.github.com` (read-only L7), `raw.githubusercontent.com` | Skill fetch, GitHub API |
|
||||
| `mcp_registry` | `registry.modelcontextprotocol.io` (read-only L7) | MCP server discovery |
|
||||
| `web_fetch_common` | readthedocs, python docs, GitHub Pages, PyPI, npm, Stack Overflow, Wikipedia | Curated web_fetch domains |
|
||||
| `bash_network_tools` | Same as `web_fetch_common` | curl/wget from bash tool |
|
||||
| `package_registries` | `pypi.org`, `files.pythonhosted.org` | pip/uv package installs |
|
||||
| `git_operations` | `github.com`, `gitlab.com` (L7: clone/fetch only, no push) | Git read-only operations |
|
||||
|
||||
### L7 enforcement
|
||||
|
||||
Endpoints marked with `protocol: rest` and `tls: terminate` get HTTP-level
|
||||
inspection. The proxy TLS-terminates using an ephemeral per-sandbox CA, parses
|
||||
each request, and evaluates method + path against the rules.
|
||||
|
||||
The `github_api`, `mcp_registry`, and `git_operations` policies use L7
|
||||
enforcement:
|
||||
|
||||
- **GitHub API / MCP Registry**: `access: read-only` -- only GET, HEAD, OPTIONS
|
||||
allowed
|
||||
- **Git operations**: explicit rules allowing only `info/refs` (GET) and
|
||||
`git-upload-pack` (POST) -- clone and fetch work, push is blocked
|
||||
|
||||
### Commented-out sections
|
||||
|
||||
The policy includes commented blocks for optional integrations. Uncomment and
|
||||
configure as needed:
|
||||
|
||||
- **OIDC** -- add your identity provider's hostname
|
||||
- **Discord** -- `discord.com`, `gateway.discord.gg`, `cdn.discordapp.com`
|
||||
- **MCP HTTP servers** -- any MCP servers using streamable-http transport
|
||||
|
||||
---
|
||||
|
||||
## Customizing the Domain Allowlist
|
||||
|
||||
The `web_fetch` tool lets the LLM fetch arbitrary public URLs, but OpenShell
|
||||
cannot allow "all HTTPS" -- bare wildcard hosts are rejected by policy
|
||||
validation. Instead, the policy ships with a curated set of common reference
|
||||
domains.
|
||||
|
||||
To add domains your workloads need:
|
||||
|
||||
```yaml
|
||||
# In turnstone-policy.yaml, under web_fetch_common.endpoints:
|
||||
- host: docs.example.com
|
||||
port: 443
|
||||
|
||||
# Also add to bash_network_tools.endpoints if curl/wget should reach it:
|
||||
- host: docs.example.com
|
||||
port: 443
|
||||
```
|
||||
|
||||
Wildcard patterns are supported:
|
||||
|
||||
- `*.example.com` -- matches one subdomain level (e.g. `api.example.com`)
|
||||
- `**.example.com` -- matches any depth (e.g. `deep.sub.example.com`)
|
||||
|
||||
Unlisted domains return connection errors, which the LLM handles gracefully by
|
||||
telling the user it cannot reach that site.
|
||||
|
||||
---
|
||||
|
||||
## Inference Routing
|
||||
|
||||
Inference routing keeps real API keys completely outside the sandbox. The
|
||||
sandbox process only sees opaque placeholder tokens in its environment
|
||||
(`openshell:resolve:env:ANTHROPIC_API_KEY`). The proxy rewrites these to real
|
||||
credentials on the wire before forwarding to the upstream API.
|
||||
|
||||
### Setup
|
||||
|
||||
1. Edit `deploy/openshell/routes.yaml` -- uncomment your provider:
|
||||
|
||||
```yaml
|
||||
routes:
|
||||
# OpenAI
|
||||
- name: inference.local
|
||||
endpoint: https://api.openai.com/v1
|
||||
model: gpt-5
|
||||
provider_type: openai
|
||||
protocols:
|
||||
- openai_chat_completions
|
||||
- model_discovery
|
||||
api_key_env: OPENAI_API_KEY
|
||||
|
||||
# Or Anthropic
|
||||
- name: inference.local
|
||||
endpoint: https://api.anthropic.com
|
||||
model: claude-sonnet-4-6
|
||||
provider_type: anthropic
|
||||
protocols:
|
||||
- anthropic_messages
|
||||
api_key_env: ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
2. Start with `--inference-routes` and point turnstone at `inference.local`:
|
||||
|
||||
```bash
|
||||
openshell sandbox run \
|
||||
--inference-routes deploy/openshell/routes.yaml \
|
||||
--base-url https://inference.local \
|
||||
...
|
||||
```
|
||||
|
||||
3. When inference routing is active, the `openai_api` and `anthropic_api`
|
||||
network policies can be removed from the sandbox policy -- the proxy handles
|
||||
LLM traffic on a separate code path that bypasses OPA entirely.
|
||||
|
||||
### Local model servers
|
||||
|
||||
For local servers (vLLM, llama.cpp) with no authentication, omit both
|
||||
`api_key` and `api_key_env` from the route config. No credential resolution
|
||||
is needed.
|
||||
|
||||
---
|
||||
|
||||
## MCP Server Subprocesses
|
||||
|
||||
MCP servers using stdio transport are spawned as child processes of turnstone.
|
||||
They automatically inherit all sandbox constraints:
|
||||
|
||||
- **Network namespace** -- kernel-level, cannot be bypassed
|
||||
- **Landlock filesystem** -- kernel-level, cannot be relaxed
|
||||
- **Seccomp socket filter** -- kernel-level, inherited on fork
|
||||
|
||||
No per-subprocess policy entries are needed for these constraints. However, if
|
||||
an MCP server makes outbound network requests (through the proxy), its binary
|
||||
must appear in a `binaries[]` entry for the relevant network policy. The proxy
|
||||
identifies the requesting process via `/proc/<pid>/exe` (not `argv[0]`, which
|
||||
is spoofable).
|
||||
|
||||
Example for a Python-based MCP server that calls an external API:
|
||||
|
||||
```yaml
|
||||
mcp_external_api:
|
||||
name: mcp-external
|
||||
endpoints:
|
||||
- host: api.example.com
|
||||
port: 443
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
```
|
||||
|
||||
MCP servers using streamable-http transport are remote -- they need a network
|
||||
policy entry for their host:port but no binary entry (the Python process making
|
||||
the HTTP call is already covered by the standard `python3*` binary entries).
|
||||
|
||||
---
|
||||
|
||||
## Security Model: Which Layer Enforces What
|
||||
|
||||
```
|
||||
OpenShell (infrastructure) Turnstone (application)
|
||||
───────────────────────────── ──────────────────────────────
|
||||
Filesystem access Landlock kernel enforcement (no enforcement)
|
||||
Network egress Netns + seccomp + proxy + OPA SSRF check on web_fetch
|
||||
Credentials Placeholder injection + proxy Output guard redaction
|
||||
Privilege level setuid drop + verification (no enforcement)
|
||||
Tool semantics (no visibility) Heuristic + LLM judge
|
||||
Tool policies (no visibility) fnmatch admin policies
|
||||
Prompt injection (no visibility) Output guard detection
|
||||
Human approval (no visibility) Approval gate + "always"
|
||||
```
|
||||
|
||||
OpenShell constrains what the process can physically reach. Turnstone constrains
|
||||
what the LLM does with its legitimate access. Neither layer is sufficient alone:
|
||||
|
||||
- Without OpenShell: a bash command can `curl` secrets to any endpoint, write to
|
||||
`/etc/crontab`, or read `~/.ssh/id_rsa` -- all gated only by human approval
|
||||
- Without Turnstone: the LLM can `rm -rf` the entire workdir, run destructive
|
||||
commands, or consume prompt injection payloads -- all within the sandbox's
|
||||
allowed scope
|
||||
|
||||
---
|
||||
|
||||
## Hardening Checklist
|
||||
|
||||
For production deployments:
|
||||
|
||||
- [ ] Set `landlock.compatibility: hard_requirement`
|
||||
- [ ] Enable inference routing (removes API keys from sandbox)
|
||||
- [ ] Remove `openai_api`/`anthropic_api` network policies when using inference
|
||||
routing (traffic goes through the router, not direct)
|
||||
- [ ] Review and trim `web_fetch_common` domains to your actual needs
|
||||
- [ ] Remove `package_registries` policy if pip/uv installs are not needed
|
||||
- [ ] Add your OIDC provider endpoint if using SSO
|
||||
- [ ] Consider removing `bash_network_tools` entirely if bash should not have
|
||||
network access
|
||||
@@ -0,0 +1,200 @@
|
||||
# PgBouncer Connection Pooling
|
||||
|
||||
Turnstone cluster deployments share a single PostgreSQL instance across
|
||||
all server nodes and the console. Each process
|
||||
maintains a small connection pool (2 base + 3 overflow = 5 max). At
|
||||
scale this adds up — a 100-node cluster opens up to 500 connections,
|
||||
and a 1000-node cluster up to 5,000.
|
||||
|
||||
PostgreSQL's default `max_connections` is 100, and each real connection
|
||||
allocates ~5–10 MB of backend memory. PgBouncer sits between turnstone
|
||||
and PostgreSQL, multiplexing thousands of lightweight client connections
|
||||
down to a small number of real database connections.
|
||||
|
||||
---
|
||||
|
||||
## Why PgBouncer works well with turnstone
|
||||
|
||||
All turnstone database operations are short-burst queries: acquire a
|
||||
connection, execute 1–3 statements, commit, release. No operation holds
|
||||
a connection for more than a few milliseconds. This makes **transaction
|
||||
pooling mode** ideal — PgBouncer assigns a real connection only for the
|
||||
duration of each transaction, then returns it to the pool.
|
||||
|
||||
| Cluster size | Client connections (max) | PgBouncer server connections needed |
|
||||
|--------------|------------------------|-------------------------------------|
|
||||
| 10 nodes | 50 | 10–20 |
|
||||
| 100 nodes | 500 | 20–40 |
|
||||
| 500 nodes | 2,500 | 30–60 |
|
||||
| 1,000 nodes | 5,000 | 40–80 |
|
||||
|
||||
The server connection count stays low because most client connections
|
||||
are idle at any given moment.
|
||||
|
||||
---
|
||||
|
||||
## Docker Compose
|
||||
|
||||
Add PgBouncer between turnstone services and PostgreSQL:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
pgbouncer:
|
||||
image: bitnami/pgbouncer:latest
|
||||
environment:
|
||||
POSTGRESQL_HOST: postgres
|
||||
POSTGRESQL_PORT: "5432"
|
||||
POSTGRESQL_DATABASE: turnstone
|
||||
POSTGRESQL_USERNAME: ${POSTGRES_USER:-turnstone}
|
||||
POSTGRESQL_PASSWORD: ${POSTGRES_PASSWORD:?}
|
||||
PGBOUNCER_POOL_MODE: transaction
|
||||
PGBOUNCER_DEFAULT_POOL_SIZE: "40"
|
||||
PGBOUNCER_MAX_CLIENT_CONN: "5000"
|
||||
PGBOUNCER_MAX_DB_CONNECTIONS: "80"
|
||||
PGBOUNCER_SERVER_IDLE_TIMEOUT: "300"
|
||||
ports:
|
||||
- "6432:6432"
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-p", "6432"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
```
|
||||
|
||||
Then point turnstone services at PgBouncer instead of PostgreSQL
|
||||
directly by changing the `DATABASE_URL` (or `TURNSTONE_DB_URL`):
|
||||
|
||||
```bash
|
||||
# Before (direct)
|
||||
TURNSTONE_DB_URL=postgresql://turnstone:secret@postgres:5432/turnstone
|
||||
|
||||
# After (via PgBouncer)
|
||||
TURNSTONE_DB_URL=postgresql://turnstone:secret@pgbouncer:6432/turnstone
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Helm / Kubernetes
|
||||
|
||||
Add a PgBouncer deployment or use a Helm chart like
|
||||
[bitnami/pgbouncer](https://github.com/bitnami/charts/tree/main/bitnami/pgbouncer).
|
||||
|
||||
In `values.yaml`, point the database at PgBouncer:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
backend: postgresql
|
||||
external:
|
||||
host: pgbouncer
|
||||
port: 6432
|
||||
database: turnstone
|
||||
username: turnstone
|
||||
existingSecret: turnstone-db-secret
|
||||
```
|
||||
|
||||
PgBouncer configuration:
|
||||
|
||||
```yaml
|
||||
pgbouncer:
|
||||
poolMode: transaction
|
||||
defaultPoolSize: 40
|
||||
maxClientConn: 5000
|
||||
maxDbConnections: 80
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration reference
|
||||
|
||||
| PgBouncer setting | Recommended | Notes |
|
||||
|-------------------|-------------|-------|
|
||||
| `pool_mode` | `transaction` | Required — turnstone uses short-burst queries with no session state |
|
||||
| `default_pool_size` | 40 | Real PostgreSQL connections per database. Start here, increase if you see `no more connections allowed` |
|
||||
| `max_client_conn` | 5000 | Upper bound on client connections. Set to `cluster_nodes × 5` |
|
||||
| `max_db_connections` | 80 | Hard cap on real connections to PostgreSQL. Keep below PG `max_connections` minus headroom for admin/monitoring |
|
||||
| `server_idle_timeout` | 300 | Close idle server connections after 5 minutes |
|
||||
| `server_lifetime` | 3600 | Recycle server connections after 1 hour |
|
||||
|
||||
On the PostgreSQL side:
|
||||
|
||||
| PostgreSQL setting | Recommended | Notes |
|
||||
|--------------------|-------------|-------|
|
||||
| `max_connections` | 100 | Default is fine — PgBouncer is the only client. Set higher than `max_db_connections` to leave room for admin connections |
|
||||
| `shared_buffers` | 25% of RAM | Standard PostgreSQL tuning |
|
||||
|
||||
---
|
||||
|
||||
## Turnstone pool settings
|
||||
|
||||
Each turnstone process maintains its own SQLAlchemy connection pool to
|
||||
PgBouncer (which then multiplexes to PostgreSQL):
|
||||
|
||||
| Environment variable | Default | Description |
|
||||
|---------------------|---------|-------------|
|
||||
| `TURNSTONE_DB_POOL_SIZE` | 2 | Base pool size per process |
|
||||
| `TURNSTONE_DB_BACKEND` | sqlite | Set to `postgresql` for cluster deployments |
|
||||
| `TURNSTONE_DB_URL` | — | Connection URL (point at PgBouncer, not PostgreSQL directly) |
|
||||
|
||||
The default pool of 2 + 3 overflow = 5 connections per process is
|
||||
intentionally small to support large clusters. You should not need to
|
||||
increase this — turnstone's database operations are all short-burst
|
||||
context-managed queries that hold connections for milliseconds.
|
||||
|
||||
SQLAlchemy `pool_pre_ping` is enabled, so stale connections (e.g. after
|
||||
PgBouncer restarts) are automatically detected and replaced.
|
||||
|
||||
---
|
||||
|
||||
## Monitoring
|
||||
|
||||
PgBouncer exposes stats via its admin console (connect to
|
||||
PgBouncer port with user `pgbouncer`):
|
||||
|
||||
```sql
|
||||
-- Active and waiting clients
|
||||
SHOW POOLS;
|
||||
|
||||
-- Per-database stats
|
||||
SHOW STATS;
|
||||
|
||||
-- Current client connections
|
||||
SHOW CLIENTS;
|
||||
```
|
||||
|
||||
Key metrics to watch:
|
||||
|
||||
- **`cl_active`** — clients with a server connection assigned. Should be
|
||||
well below `max_db_connections`.
|
||||
- **`cl_waiting`** — clients waiting for a server connection. Sustained
|
||||
non-zero values mean you need more `default_pool_size`.
|
||||
- **`sv_active`** — active server (PostgreSQL) connections. Should stay
|
||||
below PostgreSQL `max_connections`.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"no more connections allowed (max_client_conn)"** — PgBouncer is
|
||||
rejecting new client connections. Increase `max_client_conn` to match
|
||||
your cluster size × 5.
|
||||
|
||||
**"no more connections allowed (max_db_connections)"** — PgBouncer
|
||||
cannot open more connections to PostgreSQL. Increase
|
||||
`max_db_connections` and ensure PostgreSQL `max_connections` is higher.
|
||||
|
||||
**Connections timing out on startup** — If all nodes start
|
||||
simultaneously, the burst of initial connections (migrations, health
|
||||
checks) can temporarily exceed the pool. PgBouncer queues excess
|
||||
clients by default — this resolves itself within seconds.
|
||||
|
||||
**Prepared statements not supported** — PgBouncer in `transaction` mode
|
||||
does not support prepared statements. Turnstone's SQLAlchemy layer does
|
||||
not use server-side prepared statements by default, so this is not an
|
||||
issue.
|
||||
|
||||
See also: [Docker deployment](docker.md) · [Security](security.md)
|
||||
@@ -0,0 +1,80 @@
|
||||
# Release Process
|
||||
|
||||
Turnstone uses two parallel release tracks published from a single PyPI package.
|
||||
|
||||
## Release Tracks
|
||||
|
||||
| Track | Versions | Branch | Docker tags | PyPI install |
|
||||
|-------|----------|--------|-------------|--------------|
|
||||
| **Stable** | `1.0.0`, `1.0.1` | `stable/1.0` | `:1.0.1`, `:1.0`, `:stable`, `:latest` | `pip install turnstone` |
|
||||
| **Experimental** | `1.1.0a1`, `1.1.0a2` | `main` | `:1.1.0a1`, `:experimental` | `pip install turnstone --pre` |
|
||||
|
||||
- **Stable** receives bugfixes only. Production-grade.
|
||||
- **Experimental** receives new features. May be rough around the edges.
|
||||
- When experimental matures, it is promoted to stable. The previous stable branch stops receiving patches.
|
||||
|
||||
## Version Scheme
|
||||
|
||||
[PEP 440](https://peps.python.org/pep-0440/) pre-release suffixes on a single package:
|
||||
|
||||
- `1.0.0` — stable release
|
||||
- `1.1.0a1` — alpha (experimental)
|
||||
- `1.1.0b1` — beta (experimental, more stable)
|
||||
- `1.1.0rc1` — release candidate (experimental, nearly stable)
|
||||
- `1.1.0` — promoted to stable
|
||||
|
||||
## Releasing an Experimental Version (from main)
|
||||
|
||||
```bash
|
||||
scripts/release.sh 1.1.0a2 --push
|
||||
```
|
||||
|
||||
This bumps `pyproject.toml` + `turnstone/__init__.py`, regenerates `uv.lock`, commits, tags `v1.1.0a2`, and pushes. CI runs, then publish + Docker workflows fire automatically.
|
||||
|
||||
## Releasing a Stable Patch (from stable/X.Y)
|
||||
|
||||
```bash
|
||||
git checkout stable/1.0
|
||||
git cherry-pick <commit-hash> # bugfix from main
|
||||
scripts/release.sh 1.0.2 --push
|
||||
```
|
||||
|
||||
## Promoting Experimental to Stable
|
||||
|
||||
When `main` is ready for a stable release:
|
||||
|
||||
```bash
|
||||
# 1. Tag the stable release on main
|
||||
scripts/release.sh 1.1.0 --push
|
||||
|
||||
# 2. Create the stable maintenance branch from that tag
|
||||
git branch stable/1.1 v1.1.0
|
||||
git push origin stable/1.1
|
||||
|
||||
# 3. Start the next experimental cycle on main
|
||||
scripts/release.sh 1.2.0a1 --push
|
||||
```
|
||||
|
||||
The previous `stable/1.0` branch stops receiving patches at this point.
|
||||
|
||||
## CI/CD Pipeline
|
||||
|
||||
All releases are gated on CI success:
|
||||
|
||||
1. `git push` with `v*` tag triggers **CI** (lint, typecheck, test, test-postgres, lock-check, security audit)
|
||||
2. On CI success, **Publish to PyPI** fires via `workflow_run`
|
||||
3. On CI success, **Publish Docker Image** fires via `workflow_run`
|
||||
|
||||
Pre-release tags (`a`, `b`, `rc` suffixes) produce:
|
||||
- PyPI: pre-release version (not installed by default)
|
||||
- GitHub Release: marked as pre-release
|
||||
- Docker: `:experimental` alias + exact version tag
|
||||
|
||||
Stable tags produce:
|
||||
- PyPI: stable version (default `pip install`)
|
||||
- GitHub Release: full release
|
||||
- Docker: `:stable`, `:latest`, `:X.Y`, `:X.Y.Z` tags
|
||||
|
||||
## Dependency Updates
|
||||
|
||||
Renovate targets `main` (experimental) only. Stable branches receive manual dependency updates via cherry-pick when security-relevant.
|
||||
+25
-13
@@ -69,13 +69,13 @@ Both `TurnstoneServer` (sync) and `AsyncTurnstoneServer` (async) expose:
|
||||
|----------|--------|---------|
|
||||
| **Workstreams** | `list_workstreams()` | `ListWorkstreamsResponse` |
|
||||
| | `dashboard()` | `DashboardResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve, ws_template)` | `CreateWorkstreamResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve, skill)` | `CreateWorkstreamResponse` |
|
||||
| | `close_workstream(ws_id)` | `StatusResponse` |
|
||||
| **Chat** | `send(message, ws_id)` | `SendResponse` |
|
||||
| | `approve(*, ws_id, approved, feedback, always)` | `StatusResponse` |
|
||||
| | `plan_feedback(*, ws_id, feedback)` | `StatusResponse` |
|
||||
| | `command(*, ws_id, command)` | `StatusResponse` |
|
||||
| | `cancel(ws_id)` | `StatusResponse` |
|
||||
| | `cancel(ws_id, *, force=False)` | `StatusResponse` |
|
||||
| **Streaming** | `stream_events(ws_id)` | `Iterator[ServerEvent]` |
|
||||
| | `stream_global_events()` | `Iterator[ServerEvent]` |
|
||||
| **High-level** | `send_and_wait(message, ws_id, *, timeout, on_event)` | `TurnResult` |
|
||||
@@ -97,19 +97,17 @@ Both `TurnstoneConsole` (sync) and `AsyncTurnstoneConsole` (async) expose:
|
||||
| | `workstreams(*, state, node, search, sort, page, per_page)` | `ClusterWorkstreamsResponse` |
|
||||
| | `node_detail(node_id)` | `NodeDetailResponse` |
|
||||
| | `snapshot()` | `ClusterSnapshotResponse` |
|
||||
| | `create_workstream(*, node_id, name, model, initial_message, ws_template)` | `ConsoleCreateWsResponse` |
|
||||
| | `create_workstream(*, node_id, name, model, initial_message, skill)` | `ConsoleCreateWsResponse` |
|
||||
| **Schedules** | `list_schedules()` | `ListSchedulesResponse` |
|
||||
| | `create_schedule(*, name, schedule_type, initial_message, ...)` | `ScheduleInfo` |
|
||||
| | `get_schedule(task_id)` | `ScheduleInfo` |
|
||||
| | `update_schedule(task_id, *, name=..., enabled=..., ...)` | `ScheduleInfo` |
|
||||
| | `delete_schedule(task_id)` | `StatusResponse` |
|
||||
| | `list_schedule_runs(task_id, *, limit=50)` | `ListScheduleRunsResponse` |
|
||||
| **WS Templates** | `list_ws_templates()` | `ListWsTemplatesResponse` |
|
||||
| | `create_ws_template(*, name, description, ...)` | `WsTemplateInfo` |
|
||||
| | `get_ws_template(template_id)` | `WsTemplateInfo` |
|
||||
| | `update_ws_template(template_id, *, name=..., enabled=..., ...)` | `WsTemplateInfo` |
|
||||
| | `delete_ws_template(template_id)` | `StatusResponse` |
|
||||
| | `list_ws_template_versions(template_id)` | `ListWsTemplateVersionsResponse` |
|
||||
| **MCP Registry** | `search_mcp_registry(q="", *, limit=20, cursor=None)` | `RegistrySearchResponse` |
|
||||
| | `install_from_registry(registry_name, source, *, index=0, name="", variables=None, env=None, headers=None)` | `McpServerDetail` |
|
||||
| **Skill Discovery** | `discover_skills(q="", *, limit=20)` | `SkillDiscoverResponse` |
|
||||
| | `install_skill(source, *, skill_id="", url="")` | `dict` |
|
||||
| **Streaming** | `stream_cluster_events()` | `Iterator[ClusterEvent]` |
|
||||
| **Auth** | `login(username=..., password=...)` / `login(token="ts_xxx")` | `AuthLoginResponse` |
|
||||
| | `logout()` | `StatusResponse` |
|
||||
@@ -129,9 +127,9 @@ SSE events are deserialized into typed dataclasses. Use `event.type` to discrimi
|
||||
| `reasoning` | `ReasoningEvent` | `text` |
|
||||
| `tool_info` | `ToolInfoEvent` | `items` |
|
||||
| `approve_request` | `ApproveRequestEvent` | `items` |
|
||||
| `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output` |
|
||||
| `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output`, `is_error` |
|
||||
| `tool_output_chunk` | `ToolOutputChunkEvent` | `call_id`, `chunk` |
|
||||
| `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort` |
|
||||
| `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort`, `cache_creation_tokens`, `cache_read_tokens` |
|
||||
| `plan_review` | `PlanReviewEvent` | `content` |
|
||||
| `error` | `ErrorEvent` | `message` |
|
||||
| `info` | `InfoEvent` | `message` |
|
||||
@@ -228,6 +226,20 @@ await client.login({ username: "alice", password: "s3cret" });
|
||||
const overview = await client.overview();
|
||||
console.log(`Nodes: ${overview.nodes}, Workstreams: ${overview.workstreams}`);
|
||||
|
||||
// Search and install from the MCP Registry
|
||||
const results = await client.searchMcpRegistry({ q: "github", limit: 10 });
|
||||
const server = await client.installFromRegistry({
|
||||
registry_name: results.servers[0].name,
|
||||
source: "remote",
|
||||
});
|
||||
|
||||
// Search and install skills from external registries
|
||||
const skills = await client.discoverSkills({ q: "code review" });
|
||||
const skill = await client.installSkill({
|
||||
source: "github",
|
||||
url: "https://github.com/owner/skill-repo",
|
||||
});
|
||||
|
||||
// Stream cluster events
|
||||
for await (const event of client.clusterEvents()) {
|
||||
console.log(event.type, event);
|
||||
@@ -320,6 +332,6 @@ client.login(token="ts_abc123...")
|
||||
- `client.logout()` clears the stored JWT from the client.
|
||||
- If a request returns 401, the SDK raises `TurnstoneAPIError` — the caller is responsible for re-authenticating.
|
||||
|
||||
### Backward Compatibility
|
||||
### Token Types
|
||||
|
||||
The config-file token (`TURNSTONE_AUTH_TOKEN`) still works as a simple Bearer token for environments that do not use the user/JWT system. When the server receives a non-JWT Bearer token, it falls back to the legacy token check.
|
||||
The SDK accepts any Bearer token — JWTs (from `ServiceTokenManager` or login) and API tokens (`ts_` prefix) are both supported. Use `token_factory` for auto-rotating JWTs or a static `token` for API tokens.
|
||||
|
||||
+46
-67
@@ -8,23 +8,6 @@ credentials while individual server nodes validate JWTs locally.
|
||||
|
||||
## Token Types
|
||||
|
||||
### Config-file tokens
|
||||
|
||||
Static tokens defined in `config.toml` or the `TURNSTONE_AUTH_TOKEN`
|
||||
environment variable. Validated in-memory using `hmac.compare_digest`
|
||||
(timing-safe). Each token maps to a role that determines its scopes.
|
||||
|
||||
```toml
|
||||
[[auth.tokens]]
|
||||
value = "tok_legacy"
|
||||
role = "full" # full → {read, write, approve}
|
||||
```
|
||||
|
||||
Role mappings: `"read"` → `{read}`, `"full"` → `{read, write, approve}`.
|
||||
|
||||
Config tokens are sent directly as `Authorization: Bearer tok_legacy`
|
||||
on every request. No JWT exchange is needed.
|
||||
|
||||
### API tokens
|
||||
|
||||
Database-backed tokens prefixed with `ts_`. Created via the admin CLI
|
||||
@@ -149,15 +132,6 @@ The API token is hashed, looked up in the database, and exchanged for a
|
||||
JWT with the token's scopes. This is the recommended flow for SDKs and
|
||||
automated clients that need cookie-based sessions.
|
||||
|
||||
### Config-file tokens (direct)
|
||||
|
||||
Config tokens are validated per-request via `hmac.compare_digest`. No
|
||||
login exchange is needed — include the token as a `Bearer` header:
|
||||
|
||||
```
|
||||
Authorization: Bearer tok_legacy
|
||||
```
|
||||
|
||||
### First-time setup
|
||||
|
||||
When no users exist in the database:
|
||||
@@ -276,7 +250,7 @@ Setting `TURNSTONE_OIDC_PASSWORD_ENABLED=false` hides the password
|
||||
form on the login page and blocks password-based login at the API
|
||||
level. The setup wizard always works regardless of this setting — the
|
||||
first admin user is created with a password before OIDC is relevant.
|
||||
API tokens and config-file tokens are unaffected by this setting.
|
||||
API tokens are unaffected by this setting.
|
||||
|
||||
#### Known limitations
|
||||
|
||||
@@ -297,8 +271,6 @@ and classifies the token:
|
||||
|
||||
1. **Contains `.`** → JWT → validate HS256 signature and expiry
|
||||
2. **Starts with `ts_`** → API token → SHA-256 hash, database lookup
|
||||
3. **Otherwise** → config-file token → `hmac.compare_digest` against
|
||||
each configured token
|
||||
|
||||
If a session cookie is present and no `Authorization` header is sent,
|
||||
the cookie value is treated as a JWT (step 1).
|
||||
@@ -332,17 +304,10 @@ deployments.
|
||||
| Signing secret | `[auth] jwt_secret` | `TURNSTONE_JWT_SECRET` | Auto-generated ephemeral (warning logged) |
|
||||
| Expiry | `[auth] jwt_expiry_hours` | — | 24 hours |
|
||||
| Algorithm | — | — | HS256 (not configurable) |
|
||||
| Minimum secret length | — | — | 32 characters (warning if shorter) |
|
||||
| Minimum secret length | — | — | 32 characters (exits if shorter) |
|
||||
|
||||
All service nodes that need to validate JWTs must share the same signing
|
||||
secret. If no secret is configured, an ephemeral key is generated at
|
||||
startup and a warning is logged — JWTs will not survive restarts or work
|
||||
across nodes.
|
||||
|
||||
The bridge and console **require** `TURNSTONE_JWT_SECRET` when no
|
||||
`--auth-token` is provided. They exit with an error if the secret is
|
||||
missing, since ephemeral secrets would silently break inter-service
|
||||
communication.
|
||||
All services require `TURNSTONE_JWT_SECRET` and exit at startup if it is
|
||||
missing or shorter than 32 characters.
|
||||
|
||||
---
|
||||
|
||||
@@ -443,45 +408,66 @@ Console (cluster-wide) Server (per-node)
|
||||
┌──────────────────────┐ ┌──────────────────────┐
|
||||
│ User/Token CRUD (DB) │ │ JWT validation only │
|
||||
│ Login: creds → JWT │ │ (shared signing key) │
|
||||
│ Admin API endpoints │ │ Config tokens: hmac │
|
||||
│ Storage: users, │ │ No auth DB needed │
|
||||
│ Admin API endpoints │ │ No auth DB needed │
|
||||
│ Storage: users, │ │ │
|
||||
│ api_tokens tables │ │ │
|
||||
└──────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
The console owns the credential database and handles all user/token
|
||||
CRUD. Individual server nodes only need the JWT signing secret to
|
||||
validate session tokens. Config-file tokens are validated locally
|
||||
without any database.
|
||||
validate session tokens.
|
||||
|
||||
### Proxy auth forwarding
|
||||
|
||||
When the console proxies requests to server nodes (via `/node/{id}/...`
|
||||
routes), it uses a dedicated **service proxy token** with
|
||||
`aud: turnstone-server` and `write` scope. The user's console JWT
|
||||
(which has `aud: turnstone-console`) is **not** forwarded — it would be
|
||||
rejected by the server's audience validation.
|
||||
routes), it mints a **short-lived user-scoped JWT** with
|
||||
`aud: turnstone-server` carrying the real user's `user_id`, `scopes`,
|
||||
and `permissions`. The user's console JWT (which has
|
||||
`aud: turnstone-console`) is **not** forwarded directly — it would be
|
||||
rejected by the server's audience validation. Instead, the console
|
||||
re-signs a new JWT targeted at the server audience.
|
||||
|
||||
The proxy token is managed by a `ServiceTokenManager` that auto-rotates
|
||||
1-hour JWTs, refreshing at 80% of lifetime. If `--auth-token` is
|
||||
provided, that static token is used instead.
|
||||
Each proxied request gets a fresh JWT (5-minute expiry). This ensures:
|
||||
|
||||
- **Audit attribution** — the upstream server records the real user in
|
||||
`ctx_user_id` and audit events, not a generic service identity.
|
||||
- **Scope narrowing** — a read-only console user's proxied request
|
||||
carries only `read` scope, not the full `{read, write, approve}` set.
|
||||
The server enforces this as defense in depth.
|
||||
- **Permission forwarding** — granular RBAC permissions from the
|
||||
console JWT are carried through to the server.
|
||||
|
||||
The JWT `src` claim is set to `"console-proxy"`, allowing servers to
|
||||
distinguish proxied requests from direct logins in audit logs.
|
||||
|
||||
When no user context is available (auth disabled, or internal requests),
|
||||
the proxy falls back to a `ServiceTokenManager` with service identity
|
||||
`console-proxy` and full scopes.
|
||||
|
||||
### Service-to-service authentication
|
||||
|
||||
The bridge and console collector use `ServiceTokenManager` for
|
||||
auto-rotating JWTs when communicating with server nodes:
|
||||
The console collector uses `ServiceTokenManager` for auto-rotating
|
||||
JWTs when communicating with server nodes:
|
||||
|
||||
| Service | Identity | Scope | Audience | Purpose |
|
||||
|---------|----------|-------|----------|---------|
|
||||
| Bridge | `bridge` | `approve` | `turnstone-server` | Tool approval proxy, message relay |
|
||||
| Console collector | `console-collector` | `read` | `turnstone-server` | Node health polling |
|
||||
| Console proxy | `console-proxy` | `write` | `turnstone-server` | Proxied API calls |
|
||||
| Console proxy (fallback) | `console-proxy` | `approve` | `turnstone-server` | Proxied API calls when no user context |
|
||||
| Channel notify | `system` | `write` | `turnstone-channel` | Notification delivery to channel gateway |
|
||||
|
||||
Service tokens use 1-hour expiry with automatic refresh via
|
||||
`ServiceTokenManager`. The bridge injects auth headers per-request via
|
||||
httpx event hooks to ensure rotated tokens are picked up on SSE
|
||||
reconnects.
|
||||
`ServiceTokenManager`.
|
||||
|
||||
### User identity in MQ-dispatched workstreams
|
||||
|
||||
When the console creates a workstream (the normal path), the
|
||||
authenticated user's `user_id` is forwarded in the HTTP payload when
|
||||
calling the server's `POST /v1/api/workstreams/new`. The server
|
||||
accepts a `user_id` from the request body **only when the caller is a
|
||||
trusted service** — identified by `token_source` matching
|
||||
`console-proxy` or `console`. Regular API callers cannot
|
||||
override `user_id`; the server always uses their JWT identity.
|
||||
|
||||
Note that the channel gateway uses a distinct JWT audience
|
||||
(`turnstone-channel`) from the server (`turnstone-server`) and console
|
||||
@@ -496,22 +482,17 @@ channel gateway endpoint, and vice versa.
|
||||
|
||||
```toml
|
||||
[auth]
|
||||
enabled = true
|
||||
jwt_secret = "your-secret-key-here"
|
||||
jwt_expiry_hours = 24
|
||||
|
||||
[[auth.tokens]]
|
||||
value = "tok_legacy"
|
||||
role = "full"
|
||||
```
|
||||
|
||||
### Environment variables
|
||||
|
||||
Auth is always enabled. `TURNSTONE_JWT_SECRET` is required.
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `TURNSTONE_AUTH_ENABLED=1` | Enable authentication |
|
||||
| `TURNSTONE_AUTH_TOKEN=tok_xxx` | Register a config-file token with `full` access |
|
||||
| `TURNSTONE_JWT_SECRET=xxx` | JWT signing secret (must match across nodes) |
|
||||
| `TURNSTONE_JWT_SECRET=xxx` | JWT signing secret (required, must match across nodes) |
|
||||
| `TURNSTONE_CORS_ORIGINS=` | CORS allowed origins (comma-separated; empty = same-origin only) |
|
||||
|
||||
---
|
||||
@@ -549,8 +530,6 @@ and browsers enforce same-origin policy.
|
||||
|
||||
## Security Properties
|
||||
|
||||
- **Timing-safe comparison** for config-file tokens via
|
||||
`hmac.compare_digest` — no timing side-channel.
|
||||
- **Hash-based lookup** for API tokens — the database stores only
|
||||
SHA-256 hashes, eliminating timing attacks on token comparison.
|
||||
- **Local JWT validation** — no network call or database query needed
|
||||
|
||||
+7
-7
@@ -27,7 +27,7 @@ Settings resolution differs between entry points:
|
||||
|
||||
| Entry point | Chain |
|
||||
|-------------|-------|
|
||||
| **Server** (`turnstone-server`, `turnstone-bridge`) | CLI flag > ConfigStore > registry default |
|
||||
| **Server** (`turnstone-server`) | CLI flag > ConfigStore > registry default |
|
||||
| **CLI** (`turnstone`) | CLI flag > config.toml > argparse default |
|
||||
|
||||
The server's `apply_config()` ignores config.toml sections that overlap with
|
||||
@@ -46,12 +46,10 @@ connection, Redis, auth secrets, server bind address). These stay in
|
||||
|----------|---------|-------|
|
||||
| API credentials | `[api]` | config.toml / env |
|
||||
| Database | `[database]` | config.toml / env |
|
||||
| Redis | `[redis]` | config.toml / env |
|
||||
| Auth | `[auth]` | config.toml / env |
|
||||
| Bridge identity | `[bridge]` | config.toml / env |
|
||||
| Console bind | `[console]` | config.toml / env |
|
||||
|
||||
**ConfigStore settings** (~40 settings) are loaded from the database after
|
||||
**ConfigStore settings** (48 settings) are loaded from the database after
|
||||
storage initialization:
|
||||
|
||||
| Section | Settings |
|
||||
@@ -60,10 +58,12 @@ storage initialization:
|
||||
| `session` | instructions, retention_days, compact_max_tokens, auto_compact_pct |
|
||||
| `tools` | timeout, truncation, agent_max_turns, skip_permissions, search, search_threshold, search_max_results |
|
||||
| `server` | workstream_idle_timeout, max_workstreams |
|
||||
| `mcp` | config_path, refresh_interval |
|
||||
| `ratelimit` | enabled, requests_per_second, burst |
|
||||
| `cluster` | node_fan_out_limit, mcp_max_servers |
|
||||
| `mcp` | config_path, refresh_interval, registry_url |
|
||||
| `ratelimit` | enabled, requests_per_second, burst, trusted_proxies |
|
||||
| `health` | backend_probe_interval, backend_probe_timeout, circuit_breaker_threshold, circuit_breaker_cooldown |
|
||||
| `judge` | enabled, model, provider, base_url, api_key, confidence_threshold, max_context_ratio, timeout, read_only_tools |
|
||||
| `judge` | enabled, model, provider, base_url, api_key, confidence_threshold, max_context_ratio, timeout, read_only_tools, output_guard, redact_secrets |
|
||||
| `skills` | discovery_url |
|
||||
| `memory` | relevance_k, fetch_limit, max_content, nudge_cooldown, nudges |
|
||||
|
||||
Settings are addressed by dotted key (e.g. `memory.relevance_k`). Each has a
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
# Cluster Simulator
|
||||
|
||||
The simulator (`turnstone-sim`) creates lightweight simulated nodes that talk to a real Redis instance using the standard turnstone protocol. External observers — `TurnstoneClient`, `turnstone-console`, real bridges — see identical behavior. No LLM backend is needed.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
pip install turnstone[sim]
|
||||
|
||||
# 10 nodes, steady load, 60 seconds
|
||||
turnstone-sim --nodes 10 --scenario steady --duration 60 --mps 5
|
||||
|
||||
# 100 nodes via Docker
|
||||
docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
Each simulated node is an asyncio coroutine (not a thread or process), so 1000 nodes run efficiently on a single event loop. The simulator:
|
||||
|
||||
1. Registers nodes via Redis heartbeats (same keys as real bridges)
|
||||
2. Accepts messages from per-node and shared inbound queues
|
||||
3. Simulates LLM responses with configurable latency and token generation
|
||||
4. Simulates tool execution with configurable latency and failure rates
|
||||
5. Publishes real protocol events (`ContentEvent`, `StateChangeEvent`, `TurnCompleteEvent`, etc.)
|
||||
6. Reports latency, throughput, and utilization metrics at completion
|
||||
|
||||
```
|
||||
TurnstoneClient → Redis Queue → SimNode → Redis Pub/Sub → TurnstoneClient
|
||||
↓
|
||||
turnstone-console (cluster dashboard)
|
||||
```
|
||||
|
||||
## Scenarios
|
||||
|
||||
| Scenario | Description |
|
||||
|----------|-------------|
|
||||
| `steady` | Inject messages at a constant rate (`--mps`) for `--duration` seconds |
|
||||
| `burst` | Push `--burst-size` messages instantly, then wait for completion |
|
||||
| `node_failure` | Steady load + periodically kill nodes to test redistribution |
|
||||
| `directed` | Send messages to specific nodes via `target_node` routing |
|
||||
| `lifecycle` | Create, use, and close workstreams across nodes |
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
turnstone-sim [options]
|
||||
```
|
||||
|
||||
### Cluster
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--nodes` | `10` | Number of simulated nodes |
|
||||
|
||||
### Scenario
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--scenario` | `steady` | Scenario name |
|
||||
| `--duration` | `60` | Duration in seconds |
|
||||
| `--mps` | `5.0` | Messages per second (steady) |
|
||||
| `--burst-size` | `100` | Messages to send (burst) |
|
||||
| `--node-kill-interval` | `15` | Seconds between kills (node_failure) |
|
||||
| `--node-kill-count` | `1` | Nodes per kill cycle |
|
||||
|
||||
### Simulation
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--llm-latency` | `2.0` | Mean LLM response latency (seconds) |
|
||||
| `--tool-latency` | `0.5` | Mean tool execution latency (seconds) |
|
||||
| `--tool-failure-rate` | `0.02` | Tool failure probability (0.0–1.0) |
|
||||
| `--seed` | — | Random seed for reproducibility |
|
||||
|
||||
### Redis
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--redis-host` | `localhost` | Redis host |
|
||||
| `--redis-port` | `6379` | Redis port |
|
||||
| `--redis-password` | — | Redis password |
|
||||
| `--prefix` | `turnstone` | Redis key prefix |
|
||||
|
||||
### Output
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--metrics-file` | — | Write JSON report to file |
|
||||
| `--log-level` | `INFO` | Log verbosity |
|
||||
|
||||
## Example: Load Testing
|
||||
|
||||
```bash
|
||||
# 100 nodes, high throughput, 2 minutes
|
||||
turnstone-sim --nodes 100 --scenario steady --duration 120 --mps 50
|
||||
|
||||
# Burst of 500 messages across 50 nodes
|
||||
turnstone-sim --nodes 50 --scenario burst --burst-size 500 --duration 60
|
||||
|
||||
# Node failure resilience (kill 2 nodes every 10 seconds)
|
||||
turnstone-sim --nodes 20 --scenario node_failure --duration 120 \
|
||||
--node-kill-interval 10 --node-kill-count 2
|
||||
|
||||
# Fast simulation (low latency, no failures)
|
||||
turnstone-sim --nodes 10 --scenario steady --duration 30 \
|
||||
--llm-latency 0.1 --tool-latency 0.05 --tool-failure-rate 0 --mps 10
|
||||
```
|
||||
|
||||
## Metrics Report
|
||||
|
||||
The simulator prints a summary at completion:
|
||||
|
||||
```
|
||||
============================================================
|
||||
SIMULATION REPORT
|
||||
============================================================
|
||||
Scenario: steady
|
||||
Nodes: 100
|
||||
Duration: 60.2s
|
||||
Total turns: 295
|
||||
Total errors: 5
|
||||
Node kills: 0
|
||||
------------------------------------------------------------
|
||||
THROUGHPUT
|
||||
Messages/sec: 4.97
|
||||
Turns/sec: 4.89
|
||||
------------------------------------------------------------
|
||||
LATENCY (seconds)
|
||||
p50: 3.21
|
||||
p90: 5.44
|
||||
p99: 8.12
|
||||
mean: 3.56
|
||||
max: 12.1
|
||||
------------------------------------------------------------
|
||||
UTILIZATION
|
||||
Mean ws/node: 2.3
|
||||
Max ws/node: 8
|
||||
Idle nodes: 12
|
||||
============================================================
|
||||
```
|
||||
|
||||
Use `--metrics-file report.json` to write the full report as JSON.
|
||||
|
||||
## Console Integration
|
||||
|
||||
The simulator's nodes appear in `turnstone-console` exactly like real nodes. Run them together to see the dashboard populate with simulated workstreams:
|
||||
|
||||
```bash
|
||||
# Terminal 1: start Redis and console
|
||||
docker compose up redis console
|
||||
|
||||
# Terminal 2: run simulator
|
||||
docker compose --profile sim up sim
|
||||
```
|
||||
|
||||
Or all at once:
|
||||
|
||||
```bash
|
||||
SIM_NODES=50 SIM_DURATION=120 docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
Open http://localhost:8090 to see simulated nodes, workstream states, token counts, and load bars updating in real time.
|
||||
|
||||
## Architecture
|
||||
|
||||
> See also: [Simulator Architecture diagram](diagrams/png/10-simulator-architecture.png)
|
||||
|
||||
```
|
||||
turnstone/sim/
|
||||
├── __init__.py # Public API: SimCluster, SimConfig
|
||||
├── config.py # SimConfig — all simulation parameters
|
||||
├── engine.py # SimEngine — LLM + tool execution simulation
|
||||
├── node.py # SimNode + SimWorkstream — protocol-compatible node
|
||||
├── cluster.py # SimCluster + InboundDispatcher + PooledBroker
|
||||
├── scenario.py # 5 scenario classes
|
||||
├── metrics.py # MetricsCollector — latency, throughput, utilization
|
||||
└── cli.py # CLI entry point
|
||||
```
|
||||
|
||||
**Key design:** The `InboundDispatcher` batches ~50 node queues into a single Redis `BLPOP` call, keeping connection count bounded at ~20 regardless of node count. All nodes share a single `ConnectionPool(max_connections=64)`.
|
||||
|
||||
## Programmatic Use
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from turnstone.sim import SimCluster, SimConfig
|
||||
|
||||
async def main():
|
||||
config = SimConfig(
|
||||
num_nodes=10,
|
||||
scenario="steady",
|
||||
duration=30,
|
||||
messages_per_second=2.0,
|
||||
llm_latency_mean=0.5,
|
||||
)
|
||||
cluster = SimCluster(config)
|
||||
await cluster.start()
|
||||
await cluster.run_scenario()
|
||||
print(cluster.report())
|
||||
await cluster.stop()
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
# TLS / mTLS
|
||||
|
||||
Turnstone supports end-to-end transport encryption with mutual TLS (mTLS) for
|
||||
inter-service communication, powered by [lacme](https://pypi.org/project/lacme/).
|
||||
|
||||
---
|
||||
|
||||
## Quick Start (Docker Compose)
|
||||
|
||||
```bash
|
||||
docker compose -f compose.yaml -f deploy/docker-compose.tls.yml up
|
||||
```
|
||||
|
||||
This:
|
||||
1. Bootstraps an internal CA and issues certs for PostgreSQL
|
||||
2. Starts the console with TLS enabled (internal CA + ACME server)
|
||||
3. Server nodes auto-provision certs via the console's ACME endpoint
|
||||
4. All inter-service communication uses mTLS
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Console (CA + ACME Server)
|
||||
+-- CertificateAuthority (owns root key, signs certs)
|
||||
+-- ACMEResponder (mounted at /acme, RFC 8555)
|
||||
+-- GET /acme/ca.pem (root cert for node bootstrapping)
|
||||
|
|
||||
| ACME protocol (auto-approve, no challenge validation)
|
||||
+-----------+-----------+
|
||||
| | |
|
||||
Server(s) Channel GW
|
||||
(auto-cert (mTLS
|
||||
+ renewal) client)
|
||||
```
|
||||
|
||||
**Two cert paths on the console:**
|
||||
- **Internal cert** (mTLS): Always from the internal CA. Used for cluster
|
||||
service mesh communication.
|
||||
- **Frontend cert** (HTTPS): From an external ACME CA (e.g. Let's Encrypt)
|
||||
if `tls.acme_directory` is set, otherwise self-issued from the internal CA.
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### Settings (ConfigStore / Admin Settings tab)
|
||||
|
||||
| Setting | Default | Description |
|
||||
|---------|---------|-------------|
|
||||
| `tls.enabled` | `false` | Master switch for internal mTLS |
|
||||
| `tls.acme_directory` | `""` | External ACME CA URL for console frontend cert |
|
||||
|
||||
### Bootstrap Config (config.toml)
|
||||
|
||||
These are needed before storage is available:
|
||||
|
||||
```toml
|
||||
[database]
|
||||
sslmode = "prefer" # disable, allow, prefer, require, verify-full
|
||||
sslrootcert = "" # path to CA cert
|
||||
sslcert = "" # path to client cert
|
||||
sslkey = "" # path to client key
|
||||
```
|
||||
|
||||
### Hardcoded Defaults
|
||||
|
||||
| Parameter | Value | Notes |
|
||||
|-----------|-------|-------|
|
||||
| CA common name | "Turnstone CA" | |
|
||||
| CA validity | 10 years | |
|
||||
| Cert validity | 48 hours | Short-lived, auto-renewed |
|
||||
| Renewal interval | 24 hours | Half of validity |
|
||||
| ACME auto-approve | true | Internal network, no challenge validation |
|
||||
|
||||
---
|
||||
|
||||
## CLI
|
||||
|
||||
### Offline Bootstrap
|
||||
|
||||
Create a CA and infrastructure certs without a running console:
|
||||
|
||||
```bash
|
||||
# Bootstrap CA + PostgreSQL certs
|
||||
turnstone-admin tls-bootstrap --out /certs --issue postgres
|
||||
|
||||
# Output:
|
||||
# /certs/ca.pem (CA root certificate)
|
||||
# /certs/certs/postgres/ (PostgreSQL cert + key)
|
||||
```
|
||||
|
||||
The output directory is chmod 0700 (contains the CA private key).
|
||||
|
||||
### Online Cert Issuance
|
||||
|
||||
Request certs from a running console's ACME endpoint:
|
||||
|
||||
```bash
|
||||
# Download CA root cert (TOFU — verify fingerprint)
|
||||
turnstone-admin tls-ca-cert --out ca.pem --console-url http://console:8080
|
||||
|
||||
# Request a cert for a domain
|
||||
turnstone-admin tls-issue worker-1.internal --out /certs --console-url http://console:8080
|
||||
|
||||
# List issued certs
|
||||
turnstone-admin tls-list --console-url http://console:8080
|
||||
```
|
||||
|
||||
### Console URL Discovery
|
||||
|
||||
If `--console-url` is not provided, the CLI discovers it from the `services`
|
||||
table in the shared database. The console registers itself on startup.
|
||||
|
||||
---
|
||||
|
||||
## Admin UI
|
||||
|
||||
The **TLS** tab in the console admin panel (System group) shows:
|
||||
- CA status (common name, certificate count)
|
||||
- Certificate table (domain, SANs, issued, expires)
|
||||
- Force-renew and delete actions per certificate
|
||||
|
||||
---
|
||||
|
||||
## SDK
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
client = TurnstoneServer(
|
||||
base_url="https://server:8080",
|
||||
token="tok_xxx",
|
||||
ca_cert="/path/to/ca.pem",
|
||||
client_cert="/path/to/cert.pem",
|
||||
client_key="/path/to/key.pem",
|
||||
)
|
||||
```
|
||||
|
||||
### TypeScript
|
||||
|
||||
```typescript
|
||||
import { TurnstoneServer } from "@turnstone/sdk";
|
||||
import { Agent } from "undici";
|
||||
import * as fs from "fs";
|
||||
|
||||
const agent = new Agent({
|
||||
connect: {
|
||||
ca: fs.readFileSync("/path/to/ca.pem"),
|
||||
cert: fs.readFileSync("/path/to/cert.pem"),
|
||||
key: fs.readFileSync("/path/to/key.pem"),
|
||||
},
|
||||
});
|
||||
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "https://server:8080",
|
||||
token: "tok_xxx",
|
||||
// Node.js 18+ uses undici under the hood
|
||||
fetch: (url, init) =>
|
||||
fetch(url, { ...init, dispatcher: agent } as RequestInit),
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## How It Works
|
||||
|
||||
### Node Bootstrap Flow
|
||||
|
||||
1. Node starts, connects to shared database (plain connection)
|
||||
2. Discovers console URL from `services` table
|
||||
3. Fetches CA root cert from `http://console/acme/ca.pem` (plain HTTP, TOFU)
|
||||
4. Requests service cert via ACME protocol (plain HTTP, JWS-signed)
|
||||
5. Starts auto-renewal (24h interval, re-issues before expiry)
|
||||
6. All subsequent inter-service communication uses mTLS
|
||||
|
||||
### Console Startup Flow
|
||||
|
||||
1. Read `tls.enabled` from ConfigStore
|
||||
2. Initialize CA (load from DB or generate new root key)
|
||||
3. Mount ACME responder at `/acme` (serves `/ca.pem` natively)
|
||||
4. Issue console certs (internal + optional frontend)
|
||||
5. Start CA-direct auto-renewal (no network, signs directly)
|
||||
6. Register console URL in services table with heartbeat
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Cert expired / mTLS connection refused
|
||||
|
||||
Certs are valid for 48 hours. If auto-renewal stopped (e.g. console was down),
|
||||
restart the service to re-request a cert.
|
||||
|
||||
### "No console service found"
|
||||
|
||||
The console registers itself in the `services` table on startup. If the console
|
||||
hasn't started or the registration expired (1 hour TTL), nodes can't discover
|
||||
it. Use `--console-url` explicitly.
|
||||
|
||||
### Let's Encrypt for console frontend
|
||||
|
||||
Set `tls.acme_directory` to `https://acme-v02.api.letsencrypt.org/directory`
|
||||
in the admin Settings tab. The console will request a publicly trusted cert
|
||||
for its HTTPS endpoint. Internal mTLS still uses the private CA.
|
||||
|
||||
### Verifying the cert chain
|
||||
|
||||
```bash
|
||||
openssl s_client -connect server:8080 -CAfile ca.pem
|
||||
```
|
||||
+93
-29
@@ -1,6 +1,6 @@
|
||||
# Tools Reference
|
||||
|
||||
turnstone exposes 17 built-in tools plus any number of external MCP tools to the
|
||||
turnstone exposes 19 built-in tools plus any number of external MCP tools to the
|
||||
LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON
|
||||
files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`.
|
||||
MCP tools are discovered from configured MCP servers at startup by
|
||||
@@ -46,12 +46,12 @@ schema plus turnstone-specific metadata keys:
|
||||
|
||||
| Name | Description |
|
||||
|---------------------|-------------|
|
||||
| `TOOLS` | All 17 tool definitions (sent to the model). |
|
||||
| `TOOLS` | All 19 tool definitions (sent to the model). |
|
||||
| `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. |
|
||||
| `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. |
|
||||
| `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. |
|
||||
| `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 17 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 19 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. |
|
||||
|
||||
---
|
||||
@@ -69,7 +69,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools
|
||||
- Parses the JSON arguments (with fallback for malformed JSON).
|
||||
- If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string
|
||||
to the correct parameter.
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 19
|
||||
built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and
|
||||
the generic `_prepare_mcp_tool()` handler for MCP tools.
|
||||
- Validates arguments and builds a preview dict containing:
|
||||
@@ -102,10 +102,15 @@ Each item's `execute` callable is invoked:
|
||||
- Errored or denied items return their error/denial message without executing.
|
||||
- The `bash` tool streams stdout incrementally: each line calls
|
||||
`ui.on_tool_output_chunk(call_id, line)` as it is produced, then the final
|
||||
combined output (stdout + stderr) is delivered via `ui.on_tool_result(call_id, name, output)`.
|
||||
combined output (stdout + stderr) is delivered via
|
||||
`ui.on_tool_result(call_id, name, output, is_error=...)`.
|
||||
The `call_id` links `tool_info`/`approve_request` items to their streaming chunks and
|
||||
final result, enabling correct routing when multiple bash tools run in parallel.
|
||||
Other tools deliver results atomically via `ui.on_tool_result(call_id, name, output)` only.
|
||||
The `is_error` flag is `True` when the tool execution failed (e.g. bash exit code >= 2
|
||||
or signal, file not found, timeout). Exit code 1 is ambiguous and not flagged; user
|
||||
denials are tracked separately. This removes the need for text-prefix heuristics.
|
||||
Other tools deliver results atomically via
|
||||
`ui.on_tool_result(call_id, name, output, is_error=...)` only.
|
||||
- Special post-execution gate for `plan`: the plan output is shown to the user
|
||||
for review, and the user can reject or annotate it.
|
||||
|
||||
@@ -183,8 +188,11 @@ Execute a bash command and return stdout + stderr.
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `command` | string | yes | The bash command to execute. |
|
||||
| `timeout` | integer | no | Timeout in seconds (1-600). Omit to use the global `tools.timeout` setting (typically 120s). |
|
||||
| `stop_on_error` | boolean | no | Enable `set -e` so the script exits on the first command failure. Default false. |
|
||||
|
||||
- **What it does**: Runs the command in a subprocess with a configurable timeout. Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`).
|
||||
- **What it does**: Runs the command in a subprocess with a configurable timeout. Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`). Environment variables containing secrets are scrubbed (`*_KEY`, `*_SECRET`, `*_TOKEN`, etc.).
|
||||
- **Output format**: Stdout is returned directly. Stderr lines are prefixed with `[stderr]` so the model can distinguish them. When the command itself redirects stderr to stdout (`2>&1`), no prefix is added. Output exceeding 256KB is truncated (head + tail preserved, middle replaced with a truncation notice).
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only (not available to plan sub-agents).
|
||||
|
||||
@@ -216,8 +224,9 @@ Write content to a file, creating it if needed.
|
||||
|-----------|--------|----------|-------------|
|
||||
| `path` | string | yes | Absolute or relative file path. |
|
||||
| `content` | string | yes | The full file content to write. |
|
||||
| `mode` | string | no | `"overwrite"` (default) replaces the file. `"append"` adds content to the end. |
|
||||
|
||||
- **What it does**: Creates or overwrites the file at the given path. Parent directories are created as needed.
|
||||
- **What it does**: Creates or overwrites (or appends to) the file at the given path. Parent directories are created as needed.
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only.
|
||||
|
||||
@@ -225,21 +234,44 @@ Write content to a file, creating it if needed.
|
||||
|
||||
### edit_file
|
||||
|
||||
Replace an exact string in a file with new content.
|
||||
Replace exact strings in a file, or apply multiple replacements atomically.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|--------------|---------|----------|-------------|
|
||||
| `path` | string | yes | Absolute or relative file path. |
|
||||
| `old_string` | string | yes | The exact text to find and replace. |
|
||||
| `new_string` | string | yes | The replacement text. |
|
||||
| `old_string` | string | no* | The exact text to find and replace. |
|
||||
| `new_string` | string | no* | The replacement text. |
|
||||
| `near_line` | integer | no | Disambiguate when `old_string` matches multiple locations. |
|
||||
| `edits` | array | no* | Multiple replacements to apply atomically (see below). |
|
||||
| `replace_all` | boolean | no | Replace ALL occurrences of `old_string`. Cannot combine with `near_line` or `edits`. |
|
||||
|
||||
- **What it does**: Finds `old_string` in the file and replaces it with `new_string`. Fails if the string is not found or matches multiple locations (unless `near_line` is provided to pick the nearest match). Requires a prior `read_file` call on the same path.
|
||||
\* Provide either `old_string`+`new_string` (single edit) or `edits` array (batch), not both.
|
||||
|
||||
- **What it does**: Finds `old_string` in the file and replaces it with `new_string`. Fails if the string is not found or matches multiple locations (unless `near_line` or `replace_all` is provided). Requires a prior `read_file` or `diff_file` call on the same path.
|
||||
- **Batch mode**: The `edits` array accepts multiple `{old_string, new_string, near_line?}` entries applied atomically. All edits are validated before any are applied. Overlapping edits (two entries targeting the same text region) are rejected. Edits are applied in reverse file-position order so character offsets stay stable.
|
||||
- **Replace-all mode**: When `replace_all` is true, all occurrences are replaced via `str.replace()`. The approval preview shows the occurrence count.
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only.
|
||||
|
||||
---
|
||||
|
||||
### diff_file
|
||||
|
||||
Show a unified diff between two files, or between a file and a provided string.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------------|---------|----------|-------------|
|
||||
| `path_a` | string | yes | Path to the first file. |
|
||||
| `path_b` | string | no | Path to the second file. Mutually exclusive with `content_b`. |
|
||||
| `content_b` | string | no | String content to compare against `path_a`. Mutually exclusive with `path_b`. |
|
||||
| `context_lines` | integer | no | Number of context lines around changes (default 3, max 20). |
|
||||
|
||||
- **What it does**: Returns unified diff output using Python's `difflib`. Binary files (containing null bytes) are rejected with a clear error. Files read through `diff_file` satisfy `edit_file`'s read guard — you can diff then edit without a separate `read_file` call. Large diffs are streamed with early cutoff at the tool truncation limit.
|
||||
- **Auto-approve**: Yes (read-only).
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
### search
|
||||
|
||||
Search file contents for a regex pattern.
|
||||
@@ -265,8 +297,9 @@ Execute Python code for math and computation in a sandbox.
|
||||
|-----------|--------|----------|-------------|
|
||||
| `code` | string | yes | Python code to execute. Must use `print()` for output. |
|
||||
|
||||
- **What it does**: Runs Python code in a sandboxed environment with pre-imported libraries: `sympy`, `numpy`, `scipy`, `math`, `fractions`, `itertools`, `functools`, `collections`, `decimal`, `operator`, `random`, `re`, `string`. Common sympy names (`symbols`, `solve`, `simplify`, `sqrt`, `Matrix`, etc.) are pre-imported.
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **What it does**: Runs Python code in a sandboxed environment with pre-imported libraries: `sympy`, `numpy`, `scipy`, `math`, `fractions`, `itertools`, `functools`, `collections`, `decimal`, `operator`, `random`, `re`, `string`. Common sympy names (`symbols`, `solve`, `simplify`, `sqrt`, `Matrix`, etc.) are pre-imported. `pytest` is also available for import.
|
||||
- **Installation**: `sympy`, `numpy`, `scipy`, and `pytest` require the `[sandbox]` extras group: `pip install turnstone[sandbox]` (included in `[all]`).
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
@@ -492,6 +525,36 @@ data.get("mergedAt") is not None
|
||||
|
||||
---
|
||||
|
||||
### skill
|
||||
|
||||
Discover and activate skills at runtime during a conversation. The model can
|
||||
search for available skills and load one by name, replacing the current active
|
||||
skill. This enables model-driven skill selection without requiring the user to
|
||||
pre-configure skills at workstream creation.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `action` | string | yes | `load` or `search`. |
|
||||
| `name` | string | load | Skill name to activate. |
|
||||
| `query` | string | no | Search query for finding skills (for `search` action). |
|
||||
|
||||
**Actions:**
|
||||
|
||||
- `load` — Activate a skill by name. Calls `set_skill()` which handles content
|
||||
rendering with `{{model}}`/`{{ws_id}}`/`{{node_id}}` variables, system message
|
||||
reinitialization, and config persistence. Returns the skill name, description,
|
||||
and security scan tier. Warns on high/critical scan status.
|
||||
- `search` — Find available skills by query. Uses BM25 relevance ranking over
|
||||
name, description, tags, and category (same `BM25Index` used by memory
|
||||
relevance and tool search). Returns up to 10 results with name, description,
|
||||
category, scan status, and activation type.
|
||||
|
||||
- **Auto-approve**: `load` requires approval (changes session behavior); `search`
|
||||
is auto-approved (read-only).
|
||||
- **Agent availability**: Main session only — not available to plan/task sub-agents.
|
||||
|
||||
---
|
||||
|
||||
## Summary Table
|
||||
|
||||
| Tool | Category | Auto-approve | agent | task_agent | primary_key |
|
||||
@@ -513,6 +576,7 @@ data.get("mergedAt") is not None
|
||||
| `watch` | Monitor | No (create) | No | No | `command` |
|
||||
| `read_resource`| MCP | No | Yes | Yes | `uri` |
|
||||
| `use_prompt` | MCP | No | Yes | Yes | `name` |
|
||||
| `skill` | Skills | No (load) | No | No | `name` |
|
||||
| `tool_search`| Search | Yes | No | No | `query` |
|
||||
|
||||
---
|
||||
@@ -529,7 +593,7 @@ current turn and letting it search for them on demand.
|
||||
Tool search uses the best available mechanism for each provider:
|
||||
|
||||
1. **Anthropic (native)** -- Models that support it receive `defer_loading: true`
|
||||
on deferred tool definitions plus the `tool_search_tool_bm25_20251119` server-side
|
||||
on deferred tool definitions plus the `tool_search_tool_bm25` server-side
|
||||
search tool. Anthropic's API handles search and expansion transparently.
|
||||
|
||||
2. **OpenAI GPT-5.4+ (native)** -- Models with hosted tool search receive
|
||||
@@ -560,12 +624,12 @@ CLI flags override the config file:
|
||||
|
||||
### How it works
|
||||
|
||||
1. **Threshold check**: At session startup, `ToolSearchManager.should_activate()`
|
||||
counts total tools (built-in + MCP). If the count is below the threshold, tool
|
||||
search stays off and all tools are sent to the model directly.
|
||||
1. **Threshold check**: At session startup, if the total tool count (built-in + MCP)
|
||||
is below the threshold, tool search stays off and all tools are sent to the model
|
||||
directly.
|
||||
|
||||
2. **Partitioning**: When active, tools are split into two sets:
|
||||
- **Always-on** -- the 17 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
- **Always-on** -- the 19 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
These are always visible to the model.
|
||||
- **Deferred** -- all MCP tools. These are not sent in the tool list unless
|
||||
the model searches for them.
|
||||
@@ -608,7 +672,7 @@ MCP-compatible service.
|
||||
3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI
|
||||
function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`.
|
||||
|
||||
4. **Merging**: MCP tools are appended after the 17 built-in tools via
|
||||
4. **Merging**: MCP tools are appended after the 19 built-in tools via
|
||||
`merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority.
|
||||
When dynamic tool search is active, MCP tools are deferred rather than directly
|
||||
visible -- the model discovers them via search as needed (see
|
||||
@@ -626,7 +690,7 @@ that external tools are read-only. However, global overrides such as
|
||||
`--skip-permissions` will auto-approve all tools, including MCP tools. The
|
||||
interactive "Always" button adds specific tool types to the per-tool auto-approve
|
||||
set. The web UI and server use `approval_label` for MCP tools, giving
|
||||
per-prompt/per-resource granularity. The CLI and bridge use `func_name`, which
|
||||
per-prompt/per-resource granularity. The CLI uses `func_name`, which
|
||||
gives per-tool-type granularity (e.g., all `use_prompt` calls).
|
||||
|
||||
### Sub-agent availability
|
||||
@@ -812,7 +876,7 @@ the `initialize` handshake. Each prompt is stored with its prefixed name
|
||||
| `name` | string | yes | The prompt name (e.g. `mcp__server__prompt_name`). |
|
||||
| `arguments` | object | no | Key-value argument pairs for the prompt. Values must be strings. |
|
||||
|
||||
- **What it does**: Invokes an MCP prompt template by name via `MCPClientManager.get_prompt_sync()`, expanding it into messages. Returns the expanded prompt content formatted as `[role]: content` blocks joined with blank lines. The prompt catalog is listed in the system message so the model knows which prompts are available. Output is truncated by the standard tool output limiter.
|
||||
- **What it does**: Invokes an MCP prompt by name via `MCPClientManager.get_prompt_sync()`, expanding it into messages. Returns the expanded prompt content formatted as `[role]: content` blocks joined with blank lines. The prompt catalog is listed in the system message so the model knows which prompts are available. Output is truncated by the standard tool output limiter.
|
||||
- **Auto-approve**: No -- requires user confirmation (invokes external prompt servers).
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
@@ -825,18 +889,18 @@ built-in tool exposes this to the model as a function call.
|
||||
### Governance Sync
|
||||
|
||||
Discovered MCP prompts are automatically synced into the `prompt_templates`
|
||||
governance table as first-class governed templates:
|
||||
table (which stores skills) as first-class governed skills:
|
||||
|
||||
- **Origin tracking**: MCP-sourced templates have `origin="mcp"` and
|
||||
`mcp_server` set to the server name. Manual templates have
|
||||
- **Origin tracking**: MCP-sourced skills have `origin="mcp"` and
|
||||
`mcp_server` set to the server name. Manual skills have
|
||||
`origin="manual"`.
|
||||
- **Read-only**: MCP-sourced templates are `readonly=True`. The admin API
|
||||
- **Read-only**: MCP-sourced skills are `readonly=True`. The admin API
|
||||
returns 403 on update/delete attempts. The admin UI disables edit/delete
|
||||
buttons and shows an origin badge.
|
||||
- **Precedence**: If a manual template and MCP prompt share the same name,
|
||||
the manual template wins and the MCP prompt is skipped (with a log
|
||||
- **Precedence**: If a manual skill and MCP prompt share the same name,
|
||||
the manual skill wins and the MCP prompt is skipped (with a log
|
||||
warning).
|
||||
- **Lifecycle**: Templates are created on connect, updated on prompt list
|
||||
- **Lifecycle**: Skills are created on connect, updated on prompt list
|
||||
refresh, and removed when the MCP server no longer exposes the prompt.
|
||||
The sync runs automatically on connect, on `PromptListChangedNotification`,
|
||||
and on manual `/mcp refresh`.
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
# MCP Cluster Ops
|
||||
|
||||
An MCP server that exposes tools for executing commands across a [Turnstone](https://github.com/turnstonelabs/turnstone) cluster. Serves as a reference implementation for both MCP server patterns and Turnstone MQ client SDK usage.
|
||||
An MCP server that exposes tools for executing commands across a Turnstone cluster. Serves as a reference implementation for both MCP server patterns and Turnstone SDK usage.
|
||||
|
||||
## How it works
|
||||
|
||||
This server uses Turnstone's MQ client (`TurnstoneClient`) to dispatch shell commands to specific nodes via Redis. Remote agents execute the command and the raw bash output is captured directly from the `ToolResultEvent` stream — bypassing the costly "agent reads output → re-generates output as completion tokens" round-trip.
|
||||
This server uses the Turnstone console SDK (`TurnstoneConsole`) for node discovery and routing, and `TurnstoneServer` for per-node SSE streaming. The dispatch flow for each command is:
|
||||
|
||||
1. **Route** — `TurnstoneConsole.route_create_workstream(target_node=..., auto_approve=True)` creates a workstream pinned to the target node via the console's hash-ring routing proxy, returning `ws_id` and `node_url`.
|
||||
2. **Execute** — `TurnstoneServer(node_url, token=...)` connects directly to the node's SSE stream using the same `TURNSTONE_API_TOKEN`. `send_and_wait(prompt, ws_id)` runs the command and the raw bash output is captured from the `ToolResultEvent` — bypassing the costly "agent reads output then re-generates output as completion tokens" round-trip.
|
||||
3. **Cleanup** — `TurnstoneConsole.route_close(ws_id)` closes the workstream.
|
||||
|
||||
Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time is bounded by the slowest node rather than the sum.
|
||||
|
||||
@@ -19,8 +23,7 @@ Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time i
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A running Turnstone cluster (at least one `turnstone-server` + `turnstone-bridge`)
|
||||
- Redis accessible from wherever this MCP server runs
|
||||
- A running Turnstone cluster with at least one `turnstone-server` and a `turnstone-console`
|
||||
- Python 3.11+
|
||||
|
||||
## Installation
|
||||
@@ -28,10 +31,6 @@ Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time i
|
||||
```bash
|
||||
# From the turnstone repo root:
|
||||
pip install -e ./examples/mcp-cluster-ops
|
||||
|
||||
# Or install turnstone with MQ support first, then the example:
|
||||
pip install -e ".[mq]"
|
||||
pip install -e ./examples/mcp-cluster-ops
|
||||
```
|
||||
|
||||
## Configuration
|
||||
@@ -40,9 +39,8 @@ pip install -e ./examples/mcp-cluster-ops
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `REDIS_HOST` | `localhost` | Redis host |
|
||||
| `REDIS_PORT` | `6379` | Redis port |
|
||||
| `REDIS_PASSWORD` | _(none)_ | Redis password (use env vars, not config files) |
|
||||
| `TURNSTONE_CONSOLE_URL` | `http://localhost:8090` | Console URL for node discovery and routing |
|
||||
| `TURNSTONE_API_TOKEN` | _(none)_ | API token / JWT for authentication |
|
||||
| `MCP_CLUSTER_OPS_TIMEOUT` | `120` | Default command timeout (seconds, clamped 5-3600) |
|
||||
| `MCP_CLUSTER_OPS_MAX_OUTPUT` | `8192` | Max output bytes per node (0 = unlimited) |
|
||||
| `MCP_CLUSTER_OPS_MAX_NODES` | `32` | Max concurrent node dispatches |
|
||||
@@ -57,7 +55,7 @@ pip install -e ./examples/mcp-cluster-ops
|
||||
command = "mcp-cluster-ops"
|
||||
|
||||
[mcp.servers.cluster-ops.env]
|
||||
REDIS_HOST = "redis.example.com"
|
||||
TURNSTONE_CONSOLE_URL = "http://console.example.com:8090"
|
||||
```
|
||||
|
||||
**JSON** (via `--mcp-config`):
|
||||
@@ -68,7 +66,7 @@ REDIS_HOST = "redis.example.com"
|
||||
"cluster-ops": {
|
||||
"command": "mcp-cluster-ops",
|
||||
"env": {
|
||||
"REDIS_HOST": "redis.example.com"
|
||||
"TURNSTONE_CONSOLE_URL": "http://console.example.com:8090"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -90,10 +88,6 @@ node-2: /dev/sda1 500G 410G 90G 82% /
|
||||
node-3: /dev/sda1 1.0T 200G 800G 20% /
|
||||
```
|
||||
|
||||
## Why MQ client instead of HTTP SDK?
|
||||
|
||||
The HTTP SDK (`TurnstoneServer`) talks to a single server instance. The MQ client (`TurnstoneClient`) routes through Redis with `target_node` support, which is the entire point of cross-node cluster operations.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
**This MCP server grants the calling agent shell access to cluster nodes.**
|
||||
@@ -104,8 +98,8 @@ The HTTP SDK (`TurnstoneServer`) talks to a single server instance. The MQ clien
|
||||
is returned through the MCP tool result and becomes part of the LLM context.
|
||||
- The security boundary is at the MCP host layer -- use Turnstone's tool
|
||||
policy system to restrict which agents can invoke these tools.
|
||||
- Set `REDIS_PASSWORD` via your environment or a secrets manager -- avoid
|
||||
hardcoding passwords in config files.
|
||||
- Set `TURNSTONE_API_TOKEN` via your environment or a secrets manager -- avoid
|
||||
hardcoding tokens in config files.
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user