mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 25b5e32089 | |||
| 339981a258 | |||
| 06de9ff83b | |||
| 924b976f1f | |||
| d5db817391 | |||
| fc8ceb4c72 | |||
| 07234dec4d | |||
| dd4cc0b30d | |||
| e7fe8fca9d | |||
| 42b9f89988 | |||
| 77c0a7736b | |||
| 872e1770e6 | |||
| a6e929b0a0 | |||
| 047680d669 | |||
| 0fd0ad3b2d | |||
| f3dba836dd | |||
| 7adda343fc | |||
| a20a058c59 | |||
| 498f23c19e | |||
| e057c364b8 | |||
| e785a94539 | |||
| 2b58c127b1 | |||
| 5ee539c983 | |||
| 62a4ceac96 | |||
| 29c00c0cdf | |||
| b6e0f0fcca | |||
| 206e37e73e | |||
| 6c5441435b | |||
| f02972c11d | |||
| d28879208f | |||
| a1f00092f5 | |||
| bb3b735d69 | |||
| 87ffad18c2 |
@@ -12,3 +12,12 @@ venv/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.hypothesis/
|
||||
deploy/
|
||||
docs/
|
||||
tests/
|
||||
sdk/
|
||||
*.md
|
||||
!README.md
|
||||
!LICENSE
|
||||
.coverage
|
||||
.swp
|
||||
|
||||
+21
-77
@@ -1,85 +1,29 @@
|
||||
# =============================================================================
|
||||
# Turnstone Docker Compose — Environment Configuration
|
||||
# Copy to .env and fill in your values: cp .env.example .env
|
||||
# Turnstone Environment Variables
|
||||
# Copy to .env and adjust values for your deployment
|
||||
# =============================================================================
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM Backend
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenAI-compatible API URL (vLLM, llama.cpp, OpenAI, etc.)
|
||||
# -- LLM Backend --------------------------------------------------------------
|
||||
LLM_BASE_URL=http://host.docker.internal:8000/v1
|
||||
OPENAI_API_KEY=sk-...
|
||||
# ANTHROPIC_API_KEY=sk-ant-... # Set instead for Anthropic provider
|
||||
# TAVILY_API_KEY=tvly-... # For web search fallback (local models only)
|
||||
|
||||
# API key for the LLM backend ("dummy" for local servers without auth)
|
||||
OPENAI_API_KEY=dummy
|
||||
# -- Database (production profile) --------------------------------------------
|
||||
# DB_BACKEND=postgresql
|
||||
# POSTGRES_USER=turnstone
|
||||
# POSTGRES_PASSWORD=changeme
|
||||
# DATABASE_URL=postgresql+psycopg://turnstone:changeme@postgres:5432/turnstone
|
||||
|
||||
# Tavily API key for web_search tool (optional)
|
||||
TAVILY_API_KEY=
|
||||
# -- Redis ---------------------------------------------------------------------
|
||||
# REDIS_PASSWORD=
|
||||
# REDIS_PORT=6379
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Redis
|
||||
# ---------------------------------------------------------------------------
|
||||
# Redis password (leave empty for no authentication)
|
||||
REDIS_PASSWORD=
|
||||
# -- Authentication ------------------------------------------------------------
|
||||
# TURNSTONE_AUTH_ENABLED=true
|
||||
# TURNSTONE_AUTH_TOKEN=your-secret-token
|
||||
# TURNSTONE_JWT_SECRET=python -c "import secrets; print(secrets.token_hex(32))"
|
||||
|
||||
# Host port for Redis
|
||||
REDIS_PORT=6379
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server
|
||||
# ---------------------------------------------------------------------------
|
||||
# Host port for the turnstone web UI
|
||||
SERVER_PORT=8080
|
||||
|
||||
# Set to any non-empty value to auto-approve all tool calls
|
||||
SKIP_PERMISSIONS=
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bridge
|
||||
# ---------------------------------------------------------------------------
|
||||
# Heartbeat TTL in seconds
|
||||
HEARTBEAT_TTL=60
|
||||
|
||||
# Seconds to wait for external approval responses
|
||||
APPROVAL_TIMEOUT=300
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Console (Cluster Dashboard)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Host port for the cluster dashboard
|
||||
CONSOLE_PORT=8090
|
||||
|
||||
# Seconds between node polling cycles
|
||||
CONSOLE_POLL_INTERVAL=10
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth (optional)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Set to "1" to require Bearer token authentication
|
||||
TURNSTONE_AUTH_ENABLED=
|
||||
|
||||
# Bearer token for server/bridge/console authentication
|
||||
TURNSTONE_AUTH_TOKEN=
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Simulator (used with: docker compose --profile sim up)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Number of simulated nodes
|
||||
SIM_NODES=100
|
||||
|
||||
# Scenario: steady, burst, node_failure, directed, lifecycle
|
||||
SIM_SCENARIO=steady
|
||||
|
||||
# Scenario duration in seconds
|
||||
SIM_DURATION=60
|
||||
|
||||
# Messages per second (steady scenario)
|
||||
SIM_MPS=5.0
|
||||
|
||||
# Log level
|
||||
SIM_LOG_LEVEL=INFO
|
||||
|
||||
# Random seed for reproducibility (leave empty for random)
|
||||
SIM_SEED=
|
||||
|
||||
# Path to write JSON metrics report (leave empty to skip)
|
||||
SIM_METRICS_FILE=
|
||||
# -- Ports ---------------------------------------------------------------------
|
||||
# SERVER_PORT=8080
|
||||
# CONSOLE_PORT=8090
|
||||
|
||||
@@ -17,3 +17,5 @@ venv/
|
||||
.plan.md
|
||||
.plan-*.md
|
||||
.hypothesis/
|
||||
PROGRESS.md
|
||||
.coverage
|
||||
|
||||
+11
-2
@@ -25,22 +25,31 @@ FROM python:3.13-slim
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
# System dependencies for psycopg (PostgreSQL client library)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Non-root user
|
||||
RUN useradd --create-home --shell /bin/bash turnstone
|
||||
|
||||
# Install the wheel with all optional extras (redis for mq/console/sim)
|
||||
# Install the wheel with all optional extras
|
||||
COPY --from=builder /build/wheels/*.whl /tmp/wheels/
|
||||
RUN pip install --no-cache-dir "$(ls /tmp/wheels/*.whl)[mq,console,sim]" \
|
||||
RUN pip install --no-cache-dir "$(ls /tmp/wheels/*.whl)[mq,console,sim,postgres,discord]" \
|
||||
&& rm -rf /tmp/wheels
|
||||
|
||||
# Health check script (stdlib only, no pip deps needed)
|
||||
COPY docker/healthcheck.py /usr/local/bin/healthcheck.py
|
||||
|
||||
# Entrypoint script — runs migrations before starting
|
||||
COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
|
||||
# Data directory — SQLite DB is created in CWD
|
||||
WORKDIR /data
|
||||
RUN chown turnstone:turnstone /data
|
||||
|
||||
USER turnstone
|
||||
|
||||
ENTRYPOINT ["entrypoint.sh"]
|
||||
|
||||
# Default command (overridden per service in compose.yaml)
|
||||
CMD ["turnstone-server", "--host", "0.0.0.0", "--port", "8080"]
|
||||
|
||||
@@ -16,15 +16,55 @@ Turnstone gives LLMs tools — shell, files, search, web, planning — and orche
|
||||
- **Interactive sessions** — terminal CLI or browser UI with parallel workstreams
|
||||
- **Queue-driven agents** — trigger workstreams via message queue, stream progress, approve or auto-approve tool use
|
||||
- **Multi-node clusters** — generic work load-balances across nodes, directed work routes to a specific server
|
||||
- **Cluster dashboard** — real-time view of all nodes, workstreams, and resource utilization
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams, workstream creation with node targeting, reverse proxy for server UIs (only the console port needs network access)
|
||||
- **Cluster simulator** — test the stack at scale (up to 1000 nodes) without an LLM backend
|
||||
|
||||
```
|
||||
External System → Message Queue → Bridge (per node) → Turnstone Server → LLM + Tools
|
||||
↓
|
||||
Pub/Sub → Progress Events → External System
|
||||
↓
|
||||
turnstone-console → Cluster Dashboard (browser)
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph Clients
|
||||
CLI[turnstone CLI]
|
||||
UI[Browser UI]
|
||||
SDK[SDK / API]
|
||||
Discord[Discord / Slack]
|
||||
end
|
||||
|
||||
Console[turnstone-console<br/><i>dashboard + proxy</i>]
|
||||
Channel[turnstone-channel<br/><i>platform gateway</i>]
|
||||
|
||||
subgraph Cluster
|
||||
Redis[(Redis MQ)]
|
||||
DB[(PostgreSQL / SQLite)]
|
||||
|
||||
subgraph Node A
|
||||
BridgeA[bridge]
|
||||
ServerA[server]
|
||||
end
|
||||
subgraph Node B
|
||||
BridgeB[bridge]
|
||||
ServerB[server]
|
||||
end
|
||||
end
|
||||
|
||||
LLM[LLM Provider<br/><i>OpenAI · Anthropic · local</i>]
|
||||
|
||||
CLI --> ServerA
|
||||
UI --> ServerB
|
||||
SDK --> Redis
|
||||
Discord --> Channel
|
||||
|
||||
Channel <--> Redis
|
||||
Console --> Redis
|
||||
Redis --> BridgeA & BridgeB
|
||||
BridgeA --> ServerA
|
||||
BridgeB --> ServerB
|
||||
ServerA & ServerB --> LLM
|
||||
|
||||
ServerA & ServerB --> DB
|
||||
Console --> DB
|
||||
Channel --> DB
|
||||
ServerA -.->|notify| Channel
|
||||
BridgeA & BridgeB -.->|events| Redis
|
||||
Console -.->|proxy| ServerA & ServerB
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
@@ -72,13 +112,20 @@ pip install turnstone[console]
|
||||
turnstone-console --redis-host localhost --port 8090
|
||||
```
|
||||
|
||||
Then open `http://localhost:8090` for the cluster-wide dashboard.
|
||||
Then open `http://localhost:8090` for the cluster-wide dashboard. Create workstreams from the console and interact with any node's server UI through the built-in reverse proxy — no direct server port access required.
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
cp .env.example .env # edit LLM_BASE_URL, OPENAI_API_KEY, etc.
|
||||
docker compose up # starts redis + server + bridge + console
|
||||
docker compose up # starts redis + server + bridge + console (SQLite)
|
||||
```
|
||||
|
||||
For production with PostgreSQL:
|
||||
|
||||
```bash
|
||||
# Requires POSTGRES_PASSWORD and DB_BACKEND=postgresql in .env (or exported)
|
||||
docker compose --profile production up # adds PostgreSQL, uses it as database
|
||||
```
|
||||
|
||||
Console dashboard at http://localhost:8090. See [docs/docker.md](docs/docker.md) for configuration, scaling, and profiles.
|
||||
@@ -100,62 +147,11 @@ turnstone-sim --nodes 100 --scenario steady --duration 60 --mps 10
|
||||
|
||||
See [docs/simulator.md](docs/simulator.md) for scenarios, CLI reference, and metrics.
|
||||
|
||||
All frontends connect to any OpenAI-compatible API (vLLM, NVIDIA NIM/NGC, llama.cpp, OpenAI, etc.) and auto-detect the model.
|
||||
All frontends connect to any OpenAI-compatible API (vLLM, NVIDIA NIM/NGC, llama.cpp, OpenAI, etc.) or Anthropic's native Messages API, and auto-detect the model.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
turnstone/
|
||||
├── core/ # UI-agnostic engine
|
||||
│ ├── session.py # ChatSession — multi-turn loop, tool dispatch, agents
|
||||
│ ├── tools.py # Tool definitions (auto-loaded from JSON)
|
||||
│ ├── workstream.py # WorkstreamManager — parallel independent sessions
|
||||
│ ├── mcp_client.py # MCP client manager (external tool servers)
|
||||
│ ├── model_registry.py # ModelRegistry — named models, fallback routing, per-workstream selection
|
||||
│ ├── config.py # Unified TOML config (~/.config/turnstone/config.toml)
|
||||
│ ├── memory.py # SQLite persistence (memories, conversations, FTS5)
|
||||
│ ├── metrics.py # Prometheus-compatible metrics collector
|
||||
│ ├── healthcheck.py # Backend health monitor + circuit breaker
|
||||
│ ├── ratelimit.py # Per-IP token-bucket rate limiter
|
||||
│ ├── edit.py # File editing (fuzzy match, indentation)
|
||||
│ ├── safety.py # Path validation, sandbox checks
|
||||
│ ├── sandbox.py # Command sandboxing
|
||||
│ └── web.py # Web fetch/search helpers
|
||||
├── mq/ # Message queue integration
|
||||
│ ├── protocol.py # Typed message dataclasses (JSON serialization)
|
||||
│ ├── broker.py # Abstract MessageBroker + RedisBroker
|
||||
│ ├── bridge.py # Bridge service (queue ↔ HTTP API, multi-node routing)
|
||||
│ └── client.py # TurnstoneClient — Python API for external systems
|
||||
├── console/ # Cluster dashboard
|
||||
│ ├── collector.py # ClusterCollector — aggregates all nodes via Redis + HTTP
|
||||
│ ├── server.py # Dashboard HTTP server + SSE
|
||||
│ └── static/ # Cluster dashboard web UI
|
||||
├── tools/ # Tool schemas (one JSON file per tool)
|
||||
├── ui/ # Frontend assets and terminal rendering
|
||||
│ └── static/ # Web UI (HTML, CSS, JS)
|
||||
├── sim/ # Cluster simulator
|
||||
│ ├── cluster.py # SimCluster — orchestrates N nodes + dispatchers
|
||||
│ ├── node.py # SimNode + SimWorkstream — protocol-compatible node
|
||||
│ ├── engine.py # LLM + tool execution simulation
|
||||
│ ├── scenario.py # 5 workload scenarios (steady, burst, node_failure, …)
|
||||
│ ├── metrics.py # Latency, throughput, utilization collection
|
||||
│ └── cli.py # CLI entry point (turnstone-sim)
|
||||
├── cli.py # Terminal frontend (+ /cluster commands for console)
|
||||
├── server.py # Web frontend (HTTP + SSE)
|
||||
└── eval.py # Evaluation and prompt optimization harness
|
||||
docs/
|
||||
├── architecture.md # System architecture and threading model
|
||||
├── api-reference.md # Web server API and SSE event reference
|
||||
├── console.md # Cluster dashboard service (turnstone-console)
|
||||
├── docker.md # Docker Compose deployment and configuration
|
||||
├── simulator.md # Cluster simulator usage and scenarios
|
||||
├── tools.md # Tool schemas, execution pipeline, approval flow
|
||||
├── eval.md # Evaluation harness internals
|
||||
└── diagrams/ # UML architecture diagrams (PlantUML sources + PNGs)
|
||||
└── png/ # Pre-rendered diagram images
|
||||
```
|
||||
|
||||
### Architecture Diagrams
|
||||
### Diagrams
|
||||
|
||||
Detailed UML diagrams are available in [`docs/diagrams/`](docs/diagrams/):
|
||||
|
||||
@@ -163,8 +159,8 @@ Detailed UML diagrams are available in [`docs/diagrams/`](docs/diagrams/):
|
||||
|---------|-------------|
|
||||
| [System Context](docs/diagrams/png/01-system-context.png) | Top-level components and external dependencies |
|
||||
| [Package Structure](docs/diagrams/png/02-package-structure.png) | Python modules and dependency graph |
|
||||
| [Core Engine Classes](docs/diagrams/png/03-core-engine-classes.png) | SessionUI protocol, ChatSession, WorkstreamManager |
|
||||
| [Conversation Turn](docs/diagrams/png/04-conversation-turn.png) | Full message lifecycle through the engine |
|
||||
| [Core Engine Classes](docs/diagrams/png/03-core-engine-classes.png) | SessionUI protocol, ChatSession, LLMProvider, WorkstreamManager |
|
||||
| [Conversation Turn](docs/diagrams/png/04-conversation-turn.png) | Full message lifecycle through the engine (provider-agnostic) |
|
||||
| [Tool Pipeline](docs/diagrams/png/05-tool-pipeline.png) | Three-phase prepare/approve/execute |
|
||||
| [MQ Protocol](docs/diagrams/png/06-mq-protocol.png) | 9 inbound + 19 outbound message types |
|
||||
| [Message Routing](docs/diagrams/png/07-message-routing.png) | Multi-node routing scenarios |
|
||||
@@ -173,6 +169,8 @@ Detailed UML diagrams are available in [`docs/diagrams/`](docs/diagrams/):
|
||||
| [Simulator](docs/diagrams/png/10-simulator-architecture.png) | SimCluster, dispatchers, scenarios |
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection threads |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose service topology |
|
||||
| [SDK Architecture](docs/diagrams/png/13-sdk-architecture.png) | Python + TypeScript client libraries |
|
||||
| [Storage Architecture](docs/diagrams/png/14-storage-architecture.png) | Pluggable database backends (SQLite + PostgreSQL) |
|
||||
|
||||
## Multi-node routing
|
||||
|
||||
@@ -209,7 +207,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct
|
||||
| `math` | Sandboxed Python evaluation | |
|
||||
| `man` | Read man pages | yes |
|
||||
| `web_fetch` | Fetch URL content | |
|
||||
| `web_search` | Search via Tavily API | |
|
||||
| `web_search` | Web search (provider-native or Tavily) | |
|
||||
| `remember` | Save persistent facts | yes |
|
||||
| `recall` | Search memories and history | yes |
|
||||
| `forget` | Remove a memory | yes |
|
||||
@@ -241,28 +239,37 @@ turnstone-server --mcp-config ~/.config/turnstone/mcp.json
|
||||
|
||||
Use `/mcp` in the REPL to list connected tools. MCP tools require user approval by default (overridden by `--skip-permissions` or UI auto-approve).
|
||||
|
||||
### Multi-Model Support
|
||||
### Multi-Model and Multi-Provider Support
|
||||
|
||||
Turnstone supports multiple model backends per server instance. Define named models in `config.toml` and select per-workstream or switch mid-session with `/model <alias>`.
|
||||
Turnstone supports multiple model backends per server instance, including different LLM providers. `ChatSession` delegates all API communication to pluggable `LLMProvider` adapters — the internal message format stays OpenAI-like, and each provider translates at the API boundary. Define named models in `config.toml` and select per-workstream or switch mid-session with `/model <alias>`.
|
||||
|
||||
```toml
|
||||
[models.local]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
model = "qwen3-32b"
|
||||
# provider defaults to "openai" (works with vLLM, llama.cpp, etc.)
|
||||
|
||||
[models.claude]
|
||||
provider = "anthropic"
|
||||
api_key = "sk-ant-..."
|
||||
model = "claude-opus-4-6"
|
||||
context_window = 200000
|
||||
|
||||
[models.openai]
|
||||
base_url = "https://api.openai.com/v1"
|
||||
api_key = "sk-..."
|
||||
model = "gpt-4o"
|
||||
context_window = 128000
|
||||
model = "gpt-5"
|
||||
context_window = 400000
|
||||
|
||||
[model]
|
||||
default = "local" # which model to use by default
|
||||
fallback = ["openai"] # try these if the primary is unreachable
|
||||
agent_model = "local" # optional: cheaper model for plan/task sub-agents
|
||||
fallback = ["claude", "openai"] # try these if the primary is unreachable
|
||||
agent_model = "claude" # optional: separate model for plan/task sub-agents
|
||||
```
|
||||
|
||||
Use `/model` to show available models, `/model openai` to switch. Workstreams created via the API accept an optional `model` parameter.
|
||||
Supported providers: `"openai"` (default -- OpenAI, vLLM, llama.cpp, any OpenAI-compatible API) and `"anthropic"` (Anthropic Messages API, requires `pip install turnstone[anthropic]`).
|
||||
|
||||
Use `/model` to show available models, `/model claude` to switch. Workstreams created via the API accept an optional `model` parameter.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -272,7 +279,7 @@ All entry points read `~/.config/turnstone/config.toml`. CLI flags override conf
|
||||
[api]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
api_key = ""
|
||||
tavily_key = ""
|
||||
tavily_key = "" # only needed for local/vLLM models without native search
|
||||
|
||||
[model]
|
||||
name = "" # empty = auto-detect
|
||||
@@ -317,6 +324,12 @@ enabled = true
|
||||
requests_per_second = 10.0
|
||||
burst = 20
|
||||
|
||||
[database]
|
||||
backend = "sqlite" # "sqlite" (default) or "postgresql"
|
||||
path = ".turnstone.db" # SQLite file path (relative to working directory)
|
||||
# url = "postgresql+psycopg://user:pass@host:5432/turnstone" # PostgreSQL
|
||||
# pool_size = 5 # PostgreSQL connection pool size
|
||||
|
||||
[mcp]
|
||||
config_path = "" # path to MCP JSON config file (alternative to TOML sections)
|
||||
|
||||
@@ -373,8 +386,11 @@ Per-workstream metrics are labeled by `ws_id` (bounded to 10 max workstreams).
|
||||
## Requirements
|
||||
|
||||
- Python 3.11+
|
||||
- An OpenAI-compatible API endpoint ([vLLM](https://github.com/vllm-project/vllm), [NVIDIA NIM](https://build.nvidia.com/), [llama.cpp](https://github.com/ggml-org/llama.cpp), etc.)
|
||||
- An OpenAI-compatible API endpoint ([vLLM](https://github.com/vllm-project/vllm), [NVIDIA NIM](https://build.nvidia.com/), [llama.cpp](https://github.com/ggml-org/llama.cpp), etc.) or an Anthropic API key
|
||||
- Redis (for message queue bridge — `pip install turnstone[mq]`)
|
||||
- Anthropic provider (optional — `pip install turnstone[anthropic]`)
|
||||
- PostgreSQL (optional, for production — `pip install turnstone[postgres]`)
|
||||
- [Git LFS](https://git-lfs.com/) (for cloning — diagram PNGs are stored in LFS)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+85
-5
@@ -2,10 +2,11 @@
|
||||
# Turnstone Docker Compose Stack
|
||||
#
|
||||
# Usage:
|
||||
# Full stack: docker compose up
|
||||
# With simulator: docker compose --profile sim up
|
||||
# Sim only: docker compose --profile sim up redis console sim
|
||||
# Scale bridges: docker compose up --scale bridge=3
|
||||
# Default (SQLite): docker compose up
|
||||
# Production (PG): DB_BACKEND=postgresql docker compose --profile production up
|
||||
# (or set DB_BACKEND=postgresql in .env)
|
||||
# With simulator: docker compose --profile sim up
|
||||
# Scale bridges: docker compose up --scale bridge=3
|
||||
# =============================================================================
|
||||
|
||||
name: turnstone
|
||||
@@ -17,8 +18,37 @@ networks:
|
||||
volumes:
|
||||
redis-data:
|
||||
turnstone-data:
|
||||
postgres-data:
|
||||
|
||||
services:
|
||||
# -------------------------------------------------------------------
|
||||
# PostgreSQL — production database (profile: production)
|
||||
# -------------------------------------------------------------------
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
profiles:
|
||||
- production
|
||||
environment:
|
||||
POSTGRES_DB: turnstone
|
||||
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required for production profile}
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- turnstone-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-turnstone}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
cpus: '1.0'
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Redis — message broker, pub/sub, node registry
|
||||
# -------------------------------------------------------------------
|
||||
@@ -66,6 +96,7 @@ services:
|
||||
--port 8080
|
||||
--base-url "$${LLM_BASE_URL}"
|
||||
--api-key "$${OPENAI_API_KEY}"
|
||||
$${MODEL:+--model $$MODEL}
|
||||
$${SKIP_PERMISSIONS:+--skip-permissions}
|
||||
ports:
|
||||
- "${SERVER_PORT:-8080}:8080"
|
||||
@@ -78,6 +109,11 @@ services:
|
||||
- SKIP_PERMISSIONS=${SKIP_PERMISSIONS:-}
|
||||
- TURNSTONE_AUTH_ENABLED=${TURNSTONE_AUTH_ENABLED:-}
|
||||
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
- MODEL=${MODEL:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
networks:
|
||||
@@ -85,6 +121,9 @@ services:
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
|
||||
interval: 10s
|
||||
@@ -107,10 +146,11 @@ services:
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-300}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
environment:
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
@@ -140,6 +180,9 @@ services:
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- TURNSTONE_AUTH_ENABLED=${TURNSTONE_AUTH_ENABLED:-}
|
||||
- TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
@@ -153,6 +196,43 @@ services:
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-channel — Channel gateway (Discord, Slack, etc.)
|
||||
# Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter
|
||||
# -------------------------------------------------------------------
|
||||
channel:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-channel
|
||||
--redis-host=redis
|
||||
--redis-port=6379
|
||||
--http-host=0.0.0.0
|
||||
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
|
||||
environment:
|
||||
- TURNSTONE_DISCORD_TOKEN=${TURNSTONE_DISCORD_TOKEN:-}
|
||||
- TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0}
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-sim — Multi-node cluster simulator (no LLM needed)
|
||||
# Start with: docker compose --profile sim up
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: v2
|
||||
name: turnstone
|
||||
description: Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "0.3.0"
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: ~16.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
- name: redis
|
||||
version: ~20.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: redis.enabled
|
||||
@@ -0,0 +1,42 @@
|
||||
Turnstone {{ .Chart.AppVersion }} has been deployed.
|
||||
|
||||
{{- if .Values.ingress.enabled }}
|
||||
|
||||
Access the application via your ingress:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
|
||||
{{- end }}
|
||||
|
||||
{{- else }}
|
||||
|
||||
To access the Turnstone server, run:
|
||||
|
||||
kubectl port-forward svc/{{ include "turnstone.fullname" . }}-server {{ .Values.server.service.port }}:{{ .Values.server.service.port }}
|
||||
|
||||
Then open: http://localhost:{{ .Values.server.service.port }}
|
||||
|
||||
To access the Turnstone console (cluster dashboard), run:
|
||||
|
||||
kubectl port-forward svc/{{ include "turnstone.fullname" . }}-console {{ .Values.console.service.port }}:{{ .Values.console.service.port }}
|
||||
|
||||
Then open: http://localhost:{{ .Values.console.service.port }}
|
||||
|
||||
{{- end }}
|
||||
|
||||
Components deployed:
|
||||
- Server: {{ include "turnstone.fullname" . }}-server ({{ .Values.server.replicas }} replica(s))
|
||||
- Bridge: {{ include "turnstone.fullname" . }}-bridge ({{ .Values.bridge.replicas }} replica(s))
|
||||
- Console: {{ include "turnstone.fullname" . }}-console ({{ .Values.console.replicas }} replica(s))
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
- PostgreSQL (bitnami subchart)
|
||||
{{- end }}
|
||||
{{- if .Values.redis.enabled }}
|
||||
- Redis (bitnami subchart)
|
||||
{{- end }}
|
||||
|
||||
{{- if not .Values.llm.apiKey }}
|
||||
{{- if not .Values.llm.existingSecret }}
|
||||
|
||||
WARNING: No LLM API key configured. Set llm.apiKey or llm.existingSecret in your values.
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,163 @@
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "turnstone.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this
|
||||
(by the DNS naming spec). If release name contains chart name it will be used
|
||||
as a full name.
|
||||
*/}}
|
||||
{{- define "turnstone.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "turnstone.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels.
|
||||
*/}}
|
||||
{{- define "turnstone.labels" -}}
|
||||
helm.sh/chart: {{ include "turnstone.chart" . }}
|
||||
{{ include "turnstone.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels.
|
||||
*/}}
|
||||
{{- define "turnstone.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "turnstone.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use.
|
||||
*/}}
|
||||
{{- define "turnstone.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount }}
|
||||
{{- if .Values.serviceAccount.name }}
|
||||
{{- .Values.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- include "turnstone.fullname" . }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
{{- include "turnstone.fullname" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the PostgreSQL host.
|
||||
*/}}
|
||||
{{- define "turnstone.postgresql.host" -}}
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
{{- printf "%s-postgresql" .Release.Name }}
|
||||
{{- else }}
|
||||
{{- .Values.database.external.host }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the PostgreSQL port.
|
||||
*/}}
|
||||
{{- define "turnstone.postgresql.port" -}}
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
{{- printf "5432" }}
|
||||
{{- else }}
|
||||
{{- .Values.database.external.port | toString }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the PostgreSQL database name.
|
||||
*/}}
|
||||
{{- define "turnstone.postgresql.database" -}}
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
{{- .Values.postgresql.auth.database }}
|
||||
{{- else }}
|
||||
{{- .Values.database.external.database }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the PostgreSQL username.
|
||||
*/}}
|
||||
{{- define "turnstone.postgresql.username" -}}
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
{{- .Values.postgresql.auth.username }}
|
||||
{{- else }}
|
||||
{{- .Values.database.external.username }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the Redis host.
|
||||
*/}}
|
||||
{{- define "turnstone.redis.host" -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- printf "%s-redis-master" .Release.Name }}
|
||||
{{- else }}
|
||||
{{- .Values.redis.external.host }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the Redis port.
|
||||
*/}}
|
||||
{{- define "turnstone.redis.port" -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- printf "6379" }}
|
||||
{{- else }}
|
||||
{{- .Values.redis.external.port | toString }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the secret name for LLM API keys.
|
||||
*/}}
|
||||
{{- define "turnstone.llm.secretName" -}}
|
||||
{{- if .Values.llm.existingSecret }}
|
||||
{{- .Values.llm.existingSecret }}
|
||||
{{- else }}
|
||||
{{- printf "%s-secrets" (include "turnstone.fullname" .) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the secret name for auth tokens.
|
||||
*/}}
|
||||
{{- define "turnstone.auth.secretName" -}}
|
||||
{{- if .Values.auth.existingSecret }}
|
||||
{{- .Values.auth.existingSecret }}
|
||||
{{- else }}
|
||||
{{- printf "%s-secrets" (include "turnstone.fullname" .) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Container image reference.
|
||||
*/}}
|
||||
{{- define "turnstone.image" -}}
|
||||
{{- $tag := .Values.image.tag | default .Chart.AppVersion }}
|
||||
{{- printf "%s:%s" .Values.image.repository $tag }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
data:
|
||||
TURNSTONE_DB_BACKEND: {{ .Values.database.backend | quote }}
|
||||
TURNSTONE_DB_HOST: {{ include "turnstone.postgresql.host" . | quote }}
|
||||
TURNSTONE_DB_PORT: {{ include "turnstone.postgresql.port" . | quote }}
|
||||
TURNSTONE_DB_NAME: {{ include "turnstone.postgresql.database" . | quote }}
|
||||
TURNSTONE_DB_USER: {{ include "turnstone.postgresql.username" . | quote }}
|
||||
TURNSTONE_SERVER_HOST: "0.0.0.0"
|
||||
TURNSTONE_SERVER_PORT: {{ .Values.server.service.port | quote }}
|
||||
TURNSTONE_CONSOLE_HOST: "0.0.0.0"
|
||||
TURNSTONE_CONSOLE_PORT: {{ .Values.console.service.port | quote }}
|
||||
TURNSTONE_REDIS_HOST: {{ include "turnstone.redis.host" . | quote }}
|
||||
TURNSTONE_REDIS_PORT: {{ include "turnstone.redis.port" . | quote }}
|
||||
TURNSTONE_POLL_INTERVAL: "5"
|
||||
{{- if .Values.llm.baseUrl }}
|
||||
TURNSTONE_LLM_BASE_URL: {{ .Values.llm.baseUrl | quote }}
|
||||
{{- end }}
|
||||
TURNSTONE_LLM_PROVIDER: {{ .Values.llm.provider | quote }}
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-bridge
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
spec:
|
||||
replicas: {{ .Values.bridge.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
spec:
|
||||
serviceAccountName: {{ include "turnstone.serviceAccountName" . }}
|
||||
containers:
|
||||
- name: bridge
|
||||
image: {{ include "turnstone.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url={{ printf "http://%s-server:%s" (include "turnstone.fullname" .) (.Values.server.service.port | toString) }}
|
||||
- --redis-host={{ include "turnstone.redis.host" . }}
|
||||
- --redis-port={{ include "turnstone.redis.port" . }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
env:
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.bridge.resources | nindent 12 }}
|
||||
@@ -0,0 +1,62 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-console
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: console
|
||||
spec:
|
||||
replicas: {{ .Values.console.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: console
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: console
|
||||
spec:
|
||||
serviceAccountName: {{ include "turnstone.serviceAccountName" . }}
|
||||
containers:
|
||||
- name: console
|
||||
image: {{ include "turnstone.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port={{ .Values.console.service.port }}
|
||||
- --redis-host={{ include "turnstone.redis.host" . }}
|
||||
- --redis-port={{ include "turnstone.redis.port" . }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.console.service.port }}
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
env:
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
{{- toYaml .Values.console.resources | nindent 12 }}
|
||||
@@ -0,0 +1,64 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-server
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
spec:
|
||||
replicas: {{ .Values.server.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: server
|
||||
spec:
|
||||
serviceAccountName: {{ include "turnstone.serviceAccountName" . }}
|
||||
containers:
|
||||
- name: server
|
||||
image: {{ include "turnstone.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- turnstone-server
|
||||
- --host
|
||||
- "0.0.0.0"
|
||||
- --port
|
||||
- {{ .Values.server.service.port | quote }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.server.service.port }}
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
env:
|
||||
- name: TURNSTONE_DB_URL
|
||||
value: "postgresql+psycopg://$(TURNSTONE_DB_USER):$(POSTGRES_PASSWORD)@$(TURNSTONE_DB_HOST):$(TURNSTONE_DB_PORT)/$(TURNSTONE_DB_NAME)"
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
{{- toYaml .Values.server.resources | nindent 12 }}
|
||||
@@ -0,0 +1,47 @@
|
||||
{{- if .Values.ingress.enabled -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType | default "Prefix" }}
|
||||
backend:
|
||||
service:
|
||||
{{- if eq (.service | default "server") "console" }}
|
||||
name: {{ include "turnstone.fullname" $ }}-console
|
||||
port:
|
||||
number: {{ $.Values.console.service.port }}
|
||||
{{- else }}
|
||||
name: {{ include "turnstone.fullname" $ }}-server
|
||||
port:
|
||||
number: {{ $.Values.server.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-migrate
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: migrate
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade
|
||||
"helm.sh/hook-weight": "-1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
serviceAccountName: {{ include "turnstone.serviceAccountName" . }}
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: migrate
|
||||
image: {{ include "turnstone.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- turnstone.core.storage._migrate
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
env:
|
||||
- name: TURNSTONE_DB_URL
|
||||
value: "postgresql+psycopg://$(TURNSTONE_DB_USER):$(POSTGRES_PASSWORD)@$(TURNSTONE_DB_HOST):$(TURNSTONE_DB_PORT)/$(TURNSTONE_DB_NAME)"
|
||||
@@ -0,0 +1,28 @@
|
||||
{{- if not .Values.llm.existingSecret }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-secrets
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- if .Values.llm.apiKey }}
|
||||
OPENAI_API_KEY: {{ .Values.llm.apiKey | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.postgresql.enabled .Values.postgresql.auth.password }}
|
||||
POSTGRES_PASSWORD: {{ .Values.postgresql.auth.password | b64enc | quote }}
|
||||
{{- else if and (not .Values.postgresql.enabled) .Values.database.external.password }}
|
||||
POSTGRES_PASSWORD: {{ .Values.database.external.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.auth.enabled .Values.auth.token (not .Values.auth.existingSecret) }}
|
||||
TURNSTONE_AUTH_TOKEN: {{ .Values.auth.token | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.redis.enabled .Values.redis.auth }}
|
||||
{{- if .Values.redis.auth.password }}
|
||||
REDIS_PASSWORD: {{ .Values.redis.auth.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- else if and (not .Values.redis.enabled) .Values.redis.external.password }}
|
||||
REDIS_PASSWORD: {{ .Values.redis.external.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-console
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: console
|
||||
spec:
|
||||
type: {{ .Values.console.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.console.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: console
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-server
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
spec:
|
||||
type: {{ .Values.server.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.server.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "turnstone.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
@@ -0,0 +1,103 @@
|
||||
# -- Container image settings
|
||||
image:
|
||||
repository: ghcr.io/turnstonelabs/turnstone
|
||||
tag: ""
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# -- Database configuration
|
||||
database:
|
||||
# Backend type (postgresql)
|
||||
backend: postgresql
|
||||
# External database settings (used when postgresql.enabled is false)
|
||||
external:
|
||||
host: ""
|
||||
port: 5432
|
||||
database: turnstone
|
||||
username: turnstone
|
||||
existingSecret: ""
|
||||
sslmode: prefer
|
||||
|
||||
# -- Bitnami PostgreSQL subchart
|
||||
postgresql:
|
||||
enabled: true
|
||||
auth:
|
||||
database: turnstone
|
||||
username: turnstone
|
||||
|
||||
# -- Redis configuration
|
||||
redis:
|
||||
enabled: true
|
||||
architecture: standalone
|
||||
# External Redis settings (used when redis.enabled is false)
|
||||
external:
|
||||
host: ""
|
||||
port: 6379
|
||||
existingSecret: ""
|
||||
|
||||
# -- Turnstone server (main API + web UI)
|
||||
server:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
# -- Turnstone bridge (Redis MQ connector)
|
||||
bridge:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
|
||||
# -- Turnstone console (cluster dashboard)
|
||||
console:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8090
|
||||
|
||||
# -- LLM provider configuration
|
||||
llm:
|
||||
baseUrl: ""
|
||||
provider: openai
|
||||
apiKey: ""
|
||||
existingSecret: ""
|
||||
|
||||
# -- Authentication
|
||||
auth:
|
||||
enabled: false
|
||||
token: ""
|
||||
existingSecret: ""
|
||||
|
||||
# -- Ingress configuration
|
||||
ingress:
|
||||
enabled: false
|
||||
className: ""
|
||||
annotations: {}
|
||||
hosts: []
|
||||
# - host: turnstone.example.com
|
||||
# paths:
|
||||
# - path: /
|
||||
# pathType: Prefix
|
||||
# service: server
|
||||
tls: []
|
||||
# - secretName: turnstone-tls
|
||||
# hosts:
|
||||
# - turnstone.example.com
|
||||
@@ -0,0 +1,36 @@
|
||||
terraform {
|
||||
required_version = ">= 1.5"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = var.aws_region
|
||||
}
|
||||
|
||||
module "turnstone" {
|
||||
source = "../../modules/aws-ecs"
|
||||
|
||||
vpc_id = var.vpc_id
|
||||
private_subnet_ids = var.private_subnet_ids
|
||||
public_subnet_ids = var.public_subnet_ids
|
||||
|
||||
image_repository = var.image_repository
|
||||
image_tag = var.image_tag
|
||||
|
||||
llm_base_url = var.llm_base_url
|
||||
openai_api_key = var.openai_api_key
|
||||
|
||||
environment = var.environment
|
||||
name_prefix = var.name_prefix
|
||||
auth_token = var.auth_token
|
||||
|
||||
tags = {
|
||||
Example = "aws-ecs-basic"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
output "alb_dns_name" {
|
||||
description = "DNS name of the Application Load Balancer."
|
||||
value = module.turnstone.alb_dns_name
|
||||
}
|
||||
|
||||
output "server_url" {
|
||||
description = "HTTP URL for the Turnstone server."
|
||||
value = module.turnstone.server_url
|
||||
}
|
||||
|
||||
output "console_url" {
|
||||
description = "HTTP URL for the Turnstone console."
|
||||
value = module.turnstone.console_url
|
||||
}
|
||||
|
||||
output "cluster_arn" {
|
||||
description = "ARN of the ECS cluster."
|
||||
value = module.turnstone.cluster_arn
|
||||
}
|
||||
|
||||
output "rds_endpoint" {
|
||||
description = "RDS PostgreSQL endpoint."
|
||||
value = module.turnstone.rds_endpoint
|
||||
}
|
||||
|
||||
output "redis_endpoint" {
|
||||
description = "ElastiCache Redis endpoint."
|
||||
value = module.turnstone.redis_endpoint
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# --- Required ---
|
||||
|
||||
# VPC and subnet IDs from your existing AWS infrastructure.
|
||||
# The VPC must have DNS support and DNS hostnames enabled.
|
||||
vpc_id = "vpc-0123456789abcdef0"
|
||||
private_subnet_ids = ["subnet-aaa111", "subnet-bbb222"]
|
||||
public_subnet_ids = ["subnet-ccc333", "subnet-ddd444"]
|
||||
|
||||
# LLM provider configuration.
|
||||
# For OpenAI: https://api.openai.com/v1
|
||||
# For a self-hosted vLLM instance: http://your-vllm-host:8000/v1
|
||||
llm_base_url = "https://api.openai.com/v1"
|
||||
openai_api_key = "sk-..."
|
||||
|
||||
# --- Optional ---
|
||||
|
||||
# aws_region = "us-east-1"
|
||||
# image_repository = "ghcr.io/turnstonelabs/turnstone"
|
||||
# image_tag = "0.3.0"
|
||||
# environment = "production"
|
||||
# name_prefix = "turnstone"
|
||||
# auth_token = "my-secret-token"
|
||||
@@ -0,0 +1,62 @@
|
||||
variable "aws_region" {
|
||||
description = "AWS region to deploy into."
|
||||
type = string
|
||||
default = "us-east-1"
|
||||
}
|
||||
|
||||
variable "vpc_id" {
|
||||
description = "ID of the VPC where all resources will be created."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "private_subnet_ids" {
|
||||
description = "List of private subnet IDs for ECS tasks, RDS, and ElastiCache."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
variable "public_subnet_ids" {
|
||||
description = "List of public subnet IDs for the Application Load Balancer."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
variable "image_repository" {
|
||||
description = "Container image repository."
|
||||
type = string
|
||||
default = "ghcr.io/turnstonelabs/turnstone"
|
||||
}
|
||||
|
||||
variable "image_tag" {
|
||||
description = "Container image tag."
|
||||
type = string
|
||||
default = "latest"
|
||||
}
|
||||
|
||||
variable "llm_base_url" {
|
||||
description = "Base URL for the LLM provider API."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "openai_api_key" {
|
||||
description = "API key for the LLM provider."
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Deployment environment name."
|
||||
type = string
|
||||
default = "production"
|
||||
}
|
||||
|
||||
variable "name_prefix" {
|
||||
description = "Prefix for all resource names."
|
||||
type = string
|
||||
default = "turnstone"
|
||||
}
|
||||
|
||||
variable "auth_token" {
|
||||
description = "Optional authentication token for the Turnstone API."
|
||||
type = string
|
||||
sensitive = true
|
||||
default = ""
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
# ---------- Application Load Balancer ----------
|
||||
#
|
||||
# HTTP listeners are provided as a starter baseline. For production, set
|
||||
# var.certificate_arn to an ACM certificate ARN to enable HTTPS listeners
|
||||
# that redirect HTTP traffic to TLS.
|
||||
|
||||
resource "aws_lb" "this" {
|
||||
name = "${var.name_prefix}-${var.environment}"
|
||||
internal = false
|
||||
load_balancer_type = "application"
|
||||
security_groups = [aws_security_group.alb.id]
|
||||
subnets = var.public_subnet_ids
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- Server Target Group + Listeners ----------
|
||||
|
||||
resource "aws_lb_target_group" "server" {
|
||||
name = "${var.name_prefix}-server-${var.environment}"
|
||||
port = 8080
|
||||
protocol = "HTTP"
|
||||
vpc_id = var.vpc_id
|
||||
target_type = "ip"
|
||||
tags = local.common_tags
|
||||
|
||||
health_check {
|
||||
path = "/health"
|
||||
port = "traffic-port"
|
||||
protocol = "HTTP"
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 3
|
||||
timeout = 5
|
||||
interval = 30
|
||||
matcher = "200"
|
||||
}
|
||||
}
|
||||
|
||||
# HTTP listener: forwards directly when no certificate, redirects to HTTPS otherwise.
|
||||
resource "aws_lb_listener" "server" {
|
||||
count = var.certificate_arn == "" ? 1 : 0
|
||||
load_balancer_arn = aws_lb.this.arn
|
||||
port = 80
|
||||
protocol = "HTTP"
|
||||
tags = local.common_tags
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
target_group_arn = aws_lb_target_group.server.arn
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_lb_listener" "server_http_redirect" {
|
||||
count = var.certificate_arn != "" ? 1 : 0
|
||||
load_balancer_arn = aws_lb.this.arn
|
||||
port = 80
|
||||
protocol = "HTTP"
|
||||
tags = local.common_tags
|
||||
|
||||
default_action {
|
||||
type = "redirect"
|
||||
redirect {
|
||||
port = "443"
|
||||
protocol = "HTTPS"
|
||||
status_code = "HTTP_301"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_lb_listener" "server_https" {
|
||||
count = var.certificate_arn != "" ? 1 : 0
|
||||
load_balancer_arn = aws_lb.this.arn
|
||||
port = 443
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06"
|
||||
certificate_arn = var.certificate_arn
|
||||
tags = local.common_tags
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
target_group_arn = aws_lb_target_group.server.arn
|
||||
}
|
||||
}
|
||||
|
||||
# ---------- Console Target Group + Listeners ----------
|
||||
|
||||
resource "aws_lb_target_group" "console" {
|
||||
name = "${var.name_prefix}-console-${var.environment}"
|
||||
port = 8090
|
||||
protocol = "HTTP"
|
||||
vpc_id = var.vpc_id
|
||||
target_type = "ip"
|
||||
tags = local.common_tags
|
||||
|
||||
health_check {
|
||||
path = "/health"
|
||||
port = "traffic-port"
|
||||
protocol = "HTTP"
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 3
|
||||
timeout = 5
|
||||
interval = 30
|
||||
matcher = "200"
|
||||
}
|
||||
}
|
||||
|
||||
# HTTP listener: forwards directly when no certificate, redirects to HTTPS otherwise.
|
||||
resource "aws_lb_listener" "console" {
|
||||
count = var.certificate_arn == "" ? 1 : 0
|
||||
load_balancer_arn = aws_lb.this.arn
|
||||
port = 8090
|
||||
protocol = "HTTP"
|
||||
tags = local.common_tags
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
target_group_arn = aws_lb_target_group.console.arn
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_lb_listener" "console_http_redirect" {
|
||||
count = var.certificate_arn != "" ? 1 : 0
|
||||
load_balancer_arn = aws_lb.this.arn
|
||||
port = 8090
|
||||
protocol = "HTTP"
|
||||
tags = local.common_tags
|
||||
|
||||
default_action {
|
||||
type = "redirect"
|
||||
redirect {
|
||||
port = "8443"
|
||||
protocol = "HTTPS"
|
||||
status_code = "HTTP_301"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_lb_listener" "console_https" {
|
||||
count = var.certificate_arn != "" ? 1 : 0
|
||||
load_balancer_arn = aws_lb.this.arn
|
||||
port = 8443
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06"
|
||||
certificate_arn = var.certificate_arn
|
||||
tags = local.common_tags
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
target_group_arn = aws_lb_target_group.console.arn
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# ---------- ElastiCache Subnet Group ----------
|
||||
|
||||
resource "aws_elasticache_subnet_group" "this" {
|
||||
name = "${var.name_prefix}-${var.environment}"
|
||||
subnet_ids = var.private_subnet_ids
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- ElastiCache Redis Replication Group ----------
|
||||
|
||||
resource "aws_elasticache_replication_group" "this" {
|
||||
replication_group_id = "${var.name_prefix}-${var.environment}"
|
||||
description = "Turnstone Redis for MQ and session state"
|
||||
|
||||
engine = "redis"
|
||||
engine_version = "7.1"
|
||||
node_type = var.redis_node_type
|
||||
num_cache_clusters = 1
|
||||
port = 6379
|
||||
|
||||
subnet_group_name = aws_elasticache_subnet_group.this.name
|
||||
security_group_ids = [aws_security_group.redis.id]
|
||||
|
||||
at_rest_encryption_enabled = true
|
||||
transit_encryption_enabled = true
|
||||
|
||||
automatic_failover_enabled = false
|
||||
|
||||
tags = local.common_tags
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
# ---------- ECS Task Execution Role ----------
|
||||
# Used by the ECS agent to pull images and retrieve secrets.
|
||||
|
||||
resource "aws_iam_role" "ecs_execution" {
|
||||
name = "${var.name_prefix}-ecs-execution-${var.environment}"
|
||||
tags = local.common_tags
|
||||
|
||||
assume_role_policy = jsonencode({
|
||||
Version = "2012-10-17"
|
||||
Statement = [
|
||||
{
|
||||
Effect = "Allow"
|
||||
Principal = {
|
||||
Service = "ecs-tasks.amazonaws.com"
|
||||
}
|
||||
Action = "sts:AssumeRole"
|
||||
},
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "ecs_execution_base" {
|
||||
role = aws_iam_role.ecs_execution.name
|
||||
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "ecs_execution_secrets" {
|
||||
name = "${var.name_prefix}-secrets-read-${var.environment}"
|
||||
role = aws_iam_role.ecs_execution.id
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2012-10-17"
|
||||
Statement = [
|
||||
{
|
||||
Effect = "Allow"
|
||||
Action = [
|
||||
"secretsmanager:GetSecretValue",
|
||||
]
|
||||
Resource = concat(
|
||||
[
|
||||
aws_secretsmanager_secret.openai_api_key.arn,
|
||||
aws_secretsmanager_secret.db_password.arn,
|
||||
],
|
||||
var.auth_token != "" ? [aws_secretsmanager_secret.auth_token[0].arn] : [],
|
||||
)
|
||||
},
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
# ---------- ECS Task Role ----------
|
||||
# Assumed by the running container. Minimal permissions; extend as needed.
|
||||
|
||||
resource "aws_iam_role" "ecs_task" {
|
||||
name = "${var.name_prefix}-ecs-task-${var.environment}"
|
||||
tags = local.common_tags
|
||||
|
||||
assume_role_policy = jsonencode({
|
||||
Version = "2012-10-17"
|
||||
Statement = [
|
||||
{
|
||||
Effect = "Allow"
|
||||
Principal = {
|
||||
Service = "ecs-tasks.amazonaws.com"
|
||||
}
|
||||
Action = "sts:AssumeRole"
|
||||
},
|
||||
]
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
terraform {
|
||||
required_version = ">= 1.5"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 5.0"
|
||||
}
|
||||
random = {
|
||||
source = "hashicorp/random"
|
||||
version = ">= 3.5"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
full_image = "${var.image_repository}:${var.image_tag}"
|
||||
|
||||
common_tags = merge(var.tags, {
|
||||
Project = "turnstone"
|
||||
Environment = var.environment
|
||||
ManagedBy = "terraform"
|
||||
})
|
||||
|
||||
# Shared environment variables injected into every container.
|
||||
common_env = [
|
||||
{ name = "TURNSTONE_ENV", value = var.environment },
|
||||
{ name = "TURNSTONE_DB_BACKEND", value = "postgresql" },
|
||||
{ name = "TURNSTONE_LLM_BASE_URL", value = var.llm_base_url },
|
||||
{ name = "TURNSTONE_REDIS_URL", value = "redis://${aws_elasticache_replication_group.this.primary_endpoint_address}:6379/0" },
|
||||
]
|
||||
|
||||
# Secrets pulled from Secrets Manager at container start.
|
||||
common_secrets = [
|
||||
{
|
||||
name = "OPENAI_API_KEY"
|
||||
valueFrom = aws_secretsmanager_secret_version.openai_api_key.arn
|
||||
},
|
||||
{
|
||||
name = "TURNSTONE_DB_URL"
|
||||
valueFrom = aws_secretsmanager_secret_version.db_url.arn
|
||||
},
|
||||
]
|
||||
|
||||
auth_env = var.auth_token != "" ? [
|
||||
{ name = "TURNSTONE_AUTH_ENABLED", value = "true" },
|
||||
] : []
|
||||
|
||||
auth_secrets = var.auth_token != "" ? [
|
||||
{
|
||||
name = "TURNSTONE_AUTH_TOKEN"
|
||||
valueFrom = aws_secretsmanager_secret_version.auth_token[0].arn
|
||||
},
|
||||
] : []
|
||||
}
|
||||
|
||||
# ---------- Secrets Manager ----------
|
||||
|
||||
resource "aws_secretsmanager_secret" "openai_api_key" {
|
||||
name = "${var.name_prefix}-${var.environment}-openai-api-key"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "openai_api_key" {
|
||||
secret_id = aws_secretsmanager_secret.openai_api_key.id
|
||||
secret_string = var.openai_api_key
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "auth_token" {
|
||||
count = var.auth_token != "" ? 1 : 0
|
||||
name = "${var.name_prefix}-${var.environment}-auth-token"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "auth_token" {
|
||||
count = var.auth_token != "" ? 1 : 0
|
||||
secret_id = aws_secretsmanager_secret.auth_token[0].id
|
||||
secret_string = var.auth_token
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "db_password" {
|
||||
name = "${var.name_prefix}-${var.environment}-db-password"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "db_password" {
|
||||
secret_id = aws_secretsmanager_secret.db_password.id
|
||||
secret_string = random_password.db.result
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "db_url" {
|
||||
name = "${var.name_prefix}-${var.environment}-db-url"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "db_url" {
|
||||
secret_id = aws_secretsmanager_secret.db_url.id
|
||||
secret_string = "postgresql+psycopg://${aws_db_instance.this.username}:${random_password.db.result}@${aws_db_instance.this.endpoint}/turnstone"
|
||||
}
|
||||
|
||||
# ---------- ECS Cluster ----------
|
||||
|
||||
resource "aws_ecs_cluster" "this" {
|
||||
name = "${var.name_prefix}-${var.environment}"
|
||||
tags = local.common_tags
|
||||
|
||||
setting {
|
||||
name = "containerInsights"
|
||||
value = "enabled"
|
||||
}
|
||||
}
|
||||
|
||||
# ---------- CloudWatch Log Group ----------
|
||||
|
||||
resource "aws_cloudwatch_log_group" "this" {
|
||||
name = "/ecs/${var.name_prefix}-${var.environment}"
|
||||
retention_in_days = 30
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- Server Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "server" {
|
||||
family = "${var.name_prefix}-server"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = var.server_cpu
|
||||
memory = var.server_memory
|
||||
execution_role_arn = aws_iam_role.ecs_execution.arn
|
||||
task_role_arn = aws_iam_role.ecs_task.arn
|
||||
tags = local.common_tags
|
||||
|
||||
container_definitions = jsonencode([
|
||||
{
|
||||
name = "server"
|
||||
image = local.full_image
|
||||
essential = true
|
||||
command = ["turnstone-server", "--host", "0.0.0.0", "--port", "8080"]
|
||||
|
||||
portMappings = [
|
||||
{ containerPort = 8080, protocol = "tcp" },
|
||||
]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
"awslogs-group" = aws_cloudwatch_log_group.this.name
|
||||
"awslogs-region" = data.aws_region.current.name
|
||||
"awslogs-stream-prefix" = "server"
|
||||
}
|
||||
}
|
||||
|
||||
healthCheck = {
|
||||
command = ["CMD-SHELL", "curl -f http://localhost:8080/health || exit 1"]
|
||||
interval = 30
|
||||
timeout = 5
|
||||
retries = 3
|
||||
startPeriod = 10
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
resource "aws_ecs_service" "server" {
|
||||
name = "${var.name_prefix}-server"
|
||||
cluster = aws_ecs_cluster.this.id
|
||||
task_definition = aws_ecs_task_definition.server.arn
|
||||
desired_count = 1
|
||||
launch_type = "FARGATE"
|
||||
tags = local.common_tags
|
||||
|
||||
network_configuration {
|
||||
subnets = var.private_subnet_ids
|
||||
security_groups = [aws_security_group.ecs_tasks.id]
|
||||
assign_public_ip = false
|
||||
}
|
||||
|
||||
load_balancer {
|
||||
target_group_arn = aws_lb_target_group.server.arn
|
||||
container_name = "server"
|
||||
container_port = 8080
|
||||
}
|
||||
|
||||
depends_on = [aws_lb_target_group.server]
|
||||
}
|
||||
|
||||
# ---------- Bridge Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "bridge" {
|
||||
family = "${var.name_prefix}-bridge"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = var.bridge_cpu
|
||||
memory = var.bridge_memory
|
||||
execution_role_arn = aws_iam_role.ecs_execution.arn
|
||||
task_role_arn = aws_iam_role.ecs_task.arn
|
||||
tags = local.common_tags
|
||||
|
||||
container_definitions = jsonencode([
|
||||
{
|
||||
name = "bridge"
|
||||
image = local.full_image
|
||||
essential = true
|
||||
command = ["turnstone-bridge"]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
"awslogs-group" = aws_cloudwatch_log_group.this.name
|
||||
"awslogs-region" = data.aws_region.current.name
|
||||
"awslogs-stream-prefix" = "bridge"
|
||||
}
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
resource "aws_ecs_service" "bridge" {
|
||||
name = "${var.name_prefix}-bridge"
|
||||
cluster = aws_ecs_cluster.this.id
|
||||
task_definition = aws_ecs_task_definition.bridge.arn
|
||||
desired_count = 1
|
||||
launch_type = "FARGATE"
|
||||
tags = local.common_tags
|
||||
|
||||
network_configuration {
|
||||
subnets = var.private_subnet_ids
|
||||
security_groups = [aws_security_group.ecs_tasks.id]
|
||||
assign_public_ip = false
|
||||
}
|
||||
|
||||
depends_on = [aws_ecs_service.server]
|
||||
}
|
||||
|
||||
# ---------- Console Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "console" {
|
||||
family = "${var.name_prefix}-console"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = var.console_cpu
|
||||
memory = var.console_memory
|
||||
execution_role_arn = aws_iam_role.ecs_execution.arn
|
||||
task_role_arn = aws_iam_role.ecs_task.arn
|
||||
tags = local.common_tags
|
||||
|
||||
container_definitions = jsonencode([
|
||||
{
|
||||
name = "console"
|
||||
image = local.full_image
|
||||
essential = true
|
||||
command = ["turnstone-console", "--host", "0.0.0.0", "--port", "8090"]
|
||||
|
||||
portMappings = [
|
||||
{ containerPort = 8090, protocol = "tcp" },
|
||||
]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
"awslogs-group" = aws_cloudwatch_log_group.this.name
|
||||
"awslogs-region" = data.aws_region.current.name
|
||||
"awslogs-stream-prefix" = "console"
|
||||
}
|
||||
}
|
||||
|
||||
healthCheck = {
|
||||
command = ["CMD-SHELL", "curl -f http://localhost:8090/health || exit 1"]
|
||||
interval = 30
|
||||
timeout = 5
|
||||
retries = 3
|
||||
startPeriod = 10
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
resource "aws_ecs_service" "console" {
|
||||
name = "${var.name_prefix}-console"
|
||||
cluster = aws_ecs_cluster.this.id
|
||||
task_definition = aws_ecs_task_definition.console.arn
|
||||
desired_count = 1
|
||||
launch_type = "FARGATE"
|
||||
tags = local.common_tags
|
||||
|
||||
network_configuration {
|
||||
subnets = var.private_subnet_ids
|
||||
security_groups = [aws_security_group.ecs_tasks.id]
|
||||
assign_public_ip = false
|
||||
}
|
||||
|
||||
load_balancer {
|
||||
target_group_arn = aws_lb_target_group.console.arn
|
||||
container_name = "console"
|
||||
container_port = 8090
|
||||
}
|
||||
|
||||
depends_on = [aws_lb_target_group.console]
|
||||
}
|
||||
|
||||
# ---------- Data Sources ----------
|
||||
|
||||
data "aws_region" "current" {}
|
||||
data "aws_caller_identity" "current" {}
|
||||
@@ -0,0 +1,29 @@
|
||||
output "alb_dns_name" {
|
||||
description = "DNS name of the Application Load Balancer."
|
||||
value = aws_lb.this.dns_name
|
||||
}
|
||||
|
||||
output "server_url" {
|
||||
description = "HTTP URL for the Turnstone server API and web UI."
|
||||
value = "http://${aws_lb.this.dns_name}"
|
||||
}
|
||||
|
||||
output "console_url" {
|
||||
description = "HTTP URL for the Turnstone console dashboard."
|
||||
value = "http://${aws_lb.this.dns_name}:8090"
|
||||
}
|
||||
|
||||
output "cluster_arn" {
|
||||
description = "ARN of the ECS cluster."
|
||||
value = aws_ecs_cluster.this.arn
|
||||
}
|
||||
|
||||
output "rds_endpoint" {
|
||||
description = "Endpoint of the RDS PostgreSQL instance (host:port)."
|
||||
value = aws_db_instance.this.endpoint
|
||||
}
|
||||
|
||||
output "redis_endpoint" {
|
||||
description = "Primary endpoint of the ElastiCache Redis replication group."
|
||||
value = aws_elasticache_replication_group.this.primary_endpoint_address
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# ---------- Random Password ----------
|
||||
|
||||
resource "random_password" "db" {
|
||||
length = 32
|
||||
special = false
|
||||
}
|
||||
|
||||
# ---------- DB Subnet Group ----------
|
||||
|
||||
resource "aws_db_subnet_group" "this" {
|
||||
name = "${var.name_prefix}-${var.environment}"
|
||||
subnet_ids = var.private_subnet_ids
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- RDS PostgreSQL ----------
|
||||
|
||||
resource "aws_db_instance" "this" {
|
||||
identifier = "${var.name_prefix}-${var.environment}"
|
||||
|
||||
engine = "postgres"
|
||||
engine_version = "17"
|
||||
instance_class = var.db_instance_class
|
||||
allocated_storage = 20
|
||||
storage_type = "gp3"
|
||||
storage_encrypted = true
|
||||
deletion_protection = true
|
||||
skip_final_snapshot = false
|
||||
final_snapshot_identifier = "${var.name_prefix}-${var.environment}-final"
|
||||
|
||||
db_name = "turnstone"
|
||||
username = "turnstone"
|
||||
password = random_password.db.result
|
||||
|
||||
db_subnet_group_name = aws_db_subnet_group.this.name
|
||||
vpc_security_group_ids = [aws_security_group.rds.id]
|
||||
|
||||
backup_retention_period = 7
|
||||
multi_az = false
|
||||
|
||||
tags = local.common_tags
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
# ---------- ALB Security Group ----------
|
||||
|
||||
resource "aws_security_group" "alb" {
|
||||
name = "${var.name_prefix}-alb-${var.environment}"
|
||||
description = "Allow inbound HTTP to ALB for server and console"
|
||||
vpc_id = var.vpc_id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "alb_http" {
|
||||
security_group_id = aws_security_group.alb.id
|
||||
description = "HTTP traffic to server"
|
||||
from_port = 80
|
||||
to_port = 80
|
||||
ip_protocol = "tcp"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "alb_https" {
|
||||
count = var.certificate_arn != "" ? 1 : 0
|
||||
security_group_id = aws_security_group.alb.id
|
||||
description = "HTTPS traffic to server"
|
||||
from_port = 443
|
||||
to_port = 443
|
||||
ip_protocol = "tcp"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "alb_console" {
|
||||
security_group_id = aws_security_group.alb.id
|
||||
description = "HTTP traffic to console"
|
||||
from_port = 8090
|
||||
to_port = 8090
|
||||
ip_protocol = "tcp"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "alb_console_https" {
|
||||
count = var.certificate_arn != "" ? 1 : 0
|
||||
security_group_id = aws_security_group.alb.id
|
||||
description = "HTTPS traffic to console"
|
||||
from_port = 8443
|
||||
to_port = 8443
|
||||
ip_protocol = "tcp"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_egress_rule" "alb_all" {
|
||||
security_group_id = aws_security_group.alb.id
|
||||
description = "Allow all outbound"
|
||||
ip_protocol = "-1"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- ECS Tasks Security Group ----------
|
||||
|
||||
resource "aws_security_group" "ecs_tasks" {
|
||||
name = "${var.name_prefix}-ecs-tasks-${var.environment}"
|
||||
description = "Allow traffic from ALB to ECS tasks and outbound internet"
|
||||
vpc_id = var.vpc_id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "ecs_from_alb_server" {
|
||||
security_group_id = aws_security_group.ecs_tasks.id
|
||||
description = "Server port from ALB"
|
||||
from_port = 8080
|
||||
to_port = 8080
|
||||
ip_protocol = "tcp"
|
||||
referenced_security_group_id = aws_security_group.alb.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "ecs_from_alb_console" {
|
||||
security_group_id = aws_security_group.ecs_tasks.id
|
||||
description = "Console port from ALB"
|
||||
from_port = 8090
|
||||
to_port = 8090
|
||||
ip_protocol = "tcp"
|
||||
referenced_security_group_id = aws_security_group.alb.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_egress_rule" "ecs_all" {
|
||||
security_group_id = aws_security_group.ecs_tasks.id
|
||||
description = "Allow all outbound (LLM APIs, ECR, Secrets Manager, etc.)"
|
||||
ip_protocol = "-1"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- RDS Security Group ----------
|
||||
|
||||
resource "aws_security_group" "rds" {
|
||||
name = "${var.name_prefix}-rds-${var.environment}"
|
||||
description = "Allow PostgreSQL access from ECS tasks"
|
||||
vpc_id = var.vpc_id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "rds_from_ecs" {
|
||||
security_group_id = aws_security_group.rds.id
|
||||
description = "PostgreSQL from ECS tasks"
|
||||
from_port = 5432
|
||||
to_port = 5432
|
||||
ip_protocol = "tcp"
|
||||
referenced_security_group_id = aws_security_group.ecs_tasks.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- Redis Security Group ----------
|
||||
|
||||
resource "aws_security_group" "redis" {
|
||||
name = "${var.name_prefix}-redis-${var.environment}"
|
||||
description = "Allow Redis access from ECS tasks"
|
||||
vpc_id = var.vpc_id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "redis_from_ecs" {
|
||||
security_group_id = aws_security_group.redis.id
|
||||
description = "Redis from ECS tasks"
|
||||
from_port = 6379
|
||||
to_port = 6379
|
||||
ip_protocol = "tcp"
|
||||
referenced_security_group_id = aws_security_group.ecs_tasks.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
# --- Networking ---
|
||||
|
||||
variable "vpc_id" {
|
||||
description = "ID of the VPC where all resources will be created."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "private_subnet_ids" {
|
||||
description = "List of private subnet IDs for ECS tasks, RDS, and ElastiCache."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
variable "public_subnet_ids" {
|
||||
description = "List of public subnet IDs for the Application Load Balancer."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
# --- Container Image ---
|
||||
|
||||
variable "image_repository" {
|
||||
description = "Container image repository."
|
||||
type = string
|
||||
default = "ghcr.io/turnstonelabs/turnstone"
|
||||
}
|
||||
|
||||
variable "image_tag" {
|
||||
description = "Container image tag."
|
||||
type = string
|
||||
default = "latest"
|
||||
}
|
||||
|
||||
# --- LLM Provider ---
|
||||
|
||||
variable "llm_base_url" {
|
||||
description = "Base URL for the LLM provider API (e.g. https://api.openai.com/v1)."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "openai_api_key" {
|
||||
description = "API key for the LLM provider. Stored in AWS Secrets Manager."
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
# --- RDS ---
|
||||
|
||||
variable "db_instance_class" {
|
||||
description = "RDS instance class for PostgreSQL."
|
||||
type = string
|
||||
default = "db.t4g.micro"
|
||||
}
|
||||
|
||||
# --- ElastiCache ---
|
||||
|
||||
variable "redis_node_type" {
|
||||
description = "ElastiCache node type for Redis."
|
||||
type = string
|
||||
default = "cache.t4g.micro"
|
||||
}
|
||||
|
||||
# --- ECS Task Sizing ---
|
||||
|
||||
variable "server_cpu" {
|
||||
description = "CPU units for the server task (1 vCPU = 1024)."
|
||||
type = number
|
||||
default = 512
|
||||
}
|
||||
|
||||
variable "server_memory" {
|
||||
description = "Memory (MiB) for the server task."
|
||||
type = number
|
||||
default = 1024
|
||||
}
|
||||
|
||||
variable "bridge_cpu" {
|
||||
description = "CPU units for the bridge task."
|
||||
type = number
|
||||
default = 256
|
||||
}
|
||||
|
||||
variable "bridge_memory" {
|
||||
description = "Memory (MiB) for the bridge task."
|
||||
type = number
|
||||
default = 512
|
||||
}
|
||||
|
||||
variable "console_cpu" {
|
||||
description = "CPU units for the console task."
|
||||
type = number
|
||||
default = 256
|
||||
}
|
||||
|
||||
variable "console_memory" {
|
||||
description = "Memory (MiB) for the console task."
|
||||
type = number
|
||||
default = 512
|
||||
}
|
||||
|
||||
# --- General ---
|
||||
|
||||
variable "environment" {
|
||||
description = "Deployment environment name (e.g. production, staging)."
|
||||
type = string
|
||||
default = "production"
|
||||
}
|
||||
|
||||
variable "name_prefix" {
|
||||
description = "Prefix for all resource names."
|
||||
type = string
|
||||
default = "turnstone"
|
||||
}
|
||||
|
||||
variable "auth_token" {
|
||||
description = "Optional authentication token for the Turnstone API. Empty string disables auth."
|
||||
type = string
|
||||
sensitive = true
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "certificate_arn" {
|
||||
description = "ACM certificate ARN for HTTPS listeners. Leave empty for HTTP-only (not recommended for production)."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Additional tags to apply to all resources."
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Run database migrations before starting the service
|
||||
python -m turnstone.core.storage._migrate || true
|
||||
# Execute the actual command
|
||||
exec "$@"
|
||||
+244
-34
@@ -4,10 +4,10 @@
|
||||
|
||||
> See also: [MQ Protocol diagram](diagrams/png/06-mq-protocol.png) | [Message Routing diagram](diagrams/png/07-message-routing.png) | [Redis Key Schema diagram](diagrams/png/08-redis-key-schema.png)
|
||||
|
||||
`turnstone-server` exposes a browser-based chat UI backed by a Python stdlib HTTP
|
||||
server (`socketserver.ThreadingMixIn` + `http.server.HTTPServer`). The server
|
||||
uses **Server-Sent Events (SSE)** for real-time streaming and **HTTP POST** for
|
||||
user actions.
|
||||
`turnstone-server` exposes a browser-based chat UI backed by a
|
||||
**Starlette** ASGI application served by **uvicorn**. The server uses
|
||||
**Server-Sent Events (SSE)** via `sse-starlette` for real-time streaming
|
||||
and **HTTP POST** for user actions.
|
||||
|
||||
All API responses use `Content-Type: application/json` unless otherwise noted.
|
||||
CORS headers (`Access-Control-Allow-Origin: *`) are included on every response.
|
||||
@@ -17,6 +17,209 @@ an independent `ChatSession` and event queue.
|
||||
|
||||
---
|
||||
|
||||
## API Versioning
|
||||
|
||||
All API endpoints use the `/v1/` prefix. Non-API endpoints (`/`, `/health`, `/metrics`, `/openapi.json`, `/docs`, `/static/*`, `/shared/*`) are unversioned.
|
||||
|
||||
### Interactive Documentation
|
||||
|
||||
- **OpenAPI spec**: `GET /openapi.json` — machine-readable OpenAPI 3.1 schema
|
||||
- **Swagger UI**: `GET /docs` — interactive API explorer (loads from CDN)
|
||||
|
||||
### Client SDKs
|
||||
|
||||
Typed client libraries for programmatic access to both the server and console APIs.
|
||||
|
||||
**Python** (included in the `turnstone` package):
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
ws = client.create_workstream(name="demo")
|
||||
result = client.send_and_wait("Hello!", ws.ws_id)
|
||||
print(result.content)
|
||||
```
|
||||
|
||||
Async variant: `AsyncTurnstoneServer` / `AsyncTurnstoneConsole`.
|
||||
|
||||
**TypeScript** (`sdk/typescript/`):
|
||||
|
||||
```typescript
|
||||
import { TurnstoneServer } from "@turnstone/sdk";
|
||||
|
||||
const client = new TurnstoneServer({ baseUrl: "http://localhost:8080", token: "tok_xxx" });
|
||||
const ws = await client.createWorkstream({ name: "demo" });
|
||||
const result = await client.sendAndWait("Hello!", ws.ws_id);
|
||||
console.log(result.content);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Authentication
|
||||
|
||||
When auth is enabled (`[auth].enabled = true` or `TURNSTONE_AUTH_ENABLED=1`), all API endpoints except public paths require a valid token.
|
||||
|
||||
### Sending Credentials
|
||||
|
||||
Include a token in one of two ways:
|
||||
|
||||
- **Bearer header**: `Authorization: Bearer <token>`
|
||||
- **Cookie**: `turnstone_auth=<token>` (set automatically by the login endpoint)
|
||||
|
||||
The server accepts three token types:
|
||||
|
||||
| Type | Format | Example |
|
||||
|------|--------|---------|
|
||||
| JWT | Base64 segments separated by dots | `eyJhbG...` |
|
||||
| API token | `ts_` prefix + 64 hex chars | `ts_a1b2c3d4...` |
|
||||
| Config token | Arbitrary string from `config.toml` | `my-secret-token` |
|
||||
|
||||
JWTs are the recommended credential for browser sessions. API tokens are suitable for programmatic access and CI/CD. Config tokens are a simple option for single-node deployments.
|
||||
|
||||
### `POST /v1/api/auth/login`
|
||||
|
||||
Authenticate with credentials and receive a JWT. Accepts two credential formats:
|
||||
|
||||
**Username + password:**
|
||||
|
||||
```json
|
||||
{"username": "alice", "password": "hunter2"}
|
||||
```
|
||||
|
||||
**API token:**
|
||||
|
||||
```json
|
||||
{"token": "ts_a1b2c3d4e5f6..."}
|
||||
```
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"role": "full",
|
||||
"scopes": "approve,read,write",
|
||||
"jwt": "eyJhbGciOiJIUzI1NiIs...",
|
||||
"user_id": "u_abc123"
|
||||
}
|
||||
```
|
||||
|
||||
The response also sets a `turnstone_auth` HttpOnly cookie containing the JWT.
|
||||
|
||||
**Response (failure):** `401`
|
||||
|
||||
```json
|
||||
{"error": "Invalid credentials"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/auth/logout`
|
||||
|
||||
Clears the `turnstone_auth` cookie. No request body required.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok"}
|
||||
```
|
||||
|
||||
The response includes a `Set-Cookie` header that expires the auth cookie.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/auth/status`
|
||||
|
||||
Returns the current authentication state. Works with or without a valid token.
|
||||
|
||||
**Response (authenticated):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"authenticated": true,
|
||||
"user_id": "u_abc123",
|
||||
"scopes": ["approve", "read", "write"],
|
||||
"source": "jwt"
|
||||
}
|
||||
```
|
||||
|
||||
**Response (not authenticated):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"authenticated": false,
|
||||
"user_id": null,
|
||||
"scopes": [],
|
||||
"source": null
|
||||
}
|
||||
```
|
||||
|
||||
**Response (auth disabled):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"authenticated": false,
|
||||
"auth_enabled": false
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/auth/setup`
|
||||
|
||||
Creates the first admin user when no users exist in the database. This is a
|
||||
public endpoint (no authentication required) that only succeeds when auth is
|
||||
enabled and the user database is empty. Both the server and console expose
|
||||
this endpoint.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"username": "admin",
|
||||
"display_name": "Admin",
|
||||
"password": "strongpass"
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Validation |
|
||||
|----------------|--------|----------|-----------------------------|
|
||||
| `username` | string | yes | 1-64 ASCII characters |
|
||||
| `display_name` | string | yes | Non-empty |
|
||||
| `password` | string | yes | Minimum 8 characters |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"user_id": "u_abc123",
|
||||
"username": "admin",
|
||||
"role": "full",
|
||||
"scopes": "approve,read,write",
|
||||
"jwt": "eyJhbGciOiJIUzI1NiIs..."
|
||||
}
|
||||
```
|
||||
|
||||
The response also sets a `turnstone_auth` HttpOnly cookie containing the JWT.
|
||||
|
||||
**Response (already set up):** `409`
|
||||
|
||||
```json
|
||||
{"error": "Setup already completed"}
|
||||
```
|
||||
|
||||
Returned when one or more users already exist in the database.
|
||||
|
||||
**Response (auth disabled):** `400`
|
||||
|
||||
```json
|
||||
{"error": "Auth is not enabled"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Endpoints
|
||||
|
||||
### `GET /`
|
||||
@@ -29,7 +232,7 @@ below.
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/events?ws_id=<id>`
|
||||
### `GET /v1/api/events?ws_id=<id>`
|
||||
|
||||
Opens a Server-Sent Events stream scoped to a single workstream. The connection
|
||||
remains open indefinitely; the server pushes events as they occur.
|
||||
@@ -146,7 +349,7 @@ action required).
|
||||
```
|
||||
|
||||
**`approve_request`** -- one or more tool calls that require user approval. The
|
||||
client must respond via `POST /api/approve`.
|
||||
client must respond via `POST /v1/api/approve`.
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -213,7 +416,7 @@ Each item in `items` (shared by `tool_info` and `approve_request`):
|
||||
| `effort` | string | Reasoning effort level (`low`/`medium`/`high`) |
|
||||
|
||||
**`plan_review`** -- the model is proposing a plan and wants feedback. The
|
||||
client must respond via `POST /api/plan`.
|
||||
client must respond via `POST /v1/api/plan`.
|
||||
|
||||
```json
|
||||
{"type": "plan_review", "content": "Step 1: ...\nStep 2: ..."}
|
||||
@@ -267,7 +470,7 @@ connection begins streaming.
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/events/global`
|
||||
### `GET /v1/api/events/global`
|
||||
|
||||
Opens a Server-Sent Events stream that broadcasts state-change events across
|
||||
all workstreams. This is used by the tab bar to display per-workstream activity
|
||||
@@ -299,11 +502,11 @@ Possible `state` values:
|
||||
and copies each event to every client queue. If a client queue is full, the
|
||||
event is silently dropped for that client.
|
||||
|
||||
**Keepalive:** Same as `/api/events` -- an SSE comment every 5 seconds.
|
||||
**Keepalive:** Same as `/v1/api/events` -- an SSE comment every 5 seconds.
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/workstreams`
|
||||
### `GET /v1/api/workstreams`
|
||||
|
||||
Returns a list of all active workstreams.
|
||||
|
||||
@@ -325,11 +528,11 @@ Each workstream object:
|
||||
| `id` | string | Unique workstream routing identifier |
|
||||
| `name` | string | Display name (alias if set, otherwise `ws-xxxx`) |
|
||||
| `state` | string | Current state (see state values above) |
|
||||
| `session_id` | string/null | Session ID of the workstream's `ChatSession`, used for deduplication against `/api/sessions` |
|
||||
| `session_id` | string/null | Session ID of the workstream's `ChatSession`, used for deduplication against `/v1/api/sessions` |
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/sessions`
|
||||
### `GET /v1/api/sessions`
|
||||
|
||||
Returns a list of saved sessions from the database, ordered by most recently
|
||||
updated.
|
||||
@@ -355,16 +558,18 @@ Each session object:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-----------------|-------------|--------------------------------------------|
|
||||
| `session_id` | string | Unique 12-char hex session identifier |
|
||||
| `session_id` | string | Unique 32-char hex UUID session identifier |
|
||||
| `alias` | string/null | User-assigned short name |
|
||||
| `title` | string/null | LLM-generated title |
|
||||
| `created` | string | ISO timestamp of session creation |
|
||||
| `updated` | string | ISO timestamp of last message |
|
||||
| `message_count` | int | Number of messages in the session |
|
||||
| `node_id` | string/null | Server node that created the session |
|
||||
| `ws_id` | string/null | Workstream the session belongs to |
|
||||
|
||||
---
|
||||
|
||||
### `POST /api/send`
|
||||
### `POST /v1/api/send`
|
||||
|
||||
Sends a user message to a workstream. Spawns a daemon worker thread that calls
|
||||
`session.send()` and streams results back via the SSE channel.
|
||||
@@ -402,7 +607,7 @@ from a previous request. Also pushes a `busy_error` event to the SSE stream.
|
||||
|
||||
---
|
||||
|
||||
### `POST /api/approve`
|
||||
### `POST /v1/api/approve`
|
||||
|
||||
Responds to a tool approval request. The SSE stream must have previously sent
|
||||
an `approve_request` event for the given workstream.
|
||||
@@ -434,7 +639,7 @@ automatically approved without prompting.
|
||||
|
||||
---
|
||||
|
||||
### `POST /api/plan`
|
||||
### `POST /v1/api/plan`
|
||||
|
||||
Responds to a plan review dialog. The SSE stream must have previously sent a
|
||||
`plan_review` event for the given workstream.
|
||||
@@ -464,7 +669,7 @@ revision instructions).
|
||||
|
||||
---
|
||||
|
||||
### `POST /api/command`
|
||||
### `POST /v1/api/command`
|
||||
|
||||
Executes a slash command in the given workstream.
|
||||
|
||||
@@ -499,7 +704,7 @@ containing the resumed session's messages.
|
||||
|
||||
---
|
||||
|
||||
### `POST /api/workstreams/new`
|
||||
### `POST /v1/api/workstreams/new`
|
||||
|
||||
Creates a new workstream. The server supports up to 10 concurrent workstreams.
|
||||
|
||||
@@ -511,22 +716,25 @@ Creates a new workstream. The server supports up to 10 concurrent workstreams.
|
||||
|
||||
All fields are optional. The body can be empty or an empty JSON object.
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
|----------------|--------|---------|------------------------------------------------|
|
||||
| `name` | string | auto | Workstream display name |
|
||||
| `model` | string | default | Model alias from the registry (`[models.*]`) |
|
||||
| `auto_approve` | bool | false | Auto-approve all tool calls for this workstream |
|
||||
| Field | Type | Default | Description |
|
||||
|------------------|--------|---------|----------------------------------------------------------------|
|
||||
| `name` | string | auto | Workstream display name |
|
||||
| `model` | string | default | Model alias from the registry (`[models.*]`) |
|
||||
| `auto_approve` | bool | false | Auto-approve all tool calls for this workstream |
|
||||
| `resume_session` | string | "" | Session ID to resume atomically during creation (empty = fresh)|
|
||||
|
||||
**Response (success):**
|
||||
|
||||
```json
|
||||
{"ws_id": "ghi789", "name": "ws-3"}
|
||||
{"ws_id": "ghi789", "name": "ws-3", "resumed": false, "message_count": 0}
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------|--------|------------------------------------|
|
||||
| `ws_id` | string | Unique ID of the new workstream |
|
||||
| `name` | string | Auto-generated workstream name |
|
||||
| Field | Type | Description |
|
||||
|-----------------|--------|-----------------------------------------------------|
|
||||
| `ws_id` | string | Unique ID of the new workstream |
|
||||
| `name` | string | Auto-generated workstream name |
|
||||
| `resumed` | bool | Whether a previous session was successfully resumed |
|
||||
| `message_count` | int | Number of messages in the resumed session (0 if fresh) |
|
||||
|
||||
**Error (limit reached):**
|
||||
|
||||
@@ -538,7 +746,7 @@ Status code: `400`
|
||||
|
||||
---
|
||||
|
||||
### `POST /api/workstreams/close`
|
||||
### `POST /v1/api/workstreams/close`
|
||||
|
||||
Closes and removes a workstream. The last remaining workstream cannot be
|
||||
closed.
|
||||
@@ -592,8 +800,8 @@ Status code: `200` with an empty body.
|
||||
| Malformed or unparseable JSON body | Treated as an empty dict `{}`; missing fields use defaults |
|
||||
| Unknown `ws_id` | `404` with `{"error": "Unknown workstream"}` |
|
||||
| Unknown path (GET or POST) | `404` with plain-text body `Not found` |
|
||||
| Empty `message` on `/api/send` | `400` with `{"error": "Empty message"}` |
|
||||
| Empty `command` on `/api/command` | `400` with `{"error": "Empty command"}` |
|
||||
| Empty `message` on `/v1/api/send` | `400` with `{"error": "Empty message"}` |
|
||||
| Empty `command` on `/v1/api/command` | `400` with `{"error": "Empty command"}` |
|
||||
| Rate limit exceeded | `429` with `Retry-After` header (see below) |
|
||||
|
||||
### `429 Too Many Requests`
|
||||
@@ -635,7 +843,7 @@ reconnection:
|
||||
On reconnect, the server replays the full conversation history via the
|
||||
`history` event, so the client can rebuild its UI state without data loss. The
|
||||
same reconnection strategy applies to both the per-workstream SSE stream
|
||||
(`/api/events`) and the global state stream (`/api/events/global`).
|
||||
(`/v1/api/events`) and the global state stream (`/v1/api/events/global`).
|
||||
|
||||
---
|
||||
|
||||
@@ -653,7 +861,8 @@ liveness probes.
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"version": "0.3.0",
|
||||
"version": "0.4.0",
|
||||
"node_id": "worker-01_a3f2",
|
||||
"uptime_seconds": 3614.72,
|
||||
"model": "llama-3.1-70b-instruct",
|
||||
"workstreams": {
|
||||
@@ -675,6 +884,7 @@ liveness probes.
|
||||
|-------|------|-------------|
|
||||
| `status` | string | `"ok"` or `"degraded"` (degraded when backend unreachable) |
|
||||
| `version` | string | turnstone server version |
|
||||
| `node_id` | string | Server-generated node identity (`{hostname}_{4hex}`) |
|
||||
| `uptime_seconds` | number | Seconds since the server process started |
|
||||
| `model` | string | Model name detected or configured at startup |
|
||||
| `workstreams.total` | integer | Total active workstreams |
|
||||
@@ -743,7 +953,7 @@ turnstone_workstreams_active_total 1
|
||||
# TYPE turnstone_http_requests_total counter
|
||||
turnstone_http_requests_total{method="GET",endpoint="/health",status_code="200"} 42
|
||||
turnstone_http_requests_total{method="GET",endpoint="/metrics",status_code="200"} 7
|
||||
turnstone_http_requests_total{method="POST",endpoint="/api/send",status_code="200"} 18
|
||||
turnstone_http_requests_total{method="POST",endpoint="/v1/api/send",status_code="200"} 18
|
||||
# HELP turnstone_tokens_total Total tokens consumed
|
||||
# TYPE turnstone_tokens_total counter
|
||||
turnstone_tokens_total{type="prompt"} 84320
|
||||
|
||||
+474
-91
@@ -1,9 +1,10 @@
|
||||
# Turnstone Architecture
|
||||
|
||||
Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent
|
||||
memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) and
|
||||
gives the model 14 built-in tools plus external tools via MCP (Model Context
|
||||
Protocol) for reading, writing, searching, planning, and executing code.
|
||||
memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or
|
||||
Anthropic's native Messages API via pluggable provider adapters, and gives the
|
||||
model 14 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
reading, writing, searching, planning, and executing code.
|
||||
|
||||
The core design principle is a **UI-agnostic engine with pluggable frontends**.
|
||||
The engine (`ChatSession`) drives the conversation loop -- streaming, tool
|
||||
@@ -20,6 +21,8 @@ plugs in.
|
||||
| `turnstone-bridge` | `turnstone.mq.bridge` | Bridge | Message queue ↔ HTTP API bridge |
|
||||
| `turnstone-console` | `turnstone.console.server` | ClusterCollector | Cluster dashboard (aggregates all nodes) |
|
||||
| `turnstone-eval` | `turnstone.eval` | `NullUI` | Headless evaluation and prompt optimization |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) via Redis MQ |
|
||||
| `turnstone-admin` | `turnstone.core.admin_cli` | — | Offline user and API token management |
|
||||
|
||||
---
|
||||
|
||||
@@ -32,11 +35,17 @@ turnstone/
|
||||
eval.py Evaluation harness (HeadlessSession, scoring, prompt optimization)
|
||||
core/
|
||||
session.py ChatSession engine, SessionUI protocol, tool dispatch
|
||||
providers/ LLM provider adapters (pluggable backend layer)
|
||||
_protocol.py LLMProvider protocol, ModelCapabilities, StreamChunk, CompletionResult
|
||||
_openai.py OpenAIProvider — OpenAI, vLLM, llama.cpp, any compatible API
|
||||
_anthropic.py AnthropicProvider — Anthropic Messages API, native streaming, thinking
|
||||
__init__.py create_provider() + create_client() factory functions
|
||||
workstream.py Parallel workstream manager (WorkstreamState, Workstream, WorkstreamManager)
|
||||
tools.py Tool schema loader (JSON -> OpenAI function-calling format)
|
||||
mcp_client.py MCPClientManager — MCP server connections, tool discovery, async-sync bridge
|
||||
model_registry.py ModelRegistry — named model configs, lazy client creation, fallback routing
|
||||
memory.py SQLite persistence (conversations, memories, FTS5 search)
|
||||
memory.py Persistence facade (delegates to storage backend)
|
||||
storage/ Pluggable storage: StorageBackend protocol, SQLite + PostgreSQL
|
||||
metrics.py Prometheus-compatible metrics collector (MetricsCollector)
|
||||
healthcheck.py BackendHealthMonitor — periodic probe + circuit breaker
|
||||
ratelimit.py Per-IP token-bucket rate limiter (RateLimiter, TokenBucket)
|
||||
@@ -44,27 +53,52 @@ turnstone/
|
||||
safety.py Command safety validation (blocked patterns, sanitization)
|
||||
sandbox.py Math code sandboxing (AST validation, subprocess execution)
|
||||
web.py Web utilities (HTML stripping, SSRF prevention)
|
||||
api/
|
||||
schemas.py Shared Pydantic v2 models (auth, errors, WorkstreamState)
|
||||
server_schemas.py Server endpoint request/response models
|
||||
console_schemas.py Console endpoint request/response models
|
||||
openapi.py OpenAPI 3.1 spec builder
|
||||
server_spec.py Server endpoint catalog → build_server_spec()
|
||||
console_spec.py Console endpoint catalog → build_console_spec()
|
||||
docs.py /openapi.json + /docs (Swagger UI) handler factories
|
||||
sdk/
|
||||
server.py AsyncTurnstoneServer + TurnstoneServer (HTTP client)
|
||||
console.py AsyncTurnstoneConsole + TurnstoneConsole (HTTP client)
|
||||
events.py 27 SSE event dataclasses with type registry
|
||||
_base.py Shared httpx async client, auth, error handling
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
mq/
|
||||
protocol.py Inbound/outbound message dataclasses (JSON serialization)
|
||||
broker.py Abstract MessageBroker protocol + RedisBroker
|
||||
bridge.py Bridge service (queue ↔ turnstone-server HTTP API)
|
||||
client.py TurnstoneClient library + TurnResult for external systems
|
||||
client.py TurnstoneClient library + TurnResult for MQ-based access
|
||||
console/
|
||||
collector.py ClusterCollector — aggregates state from all nodes via Redis + HTTP
|
||||
scheduler.py TaskScheduler — background cron/at scheduler, dispatches via MQ
|
||||
server.py Cluster dashboard HTTP server + SSE + CLI entry point
|
||||
static/ Cluster dashboard web UI (HTML, CSS, JS)
|
||||
static/ Cluster dashboard web UI (page-specific HTML, CSS, JS)
|
||||
channels/
|
||||
cli.py Unified channel gateway entry point (turnstone-channel)
|
||||
_protocol.py ChannelAdapter protocol, ChannelEvent dataclass
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via MQ
|
||||
_config.py Base ChannelConfig dataclass
|
||||
discord/ Discord adapter (bot, cog, views, streaming, config)
|
||||
shared_static/ Shared design system (base.css, auth.js, theme.js, toast.js, utils.js, kb.js)
|
||||
ui/
|
||||
colors.py ANSI color constants with NO_COLOR support
|
||||
markdown.py Streaming terminal markdown renderer (line-buffered)
|
||||
spinner.py Braille character spinner (daemon thread)
|
||||
static/
|
||||
index.html Single-page app shell (links to CSS and JS)
|
||||
style.css All UI styles (dark/light themes, dashboard, approval blocks)
|
||||
app.js All client-side JavaScript (SSE, workstreams, dashboard, markdown)
|
||||
style.css Page-specific UI styles (dashboard layout, approval blocks)
|
||||
app.js Page-specific client-side JavaScript (SSE, workstreams, markdown)
|
||||
tools/
|
||||
*.json 14 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
```
|
||||
|
||||
Both UIs share a common design system extracted into `turnstone/shared_static/`: design tokens, login overlay, toast notifications, theme toggle, keyboard shortcuts, and utility functions. Each UI imports `base.css` and the shared JS modules at `/shared/`, then adds only page-specific code at `/static/`.
|
||||
|
||||
---
|
||||
|
||||
## Core Loop
|
||||
@@ -86,7 +120,7 @@ A user message flows through the system as follows:
|
||||
_emit_state("thinking")
|
||||
|
|
||||
v
|
||||
_create_stream_with_retry() ----> client.chat.completions.create(stream=True)
|
||||
_create_stream_with_retry() ----> provider.create_streaming(client, model, messages, ...)
|
||||
| up to 3 retries (4 total attempts), exponential backoff
|
||||
v
|
||||
_stream_response(stream) --------> dispatch tokens to UI:
|
||||
@@ -327,11 +361,11 @@ non-idle background workstreams above the input prompt.
|
||||
- **Tab bar**: Each workstream renders as a tab with a colored state indicator
|
||||
(CSS `@keyframes pulse` animation per state).
|
||||
- **Per-tab SSE**: `connectContentSSE(wsId)` opens
|
||||
`/api/events?ws_id=<id>` for the active tab's event stream.
|
||||
- **Global SSE**: `connectGlobalSSE()` opens `/api/events/global` which
|
||||
`/v1/api/events?ws_id=<id>` for the active tab's event stream.
|
||||
- **Global SSE**: `connectGlobalSSE()` opens `/v1/api/events/global` which
|
||||
receives `ws_state` broadcasts from all workstreams, used to update tab
|
||||
indicators without switching.
|
||||
- **New tab / close**: POST `/api/workstreams/new`, POST `/api/workstreams/close`.
|
||||
- **New tab / close**: POST `/v1/api/workstreams/new`, POST `/v1/api/workstreams/close`.
|
||||
|
||||
### Thread Safety
|
||||
|
||||
@@ -405,7 +439,7 @@ from each schema and builds:
|
||||
- `edit_file` -- string replacement in an existing file (requires prior `read_file`)
|
||||
- `math` -- execute Python in sandboxed subprocess (via `turnstone.core.sandbox`)
|
||||
- `web_fetch` -- fetch a URL (with SSRF protection via `turnstone.core.web`)
|
||||
- `web_search` -- search the web via Tavily API
|
||||
- `web_search` -- search the web (provider-native for Anthropic/OpenAI, Tavily fallback for local models)
|
||||
|
||||
**Agent (delegated sub-sessions)**:
|
||||
- `task` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`)
|
||||
@@ -476,6 +510,68 @@ at connection time (server names with `__` are rejected).
|
||||
servers still connect. Tool execution errors return error strings to the LLM
|
||||
rather than crashing the session.
|
||||
|
||||
### Provider Adapter Layer
|
||||
|
||||
> See also: [Core Engine Classes diagram](diagrams/png/03-core-engine-classes.png)
|
||||
|
||||
`ChatSession` is provider-agnostic — it delegates all LLM communication to an
|
||||
`LLMProvider` protocol (`turnstone/core/providers/_protocol.py`). Internally,
|
||||
messages use an OpenAI-like format; each provider translates at the API boundary.
|
||||
|
||||
```
|
||||
ChatSession
|
||||
|
|
||||
v
|
||||
LLMProvider (protocol)
|
||||
|
|
||||
+--- OpenAIProvider --- OpenAI, vLLM, llama.cpp, any /v1/chat/completions API
|
||||
+--- AnthropicProvider --- Anthropic Messages API (native streaming, thinking)
|
||||
```
|
||||
|
||||
**Protocol methods:**
|
||||
|
||||
| Method | Purpose |
|
||||
|--------|---------|
|
||||
| `create_streaming()` | Streaming request, yields normalized `StreamChunk` objects |
|
||||
| `create_completion()` | Non-streaming request, returns `CompletionResult` |
|
||||
| `get_capabilities()` | Per-model flags (`ModelCapabilities`) |
|
||||
| `convert_tools()` | Translate OpenAI tool schemas to provider format |
|
||||
| `retryable_error_names` | Exception class names that trigger retry |
|
||||
|
||||
**Normalized data types:**
|
||||
|
||||
| Type | Fields |
|
||||
|------|--------|
|
||||
| `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason` |
|
||||
| `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage` |
|
||||
| `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search` |
|
||||
| `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens` |
|
||||
|
||||
**OpenAIProvider** (`_openai.py`): passes messages through unchanged (they are
|
||||
already in OpenAI format). Model capability lookup table covers
|
||||
GPT-5/5.1/5.2, O-series, and search models (`gpt-5-search-api`).
|
||||
For search models, injects `web_search_options` and removes the `web_search`
|
||||
function tool (the model always searches). Citations from `url_citation`
|
||||
annotations are formatted as footnotes. Unknown models (local servers) get
|
||||
permissive defaults and use Tavily for web search.
|
||||
|
||||
**AnthropicProvider** (`_anthropic.py`): converts OpenAI-format messages to
|
||||
Anthropic content blocks, maps `system`/`developer` roles to the `system`
|
||||
parameter, groups consecutive `tool` result messages into user-role content
|
||||
blocks, and translates tool schemas from OpenAI function-calling format to
|
||||
Anthropic's `input_schema` format. Supports both manual and adaptive thinking
|
||||
modes, with effort parameter support for models like Claude Opus 4.6 and
|
||||
Sonnet 4.6. Replaces the `web_search` function tool with Anthropic's native
|
||||
`web_search_20250305` server-side tool — Claude decides when to search, the
|
||||
API executes it, and results stream back as `server_tool_use` /
|
||||
`web_search_tool_result` content blocks (emitted as `info_delta` for UI
|
||||
display). The `anthropic` SDK is imported lazily so it remains an optional
|
||||
dependency (`pip install turnstone[anthropic]`).
|
||||
|
||||
**Factory functions** (`__init__.py`): `create_provider(name)` returns a
|
||||
singleton provider instance (thread-safe). `create_client(name, base_url,
|
||||
api_key)` creates the appropriate SDK client.
|
||||
|
||||
### Multi-Model Registry
|
||||
|
||||
`ModelRegistry` (`turnstone/core/model_registry.py`) manages named model
|
||||
@@ -486,34 +582,47 @@ configurations so workstreams can use different LLM backends.
|
||||
[models.local]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
model = "qwen3-32b"
|
||||
# provider defaults to "openai"
|
||||
|
||||
[models.claude]
|
||||
provider = "anthropic"
|
||||
api_key = "sk-ant-..."
|
||||
model = "claude-opus-4-6"
|
||||
context_window = 200000
|
||||
|
||||
[models.openai]
|
||||
base_url = "https://api.openai.com/v1"
|
||||
api_key = "sk-..."
|
||||
model = "gpt-4o"
|
||||
context_window = 128000
|
||||
model = "gpt-5"
|
||||
context_window = 400000
|
||||
|
||||
[model]
|
||||
default = "local"
|
||||
fallback = ["openai"]
|
||||
agent_model = "local"
|
||||
fallback = ["claude", "openai"]
|
||||
agent_model = "claude"
|
||||
```
|
||||
|
||||
Each `[models.*]` entry produces a `ModelConfig` with a `provider` field
|
||||
(default: `"openai"`). Supported values: `"openai"` and `"anthropic"`.
|
||||
|
||||
**Lifecycle:**
|
||||
1. `load_model_registry()` reads `[models.*]` sections from config.toml and
|
||||
builds a `"default"` entry from CLI `--base-url`/`--model`/`--api-key` args
|
||||
2. The registry is passed to the session factory closure in both `cli.py` and
|
||||
`server.py`; each workstream resolves its model on creation
|
||||
3. `ModelRegistry.get_client()` lazily creates `OpenAI` client instances
|
||||
(thread-safe via `_client_lock`)
|
||||
4. `/model` command shows available models; `/model <alias>` switches the
|
||||
3. `ModelRegistry.get_client()` lazily creates SDK client instances via
|
||||
`create_client()` — `OpenAI` for the openai provider, `Anthropic` for
|
||||
the anthropic provider (thread-safe via `_client_lock`)
|
||||
4. `ModelRegistry.get_provider()` lazily creates `LLMProvider` instances via
|
||||
`create_provider()` (also cached and thread-safe)
|
||||
5. `/model` command shows available models; `/model <alias>` switches the
|
||||
active workstream's client, model, and context window
|
||||
5. `_create_stream_with_retry()` tries the primary model, then each fallback
|
||||
6. `_create_stream_with_retry()` tries the primary model, then each fallback
|
||||
alias in order if the primary is unreachable
|
||||
6. `_run_agent()` resolves `registry.agent_model` (if set) for plan/task
|
||||
7. `_run_agent()` resolves `registry.agent_model` (if set) for plan/task
|
||||
sub-agents, allowing a cheaper model for autonomous loops
|
||||
|
||||
**Per-workstream selection:** `POST /api/workstreams/new` accepts an optional
|
||||
**Per-workstream selection:** `POST /v1/api/workstreams/new` accepts an optional
|
||||
`"model"` field. The bridge `CreateWorkstreamMessage` carries the same field
|
||||
through the MQ protocol.
|
||||
|
||||
@@ -538,11 +647,37 @@ This truncation message is visible to the model, so it knows output was cut.
|
||||
|
||||
## Persistence
|
||||
|
||||
### Database
|
||||
### Storage Architecture
|
||||
|
||||
SQLite via `turnstone.core.memory`. Database file: `.turnstone.db` in the
|
||||
current working directory (overridable via `memory.db_override` for eval
|
||||
isolation).
|
||||
Persistence is managed by the `turnstone.core.storage` package — a pluggable
|
||||
backend behind a `StorageBackend` protocol. The `memory.py` facade provides
|
||||
backward-compatible module-level functions that delegate to the active backend.
|
||||
|
||||
```
|
||||
session.py / server.py / cli.py
|
||||
↓
|
||||
memory.py (facade — silent-failure wrappers)
|
||||
↓
|
||||
storage._registry (singleton factory)
|
||||
↓
|
||||
┌─────────────┐ ┌──────────────────┐
|
||||
│ SQLiteBackend │ │ PostgreSQLBackend │
|
||||
│ (FTS5 search) │ │ (tsvector/ILIKE) │
|
||||
└─────────────┘ └──────────────────┘
|
||||
↓ ↓
|
||||
storage._schema (SQLAlchemy Core tables — single source of truth)
|
||||
↓
|
||||
storage._migrate (programmatic Alembic)
|
||||
```
|
||||
|
||||
**SQLite** is the default (zero-config, single file at `.turnstone.db`).
|
||||
**PostgreSQL** is the production backend (connection pooling, `tsvector`
|
||||
full-text search). Select via `[database]` in `config.toml`, CLI flags, or
|
||||
environment variables (`TURNSTONE_DB_BACKEND`, `TURNSTONE_DB_URL`).
|
||||
|
||||
Schema migrations are managed by Alembic and run automatically on startup.
|
||||
Existing SQLite databases created before the migration system are auto-stamped
|
||||
at the baseline revision.
|
||||
|
||||
### Tables
|
||||
|
||||
@@ -569,40 +704,66 @@ conversations
|
||||
tool_name TEXT
|
||||
tool_args TEXT
|
||||
tool_call_id TEXT -- links tool_call ↔ tool_result for resume
|
||||
provider_data TEXT -- raw provider content (e.g. Anthropic encrypted)
|
||||
|
||||
conversations_fts -- FTS5 virtual table
|
||||
session_config
|
||||
session_id TEXT NOT NULL -- composite PK with key
|
||||
key TEXT NOT NULL
|
||||
value TEXT
|
||||
|
||||
conversations_fts -- SQLite FTS5 virtual table (optional)
|
||||
content (content=conversations, content_rowid=id)
|
||||
```
|
||||
|
||||
The `tool_call_id` column was added via schema migration (`ALTER TABLE`) for
|
||||
backwards compatibility with existing databases.
|
||||
Table definitions live in `storage/_schema.py` (SQLAlchemy Core `Table` objects)
|
||||
and are the single source of truth for both backends and Alembic migrations.
|
||||
|
||||
### Key Functions
|
||||
### StorageBackend Protocol
|
||||
|
||||
| Function | Purpose |
|
||||
|----------|---------|
|
||||
| `open_db()` | Open/create database, run migrations, initialize tables |
|
||||
| `load_memories()` | Return all `(key, value)` pairs sorted by key |
|
||||
| `save_message(session_id, role, content, ...)` | Log a message to conversations (accepts `tool_call_id`) |
|
||||
| `search_history(query, limit)` | Full-text search via FTS5 (falls back to LIKE) |
|
||||
| `search_history_recent(limit)` | Return most recent messages |
|
||||
| `register_session(session_id, title)` | Create a sessions row (no-op if exists) |
|
||||
| `update_session_title(session_id, title)` | Set/update LLM-generated title |
|
||||
| Method | Purpose |
|
||||
|--------|---------|
|
||||
| `register_session(session_id, title, node_id, ws_id)` | Create a sessions row (no-op if exists) |
|
||||
| `save_message(session_id, role, content, ...)` | Log a message to conversations |
|
||||
| `load_session_messages(session_id)` | Reconstruct OpenAI message format from DB rows |
|
||||
| `list_sessions(limit)` | List sessions with >=1 message, ordered by updated DESC |
|
||||
| `delete_session(session_id)` | Delete session and all its messages |
|
||||
| `prune_sessions(retention_days)` | Remove empty sessions and old unnamed sessions |
|
||||
| `resolve_session(alias_or_id)` | Resolve alias, exact id, or id prefix to full session_id |
|
||||
| `save_session_config(session_id, config)` | Persist session configuration key/value pairs |
|
||||
| `load_session_config(session_id)` | Retrieve session configuration |
|
||||
| `set_session_alias(session_id, alias)` | Set user-friendly alias (returns False if taken) |
|
||||
| `get_session_name(session_id)` | Return alias if set, else title, else None |
|
||||
| `resolve_session(alias_or_id)` | Resolve alias, exact id, or id prefix to full session_id |
|
||||
| `list_sessions(limit)` | List sessions with ≥1 message, ordered by updated DESC |
|
||||
| `load_session_messages(session_id)` | Reconstruct OpenAI message format from DB rows |
|
||||
| `delete_session(session_id)` | Delete session and all its messages |
|
||||
| `prune_sessions(retention_days, log_fn)` | Remove empty sessions and old unnamed sessions; called at startup |
|
||||
| `normalize_key(key)` | Normalize memory keys (`lower`, replace `-`/` ` with `_`) |
|
||||
| `fts5_query(query)` | Convert plain text to safe FTS5 query (quoted terms) |
|
||||
| `update_session_title(session_id, title)` | Set/update LLM-generated title |
|
||||
| `register_workstream(ws_id, node_id, name, state)` | Create a workstreams row (no-op if exists) |
|
||||
| `update_workstream_state(ws_id, state)` | Update workstream state and bump timestamp |
|
||||
| `update_workstream_name(ws_id, name)` | Update workstream display name |
|
||||
| `delete_workstream(ws_id)` | Delete a workstream row |
|
||||
| `list_workstreams(node_id, limit)` | List workstreams, optionally by node |
|
||||
| `kv_get(key)` / `kv_set(key, value)` / `kv_delete(key)` | Generic key-value store (backs memories table) |
|
||||
| `kv_list()` / `kv_search(query)` | List or search key-value pairs |
|
||||
| `search_history(query, limit)` | Full-text search (FTS5 on SQLite, tsvector on PostgreSQL) |
|
||||
| `search_history_recent(limit)` | Return most recent messages |
|
||||
| `close()` | Release resources (connection pool, engine) |
|
||||
|
||||
### Database Configuration
|
||||
|
||||
```toml
|
||||
[database]
|
||||
backend = "sqlite" # "sqlite" | "postgresql"
|
||||
path = ".turnstone.db" # SQLite file path
|
||||
url = "" # PostgreSQL connection URL
|
||||
pool_size = 5 # PostgreSQL connection pool size
|
||||
```
|
||||
|
||||
Environment variables: `TURNSTONE_DB_BACKEND`, `TURNSTONE_DB_URL`, `TURNSTONE_DB_PATH`.
|
||||
|
||||
### Session Persistence and Resume
|
||||
|
||||
Each `ChatSession` generates a 12-char hex `_session_id` on creation and
|
||||
registers it in the `sessions` table. Messages are saved to `conversations`
|
||||
as they happen via `save_message()`.
|
||||
Each `ChatSession` generates a full 32-char hex UUID `_session_id` on creation
|
||||
and registers it in the `sessions` table with the server's `node_id` and the
|
||||
owning `ws_id`. Messages are saved to `conversations` as they happen via
|
||||
`save_message()`. Workstreams are persisted to the `workstreams` table on
|
||||
creation, with state changes tracked via `update_workstream_state()`.
|
||||
|
||||
**Auto-titling:** After the first complete exchange (user message + assistant
|
||||
response), a background thread calls the LLM with a title-generation prompt
|
||||
@@ -738,7 +899,8 @@ HALF_OPEN ──(probe fails)──────────> OPEN
|
||||
|
||||
- `record_success()` / `record_failure()` update `_consecutive_failures` and
|
||||
transition the `_state` (`CircuitState` enum: `CLOSED`, `OPEN`, `HALF_OPEN`).
|
||||
- `should_allow_request()` returns `False` when the circuit is `OPEN`, causing
|
||||
- `acquire_request_permit()` returns `False` when the circuit is `OPEN` or when
|
||||
in `HALF_OPEN` and the single probe permit has already been consumed. Causes
|
||||
`ChatSession._create_stream_with_retry` to skip the backend and surface an
|
||||
error immediately.
|
||||
- The `/health` endpoint reads the monitor's state: `"status": "ok"` when the
|
||||
@@ -751,8 +913,12 @@ limits using a token-bucket algorithm. Each IP gets a `TokenBucket` with
|
||||
`requests_per_second` (refill rate) and `burst` (bucket capacity) from
|
||||
`[ratelimit]` config.
|
||||
|
||||
- Applied in `do_GET` / `do_POST` after authentication but before route dispatch.
|
||||
- Applied via `RateLimitMiddleware` after authentication but before route dispatch.
|
||||
- `/health` and `/metrics` are exempt (monitoring must always be reachable).
|
||||
- **X-Forwarded-For support**: when `trusted_proxies` is configured (comma-separated
|
||||
CIDRs), the middleware parses the `X-Forwarded-For` header using the
|
||||
rightmost-untrusted approach. IPv4-mapped IPv6 addresses are normalized.
|
||||
The direct client IP must be in the trusted set before XFF is considered.
|
||||
- On limit exceeded: HTTP 429 with `Retry-After` header and JSON body
|
||||
`{"error": "Rate limit exceeded", "retry_after": N}`.
|
||||
- The `turnstone_ratelimit_rejected_total` counter is incremented on each
|
||||
@@ -760,6 +926,98 @@ limits using a token-bucket algorithm. Each IP gets a `TokenBucket` with
|
||||
|
||||
---
|
||||
|
||||
## User Identity and Authentication
|
||||
|
||||
Turnstone supports three authentication mechanisms, unified behind an
|
||||
`AuthResult` dataclass that carries `user_id`, `scopes`, and `token_source`:
|
||||
|
||||
1. **Config-file tokens** — static secrets in `config.toml` `[[auth.tokens]]`
|
||||
or the `TURNSTONE_AUTH_TOKEN` env var. Validated in-memory via
|
||||
`hmac.compare_digest`. Map to scopes through their role (`read` or `full`).
|
||||
2. **API tokens** — database-backed, prefixed `ts_`, stored as SHA-256 hashes
|
||||
in the `api_tokens` table. Can be exchanged for JWTs via
|
||||
`POST /v1/api/auth/login`.
|
||||
3. **JWTs** — short-lived HMAC-SHA256 session tokens (default 24h) issued after
|
||||
successful credential validation. Contain `sub` (user_id), `scopes`, and
|
||||
`src` (origin) in claims.
|
||||
|
||||
### Scope Model
|
||||
|
||||
Three hierarchical scopes control endpoint access:
|
||||
|
||||
| Scope | Grants | Endpoints |
|
||||
|-------|--------|-----------|
|
||||
| `read` | SSE streams, workstream listing, sessions | GET endpoints |
|
||||
| `write` | `read` + send, command, workstream create/close | POST to `/api/send`, `/api/command`, etc. |
|
||||
| `approve` | `write` + tool approval, admin operations | POST to `/api/approve`, `/api/admin/*` |
|
||||
|
||||
### Middleware Flow
|
||||
|
||||
`AuthMiddleware` (ASGI) intercepts every request:
|
||||
|
||||
1. **Public path check** — `/`, `/static/*`, `/shared/*`, `/health`,
|
||||
`/metrics`, `/openapi.json`, `/docs`, `/api/auth/*`, and `/api/auth/setup`
|
||||
are always allowed.
|
||||
2. **Token extraction** — `Authorization: Bearer <token>` header first, then
|
||||
`turnstone_auth` cookie as fallback.
|
||||
3. **Token type detection** — dots in the token indicate JWT; `ts_` prefix
|
||||
indicates API token; otherwise config-file token.
|
||||
4. **Validation** — JWT signature check, API token hash lookup in storage, or
|
||||
config-token hmac comparison.
|
||||
5. **Scope check** — `required_scope(method, path)` determines the minimum
|
||||
scope; the request is rejected with 403 if the token lacks it.
|
||||
6. **Context propagation** — on success, `ctx_user_id` is set so structured
|
||||
logging includes the authenticated identity on every log event.
|
||||
|
||||
### Architecture Split
|
||||
|
||||
- **Console** is the auth management hub — it hosts the admin endpoints for
|
||||
creating users, issuing API tokens, and managing channel mappings. User
|
||||
records and token hashes live in the shared storage backend. The console
|
||||
dashboard includes an **admin panel** (Users and Tokens tabs) for managing
|
||||
credentials through the browser.
|
||||
- **Server** is a JWT validator only — it validates tokens on each request but
|
||||
never creates users or tokens. Both processes share the same `jwt_secret`
|
||||
(via `TURNSTONE_JWT_SECRET` env var or `[auth].jwt_secret` config).
|
||||
- **First-time setup** — both server and console expose
|
||||
`POST /v1/api/auth/setup`, a public endpoint that creates the initial admin
|
||||
user when no users exist. This avoids the chicken-and-egg problem of needing
|
||||
`approve` scope to create the first user via `/api/admin/users`.
|
||||
|
||||
### Auth Storage Tables
|
||||
|
||||
Three tables in `storage/_schema.py` support identity:
|
||||
|
||||
```sql
|
||||
users
|
||||
user_id TEXT PRIMARY KEY
|
||||
username TEXT NOT NULL UNIQUE
|
||||
display_name TEXT NOT NULL
|
||||
password_hash TEXT NOT NULL -- bcrypt
|
||||
created TEXT NOT NULL
|
||||
|
||||
api_tokens
|
||||
token_id TEXT PRIMARY KEY
|
||||
token_hash TEXT NOT NULL UNIQUE -- SHA-256 of raw token
|
||||
token_prefix TEXT NOT NULL -- first 8 chars for display
|
||||
user_id TEXT NOT NULL
|
||||
name TEXT NOT NULL -- human-readable label
|
||||
scopes TEXT NOT NULL -- comma-separated
|
||||
created TEXT NOT NULL
|
||||
expires TEXT -- optional expiry timestamp
|
||||
|
||||
channel_users
|
||||
channel_type TEXT NOT NULL -- e.g. "slack", "discord"
|
||||
channel_user_id TEXT NOT NULL -- platform-specific user ID
|
||||
user_id TEXT NOT NULL -- FK to users
|
||||
PRIMARY KEY (channel_type, channel_user_id)
|
||||
```
|
||||
|
||||
See [docs/security.md](security.md) for full security details including token
|
||||
lifecycle, password hashing, and deployment hardening.
|
||||
|
||||
---
|
||||
|
||||
## Threading Model
|
||||
|
||||
### CLI
|
||||
@@ -785,34 +1043,52 @@ stderr so it does not interfere with readline. Tool execution may use a
|
||||
### Server
|
||||
|
||||
```
|
||||
ThreadedHTTPServer (ThreadingMixIn + HTTPServer, daemon_threads=True)
|
||||
Starlette ASGI app (served by uvicorn)
|
||||
|
|
||||
+-- Thread per HTTP request
|
||||
| POST /api/send -> worker thread per workstream
|
||||
| POST /api/approve -> unblocks WebUI._approval_event
|
||||
| POST /api/plan -> unblocks WebUI._plan_event
|
||||
| POST /api/workstreams/new -> creates workstream + worker
|
||||
| GET /api/events -> SSE long-poll (per workstream)
|
||||
| GET /api/events/global -> SSE long-poll (fan-out)
|
||||
+-- Async request handlers (all under /v1/ prefix)
|
||||
| POST /v1/api/send -> starts worker thread per workstream
|
||||
| POST /v1/api/approve -> unblocks WebUI._approval_event
|
||||
| POST /v1/api/plan -> unblocks WebUI._plan_event
|
||||
| POST /v1/api/workstreams/new -> creates workstream + worker
|
||||
| GET /v1/api/events -> SSE via EventSourceResponse (per workstream)
|
||||
| GET /v1/api/events/global -> SSE via EventSourceResponse (fan-out)
|
||||
|
|
||||
+-- Worker thread per workstream
|
||||
| Runs session.send() in a loop
|
||||
| Blocks on WebUI._approval_event / _plan_event
|
||||
+-- ASGI middleware stack
|
||||
| MetricsMiddleware -> CORSMiddleware -> AuthMiddleware -> RateLimitMiddleware
|
||||
|
|
||||
+-- Global SSE fan-out
|
||||
WebUI._global_queue shared across all WebUI instances
|
||||
Global SSE endpoint drains this queue
|
||||
+-- Worker thread per workstream (daemon)
|
||||
| Runs session.send() synchronously -- ChatSession is fully blocking
|
||||
| Blocks on WebUI._approval_event / _plan_event (threading.Event)
|
||||
|
|
||||
+-- Background daemon threads
|
||||
Global SSE fan-out: reads global_queue, copies to per-client queues
|
||||
Idle cleanup: closes stale workstreams, cleans rate limiter buckets
|
||||
```
|
||||
|
||||
`ThreadingMixIn` ensures each HTTP request (including long-lived SSE
|
||||
connections) gets its own thread. This is necessary because SSE connections
|
||||
block indefinitely, and POST requests must be handled concurrently.
|
||||
Starlette handles all HTTP routing, CORS, and middleware. uvicorn runs
|
||||
the ASGI application with async request handling. All API endpoints live
|
||||
under the `/v1/` prefix via a Starlette `Mount`. An OpenAPI 3.1 spec is
|
||||
generated from Pydantic v2 models and served at `/openapi.json`; Swagger
|
||||
UI is available at `/docs`. SSE endpoints use `EventSourceResponse` from
|
||||
`sse-starlette` with async generators that bridge sync `queue.Queue` via
|
||||
`asyncio.get_running_loop().run_in_executor()`.
|
||||
|
||||
`ChatSession.send()` remains synchronous, running in daemon worker threads.
|
||||
WebUI keeps `threading.Event` and `queue.Queue` primitives (unchanged from
|
||||
the sync era). The `_global_fanout_thread` and `_idle_cleanup_thread` remain
|
||||
as daemon threads since they interact with sync primitives. A lifespan
|
||||
context manager handles startup/shutdown (health monitor, MCP client,
|
||||
registry).
|
||||
|
||||
Each workstream's `WebUI` has:
|
||||
- `_event_queue` (per-workstream SSE events)
|
||||
- `_event_queue` (per-workstream SSE events, `queue.Queue`)
|
||||
- `_approval_event` / `_plan_event` (`threading.Event` for blocking)
|
||||
- `_global_queue` (class variable, shared, for state broadcasts)
|
||||
|
||||
The SSE handlers bridge these sync queues to async via
|
||||
`run_in_executor()`, polling `queue.Queue.get(timeout=1)` while
|
||||
`sse-starlette` handles keepalive pings automatically.
|
||||
|
||||
### Workstream Threading (CLI)
|
||||
|
||||
```
|
||||
@@ -843,7 +1119,8 @@ bell + status line to stderr to alert the user.
|
||||
Main thread Global SSE thread Per-WS SSE threads (×N)
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| Inbound loop | | GET /events/glob | | GET /events?ws_id |
|
||||
| BLPOP on Redis | | Parse SSE data | | Parse SSE data |
|
||||
| BLPOP on Redis | | Parse SSE via | | Parse SSE via |
|
||||
| | | httpx-sse | | httpx-sse |
|
||||
| Dispatch to | | Forward state | | Forward content, |
|
||||
| handler | | changes | | tool results |
|
||||
| POST to server | | Detect turn | | Handle approval |
|
||||
@@ -859,16 +1136,19 @@ Main thread Global SSE thread Per-WS SSE threads (×N)
|
||||
|
||||
**Approval flow:** When a per-WS SSE thread receives an `approve_request`, it checks
|
||||
the workstream's `auto_approve_tools` set. If all requested tools are in the set, the
|
||||
bridge auto-approves via `POST /api/approve`. Otherwise, it publishes an
|
||||
bridge auto-approves via `POST /v1/api/approve`. Otherwise, it publishes an
|
||||
`ApprovalRequestEvent` to the outbound channel with a `request_id`, then blocks on
|
||||
`BLPOP` of a Redis response queue (`turnstone:resp:{request_id}`) until the client pushes
|
||||
a response or the approval timeout (default 300s) expires.
|
||||
a response or the approval timeout (default 3600s / 1 hour) expires.
|
||||
|
||||
**Completion detection:** The bridge tracks which `correlation_id` maps to which
|
||||
`ws_id` for active sends. When the global SSE reports `ws_state → idle` for a tracked
|
||||
workstream, the bridge emits a synthetic `TurnCompleteEvent` with the correlation ID.
|
||||
|
||||
**Multi-node routing:** Each bridge has a `node_id` (defaults to hostname) and BLPOPs
|
||||
**Multi-node routing:** Each bridge retrieves its `node_id` from the server's
|
||||
`/health` endpoint on startup (with exponential backoff retry). The server
|
||||
generates the `node_id` (`{hostname}_{4hex}`) and is the sole authority for
|
||||
node identity. The bridge BLPOPs
|
||||
from both `turnstone:inbound:{node_id}` (directed, priority) and `turnstone:inbound` (shared).
|
||||
Messages with `target_node` set are pushed to the target's per-node queue. Messages
|
||||
for existing workstreams are auto-routed via `turnstone:ws:{ws_id}` ownership keys in Redis.
|
||||
@@ -879,25 +1159,51 @@ re-routes to that node's queue (1 extra hop). Bridges publish heartbeats to
|
||||
### Cluster Console
|
||||
|
||||
```
|
||||
Event subscriber Node discovery Poll loop
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| SUBSCRIBE on | | SCAN node:* keys | | For each node: |
|
||||
| events:cluster | | every 15 seconds | | GET /api/dash |
|
||||
| Apply state | | Add/remove nodes | | GET /health |
|
||||
| changes to | | Emit join/lost | | ThreadPoolExecutor|
|
||||
| in-memory model | | events | | (50 workers) |
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| | |
|
||||
+-- Redis pub/sub +-- Redis SCAN +-- HTTP to each
|
||||
(SUBSCRIBE) (every 15s) server (every 10s)
|
||||
Monitoring (3 daemon threads) Control + Proxy (async Starlette)
|
||||
+------------------+ +----------------------------+
|
||||
| Event subscriber | | POST /v1/api/cluster/ |
|
||||
| SUBSCRIBE on | | workstreams/new |
|
||||
| events:cluster | | → LPUSH to Redis |
|
||||
+------------------+ | inbound:{node_id} |
|
||||
| Node discovery | +----------------------------+
|
||||
| SCAN node:* keys | | GET /node/{node_id}/ |
|
||||
| every 15 seconds | | → httpx.AsyncClient |
|
||||
+------------------+ | proxy to server_url |
|
||||
| Poll loop | | GET /node/{id}/v1/api/events |
|
||||
| GET /v1/api/dash | | → SSE stream proxy |
|
||||
| GET /health | | POST /node/{id}/v1/api/send |
|
||||
| ThreadPoolExec | | → forwarded to server |
|
||||
+------------------+ +----------------------------+
|
||||
```
|
||||
|
||||
The console is read-only — it never writes to Redis queues or sends commands to servers.
|
||||
Real-time events provide instant state transitions; periodic polling provides full data
|
||||
consistency (tokens, context ratios, activity strings). Clicking a workstream row in the
|
||||
console opens the node's server UI with `?ws_id=<id>` for direct deep linking — the
|
||||
server parses this on load and auto-selects the workstream. See [docs/console.md](console.md)
|
||||
for the full API reference.
|
||||
The console HTTP layer is a Starlette/ASGI app served by uvicorn. The SSE
|
||||
endpoint uses `EventSourceResponse` with the same listener queue pattern as
|
||||
the main server. `ClusterCollector`'s background threads (event subscriber,
|
||||
node discovery, poll loop) use sync Redis clients and `ThreadPoolExecutor`
|
||||
for parallel HTTP polling.
|
||||
|
||||
The console has two write-path capabilities:
|
||||
|
||||
1. **Workstream creation** — pushes `CreateWorkstreamMessage` to Redis inbound
|
||||
queues targeting specific nodes. The bridge on each node picks up the message
|
||||
and creates the workstream on the local server. Auto-selects the node with
|
||||
the most available capacity if no target is specified.
|
||||
|
||||
2. **Reverse proxy** — serves each node's server UI through the console port at
|
||||
`/node/{node_id}/`. Uses `httpx.AsyncClient` to proxy HTTP and SSE traffic.
|
||||
A JS shim is injected into the server's `app.js` to override `fetch()` and
|
||||
`EventSource()`, routing root-relative URLs through the proxy prefix. This
|
||||
eliminates the need for direct network access to individual server nodes.
|
||||
|
||||
The console also performs **version drift detection** — flagging when nodes
|
||||
report different versions via the `/health` endpoint. The overview API includes
|
||||
`version_drift` and `versions` fields; the dashboard shows a yellow warning
|
||||
indicator when versions diverge.
|
||||
|
||||
Clicking a workstream row in the console opens the proxied server UI at
|
||||
`/node/{node_id}/?ws_id=<id>` — the server's JS parses this on load and
|
||||
auto-selects the workstream. See [docs/console.md](console.md) for the full
|
||||
API reference.
|
||||
|
||||
---
|
||||
|
||||
@@ -919,3 +1225,80 @@ preserves:
|
||||
|
||||
After compaction, `_read_files` is cleared to force re-reads before edits,
|
||||
since file contents are no longer in the message history.
|
||||
|
||||
---
|
||||
|
||||
## Client SDK
|
||||
|
||||
> See also: [SDK Architecture diagram](diagrams/png/13-sdk-architecture.png) | [SDK Documentation](sdk.md)
|
||||
|
||||
The `turnstone/sdk/` package provides typed HTTP clients for programmatic access
|
||||
to both the server and console APIs. It wraps REST endpoints with methods that
|
||||
return Pydantic models, and SSE endpoints with async/sync iterators that yield
|
||||
typed event dataclasses.
|
||||
|
||||
**Two client pairs** (sync + async):
|
||||
|
||||
- `TurnstoneServer` / `AsyncTurnstoneServer` — server API (workstreams, chat, streaming, sessions)
|
||||
- `TurnstoneConsole` / `AsyncTurnstoneConsole` — console API (cluster overview, nodes, workstreams)
|
||||
|
||||
**Design**: async-first with thin sync wrappers. `_BaseClient` provides httpx
|
||||
setup, auth headers, `_request()` (REST) and `_stream_sse()` (SSE). Sync
|
||||
clients delegate through `_SyncRunner` which maintains a persistent background
|
||||
event loop on a daemon thread.
|
||||
|
||||
**Event types**: 27 standalone dataclasses in `events.py` with a type-registry
|
||||
pattern matching `OutboundEvent.from_json()` from `mq/protocol.py`. Events are
|
||||
decoupled from the MQ package so SDK consumers don't need the `redis` dependency.
|
||||
|
||||
**TypeScript SDK**: `sdk/typescript/` — separate npm package with the same API
|
||||
surface. Zero browser dependencies, SSE via `fetch` + `ReadableStream` parsing.
|
||||
|
||||
```python
|
||||
# Python quick start
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
ws = client.create_workstream(name="demo")
|
||||
result = client.send_and_wait("Hello!", ws.ws_id)
|
||||
print(result.content)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Channel Integrations
|
||||
|
||||
> See also: [Channel Integrations guide](channels.md)
|
||||
|
||||
The `turnstone-channel` gateway bridges external messaging platforms
|
||||
(Discord, Slack, Teams) to the turnstone cluster via Redis MQ. Each
|
||||
platform adapter implements the `ChannelAdapter` protocol and translates
|
||||
between platform-native events and turnstone MQ messages.
|
||||
|
||||
The `ChannelRouter` manages bidirectional routing: it maps platform
|
||||
channel/thread IDs to turnstone workstream IDs, handles workstream
|
||||
creation and stale-route recovery, and resolves platform users to
|
||||
turnstone identities via the `channel_users` table. When an evicted
|
||||
workstream is reactivated, the router uses atomic session resume via the
|
||||
`resume_session` field on `CreateWorkstreamMessage` — the server resumes
|
||||
the old session during workstream creation in a single HTTP request,
|
||||
eliminating ordering fragility. The bridge emits a `SessionResumedEvent`
|
||||
to confirm success.
|
||||
|
||||
Discord ships as the first adapter. See [channels.md](channels.md) for
|
||||
setup instructions, configuration reference, and the adapter development
|
||||
guide.
|
||||
|
||||
### Notification Subsystem
|
||||
|
||||
The `notify` tool enables the LLM to send notifications to users or
|
||||
channels without going through MQ. The server calls the channel gateway
|
||||
directly over HTTP for lower latency: `_exec_notify()` queries the
|
||||
`services` database table for healthy channel gateways (heartbeat within
|
||||
120 seconds), authenticates with a service JWT (`aud: turnstone-channel`),
|
||||
and POSTs to `POST /v1/api/notify` on the first healthy gateway. The
|
||||
gateway validates the JWT, resolves the target (username lookup via
|
||||
`channel_users` or direct `channel_type`+`channel_id`), and delegates to
|
||||
the appropriate `ChannelAdapter.send()`. Delivery retries up to 3 times
|
||||
with backoff, re-querying the service registry on each attempt. See
|
||||
[Notification Flow diagram](diagrams/png/17-notify-flow.png).
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
# Channel Integrations
|
||||
|
||||
The `turnstone-channel` gateway connects external messaging platforms to
|
||||
turnstone workstreams via Redis MQ. Each platform adapter translates
|
||||
platform-native events (messages, button clicks, slash commands) into
|
||||
turnstone MQ messages, and renders workstream output back into the
|
||||
platform's UI.
|
||||
|
||||
Discord ships as the first adapter. The adapter protocol is designed for
|
||||
future Slack and Teams integrations.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Discord Gateway
|
||||
|
|
||||
v
|
||||
turnstone-channel (Discord adapter)
|
||||
|
|
||||
v
|
||||
Redis MQ
|
||||
|
|
||||
v
|
||||
turnstone-bridge ──> turnstone-server
|
||||
```
|
||||
|
||||
Key components:
|
||||
|
||||
- **ChannelAdapter protocol** (`turnstone/channels/_protocol.py`) — generic
|
||||
interface for any messaging platform. Defines `start()`, `stop()`,
|
||||
`send()`, `edit_message()`, `send_approval_request()`,
|
||||
`send_plan_review()`, and `create_thread()`.
|
||||
- **ChannelRouter** (`turnstone/channels/_routing.py`) — maps
|
||||
channel/thread IDs to turnstone workstream IDs. Handles workstream
|
||||
creation via MQ, stale route detection, and user identity resolution.
|
||||
- **AsyncRedisBroker** (`turnstone/mq/async_broker.py`) — async Redis
|
||||
client compatible with discord.py's event loop. Used by the router for
|
||||
pub/sub and queue operations.
|
||||
- **channel_users table** — maps `(channel_type, channel_user_id)` to a
|
||||
turnstone `user_id`. Messages from unlinked users are silently dropped.
|
||||
- **channel_routes table** — persistent channel-to-workstream mappings.
|
||||
Survives bot restarts. Stale routes (evicted workstreams) are detected
|
||||
and refreshed on the next message.
|
||||
|
||||
---
|
||||
|
||||
## Discord Setup
|
||||
|
||||
### 1. Create a Discord Application
|
||||
|
||||
1. Go to https://discord.com/developers/applications
|
||||
2. Click **New Application** and give it a name
|
||||
3. Navigate to the **Bot** tab and click **Reset Token** to generate a
|
||||
bot token. Copy it immediately — it is shown only once.
|
||||
4. On the same **Bot** tab, scroll down to **Privileged Gateway Intents**
|
||||
and enable **MESSAGE CONTENT INTENT**
|
||||
5. Navigate to **OAuth2 > URL Generator**
|
||||
6. Under **Scopes**, check `bot` and `applications.commands`
|
||||
7. Under **Bot Permissions**, check:
|
||||
- View Channels
|
||||
- Send Messages
|
||||
- Send Messages in Threads
|
||||
- Create Public Threads
|
||||
- Read Message History
|
||||
- Add Reactions
|
||||
- Embed Links
|
||||
8. Copy the generated URL, open it in a browser, and add the bot to your
|
||||
Discord server
|
||||
|
||||
### 2. Configure Turnstone
|
||||
|
||||
**Environment variables** (recommended for Docker):
|
||||
|
||||
```bash
|
||||
TURNSTONE_DISCORD_TOKEN=your-bot-token-here
|
||||
TURNSTONE_DISCORD_GUILD=123456789 # optional, restrict to one guild
|
||||
```
|
||||
|
||||
**CLI flags** (bare-metal):
|
||||
|
||||
```bash
|
||||
turnstone-channel \
|
||||
--discord-token "your-bot-token" \
|
||||
--discord-guild 123456789 \
|
||||
--redis-host localhost \
|
||||
--redis-port 6379
|
||||
```
|
||||
|
||||
**Docker Compose** (production profile):
|
||||
|
||||
```bash
|
||||
# In .env file:
|
||||
TURNSTONE_DISCORD_TOKEN=your-bot-token
|
||||
TURNSTONE_DISCORD_GUILD=123456789
|
||||
```
|
||||
|
||||
Then start the stack:
|
||||
|
||||
```bash
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
The `channel` service starts automatically when
|
||||
`TURNSTONE_DISCORD_TOKEN` is set.
|
||||
|
||||
### 3. Link User Accounts
|
||||
|
||||
Discord users must link their account to a turnstone user before they can
|
||||
interact with the bot. Unlinked users' messages are silently ignored.
|
||||
|
||||
1. The user must have a turnstone API token — created via the admin panel
|
||||
or `turnstone-admin create-token`
|
||||
2. In Discord, the user runs `/link`. A modal appears prompting for the
|
||||
API token (the token is never visible in Discord audit logs because it
|
||||
is submitted via modal, not as a slash command argument).
|
||||
3. The token is validated against the database. If valid, a
|
||||
`channel_users` mapping is created.
|
||||
4. The user can now @mention the bot or use slash commands.
|
||||
|
||||
An admin can also force-link or unlink users via the console admin panel
|
||||
(Admin > Channels tab).
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
### Conversations
|
||||
|
||||
- **@mention** the bot in any allowed channel to start a new conversation.
|
||||
The bot creates a Discord thread from the message and a turnstone
|
||||
workstream behind it.
|
||||
- All subsequent messages in the thread are routed to the same workstream.
|
||||
- The bot streams responses via message edits, updated approximately every
|
||||
1.5 seconds.
|
||||
- If the workstream is evicted for capacity, the next message in the
|
||||
thread auto-creates a new workstream and atomically resumes the
|
||||
previous session via the `resume_session` field on
|
||||
`CreateWorkstreamMessage`. The server resumes the session during
|
||||
workstream creation (same HTTP request), and the bridge emits a
|
||||
`SessionResumedEvent` back to the channel. The thread receives a
|
||||
*"Session resumed: {name} ({count} messages restored)"* confirmation.
|
||||
|
||||
### Slash Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `/link` | Link Discord account to turnstone (opens modal for API token) |
|
||||
| `/unlink` | Unlink Discord account |
|
||||
| `/ask <message>` | Create a new thread and workstream with an initial message |
|
||||
| `/status` | Show workstream info for the current thread (ephemeral) |
|
||||
| `/close` | Close the workstream, delete the route, and archive the thread |
|
||||
|
||||
### Tool Approvals
|
||||
|
||||
When manual approval is enabled (the default), tool calls are displayed as
|
||||
an orange embed with:
|
||||
|
||||
- Tool name and argument preview
|
||||
- **Approve** (green), **Reject** (red), **Always Approve** (gray) buttons
|
||||
- Only linked users can interact with approval buttons
|
||||
- The approval decision is forwarded through MQ to the bridge, which
|
||||
relays it to the server
|
||||
|
||||
Buttons use static `custom_id` values so they survive bot restarts.
|
||||
Correlation data (`ws_id`, `correlation_id`) is stored in the embed footer.
|
||||
|
||||
**Auto-approval:** When `auto_approve` is true (via `--auto-approve`), or when
|
||||
all tools in the request match the `auto_approve_tools` list in the adapter
|
||||
config, the bot auto-responds with approval and posts a
|
||||
"*Tool auto-approved.*" notice to the thread instead of showing buttons. The
|
||||
`auto_approve_tools` list is set via the `ChannelConfig.auto_approve_tools`
|
||||
field (useful for allowing specific tools like `bash` or `read_file` while
|
||||
still requiring manual approval for others).
|
||||
|
||||
### Plan Reviews
|
||||
|
||||
Plan review requests are displayed as a blue embed with:
|
||||
|
||||
- **Approve Plan** (green) button — approves the plan with empty feedback
|
||||
- **Request Changes** (gray) button — opens a modal for feedback text
|
||||
(up to 2000 characters)
|
||||
- Feedback is forwarded through MQ as a `PlanFeedbackMessage`
|
||||
|
||||
---
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
| CLI Flag | Env Var | Default | Description |
|
||||
|----------|---------|---------|-------------|
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Bot token (required to enable Discord) |
|
||||
| `--discord-guild` | — | `0` (all guilds) | Restrict to a single Discord guild |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated channel IDs to allow |
|
||||
| `--redis-host` | `REDIS_HOST` | `localhost` | Redis host |
|
||||
| `--redis-port` | — | `6379` | Redis port |
|
||||
| `--redis-password` | `REDIS_PASSWORD` | — | Redis password |
|
||||
| `--redis-db` | — | `0` | Redis DB number |
|
||||
| `--model` | — | server default | Default model for new workstreams |
|
||||
| `--auto-approve` | — | `false` | Auto-approve ALL tool calls (skips approval buttons entirely) |
|
||||
| `--http-host` | — | `127.0.0.1` | HTTP server bind address for notify endpoint |
|
||||
| `--http-port` | `TURNSTONE_CHANNEL_PORT` | `8091` | HTTP server port |
|
||||
| `--auth-token` | `TURNSTONE_CHANNEL_AUTH_TOKEN` | — | Static auth token for `/v1/api/notify` (alternative to JWT) |
|
||||
| `--log-level` | `TURNSTONE_LOG_LEVEL` | `INFO` | Log level |
|
||||
| `--log-format` | `TURNSTONE_LOG_FORMAT` | `auto` | Log format (`auto`/`json`/`text`) |
|
||||
|
||||
---
|
||||
|
||||
## User Identity
|
||||
|
||||
- The `channel_users` table maps `(channel_type, channel_user_id)` to a
|
||||
turnstone `user_id`
|
||||
- Self-service linking via the `/link` slash command (modal input, not
|
||||
visible in Discord audit logs)
|
||||
- Admin can force-link or unlink via the console admin panel (Admin >
|
||||
Channels tab). Unlinking uses a styled confirmation modal.
|
||||
- Unlinked users' messages are silently dropped
|
||||
- A user can be linked across multiple platforms (e.g. Discord + Slack)
|
||||
|
||||
See [Security: Database Schema](security.md#database-schema) for the
|
||||
`channel_users` table definition.
|
||||
|
||||
---
|
||||
|
||||
## Workstream Lifecycle
|
||||
|
||||
1. **Creation** — @mention or `/ask` creates a Discord thread and a
|
||||
turnstone workstream. The `ChannelRouter` persists the mapping in the
|
||||
`channel_routes` table.
|
||||
2. **Active** — messages are routed bidirectionally. The bot streams
|
||||
responses via message edits (updated every ~1.5 seconds).
|
||||
3. **Eviction** — the server evicts an idle workstream for capacity. The
|
||||
route is preserved and the thread stays open.
|
||||
4. **Reactivation** — the next message in the thread detects the stale
|
||||
route (no MQ owner), looks up the old session via
|
||||
`get_session_id_by_ws()`, and creates a new workstream with
|
||||
`resume_session` set atomically on the `CreateWorkstreamMessage`. The
|
||||
server resumes the session during creation (no separate command
|
||||
needed). The bridge emits a `SessionResumedEvent` to the channel, and
|
||||
the thread displays *"Session resumed: {name} ({count} messages
|
||||
restored)"*. If the old session was pruned, the workstream starts
|
||||
fresh with no error.
|
||||
5. **Close** — `/close` command closes the workstream via MQ, deletes the
|
||||
route, unsubscribes from events, and archives the Discord thread.
|
||||
|
||||
---
|
||||
|
||||
## Notifications
|
||||
|
||||
> See also: [Notification Flow diagram](diagrams/png/17-notify-flow.png)
|
||||
|
||||
The `notify` tool allows the LLM to proactively send notifications to
|
||||
users or channels on external platforms. This is useful for alerting
|
||||
people about task completion, errors, or important updates without
|
||||
waiting for them to check in.
|
||||
|
||||
### Targeting
|
||||
|
||||
Two modes:
|
||||
|
||||
- **Username** — provide a turnstone `username`. The gateway resolves
|
||||
it via the `channel_users` table and sends to all linked channels
|
||||
(e.g. Discord + future Slack).
|
||||
- **Direct** — provide `channel_type` + `channel_id` to target a
|
||||
specific platform channel or user DM.
|
||||
|
||||
### Delivery Flow
|
||||
|
||||
Notifications bypass MQ for lower latency. The server calls the channel
|
||||
gateway directly over HTTP:
|
||||
|
||||
1. The LLM calls the `notify` tool with a message and target
|
||||
2. `_exec_notify()` queries the `services` table for healthy channel
|
||||
gateways (heartbeat within the last 120 seconds)
|
||||
3. The server mints a service JWT (`aud: turnstone-channel`) via
|
||||
`ServiceTokenManager` and POSTs to the first healthy gateway
|
||||
4. The gateway validates the JWT, resolves the target, and calls
|
||||
`adapter.send()` on the appropriate platform adapter
|
||||
5. On failure, the server tries the next gateway. If all fail, it
|
||||
retries up to 2 more times (delays: 1s, 3s), re-querying the
|
||||
service registry on each attempt
|
||||
|
||||
### Service Registry
|
||||
|
||||
The channel gateway registers itself in the `services` database table
|
||||
on startup and sends a heartbeat every 30 seconds. On shutdown it
|
||||
deregisters. Services are considered stale after 120 seconds (4 missed
|
||||
heartbeats) and are excluded from `list_services()` queries.
|
||||
|
||||
The `services` table schema:
|
||||
|
||||
| Column | Description |
|
||||
|--------|-------------|
|
||||
| `service_type` | Service category (e.g. `"channel"`) |
|
||||
| `service_id` | Unique instance ID (`channel-<hostname>-<random>`) |
|
||||
| `url` | HTTP base URL for the service |
|
||||
| `last_heartbeat` | ISO 8601 timestamp of last heartbeat |
|
||||
| `created` | ISO 8601 timestamp of initial registration |
|
||||
|
||||
### Security
|
||||
|
||||
- **Authentication** — the gateway's `POST /v1/api/notify` endpoint
|
||||
requires authentication. Configure either `TURNSTONE_JWT_SECRET`
|
||||
(the server mints JWTs with `aud: turnstone-channel` automatically)
|
||||
or a static token via `--auth-token`. If neither is set, the
|
||||
gateway fails closed and rejects all requests with 401. Server JWTs
|
||||
(`aud: turnstone-server`) are rejected.
|
||||
- **Rate limit** — maximum 5 notifications per turn. The counter only
|
||||
increments on successful delivery, so failures don't consume the
|
||||
budget.
|
||||
- **SSRF protection** — only `http://` and `https://` service URLs
|
||||
are allowed. Other schemes are silently skipped.
|
||||
- **Mention sanitization** — `discord.utils.escape_mentions()` is
|
||||
applied before sending, preventing `@everyone` / `@here` abuse.
|
||||
- **Error redaction** — generic error messages are returned to the
|
||||
LLM. Internal details (service IDs, URLs, exception messages) are
|
||||
logged server-side only.
|
||||
|
||||
---
|
||||
|
||||
## Adding New Adapters
|
||||
|
||||
The `ChannelAdapter` protocol defines the interface any platform adapter
|
||||
must implement:
|
||||
|
||||
```python
|
||||
class ChannelAdapter(Protocol):
|
||||
channel_type: str
|
||||
|
||||
async def start(self) -> None: ...
|
||||
async def stop(self) -> None: ...
|
||||
async def send(self, channel_id: str, content: str) -> str: ...
|
||||
async def edit_message(self, channel_id: str, message_id: str, content: str) -> None: ...
|
||||
async def send_approval_request(self, channel_id: str, ws_id: str, correlation_id: str, items: list[dict]) -> None: ...
|
||||
async def send_plan_review(self, channel_id: str, ws_id: str, correlation_id: str, content: str) -> None: ...
|
||||
async def create_thread(self, parent_channel_id: str, name: str, message_id: str = "") -> str: ...
|
||||
```
|
||||
|
||||
To add a new platform:
|
||||
|
||||
1. Create `turnstone/channels/<platform>/` package
|
||||
2. Implement the `ChannelAdapter` protocol
|
||||
3. Add a `--<platform>-token` flag and detection logic in
|
||||
`turnstone/channels/cli.py`
|
||||
4. Add the optional dependency in `pyproject.toml` (e.g.
|
||||
`turnstone[slack]`)
|
||||
|
||||
See `turnstone/channels/discord/` as a reference implementation.
|
||||
+420
-27
@@ -1,28 +1,40 @@
|
||||
# Cluster Dashboard (turnstone-console)
|
||||
|
||||
`turnstone-console` is a standalone monitoring service that provides cluster-wide visibility across all turnstone nodes. It connects to the shared Redis broker, discovers nodes via heartbeat keys, polls each node's HTTP API for workstream data, and subscribes to a cluster event channel for real-time state changes.
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It connects to the shared Redis broker, discovers nodes via heartbeat keys, polls each node's HTTP API for workstream data, and subscribes to a cluster event channel for real-time state changes.
|
||||
|
||||
The console is read-only — it observes but does not own workstreams or drive LLM sessions.
|
||||
The console also supports **workstream creation** (dispatched via MQ to target nodes) and a **reverse proxy** that serves each node's server UI through the console port — so users only need network access to the console, not to individual server nodes.
|
||||
|
||||
## Architecture
|
||||
|
||||
> See also: [Console Data Flow diagram](diagrams/png/11-console-data-flow.png)
|
||||
|
||||
```
|
||||
turnstone-server ──→ turnstone-bridge ──→ Redis ──→ turnstone-console ──→ Browser
|
||||
(per node) (per node) (shared) (one instance)
|
||||
┌── Redis ←── turnstone-bridge ←── turnstone-server
|
||||
│ (MQ) (per node) (per node)
|
||||
turnstone-console ──────┤
|
||||
(one instance) │
|
||||
└── turnstone-server (direct HTTP proxy)
|
||||
│
|
||||
▼
|
||||
Browser
|
||||
```
|
||||
|
||||
Each bridge publishes state changes to `{prefix}:events:cluster` on Redis pub/sub. The console subscribes once to that channel for real-time updates and periodically polls each node's `GET /api/dashboard` for full workstream snapshots.
|
||||
Data flows in two directions:
|
||||
|
||||
- **Inbound (monitoring):** Bridges publish state changes to `{prefix}:events:cluster` on Redis pub/sub. The console subscribes for real-time updates and periodically polls each node's `GET /v1/api/dashboard` for full workstream snapshots.
|
||||
- **Outbound (control):** The console pushes `CreateWorkstreamMessage` to Redis inbound queues targeting specific nodes. Bridges pick up these messages and create workstreams on their local servers.
|
||||
- **Proxy (pass-through):** The console reverse-proxies each node's server UI at `/node/{node_id}/`, forwarding HTTP and SSE traffic so the browser never contacts server nodes directly.
|
||||
|
||||
### Data Sources
|
||||
|
||||
| Source | Method | Frequency | Data |
|
||||
| Source | Method | Direction | Data |
|
||||
|--------|--------|-----------|------|
|
||||
| Redis heartbeats | `SCAN turnstone:node:*` | Every 15s | Node discovery (node_id, server_url, started) |
|
||||
| Redis pub/sub | `SUBSCRIBE turnstone:events:cluster` | Real-time | State changes, creates, closes, renames |
|
||||
| Node HTTP API | `GET {server_url}/api/dashboard` | Every 10s | Full workstream list with tokens, context, activity |
|
||||
| Node HTTP API | `GET {server_url}/health` | Every 10s | Node health status |
|
||||
| Redis heartbeats | `SCAN turnstone:node:*` | Read | Node discovery (node_id, server_url, started) |
|
||||
| Redis pub/sub | `SUBSCRIBE turnstone:events:cluster` | Read | State changes, creates, closes, renames |
|
||||
| Node HTTP API | `GET {server_url}/v1/api/dashboard` | Read | Full workstream list with tokens, context, activity |
|
||||
| Node HTTP API | `GET {server_url}/health` | Read | Node health status |
|
||||
| Redis inbound queue | `RPUSH turnstone:inbound:{node_id}` | Write | Workstream creation commands |
|
||||
| Node HTTP API | `GET/POST {server_url}/*` | Proxy | Server UI, API requests, SSE streams |
|
||||
|
||||
### Redis Key: Cluster Event Channel
|
||||
|
||||
@@ -47,7 +59,7 @@ The collector (`turnstone/console/collector.py`) maintains an in-memory snapshot
|
||||
|
||||
2. **Node discovery** — scans heartbeat keys every 15 seconds via `broker.list_nodes()`. Adds newly discovered nodes, removes expired ones, emits `node_joined` / `node_lost` events to SSE listeners.
|
||||
|
||||
3. **Poll loop** — fetches `GET /api/dashboard` and `GET /health` from each known node every 10 seconds. Uses `ThreadPoolExecutor(max_workers=50)` for parallelism. Each poll replaces the node's workstream list with the authoritative server data.
|
||||
3. **Poll loop** — fetches `GET /v1/api/dashboard` and `GET /health` from each known node every 10 seconds. Uses `ThreadPoolExecutor(max_workers=50)` for parallelism. Each poll replaces the node's workstream list with the authoritative server data.
|
||||
|
||||
### Thread Safety
|
||||
|
||||
@@ -64,7 +76,7 @@ All reads and writes to the node/workstream map are protected by a single `threa
|
||||
|
||||
## HTTP API
|
||||
|
||||
### `GET /api/cluster/overview`
|
||||
### `GET /v1/api/cluster/overview`
|
||||
|
||||
Cluster-wide state counts and aggregate metrics.
|
||||
|
||||
@@ -73,11 +85,15 @@ Cluster-wide state counts and aggregate metrics.
|
||||
"nodes": 847,
|
||||
"workstreams": 4219,
|
||||
"states": {"running": 1847, "thinking": 312, "attention": 89, "idle": 1940, "error": 31},
|
||||
"aggregate": {"total_tokens": 12400000, "total_tool_calls": 34200}
|
||||
"aggregate": {"total_tokens": 12400000, "total_tool_calls": 34200},
|
||||
"version_drift": true,
|
||||
"versions": ["0.3.0", "0.3.1"]
|
||||
}
|
||||
```
|
||||
|
||||
### `GET /api/cluster/nodes?sort=activity&limit=100&offset=0`
|
||||
`version_drift` is `true` when nodes report different versions. `versions` lists all unique version strings sorted alphabetically.
|
||||
|
||||
### `GET /v1/api/cluster/nodes?sort=activity&limit=100&offset=0`
|
||||
|
||||
Paginated node list. Sort options: `activity` (default, by running+attention count), `tokens`, `name`.
|
||||
|
||||
@@ -91,14 +107,15 @@ Paginated node list. Sort options: `activity` (default, by running+attention cou
|
||||
"total_tokens": 48200,
|
||||
"started": 1709294400.0,
|
||||
"reachable": true,
|
||||
"health": {}
|
||||
"health": {"status": "ok", "version": "0.3.0"},
|
||||
"version": "0.3.0"
|
||||
}
|
||||
],
|
||||
"total": 847
|
||||
}
|
||||
```
|
||||
|
||||
### `GET /api/cluster/workstreams?state=running&node=db-west-04&search=perf&page=1&per_page=50`
|
||||
### `GET /v1/api/cluster/workstreams?state=running&node=db-west-04&search=perf&page=1&per_page=50`
|
||||
|
||||
Filtered, paginated workstream list. All query parameters are optional. `per_page` is capped at 200.
|
||||
|
||||
@@ -115,7 +132,7 @@ Filtered, paginated workstream list. All query parameters are optional. `per_pag
|
||||
}
|
||||
```
|
||||
|
||||
### `GET /api/cluster/node/{node_id}`
|
||||
### `GET /v1/api/cluster/node/{node_id}`
|
||||
|
||||
Single node detail with all its workstreams.
|
||||
|
||||
@@ -129,7 +146,41 @@ Single node detail with all its workstreams.
|
||||
}
|
||||
```
|
||||
|
||||
### `GET /api/cluster/events`
|
||||
### `POST /v1/api/cluster/workstreams/new`
|
||||
|
||||
Create a new workstream on a target node. Dispatches a `CreateWorkstreamMessage` through the Redis MQ pipeline — the bridge on the target node picks it up and creates the workstream on the server. Requires `write` scope.
|
||||
|
||||
Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"node_id": "db-west-04",
|
||||
"name": "perf-analysis",
|
||||
"model": "gpt-5"
|
||||
}
|
||||
```
|
||||
|
||||
All fields are optional:
|
||||
- `node_id` — targeting mode:
|
||||
- **omitted or `"auto"`** — console picks the reachable node with the most available capacity (max_ws - ws_total) and pushes to its directed queue.
|
||||
- **`"pool"`** — pushes to the shared inbound queue; the next available bridge picks it up (true general-pool dispatch).
|
||||
- **specific node ID** — pushes to that node's directed queue.
|
||||
- `name` — workstream display name. Auto-generated if omitted.
|
||||
- `model` — model alias from the target node's registry. Uses the node's default model if omitted.
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"correlation_id": "a1b2c3d4e5f6",
|
||||
"target_node": "db-west-04"
|
||||
}
|
||||
```
|
||||
|
||||
Creation is asynchronous — the response confirms the MQ message was dispatched. A `ws_created` event on the cluster SSE stream confirms the workstream was actually created.
|
||||
|
||||
### `GET /v1/api/cluster/events`
|
||||
|
||||
Server-Sent Events stream for real-time cluster updates.
|
||||
|
||||
@@ -146,32 +197,373 @@ Keepalive comments (`: keepalive\n\n`) are sent every 5 seconds. Clients should
|
||||
### `GET /health`
|
||||
|
||||
```json
|
||||
{"status": "ok", "service": "turnstone-console", "nodes": 847, "workstreams": 4219}
|
||||
{
|
||||
"status": "ok",
|
||||
"service": "turnstone-console",
|
||||
"nodes": 847,
|
||||
"workstreams": 4219,
|
||||
"version_drift": false,
|
||||
"versions": ["0.3.0"]
|
||||
}
|
||||
```
|
||||
|
||||
### Admin API
|
||||
|
||||
User and token management endpoints. All admin endpoints require `approve` scope, except for the setup endpoint which is public.
|
||||
|
||||
#### `POST /v1/api/auth/setup`
|
||||
|
||||
Creates the first admin user when no users exist. Public endpoint (no auth required). Returns a JWT and sets a session cookie. Returns `409` if users already exist. See [Security: First-time setup](security.md#first-time-setup) for full details.
|
||||
|
||||
#### `POST /v1/api/admin/users`
|
||||
|
||||
Create a new user.
|
||||
|
||||
```json
|
||||
{
|
||||
"username": "alice",
|
||||
"password": "s3cret",
|
||||
"scopes": ["read", "write"]
|
||||
}
|
||||
```
|
||||
|
||||
#### `GET /v1/api/admin/users`
|
||||
|
||||
List all users.
|
||||
|
||||
```json
|
||||
{
|
||||
"users": [
|
||||
{"user_id": "u_abc123", "username": "alice", "scopes": ["read", "write"], "created": "2026-03-01T12:00:00Z"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### `DELETE /v1/api/admin/users/{user_id}`
|
||||
|
||||
Delete a user and revoke all their tokens.
|
||||
|
||||
#### `POST /v1/api/admin/users/{user_id}/tokens`
|
||||
|
||||
Create an API token for the given user. Returns a `ts_`-prefixed token string that can be used for Bearer auth or passed to `client.login(token="ts_xxx")`.
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "CI pipeline",
|
||||
"scopes": ["read", "write"]
|
||||
}
|
||||
```
|
||||
|
||||
#### `GET /v1/api/admin/users/{user_id}/tokens`
|
||||
|
||||
List active tokens for a user (token strings are not returned, only metadata).
|
||||
|
||||
#### `DELETE /v1/api/admin/tokens/{token_id}`
|
||||
|
||||
Revoke a specific API token.
|
||||
|
||||
### Channel links
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/v1/api/admin/users/{user_id}/channels` | List channel links for a user |
|
||||
| POST | `/v1/api/admin/users/{user_id}/channels` | Link a channel account (channel_type, channel_user_id) |
|
||||
| DELETE | `/v1/api/admin/channels/{channel_type}/{channel_user_id}` | Unlink a channel account |
|
||||
|
||||
These endpoints manage the `channel_users` table mappings that connect external platform identities (e.g. Discord user IDs) to turnstone users. See [Channel Integrations](channels.md) for details on the linking flow.
|
||||
|
||||
#### `GET /v1/api/auth/status`
|
||||
|
||||
Public endpoint for login UI state detection. Returns auth configuration, not
|
||||
current-user identity.
|
||||
|
||||
```json
|
||||
{
|
||||
"auth_enabled": true,
|
||||
"has_users": true,
|
||||
"setup_required": false
|
||||
}
|
||||
```
|
||||
|
||||
### Auth Scopes
|
||||
|
||||
The auth system uses three scopes instead of the earlier read/full role model:
|
||||
|
||||
| Scope | Grants |
|
||||
|-------|--------|
|
||||
| `read` | Read-only access: dashboards, workstream lists, SSE streams, health |
|
||||
| `write` | Send messages, create/close workstreams, approve tool calls |
|
||||
| `approve` | Admin operations: manage users and API tokens |
|
||||
|
||||
Scopes are cumulative — a user with `approve` scope can also perform `write` and `read` operations.
|
||||
|
||||
---
|
||||
|
||||
## Reverse Proxy
|
||||
|
||||
The console reverse-proxies each node's server UI at `/node/{node_id}/`. This allows users to interact with any node's workstreams through the console port alone — individual server ports do not need to be exposed to the office network.
|
||||
|
||||
### Proxy Routes
|
||||
|
||||
| Route | Behavior |
|
||||
|-------|----------|
|
||||
| `GET /node/{node_id}/` | Fetches the server's `index.html`, rewrites static and shared asset paths, injects a console-return banner and an inline JS proxy shim |
|
||||
| `GET /node/{node_id}/static/{path}` | Proxies page-specific static files |
|
||||
| `GET /node/{node_id}/shared/{path}` | Proxies shared static files (`base.css`, `auth.js`, etc.) |
|
||||
| `GET /node/{node_id}/v1/api/{path}` | Proxies GET API requests; detects SSE endpoints and streams them |
|
||||
| `POST /node/{node_id}/v1/api/{path}` | Proxies POST API requests with body forwarding |
|
||||
| `GET /node/{node_id}/{path}` | Proxies non-API endpoints (health, metrics) |
|
||||
|
||||
### URL Rewriting
|
||||
|
||||
The server UI uses root-relative URLs (`/v1/api/send`, `/static/app.js`, `/shared/base.css`, etc.). Since `<base>` tags cannot rewrite root-relative URLs, the console uses a JS shim approach:
|
||||
|
||||
1. **HTML rewriting** — when serving `index.html`, replaces `href=` and `src=` references to both `/static/` and `/shared/` with the proxy prefix (`/node/{node_id}/static/` and `/node/{node_id}/shared/` respectively).
|
||||
|
||||
2. **Inline JS shim** — injects an inline `<script>` block into the proxied HTML (after the console-return banner, before any external scripts) that overrides `window.fetch()` and `window.EventSource()` to prepend the proxy prefix to any root-relative URL. Running the shim inline ensures it executes before any external scripts load, so all API calls and SSE connections are intercepted transparently.
|
||||
|
||||
3. **Console-return banner** — injects a thin inline-styled `<div>` after `<body>` with a "← Console" link and the node ID, providing navigation back to the dashboard.
|
||||
|
||||
### SSE Proxy
|
||||
|
||||
SSE streams (`/v1/api/events`, `/v1/api/events/global`) are proxied by creating a per-connection `httpx.AsyncClient(timeout=None)`, streaming the upstream response via `aiter_text()`, parsing SSE framing (`\n\n` delimiters), and re-emitting events through `EventSourceResponse`. Each proxied SSE stream requires its own httpx client since the shared client's 30-second timeout would kill long-lived connections.
|
||||
|
||||
### Authentication
|
||||
|
||||
The proxy forwards the user's JWT to upstream server nodes — it extracts the token from the incoming request's cookie (or `Authorization` header) and adds it as a `Bearer` header on the proxied request. Since all services share the same `TURNSTONE_JWT_SECRET`, the user's JWT is valid on every node without re-authentication. The console's own auth middleware also checks proxy routes — `POST` requests to proxy write endpoints (`/v1/api/send`, `/v1/api/approve`, etc.) require `write` scope, preventing read-only tokens from escalating via proxy. The static `--auth-token` / `proxy_auth_token` is used as a fallback when no user JWT is present.
|
||||
|
||||
---
|
||||
|
||||
## Browser Dashboard
|
||||
|
||||
The web UI has three views, toggled client-side:
|
||||
The web UI has five views, toggled client-side:
|
||||
|
||||
### 1. Cluster Overview (landing)
|
||||
|
||||
- **State cards** — 5 clickable cards (running, thinking, attention, idle, error) with count and colored top border. Clicking filters to that state.
|
||||
- **Aggregate bar** — total tokens and tool calls across the cluster.
|
||||
- **Node table** — columns: NODE, WS, RUN, ATTN, TOKENS, HEALTH. Sorted by activity. Clickable rows drill down to node detail.
|
||||
- **Node table** — columns: NODE, WS, RUN, ATTN, TOKENS, VER, LOAD. Sorted by activity. Clickable rows drill down to node detail. Version column shows per-node version; hidden on mobile.
|
||||
- **Version drift indicator** — when nodes report different versions, the status bar shows a yellow "DRIFT" warning with a tooltip listing all versions. Node groups show "mixed" with a yellow badge when their members disagree.
|
||||
- **"+ new" button** — opens the workstream creation modal (see below).
|
||||
|
||||
### 2. Node Drill-down
|
||||
|
||||
Breadcrumb: `Cluster > db-west-04`. Shows the node's workstreams in a table matching the per-node dashboard layout (STATE, NAME, NODE, TASK, TOKENS, CTX) with activity sub-lines. Includes a link to the node's own dashboard (`http://{server_url}/`).
|
||||
Breadcrumb: `Cluster > db-west-04`. Shows the node's workstreams in a table matching the per-node dashboard layout (STATE, NAME, MODEL, NODE, TASK, TOKENS, CTX) with activity sub-lines. Includes a link to the node's proxied server UI.
|
||||
|
||||
**Deep linking:** Clicking a workstream row opens the node's server UI in a new tab with `?ws_id=<id>`, which auto-selects that workstream. A `↗` indicator appears on hover to signal the external navigation. Rows without a `server_url` are non-interactive.
|
||||
**Proxy deep-linking:** Clicking a workstream row opens the node's server UI in a new tab via the proxy at `/node/{node_id}/?ws_id=<id>`, which auto-selects that workstream. Users do not need direct network access to the server node.
|
||||
|
||||
### 3. Filtered Workstreams
|
||||
|
||||
Breadcrumb: `Cluster > Running` or `Cluster > db-west-04`. Server-side paginated workstream table. NODE column values are clickable to filter further. Pagination controls at bottom. Workstream rows are deep-linkable when `server_url` is available (injected by the collector from the parent node).
|
||||
Breadcrumb: `Cluster > Running` or `Cluster > db-west-04`. Server-side paginated workstream table. NODE column values are clickable to filter further. Pagination controls at bottom. Workstream rows use proxy deep-links.
|
||||
|
||||
All three views receive live updates via SSE — state cards update counts, node rows update metrics, workstream rows update state indicators.
|
||||
### 4. Workstream Creation Modal
|
||||
|
||||
Triggered by the "+ new" header button. A modal dialog with:
|
||||
|
||||
- **Node selector** — dropdown with three targeting modes: "Auto (best available)" picks the node with the most headroom, "General pool (any node)" pushes to the shared queue for any bridge to pick up, or a specific node from the list (showing capacity).
|
||||
- **Name** — optional text input. Auto-generated if left empty.
|
||||
- **Model** — optional text input for a model alias from the target node's registry.
|
||||
|
||||
On submit, `POST /v1/api/cluster/workstreams/new` dispatches the creation request. A toast confirms success; the SSE stream delivers the `ws_created` event to update the dashboard.
|
||||
|
||||
All five views receive live updates via SSE — state cards update counts, node rows update metrics, workstream rows update state indicators.
|
||||
|
||||
### 5. Admin Panel
|
||||
|
||||
Accessed via the "admin" button in the header (visible when authenticated
|
||||
with `approve` scope). Provides user, API token, and channel link management
|
||||
with three tabs:
|
||||
|
||||
**Users tab:**
|
||||
|
||||
- Grid table listing all users (username, display name, role, creation date)
|
||||
- "Create User" button opens a modal with fields for username, display name,
|
||||
and password (validated: username 1-64 ASCII, password min 8 characters)
|
||||
- Delete button on each row opens a styled confirmation modal before
|
||||
removing the user and cascading to revoke all their tokens
|
||||
|
||||
**Tokens tab:**
|
||||
|
||||
- User selector dropdown to pick which user's tokens to manage
|
||||
- Grid table listing tokens for the selected user (name, prefix, scopes,
|
||||
creation date)
|
||||
- Scope badges rendered as colored pills for visual clarity
|
||||
- "Create Token" button opens a modal with fields for token name and scope
|
||||
checkboxes
|
||||
- On creation, a "Token Created" modal displays the raw `ts_`-prefixed
|
||||
token with a copy button. The token is shown once and cannot be retrieved
|
||||
again.
|
||||
- Revoke button on each row opens a styled confirmation modal before
|
||||
deleting the token
|
||||
|
||||
**Channels tab:**
|
||||
|
||||
- User selector dropdown to pick which user's channel links to manage
|
||||
- Grid table listing linked channel accounts for the selected user
|
||||
(channel type, channel user ID, creation date)
|
||||
- "Link Channel" button opens a modal with fields for channel type
|
||||
(e.g. `discord`) and the platform user ID
|
||||
- Unlink button on each row opens a styled confirmation modal before
|
||||
removing the channel mapping
|
||||
- Admins can force-link users who have not self-linked via `/link` in
|
||||
Discord
|
||||
|
||||
**Accessibility:**
|
||||
|
||||
- Full keyboard navigation: focus traps in modals, Escape to close, arrow
|
||||
keys for tab switching
|
||||
- Responsive layout with column hiding at 700px breakpoint
|
||||
|
||||
**First-time setup:**
|
||||
|
||||
The console also exposes `POST /v1/api/auth/setup` for first-time
|
||||
bootstrap. When no users exist, the setup wizard calls this public endpoint
|
||||
to create the initial admin user and receive a JWT in one step. See
|
||||
[Security: First-time setup](security.md#first-time-setup) for details.
|
||||
|
||||
---
|
||||
|
||||
## Scheduled Tasks
|
||||
|
||||
The console includes a background **TaskScheduler** daemon that creates workstreams on a timed basis via the MQ broker. It supports cron-based recurring schedules and one-shot `at` schedules.
|
||||
|
||||
### Architecture
|
||||
|
||||
The scheduler runs as a daemon thread inside the console process. Every `check_interval` seconds (default 15) it:
|
||||
|
||||
1. Acquires a distributed lock via Redis `SET NX EX` (prevents duplicate dispatch in multi-console deployments)
|
||||
2. Queries the storage backend for tasks whose `next_run <= now` and `enabled = true`
|
||||
3. Dispatches each due task as one or more `CreateWorkstreamMessage` via MQ
|
||||
4. Updates `last_run` and computes the next `next_run` (or disables one-shot `at` tasks)
|
||||
5. Releases the lock via Lua script (safe conditional delete)
|
||||
|
||||
Run history is automatically pruned (runs older than 90 days) approximately once per hour.
|
||||
|
||||
### Schedule Types
|
||||
|
||||
| Type | Field | Behavior |
|
||||
|------|-------|----------|
|
||||
| `cron` | `cron_expr` | Recurring schedule using standard 5-field cron syntax. Requires `croniter`. |
|
||||
| `at` | `at_time` | One-shot: fires once at the given ISO 8601 timestamp (must include timezone), then auto-disables. |
|
||||
|
||||
### Target Modes
|
||||
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `auto` | Picks the reachable node with the most available capacity |
|
||||
| `pool` | Pushes to the shared inbound queue (any bridge picks it up) |
|
||||
| `all` | Fan-out to all reachable nodes (capped at `max_fan_out`, default 20) |
|
||||
| `<node_id>` | Targets a specific node by ID |
|
||||
|
||||
### Configuration
|
||||
|
||||
| Parameter | Default | Description |
|
||||
|-----------|---------|-------------|
|
||||
| `check_interval` | `15.0` | Seconds between scheduler ticks |
|
||||
| `lock_ttl` | `60` | Distributed lock TTL in seconds |
|
||||
| `max_fan_out` | `20` | Maximum nodes for `all` target mode |
|
||||
|
||||
Dependency: `croniter` (installed with turnstone).
|
||||
|
||||
### Schedule API
|
||||
|
||||
All schedule endpoints require `approve` scope. Maximum 200 schedules.
|
||||
|
||||
#### `GET /v1/api/admin/schedules`
|
||||
|
||||
List all scheduled tasks.
|
||||
|
||||
```json
|
||||
{
|
||||
"schedules": [
|
||||
{
|
||||
"task_id": "a1b2c3d4",
|
||||
"name": "nightly-checks",
|
||||
"description": "Run nightly health checks",
|
||||
"schedule_type": "cron",
|
||||
"cron_expr": "0 2 * * *",
|
||||
"at_time": "",
|
||||
"target_mode": "auto",
|
||||
"model": "",
|
||||
"initial_message": "Run the nightly health check suite.",
|
||||
"auto_approve": false,
|
||||
"auto_approve_tools": [],
|
||||
"enabled": true,
|
||||
"created_by": "u_admin",
|
||||
"last_run": "2026-03-05T02:00:00Z",
|
||||
"next_run": "2026-03-06T02:00:00Z",
|
||||
"created": "2026-03-01T12:00:00Z",
|
||||
"updated": "2026-03-05T02:00:01Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### `POST /v1/api/admin/schedules`
|
||||
|
||||
Create a scheduled task.
|
||||
|
||||
Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "nightly-checks",
|
||||
"description": "Run nightly health checks",
|
||||
"schedule_type": "cron",
|
||||
"cron_expr": "0 2 * * *",
|
||||
"target_mode": "auto",
|
||||
"initial_message": "Run the nightly health check suite.",
|
||||
"auto_approve": false,
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
Required fields: `name`, `schedule_type`, `initial_message`. For `cron` schedules provide `cron_expr`; for `at` schedules provide `at_time` (ISO 8601 with timezone, must be in the future).
|
||||
|
||||
Response: `ScheduleInfo` (same shape as list items above). Returns `400` for invalid cron syntax, naive timestamps, or past `at_time`. Returns `409` if the 200-schedule cap is reached.
|
||||
|
||||
#### `GET /v1/api/admin/schedules/{task_id}`
|
||||
|
||||
Get a single scheduled task. Returns `ScheduleInfo` or `404`.
|
||||
|
||||
#### `PUT /v1/api/admin/schedules/{task_id}`
|
||||
|
||||
Partial update — only include fields to change. If `schedule_type`, `cron_expr`, or `at_time` change, `next_run` is recomputed automatically.
|
||||
|
||||
```json
|
||||
{
|
||||
"enabled": false
|
||||
}
|
||||
```
|
||||
|
||||
Response: updated `ScheduleInfo`. Returns `400` for validation errors, `404` if not found.
|
||||
|
||||
#### `DELETE /v1/api/admin/schedules/{task_id}`
|
||||
|
||||
Delete a scheduled task and all its run history. Returns `{"status": "ok"}` or `404`.
|
||||
|
||||
#### `GET /v1/api/admin/schedules/{task_id}/runs?limit=50`
|
||||
|
||||
List execution history for a task (most recent first). `limit` defaults to 50, max 200.
|
||||
|
||||
```json
|
||||
{
|
||||
"runs": [
|
||||
{
|
||||
"run_id": "r_abc123",
|
||||
"task_id": "a1b2c3d4",
|
||||
"node_id": "db-west-04",
|
||||
"ws_id": "ws_xyz",
|
||||
"correlation_id": "corr_789",
|
||||
"started": "2026-03-05T02:00:00Z",
|
||||
"status": "dispatched",
|
||||
"error": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Status is `dispatched` on success or `failed` with an `error` message (e.g. no reachable nodes). Failed runs do not advance `next_run`.
|
||||
|
||||
---
|
||||
|
||||
@@ -201,6 +593,7 @@ CLI flags for `turnstone-console`:
|
||||
| `--redis-password` | `$REDIS_PASSWORD` | Redis password |
|
||||
| `--redis-db` | `0` | Redis DB |
|
||||
| `--poll-interval` | `10` | Node polling interval (seconds) |
|
||||
| `--auth-token` | `$TURNSTONE_AUTH_TOKEN` | Bearer token for server node communication and proxy |
|
||||
| `--log-level` | `INFO` | Log level |
|
||||
|
||||
Config file (`~/.config/turnstone/config.toml`):
|
||||
@@ -233,7 +626,7 @@ turnstone-server --port 8080
|
||||
turnstone-bridge --server-url http://localhost:8080 --node-id node-a
|
||||
|
||||
# Start cluster console (one instance)
|
||||
turnstone-console --redis-host localhost --port 8090
|
||||
turnstone-console --redis-host localhost --port 8090 --auth-token "$TURNSTONE_AUTH_TOKEN"
|
||||
```
|
||||
|
||||
Open `http://localhost:8090` for the cluster dashboard.
|
||||
Open `http://localhost:8090` for the cluster dashboard. Create workstreams via the "+ new" button. Click any workstream to open the proxied server UI — no direct access to server ports required.
|
||||
|
||||
@@ -9,7 +9,10 @@ actor "External Client\n(Python / CI)" as ext_client
|
||||
actor "Eval Harness" as eval_user
|
||||
|
||||
' External Systems
|
||||
cloud "LLM Provider\n(OpenAI-compatible API)" as llm
|
||||
cloud "LLM Providers" as llm {
|
||||
component [OpenAI-compatible API\n(OpenAI, vLLM, llama.cpp)] as llm_openai
|
||||
component [Anthropic Messages API] as llm_anthropic
|
||||
}
|
||||
database "Redis" as redis
|
||||
database "SQLite\n(.turnstone.db)" as sqlite
|
||||
|
||||
@@ -31,21 +34,21 @@ ext_client --> redis : Redis LIST\n(push commands)
|
||||
eval_user --> eval : Python API
|
||||
|
||||
' Internal connections
|
||||
cli --> llm : OpenAI Streaming API\n(HTTPS)
|
||||
cli --> llm : LLM Provider API\n(via provider adapters)
|
||||
cli --> sqlite : SQLite
|
||||
|
||||
server --> llm : OpenAI Streaming API\n(HTTPS)
|
||||
server --> llm : LLM Provider API\n(via provider adapters)
|
||||
server --> sqlite : SQLite
|
||||
|
||||
eval --> llm : OpenAI API\n(non-streaming)
|
||||
eval --> llm : LLM Provider API\n(non-streaming)
|
||||
eval --> sqlite : SQLite
|
||||
|
||||
bridge --> server : HTTP REST\n(POST /api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /api/events)
|
||||
bridge --> server : HTTP REST\n(POST /v1/api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /v1/api/events)
|
||||
bridge --> redis : Redis LIST + PUBSUB\n+ STRING (routing, heartbeats)
|
||||
|
||||
console --> redis : Redis PUBSUB + STRING\n(cluster channel, heartbeats)
|
||||
console --> server : HTTP polling\n(GET /api/dashboard)
|
||||
console --> redis : Redis PUBSUB + STRING + LIST\n(cluster events, heartbeats,\nworkstream creation commands)
|
||||
console --> server : HTTP polling + reverse proxy\n(GET /v1/api/dashboard,\nproxy /node/{id}/* traffic)
|
||||
|
||||
sim --> redis : Redis LIST + PUBSUB\n+ STRING (heartbeats)
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ skinparam component {
|
||||
BackgroundColor<<console>> #B2EBF2
|
||||
BackgroundColor<<ui>> #F0F4C3
|
||||
BackgroundColor<<artifact>> #ECEFF1
|
||||
BackgroundColor<<sdk>> #FFCDD2
|
||||
BackgroundColor<<api>> #D1C4E9
|
||||
}
|
||||
|
||||
' Entry points
|
||||
@@ -24,9 +26,11 @@ package "Entry Points" <<Rectangle>> {
|
||||
' Core engine
|
||||
package "turnstone/core/" <<Rectangle>> {
|
||||
component [session.py\nChatSession, SessionUI] as session <<core>>
|
||||
component [providers/\nLLMProvider, OpenAI, Anthropic] as providers <<core>>
|
||||
component [workstream.py\nWorkstreamManager] as workstream <<core>>
|
||||
component [tools.py\nTool loader] as tools <<core>>
|
||||
component [memory.py\nSQLite + FTS5] as memory <<core>>
|
||||
component [memory.py\nPersistence facade] as memory <<core>>
|
||||
component [storage/\nStorageBackend protocol\nSQLite + PostgreSQL] as storage <<core>>
|
||||
component [metrics.py\nPrometheus metrics] as metrics <<core>>
|
||||
component [config.py\nTOML config] as config <<core>>
|
||||
component [safety.py\nPath validation] as safety <<core>>
|
||||
@@ -72,6 +76,23 @@ package "turnstone/ui/" <<Rectangle>> {
|
||||
component [spinner.py\nTerminal spinner] as spinner <<ui>>
|
||||
}
|
||||
|
||||
' API schemas
|
||||
package "turnstone/api/" <<Rectangle>> {
|
||||
component [schemas.py\nShared Pydantic models] as apischemas <<api>>
|
||||
component [server_spec.py\nServer OpenAPI spec] as serverspec <<api>>
|
||||
component [console_spec.py\nConsole OpenAPI spec] as consolespec <<api>>
|
||||
component [openapi.py\nSpec builder] as openapi <<api>>
|
||||
component [docs.py\nSwagger UI handler] as apidocs <<api>>
|
||||
}
|
||||
|
||||
' SDK
|
||||
package "turnstone/sdk/" <<Rectangle>> {
|
||||
component [server.py\nTurnstoneServer (sync+async)] as sdkserver <<sdk>>
|
||||
component [console.py\nTurnstoneConsole (sync+async)] as sdkconsole <<sdk>>
|
||||
component [events.py\n27 SSE event types] as sdkevents <<sdk>>
|
||||
component [_base.py\nhttpx client base] as sdkbase <<sdk>>
|
||||
}
|
||||
|
||||
' Tool schemas
|
||||
package "turnstone/tools/" <<Rectangle>> {
|
||||
component [*.json\n14 tool schemas] as schemas <<artifact>>
|
||||
@@ -105,8 +126,10 @@ eval --> tools
|
||||
chat --> session
|
||||
|
||||
' Core internal deps
|
||||
session --> providers
|
||||
session --> tools
|
||||
session --> memory
|
||||
memory --> storage
|
||||
session --> safety
|
||||
session --> sandbox
|
||||
session --> edit
|
||||
@@ -114,6 +137,7 @@ session --> web
|
||||
session --> healthcheck
|
||||
session --> mcp : optional
|
||||
session --> registry : optional
|
||||
registry --> providers
|
||||
healthcheck --> metrics
|
||||
mcp --> config
|
||||
registry --> config
|
||||
@@ -149,4 +173,22 @@ consoleserver --> config
|
||||
consoleserver --> auth
|
||||
collector --> broker
|
||||
|
||||
' API dependencies
|
||||
serverspec --> openapi
|
||||
consolespec --> openapi
|
||||
serverspec --> apischemas
|
||||
consolespec --> apischemas
|
||||
server --> apidocs
|
||||
server --> serverspec
|
||||
consoleserver --> apidocs
|
||||
consoleserver --> consolespec
|
||||
|
||||
' SDK dependencies
|
||||
sdkserver --> sdkbase
|
||||
sdkconsole --> sdkbase
|
||||
sdkserver --> sdkevents
|
||||
sdkconsole --> sdkevents
|
||||
sdkserver --> apischemas : returns models
|
||||
sdkconsole --> apischemas : returns models
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b3f76042c8046560502fa3132351821d56e8526b13d38d7be8b839fcf3d5f648
|
||||
size 373463
|
||||
@@ -52,6 +52,8 @@ class "WebUI" as WebUI {
|
||||
Enqueues JSON events for SSE.
|
||||
Blocks on threading.Event for
|
||||
approval/plan review.
|
||||
SSE handlers bridge Queue to
|
||||
async via run_in_executor().
|
||||
--
|
||||
server.py
|
||||
}
|
||||
@@ -63,9 +65,55 @@ class "NullUI" as NullUI {
|
||||
eval.py
|
||||
}
|
||||
|
||||
' LLMProvider Protocol
|
||||
interface "LLMProvider" as LLMProvider <<Protocol>> {
|
||||
+ provider_name: str {property}
|
||||
+ get_capabilities(model) → ModelCapabilities
|
||||
+ create_streaming(client, model, messages, ...) → Iterator[StreamChunk]
|
||||
+ create_completion(client, model, messages, ...) → CompletionResult
|
||||
+ convert_tools(tools) → list[dict]
|
||||
+ retryable_error_names: frozenset[str] {property}
|
||||
--
|
||||
core/providers/_protocol.py
|
||||
}
|
||||
|
||||
class "OpenAIProvider" as OpenAIProv {
|
||||
Model capability lookup table
|
||||
(GPT-5.x, O-series, search)
|
||||
Passthrough: messages already
|
||||
in OpenAI format.
|
||||
Search models: web_search_options
|
||||
+ url_citation annotations.
|
||||
--
|
||||
core/providers/_openai.py
|
||||
}
|
||||
|
||||
class "AnthropicProvider" as AnthropicProv {
|
||||
Converts OpenAI messages to
|
||||
Anthropic content blocks.
|
||||
Adaptive + manual thinking.
|
||||
Native web search via
|
||||
web_search_20250305 server tool.
|
||||
Lazy anthropic SDK import.
|
||||
--
|
||||
core/providers/_anthropic.py
|
||||
}
|
||||
|
||||
' ModelCapabilities
|
||||
class "ModelCapabilities" as ModelCaps <<frozen>> {
|
||||
+ context_window: int
|
||||
+ max_output_tokens: int
|
||||
+ supports_temperature: bool
|
||||
+ token_param: str
|
||||
+ thinking_mode: str
|
||||
+ supports_effort: bool
|
||||
+ supports_web_search: bool
|
||||
}
|
||||
|
||||
' ChatSession
|
||||
class "ChatSession" as ChatSession {
|
||||
- client: OpenAI
|
||||
- client: Any
|
||||
- provider: LLMProvider
|
||||
- model: str
|
||||
- ui: SessionUI
|
||||
- messages: list[dict]
|
||||
@@ -172,20 +220,22 @@ class "MCPClientManager" as MCPMgr {
|
||||
' ModelRegistry
|
||||
class "ModelRegistry" as ModelReg {
|
||||
- _models: dict[str, ModelConfig]
|
||||
- _clients: dict[str, OpenAI]
|
||||
- _clients: dict[str, Any]
|
||||
- _providers: dict[str, LLMProvider]
|
||||
- _client_lock: Lock
|
||||
+ default: str
|
||||
+ fallback: list[str]
|
||||
+ agent_model: str | None
|
||||
--
|
||||
+ resolve(alias) → (client, model, config)
|
||||
+ get_client(alias) → OpenAI
|
||||
+ get_client(alias) → Any
|
||||
+ get_provider(alias) → LLMProvider
|
||||
+ has_alias(alias) → bool
|
||||
+ list_aliases() → list[str]
|
||||
+ shutdown()
|
||||
--
|
||||
Thread-safe lazy client creation.
|
||||
Loaded by load_model_registry()
|
||||
Thread-safe lazy client + provider
|
||||
creation. Loaded by load_model_registry()
|
||||
from CLI args + [models.*] config.
|
||||
--
|
||||
core/model_registry.py
|
||||
@@ -193,6 +243,7 @@ class "ModelRegistry" as ModelReg {
|
||||
|
||||
class "ModelConfig" as ModelCfg <<frozen>> {
|
||||
+ alias: str
|
||||
+ provider: str
|
||||
+ base_url: str
|
||||
+ model: str
|
||||
+ context_window: int
|
||||
@@ -260,7 +311,11 @@ TerminalUI <|-- WsTermUI
|
||||
SessionUI <|.. WebUI
|
||||
SessionUI <|.. NullUI
|
||||
|
||||
LLMProvider <|.. OpenAIProv
|
||||
LLMProvider <|.. AnthropicProv
|
||||
|
||||
ChatSession --> SessionUI : uses
|
||||
ChatSession --> LLMProvider : delegates LLM calls
|
||||
ChatSession --> MCPMgr : optional
|
||||
ChatSession --> ModelReg : optional
|
||||
ChatSession <|-- HeadlessSession
|
||||
@@ -273,6 +328,8 @@ Ws --> "1" WsState : has
|
||||
WsMgr ..> ChatSession : creates via\nsession_factory(ui, model_alias)
|
||||
|
||||
ModelReg --> "*" ModelCfg : holds
|
||||
ModelReg --> "*" LLMProvider : caches
|
||||
LLMProvider --> ModelCaps : returns
|
||||
|
||||
ChatSession --> HealthMon : checks circuit
|
||||
HealthMon --> "1" CircuitState : has
|
||||
@@ -282,6 +339,8 @@ note bottom of ChatSession
|
||||
Central engine: multi-turn LLM loop
|
||||
with tool dispatch, agent sub-sessions,
|
||||
context compaction, and memory persistence.
|
||||
Provider-agnostic — delegates all LLM
|
||||
communication to LLMProvider adapters.
|
||||
|
||||
core/session.py (~2700 lines)
|
||||
end note
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fa1c94a0a7489cb9e6e6cad17f7e3577c458ec89d2a47fb2669fe935768837cf
|
||||
size 276863
|
||||
@@ -8,7 +8,7 @@ skinparam sequenceLifeLineBackgroundColor #F5F5F5
|
||||
participant "User /\nHTTP Client" as User
|
||||
participant "ChatSession" as CS
|
||||
participant "SessionUI" as UI
|
||||
participant "OpenAI API\n(LLM)" as LLM
|
||||
participant "LLMProvider\n(OpenAI / Anthropic)" as LLM
|
||||
participant "Tool Executor\n(ThreadPool)" as TP
|
||||
database "SQLite" as DB
|
||||
|
||||
@@ -27,7 +27,7 @@ group loop [while tool_calls present]
|
||||
CS -> UI : on_state_change("thinking")
|
||||
CS -> UI : on_thinking_start()
|
||||
|
||||
CS -> LLM : client.chat.completions.create(\n model, messages, tools,\n stream=True, stream_options={include_usage})
|
||||
CS -> LLM : provider.create_streaming(\n client, model, messages, tools, ...)\n (normalized StreamChunk iterator)
|
||||
activate LLM
|
||||
|
||||
note right of CS
|
||||
@@ -52,6 +52,8 @@ group loop [while tool_calls present]
|
||||
CS -> UI : on_content_token(text)
|
||||
else tool_call delta
|
||||
CS -> CS : accumulate in tool_calls_acc
|
||||
else info_delta present
|
||||
CS -> UI : on_info(text)\n(e.g. server-side web search status)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -116,7 +118,7 @@ group loop [while tool_calls present]
|
||||
task/plan → _run_agent() sub-loop
|
||||
math → sandboxed subprocess
|
||||
web_fetch → httpx + LLM summarize
|
||||
web_search → Tavily API
|
||||
web_search → provider-native or Tavily fallback
|
||||
remember/recall/forget → SQLite
|
||||
end note
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:06fb9faa29c11395c6fc54ebddc79994b000dee78d56e0c13cb689fd6a82e37a
|
||||
size 237255
|
||||
@@ -105,7 +105,7 @@ partition "Phase 3: Execute" #E3F2FD {
|
||||
├─ _exec_math: sandboxed subprocess
|
||||
├─ _exec_man: man/info subprocess
|
||||
├─ _exec_web_fetch: httpx.get + LLM summary
|
||||
├─ _exec_web_search: Tavily API POST
|
||||
├─ _exec_web_search: Tavily API POST (fallback for local models)
|
||||
├─ _exec_task: _run_agent(TASK_AGENT_TOOLS)
|
||||
├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only)
|
||||
├─ _exec_remember: SQLite INSERT OR REPLACE
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:13b2da77312e5abb44c81aabb7f0addccab31d9bc7e8ef2f1c3563ef985ed503
|
||||
size 186941
|
||||
@@ -58,6 +58,7 @@ package "Inbound Messages (Client → Bridge)" #FFF3E0 {
|
||||
+ auto_approve: bool = False
|
||||
+ auto_approve_tools: list[str] = []
|
||||
+ target_node: str = ""
|
||||
+ initial_message: str = ""
|
||||
}
|
||||
|
||||
class CloseWorkstreamMessage {
|
||||
|
||||
@@ -18,20 +18,20 @@ note right of Redis : Shared queue — any bridge can pick up
|
||||
BridgeA -> Redis : BLPOP [turnstone:inbound:nodeA,\n turnstone:inbound]
|
||||
Redis --> BridgeA : SendMessage (from shared queue)
|
||||
|
||||
BridgeA -> ServerA : POST /api/workstreams/new\n{name:"", auto_approve:false}
|
||||
BridgeA -> ServerA : POST /v1/api/workstreams/new\n{name:"", auto_approve:false}
|
||||
ServerA --> BridgeA : {ws_id:"abc12345", name:"ws-abc1"}
|
||||
|
||||
BridgeA -> Redis : SET turnstone:ws:abc12345 "nodeA"
|
||||
note right : Register workstream ownership
|
||||
|
||||
BridgeA -> ServerA : GET /api/events?ws_id=abc12345
|
||||
BridgeA -> ServerA : GET /v1/api/events?ws_id=abc12345
|
||||
note right : Start per-WS SSE thread
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nWorkstreamCreatedEvent
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:cluster\nClusterStateEvent(ws_id, state:"idle", node_id:"nodeA")
|
||||
|
||||
BridgeA -> ServerA : POST /api/send\n{message:"...", ws_id:"abc12345"}
|
||||
BridgeA -> ServerA : POST /v1/api/send\n{message:"...", ws_id:"abc12345"}
|
||||
ServerA --> BridgeA : {status:"ok"}
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nAckEvent(status:"ok")
|
||||
@@ -93,7 +93,7 @@ note right : Response queue — bypasses inbound queue
|
||||
BridgeA -> Redis : BLPOP turnstone:resp:req_xyz\n(spawned approval thread, timeout 300s)
|
||||
Redis --> BridgeA : ApproveMessage
|
||||
|
||||
BridgeA -> ServerA : POST /api/approve\n{approved:true, ws_id:"abc12345"}
|
||||
BridgeA -> ServerA : POST /v1/api/approve\n{approved:true, ws_id:"abc12345"}
|
||||
|
||||
== Heartbeat (continuous) ==
|
||||
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Console Dashboard Data Collection
|
||||
title Turnstone — Console Dashboard Data Flow
|
||||
|
||||
skinparam sequenceArrowThickness 1.5
|
||||
|
||||
participant "Browser" as Browser
|
||||
participant "Console\nHTTP Server" as Server
|
||||
participant "Console\nStarlette App" as Server
|
||||
participant "ClusterCollector" as CC
|
||||
collections "Redis" as Redis
|
||||
participant "Node-A Bridge" as BridgeA
|
||||
participant "Node-A\n(real server)" as NodeA
|
||||
participant "Node-B\n(sim node)" as NodeB
|
||||
|
||||
@@ -67,14 +68,14 @@ note right of CC
|
||||
from the cluster event channel.
|
||||
end note
|
||||
|
||||
CC -> NodeA : GET /api/dashboard
|
||||
CC -> NodeA : GET /v1/api/dashboard
|
||||
activate NodeA
|
||||
NodeA --> CC : {workstreams: [...],\naggregate: {total_tokens, ...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> NodeA : GET /health
|
||||
activate NodeA
|
||||
NodeA --> CC : {status:"ok", model:"...",\nworkstreams:{total, idle, ...}}
|
||||
NodeA --> CC : {status:"ok", version:"0.3.0",\nmodel:"...", workstreams:{...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> CC : Replace NodeSnapshot["nodeA"]\n.workstreams, .health, .aggregate
|
||||
@@ -85,19 +86,19 @@ deactivate CC
|
||||
|
||||
== Browser SSE Stream ==
|
||||
|
||||
Browser -> Server : GET /api/cluster/events
|
||||
Browser -> Server : GET /v1/api/cluster/events
|
||||
activate Server
|
||||
|
||||
Server -> CC : register_listener(queue)
|
||||
note right : Per-client queue.Queue(maxsize=500)
|
||||
note right : Per-client queue.Queue(maxsize=500)\nSSE via EventSourceResponse + run_in_executor()
|
||||
|
||||
loop continuous
|
||||
CC -> Server : event via listener queue\n(from any of the 3 threads)
|
||||
Server -> Browser : data: {"type":"cluster_state",...}\n\n
|
||||
end
|
||||
|
||||
alt timeout (5s no events)
|
||||
Server -> Browser : : keepalive\n\n
|
||||
alt keepalive (sse-starlette ping=5)
|
||||
Server -> Browser : : ping\n\n
|
||||
end
|
||||
|
||||
Browser -> Server : connection closed
|
||||
@@ -106,19 +107,115 @@ deactivate Server
|
||||
|
||||
== Browser REST Requests ==
|
||||
|
||||
Browser -> Server : GET /api/cluster/overview
|
||||
Browser -> Server : GET /v1/api/cluster/overview
|
||||
Server -> CC : get_overview()
|
||||
CC --> Server : {nodes: 10, workstreams: 47,\nstates: {running:5, thinking:3, ...},\naggregate: {total_tokens: 50000}}
|
||||
CC --> Server : {nodes: 10, workstreams: 47,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.3.0"]}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /api/cluster/nodes?sort=activity
|
||||
Browser -> Server : GET /v1/api/cluster/nodes?sort=activity
|
||||
Server -> CC : get_nodes(sort_by="activity")
|
||||
CC --> Server : {nodes: [...], total: 10}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /api/cluster/workstreams\n?state=running&node=sim-0003
|
||||
Browser -> Server : GET /v1/api/cluster/workstreams\n?state=running&node=sim-0003
|
||||
Server -> CC : get_workstreams(state="running",\nnode="sim-0003")
|
||||
CC --> Server : {workstreams: [...], total: 5,\npage: 1, per_page: 50, pages: 1}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
== Workstream Creation (via MQ) ==
|
||||
|
||||
Browser -> Server : POST /v1/api/cluster/workstreams/new\n{node_id:"nodeA", name:"new-task"}
|
||||
activate Server #FFECB3
|
||||
|
||||
Server -> CC : _pick_best_node() or\nget_node_detail(node_id)
|
||||
CC --> Server : node validated
|
||||
|
||||
Server -> Server : Build CreateWorkstreamMessage\n{target_node:"nodeA", name:"new-task"}
|
||||
|
||||
Server -> Redis : RPUSH turnstone:inbound:nodeA\n(directed queue)
|
||||
Server --> Browser : {status:"ok", correlation_id:"abc",\ntarget_node:"nodeA"}
|
||||
deactivate Server
|
||||
|
||||
note right of Redis
|
||||
Bridge on Node-A picks up the
|
||||
message from its directed queue,
|
||||
POSTs to /v1/api/workstreams/new,
|
||||
registers ownership, publishes
|
||||
ws_created to cluster channel.
|
||||
end note
|
||||
|
||||
Redis --> BridgeA : BLPOP turnstone:inbound:nodeA
|
||||
activate BridgeA
|
||||
BridgeA -> NodeA : POST /v1/api/workstreams/new\n{name:"new-task"}
|
||||
NodeA --> BridgeA : {ws_id:"ws789", name:"new-task"}
|
||||
BridgeA -> Redis : SET turnstone:ws:ws789 = nodeA
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:cluster\n{type:"ws_created", ws_id:"ws789",\nnode_id:"nodeA", name:"new-task"}
|
||||
deactivate BridgeA
|
||||
|
||||
Redis --> CC : ws_created event
|
||||
CC -> CC : Add workstream to\nNodeSnapshot["nodeA"]
|
||||
CC -> CC : _fanout(event)
|
||||
Server -> Browser : SSE: data: {"type":"ws_created",...}
|
||||
|
||||
== Reverse Proxy (server UI through console port) ==
|
||||
|
||||
Browser -> Server : GET /node/nodeA/
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> CC : get_node_detail("nodeA")\n→ server_url = "http://10.0.1.1:8080"
|
||||
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/\n(via httpx.AsyncClient)
|
||||
activate NodeA
|
||||
NodeA --> Server : index.html
|
||||
deactivate NodeA
|
||||
|
||||
Server -> Server : Rewrite static paths:\nhref="/static/" → "/node/nodeA/static/"\nInject console-return banner\nafter <body>
|
||||
|
||||
Server --> Browser : Rewritten HTML
|
||||
deactivate Server
|
||||
|
||||
Browser -> Server : GET /node/nodeA/static/app.js
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/static/app.js
|
||||
activate NodeA
|
||||
NodeA --> Server : app.js
|
||||
deactivate NodeA
|
||||
|
||||
Server -> Server : Prepend JS proxy shim:\nOverride fetch() and EventSource()\nto prepend "/node/nodeA" prefix
|
||||
|
||||
Server --> Browser : Shimmed app.js
|
||||
deactivate Server
|
||||
|
||||
note right of Browser
|
||||
All fetch("/v1/api/send") calls in the
|
||||
server UI now become fetch("/node/nodeA/v1/api/send"),
|
||||
routed through the console proxy.
|
||||
end note
|
||||
|
||||
Browser -> Server : GET /node/nodeA/v1/api/events?ws_id=ws789
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/v1/api/events?ws_id=ws789\n(SSE stream via httpx.AsyncClient timeout=None)
|
||||
activate NodeA
|
||||
|
||||
loop SSE streaming
|
||||
NodeA --> Server : data: {"type":"content","text":"..."}\n\n
|
||||
Server --> Browser : data: {"type":"content","text":"..."}\n\n
|
||||
end
|
||||
|
||||
deactivate NodeA
|
||||
deactivate Server
|
||||
|
||||
Browser -> Server : POST /node/nodeA/v1/api/send\n{message:"hello", ws_id:"ws789"}
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : POST http://10.0.1.1:8080/v1/api/send\n(body forwarded)
|
||||
activate NodeA
|
||||
NodeA --> Server : {status:"ok"}
|
||||
deactivate NodeA
|
||||
|
||||
Server --> Browser : {status:"ok"}
|
||||
deactivate Server
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -83,12 +83,12 @@ mqclient --> redis : Redis protocol\nport 6379
|
||||
server --> redis : Redis protocol\n(6379)
|
||||
server --> llm_api : OpenAI API\n(HTTPS/HTTP)
|
||||
|
||||
bridge --> server : HTTP REST\n(POST /api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /api/events)
|
||||
bridge --> server : HTTP REST\n(POST /v1/api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /v1/api/events)
|
||||
bridge --> redis : Redis protocol\n(queues + pubsub)
|
||||
|
||||
console --> redis : Redis PUBSUB\n(cluster channel)
|
||||
console --> server : HTTP polling\n(GET /api/dashboard)
|
||||
console --> redis : Redis PUBSUB + LIST\n(cluster events,\nws creation commands)
|
||||
console --> server : HTTP polling + proxy\n(GET /v1/api/dashboard,\nproxy /node/{id}/*)
|
||||
|
||||
sim --> redis : Redis protocol\n(queues + pubsub + keys)
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Client SDK Architecture
|
||||
|
||||
skinparam class {
|
||||
BackgroundColor<<async>> #C8E6C9
|
||||
BackgroundColor<<sync>> #B8D4E3
|
||||
BackgroundColor<<event>> #FFE0B2
|
||||
BackgroundColor<<type>> #F0F4C3
|
||||
BackgroundColor<<ts>> #E1BEE7
|
||||
}
|
||||
|
||||
skinparam packageBorderColor #888888
|
||||
skinparam ArrowColor #555555
|
||||
|
||||
' Python SDK
|
||||
package "turnstone/sdk/ (Python)" {
|
||||
abstract class _BaseClient <<async>> {
|
||||
- _client: httpx.AsyncClient
|
||||
- _owns_client: bool
|
||||
+ _request(method, path, ...) → T
|
||||
+ _stream_sse(path, ...) → AsyncIterator
|
||||
+ aclose()
|
||||
}
|
||||
|
||||
class AsyncTurnstoneServer <<async>> {
|
||||
+ list_workstreams()
|
||||
+ dashboard()
|
||||
+ create_workstream()
|
||||
+ close_workstream()
|
||||
+ send(message, ws_id)
|
||||
+ approve()
|
||||
+ plan_feedback()
|
||||
+ command()
|
||||
+ stream_events(ws_id)
|
||||
+ stream_global_events()
|
||||
+ send_and_wait()
|
||||
+ list_sessions()
|
||||
+ login() / logout()
|
||||
+ health()
|
||||
}
|
||||
|
||||
class AsyncTurnstoneConsole <<async>> {
|
||||
+ overview()
|
||||
+ nodes()
|
||||
+ workstreams()
|
||||
+ node_detail()
|
||||
+ create_workstream()
|
||||
+ stream_cluster_events()
|
||||
+ login() / logout()
|
||||
+ health()
|
||||
}
|
||||
|
||||
class TurnstoneServer <<sync>> {
|
||||
- _async: AsyncTurnstoneServer
|
||||
- _runner: _SyncRunner
|
||||
.. delegates all methods ..
|
||||
+ __enter__ / __exit__
|
||||
}
|
||||
|
||||
class TurnstoneConsole <<sync>> {
|
||||
- _async: AsyncTurnstoneConsole
|
||||
- _runner: _SyncRunner
|
||||
.. delegates all methods ..
|
||||
+ __enter__ / __exit__
|
||||
}
|
||||
|
||||
class _SyncRunner <<sync>> {
|
||||
- _loop: EventLoop
|
||||
- _thread: Thread
|
||||
+ run(coro) → T
|
||||
+ run_iter(async_gen) → Iterator
|
||||
+ close()
|
||||
}
|
||||
|
||||
class TurnResult <<type>> {
|
||||
+ ws_id: str
|
||||
+ content_parts: list[str]
|
||||
+ reasoning_parts: list[str]
|
||||
+ tool_results: list
|
||||
+ errors: list[str]
|
||||
+ timed_out: bool
|
||||
--
|
||||
+ content: str
|
||||
+ reasoning: str
|
||||
+ ok: bool
|
||||
}
|
||||
|
||||
class ServerEvent <<event>> {
|
||||
+ type: str
|
||||
+ ws_id: str
|
||||
+ from_dict() → ServerEvent
|
||||
}
|
||||
|
||||
class ClusterEvent <<event>> {
|
||||
+ type: str
|
||||
+ from_dict() → ClusterEvent
|
||||
}
|
||||
|
||||
_BaseClient <|-- AsyncTurnstoneServer
|
||||
_BaseClient <|-- AsyncTurnstoneConsole
|
||||
TurnstoneServer --> AsyncTurnstoneServer : wraps
|
||||
TurnstoneServer --> _SyncRunner : uses
|
||||
TurnstoneConsole --> AsyncTurnstoneConsole : wraps
|
||||
TurnstoneConsole --> _SyncRunner : uses
|
||||
AsyncTurnstoneServer ..> TurnResult : returns
|
||||
AsyncTurnstoneServer ..> ServerEvent : yields
|
||||
AsyncTurnstoneConsole ..> ClusterEvent : yields
|
||||
}
|
||||
|
||||
' TypeScript SDK
|
||||
package "sdk/typescript/ (TypeScript)" {
|
||||
class "BaseClient" as TSBase <<ts>> {
|
||||
# baseUrl: string
|
||||
# token: string
|
||||
# fetchFn: fetch
|
||||
# request<T>()
|
||||
# streamSSE<T>()
|
||||
}
|
||||
|
||||
class "TurnstoneServer" as TSServer <<ts>> {
|
||||
+ listWorkstreams()
|
||||
+ send()
|
||||
+ streamEvents()
|
||||
+ sendAndWait()
|
||||
...
|
||||
}
|
||||
|
||||
class "TurnstoneConsole" as TSConsole <<ts>> {
|
||||
+ overview()
|
||||
+ nodes()
|
||||
+ clusterEvents()
|
||||
...
|
||||
}
|
||||
|
||||
TSBase <|-- TSServer
|
||||
TSBase <|-- TSConsole
|
||||
}
|
||||
|
||||
' External connections
|
||||
class "turnstone-server :8080" as Server <<artifact>>
|
||||
class "turnstone-console :8081" as Console <<artifact>>
|
||||
|
||||
AsyncTurnstoneServer --> Server : httpx REST + SSE
|
||||
AsyncTurnstoneConsole --> Console : httpx REST + SSE
|
||||
TSServer --> Server : fetch REST + SSE
|
||||
TSConsole --> Console : fetch REST + SSE
|
||||
|
||||
note right of AsyncTurnstoneServer
|
||||
Returns Pydantic models from
|
||||
turnstone.api.server_schemas
|
||||
(no type duplication)
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,173 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Storage Architecture
|
||||
|
||||
skinparam class {
|
||||
BackgroundColor<<protocol>> #E8EAF6
|
||||
BackgroundColor<<sqlite>> #C8E6C9
|
||||
BackgroundColor<<postgres>> #B3E5FC
|
||||
BackgroundColor<<facade>> #FFF9C4
|
||||
BackgroundColor<<migration>> #FFE0B2
|
||||
BackgroundColor<<schema>> #F3E5F5
|
||||
}
|
||||
|
||||
' -- Protocol --
|
||||
interface "StorageBackend" as SB <<protocol>> {
|
||||
+register_session(session_id, title, node_id, ws_id)
|
||||
+save_message(session_id, role, content, ...)
|
||||
+load_session_messages(session_id) → list[dict]
|
||||
+list_sessions(limit) → list
|
||||
+delete_session(session_id) → bool
|
||||
+prune_sessions(retention_days) → (int, int)
|
||||
+resolve_session(alias_or_id) → str | None
|
||||
+save_session_config(session_id, config)
|
||||
+load_session_config(session_id) → dict
|
||||
+set_session_alias(session_id, alias) → bool
|
||||
+get_session_name(session_id) → str | None
|
||||
+update_session_title(session_id, title)
|
||||
+register_workstream(ws_id, node_id, name, state)
|
||||
+update_workstream_state(ws_id, state)
|
||||
+update_workstream_name(ws_id, name)
|
||||
+delete_workstream(ws_id) → bool
|
||||
+list_workstreams(node_id, limit) → list
|
||||
+kv_get(key) → str | None
|
||||
+kv_set(key, value) → str | None
|
||||
+kv_delete(key) → bool
|
||||
+kv_list() → list[(str, str)]
|
||||
+kv_search(query) → list[(str, str)]
|
||||
+search_history(query, limit) → list
|
||||
+search_history_recent(limit) → list
|
||||
+create_user(user_id, username, display_name, pw_hash)
|
||||
+get_user(user_id) / get_user_by_username(username)
|
||||
+list_users() / delete_user(user_id)
|
||||
+create_api_token(...) / get_api_token_by_hash(hash)
|
||||
+list_api_tokens(user_id) / delete_api_token(id)
|
||||
+close()
|
||||
}
|
||||
|
||||
' -- Backends --
|
||||
class "SQLiteBackend" as SQLite <<sqlite>> {
|
||||
-_engine: sa.Engine
|
||||
-_fts5_available: bool
|
||||
+__init__(path: str)
|
||||
--
|
||||
FTS5 full-text search
|
||||
Default pool, check_same_thread=False
|
||||
}
|
||||
|
||||
class "PostgreSQLBackend" as PG <<postgres>> {
|
||||
-_engine: sa.Engine
|
||||
+__init__(url: str, pool_size: int)
|
||||
--
|
||||
tsvector + ILIKE search
|
||||
Connection pooling
|
||||
}
|
||||
|
||||
' -- Schema --
|
||||
class "_schema.py" as Schema <<schema>> {
|
||||
+metadata: MetaData
|
||||
+memories: Table
|
||||
+conversations: Table
|
||||
+sessions: Table (node_id, ws_id, user_id)
|
||||
+workstreams: Table (node_id, user_id, state)
|
||||
+session_config: Table
|
||||
+users: Table (username, password_hash)
|
||||
+api_tokens: Table (token_hash, scopes)
|
||||
+channel_users: Table (channel_type)
|
||||
--
|
||||
SQLAlchemy Core
|
||||
Single source of truth
|
||||
}
|
||||
|
||||
' -- Migration --
|
||||
class "_migrate.py" as Migrate <<migration>> {
|
||||
+run_migrations(storage, backend)
|
||||
-_bootstrap_existing_sqlite()
|
||||
--
|
||||
Programmatic Alembic
|
||||
Auto-bootstrap existing DBs
|
||||
}
|
||||
|
||||
class "migrations/" as Versions <<migration>> {
|
||||
001_initial_schema.py
|
||||
002_user_identity.py
|
||||
}
|
||||
|
||||
' -- Registry --
|
||||
class "_registry.py" as Registry {
|
||||
-_storage: StorageBackend | None
|
||||
+init_storage(backend, path, url) → StorageBackend
|
||||
+get_storage() → StorageBackend
|
||||
+reset_storage()
|
||||
--
|
||||
Auto-initializes SQLite
|
||||
if not configured
|
||||
}
|
||||
|
||||
' -- Facade --
|
||||
class "memory.py" as Facade <<facade>> {
|
||||
+register_session()
|
||||
+save_message()
|
||||
+load_session_messages()
|
||||
+register_workstream()
|
||||
+update_workstream_state()
|
||||
+save_memory() / delete_memory()
|
||||
+search_memories()
|
||||
+... (all 22 functions)
|
||||
--
|
||||
Thin delegation to
|
||||
get_storage()
|
||||
Silent failure behavior
|
||||
}
|
||||
|
||||
' -- Consumers --
|
||||
class "session.py\nChatSession" as Session {
|
||||
}
|
||||
|
||||
class "server.py\nWeb UI" as Server {
|
||||
}
|
||||
|
||||
class "cli.py\nTerminal" as CLI {
|
||||
}
|
||||
|
||||
' -- Relationships --
|
||||
SQLite ..|> SB
|
||||
PG ..|> SB
|
||||
|
||||
SQLite --> Schema : uses
|
||||
PG --> Schema : uses
|
||||
|
||||
Registry --> SB : creates
|
||||
Registry --> Migrate : calls
|
||||
|
||||
Migrate --> Versions : applies
|
||||
Migrate --> Schema : references
|
||||
|
||||
Facade --> Registry : get_storage()
|
||||
|
||||
Session --> Facade : imports
|
||||
Server --> Facade : imports
|
||||
CLI --> Facade : imports
|
||||
|
||||
' -- Config --
|
||||
note right of Registry
|
||||
[database]
|
||||
backend = "sqlite" | "postgresql"
|
||||
url = "postgresql+psycopg://..."
|
||||
path = ".turnstone.db"
|
||||
pool_size = 5
|
||||
end note
|
||||
|
||||
note bottom of SQLite
|
||||
Default backend.
|
||||
Zero-config for
|
||||
single-node / dev.
|
||||
end note
|
||||
|
||||
note bottom of PG
|
||||
Production backend.
|
||||
Multi-node / Docker
|
||||
default.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,179 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Authentication Architecture
|
||||
|
||||
skinparam class {
|
||||
BackgroundColor<<core>> #E8EAF6
|
||||
BackgroundColor<<jwt>> #C8E6C9
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<endpoint>> #FFE0B2
|
||||
BackgroundColor<<scope>> #F3E5F5
|
||||
}
|
||||
|
||||
' -- Core Auth --
|
||||
class "AuthConfig" as AC <<core>> {
|
||||
+enabled: bool
|
||||
+tokens: dict[str, str]
|
||||
+check(token) → role | None
|
||||
--
|
||||
Static config-file tokens
|
||||
hmac.compare_digest
|
||||
}
|
||||
|
||||
class "AuthResult" as AR <<core>> {
|
||||
+user_id: str
|
||||
+scopes: frozenset[str]
|
||||
+token_source: str
|
||||
+has_scope(scope) → bool
|
||||
}
|
||||
|
||||
class "check_request()" as CR <<core>> {
|
||||
auth_config, method, path,
|
||||
auth_header, cookie_header,
|
||||
jwt_secret, storage
|
||||
→ (allowed, status, msg, AuthResult)
|
||||
--
|
||||
1. Auth disabled → allow
|
||||
2. Public path → allow
|
||||
3. Extract Bearer / cookie
|
||||
4. Detect token type
|
||||
5. Validate → AuthResult
|
||||
6. Check scope vs path
|
||||
}
|
||||
|
||||
' -- Token Types --
|
||||
class "JWT (HS256)" as JWT <<jwt>> {
|
||||
sub: user_id
|
||||
scopes: "read,write,approve"
|
||||
src: "password" | "database"
|
||||
iat, exp (24h default)
|
||||
--
|
||||
Detected by: contains "."
|
||||
Validated locally
|
||||
No DB call
|
||||
}
|
||||
|
||||
class "API Token" as AT <<jwt>> {
|
||||
Format: ts_ + 64 hex
|
||||
Stored: SHA-256 hash
|
||||
--
|
||||
Detected by: starts with "ts_"
|
||||
Lookup by hash in DB
|
||||
Expiry check
|
||||
}
|
||||
|
||||
class "Config Token" as CT <<core>> {
|
||||
Raw value in memory
|
||||
Role: "read" | "full"
|
||||
--
|
||||
Detected by: fallback
|
||||
hmac.compare_digest
|
||||
No DB needed
|
||||
}
|
||||
|
||||
' -- Scopes --
|
||||
class "Scope Hierarchy" as SH <<scope>> {
|
||||
read: {read}
|
||||
write: {read, write}
|
||||
approve: {read, write, approve}
|
||||
--
|
||||
GET → read
|
||||
POST write paths → write
|
||||
POST /api/approve → approve
|
||||
/api/admin/* → approve
|
||||
}
|
||||
|
||||
' -- Storage --
|
||||
class "users" as UT <<storage>> {
|
||||
user_id (PK)
|
||||
username (unique)
|
||||
display_name
|
||||
password_hash (bcrypt)
|
||||
created
|
||||
}
|
||||
|
||||
class "api_tokens" as TT <<storage>> {
|
||||
token_id (PK)
|
||||
token_hash (SHA-256, unique)
|
||||
token_prefix
|
||||
user_id → users
|
||||
name, scopes
|
||||
created, expires
|
||||
}
|
||||
|
||||
' -- Endpoints --
|
||||
class "POST /api/auth/login" as Login <<endpoint>> {
|
||||
{username, password}
|
||||
OR {token: "ts_xxx"}
|
||||
→ {jwt, role, scopes, user_id}
|
||||
--
|
||||
Sets HttpOnly cookie
|
||||
}
|
||||
|
||||
class "GET /api/auth/status" as Status <<endpoint>> {
|
||||
→ {auth_enabled, has_users,
|
||||
setup_required}
|
||||
--
|
||||
Public (no auth)
|
||||
Drives UI setup wizard
|
||||
}
|
||||
|
||||
class "POST /api/auth/setup" as Setup <<endpoint>> {
|
||||
{username, display_name, password}
|
||||
→ {jwt, user_id, scopes}
|
||||
--
|
||||
Public (no auth)
|
||||
Only when zero users exist
|
||||
Returns 409 if already set up
|
||||
}
|
||||
|
||||
class "Admin API (Console)" as Admin <<endpoint>> {
|
||||
POST/GET/DELETE users
|
||||
POST/GET tokens
|
||||
DELETE tokens/{id}
|
||||
--
|
||||
Requires approve scope
|
||||
}
|
||||
|
||||
' -- Relationships --
|
||||
CR --> AC : config tokens
|
||||
CR --> JWT : validate
|
||||
CR --> AT : hash lookup
|
||||
CR --> CT : hmac check
|
||||
CR --> AR : returns
|
||||
CR --> SH : checks
|
||||
|
||||
Login --> JWT : issues
|
||||
Login --> UT : verify password
|
||||
Login --> TT : verify API token
|
||||
|
||||
Setup --> UT : create first user
|
||||
Setup --> JWT : issues
|
||||
|
||||
AT --> TT : lookup by hash
|
||||
Admin --> UT : CRUD
|
||||
Admin --> TT : CRUD
|
||||
|
||||
AR --> SH : scopes from
|
||||
|
||||
JWT ..> AR : produces
|
||||
AT ..> AR : produces
|
||||
CT ..> AR : produces
|
||||
|
||||
note right of CR
|
||||
**Middleware Flow**
|
||||
AuthMiddleware on every request:
|
||||
1. Extract token from header/cookie
|
||||
2. Detect type (JWT / ts_ / config)
|
||||
3. Validate → AuthResult
|
||||
4. Set ctx_user_id for logging
|
||||
5. Store auth_result in scope state
|
||||
end note
|
||||
|
||||
note bottom of SH
|
||||
**Console** owns admin endpoints
|
||||
**Server** validates JWT + config only
|
||||
Both share JWT signing secret
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,250 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Channel Integration Architecture
|
||||
|
||||
skinparam class {
|
||||
BackgroundColor<<platform>> #E1BEE7
|
||||
BackgroundColor<<service>> #E8EAF6
|
||||
BackgroundColor<<mq>> #FFCDD2
|
||||
BackgroundColor<<bridge>> #C8E6C9
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
}
|
||||
|
||||
' -- External Platforms --
|
||||
class "Discord" as Discord <<platform>> {
|
||||
Gateway WebSocket (v10)
|
||||
Message events
|
||||
Interaction callbacks (buttons)
|
||||
Thread-per-workstream
|
||||
--
|
||||
discord.py 2.x
|
||||
asyncio event loop
|
||||
}
|
||||
|
||||
class "Slack (future)" as Slack <<platform>> {
|
||||
Socket Mode / Events API
|
||||
Block Kit messages
|
||||
--
|
||||
Planned integration
|
||||
}
|
||||
|
||||
class "Teams (future)" as Teams <<platform>> {
|
||||
Bot Framework
|
||||
Adaptive Cards
|
||||
--
|
||||
Planned integration
|
||||
}
|
||||
|
||||
' -- Channel Service --
|
||||
class "turnstone-channel" as ChannelService <<service>> {
|
||||
entry point: turnstone-channel
|
||||
--
|
||||
One process per platform
|
||||
asyncio event loop
|
||||
Structured logging (structlog)
|
||||
--log-level, --log-format
|
||||
--
|
||||
POST /v1/api/notify (HTTP)
|
||||
GET /health
|
||||
}
|
||||
|
||||
class "DiscordBot" as Bot <<service>> {
|
||||
+on_message(msg)
|
||||
+on_interaction(interaction)
|
||||
+send(channel_id, content)
|
||||
+run(token)
|
||||
--
|
||||
discord.py Client
|
||||
Receives message events
|
||||
Sends replies + embeds
|
||||
Creates threads for workstreams
|
||||
Renders approval buttons
|
||||
escape_mentions() on send
|
||||
}
|
||||
|
||||
class "ChannelRouter" as Router <<service>> {
|
||||
+resolve_route(platform, channel_id)
|
||||
→ ws_id | None
|
||||
+register_route(channel_id, ws_id)
|
||||
+resolve_identity(platform, platform_user_id)
|
||||
→ user_id | None
|
||||
--
|
||||
Maps channels → workstreams
|
||||
Maps platform users → turnstone users
|
||||
Caches routes in memory
|
||||
}
|
||||
|
||||
class "AsyncRedisBroker" as Broker <<service>> {
|
||||
+push_inbound(msg)
|
||||
+subscribe(ws_id) → AsyncIterator
|
||||
+subscribe_global() → AsyncIterator
|
||||
+push_response(correlation_id, msg)
|
||||
--
|
||||
redis.asyncio client
|
||||
Pub/sub + queue operations
|
||||
}
|
||||
|
||||
' -- Redis MQ --
|
||||
class "Redis MQ" as Redis <<mq>> {
|
||||
turnstone:inbound (LIST)
|
||||
turnstone:events:{ws_id} (PUBSUB)
|
||||
turnstone:events:global (PUBSUB)
|
||||
turnstone:resp:{corr_id} (LIST)
|
||||
--
|
||||
Shared message bus
|
||||
Same queues as bridge protocol
|
||||
}
|
||||
|
||||
' -- Bridge + Server --
|
||||
class "turnstone-bridge" as Bridge <<bridge>> {
|
||||
BLPOP turnstone:inbound
|
||||
Drive server via HTTP
|
||||
Relay SSE → Redis pub/sub
|
||||
--
|
||||
Owns workstream lifecycle
|
||||
Auto-approve / manual approve
|
||||
}
|
||||
|
||||
class "turnstone-server" as Server <<server>> {
|
||||
POST /v1/api/send
|
||||
POST /v1/api/approve
|
||||
POST /v1/api/workstreams/new
|
||||
GET /v1/api/events?ws_id=
|
||||
--
|
||||
LLM execution + tool use
|
||||
SSE event stream
|
||||
--
|
||||
notify tool: _exec_notify()
|
||||
ServiceTokenManager (JWT)
|
||||
}
|
||||
|
||||
' -- Storage --
|
||||
class "channel_users" as CU <<storage>> {
|
||||
channel_user_id (PK)
|
||||
platform: "discord" | "slack"
|
||||
platform_user_id
|
||||
user_id → users
|
||||
linked_at
|
||||
--
|
||||
/link command creates row
|
||||
Resolved on each inbound message
|
||||
}
|
||||
|
||||
class "channel_routes" as CR <<storage>> {
|
||||
channel_type (PK)
|
||||
channel_id (PK)
|
||||
ws_id
|
||||
node_id
|
||||
created
|
||||
--
|
||||
Maps platform channels
|
||||
to turnstone workstreams
|
||||
}
|
||||
|
||||
class "services" as SVC <<storage>> {
|
||||
service_type (PK)
|
||||
service_id (PK)
|
||||
url
|
||||
last_heartbeat
|
||||
created
|
||||
--
|
||||
Heartbeat every 30s
|
||||
Stale after 120s
|
||||
ON CONFLICT DO UPDATE
|
||||
}
|
||||
|
||||
' -- Relationships --
|
||||
Discord --> Bot : gateway\nevents
|
||||
Bot --> Router : on_message\non_interaction
|
||||
Router --> Broker : SendMessage\nApproveMessage
|
||||
Router --> CU : resolve identity
|
||||
Router --> CR : resolve / register route
|
||||
Broker --> Redis : RPUSH inbound\nRPUSH resp:{id}
|
||||
|
||||
Redis --> Bridge : BLPOP inbound
|
||||
Bridge --> Server : HTTP API
|
||||
Server --> Bridge : SSE events
|
||||
Bridge --> Redis : PUBLISH events:{ws_id}\nPUBLISH events:global
|
||||
|
||||
Redis --> Broker : SUBSCRIBE events:{ws_id}
|
||||
Broker --> Bot : event stream
|
||||
Bot --> Discord : reply / embed\nbutton callback
|
||||
|
||||
Slack .[hidden]. Discord
|
||||
Teams .[hidden]. Slack
|
||||
|
||||
ChannelService --> Bot : creates + runs
|
||||
ChannelService --> Router : creates
|
||||
ChannelService --> Broker : creates
|
||||
ChannelService --> SVC : register / heartbeat /\nderegister
|
||||
|
||||
' -- Notification path (direct HTTP, bypasses MQ) --
|
||||
Server --> ChannelService : POST /v1/api/notify\n(JWT: aud=turnstone-channel)
|
||||
Server --> SVC : list_services("channel",\nmax_age_seconds=120)
|
||||
|
||||
' -- Notes --
|
||||
note right of Bot
|
||||
**Inbound Flow**
|
||||
1. Discord message arrives via gateway
|
||||
2. Bot.on_message() fires
|
||||
3. ChannelRouter resolves channel → ws_id
|
||||
(or creates new workstream)
|
||||
4. ChannelRouter resolves platform user → user_id
|
||||
via channel_users table
|
||||
5. Broker.push_inbound(SendMessage)
|
||||
6. Bridge pops from Redis, drives server
|
||||
|
||||
**Session Resume (evicted workstreams)**
|
||||
1. Stale route detected (no MQ owner)
|
||||
2. Old session looked up via get_session_id_by_ws()
|
||||
3. CreateWorkstreamMessage sent with
|
||||
resume_session=<old_session_id>
|
||||
4. Server resumes atomically during creation
|
||||
5. Bridge emits SessionResumedEvent → thread
|
||||
end note
|
||||
|
||||
note right of Broker
|
||||
**Outbound Flow**
|
||||
1. Server emits SSE events
|
||||
2. Bridge relays to Redis events:{ws_id}
|
||||
3. Broker.subscribe(ws_id) yields events
|
||||
4. Bot formats and sends to Discord thread
|
||||
end note
|
||||
|
||||
note bottom of CR
|
||||
**Approval Flow**
|
||||
1. ApprovalRequestEvent arrives via events:{ws_id}
|
||||
2. Bot renders Discord buttons (Approve / Deny)
|
||||
3. User clicks button → on_interaction()
|
||||
4. Router builds ApproveMessage
|
||||
5. Broker.push_response(correlation_id, msg)
|
||||
6. Bridge pops from resp:{id}, calls POST /api/approve
|
||||
end note
|
||||
|
||||
note bottom of CU
|
||||
**Identity Linking**
|
||||
1. User runs /link in Discord
|
||||
2. Bot opens modal requesting API token
|
||||
3. User submits ts_... API token
|
||||
4. Bot validates token against storage
|
||||
5. On success, inserts channel_users row
|
||||
6. Subsequent messages carry user_id
|
||||
7. AuthResult scopes applied by server
|
||||
end note
|
||||
|
||||
note bottom of SVC
|
||||
**Notification Flow** (direct HTTP, bypasses MQ)
|
||||
1. LLM calls notify tool → _prepare_notify()
|
||||
2. _exec_notify() checks rate limit (5/turn)
|
||||
3. Queries services table for healthy gateways
|
||||
4. Mints JWT (aud: turnstone-channel) via
|
||||
ServiceTokenManager
|
||||
5. POSTs to first healthy gateway
|
||||
6. Gateway validates JWT, resolves target
|
||||
7. adapter.send() → Discord API
|
||||
8. On failure: retry up to 3× (1s, 3s backoff)
|
||||
9. SSRF: only http(s) URLs allowed
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,116 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Notification Delivery Flow
|
||||
|
||||
skinparam participant {
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<service>> #E8EAF6
|
||||
BackgroundColor<<platform>> #E1BEE7
|
||||
}
|
||||
|
||||
participant "ChatSession\n(turnstone-server)" as Session <<server>>
|
||||
participant "StorageBackend" as Storage <<storage>>
|
||||
participant "ServiceTokenManager" as STM <<server>>
|
||||
participant "Channel Gateway\n(_http.py)" as Gateway <<service>>
|
||||
participant "ChannelAdapter\n(Discord bot)" as Adapter <<service>>
|
||||
participant "Discord API" as Discord <<platform>>
|
||||
|
||||
== Prepare Phase ==
|
||||
|
||||
Session -> Session : _prepare_notify(call_id, args)
|
||||
note right
|
||||
Validates:
|
||||
- message (required, ≤2000 chars)
|
||||
- target: username OR channel_type+channel_id
|
||||
- no ambiguous targeting (both set)
|
||||
- partial targeting errors
|
||||
end note
|
||||
|
||||
== Execute Phase ==
|
||||
|
||||
Session -> Session : _exec_notify(item)
|
||||
Session -> Session : check rate limit\n(≥5 per turn?)
|
||||
|
||||
alt rate limit exceeded
|
||||
Session --> Session : "Error: rate limit exceeded"
|
||||
end
|
||||
|
||||
loop up to 3 attempts (retry delays: 1s, 3s)
|
||||
|
||||
Session -> Storage : list_services("channel",\nmax_age_seconds=120)
|
||||
Storage --> Session : services[] (sorted by\nlast_heartbeat DESC)
|
||||
|
||||
alt no healthy services
|
||||
Session -> Session : log.warning("notify.no_services")
|
||||
Session -> Session : sleep(delay)
|
||||
else services available
|
||||
|
||||
Session -> STM : bearer_header
|
||||
note right
|
||||
Lazy-init ServiceTokenManager
|
||||
aud: turnstone-channel
|
||||
scope: write
|
||||
Auto-rotates 1h JWTs
|
||||
end note
|
||||
STM --> Session : Authorization: Bearer <jwt>
|
||||
|
||||
loop for each gateway (first-healthy)
|
||||
Session -> Session : SSRF check:\nurl.startswith("http://"|"https://")
|
||||
|
||||
Session -> Gateway : POST /v1/api/notify\n+ Authorization header
|
||||
Gateway -> Gateway : _check_auth()\nvalidate JWT (aud=turnstone-channel)\nor static token
|
||||
|
||||
alt auth failed
|
||||
Gateway --> Session : 401 Unauthorized
|
||||
else auth ok
|
||||
|
||||
alt username target
|
||||
Gateway -> Storage : get_user_by_username()
|
||||
Storage --> Gateway : user
|
||||
Gateway -> Storage : list_channel_users_by_user()
|
||||
Storage --> Gateway : linked channels
|
||||
else direct target
|
||||
Gateway -> Gateway : use channel_type + channel_id
|
||||
end
|
||||
|
||||
Gateway -> Adapter : send(channel_id, content)
|
||||
note right
|
||||
escape_mentions() applied
|
||||
Chunked for 2000-char limit
|
||||
end note
|
||||
Adapter -> Discord : POST message
|
||||
Discord --> Adapter : message_id
|
||||
Adapter --> Gateway : message_id
|
||||
Gateway --> Session : 200 {results: [{status: "sent"}]}
|
||||
|
||||
Session -> Session : _notify_count += 1
|
||||
Session --> Session : "Notification sent successfully"
|
||||
note right : Return — no further\ngateways tried
|
||||
end
|
||||
end
|
||||
|
||||
alt all gateways failed
|
||||
Session -> Session : log.warning(\n"notify.all_gateways_failed")
|
||||
Session -> Session : sleep(delay)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
alt all retries exhausted
|
||||
Session -> Session : log.warning("notify.delivery_failed")
|
||||
Session --> Session : "Error: notification delivery failed"
|
||||
end
|
||||
|
||||
== Service Registry (Background) ==
|
||||
|
||||
note over Gateway, Storage
|
||||
**Heartbeat Lifecycle**
|
||||
1. Gateway startup: register_service("channel", id, url)
|
||||
2. Every 30s: heartbeat_service("channel", id)
|
||||
3. Shutdown: deregister_service("channel", id)
|
||||
4. Stale after 120s (4 missed heartbeats)
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:341a8ab1483b1e0146878bd384a11d56bc78d29262de8262d06ef924317e2762
|
||||
size 139969
|
||||
oid sha256:9a1b0361c466327d0011a488847ea3c0365983713537d4a7c27cd7f5538ba33c
|
||||
size 164829
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:29534422fc31eee613f70a479aa14de5278b98c49bb75fce7a63b72e248f1149
|
||||
size 323269
|
||||
oid sha256:7d75da92a657525bcbb7a425dc6c8d3cafe074c3ac3bff3cf4b1d44aea607b50
|
||||
size 330156
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f85f24081d32318e079d26a4855ebb6e66df349ca7c7c703db50788af528426b
|
||||
size 376282
|
||||
oid sha256:637458e0d78df82752746e519cd7300a830c8ce211f21625694ad0c162ca316d
|
||||
size 481637
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a90dec1546dd0f8343e3e27cf6f235d95f3ffc375928d34d0df5d8f25d63e5ad
|
||||
size 269901
|
||||
oid sha256:dc3b64c9e48153641af62ed43fbc1d89a31d1a8a61e7e71cfc550c805000310d
|
||||
size 288290
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cb36b4924394cf54d6aaef454cced317b72e336e580cc6ac25cd4b9d0917bec5
|
||||
size 243422
|
||||
oid sha256:b842683d238664a3e35d04358fecfc56cefd013f7dca5f13357b0376f881e1b3
|
||||
size 245043
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e660f453a3708d1a7d827f07cd967f122500eb1c4845f5a75cc1309091bce6af
|
||||
size 185796
|
||||
oid sha256:b22d5980fe5cc4b8466ba0797113dc8fa83fab8df24b5dacceaf97e62e2e25b0
|
||||
size 187649
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8cc7c94d5ac4862c3c09450346f0af923818e02ea0550cc8023f039fdc701179
|
||||
size 221528
|
||||
oid sha256:90e4f74be795b530e711faa87bc6eb2b3bf6abb68d8fac8ebff7aaf30c6fbe53
|
||||
size 222032
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:793d7c2b28a751c6f467f2de788fcd462d3b8fd9cd5cb7adb5b32d78fb185394
|
||||
size 236004
|
||||
oid sha256:97e7210cd8f1ad195f4d5e25e778d82df3c08c5c6e0f09722e84a7a453714867
|
||||
size 411664
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0e6c1dfaef840d5228645aaad3637c973b2f71c372595814f3b743a991f5c6fc
|
||||
size 239128
|
||||
oid sha256:4c3214ef416c1dfe4fa17834c2b6f4071a8093cfdb2b862848ca79938f726a13
|
||||
size 252599
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c9823a41e09611c5c0530d9fc12ad4139cfcc3ae238dc665b2888ec94d7d6781
|
||||
size 195708
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:733aa17cbfdab60a601cac6adf439c657dd3535e3d6c33c69c2ef93ba8ec5989
|
||||
size 251042
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f4b2a2010335f986511c8dabaf49ec046ac02e577f9bc9924897e045f860bb13
|
||||
size 248808
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6049cc0b07480df88d0d93aa977a1e97f64b41588325ff41d98be0e39431fc5c
|
||||
size 431712
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f55e177e0838a16d9bc4f07b162b4b6a966cc596c9d0a022d35a3c84f23e7b02
|
||||
size 221452
|
||||
+38
-4
@@ -27,6 +27,7 @@ Console dashboard: http://localhost:8090
|
||||
| `server` | 8080 | default | Web UI + chat workstreams + LLM |
|
||||
| `bridge` | — | default | Redis-to-HTTP bridge (multi-node routing) |
|
||||
| `console` | 8090 | default | Cluster dashboard |
|
||||
| `channel` | — | production | Channel gateway (Discord, Slack, etc.) |
|
||||
| `sim` | — | sim | Multi-node cluster simulator |
|
||||
|
||||
## Profiles
|
||||
@@ -37,6 +38,12 @@ Console dashboard: http://localhost:8090
|
||||
docker compose up
|
||||
```
|
||||
|
||||
**Production** — adds PostgreSQL and the channel gateway. Requires `POSTGRES_PASSWORD` and (for Discord) `TURNSTONE_DISCORD_TOKEN`:
|
||||
|
||||
```bash
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
**Sim** — adds the simulator. Can run alongside the full stack or standalone with just Redis and the console:
|
||||
|
||||
```bash
|
||||
@@ -57,7 +64,7 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
|----------|---------|-------------|
|
||||
| `LLM_BASE_URL` | `http://host.docker.internal:8000/v1` | OpenAI-compatible API URL |
|
||||
| `OPENAI_API_KEY` | `dummy` | API key (`dummy` for local servers) |
|
||||
| `TAVILY_API_KEY` | — | Web search API key (optional) |
|
||||
| `TAVILY_API_KEY` | — | Web search API key (only needed for local/vLLM models; Anthropic and OpenAI search models use native search) |
|
||||
|
||||
### Redis
|
||||
|
||||
@@ -84,8 +91,35 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_AUTH_ENABLED` | — | Set to `1` to require Bearer token auth |
|
||||
| `TURNSTONE_AUTH_TOKEN` | — | Shared auth token for server/bridge/console |
|
||||
| `TURNSTONE_AUTH_ENABLED` | — | Set to `1` to require authentication |
|
||||
| `TURNSTONE_AUTH_TOKEN` | — | Config-file token for server/bridge/console (backward compat, works alongside JWT) |
|
||||
| `TURNSTONE_JWT_SECRET` | — | Secret key for signing JWTs (required when using user identity / JWT auth) |
|
||||
|
||||
### Database
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DB_BACKEND` | `sqlite` | Storage backend: `sqlite` or `postgresql` |
|
||||
| `TURNSTONE_DB_URL` | — | Database URL (e.g. `postgresql://user:pass@db:5432/turnstone`). For SQLite, defaults to `/data/.turnstone.db` |
|
||||
|
||||
The database stores workstream history, user accounts, and API tokens. When using JWT auth, a database backend is required for user storage.
|
||||
|
||||
> **First-time setup:** After deploying with auth enabled, create an initial admin user by running `turnstone-admin create-user` inside the container:
|
||||
>
|
||||
> ```bash
|
||||
> docker compose exec server turnstone-admin create-user --username admin --name "Admin"
|
||||
> ```
|
||||
>
|
||||
> You will be prompted to set a password. Use it to log in via the UI or SDK, then create additional users through the admin API. Pass `--token --scopes read,write,approve` to also generate an initial API token.
|
||||
|
||||
### Channel Gateway
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord adapter) |
|
||||
| `TURNSTONE_DISCORD_GUILD` | `0` | Restrict to a single Discord guild (0 = all guilds) |
|
||||
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages through Redis MQ to the bridge and server. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
### Simulator
|
||||
|
||||
@@ -128,7 +162,7 @@ docker compose build
|
||||
docker compose build --no-cache
|
||||
```
|
||||
|
||||
All five entry points are installed in a single image: `turnstone-server`, `turnstone-bridge`, `turnstone-console`, `turnstone-sim`, `turnstone-eval`.
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-bridge`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-sim`, `turnstone-eval`.
|
||||
|
||||
## Cleanup
|
||||
|
||||
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
# Turnstone Client SDK
|
||||
|
||||
> See also: [API Reference](api-reference.md) | [Architecture](architecture.md) | [SDK Class Diagram](diagrams/png/13-sdk-architecture.png)
|
||||
|
||||
Typed HTTP client libraries for programmatic access to the turnstone server and console APIs. Available in Python (sync + async) and TypeScript.
|
||||
|
||||
---
|
||||
|
||||
## Python SDK
|
||||
|
||||
The Python SDK is included in the `turnstone` package — no extra install required. It wraps the REST and SSE endpoints with typed methods that return Pydantic models directly.
|
||||
|
||||
### Quick Start
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
# Synchronous client — login with username/password
|
||||
with TurnstoneServer("http://localhost:8080") as client:
|
||||
client.login(username="alice", password="s3cret")
|
||||
|
||||
# Create a workstream
|
||||
ws = client.create_workstream(name="Analysis")
|
||||
|
||||
# Send a message and wait for the full response
|
||||
result = client.send_and_wait("Summarize this codebase.", ws.ws_id)
|
||||
print(result.content)
|
||||
|
||||
# Stream events in real time
|
||||
for event in client.stream_events(ws.ws_id):
|
||||
if event.type == "content":
|
||||
print(event.text, end="", flush=True)
|
||||
|
||||
# Close when done
|
||||
client.close_workstream(ws.ws_id)
|
||||
```
|
||||
|
||||
Alternatively, authenticate with an API token:
|
||||
|
||||
```python
|
||||
with TurnstoneServer("http://localhost:8080") as client:
|
||||
client.login(token="ts_abc123...")
|
||||
ws = client.create_workstream(name="CI run")
|
||||
result = client.send_and_wait("Run the test suite.", ws.ws_id)
|
||||
```
|
||||
|
||||
### Async Client
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from turnstone.sdk import AsyncTurnstoneServer
|
||||
|
||||
async def main():
|
||||
async with AsyncTurnstoneServer("http://localhost:8080") as client:
|
||||
await client.login(username="alice", password="s3cret")
|
||||
ws = await client.create_workstream(name="demo")
|
||||
async for event in client.stream_events(ws.ws_id):
|
||||
if event.type == "content":
|
||||
print(event.text, end="", flush=True)
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
### Server Client API
|
||||
|
||||
Both `TurnstoneServer` (sync) and `AsyncTurnstoneServer` (async) expose:
|
||||
|
||||
| Category | Method | Returns |
|
||||
|----------|--------|---------|
|
||||
| **Workstreams** | `list_workstreams()` | `ListWorkstreamsResponse` |
|
||||
| | `dashboard()` | `DashboardResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve)` | `CreateWorkstreamResponse` |
|
||||
| | `close_workstream(ws_id)` | `StatusResponse` |
|
||||
| **Chat** | `send(message, ws_id)` | `SendResponse` |
|
||||
| | `approve(*, ws_id, approved, feedback, always)` | `StatusResponse` |
|
||||
| | `plan_feedback(*, ws_id, feedback)` | `StatusResponse` |
|
||||
| | `command(*, ws_id, command)` | `StatusResponse` |
|
||||
| **Streaming** | `stream_events(ws_id)` | `Iterator[ServerEvent]` |
|
||||
| | `stream_global_events()` | `Iterator[ServerEvent]` |
|
||||
| **High-level** | `send_and_wait(message, ws_id, *, timeout, on_event)` | `TurnResult` |
|
||||
| **Sessions** | `list_sessions()` | `ListSessionsResponse` |
|
||||
| **Auth** | `login(username=..., password=...)` | `AuthLoginResponse` |
|
||||
| | `login(token="ts_xxx")` | `AuthLoginResponse` |
|
||||
| | `logout()` | `StatusResponse` |
|
||||
| | `auth_status()` | `AuthStatusResponse` |
|
||||
| **Health** | `health()` | `HealthResponse` |
|
||||
|
||||
### Console Client API
|
||||
|
||||
Both `TurnstoneConsole` (sync) and `AsyncTurnstoneConsole` (async) expose:
|
||||
|
||||
| Category | Method | Returns |
|
||||
|----------|--------|---------|
|
||||
| **Cluster** | `overview()` | `ClusterOverviewResponse` |
|
||||
| | `nodes(*, sort, limit, offset)` | `ClusterNodesResponse` |
|
||||
| | `workstreams(*, state, node, search, sort, page, per_page)` | `ClusterWorkstreamsResponse` |
|
||||
| | `node_detail(node_id)` | `NodeDetailResponse` |
|
||||
| | `create_workstream(*, node_id, name, model, initial_message)` | `ConsoleCreateWsResponse` |
|
||||
| **Schedules** | `list_schedules()` | `ListSchedulesResponse` |
|
||||
| | `create_schedule(*, name, schedule_type, initial_message, ...)` | `ScheduleInfo` |
|
||||
| | `get_schedule(task_id)` | `ScheduleInfo` |
|
||||
| | `update_schedule(task_id, *, name=..., enabled=..., ...)` | `ScheduleInfo` |
|
||||
| | `delete_schedule(task_id)` | `StatusResponse` |
|
||||
| | `list_schedule_runs(task_id, *, limit=50)` | `ListScheduleRunsResponse` |
|
||||
| **Streaming** | `stream_cluster_events()` | `Iterator[ClusterEvent]` |
|
||||
| **Auth** | `login(username=..., password=...)` / `login(token="ts_xxx")` | `AuthLoginResponse` |
|
||||
| | `logout()` | `StatusResponse` |
|
||||
| **Health** | `health()` | `ConsoleHealthResponse` |
|
||||
|
||||
### Event Types
|
||||
|
||||
SSE events are deserialized into typed dataclasses. Use `event.type` to discriminate.
|
||||
|
||||
**Per-workstream events** (from `stream_events(ws_id)`):
|
||||
|
||||
| Type | Class | Key Fields |
|
||||
|------|-------|------------|
|
||||
| `connected` | `ConnectedEvent` | `model`, `model_alias`, `skip_permissions` |
|
||||
| `history` | `HistoryEvent` | `messages` |
|
||||
| `content` | `ContentEvent` | `text` |
|
||||
| `reasoning` | `ReasoningEvent` | `text` |
|
||||
| `tool_info` | `ToolInfoEvent` | `items` |
|
||||
| `approve_request` | `ApproveRequestEvent` | `items` |
|
||||
| `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output` |
|
||||
| `tool_output_chunk` | `ToolOutputChunkEvent` | `call_id`, `chunk` |
|
||||
| `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort` |
|
||||
| `plan_review` | `PlanReviewEvent` | `content` |
|
||||
| `error` | `ErrorEvent` | `message` |
|
||||
| `info` | `InfoEvent` | `message` |
|
||||
| `stream_end` | `StreamEndEvent` | — |
|
||||
|
||||
**Global events** (from `stream_global_events()`):
|
||||
|
||||
| Type | Class | Key Fields |
|
||||
|------|-------|------------|
|
||||
| `ws_state` | `WsStateEvent` | `ws_id`, `state`, `tokens`, `activity` |
|
||||
| `ws_activity` | `WsActivityEvent` | `ws_id`, `activity`, `activity_state` |
|
||||
| `ws_rename` | `WsRenameEvent` | `ws_id`, `name` |
|
||||
| `ws_closed` | `WsClosedEvent` | `ws_id` |
|
||||
|
||||
**Cluster events** (from `stream_cluster_events()`):
|
||||
|
||||
| Type | Class | Key Fields |
|
||||
|------|-------|------------|
|
||||
| `node_joined` | `NodeJoinedEvent` | `node_id` |
|
||||
| `node_lost` | `NodeLostEvent` | `node_id` |
|
||||
| `cluster_state` | `ClusterStateEvent` | `ws_id`, `node_id`, `state`, `tokens` |
|
||||
| `ws_created` | `ClusterWsCreatedEvent` | `ws_id`, `node_id`, `name` |
|
||||
|
||||
### TurnResult
|
||||
|
||||
The `send_and_wait()` method returns a `TurnResult` that aggregates the full response:
|
||||
|
||||
```python
|
||||
result = client.send_and_wait("Hello", ws_id, timeout=60)
|
||||
result.content # Full text response
|
||||
result.reasoning # Chain-of-thought (if shown)
|
||||
result.tool_results # List of (tool_name, output) tuples
|
||||
result.errors # Any error messages
|
||||
result.ok # True if no errors and not timed out
|
||||
result.timed_out # True if timeout expired
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
Non-2xx responses raise `TurnstoneAPIError`:
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer, TurnstoneAPIError
|
||||
|
||||
try:
|
||||
client.send("hi", "bad_ws_id")
|
||||
except TurnstoneAPIError as e:
|
||||
print(e.status_code) # 404
|
||||
print(e.message) # "Unknown workstream"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## TypeScript SDK
|
||||
|
||||
Located at `sdk/typescript/`. Zero runtime dependencies for browsers; uses native `fetch` and `ReadableStream` for SSE parsing.
|
||||
|
||||
### Quick Start
|
||||
|
||||
```typescript
|
||||
import { TurnstoneServer } from "@turnstone/sdk";
|
||||
|
||||
const client = new TurnstoneServer({ baseUrl: "http://localhost:8080" });
|
||||
|
||||
// Login with username/password or API token
|
||||
await client.login({ username: "alice", password: "s3cret" });
|
||||
// or: await client.login({ token: "ts_abc123..." });
|
||||
|
||||
// Create workstream and send message
|
||||
const ws = await client.createWorkstream({ name: "demo" });
|
||||
const result = await client.sendAndWait("Hello!", ws.ws_id);
|
||||
console.log(result.content);
|
||||
|
||||
// Stream events
|
||||
for await (const event of client.streamEvents(ws.ws_id)) {
|
||||
if (event.type === "content") {
|
||||
process.stdout.write(event.text);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Console Client
|
||||
|
||||
```typescript
|
||||
import { TurnstoneConsole } from "@turnstone/sdk";
|
||||
|
||||
const client = new TurnstoneConsole({ baseUrl: "http://localhost:8090" });
|
||||
await client.login({ username: "alice", password: "s3cret" });
|
||||
|
||||
const overview = await client.overview();
|
||||
console.log(`Nodes: ${overview.nodes}, Workstreams: ${overview.workstreams}`);
|
||||
|
||||
// Stream cluster events
|
||||
for await (const event of client.clusterEvents()) {
|
||||
console.log(event.type, event);
|
||||
}
|
||||
```
|
||||
|
||||
### Type Safety
|
||||
|
||||
All event types are modeled as a discriminated union:
|
||||
|
||||
```typescript
|
||||
import { isContentEvent, isErrorEvent } from "@turnstone/sdk";
|
||||
import type { ServerEvent } from "@turnstone/sdk";
|
||||
|
||||
function handleEvent(event: ServerEvent) {
|
||||
if (isContentEvent(event)) {
|
||||
// event is narrowed to ContentEvent
|
||||
console.log(event.text);
|
||||
} else if (isErrorEvent(event)) {
|
||||
console.error(event.message);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Custom Fetch
|
||||
|
||||
The client accepts a custom `fetch` implementation for testing or Node.js environments:
|
||||
|
||||
```typescript
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://localhost:8080",
|
||||
fetch: myCustomFetch,
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
turnstone/sdk/ Python SDK (sub-package)
|
||||
_base.py Shared httpx async client, auth, error handling
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
events.py 27 SSE event dataclasses with type registry
|
||||
server.py AsyncTurnstoneServer + TurnstoneServer
|
||||
console.py AsyncTurnstoneConsole + TurnstoneConsole
|
||||
|
||||
sdk/typescript/ TypeScript SDK (npm package)
|
||||
src/base.ts fetch wrapper, auth, SSE streaming
|
||||
src/server.ts TurnstoneServer class
|
||||
src/console.ts TurnstoneConsole class
|
||||
src/events.ts Discriminated union events + type guards
|
||||
src/sse.ts ReadableStream SSE parser
|
||||
src/types.ts Request/response interfaces
|
||||
```
|
||||
|
||||
The Python SDK reuses Pydantic models from `turnstone/api/` directly — no schema duplication. The TypeScript SDK has hand-written interfaces matching those models.
|
||||
|
||||
Both SDKs follow the same design: typed methods for REST endpoints, async iterators for SSE streams, and a high-level `send_and_wait` method for simple request-response patterns.
|
||||
|
||||
---
|
||||
|
||||
## Authentication
|
||||
|
||||
When auth is enabled on the server, the SDK handles JWT-based authentication automatically.
|
||||
|
||||
### Login Flow
|
||||
|
||||
There are two ways to authenticate:
|
||||
|
||||
1. **Username + password** — calls `POST /v1/api/auth/login` with credentials. The server validates against the user database and returns a JWT.
|
||||
|
||||
2. **API token** — calls `POST /v1/api/auth/login` with a `ts_`-prefixed token string. The server looks up the token, resolves the associated user, and returns a JWT.
|
||||
|
||||
In both cases the server returns the JWT in the response body and as a `Set-Cookie` header. The SDK extracts the JWT and includes it as a `Bearer` token in the `Authorization` header on all subsequent requests.
|
||||
|
||||
```python
|
||||
# Username + password
|
||||
client.login(username="alice", password="s3cret")
|
||||
|
||||
# API token (created via admin API or turnstone-admin CLI)
|
||||
client.login(token="ts_abc123...")
|
||||
```
|
||||
|
||||
### Token Lifecycle
|
||||
|
||||
- JWTs have a configurable expiry (default: 24 hours).
|
||||
- `client.auth_status()` returns the current user identity and scopes without refreshing the token.
|
||||
- `client.logout()` clears the stored JWT from the client.
|
||||
- If a request returns 401, the SDK raises `TurnstoneAPIError` — the caller is responsible for re-authenticating.
|
||||
|
||||
### Backward Compatibility
|
||||
|
||||
The config-file token (`TURNSTONE_AUTH_TOKEN`) still works as a simple Bearer token for environments that do not use the user/JWT system. When the server receives a non-JWT Bearer token, it falls back to the legacy token check.
|
||||
@@ -0,0 +1,457 @@
|
||||
# Security and Authentication
|
||||
|
||||
Turnstone uses a layered authentication system with three token types,
|
||||
hierarchical scopes, and a split architecture where the console manages
|
||||
credentials while individual server nodes validate JWTs locally.
|
||||
|
||||
---
|
||||
|
||||
## Token Types
|
||||
|
||||
### Config-file tokens
|
||||
|
||||
Static tokens defined in `config.toml` or the `TURNSTONE_AUTH_TOKEN`
|
||||
environment variable. Validated in-memory using `hmac.compare_digest`
|
||||
(timing-safe). Each token maps to a role that determines its scopes.
|
||||
|
||||
```toml
|
||||
[[auth.tokens]]
|
||||
value = "tok_legacy"
|
||||
role = "full" # full → {read, write, approve}
|
||||
```
|
||||
|
||||
Role mappings: `"read"` → `{read}`, `"full"` → `{read, write, approve}`.
|
||||
|
||||
Config tokens are sent directly as `Authorization: Bearer tok_legacy`
|
||||
on every request. No JWT exchange is needed.
|
||||
|
||||
### API tokens
|
||||
|
||||
Database-backed tokens prefixed with `ts_`. Created via the admin CLI
|
||||
(`turnstone-admin create-token`) or the console admin API. Stored as
|
||||
SHA-256 hashes — the raw token is shown exactly once at creation and
|
||||
never persisted in plaintext.
|
||||
|
||||
```
|
||||
$ turnstone-admin create-token --user abc123 --scopes read,write --name "CI bot"
|
||||
Token created: ts_a1b2c3d4e5f6...
|
||||
(save this — it will not be shown again)
|
||||
```
|
||||
|
||||
API tokens can be used directly as `Bearer ts_xxx` headers or exchanged
|
||||
for a JWT via the login endpoint.
|
||||
|
||||
### JWTs
|
||||
|
||||
Short-lived session tokens (24 hours by default). Issued after
|
||||
authenticating with username/password or by exchanging an API token.
|
||||
HS256-signed with a shared secret. Validated locally on every service
|
||||
node — no database call per request.
|
||||
|
||||
Claims:
|
||||
|
||||
| Claim | Description |
|
||||
|-------|-------------|
|
||||
| `sub` | User ID |
|
||||
| `scopes` | Comma-separated scope list (`read,write,approve`) |
|
||||
| `src` | Token source (`password`, `api_token`, `config`) |
|
||||
| `iss` | Issuer — always `turnstone` |
|
||||
| `aud` | Audience — `turnstone-server` or `turnstone-console` |
|
||||
| `iat` | Issued-at timestamp |
|
||||
| `exp` | Expiry timestamp |
|
||||
|
||||
The `aud` claim prevents cross-service token reuse — a JWT issued for the
|
||||
console cannot be used to authenticate against a server node, and vice versa.
|
||||
Tokens without an `aud` claim are accepted during the rollout window when
|
||||
`audience` validation is not specified.
|
||||
|
||||
---
|
||||
|
||||
## Scope Model
|
||||
|
||||
Scopes are hierarchical — higher scopes imply all lower ones.
|
||||
|
||||
| Scope | Grants | Implies |
|
||||
|-------|--------|---------|
|
||||
| `read` | View workstreams, sessions, history | — |
|
||||
| `write` | Send messages, create/close workstreams | `read` |
|
||||
| `approve` | Approve tool calls, admin endpoints | `read`, `write` |
|
||||
|
||||
### Path-to-scope mapping
|
||||
|
||||
| Method | Path pattern | Required scope |
|
||||
|--------|-------------|----------------|
|
||||
| GET | Any protected path | `read` |
|
||||
| POST | `/api/send`, `/api/plan`, `/api/command` | `write` |
|
||||
| POST | `/api/workstreams/new`, `/api/workstreams/close` | `write` |
|
||||
| POST | `/api/cluster/workstreams/new` | `write` |
|
||||
| POST | `/api/approve` | `approve` |
|
||||
| Any | `/api/admin/*` | `approve` |
|
||||
|
||||
Public paths bypass authentication entirely: `/`, `/health`, `/metrics`,
|
||||
`/static/*`, `/shared/*`, `/docs`, `/openapi.json`, `/api/auth/login`,
|
||||
`/api/auth/logout`, `/api/auth/status`, `/api/auth/setup`.
|
||||
|
||||
---
|
||||
|
||||
## Login Flows
|
||||
|
||||
### Username and password
|
||||
|
||||
```
|
||||
POST /v1/api/auth/login
|
||||
Content-Type: application/json
|
||||
|
||||
{"username": "admin", "password": "s3cret"}
|
||||
```
|
||||
|
||||
Returns a JWT in the response body and sets an `HttpOnly` session cookie.
|
||||
|
||||
### API token exchange
|
||||
|
||||
```
|
||||
POST /v1/api/auth/login
|
||||
Content-Type: application/json
|
||||
|
||||
{"token": "ts_a1b2c3d4e5f6..."}
|
||||
```
|
||||
|
||||
The API token is hashed, looked up in the database, and exchanged for a
|
||||
JWT with the token's scopes. This is the recommended flow for SDKs and
|
||||
automated clients that need cookie-based sessions.
|
||||
|
||||
### Config-file tokens (direct)
|
||||
|
||||
Config tokens are validated per-request via `hmac.compare_digest`. No
|
||||
login exchange is needed — include the token as a `Bearer` header:
|
||||
|
||||
```
|
||||
Authorization: Bearer tok_legacy
|
||||
```
|
||||
|
||||
### First-time setup
|
||||
|
||||
When no users exist in the database:
|
||||
|
||||
1. `GET /v1/api/auth/status` returns `{"setup_required": true}`
|
||||
2. The UI presents a setup wizard
|
||||
3. `POST /v1/api/auth/setup` creates the first admin user and returns a
|
||||
JWT in one atomic step (no auth required — this is a public endpoint)
|
||||
4. The endpoint returns `409 Conflict` if setup has already been completed
|
||||
(i.e. users already exist in the database)
|
||||
5. Subsequent admin requests require `approve` scope
|
||||
|
||||
The `/api/auth/setup` endpoint is available on both the server and
|
||||
console. It validates input before creating the user:
|
||||
|
||||
- **username**: 1-64 ASCII characters
|
||||
- **display_name**: required (non-empty)
|
||||
- **password**: minimum 8 characters
|
||||
|
||||
```
|
||||
POST /v1/api/auth/setup
|
||||
Content-Type: application/json
|
||||
|
||||
{"username": "admin", "display_name": "Admin", "password": "strongpass"}
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"user_id": "u_abc123",
|
||||
"username": "admin",
|
||||
"role": "full",
|
||||
"scopes": "approve,read,write",
|
||||
"jwt": "eyJhbGciOiJIUzI1NiIs..."
|
||||
}
|
||||
```
|
||||
|
||||
The response also sets an `HttpOnly` session cookie containing the JWT,
|
||||
so the browser is immediately authenticated after setup completes.
|
||||
|
||||
---
|
||||
|
||||
## Token Detection Order
|
||||
|
||||
The auth middleware inspects the `Authorization: Bearer <token>` header
|
||||
and classifies the token:
|
||||
|
||||
1. **Contains `.`** → JWT → validate HS256 signature and expiry
|
||||
2. **Starts with `ts_`** → API token → SHA-256 hash, database lookup
|
||||
3. **Otherwise** → config-file token → `hmac.compare_digest` against
|
||||
each configured token
|
||||
|
||||
If a session cookie is present and no `Authorization` header is sent,
|
||||
the cookie value is treated as a JWT (step 1).
|
||||
|
||||
---
|
||||
|
||||
## Password Storage
|
||||
|
||||
Passwords are hashed with **bcrypt** using a random salt per password.
|
||||
Plaintext passwords are only accepted over HTTPS in production
|
||||
deployments.
|
||||
|
||||
---
|
||||
|
||||
## Cookie Security
|
||||
|
||||
| Attribute | Value | Purpose |
|
||||
|-----------|-------|---------|
|
||||
| `HttpOnly` | `true` | Prevents JavaScript access |
|
||||
| `SameSite` | `Lax` | CSRF protection |
|
||||
| `Path` | `/` | Available to all routes |
|
||||
| `Max-Age` | 24 hours | Matches JWT expiry |
|
||||
| `Secure` | `true` (default) | Always set unless explicitly disabled for dev |
|
||||
|
||||
---
|
||||
|
||||
## JWT Configuration
|
||||
|
||||
| Setting | Config key | Env var | Default |
|
||||
|---------|-----------|---------|---------|
|
||||
| Signing secret | `[auth] jwt_secret` | `TURNSTONE_JWT_SECRET` | Auto-generated ephemeral (warning logged) |
|
||||
| Expiry | `[auth] jwt_expiry_hours` | — | 24 hours |
|
||||
| Algorithm | — | — | HS256 (not configurable) |
|
||||
| Minimum secret length | — | — | 32 characters (warning if shorter) |
|
||||
|
||||
All service nodes that need to validate JWTs must share the same signing
|
||||
secret. If no secret is configured, an ephemeral key is generated at
|
||||
startup and a warning is logged — JWTs will not survive restarts or work
|
||||
across nodes.
|
||||
|
||||
The bridge and console **require** `TURNSTONE_JWT_SECRET` when no
|
||||
`--auth-token` is provided. They exit with an error if the secret is
|
||||
missing, since ephemeral secrets would silently break inter-service
|
||||
communication.
|
||||
|
||||
---
|
||||
|
||||
## Admin API Endpoints
|
||||
|
||||
All admin endpoints require `approve` scope.
|
||||
|
||||
### Users
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| POST | `/v1/api/admin/users` | Create user (username, display_name, password) |
|
||||
| GET | `/v1/api/admin/users` | List all users |
|
||||
| DELETE | `/v1/api/admin/users/{user_id}` | Delete user and cascade tokens |
|
||||
|
||||
### API tokens
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| POST | `/v1/api/admin/users/{user_id}/tokens` | Create API token (returns raw value once) |
|
||||
| GET | `/v1/api/admin/users/{user_id}/tokens` | List tokens (prefix only, no hashes) |
|
||||
| DELETE | `/v1/api/admin/tokens/{token_id}` | Revoke token |
|
||||
|
||||
---
|
||||
|
||||
## CLI Administration
|
||||
|
||||
The `turnstone-admin` command provides offline user and token management:
|
||||
|
||||
```
|
||||
turnstone-admin create-user --username admin --name "Admin" [--password] [--token]
|
||||
turnstone-admin create-token --user <user_id> --scopes read,write --name "CI bot"
|
||||
turnstone-admin list-users
|
||||
turnstone-admin list-tokens
|
||||
turnstone-admin revoke-token <token_id>
|
||||
```
|
||||
|
||||
When `--password` is omitted, the CLI prompts interactively. When
|
||||
`--token` is passed to `create-user`, an API token is created alongside
|
||||
the user and printed to stdout.
|
||||
|
||||
---
|
||||
|
||||
## Database Schema
|
||||
|
||||
```sql
|
||||
CREATE TABLE users (
|
||||
user_id TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE api_tokens (
|
||||
token_id TEXT PRIMARY KEY,
|
||||
token_hash TEXT NOT NULL, -- SHA-256 of raw token
|
||||
token_prefix TEXT NOT NULL, -- first 8 chars for display
|
||||
user_id TEXT NOT NULL REFERENCES users(user_id),
|
||||
name TEXT NOT NULL,
|
||||
scopes TEXT NOT NULL, -- comma-separated
|
||||
created TEXT NOT NULL,
|
||||
expires TEXT -- nullable, ISO 8601
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX ix_api_tokens_hash ON api_tokens(token_hash);
|
||||
|
||||
CREATE TABLE channel_users (
|
||||
channel_type TEXT NOT NULL,
|
||||
channel_user_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL REFERENCES users(user_id),
|
||||
created TEXT NOT NULL,
|
||||
PRIMARY KEY (channel_type, channel_user_id)
|
||||
);
|
||||
```
|
||||
|
||||
The `sessions` and `workstreams` tables have a nullable `user_id`
|
||||
column for attribution when auth is enabled.
|
||||
|
||||
---
|
||||
|
||||
## Revocation
|
||||
|
||||
- **API tokens**: Deleting a token via the admin API or CLI prevents new
|
||||
JWTs from being issued with that token. Existing JWTs derived from the
|
||||
token remain valid until they expire (at most 24 hours).
|
||||
- **Config-file tokens**: Remove the token from `config.toml` and
|
||||
restart the service. No JWTs are involved, so revocation is immediate.
|
||||
- **JWTs**: Cannot be individually revoked. Rely on short expiry (24h)
|
||||
and revoke the underlying credential to prevent renewal.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Console (cluster-wide) Server (per-node)
|
||||
┌──────────────────────┐ ┌──────────────────────┐
|
||||
│ User/Token CRUD (DB) │ │ JWT validation only │
|
||||
│ Login: creds → JWT │ │ (shared signing key) │
|
||||
│ Admin API endpoints │ │ Config tokens: hmac │
|
||||
│ Storage: users, │ │ No auth DB needed │
|
||||
│ api_tokens tables │ │ │
|
||||
└──────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
The console owns the credential database and handles all user/token
|
||||
CRUD. Individual server nodes only need the JWT signing secret to
|
||||
validate session tokens. Config-file tokens are validated locally
|
||||
without any database.
|
||||
|
||||
### Proxy auth forwarding
|
||||
|
||||
When the console proxies requests to server nodes (via `/node/{id}/...`
|
||||
routes), it uses a dedicated **service proxy token** with
|
||||
`aud: turnstone-server` and `write` scope. The user's console JWT
|
||||
(which has `aud: turnstone-console`) is **not** forwarded — it would be
|
||||
rejected by the server's audience validation.
|
||||
|
||||
The proxy token is managed by a `ServiceTokenManager` that auto-rotates
|
||||
1-hour JWTs, refreshing at 80% of lifetime. If `--auth-token` is
|
||||
provided, that static token is used instead.
|
||||
|
||||
### Service-to-service authentication
|
||||
|
||||
The bridge and console collector use `ServiceTokenManager` for
|
||||
auto-rotating JWTs when communicating with server nodes:
|
||||
|
||||
| Service | Identity | Scope | Audience | Purpose |
|
||||
|---------|----------|-------|----------|---------|
|
||||
| Bridge | `bridge` | `approve` | `turnstone-server` | Tool approval proxy, message relay |
|
||||
| Console collector | `console-collector` | `read` | `turnstone-server` | Node health polling |
|
||||
| Console proxy | `console-proxy` | `write` | `turnstone-server` | Proxied API calls |
|
||||
| Channel notify | `system` | `write` | `turnstone-channel` | Notification delivery to channel gateway |
|
||||
|
||||
Service tokens use 1-hour expiry with automatic refresh via
|
||||
`ServiceTokenManager`. The bridge injects auth headers per-request via
|
||||
httpx event hooks to ensure rotated tokens are picked up on SSE
|
||||
reconnects.
|
||||
|
||||
Note that the channel gateway uses a distinct JWT audience
|
||||
(`turnstone-channel`) from the server (`turnstone-server`) and console
|
||||
(`turnstone-console`). A server-scoped JWT cannot authenticate to the
|
||||
channel gateway endpoint, and vice versa.
|
||||
|
||||
---
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### config.toml
|
||||
|
||||
```toml
|
||||
[auth]
|
||||
enabled = true
|
||||
jwt_secret = "your-secret-key-here"
|
||||
jwt_expiry_hours = 24
|
||||
|
||||
[[auth.tokens]]
|
||||
value = "tok_legacy"
|
||||
role = "full"
|
||||
```
|
||||
|
||||
### Environment variables
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `TURNSTONE_AUTH_ENABLED=1` | Enable authentication |
|
||||
| `TURNSTONE_AUTH_TOKEN=tok_xxx` | Register a config-file token with `full` access |
|
||||
| `TURNSTONE_JWT_SECRET=xxx` | JWT signing secret (must match across nodes) |
|
||||
| `TURNSTONE_CORS_ORIGINS=` | CORS allowed origins (comma-separated; empty = same-origin only) |
|
||||
|
||||
---
|
||||
|
||||
## Login Rate Limiting
|
||||
|
||||
The `/api/auth/login` endpoint is protected by a dedicated
|
||||
`LoginRateLimiter` (separate from the general API rate limiter).
|
||||
Limits are enforced per-IP and per-username with a sliding window:
|
||||
|
||||
- **5 attempts** per **5-minute window** per key
|
||||
- Failed logins record against both `ip:{client_ip}` and `user:{username}`
|
||||
- Returns `429 Too Many Requests` with `Retry-After` header when exceeded
|
||||
- Successful logins do not consume the budget
|
||||
|
||||
---
|
||||
|
||||
## CORS Policy
|
||||
|
||||
By default, no CORS headers are sent (same-origin only). To allow
|
||||
cross-origin requests, set `TURNSTONE_CORS_ORIGINS`:
|
||||
|
||||
```bash
|
||||
# Allow specific origins
|
||||
TURNSTONE_CORS_ORIGINS=https://app.example.com,https://admin.example.com
|
||||
|
||||
# Allow all origins (development only)
|
||||
TURNSTONE_CORS_ORIGINS=*
|
||||
```
|
||||
|
||||
When the variable is empty or unset, the CORS middleware is not added
|
||||
and browsers enforce same-origin policy.
|
||||
|
||||
---
|
||||
|
||||
## Security Properties
|
||||
|
||||
- **Timing-safe comparison** for config-file tokens via
|
||||
`hmac.compare_digest` — no timing side-channel.
|
||||
- **Hash-based lookup** for API tokens — the database stores only
|
||||
SHA-256 hashes, eliminating timing attacks on token comparison.
|
||||
- **Local JWT validation** — no network call or database query needed
|
||||
per request on server nodes.
|
||||
- **One-time display** of raw API tokens at creation. The plaintext is
|
||||
never stored; `token_hash` never appears in API responses or logs.
|
||||
- **Structured logging audit trail** — `ctx_user_id` is set on every
|
||||
authenticated request and injected into all log events.
|
||||
- **Scope enforcement** at the middleware layer before any handler
|
||||
executes. Path-to-scope mapping is defined statically.
|
||||
- **JWT audience isolation** — server and console JWTs have distinct
|
||||
`aud` claims, preventing cross-service token reuse.
|
||||
- **Login brute-force protection** — per-IP and per-username rate
|
||||
limiting on the login endpoint.
|
||||
- **Secure cookies by default** — `Secure` flag set unconditionally;
|
||||
24-hour max-age matches JWT expiry.
|
||||
- **CORS restriction** — no CORS headers by default (same-origin only).
|
||||
- **Service JWT auto-rotation** — 1-hour expiry with transparent
|
||||
refresh, eliminating long-lived static tokens for inter-service auth.
|
||||
- **Secret strength validation** — warning logged when JWT secret is
|
||||
shorter than 32 characters.
|
||||
+37
-4
@@ -1,6 +1,6 @@
|
||||
# Tools Reference
|
||||
|
||||
turnstone exposes 14 built-in tools plus any number of external MCP tools to the
|
||||
turnstone exposes 15 built-in tools plus any number of external MCP tools to the
|
||||
LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON
|
||||
files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`.
|
||||
MCP tools are discovered from configured MCP servers at startup by
|
||||
@@ -46,7 +46,7 @@ schema plus turnstone-specific metadata keys:
|
||||
|
||||
| Name | Description |
|
||||
|---------------------|-------------|
|
||||
| `TOOLS` | All 14 tool definitions (sent to the model). |
|
||||
| `TOOLS` | All 15 tool definitions (sent to the model). |
|
||||
| `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. |
|
||||
| `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. |
|
||||
| `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. |
|
||||
@@ -113,6 +113,7 @@ Each item's `execute` callable is invoked:
|
||||
- `remember` -- writes to persistent memory database (lightweight, always auto-approved)
|
||||
- `recall` -- reads from persistent memory database
|
||||
- `forget` -- deletes from persistent memory database (lightweight, always auto-approved)
|
||||
- `notify` -- sends notifications to linked channels (time-sensitive, auto-approved for urgency)
|
||||
|
||||
**Requires user confirmation** (write operations, network access, side effects):
|
||||
- `bash` -- arbitrary command execution
|
||||
@@ -162,6 +163,7 @@ Every tool defines a `primary_key`. The mapping is:
|
||||
| `remember` | `key` |
|
||||
| `recall` | `query` |
|
||||
| `forget` | `key` |
|
||||
| `notify` | `message` |
|
||||
|
||||
---
|
||||
|
||||
@@ -302,8 +304,11 @@ Search the web using a text query.
|
||||
| `max_results` | integer | no | Max results to return (default 5, max 20). |
|
||||
| `topic` | string | no | Search topic: `general`, `news`, or `finance` (default `general`). |
|
||||
|
||||
- **What it does**: Searches the web via the Tavily API and returns ranked results with titles, URLs, and content snippets.
|
||||
- **Auto-approve**: No -- requires user confirmation (makes network requests).
|
||||
- **What it does**: Searches the web and returns ranked results with titles, URLs, and content snippets. Uses provider-native search when available:
|
||||
- **Anthropic**: Replaced at the API boundary with Anthropic's `web_search_20250305` server-side tool. Claude decides when to search; the API executes it and returns results with citations inline. No Tavily key needed.
|
||||
- **OpenAI search models** (`gpt-5-search-api`): Replaced with `web_search_options` parameter. The model always searches and returns `url_citation` annotations.
|
||||
- **Local/vLLM models**: Falls back to the Tavily API. Requires `tavily_key` in `config.toml` or `$TAVILY_API_KEY`.
|
||||
- **Auto-approve**: Yes (auto-approved for all tool dispatch paths).
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
@@ -384,6 +389,33 @@ Remove a persistent memory by key.
|
||||
|
||||
---
|
||||
|
||||
## Notifications
|
||||
|
||||
### notify
|
||||
|
||||
Send a notification to a user or channel on an external platform.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|----------------|--------|----------|-------------|
|
||||
| `message` | string | yes | Notification content (plain text, max 2000 chars). |
|
||||
| `username` | string | no | Turnstone username — sends to all linked channels. |
|
||||
| `channel_type` | string | no | Platform for direct targeting (`discord`). |
|
||||
| `channel_id` | string | no | Platform-specific channel or user ID for direct targeting. |
|
||||
| `title` | string | no | Optional short title (rendered as bold prefix). |
|
||||
|
||||
Provide either `username` for user-based targeting or `channel_type` +
|
||||
`channel_id` for direct targeting. Do not combine both.
|
||||
|
||||
- **What it does**: Sends a notification via the channel gateway's HTTP endpoint (`POST /v1/api/notify`). The server queries the `services` table for healthy channel gateways, authenticates with a service JWT (`aud: turnstone-channel`), and delivers to the first healthy gateway. On failure, retries up to 2 additional times with backoff (1s, 3s). Rate-limited to 5 notifications per turn (counter only increments on success).
|
||||
- **Auto-approve**: Yes — notifications are time-sensitive and auto-approved so the model can alert users urgently.
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
> See [Channel Integrations: Notifications](channels.md#notifications)
|
||||
> for the full delivery flow, service registry details, and security
|
||||
> measures.
|
||||
|
||||
---
|
||||
|
||||
## Summary Table
|
||||
|
||||
| Tool | Category | Auto-approve | agent | task_agent | primary_key |
|
||||
@@ -402,6 +434,7 @@ Remove a persistent memory by key.
|
||||
| `remember` | Memory | Yes | No | No | `key` |
|
||||
| `recall` | Memory | Yes | No | No | `query` |
|
||||
| `forget` | Memory | Yes | No | No | `key` |
|
||||
| `notify` | Notify | Yes | Yes | Yes | `message` |
|
||||
|
||||
---
|
||||
|
||||
|
||||
+60
-4
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "0.3.0"
|
||||
version = "0.4.2"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
@@ -21,7 +21,21 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
]
|
||||
dependencies = ["openai>=2.24", "httpx>=0.28", "mcp>=1.6"]
|
||||
dependencies = [
|
||||
"openai>=2.24",
|
||||
"httpx>=0.28",
|
||||
"mcp>=1.6",
|
||||
"starlette>=0.45",
|
||||
"uvicorn>=0.34",
|
||||
"sse-starlette>=2.0",
|
||||
"httpx-sse>=0.4",
|
||||
"pydantic>=2.0",
|
||||
"sqlalchemy>=2.0",
|
||||
"alembic>=1.14",
|
||||
"structlog>=24.1",
|
||||
"PyJWT>=2.8",
|
||||
"bcrypt>=4.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/turnstonelabs/turnstone"
|
||||
@@ -29,11 +43,14 @@ Repository = "https://github.com/turnstonelabs/turnstone"
|
||||
Issues = "https://github.com/turnstonelabs/turnstone/issues"
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest>=9.0", "pytest-cov>=6.0"]
|
||||
test = ["pytest>=9.0", "pytest-cov>=6.0", "croniter>=3.0"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14", "types-redis>=4.6"]
|
||||
mq = ["redis>=7.2"]
|
||||
console = ["redis>=7.2"]
|
||||
console = ["redis>=7.2", "croniter>=3.0"]
|
||||
sim = ["redis>=7.2"]
|
||||
anthropic = ["anthropic>=0.39"]
|
||||
postgres = ["psycopg[binary]>=3.2"]
|
||||
discord = ["discord.py>=2.4", "redis>=7.2"]
|
||||
|
||||
|
||||
[project.scripts]
|
||||
@@ -43,6 +60,8 @@ turnstone-server = "turnstone.server:main"
|
||||
turnstone-bridge = "turnstone.mq.bridge:main"
|
||||
turnstone-console = "turnstone.console.server:main"
|
||||
turnstone-sim = "turnstone.sim.cli:main"
|
||||
turnstone-admin = "turnstone.admin:main"
|
||||
turnstone-channel = "turnstone.channels.cli:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
include = [
|
||||
@@ -54,6 +73,9 @@ include = [
|
||||
"turnstone/console/static/*.html",
|
||||
"turnstone/console/static/*.css",
|
||||
"turnstone/console/static/*.js",
|
||||
"turnstone/shared_static/*.css",
|
||||
"turnstone/shared_static/*.js",
|
||||
"turnstone/sdk/py.typed",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
@@ -104,6 +126,40 @@ ignore_missing_imports = true
|
||||
module = ["mcp", "mcp.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["sse_starlette", "sse_starlette.*", "uvicorn", "uvicorn.*", "httpx_sse", "httpx_sse.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["sqlalchemy", "sqlalchemy.*", "alembic", "alembic.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["structlog", "structlog.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["jwt", "jwt.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["anthropic", "anthropic.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["discord", "discord.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["croniter", "croniter.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["turnstone.channels.discord.*"]
|
||||
disallow_subclassing_any = false
|
||||
disallow_untyped_decorators = false
|
||||
warn_unused_ignores = false
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
disallow_untyped_defs = false
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
node_modules/
|
||||
dist/
|
||||
*.tsbuildinfo
|
||||
@@ -0,0 +1,875 @@
|
||||
{
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "turnstone Console API",
|
||||
"version": "0.3.0",
|
||||
"description": "Cluster-wide visibility and control across all turnstone nodes."
|
||||
},
|
||||
"paths": {
|
||||
"/v1/api/cluster/overview": {
|
||||
"get": {
|
||||
"summary": "Cluster state summary",
|
||||
"operationId": "v1_api_cluster_overview_get",
|
||||
"tags": [
|
||||
"Cluster"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ClusterOverviewResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/cluster/nodes": {
|
||||
"get": {
|
||||
"summary": "Paginated node list",
|
||||
"operationId": "v1_api_cluster_nodes_get",
|
||||
"tags": [
|
||||
"Cluster"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "sort",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"default": "activity",
|
||||
"enum": [
|
||||
"activity",
|
||||
"tokens",
|
||||
"name"
|
||||
]
|
||||
},
|
||||
"description": "Sort field"
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "integer",
|
||||
"default": 100
|
||||
},
|
||||
"description": "Page size"
|
||||
},
|
||||
{
|
||||
"name": "offset",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "integer",
|
||||
"default": 0
|
||||
},
|
||||
"description": "Pagination offset"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ClusterNodesResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/cluster/workstreams": {
|
||||
"get": {
|
||||
"summary": "Filtered workstream list",
|
||||
"operationId": "v1_api_cluster_workstreams_get",
|
||||
"tags": [
|
||||
"Cluster"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "state",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"running",
|
||||
"thinking",
|
||||
"attention",
|
||||
"idle",
|
||||
"error"
|
||||
]
|
||||
},
|
||||
"description": "Filter by state"
|
||||
},
|
||||
{
|
||||
"name": "node",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Filter by node_id"
|
||||
},
|
||||
{
|
||||
"name": "search",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Search in name/title/node"
|
||||
},
|
||||
{
|
||||
"name": "sort",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"default": "state",
|
||||
"enum": [
|
||||
"state",
|
||||
"tokens",
|
||||
"name"
|
||||
]
|
||||
},
|
||||
"description": "Sort field"
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "integer",
|
||||
"default": 1
|
||||
},
|
||||
"description": "Page number"
|
||||
},
|
||||
{
|
||||
"name": "per_page",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "integer",
|
||||
"default": 50
|
||||
},
|
||||
"description": "Items per page (max 200)"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ClusterWorkstreamsResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/cluster/node/{node_id}": {
|
||||
"get": {
|
||||
"summary": "Single node detail",
|
||||
"operationId": "v1_api_cluster_node_{node_id}_get",
|
||||
"tags": [
|
||||
"Cluster"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "node_id",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/NodeDetailResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Error 404",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/cluster/workstreams/new": {
|
||||
"post": {
|
||||
"summary": "Create workstream via MQ dispatch",
|
||||
"operationId": "v1_api_cluster_workstreams_new_post",
|
||||
"tags": [
|
||||
"Cluster"
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ConsoleCreateWsRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ConsoleCreateWsResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Error 400",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Error 404",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Error 503",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/cluster/events": {
|
||||
"get": {
|
||||
"summary": "Cluster SSE event stream",
|
||||
"operationId": "v1_api_cluster_events_get",
|
||||
"tags": [
|
||||
"Streaming"
|
||||
],
|
||||
"description": "Server-Sent Events stream for real-time cluster updates. Returns text/event-stream with node_joined, node_lost, cluster_state, ws_created, ws_closed, ws_rename events.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/auth/login": {
|
||||
"post": {
|
||||
"summary": "Authenticate with a token",
|
||||
"operationId": "v1_api_auth_login_post",
|
||||
"tags": [
|
||||
"Auth"
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/AuthLoginRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/AuthLoginResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Error 401",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/api/auth/logout": {
|
||||
"post": {
|
||||
"summary": "Clear auth cookie",
|
||||
"operationId": "v1_api_auth_logout_post",
|
||||
"tags": [
|
||||
"Auth"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/StatusResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"summary": "Console health check",
|
||||
"operationId": "health_get",
|
||||
"tags": [
|
||||
"Observability"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ConsoleHealthResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"components": {
|
||||
"schemas": {
|
||||
"ErrorResponse": {
|
||||
"description": "Standard error response body.",
|
||||
"properties": {
|
||||
"error": {
|
||||
"description": "Error message",
|
||||
"title": "Error",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"error"
|
||||
],
|
||||
"title": "ErrorResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"StatusResponse": {
|
||||
"description": "Generic success response.",
|
||||
"properties": {
|
||||
"status": {
|
||||
"default": "ok",
|
||||
"examples": [
|
||||
"ok"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"title": "StatusResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"AuthLoginRequest": {
|
||||
"description": "POST /v1/api/auth/login request body.",
|
||||
"properties": {
|
||||
"token": {
|
||||
"description": "Bearer token to authenticate",
|
||||
"title": "Token",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"title": "AuthLoginRequest",
|
||||
"type": "object"
|
||||
},
|
||||
"AuthLoginResponse": {
|
||||
"description": "POST /v1/api/auth/login success response.",
|
||||
"properties": {
|
||||
"status": {
|
||||
"default": "ok",
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"role": {
|
||||
"description": "Assigned role",
|
||||
"examples": [
|
||||
"full",
|
||||
"read"
|
||||
],
|
||||
"title": "Role",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role"
|
||||
],
|
||||
"title": "AuthLoginResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ClusterOverviewResponse": {
|
||||
"properties": {
|
||||
"nodes": {
|
||||
"default": 0,
|
||||
"title": "Nodes",
|
||||
"type": "integer"
|
||||
},
|
||||
"workstreams": {
|
||||
"default": 0,
|
||||
"title": "Workstreams",
|
||||
"type": "integer"
|
||||
},
|
||||
"states": {
|
||||
"$ref": "#/components/schemas/StateCounts",
|
||||
"default": {
|
||||
"running": 0,
|
||||
"thinking": 0,
|
||||
"attention": 0,
|
||||
"idle": 0,
|
||||
"error": 0
|
||||
}
|
||||
},
|
||||
"aggregate": {
|
||||
"$ref": "#/components/schemas/ClusterAggregate",
|
||||
"default": {
|
||||
"total_tokens": 0,
|
||||
"total_tool_calls": 0
|
||||
}
|
||||
},
|
||||
"version_drift": {
|
||||
"default": false,
|
||||
"title": "Version Drift",
|
||||
"type": "boolean"
|
||||
},
|
||||
"versions": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Versions",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"title": "ClusterOverviewResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ClusterAggregate": {
|
||||
"properties": {
|
||||
"total_tokens": {
|
||||
"default": 0,
|
||||
"title": "Total Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"total_tool_calls": {
|
||||
"default": 0,
|
||||
"title": "Total Tool Calls",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"title": "ClusterAggregate",
|
||||
"type": "object"
|
||||
},
|
||||
"StateCounts": {
|
||||
"properties": {
|
||||
"running": {
|
||||
"default": 0,
|
||||
"title": "Running",
|
||||
"type": "integer"
|
||||
},
|
||||
"thinking": {
|
||||
"default": 0,
|
||||
"title": "Thinking",
|
||||
"type": "integer"
|
||||
},
|
||||
"attention": {
|
||||
"default": 0,
|
||||
"title": "Attention",
|
||||
"type": "integer"
|
||||
},
|
||||
"idle": {
|
||||
"default": 0,
|
||||
"title": "Idle",
|
||||
"type": "integer"
|
||||
},
|
||||
"error": {
|
||||
"default": 0,
|
||||
"title": "Error",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"title": "StateCounts",
|
||||
"type": "object"
|
||||
},
|
||||
"ClusterNodesResponse": {
|
||||
"properties": {
|
||||
"nodes": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ClusterNodeInfo"
|
||||
},
|
||||
"title": "Nodes",
|
||||
"type": "array"
|
||||
},
|
||||
"total": {
|
||||
"default": 0,
|
||||
"title": "Total",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"nodes"
|
||||
],
|
||||
"title": "ClusterNodesResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ClusterNodeInfo": {
|
||||
"properties": {
|
||||
"node_id": {
|
||||
"title": "Node Id",
|
||||
"type": "string"
|
||||
},
|
||||
"server_url": {
|
||||
"default": "",
|
||||
"title": "Server Url",
|
||||
"type": "string"
|
||||
},
|
||||
"ws_total": {
|
||||
"default": 0,
|
||||
"title": "Ws Total",
|
||||
"type": "integer"
|
||||
},
|
||||
"ws_running": {
|
||||
"default": 0,
|
||||
"title": "Ws Running",
|
||||
"type": "integer"
|
||||
},
|
||||
"ws_thinking": {
|
||||
"default": 0,
|
||||
"title": "Ws Thinking",
|
||||
"type": "integer"
|
||||
},
|
||||
"ws_attention": {
|
||||
"default": 0,
|
||||
"title": "Ws Attention",
|
||||
"type": "integer"
|
||||
},
|
||||
"ws_idle": {
|
||||
"default": 0,
|
||||
"title": "Ws Idle",
|
||||
"type": "integer"
|
||||
},
|
||||
"ws_error": {
|
||||
"default": 0,
|
||||
"title": "Ws Error",
|
||||
"type": "integer"
|
||||
},
|
||||
"total_tokens": {
|
||||
"default": 0,
|
||||
"title": "Total Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"started": {
|
||||
"default": 0.0,
|
||||
"title": "Started",
|
||||
"type": "number"
|
||||
},
|
||||
"reachable": {
|
||||
"default": true,
|
||||
"title": "Reachable",
|
||||
"type": "boolean"
|
||||
},
|
||||
"health": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Health",
|
||||
"type": "object"
|
||||
},
|
||||
"version": {
|
||||
"default": "",
|
||||
"title": "Version",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"node_id"
|
||||
],
|
||||
"title": "ClusterNodeInfo",
|
||||
"type": "object"
|
||||
},
|
||||
"ClusterWorkstreamsResponse": {
|
||||
"properties": {
|
||||
"workstreams": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ClusterWorkstreamInfo"
|
||||
},
|
||||
"title": "Workstreams",
|
||||
"type": "array"
|
||||
},
|
||||
"total": {
|
||||
"default": 0,
|
||||
"title": "Total",
|
||||
"type": "integer"
|
||||
},
|
||||
"page": {
|
||||
"default": 1,
|
||||
"title": "Page",
|
||||
"type": "integer"
|
||||
},
|
||||
"per_page": {
|
||||
"default": 50,
|
||||
"title": "Per Page",
|
||||
"type": "integer"
|
||||
},
|
||||
"pages": {
|
||||
"default": 1,
|
||||
"title": "Pages",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"workstreams"
|
||||
],
|
||||
"title": "ClusterWorkstreamsResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ClusterWorkstreamInfo": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"title": "Id",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"default": "",
|
||||
"title": "Name",
|
||||
"type": "string"
|
||||
},
|
||||
"state": {
|
||||
"default": "",
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"node": {
|
||||
"default": "",
|
||||
"title": "Node",
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"default": "",
|
||||
"title": "Title",
|
||||
"type": "string"
|
||||
},
|
||||
"tokens": {
|
||||
"default": 0,
|
||||
"title": "Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"context_ratio": {
|
||||
"default": 0.0,
|
||||
"title": "Context Ratio",
|
||||
"type": "number"
|
||||
},
|
||||
"activity": {
|
||||
"default": "",
|
||||
"title": "Activity",
|
||||
"type": "string"
|
||||
},
|
||||
"activity_state": {
|
||||
"default": "",
|
||||
"title": "Activity State",
|
||||
"type": "string"
|
||||
},
|
||||
"tool_calls": {
|
||||
"default": 0,
|
||||
"title": "Tool Calls",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id"
|
||||
],
|
||||
"title": "ClusterWorkstreamInfo",
|
||||
"type": "object"
|
||||
},
|
||||
"NodeDetailResponse": {
|
||||
"properties": {
|
||||
"node_id": {
|
||||
"title": "Node Id",
|
||||
"type": "string"
|
||||
},
|
||||
"server_url": {
|
||||
"default": "",
|
||||
"title": "Server Url",
|
||||
"type": "string"
|
||||
},
|
||||
"health": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Health",
|
||||
"type": "object"
|
||||
},
|
||||
"workstreams": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ClusterWorkstreamInfo"
|
||||
},
|
||||
"title": "Workstreams",
|
||||
"type": "array"
|
||||
},
|
||||
"aggregate": {
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
},
|
||||
"title": "Aggregate",
|
||||
"type": "object"
|
||||
},
|
||||
"reachable": {
|
||||
"default": true,
|
||||
"title": "Reachable",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"node_id"
|
||||
],
|
||||
"title": "NodeDetailResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ConsoleCreateWsRequest": {
|
||||
"properties": {
|
||||
"node_id": {
|
||||
"default": "",
|
||||
"description": "Target node: specific ID, 'auto', 'pool', or empty for auto",
|
||||
"title": "Node Id",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"default": "",
|
||||
"description": "Workstream name (auto-generated if empty)",
|
||||
"title": "Name",
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"default": "",
|
||||
"description": "Model alias from node registry",
|
||||
"title": "Model",
|
||||
"type": "string"
|
||||
},
|
||||
"initial_message": {
|
||||
"default": "",
|
||||
"description": "Optional first message sent after creation",
|
||||
"title": "Initial Message",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"title": "ConsoleCreateWsRequest",
|
||||
"type": "object"
|
||||
},
|
||||
"ConsoleCreateWsResponse": {
|
||||
"properties": {
|
||||
"status": {
|
||||
"default": "ok",
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"correlation_id": {
|
||||
"default": "",
|
||||
"title": "Correlation Id",
|
||||
"type": "string"
|
||||
},
|
||||
"target_node": {
|
||||
"default": "",
|
||||
"title": "Target Node",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"title": "ConsoleCreateWsResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ConsoleHealthResponse": {
|
||||
"properties": {
|
||||
"status": {
|
||||
"default": "ok",
|
||||
"examples": [
|
||||
"ok"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"service": {
|
||||
"default": "turnstone-console",
|
||||
"title": "Service",
|
||||
"type": "string"
|
||||
},
|
||||
"nodes": {
|
||||
"default": 0,
|
||||
"title": "Nodes",
|
||||
"type": "integer"
|
||||
},
|
||||
"workstreams": {
|
||||
"default": 0,
|
||||
"title": "Workstreams",
|
||||
"type": "integer"
|
||||
},
|
||||
"version_drift": {
|
||||
"default": false,
|
||||
"title": "Version Drift",
|
||||
"type": "boolean"
|
||||
},
|
||||
"versions": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Versions",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"title": "ConsoleHealthResponse",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Generated
+1346
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"name": "@turnstone/sdk",
|
||||
"version": "0.3.0",
|
||||
"description": "TypeScript client SDK for the turnstone AI orchestration platform",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"generate-types": "python scripts/generate-types.py"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src"
|
||||
],
|
||||
"keywords": [
|
||||
"turnstone",
|
||||
"ai",
|
||||
"llm",
|
||||
"agent",
|
||||
"sdk",
|
||||
"client"
|
||||
],
|
||||
"license": "BUSL-1.1",
|
||||
"devDependencies": {
|
||||
"typescript": "^5.4",
|
||||
"vitest": "^2.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export OpenAPI specs to JSON files for TypeScript type reference.
|
||||
|
||||
Usage:
|
||||
python scripts/generate-types.py
|
||||
|
||||
Writes:
|
||||
openapi-server.json — Server API OpenAPI 3.1 spec
|
||||
openapi-console.json — Console API OpenAPI 3.1 spec
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure the turnstone package is importable (repo root is 3 levels up)
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||
|
||||
from turnstone.api.console_spec import build_console_spec
|
||||
from turnstone.api.server_spec import build_server_spec
|
||||
|
||||
output_dir = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def main() -> None:
|
||||
server_spec = build_server_spec()
|
||||
console_spec = build_console_spec()
|
||||
|
||||
server_path = output_dir / "openapi-server.json"
|
||||
console_path = output_dir / "openapi-console.json"
|
||||
|
||||
server_path.write_text(json.dumps(server_spec, indent=2) + "\n")
|
||||
console_path.write_text(json.dumps(console_spec, indent=2) + "\n")
|
||||
|
||||
print(f"Wrote {server_path} ({len(server_spec['paths'])} paths)")
|
||||
print(f"Wrote {console_path} ({len(console_spec['paths'])} paths)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,107 @@
|
||||
import { TurnstoneAPIError } from "./errors.js";
|
||||
import { parseSSEStream } from "./sse.js";
|
||||
|
||||
export interface ClientOptions {
|
||||
/** Server base URL (e.g. "http://localhost:8080"). */
|
||||
baseUrl: string;
|
||||
/** Bearer token for authentication. */
|
||||
token?: string;
|
||||
/** Custom fetch implementation (defaults to globalThis.fetch). */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
export interface RequestOptions {
|
||||
json?: object;
|
||||
params?: Record<string, string | number>;
|
||||
}
|
||||
|
||||
export class BaseClient {
|
||||
protected readonly baseUrl: string;
|
||||
protected readonly token: string;
|
||||
protected readonly fetchFn: typeof globalThis.fetch;
|
||||
|
||||
constructor(options: ClientOptions) {
|
||||
this.baseUrl = options.baseUrl.replace(/\/$/, "");
|
||||
this.token = options.token ?? "";
|
||||
this.fetchFn = options.fetch ?? globalThis.fetch.bind(globalThis);
|
||||
}
|
||||
|
||||
protected async request<T>(
|
||||
method: string,
|
||||
path: string,
|
||||
options?: RequestOptions,
|
||||
): Promise<T> {
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
if (this.token) {
|
||||
headers["Authorization"] = `Bearer ${this.token}`;
|
||||
}
|
||||
|
||||
let url = `${this.baseUrl}${path}`;
|
||||
if (options?.params) {
|
||||
const searchParams = new URLSearchParams();
|
||||
for (const [key, value] of Object.entries(options.params)) {
|
||||
if (value !== undefined && value !== "") {
|
||||
searchParams.set(key, String(value));
|
||||
}
|
||||
}
|
||||
const qs = searchParams.toString();
|
||||
if (qs) url += `?${qs}`;
|
||||
}
|
||||
|
||||
const resp = await this.fetchFn(url, {
|
||||
method,
|
||||
headers,
|
||||
body: options?.json ? JSON.stringify(options.json) : undefined,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
let msg = "";
|
||||
try {
|
||||
const body = (await resp.json()) as Record<string, unknown>;
|
||||
msg = (body.error as string) ?? (body.detail as string) ?? "";
|
||||
} catch {
|
||||
msg = await resp.text().catch(() => "");
|
||||
}
|
||||
throw new TurnstoneAPIError(resp.status, msg || `HTTP ${resp.status}`);
|
||||
}
|
||||
|
||||
return (await resp.json()) as T;
|
||||
}
|
||||
|
||||
protected async *streamSSE<T = Record<string, unknown>>(
|
||||
path: string,
|
||||
params?: Record<string, string | number>,
|
||||
signal?: AbortSignal,
|
||||
): AsyncIterableIterator<T> {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "text/event-stream",
|
||||
};
|
||||
if (this.token) {
|
||||
headers["Authorization"] = `Bearer ${this.token}`;
|
||||
}
|
||||
|
||||
let url = `${this.baseUrl}${path}`;
|
||||
if (params) {
|
||||
const searchParams = new URLSearchParams();
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
if (value !== undefined && value !== "") {
|
||||
searchParams.set(key, String(value));
|
||||
}
|
||||
}
|
||||
const qs = searchParams.toString();
|
||||
if (qs) url += `?${qs}`;
|
||||
}
|
||||
|
||||
const resp = await this.fetchFn(url, { method: "GET", headers, signal });
|
||||
if (!resp.ok) {
|
||||
throw new TurnstoneAPIError(
|
||||
resp.status,
|
||||
`SSE connection failed: HTTP ${resp.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
yield* parseSSEStream<T>(resp);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { BaseClient, type ClientOptions } from "./base.js";
|
||||
import type { ClusterEvent } from "./events.js";
|
||||
import type {
|
||||
AuthLoginResponse,
|
||||
AuthSetupResponse,
|
||||
AuthStatusResponse,
|
||||
ClusterNodesResponse,
|
||||
ClusterOverviewResponse,
|
||||
ClusterWorkstreamsResponse,
|
||||
ConsoleCreateWsRequest,
|
||||
ConsoleCreateWsResponse,
|
||||
ConsoleHealthResponse,
|
||||
CreateScheduleRequest,
|
||||
ListScheduleRunsResponse,
|
||||
ListSchedulesResponse,
|
||||
NodeDetailResponse,
|
||||
NodesOptions,
|
||||
ScheduleInfo,
|
||||
StatusResponse,
|
||||
UpdateScheduleRequest,
|
||||
WorkstreamsOptions,
|
||||
} from "./types.js";
|
||||
|
||||
/** Async client for the turnstone console API. */
|
||||
export class TurnstoneConsole extends BaseClient {
|
||||
constructor(options: ClientOptions) {
|
||||
super(options);
|
||||
}
|
||||
|
||||
// -- Cluster overview -----------------------------------------------------
|
||||
|
||||
async overview(): Promise<ClusterOverviewResponse> {
|
||||
return this.request("GET", "/v1/api/cluster/overview");
|
||||
}
|
||||
|
||||
async nodes(opts?: NodesOptions): Promise<ClusterNodesResponse> {
|
||||
return this.request("GET", "/v1/api/cluster/nodes", {
|
||||
params: {
|
||||
sort: opts?.sort ?? "activity",
|
||||
limit: opts?.limit ?? 100,
|
||||
offset: opts?.offset ?? 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async workstreams(
|
||||
opts?: WorkstreamsOptions,
|
||||
): Promise<ClusterWorkstreamsResponse> {
|
||||
const params: Record<string, string | number> = {
|
||||
sort: opts?.sort ?? "state",
|
||||
page: opts?.page ?? 1,
|
||||
per_page: opts?.per_page ?? 50,
|
||||
};
|
||||
if (opts?.state) params.state = opts.state;
|
||||
if (opts?.node) params.node = opts.node;
|
||||
if (opts?.search) params.search = opts.search;
|
||||
return this.request("GET", "/v1/api/cluster/workstreams", { params });
|
||||
}
|
||||
|
||||
async nodeDetail(nodeId: string): Promise<NodeDetailResponse> {
|
||||
return this.request("GET", `/v1/api/cluster/node/${nodeId}`);
|
||||
}
|
||||
|
||||
async createWorkstream(
|
||||
opts?: ConsoleCreateWsRequest,
|
||||
): Promise<ConsoleCreateWsResponse> {
|
||||
return this.request("POST", "/v1/api/cluster/workstreams/new", {
|
||||
json: opts,
|
||||
});
|
||||
}
|
||||
|
||||
// -- Streaming ------------------------------------------------------------
|
||||
|
||||
async *clusterEvents(): AsyncIterableIterator<ClusterEvent> {
|
||||
yield* this.streamSSE<ClusterEvent>("/v1/api/cluster/events");
|
||||
}
|
||||
|
||||
// -- Auth -----------------------------------------------------------------
|
||||
|
||||
async login(opts: {
|
||||
token?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
}): Promise<AuthLoginResponse> {
|
||||
const body =
|
||||
opts.username && opts.password
|
||||
? { username: opts.username, password: opts.password }
|
||||
: { token: opts.token ?? "" };
|
||||
return this.request("POST", "/v1/api/auth/login", { json: body });
|
||||
}
|
||||
|
||||
async authStatus(): Promise<AuthStatusResponse> {
|
||||
return this.request("GET", "/v1/api/auth/status");
|
||||
}
|
||||
|
||||
async setup(opts: {
|
||||
username: string;
|
||||
displayName: string;
|
||||
password: string;
|
||||
}): Promise<AuthSetupResponse> {
|
||||
return this.request("POST", "/v1/api/auth/setup", {
|
||||
json: {
|
||||
username: opts.username,
|
||||
display_name: opts.displayName,
|
||||
password: opts.password,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async logout(): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/auth/logout");
|
||||
}
|
||||
|
||||
// -- Health ---------------------------------------------------------------
|
||||
|
||||
async health(): Promise<ConsoleHealthResponse> {
|
||||
return this.request("GET", "/health");
|
||||
}
|
||||
|
||||
// -- Schedules ------------------------------------------------------------
|
||||
|
||||
async listSchedules(): Promise<ListSchedulesResponse> {
|
||||
return this.request("GET", "/v1/api/admin/schedules");
|
||||
}
|
||||
|
||||
async createSchedule(opts: CreateScheduleRequest): Promise<ScheduleInfo> {
|
||||
return this.request("POST", "/v1/api/admin/schedules", { json: opts });
|
||||
}
|
||||
|
||||
async getSchedule(taskId: string): Promise<ScheduleInfo> {
|
||||
return this.request("GET", `/v1/api/admin/schedules/${taskId}`);
|
||||
}
|
||||
|
||||
async updateSchedule(
|
||||
taskId: string,
|
||||
opts: UpdateScheduleRequest,
|
||||
): Promise<ScheduleInfo> {
|
||||
return this.request("PUT", `/v1/api/admin/schedules/${taskId}`, {
|
||||
json: opts,
|
||||
});
|
||||
}
|
||||
|
||||
async deleteSchedule(taskId: string): Promise<StatusResponse> {
|
||||
return this.request("DELETE", `/v1/api/admin/schedules/${taskId}`);
|
||||
}
|
||||
|
||||
async listScheduleRuns(
|
||||
taskId: string,
|
||||
opts?: { limit?: number },
|
||||
): Promise<ListScheduleRunsResponse> {
|
||||
return this.request("GET", `/v1/api/admin/schedules/${taskId}/runs`, {
|
||||
params: { limit: opts?.limit ?? 50 },
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Raised when a turnstone server returns a non-2xx response. */
|
||||
export class TurnstoneAPIError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
public readonly errorMessage: string,
|
||||
) {
|
||||
super(`HTTP ${statusCode}: ${errorMessage}`);
|
||||
this.name = "TurnstoneAPIError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server SSE events
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ConnectedEvent {
|
||||
type: "connected";
|
||||
model: string;
|
||||
model_alias: string;
|
||||
skip_permissions: boolean;
|
||||
}
|
||||
|
||||
export interface HistoryEvent {
|
||||
type: "history";
|
||||
messages: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface ThinkingStartEvent {
|
||||
type: "thinking_start";
|
||||
}
|
||||
|
||||
export interface ThinkingStopEvent {
|
||||
type: "thinking_stop";
|
||||
}
|
||||
|
||||
export interface ContentEvent {
|
||||
type: "content";
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ReasoningEvent {
|
||||
type: "reasoning";
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface StreamEndEvent {
|
||||
type: "stream_end";
|
||||
}
|
||||
|
||||
export interface ToolInfoEvent {
|
||||
type: "tool_info";
|
||||
items: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface ApproveRequestEvent {
|
||||
type: "approve_request";
|
||||
items: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface ToolResultEvent {
|
||||
type: "tool_result";
|
||||
call_id: string;
|
||||
name: string;
|
||||
output: string;
|
||||
}
|
||||
|
||||
export interface ToolOutputChunkEvent {
|
||||
type: "tool_output_chunk";
|
||||
call_id: string;
|
||||
chunk: string;
|
||||
}
|
||||
|
||||
export interface StatusEvent {
|
||||
type: "status";
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
context_window: number;
|
||||
pct: number;
|
||||
effort: string;
|
||||
}
|
||||
|
||||
export interface PlanReviewEvent {
|
||||
type: "plan_review";
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface InfoEvent {
|
||||
type: "info";
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface ErrorEvent {
|
||||
type: "error";
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface BusyErrorEvent {
|
||||
type: "busy_error";
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface ClearUiEvent {
|
||||
type: "clear_ui";
|
||||
}
|
||||
|
||||
// Global events
|
||||
|
||||
export interface WsStateEvent {
|
||||
type: "ws_state";
|
||||
ws_id: string;
|
||||
state: string;
|
||||
tokens: number;
|
||||
context_ratio: number;
|
||||
activity: string;
|
||||
activity_state: string;
|
||||
}
|
||||
|
||||
export interface WsActivityEvent {
|
||||
type: "ws_activity";
|
||||
ws_id: string;
|
||||
activity: string;
|
||||
activity_state: string;
|
||||
}
|
||||
|
||||
export interface WsRenameEvent {
|
||||
type: "ws_rename";
|
||||
ws_id: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface WsClosedEvent {
|
||||
type: "ws_closed";
|
||||
ws_id: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
/** Discriminated union of all server SSE event types. */
|
||||
export type ServerEvent =
|
||||
| ConnectedEvent
|
||||
| HistoryEvent
|
||||
| ThinkingStartEvent
|
||||
| ThinkingStopEvent
|
||||
| ContentEvent
|
||||
| ReasoningEvent
|
||||
| StreamEndEvent
|
||||
| ToolInfoEvent
|
||||
| ApproveRequestEvent
|
||||
| ToolResultEvent
|
||||
| ToolOutputChunkEvent
|
||||
| StatusEvent
|
||||
| PlanReviewEvent
|
||||
| InfoEvent
|
||||
| ErrorEvent
|
||||
| BusyErrorEvent
|
||||
| ClearUiEvent
|
||||
| WsStateEvent
|
||||
| WsActivityEvent
|
||||
| WsRenameEvent
|
||||
| WsClosedEvent;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Console cluster SSE events
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface NodeJoinedEvent {
|
||||
type: "node_joined";
|
||||
node_id: string;
|
||||
}
|
||||
|
||||
export interface NodeLostEvent {
|
||||
type: "node_lost";
|
||||
node_id: string;
|
||||
}
|
||||
|
||||
export interface ClusterStateEvent {
|
||||
type: "cluster_state";
|
||||
ws_id: string;
|
||||
node_id: string;
|
||||
state: string;
|
||||
tokens: number;
|
||||
context_ratio: number;
|
||||
activity: string;
|
||||
activity_state: string;
|
||||
}
|
||||
|
||||
export interface ClusterWsCreatedEvent {
|
||||
type: "ws_created";
|
||||
ws_id: string;
|
||||
node_id: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface ClusterWsClosedEvent {
|
||||
type: "ws_closed";
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface ClusterWsRenameEvent {
|
||||
type: "ws_rename";
|
||||
ws_id: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/** Discriminated union of all console cluster SSE event types. */
|
||||
export type ClusterEvent =
|
||||
| NodeJoinedEvent
|
||||
| NodeLostEvent
|
||||
| ClusterStateEvent
|
||||
| ClusterWsCreatedEvent
|
||||
| ClusterWsClosedEvent
|
||||
| ClusterWsRenameEvent;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Type guards
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function isContentEvent(e: ServerEvent): e is ContentEvent {
|
||||
return e.type === "content";
|
||||
}
|
||||
|
||||
export function isReasoningEvent(e: ServerEvent): e is ReasoningEvent {
|
||||
return e.type === "reasoning";
|
||||
}
|
||||
|
||||
export function isErrorEvent(e: ServerEvent): e is ErrorEvent {
|
||||
return e.type === "error";
|
||||
}
|
||||
|
||||
export function isStreamEndEvent(e: ServerEvent): e is StreamEndEvent {
|
||||
return e.type === "stream_end";
|
||||
}
|
||||
|
||||
export function isToolResultEvent(e: ServerEvent): e is ToolResultEvent {
|
||||
return e.type === "tool_result";
|
||||
}
|
||||
|
||||
export function isWsStateEvent(e: ServerEvent): e is WsStateEvent {
|
||||
return e.type === "ws_state";
|
||||
}
|
||||
|
||||
export function isApproveRequestEvent(
|
||||
e: ServerEvent,
|
||||
): e is ApproveRequestEvent {
|
||||
return e.type === "approve_request";
|
||||
}
|
||||
|
||||
export function isPlanReviewEvent(e: ServerEvent): e is PlanReviewEvent {
|
||||
return e.type === "plan_review";
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* @turnstone/sdk — TypeScript client SDK for the turnstone AI orchestration platform.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import { TurnstoneServer } from "@turnstone/sdk";
|
||||
*
|
||||
* const client = new TurnstoneServer({
|
||||
* baseUrl: "http://localhost:8080",
|
||||
* token: "tok_xxx",
|
||||
* });
|
||||
*
|
||||
* const ws = await client.createWorkstream({ name: "demo" });
|
||||
* const result = await client.sendAndWait("Hello!", ws.ws_id);
|
||||
* console.log(result.content);
|
||||
* ```
|
||||
*/
|
||||
|
||||
// Clients
|
||||
export { TurnstoneServer } from "./server.js";
|
||||
export { TurnstoneConsole } from "./console.js";
|
||||
export type { ClientOptions } from "./base.js";
|
||||
|
||||
// Errors
|
||||
export { TurnstoneAPIError } from "./errors.js";
|
||||
|
||||
// Event types and guards
|
||||
export type {
|
||||
ServerEvent,
|
||||
ClusterEvent,
|
||||
ConnectedEvent,
|
||||
HistoryEvent,
|
||||
ThinkingStartEvent,
|
||||
ThinkingStopEvent,
|
||||
ContentEvent,
|
||||
ReasoningEvent,
|
||||
StreamEndEvent,
|
||||
ToolInfoEvent,
|
||||
ApproveRequestEvent,
|
||||
ToolResultEvent,
|
||||
ToolOutputChunkEvent,
|
||||
StatusEvent,
|
||||
PlanReviewEvent,
|
||||
InfoEvent,
|
||||
ErrorEvent,
|
||||
BusyErrorEvent,
|
||||
ClearUiEvent,
|
||||
WsStateEvent,
|
||||
WsActivityEvent,
|
||||
WsRenameEvent,
|
||||
WsClosedEvent,
|
||||
NodeJoinedEvent,
|
||||
NodeLostEvent,
|
||||
ClusterStateEvent,
|
||||
ClusterWsCreatedEvent,
|
||||
ClusterWsClosedEvent,
|
||||
ClusterWsRenameEvent,
|
||||
} from "./events.js";
|
||||
|
||||
export {
|
||||
isContentEvent,
|
||||
isReasoningEvent,
|
||||
isErrorEvent,
|
||||
isStreamEndEvent,
|
||||
isToolResultEvent,
|
||||
isWsStateEvent,
|
||||
isApproveRequestEvent,
|
||||
isPlanReviewEvent,
|
||||
} from "./events.js";
|
||||
|
||||
// Request/response types
|
||||
export type {
|
||||
SendRequest,
|
||||
SendResponse,
|
||||
ApproveRequest,
|
||||
PlanFeedbackRequest,
|
||||
CommandRequest,
|
||||
CreateWorkstreamRequest,
|
||||
CreateWorkstreamResponse,
|
||||
CloseWorkstreamRequest,
|
||||
WorkstreamInfo,
|
||||
ListWorkstreamsResponse,
|
||||
DashboardWorkstream,
|
||||
DashboardAggregate,
|
||||
DashboardResponse,
|
||||
SessionInfo,
|
||||
ListSessionsResponse,
|
||||
BackendStatus,
|
||||
WorkstreamCounts,
|
||||
HealthResponse,
|
||||
AuthLoginRequest,
|
||||
AuthLoginResponse,
|
||||
AuthStatusResponse,
|
||||
AuthSetupResponse,
|
||||
StatusResponse,
|
||||
ErrorResponse,
|
||||
ClusterOverviewResponse,
|
||||
ClusterNodeInfo,
|
||||
ClusterNodesResponse,
|
||||
ClusterWorkstreamInfo,
|
||||
ClusterWorkstreamsResponse,
|
||||
NodeDetailResponse,
|
||||
ConsoleCreateWsRequest,
|
||||
ConsoleCreateWsResponse,
|
||||
ConsoleHealthResponse,
|
||||
CreateScheduleRequest,
|
||||
UpdateScheduleRequest,
|
||||
ScheduleInfo,
|
||||
ScheduleRunInfo,
|
||||
ListSchedulesResponse,
|
||||
ListScheduleRunsResponse,
|
||||
TurnResult,
|
||||
SendAndWaitOptions,
|
||||
NodesOptions,
|
||||
WorkstreamsOptions,
|
||||
} from "./types.js";
|
||||
|
||||
// SSE parser (for advanced usage)
|
||||
export { parseSSEStream } from "./sse.js";
|
||||
@@ -0,0 +1,228 @@
|
||||
import { BaseClient, type ClientOptions } from "./base.js";
|
||||
import type { ServerEvent } from "./events.js";
|
||||
import type {
|
||||
AuthLoginResponse,
|
||||
AuthSetupResponse,
|
||||
AuthStatusResponse,
|
||||
CreateWorkstreamRequest,
|
||||
CreateWorkstreamResponse,
|
||||
DashboardResponse,
|
||||
HealthResponse,
|
||||
ListSessionsResponse,
|
||||
ListWorkstreamsResponse,
|
||||
SendAndWaitOptions,
|
||||
SendResponse,
|
||||
StatusResponse,
|
||||
TurnResult,
|
||||
} from "./types.js";
|
||||
|
||||
/** Async client for the turnstone server API. */
|
||||
export class TurnstoneServer extends BaseClient {
|
||||
constructor(options: ClientOptions) {
|
||||
super(options);
|
||||
}
|
||||
|
||||
// -- Workstream management ------------------------------------------------
|
||||
|
||||
async listWorkstreams(): Promise<ListWorkstreamsResponse> {
|
||||
return this.request("GET", "/v1/api/workstreams");
|
||||
}
|
||||
|
||||
async dashboard(): Promise<DashboardResponse> {
|
||||
return this.request("GET", "/v1/api/dashboard");
|
||||
}
|
||||
|
||||
async createWorkstream(
|
||||
opts?: CreateWorkstreamRequest,
|
||||
): Promise<CreateWorkstreamResponse> {
|
||||
return this.request("POST", "/v1/api/workstreams/new", { json: opts });
|
||||
}
|
||||
|
||||
async closeWorkstream(wsId: string): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/workstreams/close", {
|
||||
json: { ws_id: wsId },
|
||||
});
|
||||
}
|
||||
|
||||
// -- Chat interaction -----------------------------------------------------
|
||||
|
||||
async send(message: string, wsId: string): Promise<SendResponse> {
|
||||
return this.request("POST", "/v1/api/send", {
|
||||
json: { message, ws_id: wsId },
|
||||
});
|
||||
}
|
||||
|
||||
async approve(opts: {
|
||||
wsId: string;
|
||||
approved?: boolean;
|
||||
feedback?: string | null;
|
||||
always?: boolean;
|
||||
}): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/approve", {
|
||||
json: {
|
||||
ws_id: opts.wsId,
|
||||
approved: opts.approved ?? true,
|
||||
feedback: opts.feedback,
|
||||
always: opts.always,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async planFeedback(opts: {
|
||||
wsId: string;
|
||||
feedback?: string;
|
||||
}): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/plan", {
|
||||
json: { ws_id: opts.wsId, feedback: opts.feedback ?? "" },
|
||||
});
|
||||
}
|
||||
|
||||
async command(opts: {
|
||||
wsId: string;
|
||||
command: string;
|
||||
}): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/command", {
|
||||
json: { ws_id: opts.wsId, command: opts.command },
|
||||
});
|
||||
}
|
||||
|
||||
// -- Streaming ------------------------------------------------------------
|
||||
|
||||
async *streamEvents(wsId: string): AsyncIterableIterator<ServerEvent> {
|
||||
yield* this.streamSSE<ServerEvent>("/v1/api/events", { ws_id: wsId });
|
||||
}
|
||||
|
||||
async *streamGlobalEvents(): AsyncIterableIterator<ServerEvent> {
|
||||
yield* this.streamSSE<ServerEvent>("/v1/api/events/global");
|
||||
}
|
||||
|
||||
// -- High-level convenience -----------------------------------------------
|
||||
|
||||
async sendAndWait(
|
||||
message: string,
|
||||
wsId: string,
|
||||
opts?: SendAndWaitOptions,
|
||||
): Promise<TurnResult> {
|
||||
const result: TurnResult = {
|
||||
wsId,
|
||||
contentParts: [],
|
||||
reasoningParts: [],
|
||||
toolResults: [],
|
||||
errors: [],
|
||||
timedOut: false,
|
||||
get content() {
|
||||
return this.contentParts.join("");
|
||||
},
|
||||
get reasoning() {
|
||||
return this.reasoningParts.join("");
|
||||
},
|
||||
get ok() {
|
||||
return !this.timedOut && this.errors.length === 0;
|
||||
},
|
||||
};
|
||||
|
||||
// Open SSE stream BEFORE sending to avoid missing early events
|
||||
const timeoutMs = opts?.timeout ?? 600_000;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
// Start consuming the per-workstream SSE stream first
|
||||
const events = this.streamSSE<ServerEvent>(
|
||||
"/v1/api/events",
|
||||
{ ws_id: wsId },
|
||||
controller.signal,
|
||||
);
|
||||
|
||||
const sendResp = await this.send(message, wsId);
|
||||
if (sendResp.status === "busy") {
|
||||
result.errors.push("Workstream is busy");
|
||||
return result;
|
||||
}
|
||||
|
||||
for await (const event of events) {
|
||||
opts?.onEvent?.(event);
|
||||
|
||||
switch (event.type) {
|
||||
case "content":
|
||||
result.contentParts.push(event.text);
|
||||
break;
|
||||
case "reasoning":
|
||||
result.reasoningParts.push(event.text);
|
||||
break;
|
||||
case "tool_result":
|
||||
result.toolResults.push({
|
||||
name: event.name,
|
||||
output: event.output,
|
||||
});
|
||||
break;
|
||||
case "error":
|
||||
result.errors.push(event.message);
|
||||
break;
|
||||
case "ws_state":
|
||||
if (event.state === "idle") return result;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name === "AbortError") {
|
||||
result.timedOut = true;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
controller.abort();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// -- Sessions -------------------------------------------------------------
|
||||
|
||||
async listSessions(): Promise<ListSessionsResponse> {
|
||||
return this.request("GET", "/v1/api/sessions");
|
||||
}
|
||||
|
||||
// -- Auth -----------------------------------------------------------------
|
||||
|
||||
async login(opts: {
|
||||
token?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
}): Promise<AuthLoginResponse> {
|
||||
const body =
|
||||
opts.username && opts.password
|
||||
? { username: opts.username, password: opts.password }
|
||||
: { token: opts.token ?? "" };
|
||||
return this.request("POST", "/v1/api/auth/login", { json: body });
|
||||
}
|
||||
|
||||
async authStatus(): Promise<AuthStatusResponse> {
|
||||
return this.request("GET", "/v1/api/auth/status");
|
||||
}
|
||||
|
||||
async setup(opts: {
|
||||
username: string;
|
||||
displayName: string;
|
||||
password: string;
|
||||
}): Promise<AuthSetupResponse> {
|
||||
return this.request("POST", "/v1/api/auth/setup", {
|
||||
json: {
|
||||
username: opts.username,
|
||||
display_name: opts.displayName,
|
||||
password: opts.password,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async logout(): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/auth/logout");
|
||||
}
|
||||
|
||||
// -- Health ---------------------------------------------------------------
|
||||
|
||||
async health(): Promise<HealthResponse> {
|
||||
return this.request("GET", "/health");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* SSE stream parser for fetch ReadableStream.
|
||||
*
|
||||
* Parses standard Server-Sent Events from a `Response.body` stream.
|
||||
* Works in browsers and Node.js 18+ natively (no dependencies).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Parse an SSE stream and yield JSON-parsed data payloads.
|
||||
*
|
||||
* Handles the standard SSE format including multi-line `data:` fields
|
||||
* (joined with `\n` per the SSE spec) and CRLF line endings.
|
||||
*/
|
||||
export async function* parseSSEStream<T = Record<string, unknown>>(
|
||||
response: Response,
|
||||
): AsyncIterableIterator<T> {
|
||||
const body = response.body;
|
||||
if (!body) return;
|
||||
|
||||
const reader = body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
// Normalize CRLF to LF
|
||||
buffer = buffer.replace(/\r\n/g, "\n");
|
||||
|
||||
// Process complete SSE frames (separated by double newlines)
|
||||
const frames = buffer.split("\n\n");
|
||||
// Keep the last (possibly incomplete) frame in the buffer
|
||||
buffer = frames.pop() ?? "";
|
||||
|
||||
for (const frame of frames) {
|
||||
if (!frame.trim()) continue;
|
||||
|
||||
// Extract data lines from the frame, joining with \n per SSE spec
|
||||
const dataLines: string[] = [];
|
||||
for (const line of frame.split("\n")) {
|
||||
if (line.startsWith("data: ")) {
|
||||
dataLines.push(line.slice(6));
|
||||
} else if (line.startsWith("data:")) {
|
||||
dataLines.push(line.slice(5));
|
||||
}
|
||||
}
|
||||
|
||||
if (dataLines.length === 0) continue;
|
||||
const data = dataLines.join("\n");
|
||||
if (!data.trim()) continue;
|
||||
|
||||
try {
|
||||
yield JSON.parse(data) as T;
|
||||
} catch {
|
||||
// Skip malformed JSON
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ErrorResponse {
|
||||
error: string;
|
||||
}
|
||||
|
||||
export interface StatusResponse {
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface AuthLoginRequest {
|
||||
token: string;
|
||||
}
|
||||
|
||||
export interface AuthLoginResponse {
|
||||
status: string;
|
||||
role: string;
|
||||
scopes?: string;
|
||||
jwt?: string;
|
||||
user_id?: string;
|
||||
}
|
||||
|
||||
export interface AuthStatusResponse {
|
||||
auth_enabled: boolean;
|
||||
has_users: boolean;
|
||||
setup_required: boolean;
|
||||
}
|
||||
|
||||
export interface AuthSetupResponse {
|
||||
status: string;
|
||||
user_id: string;
|
||||
username: string;
|
||||
role: string;
|
||||
scopes: string;
|
||||
jwt?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server API — Workstream management
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SendRequest {
|
||||
message: string;
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface SendResponse {
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface ApproveRequest {
|
||||
approved: boolean;
|
||||
feedback?: string | null;
|
||||
always?: boolean;
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface PlanFeedbackRequest {
|
||||
feedback: string;
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface CommandRequest {
|
||||
command: string;
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface CreateWorkstreamRequest {
|
||||
name?: string;
|
||||
model?: string;
|
||||
auto_approve?: boolean;
|
||||
resume_session?: string;
|
||||
}
|
||||
|
||||
export interface CreateWorkstreamResponse {
|
||||
ws_id: string;
|
||||
name: string;
|
||||
resumed?: boolean;
|
||||
session_id?: string;
|
||||
message_count?: number;
|
||||
}
|
||||
|
||||
export interface CloseWorkstreamRequest {
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface WorkstreamInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
state: string;
|
||||
session_id?: string | null;
|
||||
}
|
||||
|
||||
export interface ListWorkstreamsResponse {
|
||||
workstreams: WorkstreamInfo[];
|
||||
}
|
||||
|
||||
export interface DashboardWorkstream {
|
||||
id: string;
|
||||
name: string;
|
||||
state: string;
|
||||
session_id?: string | null;
|
||||
title?: string;
|
||||
tokens?: number;
|
||||
context_ratio?: number;
|
||||
activity?: string;
|
||||
activity_state?: string;
|
||||
tool_calls?: number;
|
||||
node?: string;
|
||||
model?: string;
|
||||
model_alias?: string;
|
||||
}
|
||||
|
||||
export interface DashboardAggregate {
|
||||
total_tokens: number;
|
||||
total_tool_calls: number;
|
||||
active_count: number;
|
||||
total_count: number;
|
||||
uptime_seconds?: number;
|
||||
node?: string;
|
||||
}
|
||||
|
||||
export interface DashboardResponse {
|
||||
workstreams: DashboardWorkstream[];
|
||||
aggregate: DashboardAggregate;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server API — Sessions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SessionInfo {
|
||||
session_id: string;
|
||||
alias?: string | null;
|
||||
title?: string | null;
|
||||
created: string;
|
||||
updated: string;
|
||||
message_count: number;
|
||||
}
|
||||
|
||||
export interface ListSessionsResponse {
|
||||
sessions: SessionInfo[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server API — Health
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface BackendStatus {
|
||||
status: string;
|
||||
circuit_state: string;
|
||||
}
|
||||
|
||||
export interface WorkstreamCounts {
|
||||
total: number;
|
||||
idle?: number;
|
||||
thinking?: number;
|
||||
running?: number;
|
||||
attention?: number;
|
||||
error?: number;
|
||||
}
|
||||
|
||||
export interface HealthResponse {
|
||||
status: string;
|
||||
version?: string;
|
||||
uptime_seconds?: number;
|
||||
model?: string;
|
||||
workstreams?: WorkstreamCounts;
|
||||
backend?: BackendStatus | null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Console API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface StateCounts {
|
||||
running?: number;
|
||||
thinking?: number;
|
||||
attention?: number;
|
||||
idle?: number;
|
||||
error?: number;
|
||||
}
|
||||
|
||||
export interface ClusterAggregate {
|
||||
total_tokens: number;
|
||||
total_tool_calls: number;
|
||||
}
|
||||
|
||||
export interface ClusterOverviewResponse {
|
||||
nodes: number;
|
||||
workstreams: number;
|
||||
states: StateCounts;
|
||||
aggregate: ClusterAggregate;
|
||||
version_drift: boolean;
|
||||
versions: string[];
|
||||
}
|
||||
|
||||
export interface ClusterNodeInfo {
|
||||
node_id: string;
|
||||
server_url: string;
|
||||
ws_total: number;
|
||||
ws_running: number;
|
||||
ws_thinking: number;
|
||||
ws_attention: number;
|
||||
ws_idle: number;
|
||||
ws_error: number;
|
||||
total_tokens: number;
|
||||
started: number;
|
||||
reachable: boolean;
|
||||
health: Record<string, string>;
|
||||
version: string;
|
||||
}
|
||||
|
||||
export interface ClusterNodesResponse {
|
||||
nodes: ClusterNodeInfo[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ClusterWorkstreamInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
state: string;
|
||||
node: string;
|
||||
title?: string;
|
||||
tokens?: number;
|
||||
context_ratio?: number;
|
||||
activity?: string;
|
||||
activity_state?: string;
|
||||
tool_calls?: number;
|
||||
}
|
||||
|
||||
export interface ClusterWorkstreamsResponse {
|
||||
workstreams: ClusterWorkstreamInfo[];
|
||||
total: number;
|
||||
page: number;
|
||||
per_page: number;
|
||||
pages: number;
|
||||
}
|
||||
|
||||
export interface NodeDetailResponse {
|
||||
node_id: string;
|
||||
server_url: string;
|
||||
health: Record<string, string>;
|
||||
workstreams: ClusterWorkstreamInfo[];
|
||||
aggregate: ClusterAggregate;
|
||||
}
|
||||
|
||||
export interface ConsoleCreateWsRequest {
|
||||
node_id?: string;
|
||||
name?: string;
|
||||
model?: string;
|
||||
initial_message?: string;
|
||||
}
|
||||
|
||||
export interface ConsoleCreateWsResponse {
|
||||
status: string;
|
||||
correlation_id: string;
|
||||
target_node: string;
|
||||
}
|
||||
|
||||
export interface ConsoleHealthResponse {
|
||||
status: string;
|
||||
service: string;
|
||||
nodes: number;
|
||||
workstreams: number;
|
||||
version_drift: boolean;
|
||||
versions: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Console API — Schedules
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface CreateScheduleRequest {
|
||||
name: string;
|
||||
schedule_type: string;
|
||||
initial_message: string;
|
||||
description?: string;
|
||||
cron_expr?: string;
|
||||
at_time?: string;
|
||||
target_mode?: string;
|
||||
model?: string;
|
||||
auto_approve?: boolean;
|
||||
auto_approve_tools?: string[];
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
export interface UpdateScheduleRequest {
|
||||
name?: string;
|
||||
description?: string;
|
||||
schedule_type?: string;
|
||||
cron_expr?: string;
|
||||
at_time?: string;
|
||||
target_mode?: string;
|
||||
model?: string;
|
||||
initial_message?: string;
|
||||
auto_approve?: boolean;
|
||||
auto_approve_tools?: string[];
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
export interface ScheduleInfo {
|
||||
task_id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
schedule_type: string;
|
||||
cron_expr: string;
|
||||
at_time: string;
|
||||
target_mode: string;
|
||||
model: string;
|
||||
initial_message: string;
|
||||
auto_approve: boolean;
|
||||
auto_approve_tools: string[];
|
||||
enabled: boolean;
|
||||
created_by: string;
|
||||
last_run: string | null;
|
||||
next_run: string | null;
|
||||
created: string;
|
||||
updated: string;
|
||||
}
|
||||
|
||||
export interface ListSchedulesResponse {
|
||||
schedules: ScheduleInfo[];
|
||||
}
|
||||
|
||||
export interface ScheduleRunInfo {
|
||||
run_id: string;
|
||||
task_id: string;
|
||||
node_id: string;
|
||||
ws_id: string;
|
||||
correlation_id: string;
|
||||
started: string;
|
||||
status: string;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export interface ListScheduleRunsResponse {
|
||||
runs: ScheduleRunInfo[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SDK-specific types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface TurnResult {
|
||||
wsId: string;
|
||||
contentParts: string[];
|
||||
reasoningParts: string[];
|
||||
toolResults: Array<{ name: string; output: string }>;
|
||||
errors: string[];
|
||||
timedOut: boolean;
|
||||
content: string;
|
||||
reasoning: string;
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
export interface SendAndWaitOptions {
|
||||
/** Timeout in milliseconds (default: 600000 = 10 minutes). */
|
||||
timeout?: number;
|
||||
onEvent?: (event: import("./events.js").ServerEvent) => void;
|
||||
}
|
||||
|
||||
export interface NodesOptions {
|
||||
sort?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export interface WorkstreamsOptions {
|
||||
state?: string;
|
||||
node?: string;
|
||||
search?: string;
|
||||
sort?: string;
|
||||
page?: number;
|
||||
per_page?: number;
|
||||
}
|
||||
|
||||
// Re-export event types for convenience
|
||||
export type { ServerEvent, ClusterEvent } from "./events.js";
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { TurnstoneConsole } from "../src/console.js";
|
||||
|
||||
function mockFetch(response: object): typeof globalThis.fetch {
|
||||
return vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify(response), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("TurnstoneConsole", () => {
|
||||
it("overview returns parsed response", async () => {
|
||||
const fetchFn = mockFetch({
|
||||
nodes: 2,
|
||||
workstreams: 5,
|
||||
states: { idle: 5 },
|
||||
aggregate: { total_tokens: 1000, total_tool_calls: 0 },
|
||||
version_drift: false,
|
||||
versions: ["0.3.0"],
|
||||
});
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const resp = await client.overview();
|
||||
expect(resp.nodes).toBe(2);
|
||||
expect(resp.workstreams).toBe(5);
|
||||
});
|
||||
|
||||
it("nodes passes query parameters", async () => {
|
||||
const fetchFn = mockFetch({ nodes: [], total: 0 });
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await client.nodes({ sort: "tokens", limit: 50, offset: 10 });
|
||||
|
||||
const [url] = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(url).toContain("sort=tokens");
|
||||
expect(url).toContain("limit=50");
|
||||
expect(url).toContain("offset=10");
|
||||
});
|
||||
|
||||
it("workstreams passes filter parameters", async () => {
|
||||
const fetchFn = mockFetch({
|
||||
workstreams: [],
|
||||
total: 0,
|
||||
page: 1,
|
||||
per_page: 50,
|
||||
pages: 0,
|
||||
});
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await client.workstreams({ state: "running", page: 2 });
|
||||
|
||||
const [url] = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(url).toContain("state=running");
|
||||
expect(url).toContain("page=2");
|
||||
});
|
||||
|
||||
it("health returns parsed response", async () => {
|
||||
const fetchFn = mockFetch({
|
||||
status: "ok",
|
||||
service: "turnstone-console",
|
||||
nodes: 2,
|
||||
workstreams: 5,
|
||||
version_drift: false,
|
||||
versions: ["0.3.0"],
|
||||
});
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const resp = await client.health();
|
||||
expect(resp.status).toBe("ok");
|
||||
expect(resp.nodes).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isContentEvent,
|
||||
isErrorEvent,
|
||||
isStreamEndEvent,
|
||||
isToolResultEvent,
|
||||
isWsStateEvent,
|
||||
isApproveRequestEvent,
|
||||
isPlanReviewEvent,
|
||||
isReasoningEvent,
|
||||
} from "../src/events.js";
|
||||
import type { ServerEvent } from "../src/events.js";
|
||||
|
||||
describe("event type guards", () => {
|
||||
it("isContentEvent", () => {
|
||||
const e: ServerEvent = { type: "content", text: "hello" };
|
||||
expect(isContentEvent(e)).toBe(true);
|
||||
expect(isErrorEvent(e)).toBe(false);
|
||||
});
|
||||
|
||||
it("isReasoningEvent", () => {
|
||||
const e: ServerEvent = { type: "reasoning", text: "step 1" };
|
||||
expect(isReasoningEvent(e)).toBe(true);
|
||||
expect(isContentEvent(e)).toBe(false);
|
||||
});
|
||||
|
||||
it("isErrorEvent", () => {
|
||||
const e: ServerEvent = { type: "error", message: "bad" };
|
||||
expect(isErrorEvent(e)).toBe(true);
|
||||
});
|
||||
|
||||
it("isStreamEndEvent", () => {
|
||||
const e: ServerEvent = { type: "stream_end" };
|
||||
expect(isStreamEndEvent(e)).toBe(true);
|
||||
});
|
||||
|
||||
it("isToolResultEvent", () => {
|
||||
const e: ServerEvent = {
|
||||
type: "tool_result",
|
||||
call_id: "c1",
|
||||
name: "search",
|
||||
output: "found",
|
||||
};
|
||||
expect(isToolResultEvent(e)).toBe(true);
|
||||
});
|
||||
|
||||
it("isWsStateEvent", () => {
|
||||
const e: ServerEvent = {
|
||||
type: "ws_state",
|
||||
ws_id: "ws1",
|
||||
state: "idle",
|
||||
tokens: 0,
|
||||
context_ratio: 0,
|
||||
activity: "",
|
||||
activity_state: "",
|
||||
};
|
||||
expect(isWsStateEvent(e)).toBe(true);
|
||||
});
|
||||
|
||||
it("isApproveRequestEvent", () => {
|
||||
const e: ServerEvent = { type: "approve_request", items: [] };
|
||||
expect(isApproveRequestEvent(e)).toBe(true);
|
||||
});
|
||||
|
||||
it("isPlanReviewEvent", () => {
|
||||
const e: ServerEvent = { type: "plan_review", content: "## Plan" };
|
||||
expect(isPlanReviewEvent(e)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { TurnstoneServer } from "../src/server.js";
|
||||
import { TurnstoneAPIError } from "../src/errors.js";
|
||||
|
||||
function mockFetch(response: object, status = 200): typeof globalThis.fetch {
|
||||
return vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify(response), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function mockFetchError(
|
||||
error: object,
|
||||
status: number,
|
||||
): typeof globalThis.fetch {
|
||||
return vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify(error), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("TurnstoneServer", () => {
|
||||
it("listWorkstreams returns parsed response", async () => {
|
||||
const fetchFn = mockFetch({
|
||||
workstreams: [{ id: "ws1", name: "test", state: "idle" }],
|
||||
});
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const resp = await client.listWorkstreams();
|
||||
expect(resp.workstreams).toHaveLength(1);
|
||||
expect(resp.workstreams[0].id).toBe("ws1");
|
||||
expect(fetchFn).toHaveBeenCalledWith(
|
||||
"http://test/v1/api/workstreams",
|
||||
expect.objectContaining({ method: "GET" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("createWorkstream sends correct body", async () => {
|
||||
const fetchFn = mockFetch({ ws_id: "ws_new", name: "Analysis" });
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const resp = await client.createWorkstream({ name: "Analysis" });
|
||||
expect(resp.ws_id).toBe("ws_new");
|
||||
|
||||
const [, init] = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(JSON.parse(init.body)).toEqual({ name: "Analysis" });
|
||||
});
|
||||
|
||||
it("send posts correct payload", async () => {
|
||||
const fetchFn = mockFetch({ status: "ok" });
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await client.send("Hello", "ws1");
|
||||
|
||||
const [url, init] = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(url).toBe("http://test/v1/api/send");
|
||||
expect(JSON.parse(init.body)).toEqual({ message: "Hello", ws_id: "ws1" });
|
||||
});
|
||||
|
||||
it("injects auth header when token provided", async () => {
|
||||
const fetchFn = mockFetch({ workstreams: [] });
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
token: "tok_abc",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await client.listWorkstreams();
|
||||
|
||||
const [, init] = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(init.headers.Authorization).toBe("Bearer tok_abc");
|
||||
});
|
||||
|
||||
it("throws TurnstoneAPIError on 404", async () => {
|
||||
const fetchFn = mockFetchError({ error: "Not found" }, 404);
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await expect(client.send("hi", "bad_ws")).rejects.toThrow(
|
||||
TurnstoneAPIError,
|
||||
);
|
||||
try {
|
||||
await client.send("hi", "bad_ws");
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(TurnstoneAPIError);
|
||||
expect((e as TurnstoneAPIError).statusCode).toBe(404);
|
||||
}
|
||||
});
|
||||
|
||||
it("health returns parsed response", async () => {
|
||||
const fetchFn = mockFetch({
|
||||
status: "ok",
|
||||
version: "0.3.0",
|
||||
uptime_seconds: 120,
|
||||
});
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const resp = await client.health();
|
||||
expect(resp.status).toBe("ok");
|
||||
expect(resp.version).toBe("0.3.0");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseSSEStream } from "../src/sse.js";
|
||||
|
||||
function makeSSEResponse(...events: string[]): Response {
|
||||
const body = events.map((e) => `data: ${e}\n\n`).join("");
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(body));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
return new Response(stream, {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}
|
||||
|
||||
describe("parseSSEStream", () => {
|
||||
it("yields parsed JSON from SSE data lines", async () => {
|
||||
const resp = makeSSEResponse(
|
||||
'{"type": "content", "text": "hello"}',
|
||||
'{"type": "stream_end"}',
|
||||
);
|
||||
const events: unknown[] = [];
|
||||
for await (const event of parseSSEStream(resp)) {
|
||||
events.push(event);
|
||||
}
|
||||
expect(events).toHaveLength(2);
|
||||
expect(events[0]).toEqual({ type: "content", text: "hello" });
|
||||
expect(events[1]).toEqual({ type: "stream_end" });
|
||||
});
|
||||
|
||||
it("skips malformed JSON", async () => {
|
||||
const resp = makeSSEResponse(
|
||||
"not-json",
|
||||
'{"type": "info", "message": "ok"}',
|
||||
);
|
||||
const events: unknown[] = [];
|
||||
for await (const event of parseSSEStream(resp)) {
|
||||
events.push(event);
|
||||
}
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toEqual({ type: "info", message: "ok" });
|
||||
});
|
||||
|
||||
it("handles multiple events in sequence", async () => {
|
||||
const resp = makeSSEResponse(
|
||||
'{"type": "connected", "model": "gpt-5"}',
|
||||
'{"type": "content", "text": "a"}',
|
||||
'{"type": "content", "text": "b"}',
|
||||
'{"type": "status", "total_tokens": 10}',
|
||||
'{"type": "stream_end"}',
|
||||
);
|
||||
const events: unknown[] = [];
|
||||
for await (const event of parseSSEStream(resp)) {
|
||||
events.push(event);
|
||||
}
|
||||
expect(events).toHaveLength(5);
|
||||
const types = events.map((e) => (e as Record<string, unknown>).type);
|
||||
expect(types).toEqual([
|
||||
"connected",
|
||||
"content",
|
||||
"content",
|
||||
"status",
|
||||
"stream_end",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"lib": ["ES2022", "DOM"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "tests"]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["tests/**/*.test.ts"],
|
||||
},
|
||||
});
|
||||
+6
-6
@@ -4,15 +4,15 @@ import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_db(tmp_path, monkeypatch):
|
||||
"""Provide a temporary SQLite database."""
|
||||
import turnstone.core.memory as memory
|
||||
def tmp_db(tmp_path):
|
||||
"""Provide a temporary SQLite storage backend."""
|
||||
from turnstone.core.storage import init_storage, reset_storage
|
||||
|
||||
db_path = str(tmp_path / "test.db")
|
||||
monkeypatch.setattr(memory, "db_override", db_path)
|
||||
memory.db_initialized.discard(db_path)
|
||||
reset_storage()
|
||||
init_storage("sqlite", path=db_path, run_migrations=False)
|
||||
yield db_path
|
||||
memory.db_initialized.discard(db_path)
|
||||
reset_storage()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Integration tests for API versioning and OpenAPI/docs endpoints."""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestServerVersioning:
|
||||
"""Test /v1/ routes and OpenAPI endpoints on the server."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.core.auth import AuthConfig
|
||||
from turnstone.server import create_app
|
||||
|
||||
mock_mgr = MagicMock()
|
||||
mock_mgr.list_all.return_value = []
|
||||
app = create_app(
|
||||
workstreams=mock_mgr,
|
||||
global_queue=queue.Queue(),
|
||||
global_listeners=[],
|
||||
global_listeners_lock=threading.Lock(),
|
||||
skip_permissions=False,
|
||||
auth_config=AuthConfig(),
|
||||
)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_v1_workstreams(self, client):
|
||||
resp = client.get("/v1/api/workstreams")
|
||||
assert resp.status_code == 200
|
||||
assert "workstreams" in resp.json()
|
||||
|
||||
def test_unversioned_api_404(self, client):
|
||||
resp = client.get("/api/workstreams")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_openapi_json(self, client):
|
||||
resp = client.get("/openapi.json")
|
||||
assert resp.status_code == 200
|
||||
spec = resp.json()
|
||||
assert spec["openapi"] == "3.1.0"
|
||||
assert "/v1/api/send" in spec["paths"]
|
||||
|
||||
def test_docs_page(self, client):
|
||||
resp = client.get("/docs")
|
||||
assert resp.status_code == 200
|
||||
assert "swagger-ui" in resp.text.lower()
|
||||
|
||||
def test_health_unversioned(self, client):
|
||||
resp = client.get("/health")
|
||||
assert resp.status_code == 200
|
||||
assert "status" in resp.json()
|
||||
|
||||
def test_shared_static_unversioned(self, client):
|
||||
resp = client.get("/shared/base.css")
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
class TestConsoleVersioning:
|
||||
"""Test /v1/ routes and OpenAPI endpoints on the console."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.server import _load_static, create_app
|
||||
from turnstone.core.auth import AuthConfig
|
||||
|
||||
_load_static()
|
||||
collector = MagicMock(spec=ClusterCollector)
|
||||
collector.get_overview.return_value = {
|
||||
"nodes": 0,
|
||||
"workstreams": 0,
|
||||
"states": {},
|
||||
"aggregate": {},
|
||||
}
|
||||
app = create_app(
|
||||
collector=collector,
|
||||
broker=MagicMock(),
|
||||
auth_config=AuthConfig(),
|
||||
)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_v1_cluster_overview(self, client):
|
||||
resp = client.get("/v1/api/cluster/overview")
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_unversioned_api_404(self, client):
|
||||
resp = client.get("/api/cluster/overview")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_openapi_json(self, client):
|
||||
resp = client.get("/openapi.json")
|
||||
assert resp.status_code == 200
|
||||
spec = resp.json()
|
||||
assert spec["openapi"] == "3.1.0"
|
||||
assert "/v1/api/cluster/overview" in spec["paths"]
|
||||
|
||||
def test_docs_page(self, client):
|
||||
resp = client.get("/docs")
|
||||
assert resp.status_code == 200
|
||||
assert "swagger-ui" in resp.text.lower()
|
||||
|
||||
def test_health_unversioned(self, client):
|
||||
resp = client.get("/health")
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_console_app_js_uses_v1_paths(self, client):
|
||||
resp = client.get("/static/app.js")
|
||||
body = resp.text
|
||||
assert "/v1/api/cluster" in body
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Tests for turnstone.mq.async_broker.AsyncRedisBroker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.mq.async_broker import AsyncRedisBroker
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def broker() -> AsyncRedisBroker:
|
||||
return AsyncRedisBroker(host="localhost", port=6379, db=0, prefix="test", response_ttl=120)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis() -> AsyncMock:
|
||||
"""Return a mock Redis client with common async methods."""
|
||||
r = AsyncMock()
|
||||
r.rpush = AsyncMock()
|
||||
r.publish = AsyncMock()
|
||||
r.expire = AsyncMock()
|
||||
r.get = AsyncMock(return_value=None)
|
||||
r.set = AsyncMock()
|
||||
r.delete = AsyncMock()
|
||||
r.blpop = AsyncMock(return_value=None)
|
||||
ps = AsyncMock()
|
||||
ps.subscribe = AsyncMock()
|
||||
ps.unsubscribe = AsyncMock()
|
||||
ps.close = AsyncMock()
|
||||
ps.get_message = AsyncMock(return_value=None)
|
||||
r.pubsub = MagicMock(return_value=ps)
|
||||
return r
|
||||
|
||||
|
||||
def _inject_redis(broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
"""Inject a mock Redis client into the broker, simulating connect()."""
|
||||
broker._redis = mock_redis
|
||||
broker._pubsub = mock_redis.pubsub()
|
||||
|
||||
|
||||
class TestConstructor:
|
||||
def test_stores_config(self) -> None:
|
||||
b = AsyncRedisBroker(host="h", port=1234, db=2, prefix="pfx", password="pw")
|
||||
assert b._host == "h"
|
||||
assert b._port == 1234
|
||||
assert b._db == 2
|
||||
assert b._prefix == "pfx"
|
||||
assert b._password == "pw"
|
||||
assert b._redis is None
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
b = AsyncRedisBroker()
|
||||
assert b._host == "localhost"
|
||||
assert b._port == 6379
|
||||
assert b._prefix == "turnstone"
|
||||
|
||||
|
||||
class TestConnect:
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_connection(self) -> None:
|
||||
b = AsyncRedisBroker()
|
||||
mock_r = AsyncMock()
|
||||
mock_r.pubsub = MagicMock(return_value=AsyncMock())
|
||||
with patch("redis.asyncio.Redis", return_value=mock_r):
|
||||
await b.connect()
|
||||
assert b._redis is mock_r
|
||||
assert b._pubsub is not None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_connect_idempotent(
|
||||
self, broker: AsyncRedisBroker, mock_redis: AsyncMock
|
||||
) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
old = broker._redis
|
||||
await broker.connect()
|
||||
assert broker._redis is old
|
||||
|
||||
|
||||
class TestPushInbound:
|
||||
@pytest.mark.anyio
|
||||
async def test_shared_queue(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.push_inbound('{"type":"send"}')
|
||||
mock_redis.rpush.assert_awaited_once_with("test:inbound", '{"type":"send"}')
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_per_node_queue(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.push_inbound('{"type":"send"}', node_id="node-1")
|
||||
mock_redis.rpush.assert_awaited_once_with("test:inbound:node-1", '{"type":"send"}')
|
||||
|
||||
|
||||
class TestPublishOutbound:
|
||||
@pytest.mark.anyio
|
||||
async def test_publishes(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.publish_outbound("test:events:global", '{"event":"data"}')
|
||||
mock_redis.publish.assert_awaited_once_with("test:events:global", '{"event":"data"}')
|
||||
|
||||
|
||||
class TestPushResponse:
|
||||
@pytest.mark.anyio
|
||||
async def test_rpush_and_expire(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.push_response("req-123", '{"ok":true}')
|
||||
mock_redis.rpush.assert_awaited_once_with("test:resp:req-123", '{"ok":true}')
|
||||
mock_redis.expire.assert_awaited_once_with("test:resp:req-123", 120)
|
||||
|
||||
|
||||
class TestSubscribe:
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_task(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.subscribe("test:events:global", lambda msg: None)
|
||||
assert "test:events:global" in broker._callbacks
|
||||
assert broker._listener_task is not None
|
||||
assert isinstance(broker._listener_task, asyncio.Task)
|
||||
# Clean up.
|
||||
broker._listener_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await broker._listener_task
|
||||
|
||||
|
||||
class TestUnsubscribe:
|
||||
@pytest.mark.anyio
|
||||
async def test_cancels_task(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.subscribe("test:events:ch", lambda msg: None)
|
||||
assert "test:events:ch" in broker._callbacks
|
||||
await broker.unsubscribe("test:events:ch")
|
||||
assert "test:events:ch" not in broker._callbacks
|
||||
|
||||
|
||||
class TestRoutingPrimitives:
|
||||
@pytest.mark.anyio
|
||||
async def test_get_ws_owner(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
mock_redis.get.return_value = "node-1"
|
||||
result = await broker.get_ws_owner("ws-abc")
|
||||
mock_redis.get.assert_awaited_once_with("test:ws:ws-abc")
|
||||
assert result == "node-1"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_set_ws_owner(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.set_ws_owner("ws-abc", "node-2")
|
||||
mock_redis.set.assert_awaited_once_with("test:ws:ws-abc", "node-2")
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_set_ws_owner_with_ttl(
|
||||
self, broker: AsyncRedisBroker, mock_redis: AsyncMock
|
||||
) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.set_ws_owner("ws-abc", "node-2", ttl=300)
|
||||
mock_redis.set.assert_awaited_once_with("test:ws:ws-abc", "node-2", ex=300)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_del_ws_owner(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.del_ws_owner("ws-abc")
|
||||
mock_redis.delete.assert_awaited_once_with("test:ws:ws-abc")
|
||||
|
||||
|
||||
class TestClose:
|
||||
@pytest.mark.anyio
|
||||
async def test_cancels_tasks_and_closes(
|
||||
self, broker: AsyncRedisBroker, mock_redis: AsyncMock
|
||||
) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.subscribe("ch1", lambda m: None)
|
||||
assert len(broker._callbacks) == 1
|
||||
assert broker._listener_task is not None
|
||||
await broker.close()
|
||||
assert len(broker._callbacks) == 0
|
||||
assert broker._listener_task is None
|
||||
assert broker._redis is None
|
||||
assert broker._pubsub is None
|
||||
+620
-260
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,357 @@
|
||||
"""Tests for user identity, API tokens, JWT, and scoped auth."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.auth import (
|
||||
AuthConfig,
|
||||
AuthResult,
|
||||
_authenticate_token,
|
||||
check_request,
|
||||
create_jwt,
|
||||
generate_token,
|
||||
hash_password,
|
||||
hash_token,
|
||||
parse_scopes,
|
||||
required_scope,
|
||||
token_prefix,
|
||||
validate_jwt,
|
||||
verify_password,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AuthResult
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAuthResult:
|
||||
def test_frozen(self):
|
||||
r = AuthResult(user_id="u1", scopes=frozenset({"read"}), token_source="config")
|
||||
with pytest.raises(AttributeError):
|
||||
r.user_id = "u2" # type: ignore[misc]
|
||||
|
||||
def test_has_scope(self):
|
||||
r = AuthResult(user_id="", scopes=frozenset({"read", "write"}), token_source="config")
|
||||
assert r.has_scope("read")
|
||||
assert r.has_scope("write")
|
||||
assert not r.has_scope("approve")
|
||||
|
||||
def test_empty_scopes(self):
|
||||
r = AuthResult(user_id="", scopes=frozenset(), token_source="config")
|
||||
assert not r.has_scope("read")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token generation and hashing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTokenHelpers:
|
||||
def test_generate_token_format(self):
|
||||
tok = generate_token()
|
||||
assert tok.startswith("ts_")
|
||||
assert len(tok) == 3 + 64 # ts_ + 64 hex chars
|
||||
|
||||
def test_generate_token_unique(self):
|
||||
tokens = {generate_token() for _ in range(10)}
|
||||
assert len(tokens) == 10
|
||||
|
||||
def test_hash_token_deterministic(self):
|
||||
assert hash_token("ts_abc") == hash_token("ts_abc")
|
||||
|
||||
def test_hash_token_hex(self):
|
||||
h = hash_token("test")
|
||||
assert len(h) == 64 # SHA-256 hex
|
||||
int(h, 16) # valid hex
|
||||
|
||||
def test_token_prefix(self):
|
||||
assert token_prefix("ts_abcdefgh1234") == "ts_abcde"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Password hashing (bcrypt)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPasswordHashing:
|
||||
def test_hash_and_verify(self):
|
||||
pw = "hunter2"
|
||||
hashed = hash_password(pw)
|
||||
assert verify_password(pw, hashed)
|
||||
|
||||
def test_wrong_password(self):
|
||||
hashed = hash_password("correct")
|
||||
assert not verify_password("wrong", hashed)
|
||||
|
||||
def test_hash_is_different_each_time(self):
|
||||
h1 = hash_password("same")
|
||||
h2 = hash_password("same")
|
||||
assert h1 != h2 # different salts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scope parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseScopes:
|
||||
def test_single_scope(self):
|
||||
assert parse_scopes("read") == frozenset({"read"})
|
||||
|
||||
def test_hierarchy_write(self):
|
||||
assert parse_scopes("write") == frozenset({"read", "write"})
|
||||
|
||||
def test_hierarchy_approve(self):
|
||||
assert parse_scopes("approve") == frozenset({"read", "write", "approve"})
|
||||
|
||||
def test_comma_separated(self):
|
||||
assert parse_scopes("read,write") == frozenset({"read", "write"})
|
||||
|
||||
def test_redundant_scopes(self):
|
||||
# approve already includes read,write
|
||||
assert parse_scopes("read,approve") == frozenset({"read", "write", "approve"})
|
||||
|
||||
def test_empty_string(self):
|
||||
assert parse_scopes("") == frozenset()
|
||||
|
||||
def test_invalid_scope_filtered(self):
|
||||
assert parse_scopes("bogus") == frozenset()
|
||||
|
||||
def test_mixed_valid_invalid(self):
|
||||
assert parse_scopes("read,bogus,approve") == frozenset({"read", "write", "approve"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JWT create / validate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJWT:
|
||||
SECRET = "test-secret-key-for-jwt"
|
||||
|
||||
def test_round_trip(self):
|
||||
scopes = frozenset({"read", "write"})
|
||||
token = create_jwt("user123", scopes, "database", self.SECRET, expiry_hours=1)
|
||||
result = validate_jwt(token, self.SECRET)
|
||||
assert result is not None
|
||||
assert result.user_id == "user123"
|
||||
assert result.scopes == frozenset({"read", "write"})
|
||||
|
||||
def test_expired_token(self):
|
||||
import jwt
|
||||
|
||||
payload = {
|
||||
"sub": "user1",
|
||||
"scopes": "read",
|
||||
"src": "database",
|
||||
"iat": int(time.time()) - 7200,
|
||||
"exp": int(time.time()) - 3600,
|
||||
}
|
||||
token = jwt.encode(payload, self.SECRET, algorithm="HS256")
|
||||
assert validate_jwt(token, self.SECRET) is None
|
||||
|
||||
def test_invalid_signature(self):
|
||||
token = create_jwt("user1", frozenset({"read"}), "db", self.SECRET)
|
||||
assert validate_jwt(token, "wrong-secret") is None
|
||||
|
||||
def test_malformed_token(self):
|
||||
assert validate_jwt("not.a.jwt", self.SECRET) is None
|
||||
|
||||
def test_contains_dots(self):
|
||||
"""JWTs contain dots, used for detection."""
|
||||
token = create_jwt("u1", frozenset({"read"}), "db", self.SECRET)
|
||||
assert "." in token
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# required_scope
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRequiredScope:
|
||||
def test_get_read(self):
|
||||
assert required_scope("GET", "/api/workstreams") == "read"
|
||||
|
||||
def test_post_write(self):
|
||||
assert required_scope("POST", "/api/send") == "write"
|
||||
|
||||
def test_post_approve(self):
|
||||
assert required_scope("POST", "/api/approve") == "approve"
|
||||
|
||||
def test_admin_prefix(self):
|
||||
assert required_scope("GET", "/api/admin/users") == "approve"
|
||||
assert required_scope("POST", "/api/admin/users") == "approve"
|
||||
assert required_scope("DELETE", "/api/admin/users/abc") == "approve"
|
||||
|
||||
def test_versioned_path(self):
|
||||
assert required_scope("POST", "/v1/api/send") == "write"
|
||||
assert required_scope("POST", "/v1/api/approve") == "approve"
|
||||
|
||||
def test_proxy_write(self):
|
||||
assert required_scope("POST", "/node/n1/api/send") == "write"
|
||||
|
||||
def test_proxy_approve(self):
|
||||
assert required_scope("POST", "/node/n1/api/approve") == "approve"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _authenticate_token
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAuthenticateToken:
|
||||
def test_config_token_read(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_read": "read"})
|
||||
result = _authenticate_token("tok_read", cfg)
|
||||
assert result is not None
|
||||
assert result.scopes == frozenset({"read"})
|
||||
assert result.token_source == "config"
|
||||
|
||||
def test_config_token_full(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_full": "full"})
|
||||
result = _authenticate_token("tok_full", cfg)
|
||||
assert result is not None
|
||||
assert result.scopes == frozenset({"read", "write", "approve"})
|
||||
|
||||
def test_jwt_token(self):
|
||||
secret = "test-secret"
|
||||
jwt_tok = create_jwt("user1", frozenset({"read", "write"}), "db", secret)
|
||||
cfg = AuthConfig(enabled=True)
|
||||
result = _authenticate_token(jwt_tok, cfg, jwt_secret=secret)
|
||||
assert result is not None
|
||||
assert result.user_id == "user1"
|
||||
assert result.token_source == "db"
|
||||
|
||||
def test_api_token_with_storage(self):
|
||||
"""API tokens are looked up by hash in storage."""
|
||||
raw = generate_token()
|
||||
|
||||
class MockStorage:
|
||||
def get_api_token_by_hash(self, token_hash):
|
||||
expected = hash_token(raw)
|
||||
if token_hash == expected:
|
||||
return {
|
||||
"token_id": "tid",
|
||||
"token_prefix": "ts_abcde",
|
||||
"user_id": "user1",
|
||||
"name": "test",
|
||||
"scopes": "read,write",
|
||||
"created": "2026-01-01T00:00:00",
|
||||
}
|
||||
return None
|
||||
|
||||
cfg = AuthConfig(enabled=True)
|
||||
result = _authenticate_token(raw, cfg, storage=MockStorage())
|
||||
assert result is not None
|
||||
assert result.user_id == "user1"
|
||||
assert result.has_scope("write")
|
||||
assert result.token_source == "database"
|
||||
|
||||
def test_api_token_expired(self):
|
||||
"""Expired API tokens are rejected."""
|
||||
raw = generate_token()
|
||||
|
||||
class MockStorage:
|
||||
def get_api_token_by_hash(self, token_hash):
|
||||
return {
|
||||
"token_id": "tid",
|
||||
"token_prefix": "ts_abcde",
|
||||
"user_id": "user1",
|
||||
"name": "test",
|
||||
"scopes": "read",
|
||||
"created": "2020-01-01T00:00:00",
|
||||
"expires": "2020-01-02T00:00:00",
|
||||
}
|
||||
|
||||
cfg = AuthConfig(enabled=True)
|
||||
result = _authenticate_token(raw, cfg, storage=MockStorage())
|
||||
assert result is None
|
||||
|
||||
def test_unknown_token(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok": "full"})
|
||||
result = _authenticate_token("unknown", cfg)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_request with scopes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckRequestScopes:
|
||||
def test_config_read_on_write_403(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_read": "read"})
|
||||
allowed, status, msg, _ = check_request(cfg, "POST", "/api/send", "Bearer tok_read")
|
||||
assert not allowed
|
||||
assert status == 403
|
||||
assert "write" in msg
|
||||
|
||||
def test_config_read_on_approve_403(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_read": "read"})
|
||||
allowed, status, msg, _ = check_request(cfg, "POST", "/api/approve", "Bearer tok_read")
|
||||
assert not allowed
|
||||
assert status == 403
|
||||
assert "approve" in msg
|
||||
|
||||
def test_config_full_on_approve_ok(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_full": "full"})
|
||||
allowed, status, msg, result = check_request(cfg, "POST", "/api/approve", "Bearer tok_full")
|
||||
assert allowed
|
||||
assert result is not None
|
||||
assert result.has_scope("approve")
|
||||
|
||||
def test_jwt_with_scopes(self):
|
||||
secret = "test"
|
||||
jwt_tok = create_jwt("u1", frozenset({"read", "write"}), "db", secret)
|
||||
cfg = AuthConfig(enabled=True)
|
||||
allowed, status, msg, result = check_request(
|
||||
cfg,
|
||||
"POST",
|
||||
"/api/send",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=secret,
|
||||
)
|
||||
assert allowed
|
||||
assert result is not None
|
||||
assert result.user_id == "u1"
|
||||
|
||||
def test_jwt_insufficient_scope(self):
|
||||
secret = "test"
|
||||
jwt_tok = create_jwt("u1", frozenset({"read"}), "db", secret)
|
||||
cfg = AuthConfig(enabled=True)
|
||||
allowed, status, msg, _ = check_request(
|
||||
cfg,
|
||||
"POST",
|
||||
"/api/send",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=secret,
|
||||
)
|
||||
assert not allowed
|
||||
assert status == 403
|
||||
|
||||
def test_admin_path_requires_approve(self):
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_read": "read"})
|
||||
allowed, status, msg, _ = check_request(
|
||||
cfg,
|
||||
"GET",
|
||||
"/v1/api/admin/users",
|
||||
"Bearer tok_read",
|
||||
)
|
||||
assert not allowed
|
||||
assert status == 403
|
||||
|
||||
def test_backward_compat_role_full(self):
|
||||
"""Config tokens with role='full' get all scopes."""
|
||||
cfg = AuthConfig(enabled=True, tokens={"tok_full": "full"})
|
||||
allowed, _, _, result = check_request(
|
||||
cfg,
|
||||
"GET",
|
||||
"/v1/api/admin/users",
|
||||
"Bearer tok_full",
|
||||
)
|
||||
assert allowed
|
||||
assert result is not None
|
||||
assert result.has_scope("approve")
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Tests for the Discord channel adapter (bot, cog, views, config, CLI)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
discord = pytest.importorskip("discord")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run(coro):
|
||||
"""Run an async coroutine in a fresh event loop (no pytest-asyncio needed)."""
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _make_message(*, bot=False, guild=True, content="hello", channel=None):
|
||||
"""Build a mock ``discord.Message``."""
|
||||
msg = MagicMock(spec=discord.Message)
|
||||
msg.author = MagicMock()
|
||||
msg.author.bot = bot
|
||||
msg.author.id = 12345
|
||||
msg.content = content
|
||||
msg.guild = MagicMock() if guild else None
|
||||
msg.channel = channel or MagicMock()
|
||||
msg.mentions = []
|
||||
return msg
|
||||
|
||||
|
||||
def _make_interaction(*, footer_text=None, has_embeds=True):
|
||||
"""Build a mock ``discord.Interaction``."""
|
||||
interaction = MagicMock(spec=discord.Interaction)
|
||||
interaction.user = MagicMock()
|
||||
interaction.user.id = 67890
|
||||
interaction.response = MagicMock()
|
||||
interaction.response.send_message = AsyncMock()
|
||||
|
||||
if has_embeds and footer_text is not None:
|
||||
embed = MagicMock()
|
||||
embed.footer.text = footer_text
|
||||
interaction.message = MagicMock()
|
||||
interaction.message.embeds = [embed]
|
||||
elif not has_embeds:
|
||||
interaction.message = MagicMock()
|
||||
interaction.message.embeds = []
|
||||
else:
|
||||
interaction.message = None
|
||||
|
||||
return interaction
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DiscordConfig
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDiscordConfig:
|
||||
"""Tests for DiscordConfig default and custom values."""
|
||||
|
||||
def test_defaults(self):
|
||||
from turnstone.channels.discord.config import DiscordConfig
|
||||
|
||||
cfg = DiscordConfig()
|
||||
assert cfg.bot_token == ""
|
||||
assert cfg.guild_id == 0
|
||||
assert cfg.allowed_channels == []
|
||||
assert cfg.thread_auto_archive == 1440
|
||||
assert cfg.max_message_length == 2000
|
||||
assert cfg.streaming_edit_interval == 1.5
|
||||
# Inherited from ChannelConfig
|
||||
assert cfg.redis_host == "localhost"
|
||||
assert cfg.redis_port == 6379
|
||||
assert cfg.model == ""
|
||||
assert cfg.auto_approve is False
|
||||
|
||||
def test_custom_values(self):
|
||||
from turnstone.channels.discord.config import DiscordConfig
|
||||
|
||||
cfg = DiscordConfig(
|
||||
bot_token="tok_123",
|
||||
guild_id=999,
|
||||
allowed_channels=[1, 2, 3],
|
||||
thread_auto_archive=60,
|
||||
max_message_length=4000,
|
||||
streaming_edit_interval=0.5,
|
||||
model="gpt-5",
|
||||
auto_approve=True,
|
||||
)
|
||||
assert cfg.bot_token == "tok_123"
|
||||
assert cfg.guild_id == 999
|
||||
assert cfg.allowed_channels == [1, 2, 3]
|
||||
assert cfg.thread_auto_archive == 60
|
||||
assert cfg.max_message_length == 4000
|
||||
assert cfg.streaming_edit_interval == 0.5
|
||||
assert cfg.model == "gpt-5"
|
||||
assert cfg.auto_approve is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StreamingMessage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStreamingMessage:
|
||||
"""Tests for the StreamingMessage helper in bot.py."""
|
||||
|
||||
def test_append_accumulates(self):
|
||||
from turnstone.channels.discord.bot import StreamingMessage
|
||||
|
||||
channel = MagicMock()
|
||||
channel.send = AsyncMock()
|
||||
sm = StreamingMessage(channel=channel, edit_interval=999.0)
|
||||
|
||||
_run(sm.append("hello "))
|
||||
_run(sm.append("world"))
|
||||
|
||||
assert "".join(sm._buffer) == "hello world"
|
||||
|
||||
def test_finalize_sends_when_no_prior_message(self):
|
||||
from turnstone.channels.discord.bot import StreamingMessage
|
||||
|
||||
channel = MagicMock()
|
||||
channel.send = AsyncMock()
|
||||
sm = StreamingMessage(channel=channel, edit_interval=999.0)
|
||||
|
||||
_run(sm.append("hello"))
|
||||
_run(sm.finalize())
|
||||
|
||||
channel.send.assert_awaited_once_with("hello")
|
||||
|
||||
def test_finalize_edits_existing_message(self):
|
||||
from turnstone.channels.discord.bot import StreamingMessage
|
||||
|
||||
channel = MagicMock()
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.edit = AsyncMock()
|
||||
channel.send = AsyncMock(return_value=sent_msg)
|
||||
sm = StreamingMessage(channel=channel, edit_interval=0.0)
|
||||
|
||||
# First append triggers flush (interval=0) which creates the message.
|
||||
_run(sm.append("hi"))
|
||||
assert sm._message is sent_msg
|
||||
|
||||
_run(sm.append(" there"))
|
||||
_run(sm.finalize())
|
||||
|
||||
# finalize edits the existing message with full content.
|
||||
sent_msg.edit.assert_awaited_with(content="hi there")
|
||||
|
||||
def test_finalize_chunks_long_content(self):
|
||||
from turnstone.channels.discord.bot import StreamingMessage
|
||||
|
||||
channel = MagicMock()
|
||||
channel.send = AsyncMock()
|
||||
sm = StreamingMessage(channel=channel, max_length=10, edit_interval=999.0)
|
||||
|
||||
# Content longer than max_length should be chunked on finalize.
|
||||
_run(sm.append("a" * 25))
|
||||
_run(sm.finalize())
|
||||
|
||||
# Should have sent multiple chunks via channel.send.
|
||||
assert channel.send.await_count >= 2
|
||||
|
||||
def test_finalize_empty_is_noop(self):
|
||||
from turnstone.channels.discord.bot import StreamingMessage
|
||||
|
||||
channel = MagicMock()
|
||||
channel.send = AsyncMock()
|
||||
sm = StreamingMessage(channel=channel)
|
||||
|
||||
_run(sm.finalize())
|
||||
channel.send.assert_not_awaited()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MessageCog._on_message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMessageCog:
|
||||
"""Tests for the MessageCog on_message filtering logic."""
|
||||
|
||||
def _make_cog(self):
|
||||
"""Build a MessageCog with a fully mocked bot and TurnstoneBot."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
bot.user.mentioned_in = MagicMock(return_value=False)
|
||||
|
||||
ts = MagicMock()
|
||||
ts._is_allowed_channel = MagicMock(return_value=True)
|
||||
ts.storage = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.resolve_user = AsyncMock(return_value="u_abc")
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts.config = MagicMock()
|
||||
ts._ws_tasks = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
return cog, ts, bot
|
||||
|
||||
def test_ignores_bot_messages(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
msg = _make_message(bot=True)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
# No router interaction means the message was ignored.
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_ignores_own_messages(self):
|
||||
cog, ts, bot = self._make_cog()
|
||||
msg = _make_message(bot=False)
|
||||
msg.author = bot.user # message from ourselves
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_ignores_dms(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
msg = _make_message(guild=False)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_ignores_non_allowed_channels(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
ts._is_allowed_channel = MagicMock(return_value=False)
|
||||
|
||||
thread = MagicMock(spec=discord.Thread)
|
||||
thread.id = 111
|
||||
thread.parent_id = 222
|
||||
msg = _make_message(channel=thread)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_footer (views.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseFooter:
|
||||
"""Tests for _parse_footer in views.py."""
|
||||
|
||||
def test_valid_footer(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = _make_interaction(footer_text="ws_abc|corr_123")
|
||||
result = _parse_footer(interaction)
|
||||
assert result == ("ws_abc", "corr_123")
|
||||
|
||||
def test_footer_with_pipe_in_correlation(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = _make_interaction(footer_text="ws_abc|corr|extra")
|
||||
result = _parse_footer(interaction)
|
||||
# split("|", 1) means the second part includes everything after first pipe.
|
||||
assert result == ("ws_abc", "corr|extra")
|
||||
|
||||
def test_no_message_returns_none(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = MagicMock()
|
||||
interaction.message = None
|
||||
assert _parse_footer(interaction) is None
|
||||
|
||||
def test_no_embeds_returns_none(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = _make_interaction(has_embeds=False)
|
||||
assert _parse_footer(interaction) is None
|
||||
|
||||
def test_empty_footer_returns_none(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
# Build an interaction whose embed has footer.text = None.
|
||||
interaction = MagicMock(spec=discord.Interaction)
|
||||
embed = MagicMock()
|
||||
embed.footer.text = None
|
||||
interaction.message = MagicMock()
|
||||
interaction.message.embeds = [embed]
|
||||
assert _parse_footer(interaction) is None
|
||||
|
||||
def test_footer_without_pipe_returns_none(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = _make_interaction(footer_text="no_pipe_here")
|
||||
# footer text has no "|" separator
|
||||
embed = MagicMock()
|
||||
embed.footer.text = "no_pipe_here"
|
||||
interaction.message.embeds = [embed]
|
||||
assert _parse_footer(interaction) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI main() — no adapter configured
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChannelCLI:
|
||||
"""Tests for the channel CLI entry point."""
|
||||
|
||||
def test_exits_without_adapter_token(self):
|
||||
from turnstone.channels.cli import main
|
||||
|
||||
with (
|
||||
patch.object(sys, "argv", ["turnstone-channel"]),
|
||||
patch.dict("os.environ", {}, clear=True),
|
||||
pytest.raises(SystemExit) as exc_info,
|
||||
):
|
||||
main()
|
||||
|
||||
assert exc_info.value.code == 1
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user