mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 07:22:24 -06:00
Compare commits
77 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8c414feba2 | |||
| d7cea053b6 | |||
| c45e98462b | |||
| e17cbe35a5 | |||
| fd47c23177 | |||
| 9fe988b1be | |||
| 3ce66960bc | |||
| 8a852a12e3 | |||
| 9a518657a3 | |||
| c424176c73 | |||
| 2c32e89de3 | |||
| 06310c74ee | |||
| dfad58a3d2 | |||
| e31197d64a | |||
| 5f9200f6a0 | |||
| 84b0d5615c | |||
| d41621877f | |||
| a4f3d205d1 | |||
| 6078a88533 | |||
| 843fa04e65 | |||
| 473298199d | |||
| c251e2dac8 | |||
| ac47476d0a | |||
| 055bd5a88f | |||
| a7d9461735 | |||
| 9de77c3ee3 | |||
| 0cfe521ce7 | |||
| c2750de7a4 | |||
| bd782f804e | |||
| 87b69a318b | |||
| a315cabe71 | |||
| be17d8c5d0 | |||
| 62ce450b06 | |||
| d19dad05bd | |||
| 262a6a9918 | |||
| 2bb55590bf | |||
| 0e02d1b52c | |||
| 9b29453e9b | |||
| c4ff1caf09 | |||
| 22245145db | |||
| 8eacc4d632 | |||
| 23fed785c4 | |||
| 688c27e68a | |||
| 405baf7cb2 | |||
| 1027c22333 | |||
| 381651049b | |||
| 322b7dabc4 | |||
| d5e86c8493 | |||
| c154ea3966 | |||
| 755ab51802 | |||
| 9df8ab836f | |||
| 8d88e6a7eb | |||
| cce292f793 | |||
| 6adc577d30 | |||
| 02c50b81c1 | |||
| 753cd04b4e | |||
| c3217748dc | |||
| 8cbff49694 | |||
| 4f26d63c14 | |||
| 74347fb29f | |||
| 2ace8cccc8 | |||
| e95b8f5ca1 | |||
| 7a32c51a1c | |||
| dce663105b | |||
| cfef3616e6 | |||
| c22d39a798 | |||
| 63921450b1 | |||
| 929fad63be | |||
| 976e9df3b6 | |||
| 7cb21b84f1 | |||
| 7263edd48d | |||
| 6742c7e405 | |||
| 1aa6982868 | |||
| 42d1abbd04 | |||
| f543ed714a | |||
| 2c6abb0fde | |||
| 491fc6748a |
@@ -10,7 +10,7 @@ repos:
|
||||
rev: v1.19.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
additional_dependencies: [types-redis>=4.6, redis>=7.2]
|
||||
additional_dependencies: []
|
||||
args: [--config-file=pyproject.toml]
|
||||
pass_filenames: false
|
||||
entry: mypy turnstone/
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# libexpat integer overflow — no fix available in Debian repos yet
|
||||
# https://avd.aquasec.com/nvd/cve-2026-25210
|
||||
# Review: remove this entry once a patched libexpat1 is published
|
||||
CVE-2026-25210
|
||||
|
||||
# ncurses buffer overflow — no fix in Debian 13 repos yet
|
||||
# Affects libncursesw6, libtinfo6, ncurses-base, ncurses-bin
|
||||
# https://avd.aquasec.com/nvd/cve-2025-69720
|
||||
CVE-2025-69720
|
||||
|
||||
# nghttp2 DoS via malformed HTTP/2 frames — no fix in Debian 13 repos yet
|
||||
# Affects libnghttp2-14
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27135
|
||||
CVE-2026-27135
|
||||
|
||||
# systemd arbitrary code execution via spurious IPC — no fix in Debian 13 repos yet
|
||||
# Affects libsystemd0, libudev1
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29111
|
||||
CVE-2026-29111
|
||||
+12
-8
@@ -1,6 +1,6 @@
|
||||
# =============================================================================
|
||||
# Turnstone — Docker build with uv for reproducible, locked installs
|
||||
# Single image for all services: server, bridge, console, sim, eval
|
||||
# Single image for all services: server, console, channel, eval
|
||||
# =============================================================================
|
||||
|
||||
FROM python:3.14-slim
|
||||
@@ -10,8 +10,12 @@ LABEL org.opencontainers.image.title="turnstone" \
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.2 /uv /usr/local/bin/uv
|
||||
|
||||
# System dependencies for psycopg (PostgreSQL client library)
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libpq5 \
|
||||
# Remove the slim image's man page exclusion so man-db has actual content
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/docker
|
||||
|
||||
# System dependencies: psycopg (libpq5), developer tooling for agent workflows
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
|
||||
libpq5 git curl jq man-db manpages procps file \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Non-root user
|
||||
@@ -19,18 +23,18 @@ RUN useradd --create-home --shell /bin/bash turnstone
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Compile bytecode for faster startup
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
|
||||
# Install dependencies first (cached layer — only re-runs when deps change)
|
||||
COPY pyproject.toml uv.lock README.md LICENSE ./
|
||||
RUN uv sync --frozen --no-install-project --no-dev \
|
||||
--extra all
|
||||
--no-compile --extra all
|
||||
|
||||
# Install the project itself
|
||||
COPY turnstone/ turnstone/
|
||||
RUN uv sync --frozen --no-dev \
|
||||
--extra all
|
||||
--no-compile --extra all
|
||||
|
||||
# Compile bytecode in a separate step (avoids fd exhaustion during install)
|
||||
RUN python -m compileall -q .venv turnstone/
|
||||
|
||||
# Add venv to PATH so entry points are found
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
+2
-2
@@ -45,9 +45,9 @@ That's it — no flags, no arguments. The wizard prompts for everything.
|
||||
The wizard supports two deployment modes:
|
||||
|
||||
- **Single-node production** (`docker compose --profile production up`) —
|
||||
1 server + bridge + console + PostgreSQL + Redis. Good for most use cases.
|
||||
1 server + console + PostgreSQL. Good for most use cases.
|
||||
- **Multi-node cluster** (`docker compose --profile cluster up`) —
|
||||
10-node server/bridge fleet + PostgreSQL + Redis. For high-throughput or
|
||||
10-node server fleet + console + PostgreSQL. For high-throughput or
|
||||
HA deployments.
|
||||
|
||||
## Example Session
|
||||
|
||||
@@ -5,418 +5,125 @@
|
||||
[](https://pypi.org/project/turnstone/)
|
||||
[](LICENSE)
|
||||
|
||||
Experimental multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers, driven by message queues or interactive interfaces.
|
||||
Multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers with direct HTTP routing, interactive interfaces, and enterprise governance.
|
||||
|
||||
> **Beta — Use at your own risk.** Turnstone is under active development and has not reached a stable release. APIs, configuration formats, and database schemas may change between versions without migration paths. We make no guarantees of determinism, reliability, or backward compatibility. Evaluate thoroughly before deploying to any environment where these properties matter.
|
||||
> **Beta — Use at your own risk.** APIs, configuration formats, and database schemas may change between versions without migration paths.
|
||||
|
||||
Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) (*Arenaria interpres*) — a shorebird that flips stones to discover what's hiding underneath.
|
||||
|
||||
## What it does
|
||||
|
||||
Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports. It runs as:
|
||||
Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports.
|
||||
|
||||
- **Interactive sessions** — terminal CLI or browser UI with parallel workstreams
|
||||
- **Queue-driven agents** — trigger workstreams via message queue, stream progress, approve or auto-approve tool use
|
||||
- **Multi-node clusters** — generic work load-balances across nodes, directed work routes to a specific server
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams, reverse proxy for server UIs
|
||||
- **Intent validation** — an LLM judge evaluates every tool call before approval, presenting risk assessments and evidence-based recommendations so users can make informed decisions instead of blindly approving raw tool calls
|
||||
- **Governance & compliance** — RBAC, OIDC SSO (Okta, Azure AD, Google, Keycloak), tool policies, skills (reusable behavioral profiles with security scanning), usage tracking, and append-only audit logs
|
||||
- **Cluster simulator** — test the stack at scale (up to 1000 nodes) without an LLM backend
|
||||
|
||||
Works with any OpenAI-compatible API (vLLM, llama.cpp, NVIDIA NIM) or Anthropic's native Messages API. Supports [MCP](https://modelcontextprotocol.io/) for external tool servers with native deferred tool loading on Anthropic and OpenAI APIs (BM25 fallback for local models).
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams with console routing proxy
|
||||
- **Intent validation** — LLM judge evaluates every tool call with risk assessments and evidence
|
||||
- **Governance** — RBAC, OIDC SSO, tool policies, skills, usage tracking, audit logs
|
||||
- **Multi-provider** — OpenAI-compatible APIs (vLLM, llama.cpp, NIM) and Anthropic Messages API
|
||||
- **MCP support** — external tool servers with native deferred loading (Anthropic/OpenAI) or BM25 fallback
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/diagrams/architecture-overview.svg" alt="Turnstone system architecture — data flow from clients through gateways, Redis MQ, cluster nodes, to LLM providers" width="960"/>
|
||||
<img src="docs/diagrams/architecture-overview.svg" alt="Turnstone system architecture" width="960"/>
|
||||
</p>
|
||||
|
||||
## Quickstart
|
||||
|
||||
### Interactive (terminal)
|
||||
|
||||
```bash
|
||||
pip install turnstone
|
||||
|
||||
# Terminal REPL
|
||||
turnstone --base-url http://localhost:8000/v1
|
||||
```
|
||||
|
||||
### Interactive (browser)
|
||||
|
||||
```bash
|
||||
# Browser UI
|
||||
turnstone-server --port 8080 --base-url http://localhost:8000/v1
|
||||
```
|
||||
|
||||
### Queue-driven (programmatic)
|
||||
|
||||
```bash
|
||||
pip install turnstone[mq]
|
||||
turnstone-bridge --server-url http://localhost:8080 --redis-host localhost
|
||||
```
|
||||
|
||||
```python
|
||||
from turnstone.mq import TurnstoneClient
|
||||
|
||||
with TurnstoneClient() as client:
|
||||
# Generic — any available node picks it up
|
||||
result = client.send_and_wait("Analyze the error logs", auto_approve=True)
|
||||
print(result.content)
|
||||
|
||||
# Directed — must run on a specific server
|
||||
result = client.send_and_wait(
|
||||
"Check disk I/O on this server",
|
||||
target_node="server-12",
|
||||
auto_approve=True,
|
||||
)
|
||||
```
|
||||
|
||||
### Cluster dashboard
|
||||
|
||||
```bash
|
||||
# Cluster dashboard
|
||||
pip install turnstone[console]
|
||||
turnstone-console --redis-host localhost --port 8090
|
||||
turnstone-console --port 8090
|
||||
```
|
||||
|
||||
Then open `http://localhost:8090` for the cluster-wide dashboard. Create workstreams from the console and interact with any node's server UI through the built-in reverse proxy — no direct server port access required.
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
cp .env.example .env # edit LLM_BASE_URL, OPENAI_API_KEY, etc.
|
||||
docker compose up # starts redis + server + bridge + console (SQLite)
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
For production with PostgreSQL:
|
||||
See [QUICKSTART.md](QUICKSTART.md) for the bootstrap wizard and [docs/docker.md](docs/docker.md) for Docker configuration and profiles.
|
||||
|
||||
```bash
|
||||
# Requires POSTGRES_PASSWORD and DB_BACKEND=postgresql in .env (or exported)
|
||||
docker compose --profile production up # adds PostgreSQL, uses it as database
|
||||
### Programmatic (SDK)
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
ws = client.create_workstream(name="demo")
|
||||
result = client.send_and_wait("Analyze the error logs", ws.ws_id, auto_approve=True)
|
||||
print(result.content)
|
||||
```
|
||||
|
||||
Console dashboard at http://localhost:8090. See [docs/docker.md](docs/docker.md) for configuration, scaling, and profiles.
|
||||
|
||||
### Simulator
|
||||
|
||||
Test the multi-node stack at scale without an LLM backend:
|
||||
|
||||
```bash
|
||||
docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
Or standalone:
|
||||
|
||||
```bash
|
||||
pip install turnstone[sim]
|
||||
turnstone-sim --nodes 100 --scenario steady --duration 60 --mps 10
|
||||
```
|
||||
|
||||
See [docs/simulator.md](docs/simulator.md) for scenarios, CLI reference, and metrics.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Diagrams
|
||||
|
||||
Detailed UML diagrams are available in [`docs/diagrams/`](docs/diagrams/):
|
||||
|
||||
| Diagram | Description |
|
||||
|---------|-------------|
|
||||
| [System Context](docs/diagrams/png/01-system-context.png) | Top-level components and external dependencies |
|
||||
| [Package Structure](docs/diagrams/png/02-package-structure.png) | Python modules and dependency graph |
|
||||
| [Core Engine Classes](docs/diagrams/png/03-core-engine-classes.png) | SessionUI protocol, ChatSession, LLMProvider, WorkstreamManager |
|
||||
| [Conversation Turn](docs/diagrams/png/04-conversation-turn.png) | Full message lifecycle through the engine (provider-agnostic) |
|
||||
| [Tool Pipeline](docs/diagrams/png/05-tool-pipeline.png) | Three-phase prepare/approve/execute |
|
||||
| [MQ Protocol](docs/diagrams/png/06-mq-protocol.png) | 9 inbound + 19 outbound message types |
|
||||
| [Message Routing](docs/diagrams/png/07-message-routing.png) | Multi-node routing scenarios |
|
||||
| [Redis Key Schema](docs/diagrams/png/08-redis-key-schema.png) | All Redis keys, types, and TTLs |
|
||||
| [Workstream States](docs/diagrams/png/09-workstream-states.png) | State machine transitions |
|
||||
| [Simulator](docs/diagrams/png/10-simulator-architecture.png) | SimCluster, dispatchers, scenarios |
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection threads |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose service topology |
|
||||
| [SDK Architecture](docs/diagrams/png/13-sdk-architecture.png) | Python + TypeScript client libraries |
|
||||
| [Storage Architecture](docs/diagrams/png/14-storage-architecture.png) | Pluggable database backends (SQLite + PostgreSQL) |
|
||||
| [Auth Architecture](docs/diagrams/png/15-auth-architecture.png) | JWT, scopes, token types, login flows |
|
||||
| [Channel Architecture](docs/diagrams/png/16-channel-architecture.png) | Discord/Slack adapter protocol and routing |
|
||||
| [Notify Flow](docs/diagrams/png/17-notify-flow.png) | Channel notification dispatch |
|
||||
| [Watch Architecture](docs/diagrams/png/18-watch-architecture.png) | Periodic command polling daemon |
|
||||
| [Governance Architecture](docs/diagrams/png/19-governance-architecture.png) | RBAC, policies, audit, usage enforcement flow |
|
||||
| [WS Template Architecture](docs/diagrams/png/21-ws-template-architecture.png) | Workstream template application and lifecycle |
|
||||
| [Judge Architecture](docs/diagrams/png/22-judge-architecture.png) | Intent validation two-tier evaluation pipeline |
|
||||
| [OIDC Architecture](docs/diagrams/png/25-oidc-architecture.png) | OIDC SSO authorization code flow with PKCE |
|
||||
|
||||
### Governance
|
||||
|
||||
Turnstone includes a built-in governance layer for enterprise deployments — manage who can do what, which tools run unattended, and where every token goes.
|
||||
|
||||
- **RBAC** — 15 granular permissions, 3 built-in roles (admin / operator / viewer), custom roles, privilege escalation prevention
|
||||
- **OIDC SSO** — single sign-on via any OpenID Connect provider (Okta, Azure AD, Google, Keycloak); Authorization Code Flow with PKCE, auto-provisioning, claim-based role mapping with demotion propagation; see [docs/oidc.md](docs/oidc.md)
|
||||
- **Tool policies** — glob-pattern rules (`allow` / `deny` / `ask`) with priority ordering; automate approvals or lock down dangerous tools
|
||||
- **Skills** — reusable behavioral profiles with system prompts, `{{variable}}` substitution, session config (model, temperature, token budget), install-time security scanning, version history, external discovery (skills.sh / GitHub), and runtime `skill` tool for model-driven skill activation
|
||||
- **Usage tracking** — per-request token and tool metrics, aggregation by day / model / user, automatic 90-day pruning
|
||||
- **Audit logging** — append-only event trail for all admin mutations, IP-aware, 365-day retention
|
||||
|
||||
All governance features are managed through the console admin panel (13 tabs) and the full REST API. Runtime settings (model, tools, rate limiting, health, judge, memory) are configurable via the admin Settings tab — no config file edits or restarts needed for most changes. See [docs/governance.md](docs/governance.md) for setup and [docs/settings.md](docs/settings.md) for the settings reference.
|
||||
|
||||
### Intent Validation (LLM Judge)
|
||||
|
||||
Every tool call that requires human approval is evaluated by an intent validation judge that provides a structured risk assessment alongside the approval prompt — so instead of "approve this bash command?", users see a verdict with risk level, confidence, recommendation, and reasoning.
|
||||
|
||||
The system uses a two-tier evaluation pipeline:
|
||||
|
||||
1. **Heuristic tier** (instant, free) — 36 pattern-based rules classify tool calls by severity. Catches destructive commands (`rm -rf /`, `DROP TABLE`), privilege escalation (`sudo`), credential access, supply chain risks, browser data export, cloud infrastructure mutations, and more. Results appear immediately.
|
||||
2. **LLM judge tier** (async) — A full LLM evaluation runs in the background with access to `read_file` and `list_directory` for evidence gathering. The judge can inspect files that a write would overwrite, check directory contents before a delete, and cite specific evidence in its reasoning. Results update the UI progressively when ready.
|
||||
|
||||
The judge defaults to the same model as the session (self-consistency) but can be configured to use a separate model — useful when running a small local model for tasks but wanting a commercial model for safety evaluation.
|
||||
|
||||
```toml
|
||||
[judge]
|
||||
enabled = true # on by default
|
||||
model = "" # empty = same as session model
|
||||
provider = "" # empty = same as session provider
|
||||
timeout = 60.0 # generous for local models
|
||||
```
|
||||
|
||||
Verdicts are persisted for audit and exposed via Prometheus metrics (`turnstone_judge_verdicts_total`, `turnstone_judge_llm_latency_seconds`).
|
||||
|
||||
Skills are also scanned at install time — the scanner evaluates content, supply chain, vulnerability, and declared capability risk across four independent axes. Results populate `scan_status` (tier) and `scan_report` (structured JSON breakdown) on the skill record so administrators can assess risk before enabling a skill.
|
||||
|
||||
Tool execution results are evaluated by an output guard before entering the conversation — detecting prompt injection payloads in fetched content, credential leakage in command output, and encoded payloads. Detected credentials are automatically redacted.
|
||||
|
||||
See [docs/judge.md](docs/judge.md) for the full guide.
|
||||
|
||||
## Multi-node routing
|
||||
|
||||
Each Turnstone server runs a bridge process. Bridges share a Redis instance for coordination:
|
||||
|
||||
| Redis Key | Purpose |
|
||||
|-----------|---------|
|
||||
| `turnstone:inbound` | Shared work queue — generic tasks, any node |
|
||||
| `turnstone:inbound:{node_id}` | Per-node queue — directed tasks |
|
||||
| `turnstone:ws:{ws_id}` | Workstream ownership — auto-routes follow-ups |
|
||||
| `turnstone:node:{node_id}` | Node heartbeat + metadata for discovery |
|
||||
| `turnstone:events:{ws_id}` | Per-workstream event pub/sub |
|
||||
| `turnstone:events:global` | Global event pub/sub |
|
||||
| `turnstone:events:cluster` | Cluster-wide state changes (for turnstone-console) |
|
||||
|
||||
**Routing rules:**
|
||||
1. Message has `target_node` → routes to that node's queue
|
||||
2. Message has `ws_id` → looks up owner, routes to owning node
|
||||
3. Neither → shared queue, next available bridge picks it up
|
||||
|
||||
Bridges BLPOP from their per-node queue (priority) then the shared queue. Directed work always takes precedence.
|
||||
|
||||
## Tools
|
||||
|
||||
15 built-in tools, 2 agent tools, plus external tools via MCP:
|
||||
Built-in tools for shell, files, search, web, memory, notifications, and autonomous sub-agents — plus external tools via [MCP](https://modelcontextprotocol.io/) with native deferred loading. See [docs/tools.md](docs/tools.md) for the full reference and [docs/mcp.md](docs/mcp.md) for MCP configuration.
|
||||
|
||||
| Tool | Description | Auto-approved |
|
||||
|------|-------------|:---:|
|
||||
| `bash` | Execute shell commands | |
|
||||
| `read_file` | Read file contents (text or images with vision models) | yes |
|
||||
| `write_file` | Write/create files | |
|
||||
| `edit_file` | Fuzzy-match file editing | |
|
||||
| `search` | Search files by name/content | yes |
|
||||
| `math` | Sandboxed Python evaluation | |
|
||||
| `man` | Read man pages | yes |
|
||||
| `web_fetch` | Fetch URL content | |
|
||||
| `web_search` | Web search (provider-native or Tavily) | |
|
||||
| `memory` | Structured persistent memory (save/search/delete/list) | yes |
|
||||
| `recall` | Search conversation history | yes |
|
||||
| `notify` | Send notifications to linked channels | yes |
|
||||
| `watch` | Periodic command polling with conditions | |
|
||||
| `task` | Spawn autonomous sub-agent | |
|
||||
| `plan` | Explore codebase, write .plan.md | |
|
||||
| `mcp__*` | External tools from MCP servers | |
|
||||
## Architecture
|
||||
|
||||
When the total tool count exceeds a configurable threshold (default 20), MCP tools are automatically deferred using native `defer_loading` on Anthropic and OpenAI APIs, or a transparent client-side BM25 search for local models. The LLM discovers deferred tools on demand via a `tool_search` capability — no configuration needed beyond `--tool-search auto` (the default).
|
||||
**Single-node**: Client → Server (direct HTTP + SSE). No external dependencies beyond the database.
|
||||
|
||||
### MCP Tool Servers
|
||||
**Multi-node**: Client → Console (hash ring routing proxy) → Server nodes. The console maintains a 65536-entry bucket cache for O(1) workstream routing. A rebalancer daemon redistributes buckets when nodes join or leave.
|
||||
|
||||
Turnstone supports the [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) for connecting external tool servers. MCP tools are discovered at startup, converted to OpenAI function-calling format, and merged with built-in tools. Each MCP tool is prefixed with `mcp__{server}__{tool}` to avoid name collisions. Tool lists stay fresh via push notifications (`tools.listChanged`), periodic polling for servers without push, and manual `/mcp refresh`.
|
||||
| Component | Purpose |
|
||||
|-----------|---------|
|
||||
| `turnstone` | Terminal CLI (REPL) |
|
||||
| `turnstone-server` | Web UI + REST API + SSE events |
|
||||
| `turnstone-console` | Cluster dashboard + routing proxy + admin panel |
|
||||
| `turnstone-channel` | Channel gateway (Discord, with adapters for Slack/Teams planned) |
|
||||
| `turnstone-admin` | User/token management CLI |
|
||||
| `turnstone-eval` | Eval harness for prompt/tool optimization |
|
||||
| `turnstone-bootstrap` | LLM-guided setup wizard |
|
||||
|
||||
Configure via `config.toml` or `--mcp-config`:
|
||||
### Diagrams
|
||||
|
||||
```toml
|
||||
[mcp.servers.github]
|
||||
command = "npx"
|
||||
args = ["-y", "@modelcontextprotocol/server-github"]
|
||||
UML diagrams in [`docs/diagrams/`](docs/diagrams/):
|
||||
|
||||
[mcp.servers.github.env]
|
||||
GITHUB_TOKEN = "ghp_..."
|
||||
```
|
||||
| Diagram | Description |
|
||||
|---------|-------------|
|
||||
| [System Context](docs/diagrams/png/01-system-context.png) | Components and external dependencies |
|
||||
| [Package Structure](docs/diagrams/png/02-package-structure.png) | Python modules and dependency graph |
|
||||
| [Core Engine](docs/diagrams/png/03-core-engine-classes.png) | SessionUI, ChatSession, LLMProvider |
|
||||
| [Conversation Turn](docs/diagrams/png/04-conversation-turn.png) | Message lifecycle through the engine |
|
||||
| [Tool Pipeline](docs/diagrams/png/05-tool-pipeline.png) | Prepare / approve / execute |
|
||||
| [Workstream States](docs/diagrams/png/09-workstream-states.png) | State machine transitions |
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose topology |
|
||||
| [Auth](docs/diagrams/png/15-auth-architecture.png) | JWT, scopes, login flows |
|
||||
| [Channels](docs/diagrams/png/16-channel-architecture.png) | Discord adapter + routing |
|
||||
| [Judge](docs/diagrams/png/22-judge-architecture.png) | Intent validation pipeline |
|
||||
| [OIDC](docs/diagrams/png/25-oidc-architecture.png) | SSO authorization code flow |
|
||||
|
||||
Or use a standard MCP JSON config file:
|
||||
## Documentation
|
||||
|
||||
```bash
|
||||
turnstone --mcp-config ~/.config/turnstone/mcp.json
|
||||
turnstone-server --mcp-config ~/.config/turnstone/mcp.json
|
||||
```
|
||||
|
||||
Use `/mcp` in the REPL to list connected tools, `/mcp refresh` to re-fetch tool lists from servers. MCP tools require user approval by default (overridden by `--skip-permissions` or UI auto-approve).
|
||||
|
||||
### Multi-Model and Multi-Provider Support
|
||||
|
||||
Turnstone supports multiple model backends per server instance, including different LLM providers. `ChatSession` delegates all API communication to pluggable `LLMProvider` adapters — the internal message format stays OpenAI-like, and each provider translates at the API boundary. Define named models in `config.toml` and select per-workstream or switch mid-session with `/model <alias>`.
|
||||
|
||||
```toml
|
||||
[models.local]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
model = "qwen3-32b"
|
||||
# provider defaults to "openai" (works with vLLM, llama.cpp, etc.)
|
||||
|
||||
[models.claude]
|
||||
provider = "anthropic"
|
||||
api_key = "sk-ant-..."
|
||||
model = "claude-opus-4-6"
|
||||
context_window = 200000
|
||||
|
||||
[models.openai]
|
||||
base_url = "https://api.openai.com/v1"
|
||||
api_key = "sk-..."
|
||||
model = "gpt-5"
|
||||
context_window = 400000
|
||||
|
||||
[model]
|
||||
default = "local" # which model to use by default
|
||||
fallback = ["claude", "openai"] # try these if the primary is unreachable
|
||||
agent_model = "claude" # optional: separate model for plan/task sub-agents
|
||||
```
|
||||
|
||||
Supported providers: `"openai"` (default -- OpenAI, vLLM, llama.cpp, any OpenAI-compatible API) and `"anthropic"` (Anthropic Messages API, requires `pip install turnstone[anthropic]`).
|
||||
|
||||
Use `/model` to show available models, `/model claude` to switch. Workstreams created via the API accept an optional `model` parameter.
|
||||
|
||||
## Configuration
|
||||
|
||||
All entry points read `~/.config/turnstone/config.toml`. CLI flags override config values.
|
||||
|
||||
```toml
|
||||
[api]
|
||||
base_url = "http://localhost:8000/v1"
|
||||
api_key = ""
|
||||
# tavily_key = "" # only needed for local/vLLM models without native search
|
||||
|
||||
[model]
|
||||
name = "" # empty = auto-detect
|
||||
temperature = 0.5
|
||||
reasoning_effort = "medium"
|
||||
default = "default" # model alias for new workstreams
|
||||
fallback = [] # ordered list of fallback model aliases
|
||||
agent_model = "" # model alias for plan/task sub-agents
|
||||
|
||||
[tools]
|
||||
timeout = 30
|
||||
skip_permissions = false
|
||||
search = "auto" # "auto" (enable when >threshold tools), "on", "off"
|
||||
search_threshold = 20 # min tools before tool search activates
|
||||
search_max_results = 5 # max tools returned per search query
|
||||
|
||||
[server]
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
max_workstreams = 50 # auto-evicts oldest idle when full
|
||||
|
||||
[redis]
|
||||
host = "localhost"
|
||||
port = 6379
|
||||
password = ""
|
||||
|
||||
[bridge]
|
||||
server_url = "http://localhost:8080"
|
||||
node_id = "" # empty = hostname_xxxx
|
||||
|
||||
[console]
|
||||
host = "0.0.0.0"
|
||||
port = 8090
|
||||
url = "http://localhost:8090" # used by CLI /cluster commands
|
||||
poll_interval = 10
|
||||
|
||||
[health]
|
||||
backend_probe_interval = 30
|
||||
backend_probe_timeout = 5
|
||||
circuit_breaker_threshold = 5
|
||||
circuit_breaker_cooldown = 60
|
||||
|
||||
[ratelimit]
|
||||
enabled = true
|
||||
requests_per_second = 10.0
|
||||
burst = 20
|
||||
|
||||
[database]
|
||||
backend = "sqlite" # "sqlite" (default) or "postgresql"
|
||||
path = ".turnstone.db" # SQLite file path (relative to working directory)
|
||||
# url = "postgresql+psycopg://user:pass@host:5432/turnstone" # PostgreSQL
|
||||
# pool_size = 2 # PostgreSQL connection pool size (per process)
|
||||
|
||||
[judge]
|
||||
enabled = true # intent validation for tool approvals (--no-judge to disable)
|
||||
model = "" # empty = same as session model (self-consistency)
|
||||
provider = "" # empty = same as session provider
|
||||
timeout = 60.0 # LLM judge timeout in seconds
|
||||
confidence_threshold = 0.7
|
||||
|
||||
[mcp]
|
||||
config_path = "" # path to MCP JSON config file (alternative to TOML sections)
|
||||
refresh_interval = 14400 # periodic refresh for servers without push notifications (seconds, 0 to disable)
|
||||
|
||||
[mcp.servers.example] # one section per MCP server
|
||||
command = "npx"
|
||||
args = ["-y", "@modelcontextprotocol/server-example"]
|
||||
# type = "stdio" # "stdio" (default) or "http"
|
||||
# url = "" # for HTTP transport
|
||||
```
|
||||
|
||||
Precedence: CLI args > environment variables > config.toml > defaults.
|
||||
|
||||
## Workstreams
|
||||
|
||||
Parallel independent conversations, each with its own session and state:
|
||||
|
||||
| Symbol | State | Meaning |
|
||||
|--------|-------|---------|
|
||||
| `·` | idle | Waiting for input |
|
||||
| `◌` | thinking | Model is generating |
|
||||
| `▸` | running | Tool execution in progress |
|
||||
| `◆` | attention | Waiting for approval |
|
||||
| `✖` | error | Something went wrong |
|
||||
|
||||
Idle workstreams are automatically cleaned up after 2 hours (configurable). In multi-node deployments, workstream ownership is tracked in Redis — follow-up messages auto-route to the owning node.
|
||||
|
||||
## Monitoring
|
||||
|
||||
`/metrics` endpoint exposes Prometheus-format metrics:
|
||||
|
||||
- `turnstone_tokens_total{direction}` — prompt/completion token counters
|
||||
- `turnstone_tool_calls_total{tool}` — per-tool invocation counts
|
||||
- `turnstone_workstream_context_ratio{ws_id}` — per-workstream context utilization
|
||||
- `turnstone_http_request_duration_seconds` — request latency histogram
|
||||
- `turnstone_workstreams_by_state{state}` — workstream state gauges
|
||||
- `turnstone_sse_connections_active` — current open SSE connections
|
||||
- `turnstone_ratelimit_rejected_total` — requests rejected by rate limiter
|
||||
- `turnstone_backend_up` — LLM backend reachability (0/1)
|
||||
- `turnstone_circuit_state` — circuit breaker state (0=closed, 1=open, 2=half_open)
|
||||
- `turnstone_workstreams_evicted_total` — workstreams auto-evicted at capacity
|
||||
- `turnstone_judge_verdicts_total{tier,risk_level}` — intent validation verdicts by tier and risk
|
||||
- `turnstone_judge_llm_latency_seconds` — LLM judge evaluation latency histogram
|
||||
- `turnstone_judge_enabled` — whether the intent validation judge is active (0/1)
|
||||
|
||||
Per-workstream metrics are labeled by `ws_id` (bounded by `[server].max_workstreams`).
|
||||
|
||||
### Health & Rate Limiting
|
||||
|
||||
**Health degradation.** A background `BackendHealthMonitor` probes the LLM backend every `backend_probe_interval` seconds. When the backend is unreachable, `/health` reports `"status": "degraded"` (HTTP 200) and the `turnstone_backend_up` gauge drops to 0.
|
||||
|
||||
**Circuit breaker.** After `circuit_breaker_threshold` consecutive probe failures the circuit opens (CLOSED -> OPEN). While open, `ChatSession._create_stream_with_retry` skips the backend entirely and returns an error. After `circuit_breaker_cooldown` seconds the circuit enters HALF_OPEN, allowing a single probe. A successful probe closes the circuit; a failure re-opens it.
|
||||
|
||||
**Per-IP rate limiting.** When `[ratelimit].enabled` is true, each client IP is tracked with a token-bucket limiter (`requests_per_second` / `burst`). Rate limiting is applied in `do_GET`/`do_POST` after authentication but before route dispatch. `/health` and `/metrics` are exempt. Requests that exceed the limit receive HTTP 429 with a `Retry-After` header.
|
||||
|
||||
**Workstream eviction.** When `WorkstreamManager.create()` would exceed `max_workstreams`, the oldest IDLE workstream is automatically evicted and the `turnstone_workstreams_evicted_total` counter is incremented. Configure via `[server].max_workstreams` (default 50).
|
||||
| Topic | Link |
|
||||
|-------|------|
|
||||
| Configuration reference | [docs/settings.md](docs/settings.md) |
|
||||
| API reference | [docs/api-reference.md](docs/api-reference.md) |
|
||||
| Docker deployment | [docs/docker.md](docs/docker.md) |
|
||||
| Intent validation (judge) | [docs/judge.md](docs/judge.md) |
|
||||
| Governance & RBAC | [docs/governance.md](docs/governance.md) |
|
||||
| OIDC SSO | [docs/oidc.md](docs/oidc.md) |
|
||||
| TLS / mTLS | [docs/tls.md](docs/tls.md) |
|
||||
| Channel integrations | [docs/channels.md](docs/channels.md) |
|
||||
| Console dashboard | [docs/console.md](docs/console.md) |
|
||||
| Eval harness | [docs/eval.md](docs/eval.md) |
|
||||
| Tools reference | [docs/tools.md](docs/tools.md) |
|
||||
| MCP integration | [docs/mcp.md](docs/mcp.md) |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.11+
|
||||
- An OpenAI-compatible API endpoint ([vLLM](https://github.com/vllm-project/vllm), [NVIDIA NIM](https://build.nvidia.com/), [llama.cpp](https://github.com/ggml-org/llama.cpp), etc.) or an Anthropic API key
|
||||
- Redis (for message queue bridge — `pip install turnstone[mq]`)
|
||||
- Anthropic provider (optional — `pip install turnstone[anthropic]`)
|
||||
- PostgreSQL (optional, for production — `pip install turnstone[postgres]`)
|
||||
- Math sandbox packages (optional — `pip install turnstone[sandbox]` for sympy, numpy, scipy, pytest)
|
||||
- [Git LFS](https://git-lfs.com/) (for cloning — diagram PNGs are stored in LFS)
|
||||
- An OpenAI-compatible API endpoint or Anthropic API key
|
||||
- Optional: PostgreSQL (`pip install turnstone[postgres]`), Anthropic (`pip install turnstone[anthropic]`)
|
||||
- [Git LFS](https://git-lfs.com/) for cloning (diagram PNGs)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+18
-2607
File diff suppressed because it is too large
Load Diff
@@ -3,12 +3,12 @@
|
||||
# Usage (requires base compose.yaml with production profile):
|
||||
# docker compose -f compose.yaml -f deploy/docker-compose.tls.yml --profile production up
|
||||
#
|
||||
# The tls-init service bootstraps a CA and issues a cert for Redis.
|
||||
# The tls-init service bootstraps a CA and issues certs.
|
||||
# All turnstone services auto-provision their own certs via the
|
||||
# console's ACME endpoint.
|
||||
|
||||
services:
|
||||
# Bootstrap: create CA + Redis cert before anything starts.
|
||||
# Bootstrap: create CA before anything starts.
|
||||
# Runs as root to create directories in the volume, then chowns
|
||||
# to turnstone:turnstone with restrictive perms (keys 0600).
|
||||
tls-init:
|
||||
@@ -19,7 +19,7 @@ services:
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
turnstone-admin tls-bootstrap --out /certs --issue redis
|
||||
turnstone-admin tls-bootstrap --out /certs
|
||||
chown -R turnstone:turnstone /certs
|
||||
find /certs -type d -exec chmod 750 {} +
|
||||
find /certs -type f -name '*key.pem' -exec chmod 600 {} +
|
||||
@@ -45,11 +45,7 @@ services:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port=8090
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --poll-interval=${CONSOLE_POLL_INTERVAL:-10}
|
||||
- --redis-tls
|
||||
- --redis-tls-ca=/certs/ca.pem
|
||||
|
||||
# Server: auto-provisions certs via console ACME, serves HTTPS
|
||||
server:
|
||||
@@ -64,41 +60,14 @@ services:
|
||||
# Disable healthcheck — server serves HTTPS with mTLS which the
|
||||
# stdlib healthcheck script can't satisfy. The base compose
|
||||
# healthcheck uses plain HTTP which won't work on an HTTPS listener.
|
||||
# TODO: wire healthcheck with client cert from /certs volume
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
# Bridge: mTLS to server + Redis TLS
|
||||
bridge:
|
||||
depends_on:
|
||||
console:
|
||||
condition: service_healthy
|
||||
server:
|
||||
condition: service_started
|
||||
redis:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- tls-certs:/certs:ro
|
||||
environment:
|
||||
TURNSTONE_TLS_ENABLED: "true"
|
||||
TURNSTONE_TLS_SANS: "bridge"
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url=http://server:8080
|
||||
- --redis-host=redis
|
||||
- --redis-port=6379
|
||||
- --heartbeat-ttl=${HEARTBEAT_TTL:-60}
|
||||
- --approval-timeout=${APPROVAL_TIMEOUT:-3600}
|
||||
- --redis-tls
|
||||
- --redis-tls-ca=/certs/ca.pem
|
||||
|
||||
# Channel: Redis TLS
|
||||
# Channel: TLS
|
||||
channel:
|
||||
depends_on:
|
||||
console:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- tls-certs:/certs:ro
|
||||
environment:
|
||||
@@ -109,38 +78,8 @@ services:
|
||||
- -c
|
||||
- >-
|
||||
turnstone-channel
|
||||
--redis-host=redis
|
||||
--redis-port=6379
|
||||
--redis-tls
|
||||
--redis-tls-ca=/certs/ca.pem
|
||||
--http-host=0.0.0.0
|
||||
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
|
||||
|
||||
# Redis: TLS with certs from bootstrap
|
||||
redis:
|
||||
depends_on:
|
||||
tls-init:
|
||||
condition: service_completed_successfully
|
||||
volumes:
|
||||
- tls-certs:/certs:ro
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
ARGS="--tls-port 6379 --port 0 \
|
||||
--tls-cert-file /certs/certs/redis/cert.pem \
|
||||
--tls-key-file /certs/certs/redis/key.pem \
|
||||
--tls-ca-cert-file /certs/ca.pem \
|
||||
--tls-auth-clients no"
|
||||
if [ -n "$$REDIS_PASSWORD" ]; then
|
||||
ARGS="$$ARGS --requirepass $$REDIS_PASSWORD"
|
||||
fi
|
||||
exec redis-server $$ARGS
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "if [ -n \"$$REDIS_PASSWORD\" ]; then redis-cli --tls --cacert /certs/ca.pem -a $$REDIS_PASSWORD ping; else redis-cli --tls --cacert /certs/ca.pem ping; fi"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
tls-certs:
|
||||
|
||||
@@ -10,7 +10,3 @@ dependencies:
|
||||
version: ~18.5.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
- name: redis
|
||||
version: ~25.3.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: redis.enabled
|
||||
|
||||
@@ -25,14 +25,10 @@ Then open: http://localhost:{{ .Values.console.service.port }}
|
||||
|
||||
Components deployed:
|
||||
- Server: {{ include "turnstone.fullname" . }}-server ({{ .Values.server.replicas }} replica(s))
|
||||
- Bridge: {{ include "turnstone.fullname" . }}-bridge ({{ .Values.bridge.replicas }} replica(s))
|
||||
- Console: {{ include "turnstone.fullname" . }}-console ({{ .Values.console.replicas }} replica(s))
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
- PostgreSQL (bitnami subchart)
|
||||
{{- end }}
|
||||
{{- if .Values.redis.enabled }}
|
||||
- Redis (bitnami subchart)
|
||||
{{- end }}
|
||||
|
||||
{{- if not .Values.llm.apiKey }}
|
||||
{{- if not .Values.llm.existingSecret }}
|
||||
|
||||
@@ -110,28 +110,6 @@ Determine the PostgreSQL username.
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the Redis host.
|
||||
*/}}
|
||||
{{- define "turnstone.redis.host" -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- printf "%s-redis-master" .Release.Name }}
|
||||
{{- else }}
|
||||
{{- .Values.redis.external.host }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the Redis port.
|
||||
*/}}
|
||||
{{- define "turnstone.redis.port" -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- printf "6379" }}
|
||||
{{- else }}
|
||||
{{- .Values.redis.external.port | toString }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Determine the secret name for LLM API keys.
|
||||
*/}}
|
||||
|
||||
@@ -14,8 +14,6 @@ data:
|
||||
TURNSTONE_SERVER_PORT: {{ .Values.server.service.port | quote }}
|
||||
TURNSTONE_CONSOLE_HOST: "0.0.0.0"
|
||||
TURNSTONE_CONSOLE_PORT: {{ .Values.console.service.port | quote }}
|
||||
TURNSTONE_REDIS_HOST: {{ include "turnstone.redis.host" . | quote }}
|
||||
TURNSTONE_REDIS_PORT: {{ include "turnstone.redis.port" . | quote }}
|
||||
TURNSTONE_POLL_INTERVAL: "5"
|
||||
{{- if .Values.llm.baseUrl }}
|
||||
TURNSTONE_LLM_BASE_URL: {{ .Values.llm.baseUrl | quote }}
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "turnstone.fullname" . }}-bridge
|
||||
labels:
|
||||
{{- include "turnstone.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
spec:
|
||||
replicas: {{ .Values.bridge.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "turnstone.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: bridge
|
||||
spec:
|
||||
serviceAccountName: {{ include "turnstone.serviceAccountName" . }}
|
||||
containers:
|
||||
- name: bridge
|
||||
image: {{ include "turnstone.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- turnstone-bridge
|
||||
- --server-url={{ printf "http://%s-server:%s" (include "turnstone.fullname" .) (.Values.server.service.port | toString) }}
|
||||
- --redis-host={{ include "turnstone.redis.host" . }}
|
||||
- --redis-port={{ include "turnstone.redis.port" . }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "turnstone.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "turnstone.llm.secretName" . }}
|
||||
optional: true
|
||||
{{- if and .Values.auth.enabled .Values.auth.existingSecret }}
|
||||
env:
|
||||
- name: TURNSTONE_AUTH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.auth.existingSecret }}
|
||||
key: TURNSTONE_AUTH_TOKEN
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.bridge.resources | nindent 12 }}
|
||||
@@ -26,8 +26,6 @@ spec:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port={{ .Values.console.service.port }}
|
||||
- --redis-host={{ include "turnstone.redis.host" . }}
|
||||
- --redis-port={{ include "turnstone.redis.port" . }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.console.service.port }}
|
||||
|
||||
@@ -18,11 +18,4 @@ data:
|
||||
{{- if and .Values.auth.enabled .Values.auth.token (not .Values.auth.existingSecret) }}
|
||||
TURNSTONE_AUTH_TOKEN: {{ .Values.auth.token | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.redis.enabled .Values.redis.auth }}
|
||||
{{- if .Values.redis.auth.password }}
|
||||
REDIS_PASSWORD: {{ .Values.redis.auth.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- else if and (not .Values.redis.enabled) .Values.redis.external.password }}
|
||||
REDIS_PASSWORD: {{ .Values.redis.external.password | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -24,16 +24,6 @@ postgresql:
|
||||
database: turnstone
|
||||
username: turnstone
|
||||
|
||||
# -- Redis configuration
|
||||
redis:
|
||||
enabled: true
|
||||
architecture: standalone
|
||||
# External Redis settings (used when redis.enabled is false)
|
||||
external:
|
||||
host: ""
|
||||
port: 6379
|
||||
existingSecret: ""
|
||||
|
||||
# -- Turnstone server (main API + web UI)
|
||||
server:
|
||||
replicas: 1
|
||||
@@ -48,17 +38,6 @@ server:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
# -- Turnstone bridge (Redis MQ connector)
|
||||
bridge:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
|
||||
# -- Turnstone console (cluster dashboard)
|
||||
console:
|
||||
replicas: 1
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# OpenShell sandbox policy for Turnstone AI orchestration platform.
|
||||
#
|
||||
# This policy wraps a turnstone-server process (the primary sandbox target).
|
||||
# The bridge, console, and channel gateway are separate processes that would
|
||||
# each need their own sandbox with a tailored policy variant.
|
||||
# The console and channel gateway are separate processes that would each
|
||||
# need their own sandbox with a tailored policy variant.
|
||||
#
|
||||
# Usage:
|
||||
# openshell sandbox run \
|
||||
@@ -23,7 +23,6 @@
|
||||
#
|
||||
# Customization points (search for "CUSTOMIZE"):
|
||||
# - OIDC issuer endpoint
|
||||
# - Redis host/port (if not localhost)
|
||||
# - MCP HTTP server endpoints
|
||||
# - Additional tool binaries
|
||||
# - web_fetch domain allowlist
|
||||
@@ -166,20 +165,6 @@ network_policies:
|
||||
# - path: /usr/bin/python3*
|
||||
# - path: /usr/local/bin/python3*
|
||||
|
||||
# --- Redis (MQ) ---
|
||||
# CUSTOMIZE: if Redis is not on localhost, add host + allowed_ips.
|
||||
# localhost is blocked by default SSRF protection, so we need allowed_ips.
|
||||
|
||||
redis:
|
||||
name: redis-mq
|
||||
endpoints:
|
||||
- port: 6379
|
||||
allowed_ips:
|
||||
- "127.0.0.1"
|
||||
binaries:
|
||||
- path: /usr/bin/python3*
|
||||
- path: /usr/local/bin/python3*
|
||||
|
||||
# --- Discord (channel integration) ---
|
||||
# Uncomment if using turnstone-channel with Discord adapter.
|
||||
|
||||
|
||||
@@ -22,8 +22,3 @@ output "rds_endpoint" {
|
||||
description = "RDS PostgreSQL endpoint."
|
||||
value = module.turnstone.rds_endpoint
|
||||
}
|
||||
|
||||
output "redis_endpoint" {
|
||||
description = "ElastiCache Redis endpoint."
|
||||
value = module.turnstone.redis_endpoint
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ variable "vpc_id" {
|
||||
}
|
||||
|
||||
variable "private_subnet_ids" {
|
||||
description = "List of private subnet IDs for ECS tasks, RDS, and ElastiCache."
|
||||
description = "List of private subnet IDs for ECS tasks and RDS."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# ---------- ElastiCache Subnet Group ----------
|
||||
|
||||
resource "aws_elasticache_subnet_group" "this" {
|
||||
name = "${var.name_prefix}-${var.environment}"
|
||||
subnet_ids = var.private_subnet_ids
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- ElastiCache Redis Replication Group ----------
|
||||
|
||||
resource "aws_elasticache_replication_group" "this" {
|
||||
replication_group_id = "${var.name_prefix}-${var.environment}"
|
||||
description = "Turnstone Redis for MQ and session state"
|
||||
|
||||
engine = "redis"
|
||||
engine_version = "7.1"
|
||||
node_type = var.redis_node_type
|
||||
num_cache_clusters = 1
|
||||
port = 6379
|
||||
|
||||
subnet_group_name = aws_elasticache_subnet_group.this.name
|
||||
security_group_ids = [aws_security_group.redis.id]
|
||||
|
||||
at_rest_encryption_enabled = true
|
||||
transit_encryption_enabled = true
|
||||
|
||||
automatic_failover_enabled = false
|
||||
|
||||
tags = local.common_tags
|
||||
}
|
||||
@@ -27,7 +27,6 @@ locals {
|
||||
{ name = "TURNSTONE_ENV", value = var.environment },
|
||||
{ name = "TURNSTONE_DB_BACKEND", value = "postgresql" },
|
||||
{ name = "TURNSTONE_LLM_BASE_URL", value = var.llm_base_url },
|
||||
{ name = "TURNSTONE_REDIS_URL", value = "redis://${aws_elasticache_replication_group.this.primary_endpoint_address}:6379/0" },
|
||||
]
|
||||
|
||||
# Secrets pulled from Secrets Manager at container start.
|
||||
@@ -187,57 +186,6 @@ resource "aws_ecs_service" "server" {
|
||||
depends_on = [aws_lb_target_group.server]
|
||||
}
|
||||
|
||||
# ---------- Bridge Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "bridge" {
|
||||
family = "${var.name_prefix}-bridge"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = var.bridge_cpu
|
||||
memory = var.bridge_memory
|
||||
execution_role_arn = aws_iam_role.ecs_execution.arn
|
||||
task_role_arn = aws_iam_role.ecs_task.arn
|
||||
tags = local.common_tags
|
||||
|
||||
container_definitions = jsonencode([
|
||||
{
|
||||
name = "bridge"
|
||||
image = local.full_image
|
||||
essential = true
|
||||
command = ["turnstone-bridge"]
|
||||
|
||||
environment = concat(local.common_env, local.auth_env)
|
||||
secrets = concat(local.common_secrets, local.auth_secrets)
|
||||
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
"awslogs-group" = aws_cloudwatch_log_group.this.name
|
||||
"awslogs-region" = data.aws_region.current.name
|
||||
"awslogs-stream-prefix" = "bridge"
|
||||
}
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
resource "aws_ecs_service" "bridge" {
|
||||
name = "${var.name_prefix}-bridge"
|
||||
cluster = aws_ecs_cluster.this.id
|
||||
task_definition = aws_ecs_task_definition.bridge.arn
|
||||
desired_count = 1
|
||||
launch_type = "FARGATE"
|
||||
tags = local.common_tags
|
||||
|
||||
network_configuration {
|
||||
subnets = var.private_subnet_ids
|
||||
security_groups = [aws_security_group.ecs_tasks.id]
|
||||
assign_public_ip = false
|
||||
}
|
||||
|
||||
depends_on = [aws_ecs_service.server]
|
||||
}
|
||||
|
||||
# ---------- Console Task Definition + Service ----------
|
||||
|
||||
resource "aws_ecs_task_definition" "console" {
|
||||
|
||||
@@ -22,8 +22,3 @@ output "rds_endpoint" {
|
||||
description = "Endpoint of the RDS PostgreSQL instance (host:port)."
|
||||
value = aws_db_instance.this.endpoint
|
||||
}
|
||||
|
||||
output "redis_endpoint" {
|
||||
description = "Primary endpoint of the ElastiCache Redis replication group."
|
||||
value = aws_elasticache_replication_group.this.primary_endpoint_address
|
||||
}
|
||||
|
||||
@@ -112,22 +112,3 @@ resource "aws_vpc_security_group_ingress_rule" "rds_from_ecs" {
|
||||
referenced_security_group_id = aws_security_group.ecs_tasks.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
# ---------- Redis Security Group ----------
|
||||
|
||||
resource "aws_security_group" "redis" {
|
||||
name = "${var.name_prefix}-redis-${var.environment}"
|
||||
description = "Allow Redis access from ECS tasks"
|
||||
vpc_id = var.vpc_id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "redis_from_ecs" {
|
||||
security_group_id = aws_security_group.redis.id
|
||||
description = "Redis from ECS tasks"
|
||||
from_port = 6379
|
||||
to_port = 6379
|
||||
ip_protocol = "tcp"
|
||||
referenced_security_group_id = aws_security_group.ecs_tasks.id
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ variable "vpc_id" {
|
||||
}
|
||||
|
||||
variable "private_subnet_ids" {
|
||||
description = "List of private subnet IDs for ECS tasks, RDS, and ElastiCache."
|
||||
description = "List of private subnet IDs for ECS tasks and RDS."
|
||||
type = list(string)
|
||||
}
|
||||
|
||||
@@ -50,14 +50,6 @@ variable "db_instance_class" {
|
||||
default = "db.t4g.micro"
|
||||
}
|
||||
|
||||
# --- ElastiCache ---
|
||||
|
||||
variable "redis_node_type" {
|
||||
description = "ElastiCache node type for Redis."
|
||||
type = string
|
||||
default = "cache.t4g.micro"
|
||||
}
|
||||
|
||||
# --- ECS Task Sizing ---
|
||||
|
||||
variable "server_cpu" {
|
||||
@@ -72,18 +64,6 @@ variable "server_memory" {
|
||||
default = 1024
|
||||
}
|
||||
|
||||
variable "bridge_cpu" {
|
||||
description = "CPU units for the bridge task."
|
||||
type = number
|
||||
default = 256
|
||||
}
|
||||
|
||||
variable "bridge_memory" {
|
||||
description = "Memory (MiB) for the bridge task."
|
||||
type = number
|
||||
default = 512
|
||||
}
|
||||
|
||||
variable "console_cpu" {
|
||||
description = "CPU units for the console task."
|
||||
type = number
|
||||
|
||||
+52
-3
@@ -2,8 +2,6 @@
|
||||
|
||||
## Overview
|
||||
|
||||
> See also: [MQ Protocol diagram](diagrams/png/06-mq-protocol.png) | [Message Routing diagram](diagrams/png/07-message-routing.png) | [Redis Key Schema diagram](diagrams/png/08-redis-key-schema.png)
|
||||
|
||||
`turnstone-server` exposes a browser-based chat UI backed by a
|
||||
**Starlette** ASGI application served by **uvicorn**. The server uses
|
||||
**Server-Sent Events (SSE)** via `sse-starlette` for real-time streaming
|
||||
@@ -523,7 +521,7 @@ inactivity.
|
||||
|
||||
Each SSE connection to a workstream receives its own delivery queue. Events
|
||||
produced by the worker thread are fanned out to all registered listener queues,
|
||||
so multiple consumers (browser, bridge, console proxy, SDK) can connect
|
||||
so multiple consumers (browser, console proxy, SDK) can connect
|
||||
simultaneously and each receives every event. On reconnect the client receives
|
||||
a full history replay, so no catch-up mechanism is needed.
|
||||
|
||||
@@ -1859,3 +1857,54 @@ turnstone_tokens_total{type="completion"} 12150
|
||||
turnstone_tool_calls_total{tool="bash"} 7
|
||||
turnstone_tool_calls_total{tool="read_file"} 3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Console Routing Proxy Endpoints
|
||||
|
||||
These endpoints are served by the console (`turnstone-console`) and proxy
|
||||
requests to the correct server node via the hash ring bucket cache. In
|
||||
multi-node deployments, clients (SDK, channel gateway) talk to the console
|
||||
instead of individual server nodes.
|
||||
|
||||
### `POST /v1/api/route/workstreams/new`
|
||||
|
||||
Create a workstream via hash-ring routing. The console generates the `ws_id`,
|
||||
routes to the assigned node, and includes `node_url` in the response for
|
||||
direct SSE connections.
|
||||
|
||||
### `POST /v1/api/route/send`
|
||||
|
||||
Proxy a message to the workstream's assigned server node.
|
||||
|
||||
### `POST /v1/api/route/approve`
|
||||
|
||||
Proxy an approval response to the workstream's assigned server node.
|
||||
|
||||
### `POST /v1/api/route/cancel`
|
||||
|
||||
Cancel generation on a workstream.
|
||||
|
||||
### `POST /v1/api/route/command`
|
||||
|
||||
Send a slash command to a workstream.
|
||||
|
||||
### `POST /v1/api/route/plan`
|
||||
|
||||
Send plan review feedback to a workstream.
|
||||
|
||||
### `POST /v1/api/route/workstreams/close`
|
||||
|
||||
Close a workstream.
|
||||
|
||||
### `GET /v1/api/route?ws_id=X`
|
||||
|
||||
Look up which server node owns a workstream. Returns `{"node_url": "...", "node_id": "..."}`.
|
||||
Used by channel adapters to open direct SSE connections to the correct server node.
|
||||
|
||||
### `GET /metrics` (Console)
|
||||
|
||||
Prometheus metrics for the console routing layer. Includes:
|
||||
`turnstone_router_requests_total`, `turnstone_router_request_duration_seconds`,
|
||||
`turnstone_ring_membership_size`, `turnstone_ring_version`,
|
||||
`turnstone_ring_rebalance_total`, `turnstone_ring_migrations_total`.
|
||||
|
||||
+35
-99
@@ -3,7 +3,7 @@
|
||||
Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent
|
||||
memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or
|
||||
Anthropic's native Messages API via pluggable provider adapters, and gives the
|
||||
model 17 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
model 19 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
reading, writing, searching, planning, and executing code.
|
||||
|
||||
The core design principle is a **UI-agnostic engine with pluggable frontends**.
|
||||
@@ -18,10 +18,9 @@ plugs in.
|
||||
|---------|--------|----------|---------|
|
||||
| `turnstone` | `turnstone.cli` | `TerminalUI` | Interactive terminal REPL |
|
||||
| `turnstone-server` | `turnstone.server` | `WebUI` | Browser-based chat (HTTP + SSE) |
|
||||
| `turnstone-bridge` | `turnstone.mq.bridge` | Bridge | Message queue ↔ HTTP API bridge |
|
||||
| `turnstone-console` | `turnstone.console.server` | ClusterCollector | Cluster dashboard (aggregates all nodes) |
|
||||
| `turnstone-eval` | `turnstone.eval` | `NullUI` | Headless evaluation and prompt optimization |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) via Redis MQ |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) |
|
||||
| `turnstone-admin` | `turnstone.core.admin_cli` | — | Offline user and API token management |
|
||||
|
||||
---
|
||||
@@ -42,7 +41,7 @@ turnstone/
|
||||
__init__.py create_provider() + create_client() factory functions
|
||||
workstream.py Parallel workstream manager (WorkstreamState, Workstream, WorkstreamManager)
|
||||
tools.py Tool schema loader (JSON -> OpenAI function-calling format)
|
||||
mcp_client.py MCPClientManager — MCP server connections, tool discovery, dynamic refresh, async-sync bridge
|
||||
mcp_client.py MCPClientManager — MCP server connections, tool discovery, dynamic refresh
|
||||
tool_search.py Dynamic tool search — BM25 index, session-scoped tool visibility
|
||||
watch.py WatchRunner daemon — periodic command polling, condition DSL, result dispatch
|
||||
judge.py Intent validation — heuristic rules + LLM judge, advisory verdicts
|
||||
@@ -74,20 +73,15 @@ turnstone/
|
||||
_base.py Shared httpx async client, auth, error handling
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
mq/
|
||||
protocol.py Inbound/outbound message dataclasses (JSON serialization)
|
||||
broker.py Abstract MessageBroker protocol + RedisBroker
|
||||
bridge.py Bridge service (queue ↔ turnstone-server HTTP API)
|
||||
client.py TurnstoneClient library + TurnResult for MQ-based access
|
||||
console/
|
||||
collector.py ClusterCollector — aggregates state from all nodes via Redis + HTTP
|
||||
scheduler.py TaskScheduler — background cron/at scheduler, dispatches via MQ
|
||||
collector.py ClusterCollector — aggregates state from all nodes via SSE
|
||||
scheduler.py TaskScheduler — background cron/at scheduler, dispatches via HTTP
|
||||
server.py Cluster dashboard HTTP server + SSE + CLI entry point
|
||||
static/ Cluster dashboard web UI (page-specific HTML, CSS, JS)
|
||||
channels/
|
||||
cli.py Unified channel gateway entry point (turnstone-channel)
|
||||
_protocol.py ChannelAdapter protocol, ChannelEvent dataclass
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via MQ
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via HTTP
|
||||
_config.py Base ChannelConfig dataclass
|
||||
discord/ Discord adapter (bot, cog, views, streaming, config)
|
||||
shared_static/ Shared design system (base.css, auth.js, theme.js, toast.js, utils.js, kb.js)
|
||||
@@ -704,8 +698,7 @@ supports_vision = true
|
||||
sub-agents, allowing a cheaper model for autonomous loops
|
||||
|
||||
**Per-workstream selection:** `POST /v1/api/workstreams/new` accepts an optional
|
||||
`"model"` field. The bridge `CreateWorkstreamMessage` carries the same field
|
||||
through the MQ protocol, along with `skill` (skill name)
|
||||
`"model"` field, along with `skill` (skill name)
|
||||
which can override the model before workstream creation.
|
||||
|
||||
### Tool Output Truncation
|
||||
@@ -1205,95 +1198,39 @@ calls `_fg_event.wait()`, which blocks the worker thread until the user
|
||||
switches to that workstream. The `_bg_attention_notify` callback writes a
|
||||
bell + status line to stderr to alert the user.
|
||||
|
||||
### Message Queue Bridge
|
||||
|
||||
```
|
||||
Main thread Global SSE thread Per-WS SSE threads (×N)
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| Inbound loop | | GET /events/glob | | GET /events?ws_id |
|
||||
| BLPOP on Redis | | Parse SSE via | | Parse SSE via |
|
||||
| | | httpx-sse | | httpx-sse |
|
||||
| Dispatch to | | Forward state | | Forward content, |
|
||||
| handler | | changes | | tool results |
|
||||
| POST to server | | Detect turn | | Handle approval |
|
||||
| Publish ACK | | completion | | forwarding |
|
||||
+------------------+ +------------------+ +-------------------+
|
||||
| | |
|
||||
+-- Redis inbound queue +-- Redis pub/sub +-- Redis pub/sub
|
||||
(RPUSH/BLPOP) (PUBLISH) (PUBLISH)
|
||||
+ response queue
|
||||
(BLPOP on
|
||||
approval)
|
||||
```
|
||||
|
||||
**Approval flow:** When a per-WS SSE thread receives an `approve_request`, it checks
|
||||
the workstream's `auto_approve_tools` set. If all requested tools are in the set, the
|
||||
bridge auto-approves via `POST /v1/api/approve`. Otherwise, it publishes an
|
||||
`ApprovalRequestEvent` to the outbound channel with a `request_id`, then blocks on
|
||||
`BLPOP` of a Redis response queue (`turnstone:resp:{request_id}`) until the client pushes
|
||||
a response or the approval timeout (default 3600s / 1 hour) expires.
|
||||
|
||||
**Cancellation:** The `CancelMessage` (type `"cancel"`) is a routed inbound message.
|
||||
The bridge dispatches it to `POST /v1/api/cancel` on the server owning the workstream,
|
||||
which sets the cooperative cancel flag and unblocks any pending approval/plan waits.
|
||||
|
||||
**Completion detection:** The bridge tracks which `correlation_id` maps to which
|
||||
`ws_id` for active sends. The server accumulates content tokens in the WebUI and
|
||||
piggybacks the full response text onto the `ws_state → idle` global SSE event.
|
||||
When the bridge receives this event, it emits a synthetic `TurnCompleteEvent`
|
||||
carrying the correlation ID and the server-provided `content`. This lets downstream
|
||||
consumers (e.g. the Discord bot) recover the full response when individual
|
||||
`ContentEvent`s were missed, and serves as the primary delivery path for
|
||||
bidirectional notification DM forwarding.
|
||||
|
||||
**Multi-node routing:** Each bridge retrieves its `node_id` from the server's
|
||||
`/health` endpoint on startup (with exponential backoff retry). The server
|
||||
generates the `node_id` (`{hostname}_{4hex}`) and is the sole authority for
|
||||
node identity. The bridge BLPOPs
|
||||
from both `turnstone:inbound:{node_id}` (directed, priority) and `turnstone:inbound` (shared).
|
||||
Messages with `target_node` set are pushed to the target's per-node queue. Messages
|
||||
for existing workstreams are auto-routed via `turnstone:ws:{ws_id}` ownership keys in Redis.
|
||||
If a bridge picks up a shared-queue message for a workstream owned by another node, it
|
||||
re-routes to that node's queue (1 extra hop). Bridges publish heartbeats to
|
||||
`turnstone:node:{node_id}` with configurable TTL for node discovery.
|
||||
On startup, `_recover_workstreams` re-registers ownership of existing
|
||||
workstreams and publishes `WorkstreamCreatedEvent` to the cluster channel
|
||||
so the console collector picks them up immediately.
|
||||
|
||||
### Cluster Console
|
||||
|
||||
```
|
||||
Monitoring (3 daemon threads) Control + Proxy (async Starlette)
|
||||
Monitoring (2 daemon threads) Control + Proxy (async Starlette)
|
||||
+------------------+ +----------------------------+
|
||||
| Event subscriber | | POST /v1/api/cluster/ |
|
||||
| SUBSCRIBE on | | workstreams/new |
|
||||
| events:cluster | | → LPUSH to Redis |
|
||||
+------------------+ | inbound:{node_id} |
|
||||
| Node discovery | +----------------------------+
|
||||
| SCAN node:* keys | | GET /node/{node_id}/ |
|
||||
| every 15 seconds | | → httpx.AsyncClient |
|
||||
+------------------+ | proxy to server_url |
|
||||
| Poll loop | | GET /node/{id}/v1/api/events |
|
||||
| GET /v1/api/dash | | → SSE stream proxy |
|
||||
| GET /health | | POST /node/{id}/v1/api/send |
|
||||
| ThreadPoolExec | | → forwarded to server |
|
||||
| Node discovery | | POST /v1/api/cluster/ |
|
||||
| Service registry | | workstreams/new |
|
||||
| every 60 seconds | | → POST to target server |
|
||||
+------------------+ +----------------------------+
|
||||
| SSE manager | | GET /node/{node_id}/ |
|
||||
| asyncio loop | | → httpx.AsyncClient |
|
||||
| 1 task per node | | proxy to server_url |
|
||||
| /events/global | | GET /node/{id}/v1/api/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/send |
|
||||
| → forwarded to server |
|
||||
+----------------------------+
|
||||
```
|
||||
|
||||
The console HTTP layer is a Starlette/ASGI app served by uvicorn. The SSE
|
||||
endpoint uses `EventSourceResponse` with the same listener queue pattern as
|
||||
the main server. `ClusterCollector`'s background threads (event subscriber,
|
||||
node discovery, poll loop) use sync Redis clients and `ThreadPoolExecutor`
|
||||
for parallel HTTP polling. The poll loop diffs workstream IDs between poll
|
||||
cycles and fans out synthetic `ws_created`/`ws_closed` SSE events for any
|
||||
changes, ensuring browser clients stay in sync even when real-time cluster
|
||||
events are missed (e.g. bridge startup recovery).
|
||||
the main server. `ClusterCollector` runs two daemon threads: a discovery loop
|
||||
that queries the service registry every 60 seconds, and an SSE manager that
|
||||
runs a single asyncio event loop multiplexing persistent SSE connections to
|
||||
all nodes via `GET /v1/api/events/global`. Each node delivers a full snapshot
|
||||
on connect followed by real-time delta events — state changes, health
|
||||
transitions, and aggregate metrics arrive sub-second instead of on a 15-second
|
||||
poll cycle.
|
||||
|
||||
The console has two write-path capabilities:
|
||||
|
||||
1. **Workstream creation** — pushes `CreateWorkstreamMessage` to Redis inbound
|
||||
queues targeting specific nodes. The bridge on each node picks up the message
|
||||
and creates the workstream on the local server. Auto-selects the node with
|
||||
1. **Workstream creation** — sends HTTP requests to target server nodes
|
||||
to create workstreams. Auto-selects the node with
|
||||
the most available capacity if no target is specified. When a `skill`
|
||||
field is present, the server resolves the skill BEFORE `mgr.create()`
|
||||
(applying the model override to the creation request) and snapshot-applies
|
||||
@@ -1360,7 +1297,7 @@ event loop on a daemon thread.
|
||||
|
||||
**Event types**: 27 standalone dataclasses in `events.py` with a type-registry
|
||||
pattern matching `OutboundEvent.from_json()` from `mq/protocol.py`. Events are
|
||||
decoupled from the MQ package so SDK consumers don't need the `redis` dependency.
|
||||
decoupled from server internals.
|
||||
|
||||
**TypeScript SDK**: `sdk/typescript/` — separate npm package with the same API
|
||||
surface. Zero browser dependencies, SSE via `fetch` + `ReadableStream` parsing.
|
||||
@@ -1381,20 +1318,19 @@ with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
|
||||
> See also: [Channel Integrations guide](channels.md)
|
||||
|
||||
The `turnstone-channel` gateway bridges external messaging platforms
|
||||
(Discord, Slack, Teams) to the turnstone cluster via Redis MQ. Each
|
||||
The `turnstone-channel` gateway connects external messaging platforms
|
||||
(Discord, Slack, Teams) to the turnstone cluster via HTTP. Each
|
||||
platform adapter implements the `ChannelAdapter` protocol and translates
|
||||
between platform-native events and turnstone MQ messages.
|
||||
between platform-native events and turnstone server API calls.
|
||||
|
||||
The `ChannelRouter` manages bidirectional routing: it maps platform
|
||||
channel/thread IDs to turnstone workstream IDs, handles workstream
|
||||
creation and stale-route recovery, and resolves platform users to
|
||||
turnstone identities via the `channel_users` table. When an evicted
|
||||
workstream is reactivated, the router uses atomic resume via the
|
||||
`resume_ws` field on `CreateWorkstreamMessage` — the server resumes
|
||||
`resume_ws` field on the workstream creation request — the server resumes
|
||||
the old workstream's conversation during creation in a single HTTP
|
||||
request, eliminating ordering fragility. The bridge emits a
|
||||
`WorkstreamResumedEvent` to confirm success.
|
||||
request, eliminating ordering fragility.
|
||||
|
||||
Discord ships as the first adapter. See [channels.md](channels.md) for
|
||||
setup instructions, configuration reference, and the adapter development
|
||||
@@ -1403,7 +1339,7 @@ guide.
|
||||
### Notification Subsystem
|
||||
|
||||
The `notify` tool enables the LLM to send notifications to users or
|
||||
channels without going through MQ. The server calls the channel gateway
|
||||
channels directly. The server calls the channel gateway
|
||||
directly over HTTP for lower latency: `_exec_notify()` queries the
|
||||
`services` database table for healthy channel gateways (heartbeat within
|
||||
120 seconds), authenticates with a service JWT (`aud: turnstone-channel`),
|
||||
|
||||
+18
-25
@@ -1,9 +1,10 @@
|
||||
# Channel Integrations
|
||||
|
||||
The `turnstone-channel` gateway connects external messaging platforms to
|
||||
turnstone workstreams via Redis MQ. Each platform adapter translates
|
||||
turnstone workstreams via direct HTTP to the server (single-node) or the
|
||||
console routing proxy (multi-node). Each platform adapter translates
|
||||
platform-native events (messages, button clicks, slash commands) into
|
||||
turnstone MQ messages, and renders workstream output back into the
|
||||
turnstone API calls, and renders workstream output back into the
|
||||
platform's UI.
|
||||
|
||||
Discord ships as the first adapter. The adapter protocol is designed for
|
||||
@@ -20,10 +21,9 @@ Discord Gateway
|
||||
turnstone-channel (Discord adapter)
|
||||
|
|
||||
v
|
||||
Redis MQ
|
||||
|
|
||||
v
|
||||
turnstone-bridge ──> turnstone-server
|
||||
turnstone-server (direct HTTP)
|
||||
or
|
||||
turnstone-console (routing proxy, multi-node)
|
||||
```
|
||||
|
||||
Key components:
|
||||
@@ -34,10 +34,7 @@ Key components:
|
||||
`send_approval_request()`, `send_plan_review()`, and `create_thread()`.
|
||||
- **ChannelRouter** (`turnstone/channels/_routing.py`) — maps
|
||||
channel/thread IDs to turnstone workstream IDs. Handles workstream
|
||||
creation via MQ, stale route detection, and user identity resolution.
|
||||
- **AsyncRedisBroker** (`turnstone/mq/async_broker.py`) — async Redis
|
||||
client compatible with discord.py's event loop. Used by the router for
|
||||
pub/sub and queue operations.
|
||||
creation via HTTP, stale route detection, and user identity resolution.
|
||||
- **channel_users table** — maps `(channel_type, channel_user_id)` to a
|
||||
turnstone `user_id`. Messages from unlinked users are silently dropped.
|
||||
- **channel_routes table** — persistent channel-to-workstream mappings.
|
||||
@@ -84,8 +81,7 @@ TURNSTONE_DISCORD_GUILD=123456789 # optional, restrict to one guild
|
||||
turnstone-channel \
|
||||
--discord-token "your-bot-token" \
|
||||
--discord-guild 123456789 \
|
||||
--redis-host localhost \
|
||||
--redis-port 6379
|
||||
--server-url http://localhost:8080
|
||||
```
|
||||
|
||||
**Docker Compose** (production profile):
|
||||
@@ -138,7 +134,7 @@ An admin can also force-link or unlink users via the console admin panel
|
||||
thread auto-creates a new workstream and atomically resumes the
|
||||
previous workstream via the `resume_ws` field on
|
||||
`CreateWorkstreamMessage`. The server resumes the workstream during
|
||||
creation (same HTTP request), and the bridge emits a
|
||||
creation (same HTTP request), and the server emits a
|
||||
`WorkstreamResumedEvent` back to the channel. The thread receives a
|
||||
*"Resumed: {name} ({count} messages restored)"* confirmation.
|
||||
|
||||
@@ -160,8 +156,7 @@ an orange embed with:
|
||||
- Tool name and argument preview
|
||||
- **Approve** (green), **Reject** (red), **Always Approve** (gray) buttons
|
||||
- Only linked users can interact with approval buttons
|
||||
- The approval decision is forwarded through MQ to the bridge, which
|
||||
relays it to the server
|
||||
- The approval decision is forwarded to the server via HTTP
|
||||
|
||||
Buttons use static `custom_id` values so they survive bot restarts.
|
||||
Correlation data (`ws_id`, `correlation_id`) is stored in the embed footer.
|
||||
@@ -181,7 +176,7 @@ Plan review requests are displayed as a blue embed with:
|
||||
- **Approve Plan** (green) button — approves the plan with empty feedback
|
||||
- **Request Changes** (gray) button — opens a modal for feedback text
|
||||
(up to 2000 characters)
|
||||
- Feedback is forwarded through MQ as a `PlanFeedbackMessage`
|
||||
- Feedback is forwarded to the server via HTTP
|
||||
|
||||
---
|
||||
|
||||
@@ -192,10 +187,8 @@ Plan review requests are displayed as a blue embed with:
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Bot token (required to enable Discord) |
|
||||
| `--discord-guild` | — | `0` (all guilds) | Restrict to a single Discord guild |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated channel IDs to allow |
|
||||
| `--redis-host` | `REDIS_HOST` | `localhost` | Redis host |
|
||||
| `--redis-port` | — | `6379` | Redis port |
|
||||
| `--redis-password` | `REDIS_PASSWORD` | — | Redis password |
|
||||
| `--redis-db` | — | `0` | Redis DB number |
|
||||
| `--server-url` | `TURNSTONE_SERVER_URL` | `http://localhost:8080` | Server URL (single-node) |
|
||||
| `--console-url` | `TURNSTONE_CONSOLE_URL` | — | Console URL (multi-node routing proxy) |
|
||||
| `--model` | — | server default | Default model for new workstreams |
|
||||
| `--auto-approve` | — | `false` | Auto-approve ALL tool calls (skips approval buttons entirely) |
|
||||
| `--http-host` | — | `127.0.0.1` | HTTP server bind address for notify endpoint |
|
||||
@@ -232,13 +225,13 @@ See [Security: Database Schema](security.md#database-schema) for the
|
||||
3. **Eviction** — the server evicts an idle workstream for capacity. The
|
||||
route is preserved and the thread stays open.
|
||||
4. **Reactivation** — the next message in the thread detects the stale
|
||||
route (no MQ owner) and creates a new workstream with the old `ws_id`
|
||||
as `resume_ws` on the `CreateWorkstreamMessage`. The server resumes
|
||||
route and creates a new workstream with the old `ws_id`
|
||||
as `resume_ws` on the creation request. The server resumes
|
||||
the workstream during creation (no separate command or reverse lookup
|
||||
needed). The bridge emits a `WorkstreamResumedEvent` to the channel, and
|
||||
needed). The channel receives a `WorkstreamResumedEvent`, and
|
||||
the thread displays *"Resumed: {name} ({count} messages restored)"*.
|
||||
If the old workstream was pruned, a fresh one starts with no error.
|
||||
5. **Close** — `/close` command closes the workstream via MQ, deletes the
|
||||
5. **Close** — `/close` command closes the workstream via HTTP, deletes the
|
||||
route, unsubscribes from events, and archives the Discord thread.
|
||||
|
||||
---
|
||||
@@ -264,7 +257,7 @@ Two modes:
|
||||
|
||||
### Delivery Flow
|
||||
|
||||
Notifications bypass MQ for lower latency. The server calls the channel
|
||||
Notifications use direct HTTP for low latency. The server calls the channel
|
||||
gateway directly over HTTP:
|
||||
|
||||
1. The LLM calls the `notify` tool with a message and target
|
||||
|
||||
+25
-59
@@ -1,16 +1,16 @@
|
||||
# Cluster Dashboard (turnstone-console)
|
||||
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It connects to the shared Redis broker, discovers nodes via heartbeat keys, polls each node's HTTP API for workstream data, and subscribes to a cluster event channel for real-time state changes.
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It discovers nodes via the `services` database table and subscribes to each node's SSE event stream for real-time workstream, health, and metric updates.
|
||||
|
||||
The console also supports **workstream creation** (dispatched via MQ to target nodes) and a **reverse proxy** that serves each node's server UI through the console port — so users only need network access to the console, not to individual server nodes.
|
||||
The console also supports **workstream creation** (dispatched via HTTP proxy to target nodes) and a **reverse proxy** that serves each node's server UI through the console port — so users only need network access to the console, not to individual server nodes.
|
||||
|
||||
## Architecture
|
||||
|
||||
> See also: [Console Data Flow diagram](diagrams/png/11-console-data-flow.png)
|
||||
|
||||
```
|
||||
┌── Redis ←── turnstone-bridge ←── turnstone-server
|
||||
│ (MQ) (per node) (per node)
|
||||
┌── services table ── turnstone-server
|
||||
│ (node registry) (per node)
|
||||
turnstone-console ──────┤
|
||||
(one instance) │
|
||||
└── turnstone-server (direct HTTP proxy)
|
||||
@@ -21,45 +21,28 @@ turnstone-console ──────┤
|
||||
|
||||
Data flows in two directions:
|
||||
|
||||
- **Inbound (monitoring):** Bridges publish state changes to `{prefix}:events:cluster` on Redis pub/sub. The console subscribes for real-time updates and periodically polls each node's `GET /v1/api/dashboard` for full workstream snapshots.
|
||||
- **Outbound (control):** The console pushes `CreateWorkstreamMessage` to Redis inbound queues targeting specific nodes. Bridges pick up these messages and create workstreams on their local servers.
|
||||
- **Inbound (monitoring):** The console discovers nodes via the `services` database table (nodes register on startup and send periodic heartbeats). It opens a persistent SSE connection to each node's `GET /v1/api/events/global` endpoint, receiving a full snapshot on connect followed by real-time delta events (state changes, health transitions, aggregate metrics).
|
||||
- **Outbound (control):** The console proxies workstream creation requests to target nodes via HTTP.
|
||||
- **Proxy (pass-through):** The console reverse-proxies each node's server UI at `/node/{node_id}/`, forwarding HTTP and SSE traffic so the browser never contacts server nodes directly.
|
||||
|
||||
### Data Sources
|
||||
|
||||
| Source | Method | Direction | Data |
|
||||
|--------|--------|-----------|------|
|
||||
| Redis heartbeats | `SCAN turnstone:node:*` | Read | Node discovery (node_id, server_url, started) |
|
||||
| Redis pub/sub | `SUBSCRIBE turnstone:events:cluster` | Read | State changes, creates, closes, renames |
|
||||
| Node HTTP API | `GET {server_url}/v1/api/dashboard` | Read | Full workstream list with tokens, context, activity |
|
||||
| Node HTTP API | `GET {server_url}/health` | Read | Node health status |
|
||||
| Redis inbound queue | `RPUSH turnstone:inbound:{node_id}` | Write | Workstream creation commands |
|
||||
| `services` table | Database query | Read | Node discovery (node_id, server_url, started) |
|
||||
| Node SSE | `GET {server_url}/v1/api/events/global` | Stream | Snapshot on connect, then real-time delta events (state, health, aggregate) |
|
||||
| Node HTTP API | `POST {server_url}/v1/api/workstreams/new` | Write | Workstream creation |
|
||||
| Node HTTP API | `GET/POST {server_url}/*` | Proxy | Server UI, API requests, SSE streams |
|
||||
|
||||
### Redis Key: Cluster Event Channel
|
||||
|
||||
Bridges publish to `{prefix}:events:cluster` whenever a workstream state change, creation, closure, or rename occurs. Events include `node_id` so the console can attribute them to the correct node.
|
||||
|
||||
Event types on the cluster channel:
|
||||
|
||||
| Event | Fields | Trigger |
|
||||
|-------|--------|---------|
|
||||
| `cluster_state` | ws_id, state, node_id, tokens, context_ratio, activity | Workstream state transition |
|
||||
| `ws_created` | ws_id, name, node_id | New workstream created |
|
||||
| `ws_closed` | ws_id | Workstream closed |
|
||||
| `ws_rename` | ws_id, name | Workstream renamed |
|
||||
|
||||
---
|
||||
|
||||
## ClusterCollector
|
||||
|
||||
The collector (`turnstone/console/collector.py`) maintains an in-memory snapshot of all nodes and workstreams. Three daemon threads handle data acquisition:
|
||||
The collector (`turnstone/console/collector.py`) maintains an in-memory snapshot of all nodes and workstreams. Two daemon threads handle data acquisition:
|
||||
|
||||
1. **Event subscriber** — subscribes to `{prefix}:events:cluster` via `RedisBroker.subscribe_cluster()`. Applies state changes, creates, closes, and renames to the in-memory model immediately.
|
||||
1. **Node discovery** — queries the `services` database table every 60 seconds. Adds newly discovered nodes, removes expired ones (stale heartbeats), emits `node_joined` / `node_lost` events to SSE listeners, and spawns/cancels SSE tasks for new/lost nodes.
|
||||
|
||||
2. **Node discovery** — scans heartbeat keys every 15 seconds via `broker.list_nodes()`. Adds newly discovered nodes, removes expired ones, emits `node_joined` / `node_lost` events to SSE listeners.
|
||||
|
||||
3. **Poll loop** — fetches `GET /v1/api/dashboard` and `GET /health` from each known node every 10 seconds. Uses `ThreadPoolExecutor(max_workers=50)` for parallelism. Each poll replaces the node's workstream list with the authoritative server data.
|
||||
2. **SSE manager** — a single asyncio event loop on one thread multiplexes persistent SSE connections to all discovered nodes via `GET /v1/api/events/global`. Each connection receives a `node_snapshot` on connect (workstreams, health, aggregate) followed by real-time delta events (`ws_state`, `ws_created`, `ws_closed`, `ws_rename`, `health_changed`, `aggregate`). On disconnect, the node is marked unreachable and the connection is retried with exponential backoff (1s–30s). An `?expected_node_id=` query parameter provides identity verification against IP reuse (server returns 409 on mismatch).
|
||||
|
||||
A `get_snapshot()` method builds the full cluster state under a single lock acquisition — overview aggregates and per-node workstream lists in one atomic read. This is served both as a REST endpoint and as the initial SSE event on client connect.
|
||||
|
||||
@@ -70,7 +53,7 @@ All reads and writes to the node/workstream map are protected by a single `threa
|
||||
### Scale Considerations
|
||||
|
||||
- **50,000 workstreams** (1,000 nodes × 50 per node) at ~500 bytes each = ~25 MB in memory
|
||||
- **1,000 nodes** polled in parallel — fan-out concurrency is configurable via `cluster.node_fan_out_limit` (default 200), yielding 5 batches at ~100ms each = ~0.5 second poll cycle
|
||||
- **1,000 nodes** connected via persistent SSE — a single asyncio event loop multiplexes all connections with negligible overhead. Ensure `ulimit -n` >= 4096 for fd headroom
|
||||
- **Filtering and pagination** run in-memory on the full workstream list — sub-millisecond at this scale
|
||||
- **SSE fan-out** uses per-client queues (2,000 events) — backed-up clients get events dropped, not blocking
|
||||
- **Database** — for clusters sharing PostgreSQL, use [PgBouncer](pgbouncer.md) in transaction pooling mode
|
||||
@@ -183,7 +166,7 @@ Full cluster state in a single response — all nodes with their workstreams plu
|
||||
|
||||
### `POST /v1/api/cluster/workstreams/new`
|
||||
|
||||
Create a new workstream on a target node. Dispatches a `CreateWorkstreamMessage` through the Redis MQ pipeline — the bridge on the target node picks it up and creates the workstream on the server. Requires `write` scope.
|
||||
Create a new workstream on a target node. The console proxies the creation request to the target node's HTTP API. Requires `write` scope.
|
||||
|
||||
Request:
|
||||
|
||||
@@ -197,9 +180,9 @@ Request:
|
||||
|
||||
All fields are optional:
|
||||
- `node_id` — targeting mode:
|
||||
- **omitted or `"auto"`** — console picks the reachable node with the most available capacity (max_ws - ws_total) and pushes to its directed queue.
|
||||
- **`"pool"`** — pushes to the shared inbound queue; the next available bridge picks it up (true general-pool dispatch).
|
||||
- **specific node ID** — pushes to that node's directed queue.
|
||||
- **omitted or `"auto"`** — console picks the reachable node with the most available capacity (max_ws - ws_total) and proxies the request to it.
|
||||
- **`"pool"`** — console picks a reachable node with available capacity using round-robin selection.
|
||||
- **specific node ID** — proxies the request to that node directly.
|
||||
- `name` — workstream display name. Auto-generated if omitted.
|
||||
- `model` — model alias from the target node's registry. Uses the node's default model if omitted.
|
||||
|
||||
@@ -213,7 +196,7 @@ Response:
|
||||
}
|
||||
```
|
||||
|
||||
Creation is asynchronous — the response confirms the MQ message was dispatched. A `ws_created` event on the cluster SSE stream confirms the workstream was actually created.
|
||||
The response confirms the workstream creation request was proxied to the target node. A `ws_created` event on the cluster SSE stream confirms the workstream was actually created.
|
||||
|
||||
### `GET /v1/api/cluster/events`
|
||||
|
||||
@@ -395,7 +378,7 @@ Breadcrumb: `Cluster > Running` or `Cluster > db-west-04`. Server-side paginated
|
||||
|
||||
Triggered by the "+ new" header button. A modal dialog with:
|
||||
|
||||
- **Node selector** — dropdown with three targeting modes: "Auto (best available)" picks the node with the most headroom, "General pool (any node)" pushes to the shared queue for any bridge to pick up, or a specific node from the list (showing capacity).
|
||||
- **Node selector** — dropdown with three targeting modes: "Auto (best available)" picks the node with the most headroom, "General pool (any node)" picks a node with available capacity using round-robin, or a specific node from the list (showing capacity).
|
||||
- **Profile** — optional dropdown listing enabled skills. Applies the skill's model, auto-approve policy, token budget, and other behavioral settings at creation time.
|
||||
- **Name** — optional text input. Auto-generated if left empty.
|
||||
- **Model** — optional text input for a model alias from the target node's registry.
|
||||
@@ -482,17 +465,17 @@ to create the initial admin user and receive a JWT in one step. See
|
||||
|
||||
## Scheduled Tasks
|
||||
|
||||
The console includes a background **TaskScheduler** daemon that creates workstreams on a timed basis via the MQ broker. It supports cron-based recurring schedules and one-shot `at` schedules.
|
||||
The console includes a background **TaskScheduler** daemon that creates workstreams on a timed basis via HTTP proxy to target nodes. It supports cron-based recurring schedules and one-shot `at` schedules.
|
||||
|
||||
### Architecture
|
||||
|
||||
The scheduler runs as a daemon thread inside the console process. Every `check_interval` seconds (default 15) it:
|
||||
|
||||
1. Acquires a distributed lock via Redis `SET NX EX` (prevents duplicate dispatch in multi-console deployments)
|
||||
1. Acquires a distributed lock via the `system_settings` table (prevents duplicate dispatch in multi-console deployments)
|
||||
2. Queries the storage backend for tasks whose `next_run <= now` and `enabled = true`
|
||||
3. Dispatches each due task as one or more `CreateWorkstreamMessage` via MQ
|
||||
3. Dispatches each due task as one or more workstream creation requests via HTTP proxy
|
||||
4. Updates `last_run` and computes the next `next_run` (or disables one-shot `at` tasks)
|
||||
5. Releases the lock via Lua script (safe conditional delete)
|
||||
5. Releases the lock
|
||||
|
||||
Run history is automatically pruned (runs older than 90 days) approximately once per hour.
|
||||
|
||||
@@ -508,7 +491,7 @@ Run history is automatically pruned (runs older than 90 days) approximately once
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `auto` | Picks the reachable node with the most available capacity |
|
||||
| `pool` | Pushes to the shared inbound queue (any bridge picks it up) |
|
||||
| `pool` | Picks a reachable node with available capacity using round-robin |
|
||||
| `all` | Fan-out to all reachable nodes (capped at `max_fan_out`, default 20) |
|
||||
| `<node_id>` | Targets a specific node by ID |
|
||||
|
||||
@@ -645,11 +628,6 @@ CLI flags for `turnstone-console`:
|
||||
|------|---------|-------------|
|
||||
| `--host` | `0.0.0.0` | Bind host |
|
||||
| `--port` | `8090` | HTTP port |
|
||||
| `--redis-host` | `localhost` | Redis host |
|
||||
| `--redis-port` | `6379` | Redis port |
|
||||
| `--redis-password` | `$REDIS_PASSWORD` | Redis password |
|
||||
| `--redis-db` | `0` | Redis DB |
|
||||
| `--poll-interval` | `10` | Node polling interval (seconds) |
|
||||
| `--auth-token` | `$TURNSTONE_AUTH_TOKEN` | Bearer token for server node communication and proxy |
|
||||
| `--log-level` | `INFO` | Log level |
|
||||
|
||||
@@ -660,12 +638,6 @@ Config file (`~/.config/turnstone/config.toml`):
|
||||
host = "0.0.0.0"
|
||||
port = 8090
|
||||
url = "http://localhost:8090" # used by CLI /cluster commands
|
||||
poll_interval = 10
|
||||
|
||||
[redis]
|
||||
host = "localhost"
|
||||
port = 6379
|
||||
password = "my-redis-password"
|
||||
```
|
||||
|
||||
---
|
||||
@@ -673,17 +645,11 @@ password = "my-redis-password"
|
||||
## Deployment
|
||||
|
||||
```bash
|
||||
# Start Redis
|
||||
redis-server
|
||||
|
||||
# Start turnstone servers (one per node)
|
||||
turnstone-server --port 8080
|
||||
|
||||
# Start bridges (one per server)
|
||||
turnstone-bridge --server-url http://localhost:8080 --node-id node-a
|
||||
|
||||
# Start cluster console (one instance)
|
||||
turnstone-console --redis-host localhost --port 8090 --auth-token "$TURNSTONE_AUTH_TOKEN"
|
||||
turnstone-console --port 8090 --auth-token "$TURNSTONE_AUTH_TOKEN"
|
||||
```
|
||||
|
||||
Open `http://localhost:8090` for the cluster dashboard. Create workstreams via the "+ new" button. Click any workstream to open the proxied server UI — no direct access to server ports required.
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# Consistent Hash Ring — Reference Design
|
||||
|
||||
**Status**: Reference (not currently in the hot path)
|
||||
**Date**: 2026-03-30
|
||||
|
||||
## Overview
|
||||
|
||||
This document describes a consistent hash ring algorithm evaluated during
|
||||
the design of the direct HTTP transport routing system. The current
|
||||
implementation uses weight-proportional bucket assignment with a
|
||||
donor/recipient rebalancing algorithm (see `direct-http-transport.md`).
|
||||
The consistent hash ring is documented here as a reference for future
|
||||
scalability work — if the cluster grows beyond the point where the
|
||||
weight-proportional approach is sufficient, the ring provides a
|
||||
proven alternative with stronger stability guarantees.
|
||||
|
||||
## When to consider the ring approach
|
||||
|
||||
The current weight-proportional seeding + donor/recipient rebalancer works
|
||||
well when:
|
||||
- Cluster size is moderate (< 50 nodes)
|
||||
- Nodes join/leave infrequently
|
||||
- The rebalancer runs centrally (in the console)
|
||||
|
||||
The consistent hash ring becomes advantageous when:
|
||||
- Cluster size grows large (50+ nodes) and frequent membership changes
|
||||
cause the donor/recipient algorithm to churn
|
||||
- Decentralized routing is needed (each node computes the ring locally,
|
||||
no central console required)
|
||||
- Cross-language determinism is important (multiple implementations must
|
||||
agree on the same assignment without sharing state)
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Hash function: FNV-1a (32-bit)
|
||||
|
||||
```python
|
||||
def fnv1a_32(data: bytes) -> int:
|
||||
"""FNV-1a 32-bit hash.
|
||||
|
||||
Basis: 0x811C9DC5, Prime: 0x01000193.
|
||||
XOR each byte, then multiply by prime (masked to 32 bits).
|
||||
"""
|
||||
h = 0x811C9DC5
|
||||
for b in data:
|
||||
h ^= b
|
||||
h = (h * 0x01000193) & 0xFFFFFFFF
|
||||
return h
|
||||
```
|
||||
|
||||
Known test vectors:
|
||||
- `fnv1a_32(b"")` = `0x811C9DC5` (basis value)
|
||||
- `fnv1a_32(b"foobar")` = `0xBF9CF968`
|
||||
|
||||
Cross-language implementations:
|
||||
- **Python**: loop above (no dependencies)
|
||||
- **Go**: same algorithm with `uint32` arithmetic
|
||||
- **TypeScript**: same algorithm with `>>> 0` for unsigned 32-bit
|
||||
|
||||
### Virtual nodes
|
||||
|
||||
Each physical node with weight `w` gets `w * 150` virtual positions on a
|
||||
16-bit ring (65536 positions). Virtual node `i` of physical node `N` is
|
||||
placed at:
|
||||
|
||||
```
|
||||
position = fnv1a_32(f"{N.node_id}:{i}".encode()) % 65536
|
||||
```
|
||||
|
||||
With 150 vnodes per unit weight:
|
||||
- 2 equal-weight nodes: ~50/50 split (measured: 38-62% range due to
|
||||
hash variance, stddev ~3% with large vnode counts)
|
||||
- 3 nodes at weights 2:1:1: ~50/25/25 (within 10% tolerance)
|
||||
|
||||
### Lookup
|
||||
|
||||
```python
|
||||
def owner(bucket: int) -> str:
|
||||
"""O(log n) bisect-right walk to find the next virtual node clockwise."""
|
||||
idx = bisect_right(positions, bucket)
|
||||
if idx >= len(positions):
|
||||
idx = 0 # wrap around
|
||||
return vnode_map[positions[idx]]
|
||||
```
|
||||
|
||||
### Stability properties
|
||||
|
||||
The consistent hash ring guarantees:
|
||||
- **Node addition**: adding a node moves at most `1/N` of buckets (where N
|
||||
is the new node count). Other nodes' buckets are unaffected.
|
||||
- **Node removal**: only the removed node's buckets are reassigned. Buckets
|
||||
owned by surviving nodes don't move.
|
||||
- **Determinism**: same membership list always produces the same ring.
|
||||
No coordination needed between processes.
|
||||
|
||||
### Full assignment precomputation
|
||||
|
||||
```python
|
||||
def assignments() -> list[tuple[int, str]]:
|
||||
"""Compute all 65536 bucket-to-node mappings."""
|
||||
return [(b, owner(b)) for b in range(65536)]
|
||||
```
|
||||
|
||||
This produces a complete assignment table that can be loaded into a flat
|
||||
array for O(1) request-time lookup. The ring itself is never consulted
|
||||
on the hot path.
|
||||
|
||||
## Data structures
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RingNode:
|
||||
node_id: str
|
||||
url: str
|
||||
weight: int = 1
|
||||
|
||||
class HashRing:
|
||||
"""Immutable consistent hash ring. Thread-safe (no mutable state)."""
|
||||
|
||||
def __init__(self, nodes: Sequence[RingNode], vnodes_per_unit: int = 150):
|
||||
# Validate no duplicate node_ids
|
||||
# Build sorted array of (position, node_id) tuples
|
||||
# positions[i] = fnv1a_32(f"{node_id}:{i}".encode()) % RING_SIZE
|
||||
|
||||
def owner(self, bucket: int) -> RingNode | None:
|
||||
# bisect_right + wrap
|
||||
|
||||
@property
|
||||
def version(self) -> int:
|
||||
# Deterministic hash of membership: fnv1a_32 of sorted node_id:weight pairs
|
||||
|
||||
def assignments(self) -> list[tuple[int, str]]:
|
||||
# Precompute all 65536 bucket assignments
|
||||
```
|
||||
|
||||
## Comparison with current approach
|
||||
|
||||
| Aspect | Weight-proportional (current) | Consistent hash ring |
|
||||
|--------|------------------------------|---------------------|
|
||||
| Seeding | Exact weight split, deterministic | Hash-based, ~3% variance |
|
||||
| Node addition | Donor/recipient moves only excess | Ring moves ~1/N buckets |
|
||||
| Node removal | Dead buckets → most underloaded | Ring redistributes to clockwise neighbors |
|
||||
| Cross-node churn | Zero (only donor→recipient) | Zero (ring stability guarantee) |
|
||||
| Decentralized | No (needs central rebalancer) | Yes (each node computes locally) |
|
||||
| Complexity | Simple weight arithmetic | Virtual node construction + bisect |
|
||||
|
||||
## Test vectors
|
||||
|
||||
For cross-language implementation validation:
|
||||
|
||||
```json
|
||||
{
|
||||
"fnv1a_32": [
|
||||
{"input": "", "output": 2166136261},
|
||||
{"input": "foobar", "output": 3215766888}
|
||||
],
|
||||
"bucket_of": [
|
||||
{"ws_id": "a3f100000000000000000000000000000", "bucket": 41969},
|
||||
{"ws_id": "00000000000000000000000000000000", "bucket": 0},
|
||||
{"ws_id": "ffff0000000000000000000000000000", "bucket": 65535}
|
||||
],
|
||||
"ring_single_node": {
|
||||
"nodes": [{"node_id": "n1", "weight": 1}],
|
||||
"vnodes_per_unit": 150,
|
||||
"expected_n1_buckets": 65536
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -13,24 +13,22 @@ cloud "LLM Providers" as llm {
|
||||
component [OpenAI-compatible API\n(OpenAI, vLLM, llama.cpp)] as llm_openai
|
||||
component [Anthropic Messages API] as llm_anthropic
|
||||
}
|
||||
database "Redis" as redis
|
||||
database "SQLite\n(.turnstone.db)" as sqlite
|
||||
|
||||
' Turnstone System Boundary
|
||||
package "Turnstone Platform" {
|
||||
component [turnstone\n(CLI)] as cli <<entry point>>
|
||||
component [turnstone-server\n(HTTP + SSE)] as server <<entry point>>
|
||||
component [turnstone-bridge\n(Queue ↔ HTTP)] as bridge <<service>>
|
||||
component [turnstone-console\n(Dashboard)] as console <<service>>
|
||||
component [turnstone-console\n(Dashboard + Router)] as console <<service>>
|
||||
component [turnstone-eval\n(Headless)] as eval <<entry point>>
|
||||
component [turnstone-sim\n(Simulator)] as sim <<service>>
|
||||
component [turnstone-channel\n(Channel Gateway)] as channel <<service>>
|
||||
}
|
||||
|
||||
' User connections
|
||||
cli_user --> cli : stdin / stdout
|
||||
browser_user --> server : HTTP + SSE\n(port 8080)
|
||||
browser_user --> console : HTTP + SSE\n(port 8090)
|
||||
ext_client --> redis : Redis LIST\n(push commands)
|
||||
ext_client --> server : HTTP + SSE\n(SDK / API)
|
||||
eval_user --> eval : Python API
|
||||
|
||||
' Internal connections
|
||||
@@ -43,26 +41,16 @@ server --> sqlite : SQLite
|
||||
eval --> llm : LLM Provider API\n(non-streaming)
|
||||
eval --> sqlite : SQLite
|
||||
|
||||
bridge --> server : HTTP REST\n(POST /v1/api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /v1/api/events)
|
||||
bridge --> redis : Redis LIST + PUBSUB\n+ STRING (routing, heartbeats)
|
||||
console --> server : HTTP proxy\n(hash-ring bucket lookup,\nproxy /node/{id}/* traffic)
|
||||
|
||||
console --> redis : Redis PUBSUB + STRING + LIST\n(cluster events, heartbeats,\nworkstream creation commands)
|
||||
console --> server : HTTP polling + reverse proxy\n(GET /v1/api/dashboard,\nproxy /node/{id}/* traffic)
|
||||
|
||||
sim --> redis : Redis LIST + PUBSUB\n+ STRING (heartbeats)
|
||||
channel --> server : HTTP + SSE\n(POST /v1/api/send,\nGET /v1/api/events)
|
||||
|
||||
' Notes
|
||||
note right of sim
|
||||
Simulator replaces Server+Bridge
|
||||
with lightweight SimNodes that
|
||||
publish to the same Redis channels.
|
||||
end note
|
||||
|
||||
note right of redis
|
||||
Shared message broker:
|
||||
- LIST: command queues
|
||||
- STRING: heartbeats, routing
|
||||
- PUBSUB: event broadcast
|
||||
note right of console
|
||||
Multi-node router:
|
||||
- Hash-ring bucket lookup
|
||||
- Proxies create/send/approve
|
||||
- Direct SSE from client to node
|
||||
- HTTP polling for dashboard
|
||||
end note
|
||||
@enduml
|
||||
|
||||
@@ -6,13 +6,12 @@ title Turnstone — Package & Module Structure
|
||||
skinparam component {
|
||||
BackgroundColor<<entry>> #B8D4E3
|
||||
BackgroundColor<<core>> #C8E6C9
|
||||
BackgroundColor<<mq>> #FFE0B2
|
||||
BackgroundColor<<sim>> #E1BEE7
|
||||
BackgroundColor<<console>> #B2EBF2
|
||||
BackgroundColor<<ui>> #F0F4C3
|
||||
BackgroundColor<<artifact>> #ECEFF1
|
||||
BackgroundColor<<sdk>> #FFCDD2
|
||||
BackgroundColor<<api>> #D1C4E9
|
||||
BackgroundColor<<channel>> #FFE0B2
|
||||
}
|
||||
|
||||
' Entry points
|
||||
@@ -45,23 +44,11 @@ package "turnstone/core/" <<Rectangle>> {
|
||||
component [model_registry.py\nModelRegistry] as registry <<core>>
|
||||
}
|
||||
|
||||
' MQ subsystem
|
||||
package "turnstone/mq/" <<Rectangle>> {
|
||||
component [protocol.py\n28 message types] as protocol <<mq>>
|
||||
component [broker.py\nMessageBroker, RedisBroker] as broker <<mq>>
|
||||
component [bridge.py\nturnstone-bridge] as bridge <<mq>>
|
||||
component [client.py\nTurnstoneClient] as client <<mq>>
|
||||
}
|
||||
|
||||
' Simulator
|
||||
package "turnstone/sim/" <<Rectangle>> {
|
||||
component [cluster.py\nSimCluster] as simcluster <<sim>>
|
||||
component [node.py\nSimNode, SimWorkstream] as simnode <<sim>>
|
||||
component [engine.py\nSimEngine] as simengine <<sim>>
|
||||
component [scenario.py\n5 scenarios] as scenario <<sim>>
|
||||
component [sim/config.py\nSimConfig] as simconfig <<sim>>
|
||||
component [sim/metrics.py\nSim metrics] as simmetrics <<sim>>
|
||||
component [sim/cli.py\nturnstone-sim] as simcli <<sim>>
|
||||
' Channels
|
||||
package "turnstone/channels/" <<Rectangle>> {
|
||||
component [_routing.py\nChannelRouter] as router <<channel>>
|
||||
component [discord/bot.py\nDiscordBot] as discordbot <<channel>>
|
||||
component [gateway.py\nturnstone-channel] as gateway <<channel>>
|
||||
}
|
||||
|
||||
' Console
|
||||
@@ -97,7 +84,7 @@ package "turnstone/sdk/" <<Rectangle>> {
|
||||
|
||||
' Tool schemas
|
||||
package "turnstone/tools/" <<Rectangle>> {
|
||||
component [*.json\n18 tool schemas] as schemas <<artifact>>
|
||||
component [*.json\n19 tool schemas] as schemas <<artifact>>
|
||||
}
|
||||
|
||||
' Entry point dependencies
|
||||
@@ -146,35 +133,17 @@ mcp --> config
|
||||
registry --> config
|
||||
tools --> schemas
|
||||
|
||||
' MQ dependencies
|
||||
bridge --> protocol
|
||||
bridge --> broker
|
||||
bridge --> config
|
||||
client --> protocol
|
||||
client --> broker
|
||||
|
||||
' Sim dependencies
|
||||
simcli --> simcluster
|
||||
simcli --> simconfig
|
||||
simcli --> scenario
|
||||
simcluster --> simnode
|
||||
simcluster --> broker
|
||||
simcluster --> simmetrics
|
||||
simcluster --> simconfig
|
||||
simnode --> simengine
|
||||
simnode --> protocol
|
||||
simnode --> simconfig
|
||||
simnode --> simmetrics
|
||||
scenario --> broker
|
||||
scenario --> protocol
|
||||
scenario --> simconfig
|
||||
scenario --> simmetrics
|
||||
' Channel dependencies
|
||||
gateway --> discordbot
|
||||
gateway --> router
|
||||
discordbot --> sdkserver : HTTP + SSE
|
||||
router --> storage : channel_routes
|
||||
|
||||
' Console dependencies
|
||||
consoleserver --> collector
|
||||
consoleserver --> config
|
||||
consoleserver --> auth
|
||||
collector --> broker
|
||||
collector --> server : HTTP polling
|
||||
|
||||
' API dependencies
|
||||
serverspec --> openapi
|
||||
|
||||
@@ -24,7 +24,7 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
:Dispatch to _prepare_{func_name}();
|
||||
|
||||
note right
|
||||
**Dispatch table (17 tools):**
|
||||
**Dispatch table (19 built-in + tool_search):**
|
||||
┌───────────────┬──────────────────┐
|
||||
│ Tool │ Needs Approval? │
|
||||
├───────────────┼──────────────────┤
|
||||
@@ -33,16 +33,19 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
│ write_file │ ✓ Yes │
|
||||
│ edit_file │ ✓ Yes │
|
||||
│ search │ ✗ Auto-approve │
|
||||
│ diff_file │ ✗ Auto-approve │
|
||||
│ math │ ✗ Auto-approve │
|
||||
│ man │ ✗ Auto-approve │
|
||||
│ web_fetch │ ✗ Auto-approve │
|
||||
│ web_search │ ✗ Auto-approve │
|
||||
│ tool_search │ ✗ Auto-approve │
|
||||
│ task │ ✓ Yes │
|
||||
│ plan │ ✓ Yes │
|
||||
│ task_agent │ ✓ Yes │
|
||||
│ plan_agent │ ✓ Yes │
|
||||
│ memory │ ✗ Auto-approve │
|
||||
│ recall │ ✗ Auto-approve │
|
||||
│ notify │ ✗ Auto-approve │
|
||||
│ watch │ ✓ create only │
|
||||
│ skill │ ✓ load only │
|
||||
│ read_resource │ ✓ Yes │
|
||||
│ use_prompt │ ✓ Yes │
|
||||
├───────────────┼──────────────────┤
|
||||
|
||||
@@ -1,265 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Message Queue Protocol Types
|
||||
skinparam classAttributeIconSize 0
|
||||
skinparam packageStyle rectangle
|
||||
skinparam packageBorderThickness 2
|
||||
|
||||
package "Inbound Messages (Client → Bridge)" as InPkg #FFF3E0 {
|
||||
|
||||
abstract class "InboundMessage" as IM {
|
||||
+ type: str
|
||||
+ correlation_id: str {auto: uuid4().hex[:12]}
|
||||
+ timestamp: float {auto: time.time()}
|
||||
--
|
||||
+ to_json() → str
|
||||
+ {static} from_json(raw) → InboundMessage
|
||||
}
|
||||
|
||||
class SendMessage {
|
||||
type = "send"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ message: str
|
||||
+ auto_approve: bool = False
|
||||
+ auto_approve_tools: list[str] = []
|
||||
+ name: str = ""
|
||||
+ target_node: str = ""
|
||||
}
|
||||
|
||||
class ApproveMessage {
|
||||
type = "approve"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ request_id: str
|
||||
+ approved: bool = True
|
||||
+ feedback: str | None
|
||||
+ always: bool = False
|
||||
}
|
||||
|
||||
class PlanFeedbackMessage {
|
||||
type = "plan_feedback"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ request_id: str
|
||||
+ feedback: str
|
||||
}
|
||||
|
||||
class CommandMessage {
|
||||
type = "command"
|
||||
--
|
||||
+ ws_id: str
|
||||
+ command: str
|
||||
}
|
||||
|
||||
class CreateWorkstreamMessage {
|
||||
type = "create_workstream"
|
||||
--
|
||||
+ name: str = ""
|
||||
+ auto_approve: bool = False
|
||||
+ auto_approve_tools: list[str] = []
|
||||
+ target_node: str = ""
|
||||
+ initial_message: str = ""
|
||||
+ skill: str = ""
|
||||
+ user_id: str = ""
|
||||
}
|
||||
|
||||
class CloseWorkstreamMessage {
|
||||
type = "close_workstream"
|
||||
--
|
||||
+ ws_id: str
|
||||
}
|
||||
|
||||
class ListWorkstreamsMessage {
|
||||
type = "list_workstreams"
|
||||
}
|
||||
|
||||
class HealthMessage {
|
||||
type = "health"
|
||||
}
|
||||
|
||||
class ListNodesMessage {
|
||||
type = "list_nodes"
|
||||
}
|
||||
|
||||
class CancelMessage {
|
||||
type = "cancel"
|
||||
--
|
||||
+ ws_id: str
|
||||
}
|
||||
|
||||
IM <|-- SendMessage
|
||||
IM <|-- ApproveMessage
|
||||
IM <|-- PlanFeedbackMessage
|
||||
IM <|-- CommandMessage
|
||||
IM <|-- CreateWorkstreamMessage
|
||||
IM <|-- CloseWorkstreamMessage
|
||||
IM <|-- ListWorkstreamsMessage
|
||||
IM <|-- HealthMessage
|
||||
IM <|-- ListNodesMessage
|
||||
IM <|-- CancelMessage
|
||||
}
|
||||
|
||||
package "Outbound Events (Bridge → Client)" as OutPkg #E3F2FD {
|
||||
|
||||
abstract class "OutboundEvent" as OE {
|
||||
+ type: str
|
||||
+ ws_id: str
|
||||
+ correlation_id: str
|
||||
+ timestamp: float
|
||||
--
|
||||
+ to_json() → str
|
||||
+ {static} from_json(raw) → OutboundEvent
|
||||
}
|
||||
|
||||
package "Streaming" #BBDEFB {
|
||||
class ContentEvent {
|
||||
type = "content"
|
||||
+ text: str
|
||||
}
|
||||
class ReasoningEvent {
|
||||
type = "reasoning"
|
||||
+ text: str
|
||||
}
|
||||
class StreamEndEvent {
|
||||
type = "stream_end"
|
||||
}
|
||||
}
|
||||
|
||||
package "Tools" #C8E6C9 {
|
||||
class ToolInfoEvent {
|
||||
type = "tool_info"
|
||||
+ items: list
|
||||
}
|
||||
class ApprovalRequestEvent {
|
||||
type = "approval_request"
|
||||
+ items: list
|
||||
..
|
||||
correlation_id = request_id
|
||||
}
|
||||
class ToolOutputChunkEvent {
|
||||
type = "tool_output_chunk"
|
||||
+ call_id: str
|
||||
+ chunk: str
|
||||
}
|
||||
class ToolResultEvent {
|
||||
type = "tool_result"
|
||||
+ call_id: str
|
||||
+ name: str
|
||||
+ output: str
|
||||
+ is_error: bool
|
||||
}
|
||||
class PlanReviewEvent {
|
||||
type = "plan_review"
|
||||
+ content: str
|
||||
}
|
||||
}
|
||||
|
||||
package "Status" #FFF9C4 {
|
||||
class AckEvent {
|
||||
type = "ack"
|
||||
+ status: str
|
||||
+ detail: str
|
||||
}
|
||||
class StatusEvent {
|
||||
type = "status"
|
||||
+ prompt_tokens: int
|
||||
+ completion_tokens: int
|
||||
+ total_tokens: int
|
||||
+ context_window: int
|
||||
+ pct: float
|
||||
+ effort: str
|
||||
+ cache_creation_tokens: int
|
||||
+ cache_read_tokens: int
|
||||
}
|
||||
class StateChangeEvent {
|
||||
type = "state_change"
|
||||
+ state: str
|
||||
}
|
||||
class TurnCompleteEvent {
|
||||
type = "turn_complete"
|
||||
+ content: str
|
||||
}
|
||||
}
|
||||
|
||||
package "Lifecycle" #F8BBD0 {
|
||||
class WorkstreamCreatedEvent {
|
||||
type = "ws_created"
|
||||
+ name: str
|
||||
}
|
||||
class WorkstreamClosedEvent {
|
||||
type = "ws_closed"
|
||||
}
|
||||
class WorkstreamListEvent {
|
||||
type = "ws_list"
|
||||
+ workstreams: list
|
||||
}
|
||||
class WorkstreamRenameEvent {
|
||||
type = "ws_rename"
|
||||
+ name: str
|
||||
}
|
||||
}
|
||||
|
||||
package "System" #E0E0E0 {
|
||||
class HealthResponseEvent {
|
||||
type = "health_response"
|
||||
+ data: dict
|
||||
}
|
||||
class ErrorEvent {
|
||||
type = "error"
|
||||
+ message: str
|
||||
}
|
||||
class InfoEvent {
|
||||
type = "info"
|
||||
+ message: str
|
||||
}
|
||||
class NodeListEvent {
|
||||
type = "node_list"
|
||||
+ nodes: list
|
||||
}
|
||||
class ClusterStateEvent {
|
||||
type = "cluster_state"
|
||||
+ state: str
|
||||
+ node_id: str
|
||||
+ tokens: int
|
||||
+ context_ratio: float
|
||||
+ activity: str
|
||||
+ activity_state: str
|
||||
}
|
||||
}
|
||||
|
||||
OE <|-- ContentEvent
|
||||
OE <|-- ReasoningEvent
|
||||
OE <|-- StreamEndEvent
|
||||
OE <|-- ToolInfoEvent
|
||||
OE <|-- ApprovalRequestEvent
|
||||
OE <|-- ToolResultEvent
|
||||
OE <|-- PlanReviewEvent
|
||||
OE <|-- AckEvent
|
||||
OE <|-- StatusEvent
|
||||
OE <|-- StateChangeEvent
|
||||
OE <|-- TurnCompleteEvent
|
||||
OE <|-- WorkstreamCreatedEvent
|
||||
OE <|-- WorkstreamClosedEvent
|
||||
OE <|-- WorkstreamListEvent
|
||||
OE <|-- WorkstreamRenameEvent
|
||||
OE <|-- HealthResponseEvent
|
||||
OE <|-- ErrorEvent
|
||||
OE <|-- InfoEvent
|
||||
OE <|-- NodeListEvent
|
||||
OE <|-- ClusterStateEvent
|
||||
}
|
||||
|
||||
SendMessage -[hidden]down- OE
|
||||
|
||||
note bottom of IM
|
||||
**Deserialization**: Strict type-dispatch via _INBOUND_REGISTRY.
|
||||
Unknown type raises ValueError.
|
||||
end note
|
||||
|
||||
note bottom of OE
|
||||
**Deserialization**: Lenient type-dispatch via _OUTBOUND_REGISTRY.
|
||||
Unknown type falls back to base OutboundEvent.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -1,105 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Multi-Node Message Routing
|
||||
|
||||
skinparam sequenceArrowThickness 1.5
|
||||
|
||||
participant "TurnstoneClient" as Client
|
||||
collections "Redis" as Redis
|
||||
participant "Bridge-A\n(node_id: nodeA)" as BridgeA
|
||||
participant "Bridge-B\n(node_id: nodeB)" as BridgeB
|
||||
participant "Server-A" as ServerA
|
||||
|
||||
== Scenario A: New Message — No Workstream Affinity ==
|
||||
|
||||
Client -> Redis : RPUSH turnstone:inbound\n{type:"send", message:"...", ws_id:""}
|
||||
note right of Redis : Shared queue — any bridge can pick up
|
||||
|
||||
BridgeA -> Redis : BLPOP [turnstone:inbound:nodeA,\n turnstone:inbound]
|
||||
Redis --> BridgeA : SendMessage (from shared queue)
|
||||
|
||||
BridgeA -> ServerA : POST /v1/api/workstreams/new\n{name:"", auto_approve:false}
|
||||
ServerA --> BridgeA : {ws_id:"abc12345", name:"ws-abc1"}
|
||||
|
||||
BridgeA -> Redis : SET turnstone:ws:abc12345 "nodeA"
|
||||
note right : Register workstream ownership
|
||||
|
||||
BridgeA -> ServerA : GET /v1/api/events?ws_id=abc12345
|
||||
note right : Start per-WS SSE thread
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nWorkstreamCreatedEvent
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:cluster\nClusterStateEvent(ws_id, state:"idle", node_id:"nodeA")
|
||||
|
||||
BridgeA -> ServerA : POST /v1/api/send\n{message:"...", ws_id:"abc12345"}
|
||||
ServerA --> BridgeA : {status:"ok"}
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nAckEvent(status:"ok")
|
||||
|
||||
... SSE events flow: content, tool_output_chunk, tool_result, status, state_change ...
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nContentEvent, ToolResultEvent, ...
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nStateChangeEvent(state:"idle", content:"...")
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nTurnCompleteEvent(content:"...")
|
||||
|
||||
== Scenario B: Directed Message to Specific Node ==
|
||||
|
||||
Client -> Redis : RPUSH turnstone:inbound:nodeB\n{type:"send", target_node:"nodeB", ...}
|
||||
note right : Per-node queue — only nodeB picks up
|
||||
|
||||
BridgeB -> Redis : BLPOP [turnstone:inbound:nodeB,\n turnstone:inbound]
|
||||
Redis --> BridgeB : SendMessage (from per-node queue, priority)
|
||||
|
||||
note right of BridgeB : Process locally on nodeB
|
||||
|
||||
== Scenario C: Re-routing (Lands on Wrong Node) ==
|
||||
|
||||
Client -> Redis : RPUSH turnstone:inbound\n{type:"send", ws_id:"abc12345"}
|
||||
|
||||
BridgeB -> Redis : BLPOP [..., turnstone:inbound]
|
||||
Redis --> BridgeB : SendMessage (ws_id: abc12345)
|
||||
|
||||
BridgeB -> Redis : GET turnstone:ws:abc12345
|
||||
Redis --> BridgeB : "nodeA"
|
||||
|
||||
note right of BridgeB : Owner is nodeA, not me — re-route
|
||||
|
||||
BridgeB -> Redis : RPUSH turnstone:inbound:nodeA\n(re-routed message)
|
||||
|
||||
BridgeA -> Redis : BLPOP [turnstone:inbound:nodeA, ...]
|
||||
Redis --> BridgeA : SendMessage (from per-node queue)
|
||||
note right of BridgeA : Process locally — I own this workstream
|
||||
|
||||
== Scenario D: Approval via Response Queue ==
|
||||
|
||||
BridgeA <- ServerA : SSE: {type:"approve_request", items:[...]}
|
||||
|
||||
note right of BridgeA
|
||||
Bridge checks auto-approve:
|
||||
1. _ws_auto_approve[ws_id]? → auto
|
||||
2. All tools in safe set? → auto
|
||||
(read_file, search, man,
|
||||
memory, recall)
|
||||
3. Otherwise → manual approval
|
||||
end note
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nApprovalRequestEvent(correlation_id: req_xyz)
|
||||
|
||||
Client <- Redis : (subscribed) ApprovalRequestEvent
|
||||
|
||||
Client -> Redis : RPUSH turnstone:resp:req_xyz\nApproveMessage(approved:true)
|
||||
note right : Response queue — bypasses inbound queue
|
||||
|
||||
BridgeA -> Redis : BLPOP turnstone:resp:req_xyz\n(spawned approval thread, timeout 300s)
|
||||
Redis --> BridgeA : ApproveMessage
|
||||
|
||||
BridgeA -> ServerA : POST /v1/api/approve\n{approved:true, ws_id:"abc12345"}
|
||||
|
||||
== Heartbeat (continuous) ==
|
||||
|
||||
BridgeA -> Redis : SET turnstone:node:nodeA\n{server_url, started} EX 60
|
||||
note right : Every 30s — TTL 60s
|
||||
|
||||
BridgeB -> Redis : SET turnstone:node:nodeB\n{server_url, started} EX 60
|
||||
|
||||
@enduml
|
||||
@@ -1,98 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Redis Key Schema
|
||||
|
||||
skinparam component {
|
||||
BackgroundColor<<LIST>> #BBDEFB
|
||||
BackgroundColor<<STRING>> #C8E6C9
|
||||
BackgroundColor<<PUBSUB>> #FFE0B2
|
||||
}
|
||||
|
||||
skinparam note {
|
||||
BackgroundColor #FAFAFA
|
||||
}
|
||||
|
||||
package "Queues (Redis LIST)" #E3F2FD {
|
||||
component [**turnstone:inbound**\n\nShared command queue.\nAny bridge can consume.\n\nOps: RPUSH (write), BLPOP (read)] as inbound <<LIST>>
|
||||
|
||||
component [**turnstone:inbound:{node_id}**\n\nPer-node directed queue.\nPriority over shared queue.\n\nOps: RPUSH (write), BLPOP (read)] as inbound_node <<LIST>>
|
||||
|
||||
component [**turnstone:resp:{request_id}**\n\nPer-request response queue.\nFor approval / plan feedback.\nTTL: 600s\n\nOps: RPUSH + EXPIRE (write), BLPOP (read)] as resp <<LIST>>
|
||||
}
|
||||
|
||||
package "Routing (Redis STRING)" #E8F5E9 {
|
||||
component [**turnstone:ws:{ws_id}**\n\nWorkstream → node ownership.\nValue: node_id string.\nNo TTL.\n\nOps: SET, GET, DEL] as ws_owner <<STRING>>
|
||||
|
||||
component [**turnstone:node:{node_id}**\n\nNode heartbeat + metadata.\nValue: JSON {server_url, started, ...}\nTTL: 60s (refreshed every 30s)\n\nOps: SET with EX, GET, SCAN] as node_hb <<STRING>>
|
||||
}
|
||||
|
||||
package "Event Channels (Redis PUBSUB)" #FFF3E0 {
|
||||
component [**turnstone:events:global**\n\nGlobal event broadcast.\nAll state changes, ws lifecycle.\n\nOps: PUBLISH, SUBSCRIBE] as evt_global <<PUBSUB>>
|
||||
|
||||
component [**turnstone:events:{ws_id}**\n\nPer-workstream events.\nContent, tools, status.\n\nOps: PUBLISH, SUBSCRIBE] as evt_ws <<PUBSUB>>
|
||||
|
||||
component [**turnstone:events:cluster**\n\nCluster-wide state changes.\nUsed by Console dashboard.\n\nOps: PUBLISH, SUBSCRIBE] as evt_cluster <<PUBSUB>>
|
||||
}
|
||||
|
||||
' Readers / Writers
|
||||
|
||||
actor "TurnstoneClient" as client
|
||||
actor "Bridge" as bridge
|
||||
actor "SimNode" as sim
|
||||
actor "Console\nCollector" as console
|
||||
actor "Scenario\n(injector)" as scenario
|
||||
|
||||
' Queue interactions
|
||||
client --> inbound : RPUSH\n(send commands)
|
||||
client --> inbound_node : RPUSH\n(directed)
|
||||
scenario --> inbound : RPUSH\n(inject load)
|
||||
scenario --> inbound_node : RPUSH\n(directed scenario)
|
||||
bridge --> inbound : BLPOP\n(consume)
|
||||
bridge --> inbound_node : BLPOP\n(priority)
|
||||
bridge --> inbound_node : RPUSH\n(re-route)
|
||||
sim --> inbound_node : BLPOP\n(via dispatcher)
|
||||
|
||||
client --> resp : RPUSH\n(approval response)
|
||||
bridge --> resp : BLPOP\n(wait for approval)
|
||||
|
||||
' Routing interactions
|
||||
bridge --> ws_owner : SET / GET / DEL
|
||||
client --> ws_owner : GET\n(route lookup)
|
||||
sim --> ws_owner : SET / DEL
|
||||
|
||||
bridge --> node_hb : SET with EX\n(heartbeat)
|
||||
sim --> node_hb : SET with EX\n(heartbeat)
|
||||
console --> node_hb : SCAN + GET\n(discovery)
|
||||
client --> node_hb : SCAN + GET\n(list_nodes)
|
||||
|
||||
' Pub/sub interactions
|
||||
bridge --> evt_global : PUBLISH
|
||||
bridge --> evt_ws : PUBLISH
|
||||
bridge --> evt_cluster : PUBLISH
|
||||
client --> evt_global : SUBSCRIBE
|
||||
client --> evt_ws : SUBSCRIBE
|
||||
sim --> evt_global : PUBLISH
|
||||
sim --> evt_ws : PUBLISH
|
||||
sim --> evt_cluster : PUBLISH
|
||||
console --> evt_cluster : SUBSCRIBE
|
||||
|
||||
note bottom of inbound
|
||||
**BLPOP priority**: Bridges call
|
||||
BLPOP [per-node, shared] so the
|
||||
per-node queue is always checked first.
|
||||
end note
|
||||
|
||||
note bottom of resp
|
||||
**Bypasses inbound queue**: Approval
|
||||
responses go directly to the response
|
||||
queue, not through the inbound queue.
|
||||
Auto-cleaned after 600s TTL.
|
||||
end note
|
||||
|
||||
note bottom of evt_cluster
|
||||
**ClusterStateEvent** includes node_id,
|
||||
tokens, and context_ratio — enriched
|
||||
data not available on the global channel.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -64,7 +64,7 @@ note right of thinking
|
||||
|
||||
**Propagation:**
|
||||
• WebUI → global SSE queue (ws_state)
|
||||
• Bridge → PUBLISH to global + cluster channels
|
||||
• Console → HTTP polling picks up state
|
||||
• CLI → WorkstreamManager.set_state()
|
||||
end note
|
||||
|
||||
@@ -72,27 +72,8 @@ note left of attention
|
||||
**Blocking mechanisms:**
|
||||
• TerminalUI: input() prompt
|
||||
• WebUI: threading.Event.wait()
|
||||
• Bridge: BLPOP on response queue
|
||||
• ChannelBot: SSE event + Discord button
|
||||
• NullUI: auto-approve (never reaches)
|
||||
end note
|
||||
|
||||
state "SimWorkstream (simplified)" as sim_group {
|
||||
state "sim_idle" as si <<idle>>
|
||||
state "sim_thinking" as st <<thinking>>
|
||||
state "sim_running" as sr <<running>>
|
||||
state "sim_error" as se <<error>>
|
||||
|
||||
[*] --> si
|
||||
si --> st : process_turn() called
|
||||
st --> sr : Tool calls generated
|
||||
sr --> st : More rounds
|
||||
st --> si : No tools / max rounds
|
||||
st --> se : Uncaught exception
|
||||
}
|
||||
|
||||
note right of sim_group
|
||||
SimWorkstream has no ATTENTION state —
|
||||
tool approval is not simulated.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Simulator Architecture
|
||||
|
||||
skinparam component {
|
||||
BackgroundColor<<cluster>> #E1BEE7
|
||||
BackgroundColor<<node>> #CE93D8
|
||||
BackgroundColor<<engine>> #F3E5F5
|
||||
BackgroundColor<<scenario>> #FFF3E0
|
||||
BackgroundColor<<metrics>> #E8F5E9
|
||||
BackgroundColor<<redis>> #FFCDD2
|
||||
}
|
||||
|
||||
package "SimCluster" as cluster <<cluster>> {
|
||||
|
||||
component [**ThreadPoolExecutor**\nmax_workers=64\n(blocking Redis ops)] as executor <<cluster>>
|
||||
|
||||
component [**redis.ConnectionPool**\nmax_connections=64\ndecode_responses=True\n(shared across all nodes)] as pool <<redis>>
|
||||
|
||||
package "InboundDispatchers" {
|
||||
component [**Dispatcher 0**\nnodes 0-49] as d0
|
||||
component [**Dispatcher 1**\nnodes 50-99] as d1
|
||||
component [**...**\n(ceil(N/50) total)] as dn
|
||||
|
||||
note bottom of d0
|
||||
Each dispatcher calls BLPOP on a single Redis
|
||||
connection for up to 50 node queues + shared queue.
|
||||
Keys: [prefix:inbound:sim-0000, ..., prefix:inbound]
|
||||
Per-node keys have BLPOP priority over shared.
|
||||
end note
|
||||
}
|
||||
|
||||
package "SimNodes (N instances)" {
|
||||
component [**SimNode sim-0000**] as n0 <<node>>
|
||||
component [**SimNode sim-0001**] as n1 <<node>>
|
||||
component [**...**] as nn <<node>>
|
||||
|
||||
component [**SimEngine**\n(per node, seeded RNG)\n\nLLM simulation:\n gaussian(μ=2s, σ=0.5s) latency\n gaussian(μ=200, σ=50) tokens\n random word content\n P(tool_calls) = 0.6/0.3\n\nTool simulation:\n gaussian(μ=0.5s, σ=0.2s) latency\n P(failure) = 0.02] as engine <<engine>>
|
||||
|
||||
component [**SimWorkstream**\n(0..max_ws per node)\n\nState: idle→thinking→running→idle\nToken accounting: word_count × 3\nContent: 8-chunk streaming] as ws <<node>>
|
||||
}
|
||||
|
||||
component [**MetricsCollector**\n(thread-safe, shared)\n\nTracks: turn latencies,\nthroughput, utilization,\nerrors, node kills] as metrics <<metrics>>
|
||||
}
|
||||
|
||||
package "Scenarios (5 workload patterns)" <<scenario>> {
|
||||
component [**SteadyState**\nConstant rate:\n1/mps interval\nfor duration secs] as steady <<scenario>>
|
||||
|
||||
component [**Burst**\nburst_size messages\nas fast as possible\nthen wait] as burst <<scenario>>
|
||||
|
||||
component [**NodeFailure**\nSteadyState + periodic\nnode kills (up to N/2)] as failure <<scenario>>
|
||||
|
||||
component [**Directed**\nMessages targeted to\nspecific nodes via\ntarget_node field] as directed <<scenario>>
|
||||
|
||||
component [**Lifecycle**\n3 phases:\n1. Create workstreams\n2. Send messages\n3. Close half] as lifecycle <<scenario>>
|
||||
}
|
||||
|
||||
database "Redis" as redis <<redis>>
|
||||
|
||||
' Scenario -> Redis
|
||||
steady --> redis : RPUSH prefix:inbound\n(SendMessage)
|
||||
burst --> redis : RPUSH prefix:inbound\n(burst)
|
||||
failure --> redis : RPUSH prefix:inbound
|
||||
directed --> redis : RPUSH prefix:inbound:{node}\n(directed)
|
||||
lifecycle --> redis : RPUSH prefix:inbound\n(Create/Send/Close)
|
||||
|
||||
' Dispatchers -> Redis -> Nodes
|
||||
d0 --> redis : BLPOP [per-node..., shared]
|
||||
d1 --> redis : BLPOP [per-node..., shared]
|
||||
d0 --> n0 : handle_message(raw)
|
||||
d0 --> n1 : handle_message(raw)
|
||||
|
||||
' Nodes internal
|
||||
n0 --> engine : simulate_llm_response()\nsimulate_tool_execution()
|
||||
n0 --> ws : process_turn()
|
||||
|
||||
' Nodes -> Redis (events)
|
||||
n0 --> redis : PUBLISH prefix:events:global\n(StateChangeEvent)
|
||||
n0 --> redis : PUBLISH prefix:events:{ws_id}\n(ContentEvent, ToolResultEvent, ...)
|
||||
n0 --> redis : PUBLISH prefix:events:cluster\n(ClusterStateEvent)
|
||||
n0 --> redis : SET prefix:node:sim-0000\nEX 60 (heartbeat)
|
||||
n0 --> redis : SET prefix:ws:{ws_id}\n(ownership)
|
||||
|
||||
' Shared pool
|
||||
n0 ..> pool : PooledBroker\n(shared connection)
|
||||
n1 ..> pool : PooledBroker
|
||||
d0 ..> pool
|
||||
d0 ..> executor : asyncio.to_thread()
|
||||
|
||||
' Metrics
|
||||
ws --> metrics : record_turn(ws_id, node_id, latency)
|
||||
steady --> metrics : record_inject()
|
||||
burst --> metrics : record_inject()
|
||||
directed --> metrics : record_inject()
|
||||
lifecycle --> metrics : record_inject()
|
||||
cluster --> metrics : record_node_kill(node_id)
|
||||
cluster --> metrics : snapshot_utilization()\n(every metrics_interval)
|
||||
|
||||
note bottom of cluster
|
||||
**SimConfig** controls all simulation parameters:
|
||||
num_nodes, max_ws_per_node, redis settings,
|
||||
llm_latency_mean/stddev, tool_failure_rate,
|
||||
scenario, duration, messages_per_second, seed
|
||||
end note
|
||||
|
||||
note right of redis
|
||||
Simulator uses **real Redis** —
|
||||
not a mock. Console dashboard
|
||||
can monitor a running simulation
|
||||
via the same cluster channel.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -7,110 +7,79 @@ skinparam sequenceArrowThickness 1.5
|
||||
participant "Browser" as Browser
|
||||
participant "Console\nStarlette App" as Server
|
||||
participant "ClusterCollector" as CC
|
||||
collections "Redis" as Redis
|
||||
participant "Node-A Bridge" as BridgeA
|
||||
participant "Node-A\n(real server)" as NodeA
|
||||
participant "Node-B\n(sim node)" as NodeB
|
||||
participant "Node-A\n(server)" as NodeA
|
||||
participant "Node-B\n(server)" as NodeB
|
||||
|
||||
== Thread 1: Cluster Event Subscriber (real-time) ==
|
||||
== Thread 1: Node Discovery (every 60s) ==
|
||||
|
||||
CC -> Redis : SUBSCRIBE turnstone:events:cluster
|
||||
activate CC #E1BEE7
|
||||
|
||||
Redis --> CC : ClusterStateEvent\n{ws_id, state:"thinking",\nnode_id:"nodeA", tokens:500,\ncontext_ratio:0.05}
|
||||
CC -> CC : Update NodeSnapshot["nodeA"]\n.workstreams["ws123"].state = "thinking"
|
||||
CC -> CC : _fanout(event) → all SSE listeners
|
||||
|
||||
Redis --> CC : {"type":"ws_created",\nws_id:"ws456", name:"task-1",\nnode_id:"sim-0003"}
|
||||
CC -> CC : Add workstream to\nNodeSnapshot["sim-0003"]
|
||||
CC -> CC : _fanout(event)
|
||||
|
||||
Redis --> CC : ClusterStateEvent\n{ws_id:"ws456", state:"idle"}
|
||||
CC -> CC : Update workstream state
|
||||
|
||||
note right of CC
|
||||
Handles: cluster_state,
|
||||
ws_created, ws_closed, ws_rename
|
||||
|
||||
Thread runs continuously.
|
||||
All updates are thread-safe
|
||||
via threading.Lock.
|
||||
end note
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 2: Node Discovery (every 15s) ==
|
||||
|
||||
CC -> Redis : SCAN 0 MATCH turnstone:node:*
|
||||
activate CC #B2EBF2
|
||||
Redis --> CC : [turnstone:node:nodeA, turnstone:node:sim-0003, ...]
|
||||
|
||||
loop for each discovered key
|
||||
CC -> Redis : GET turnstone:node:{id}
|
||||
Redis --> CC : JSON: {server_url, started, max_ws, sim:true/false}
|
||||
end
|
||||
|
||||
CC -> CC : Create new NodeSnapshot\nfor newly discovered nodes
|
||||
CC -> CC : Remove NodeSnapshot\nfor disappeared nodes
|
||||
|
||||
CC -> CC : _fanout({type: "node_joined", ...})\n_fanout({type: "node_lost", ...})
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 3: HTTP Polling (every 10s, real nodes only) ==
|
||||
|
||||
CC -> CC : Filter nodes where\nserver_url.startswith("http")
|
||||
CC -> CC : list_services("server",\nmax_age_seconds=120)
|
||||
activate CC #C8E6C9
|
||||
|
||||
note right of CC
|
||||
sim:// nodes are SKIPPED.
|
||||
Their data comes exclusively
|
||||
from the cluster event channel.
|
||||
end note
|
||||
|
||||
CC -> NodeA : GET /v1/api/dashboard
|
||||
activate NodeA
|
||||
NodeA --> CC : {workstreams: [...],\naggregate: {total_tokens, ...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> NodeA : GET /health
|
||||
activate NodeA
|
||||
NodeA --> CC : {status:"ok", version:"0.3.0",\nmodel:"...", workstreams:{...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> CC : Diff old vs new workstream IDs
|
||||
CC -> CC : Replace NodeSnapshot["nodeA"]\n.workstreams, .health, .aggregate
|
||||
CC -> CC : _fanout(ws_created) for\nnewly appeared workstreams
|
||||
CC -> CC : _fanout(ws_closed) for\nremoved workstreams
|
||||
|
||||
note right of CC
|
||||
Poll-diff fanout ensures
|
||||
browser SSE clients learn
|
||||
about workstreams that
|
||||
appeared without a real-time
|
||||
cluster event (e.g. bridge
|
||||
startup recovery).
|
||||
end note
|
||||
|
||||
CC -x NodeB : (SKIPPED: sim:// URL)
|
||||
CC -> CC : New node? → spawn SSE task\nLost node? → cancel SSE task
|
||||
CC -> CC : _fanout(node_joined)\n_fanout(node_lost)
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 2: SSE Manager (asyncio event loop) ==
|
||||
|
||||
note over CC
|
||||
Single asyncio event loop multiplexes
|
||||
one persistent SSE connection per node.
|
||||
Scales to 1000+ nodes.
|
||||
end note
|
||||
|
||||
CC -> NodeA : GET /v1/api/events/global\n?expected_node_id=nodeA
|
||||
activate NodeA
|
||||
activate CC #BBDEFB
|
||||
|
||||
NodeA --> CC : data: {"type":"node_snapshot",\n"node_id":"nodeA",\n"workstreams":[...],\n"health":{...},\n"aggregate":{...}}
|
||||
|
||||
note right of CC
|
||||
Snapshot populates NodeSnapshot
|
||||
in-memory state. Reconciles
|
||||
against stale data (emits
|
||||
ws_created/ws_closed diffs).
|
||||
end note
|
||||
|
||||
loop real-time delta events
|
||||
NodeA --> CC : data: {"type":"ws_state",\n"ws_id":"ws1","state":"running"}
|
||||
CC -> CC : Update NodeSnapshot\n_fanout(cluster_state)
|
||||
end
|
||||
|
||||
alt health transition
|
||||
NodeA --> CC : data: {"type":"health_changed",\n"circuit_state":"open"}
|
||||
CC -> CC : Update node.health
|
||||
end
|
||||
|
||||
alt periodic aggregate (every 10s)
|
||||
NodeA --> CC : data: {"type":"aggregate",\n"total_tokens":50000}
|
||||
CC -> CC : Update node.aggregate
|
||||
end
|
||||
|
||||
deactivate CC
|
||||
deactivate NodeA
|
||||
|
||||
alt SSE disconnect
|
||||
CC -> CC : Mark node unreachable\nReconnect with backoff\n(1s → 30s cap)
|
||||
end
|
||||
|
||||
alt identity mismatch (409 or snapshot node_id differs)
|
||||
CC -> CC : Mark node unreachable\nStop reconnecting to this URL
|
||||
end
|
||||
|
||||
== Browser SSE Stream ==
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/events
|
||||
activate Server
|
||||
|
||||
Server -> CC : get_snapshot()
|
||||
Server -> CC : get_snapshot_and_register(queue)
|
||||
note right : Atomic: snapshot + listener\nregistration under both locks\n→ no event gap
|
||||
CC --> Server : ClusterSnapshot\n(full current state)
|
||||
|
||||
Server -> CC : register_listener(queue)
|
||||
note right : Per-client queue.Queue(maxsize=2000)\nSSE via EventSourceResponse + run_in_executor()
|
||||
|
||||
Server -> Browser : data: {"type":"snapshot",...}\n(full state as first SSE event)
|
||||
|
||||
loop continuous (incremental updates)
|
||||
CC -> Server : event via listener queue\n(from any of the 3 threads)
|
||||
CC -> Server : event via listener queue\n(from SSE manager thread)
|
||||
Server -> Browser : data: {"type":"cluster_state",...}\n\n
|
||||
end
|
||||
|
||||
@@ -133,20 +102,20 @@ Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/overview
|
||||
Server -> CC : get_overview()
|
||||
CC --> Server : {nodes: 10, workstreams: 47,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.3.0"]}
|
||||
CC --> Server : {nodes: 2, workstreams: 12,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.9.7"]}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/nodes?sort=activity
|
||||
Server -> CC : get_nodes(sort_by="activity")
|
||||
CC --> Server : {nodes: [...], total: 10}
|
||||
CC --> Server : {nodes: [...], total: 2}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/workstreams\n?state=running&node=sim-0003
|
||||
Server -> CC : get_workstreams(state="running",\nnode="sim-0003")
|
||||
Browser -> Server : GET /v1/api/cluster/workstreams\n?state=running&node=nodeA
|
||||
Server -> CC : get_workstreams(state="running",\nnode="nodeA")
|
||||
CC --> Server : {workstreams: [...], total: 5,\npage: 1, per_page: 50, pages: 1}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
== Workstream Creation (via MQ) ==
|
||||
== Workstream Creation (via Console proxy) ==
|
||||
|
||||
Browser -> Server : POST /v1/api/cluster/workstreams/new\n{node_id:"nodeA", name:"new-task"}
|
||||
activate Server #FFECB3
|
||||
@@ -154,34 +123,22 @@ activate Server #FFECB3
|
||||
Server -> CC : _pick_best_node() or\nget_node_detail(node_id)
|
||||
CC --> Server : node validated
|
||||
|
||||
Server -> Server : Build CreateWorkstreamMessage\n{target_node:"nodeA", name:"new-task",\nuser_id: from auth_result}
|
||||
Server -> NodeA : POST http://nodeA:8080/v1/api/workstreams/new\n{name:"new-task", user_id: from auth_result}
|
||||
activate NodeA
|
||||
NodeA --> Server : {ws_id:"ws789", name:"new-task",\nnode_url:"http://nodeA:8080"}
|
||||
deactivate NodeA
|
||||
|
||||
Server -> Redis : RPUSH turnstone:inbound:nodeA\n(directed queue)
|
||||
Server --> Browser : {status:"ok", correlation_id:"abc",\ntarget_node:"nodeA"}
|
||||
Server --> Browser : {status:"ok", ws_id:"ws789",\nnode_url:"http://nodeA:8080"}
|
||||
deactivate Server
|
||||
|
||||
note right of Redis
|
||||
Bridge on Node-A picks up the
|
||||
message from its directed queue,
|
||||
POSTs to /v1/api/workstreams/new
|
||||
(forwarding user_id in payload),
|
||||
registers ownership, publishes
|
||||
ws_created to cluster channel.
|
||||
note right of Server
|
||||
Console proxies the create request
|
||||
directly to the target node via HTTP.
|
||||
The response includes node_url so the
|
||||
client can establish a direct SSE
|
||||
connection for the data plane.
|
||||
end note
|
||||
|
||||
Redis --> BridgeA : BLPOP turnstone:inbound:nodeA
|
||||
activate BridgeA
|
||||
BridgeA -> NodeA : POST /v1/api/workstreams/new\n{name:"new-task"}
|
||||
NodeA --> BridgeA : {ws_id:"ws789", name:"new-task"}
|
||||
BridgeA -> Redis : SET turnstone:ws:ws789 = nodeA
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:cluster\n{type:"ws_created", ws_id:"ws789",\nnode_id:"nodeA", name:"new-task"}
|
||||
deactivate BridgeA
|
||||
|
||||
Redis --> CC : ws_created event
|
||||
CC -> CC : Add workstream to\nNodeSnapshot["nodeA"]
|
||||
CC -> CC : _fanout(event)
|
||||
Server -> Browser : SSE: data: {"type":"ws_created",...}
|
||||
|
||||
== Reverse Proxy (server UI through console port) ==
|
||||
|
||||
Browser -> Server : GET /node/nodeA/
|
||||
|
||||
@@ -14,45 +14,25 @@ node "Docker Host" as host {
|
||||
|
||||
frame "turnstone-net (bridge network)" as net {
|
||||
|
||||
node "redis" <<redis:7.4-alpine>> as redis_node {
|
||||
component [Redis Server\nport 6379] as redis
|
||||
note bottom of redis
|
||||
Healthcheck: redis-cli ping
|
||||
Volume: redis-data
|
||||
end note
|
||||
}
|
||||
|
||||
node "server" <<turnstone image>> as server_node {
|
||||
component [turnstone-server\nport 8080] as server
|
||||
note bottom of server
|
||||
Command: turnstone-server
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
Depends: redis (healthy)
|
||||
Volume: turnstone-data
|
||||
(/data)
|
||||
end note
|
||||
}
|
||||
|
||||
node "bridge ×N" <<turnstone image>> as bridge_node {
|
||||
component [turnstone-bridge] as bridge
|
||||
note bottom of bridge
|
||||
Command: turnstone-bridge
|
||||
--server-url http://server:8080
|
||||
--redis-host redis
|
||||
Depends: server + redis
|
||||
Scalable: --scale bridge=N
|
||||
node_id: auto from hostname
|
||||
end note
|
||||
}
|
||||
|
||||
node "console" <<turnstone image>> as console_node {
|
||||
component [turnstone-console\nport 8090] as console
|
||||
note bottom of console
|
||||
Command: turnstone-console
|
||||
--redis-host redis
|
||||
--port 8090
|
||||
Depends: redis
|
||||
Depends: server
|
||||
Hash-ring router for
|
||||
multi-node clusters
|
||||
end note
|
||||
}
|
||||
|
||||
@@ -75,42 +55,21 @@ node "Docker Host" as host {
|
||||
See docs/pgbouncer.md
|
||||
end note
|
||||
}
|
||||
|
||||
node "sim (profile: sim)" <<turnstone image>> as sim_node {
|
||||
component [turnstone-sim] as sim
|
||||
note bottom of sim
|
||||
Command: turnstone-sim
|
||||
--redis-host redis
|
||||
--nodes 100
|
||||
--scenario steady
|
||||
Depends: redis
|
||||
Optional: only with
|
||||
--profile sim
|
||||
end note
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
actor "Browser\nUser" as browser
|
||||
actor "MQ Client" as mqclient
|
||||
actor "SDK /\nAPI Client" as apiclient
|
||||
|
||||
' External connections
|
||||
browser --> server : HTTP + SSE\nport 8080
|
||||
browser --> console : HTTP + SSE\nport 8090
|
||||
mqclient --> redis : Redis protocol\nport 6379
|
||||
apiclient --> server : HTTP + SSE\nport 8080
|
||||
|
||||
' Internal connections
|
||||
server --> redis : Redis protocol\n(6379)
|
||||
server --> llm_api : OpenAI API\n(HTTPS/HTTP)
|
||||
|
||||
bridge --> server : HTTP REST\n(POST /v1/api/send, etc.)
|
||||
bridge <-- server : SSE\n(GET /v1/api/events)
|
||||
bridge --> redis : Redis protocol\n(queues + pubsub)
|
||||
|
||||
console --> redis : Redis PUBSUB + LIST\n(cluster events,\nws creation commands)
|
||||
console --> server : HTTP polling + proxy\n(GET /v1/api/dashboard,\nproxy /node/{id}/*)
|
||||
|
||||
sim --> redis : Redis protocol\n(queues + pubsub + keys)
|
||||
console --> server : HTTP proxy\n(hash-ring lookup,\nproxy /node/{id}/*)
|
||||
|
||||
' Database connections (production/cluster profiles)
|
||||
server ..> pgbouncer : PostgreSQL\n(pool_size=2)
|
||||
@@ -120,20 +79,17 @@ pgbouncer --> postgres : transaction\npooling
|
||||
' Environment variables
|
||||
note right of host
|
||||
**Environment Variables:**
|
||||
• LLM_BASE_URL — LLM endpoint
|
||||
• OPENAI_API_KEY — API key
|
||||
• REDIS_PASSWORD — Redis auth
|
||||
• TURNSTONE_AUTH_TOKEN — API auth
|
||||
• TURNSTONE_DB_URL — PostgreSQL URL
|
||||
• POSTGRES_PASSWORD — DB password
|
||||
* LLM_BASE_URL -- LLM endpoint
|
||||
* OPENAI_API_KEY -- API key
|
||||
* TURNSTONE_AUTH_TOKEN -- API auth
|
||||
* TURNSTONE_DB_URL -- PostgreSQL URL
|
||||
* POSTGRES_PASSWORD -- DB password
|
||||
end note
|
||||
|
||||
' Volumes
|
||||
database "redis-data" as rv
|
||||
database "turnstone-data" as tv
|
||||
database "postgres-data" as pv
|
||||
|
||||
redis_node --> rv
|
||||
server_node --> tv
|
||||
pg_node --> pv
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@ title Turnstone — Channel Integration Architecture
|
||||
skinparam class {
|
||||
BackgroundColor<<platform>> #E1BEE7
|
||||
BackgroundColor<<service>> #E8EAF6
|
||||
BackgroundColor<<mq>> #FFCDD2
|
||||
BackgroundColor<<bridge>> #C8E6C9
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
}
|
||||
@@ -63,53 +61,23 @@ class "DiscordBot" as Bot <<service>> {
|
||||
Renders approval buttons
|
||||
escape_mentions() on send
|
||||
--
|
||||
_notify_ws_map: msg_id → (ws_id, user_id)
|
||||
_notify_reply_channels: ws_id → (dm, user_id)
|
||||
_notify_ws_map: msg_id -> (ws_id, user_id)
|
||||
_notify_reply_channels: ws_id -> (dm, user_id)
|
||||
}
|
||||
|
||||
class "ChannelRouter" as Router <<service>> {
|
||||
+resolve_route(platform, channel_id)
|
||||
→ ws_id | None
|
||||
-> ws_id | None
|
||||
+register_route(channel_id, ws_id)
|
||||
+resolve_identity(platform, platform_user_id)
|
||||
→ user_id | None
|
||||
-> user_id | None
|
||||
--
|
||||
Maps channels → workstreams
|
||||
Maps platform users → turnstone users
|
||||
Maps channels -> workstreams
|
||||
Maps platform users -> turnstone users
|
||||
Caches routes in memory
|
||||
}
|
||||
|
||||
class "AsyncRedisBroker" as Broker <<service>> {
|
||||
+push_inbound(msg)
|
||||
+subscribe(ws_id) → AsyncIterator
|
||||
+subscribe_global() → AsyncIterator
|
||||
+push_response(correlation_id, msg)
|
||||
--
|
||||
redis.asyncio client
|
||||
Pub/sub + queue operations
|
||||
}
|
||||
|
||||
' -- Redis MQ --
|
||||
class "Redis MQ" as Redis <<mq>> {
|
||||
turnstone:inbound (LIST)
|
||||
turnstone:events:{ws_id} (PUBSUB)
|
||||
turnstone:events:global (PUBSUB)
|
||||
turnstone:resp:{corr_id} (LIST)
|
||||
--
|
||||
Shared message bus
|
||||
Same queues as bridge protocol
|
||||
}
|
||||
|
||||
' -- Bridge + Server --
|
||||
class "turnstone-bridge" as Bridge <<bridge>> {
|
||||
BLPOP turnstone:inbound
|
||||
Drive server via HTTP
|
||||
Relay SSE → Redis pub/sub
|
||||
--
|
||||
Owns workstream lifecycle
|
||||
Auto-approve / manual approve
|
||||
}
|
||||
|
||||
' -- Server --
|
||||
class "turnstone-server" as Server <<server>> {
|
||||
POST /v1/api/send
|
||||
POST /v1/api/approve
|
||||
@@ -128,7 +96,7 @@ class "channel_users" as CU <<storage>> {
|
||||
channel_user_id (PK)
|
||||
platform: "discord" | "slack"
|
||||
platform_user_id
|
||||
user_id → users
|
||||
user_id -> users
|
||||
linked_at
|
||||
--
|
||||
/link command creates row
|
||||
@@ -161,18 +129,13 @@ class "services" as SVC <<storage>> {
|
||||
' -- Relationships --
|
||||
Discord --> Bot : gateway\nevents
|
||||
Bot --> Router : on_message\non_interaction
|
||||
Router --> Broker : SendMessage\nApproveMessage
|
||||
Router --> CU : resolve identity
|
||||
Router --> CR : resolve / register route
|
||||
Broker --> Redis : RPUSH inbound\nRPUSH resp:{id}
|
||||
|
||||
Redis --> Bridge : BLPOP inbound
|
||||
Bridge --> Server : HTTP API
|
||||
Server --> Bridge : SSE events
|
||||
Bridge --> Redis : PUBLISH events:{ws_id}\nPUBLISH events:global
|
||||
Router --> Server : POST /v1/api/send\nPOST /v1/api/approve\nPOST /v1/api/workstreams/new
|
||||
Bot --> Server : GET /v1/api/events?ws_id=\n(SSE via httpx-sse)
|
||||
Server --> Bot : SSE event stream
|
||||
|
||||
Redis --> Broker : SUBSCRIBE events:{ws_id}
|
||||
Broker --> Bot : event stream
|
||||
Bot --> Discord : reply / embed\nbutton callback
|
||||
|
||||
Slack .[hidden]. Discord
|
||||
@@ -180,10 +143,9 @@ Teams .[hidden]. Slack
|
||||
|
||||
ChannelService --> Bot : creates + runs
|
||||
ChannelService --> Router : creates
|
||||
ChannelService --> Broker : creates
|
||||
ChannelService --> SVC : register / heartbeat /\nderegister
|
||||
|
||||
' -- Notification path (direct HTTP, bypasses MQ) --
|
||||
' -- Notification path (direct HTTP) --
|
||||
Server --> ChannelService : POST /v1/api/notify\n(JWT: aud=turnstone-channel)
|
||||
Server --> SVC : list_services("channel",\nmax_age_seconds=120)
|
||||
|
||||
@@ -192,38 +154,36 @@ note right of Bot
|
||||
**Inbound Flow**
|
||||
1. Discord message arrives via gateway
|
||||
2. Bot.on_message() fires
|
||||
3. ChannelRouter resolves channel → ws_id
|
||||
3. ChannelRouter resolves channel -> ws_id
|
||||
(or creates new workstream)
|
||||
4. ChannelRouter resolves platform user → user_id
|
||||
4. ChannelRouter resolves platform user -> user_id
|
||||
via channel_users table
|
||||
5. Broker.push_inbound(SendMessage)
|
||||
6. Bridge pops from Redis, drives server
|
||||
5. Router sends POST /v1/api/send to server
|
||||
|
||||
**Workstream Resume (evicted workstreams)**
|
||||
1. Stale route detected (no MQ owner)
|
||||
1. Stale route detected (no active SSE listener)
|
||||
2. Existing ws_id reused directly from route
|
||||
3. CreateWorkstreamMessage sent with
|
||||
3. POST /v1/api/workstreams/new with
|
||||
resume_ws=<ws_id>
|
||||
4. Server resumes atomically during creation
|
||||
5. Bridge emits WorkstreamResumedEvent → thread
|
||||
5. SSE emits WorkstreamResumedEvent -> thread
|
||||
end note
|
||||
|
||||
note right of Broker
|
||||
note right of Server
|
||||
**Outbound Flow**
|
||||
1. Server emits SSE events
|
||||
2. Bridge relays to Redis events:{ws_id}
|
||||
3. Broker.subscribe(ws_id) yields events
|
||||
4. Bot formats and sends to Discord thread
|
||||
1. Server emits SSE events on
|
||||
GET /v1/api/events?ws_id=
|
||||
2. Bot subscribes via httpx-sse
|
||||
3. Bot formats and sends to Discord thread
|
||||
end note
|
||||
|
||||
note bottom of CR
|
||||
**Approval Flow**
|
||||
1. ApprovalRequestEvent arrives via events:{ws_id}
|
||||
1. ApprovalRequestEvent arrives via SSE
|
||||
2. Bot renders Discord buttons (Approve / Deny)
|
||||
3. User clicks button → on_interaction()
|
||||
3. User clicks button -> on_interaction()
|
||||
4. Router builds ApproveMessage
|
||||
5. Broker.push_response(correlation_id, msg)
|
||||
6. Bridge pops from resp:{id}, calls POST /api/approve
|
||||
5. Router sends POST /v1/api/approve to server
|
||||
end note
|
||||
|
||||
note bottom of CU
|
||||
@@ -238,17 +198,17 @@ note bottom of CU
|
||||
end note
|
||||
|
||||
note bottom of SVC
|
||||
**Notification Flow** (direct HTTP, bypasses MQ)
|
||||
1. LLM calls notify tool → _prepare_notify()
|
||||
**Notification Flow** (direct HTTP)
|
||||
1. LLM calls notify tool -> _prepare_notify()
|
||||
2. _exec_notify() checks rate limit (5/turn)
|
||||
3. Queries services table for healthy gateways
|
||||
4. Mints JWT (aud: turnstone-channel) via
|
||||
ServiceTokenManager
|
||||
5. POSTs to first healthy gateway (incl. ws_id)
|
||||
6. Gateway validates JWT, resolves target
|
||||
7. adapter.send_notification() → Discord API
|
||||
(tracks msg_id → ws_id for reply routing)
|
||||
8. On failure: retry up to 3× (1s, 3s backoff)
|
||||
7. adapter.send_notification() -> Discord API
|
||||
(tracks msg_id -> ws_id for reply routing)
|
||||
8. On failure: retry up to 3x (1s, 3s backoff)
|
||||
9. SSRF: only http(s) URLs allowed
|
||||
|
||||
**Bidirectional DM Replies**
|
||||
|
||||
@@ -61,7 +61,7 @@ end
|
||||
|
||||
Session -> Session : _init_system_messages()\nevery conversation turn
|
||||
|
||||
Session -> Session : _get_visible_memories(\nlimit=fetch_limit)
|
||||
Session -> Session : _list_visible_memories(\nlimit=fetch_limit)
|
||||
note right
|
||||
**Scope resolution:**
|
||||
1. global scope (always)
|
||||
|
||||
@@ -144,7 +144,7 @@ note over Server, Registry
|
||||
**CLI entry point:**
|
||||
CLI flag > config.toml > argparse default
|
||||
|
||||
**Bootstrap settings** (database, Redis, auth, server bind):
|
||||
**Bootstrap settings** (database, auth, server bind):
|
||||
Always from config.toml / env vars — never in ConfigStore.
|
||||
end note
|
||||
|
||||
|
||||
@@ -34,253 +34,214 @@
|
||||
<text x="600" y="54" text-anchor="middle" fill="#8b949e" font-size="11" letter-spacing="1">SYSTEM ARCHITECTURE</text>
|
||||
|
||||
<!-- ==================== COLUMN HEADERS ==================== -->
|
||||
<text x="90" y="86" text-anchor="middle" fill="#58a6ff" font-size="9" font-weight="600" letter-spacing="2">CLIENTS</text>
|
||||
<text x="276" y="86" text-anchor="middle" fill="#3fb950" font-size="9" font-weight="600" letter-spacing="2">GATEWAYS</text>
|
||||
<text x="480" y="86" text-anchor="middle" fill="#f0883e" font-size="9" font-weight="600" letter-spacing="2">MESSAGE QUEUE</text>
|
||||
<text x="700" y="86" text-anchor="middle" fill="#f47067" font-size="9" font-weight="600" letter-spacing="2">CLUSTER NODES</text>
|
||||
<text x="940" y="86" text-anchor="middle" fill="#f778ba" font-size="9" font-weight="600" letter-spacing="2">LLM PROVIDERS</text>
|
||||
<text x="110" y="86" text-anchor="middle" fill="#58a6ff" font-size="9" font-weight="600" letter-spacing="2">CLIENTS</text>
|
||||
<text x="380" y="86" text-anchor="middle" fill="#3fb950" font-size="9" font-weight="600" letter-spacing="2">CONSOLE ROUTER</text>
|
||||
<text x="700" y="86" text-anchor="middle" fill="#f47067" font-size="9" font-weight="600" letter-spacing="2">SERVER NODES</text>
|
||||
<text x="1010" y="86" text-anchor="middle" fill="#f778ba" font-size="9" font-weight="600" letter-spacing="2">LLM PROVIDERS</text>
|
||||
|
||||
<!-- ==================== CLIENT BOXES ==================== -->
|
||||
<!-- CLI -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="108" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="108" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="108" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="111" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="130" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">CLI</text>
|
||||
<text x="90" y="145" text-anchor="middle" fill="#8b949e" font-size="9">terminal REPL</text>
|
||||
<rect x="40" y="108" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="108" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="108" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="111" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="130" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">CLI</text>
|
||||
<text x="110" y="145" text-anchor="middle" fill="#8b949e" font-size="9">terminal REPL</text>
|
||||
</g>
|
||||
|
||||
<!-- Browser UI -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="174" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="174" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="174" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="177" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="196" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Browser UI</text>
|
||||
<text x="90" y="211" text-anchor="middle" fill="#8b949e" font-size="9">HTTP + SSE</text>
|
||||
<rect x="40" y="174" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="174" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="174" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="177" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="196" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Browser UI</text>
|
||||
<text x="110" y="211" text-anchor="middle" fill="#8b949e" font-size="9">HTTP + SSE</text>
|
||||
</g>
|
||||
|
||||
<!-- SDK / API -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="244" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="244" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="244" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="247" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="266" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">SDK / API</text>
|
||||
<text x="90" y="281" text-anchor="middle" fill="#8b949e" font-size="9">programmatic</text>
|
||||
</g>
|
||||
|
||||
<!-- Discord / Slack -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="30" y="314" width="120" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="30" y="314" width="120" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="314" width="120" height="5" fill="#58a6ff"/>
|
||||
<rect x="30" y="317" width="120" height="2" fill="#161b22"/>
|
||||
<text x="90" y="336" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Discord / Slack</text>
|
||||
<text x="90" y="351" text-anchor="middle" fill="#8b949e" font-size="9">chat platforms</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== GATEWAY BOXES ==================== -->
|
||||
<!-- Console -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="216" y="118" width="120" height="58" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="216" y="118" width="120" height="5" rx="5" fill="#3fb950"/>
|
||||
<rect x="216" y="118" width="120" height="5" fill="#3fb950"/>
|
||||
<rect x="216" y="121" width="120" height="2" fill="#161b22"/>
|
||||
<text x="276" y="142" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Console</text>
|
||||
<text x="276" y="157" text-anchor="middle" fill="#8b949e" font-size="9">dashboard + proxy</text>
|
||||
<text x="276" y="169" text-anchor="middle" fill="#8b949e" font-size="9">cluster management</text>
|
||||
<rect x="40" y="244" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="244" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="244" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="247" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="266" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">SDK / API</text>
|
||||
<text x="110" y="281" text-anchor="middle" fill="#8b949e" font-size="9">programmatic</text>
|
||||
</g>
|
||||
|
||||
<!-- Channel Gateway -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="216" y="292" width="120" height="58" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="216" y="292" width="120" height="5" rx="5" fill="#3fb950"/>
|
||||
<rect x="216" y="292" width="120" height="5" fill="#3fb950"/>
|
||||
<rect x="216" y="295" width="120" height="2" fill="#161b22"/>
|
||||
<text x="276" y="316" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Channel Gateway</text>
|
||||
<text x="276" y="331" text-anchor="middle" fill="#8b949e" font-size="9">platform adapter</text>
|
||||
<text x="276" y="343" text-anchor="middle" fill="#8b949e" font-size="9">Discord, Slack, ...</text>
|
||||
<rect x="40" y="314" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="40" y="314" width="140" height="5" rx="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="314" width="140" height="5" fill="#58a6ff"/>
|
||||
<rect x="40" y="317" width="140" height="2" fill="#161b22"/>
|
||||
<text x="110" y="336" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Channel Gateway</text>
|
||||
<text x="110" y="351" text-anchor="middle" fill="#8b949e" font-size="9">Discord, Slack, ...</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== REDIS MQ ==================== -->
|
||||
<!-- ==================== CONSOLE ROUTER ==================== -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="420" y="168" width="120" height="132" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="420" y="168" width="120" height="5" rx="5" fill="#f0883e"/>
|
||||
<rect x="420" y="168" width="120" height="5" fill="#f0883e"/>
|
||||
<rect x="420" y="171" width="120" height="2" fill="#161b22"/>
|
||||
<text x="480" y="198" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Redis MQ</text>
|
||||
<line x1="438" y1="210" x2="522" y2="210" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="480" y="228" text-anchor="middle" fill="#8b949e" font-size="9">inbound queues</text>
|
||||
<text x="480" y="243" text-anchor="middle" fill="#8b949e" font-size="9">event pub/sub</text>
|
||||
<text x="480" y="258" text-anchor="middle" fill="#8b949e" font-size="9">node heartbeats</text>
|
||||
<text x="480" y="273" text-anchor="middle" fill="#8b949e" font-size="9">workstream routing</text>
|
||||
<text x="480" y="288" text-anchor="middle" fill="#8b949e" font-size="9">cluster state</text>
|
||||
<rect x="300" y="148" width="160" height="170" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="300" y="148" width="160" height="5" rx="5" fill="#3fb950"/>
|
||||
<rect x="300" y="148" width="160" height="5" fill="#3fb950"/>
|
||||
<rect x="300" y="151" width="160" height="2" fill="#161b22"/>
|
||||
<text x="380" y="178" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Console</text>
|
||||
<line x1="318" y1="190" x2="442" y2="190" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="380" y="208" text-anchor="middle" fill="#8b949e" font-size="9">hash-ring router</text>
|
||||
<text x="380" y="223" text-anchor="middle" fill="#8b949e" font-size="9">cluster dashboard</text>
|
||||
<text x="380" y="238" text-anchor="middle" fill="#8b949e" font-size="9">reverse proxy</text>
|
||||
<line x1="318" y1="250" x2="442" y2="250" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="380" y="268" text-anchor="middle" fill="#484f58" font-size="8">control plane:</text>
|
||||
<text x="380" y="282" text-anchor="middle" fill="#484f58" font-size="8">create / send / approve</text>
|
||||
<text x="380" y="296" text-anchor="middle" fill="#484f58" font-size="8">cancel / command / close</text>
|
||||
<text x="380" y="310" text-anchor="middle" fill="#484f58" font-size="8">port 8090</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== CLUSTER NODES ==================== -->
|
||||
<!-- ==================== SERVER NODES ==================== -->
|
||||
<!-- Cluster outline -->
|
||||
<rect x="598" y="100" width="204" height="310" rx="8" fill="none" stroke="#30363d" stroke-width="1" stroke-dasharray="4,3"/>
|
||||
<rect x="570" y="100" width="260" height="310" rx="8" fill="none" stroke="#30363d" stroke-width="1" stroke-dasharray="4,3"/>
|
||||
<text x="700" y="422" text-anchor="middle" fill="#30363d" font-size="9" letter-spacing="1">CLUSTER</text>
|
||||
|
||||
<!-- Node A -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="614" y="118" width="170" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="614" y="118" width="170" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="614" y="118" width="170" height="5" fill="#f47067"/>
|
||||
<rect x="614" y="121" width="170" height="2" fill="#161b22"/>
|
||||
<text x="699" y="142" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Node A</text>
|
||||
<line x1="632" y1="152" x2="766" y2="152" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Bridge -->
|
||||
<rect x="626" y="162" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="661" y="180" text-anchor="middle" fill="#8b949e" font-size="9">bridge</text>
|
||||
<!-- Server -->
|
||||
<rect x="704" y="162" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="739" y="180" text-anchor="middle" fill="#8b949e" font-size="9">server</text>
|
||||
<!-- Arrow bridge to server -->
|
||||
<line x1="696" y1="176" x2="702" y2="176" stroke="#484f58" stroke-width="1" marker-end="url(#arrow)"/>
|
||||
<rect x="590" y="118" width="220" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="590" y="118" width="220" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="590" y="118" width="220" height="5" fill="#f47067"/>
|
||||
<rect x="590" y="121" width="220" height="2" fill="#161b22"/>
|
||||
<text x="700" y="142" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Server Node A</text>
|
||||
<line x1="608" y1="152" x2="792" y2="152" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Server process -->
|
||||
<rect x="608" y="162" width="184" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="700" y="180" text-anchor="middle" fill="#8b949e" font-size="9">turnstone-server :8080</text>
|
||||
<!-- Tools label -->
|
||||
<text x="699" y="206" text-anchor="middle" fill="#484f58" font-size="8">14 tools + MCP</text>
|
||||
<text x="700" y="206" text-anchor="middle" fill="#484f58" font-size="8">19 tools + MCP</text>
|
||||
</g>
|
||||
|
||||
<!-- Node B -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="614" y="238" width="170" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="614" y="238" width="170" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="614" y="238" width="170" height="5" fill="#f47067"/>
|
||||
<rect x="614" y="241" width="170" height="2" fill="#161b22"/>
|
||||
<text x="699" y="262" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Node B</text>
|
||||
<line x1="632" y1="272" x2="766" y2="272" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Bridge -->
|
||||
<rect x="626" y="282" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="661" y="300" text-anchor="middle" fill="#8b949e" font-size="9">bridge</text>
|
||||
<!-- Server -->
|
||||
<rect x="704" y="282" width="70" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="739" y="300" text-anchor="middle" fill="#8b949e" font-size="9">server</text>
|
||||
<!-- Arrow bridge to server -->
|
||||
<line x1="696" y1="296" x2="702" y2="296" stroke="#484f58" stroke-width="1" marker-end="url(#arrow)"/>
|
||||
<rect x="590" y="238" width="220" height="100" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="590" y="238" width="220" height="5" rx="5" fill="#f47067"/>
|
||||
<rect x="590" y="238" width="220" height="5" fill="#f47067"/>
|
||||
<rect x="590" y="241" width="220" height="2" fill="#161b22"/>
|
||||
<text x="700" y="262" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Server Node B</text>
|
||||
<line x1="608" y1="272" x2="792" y2="272" stroke="#30363d" stroke-width="1"/>
|
||||
<!-- Server process -->
|
||||
<rect x="608" y="282" width="184" height="28" rx="3" fill="#1c2128" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="700" y="300" text-anchor="middle" fill="#8b949e" font-size="9">turnstone-server :8080</text>
|
||||
<!-- Tools label -->
|
||||
<text x="699" y="326" text-anchor="middle" fill="#484f58" font-size="8">14 tools + MCP</text>
|
||||
<text x="700" y="326" text-anchor="middle" fill="#484f58" font-size="8">19 tools + MCP</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== LLM PROVIDERS ==================== -->
|
||||
<!-- OpenAI -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="870" y="130" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="870" y="130" width="140" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="870" y="130" width="140" height="5" fill="#f778ba"/>
|
||||
<rect x="870" y="133" width="140" height="2" fill="#161b22"/>
|
||||
<text x="940" y="153" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">OpenAI</text>
|
||||
<text x="940" y="167" text-anchor="middle" fill="#8b949e" font-size="9">GPT-5, o-series</text>
|
||||
<rect x="930" y="130" width="160" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="930" y="130" width="160" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="930" y="130" width="160" height="5" fill="#f778ba"/>
|
||||
<rect x="930" y="133" width="160" height="2" fill="#161b22"/>
|
||||
<text x="1010" y="153" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">OpenAI</text>
|
||||
<text x="1010" y="167" text-anchor="middle" fill="#8b949e" font-size="9">GPT-5, o-series</text>
|
||||
</g>
|
||||
|
||||
<!-- Anthropic -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="870" y="196" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="870" y="196" width="140" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="870" y="196" width="140" height="5" fill="#f778ba"/>
|
||||
<rect x="870" y="199" width="140" height="2" fill="#161b22"/>
|
||||
<text x="940" y="219" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Anthropic</text>
|
||||
<text x="940" y="233" text-anchor="middle" fill="#8b949e" font-size="9">Claude 4.5 / 4.6</text>
|
||||
<rect x="930" y="196" width="160" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="930" y="196" width="160" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="930" y="196" width="160" height="5" fill="#f778ba"/>
|
||||
<rect x="930" y="199" width="160" height="2" fill="#161b22"/>
|
||||
<text x="1010" y="219" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Anthropic</text>
|
||||
<text x="1010" y="233" text-anchor="middle" fill="#8b949e" font-size="9">Claude 4.5 / 4.6</text>
|
||||
</g>
|
||||
|
||||
<!-- Local / vLLM -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="870" y="262" width="140" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="870" y="262" width="140" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="870" y="262" width="140" height="5" fill="#f778ba"/>
|
||||
<rect x="870" y="265" width="140" height="2" fill="#161b22"/>
|
||||
<text x="940" y="285" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Local / vLLM</text>
|
||||
<text x="940" y="299" text-anchor="middle" fill="#8b949e" font-size="9">llama.cpp, NIM</text>
|
||||
<rect x="930" y="262" width="160" height="46" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="930" y="262" width="160" height="5" rx="5" fill="#f778ba"/>
|
||||
<rect x="930" y="262" width="160" height="5" fill="#f778ba"/>
|
||||
<rect x="930" y="265" width="160" height="2" fill="#161b22"/>
|
||||
<text x="1010" y="285" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">Local / vLLM</text>
|
||||
<text x="1010" y="299" text-anchor="middle" fill="#8b949e" font-size="9">llama.cpp, NIM</text>
|
||||
</g>
|
||||
|
||||
<!-- ==================== STORAGE ==================== -->
|
||||
<g filter="url(#shadow)">
|
||||
<rect x="614" y="450" width="170" height="52" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="614" y="450" width="170" height="5" rx="5" fill="#bc8cff"/>
|
||||
<rect x="614" y="450" width="170" height="5" fill="#bc8cff"/>
|
||||
<rect x="614" y="453" width="170" height="2" fill="#161b22"/>
|
||||
<text x="699" y="476" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">PostgreSQL / SQLite</text>
|
||||
<text x="699" y="492" text-anchor="middle" fill="#8b949e" font-size="9">conversations, memory, auth</text>
|
||||
<rect x="590" y="450" width="220" height="52" rx="5" fill="#161b22" stroke="#30363d" stroke-width="1"/>
|
||||
<rect x="590" y="450" width="220" height="5" rx="5" fill="#bc8cff"/>
|
||||
<rect x="590" y="450" width="220" height="5" fill="#bc8cff"/>
|
||||
<rect x="590" y="453" width="220" height="2" fill="#161b22"/>
|
||||
<text x="700" y="476" text-anchor="middle" fill="#e6edf3" font-size="11" font-weight="600">PostgreSQL / SQLite</text>
|
||||
<text x="700" y="492" text-anchor="middle" fill="#8b949e" font-size="9">conversations, memory, auth</text>
|
||||
</g>
|
||||
<text x="699" y="444" text-anchor="middle" fill="#bc8cff" font-size="9" font-weight="600" letter-spacing="2">STORAGE</text>
|
||||
<text x="700" y="444" text-anchor="middle" fill="#bc8cff" font-size="9" font-weight="600" letter-spacing="2">STORAGE</text>
|
||||
|
||||
<!-- ==================== CONNECTION LINES ==================== -->
|
||||
|
||||
<!-- CLIENT -> GATEWAY connections -->
|
||||
<!-- CLIENT -> CONSOLE connections (control plane) -->
|
||||
<!-- Browser -> Console -->
|
||||
<line x1="150" y1="197" x2="214" y2="155" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- Discord -> Channel -->
|
||||
<line x1="150" y1="337" x2="214" y2="325" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<line x1="180" y1="197" x2="298" y2="210" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- Channel -> Console -->
|
||||
<line x1="180" y1="337" x2="298" y2="290" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- SDK -> Console -->
|
||||
<line x1="180" y1="267" x2="298" y2="248" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<text x="240" y="238" fill="#484f58" font-size="8" text-anchor="middle">HTTP</text>
|
||||
|
||||
<!-- CLI -> direct to Node A server (top path, curved) -->
|
||||
<path d="M 150 131 C 200 131, 200 100, 400 100 L 400 100 C 500 100, 570 140, 612 168" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.5" fill="none" stroke-dasharray="6,3" marker-end="url(#arrow-blue)"/>
|
||||
<text x="370" y="96" fill="#484f58" font-size="8" text-anchor="middle">direct</text>
|
||||
<!-- CLI -> direct to Node A (single-node mode, above everything) -->
|
||||
<path d="M 180 120 L 588 120" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.4" fill="none" stroke-dasharray="6,3" marker-end="url(#arrow-blue)"/>
|
||||
<text x="390" y="114" fill="#484f58" font-size="8" text-anchor="middle">direct (single-node)</text>
|
||||
|
||||
<!-- SDK -> Redis (direct push) -->
|
||||
<line x1="150" y1="267" x2="418" y2="240" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-blue)"/>
|
||||
<!-- CONSOLE -> NODE connections (proxy) -->
|
||||
<!-- Console -> Node A -->
|
||||
<line x1="460" y1="200" x2="588" y2="176" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
<!-- Console -> Node B -->
|
||||
<line x1="460" y1="260" x2="588" y2="296" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
<text x="520" y="222" fill="#484f58" font-size="8" text-anchor="middle">proxy</text>
|
||||
|
||||
<!-- GATEWAY -> REDIS connections -->
|
||||
<!-- Console -> Redis -->
|
||||
<line x1="336" y1="160" x2="418" y2="200" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
<!-- Channel -> Redis -->
|
||||
<line x1="336" y1="318" x2="418" y2="272" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-green)"/>
|
||||
|
||||
<!-- REDIS -> NODE connections -->
|
||||
<!-- Redis -> Node A bridge -->
|
||||
<line x1="540" y1="210" x2="624" y2="176" stroke="#f0883e" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-orange)"/>
|
||||
<!-- Redis -> Node B bridge -->
|
||||
<line x1="540" y1="260" x2="624" y2="296" stroke="#f0883e" stroke-width="1.2" stroke-opacity="0.6" marker-end="url(#arrow-orange)"/>
|
||||
|
||||
<!-- Console -> Node (proxy, dashed) -->
|
||||
<path d="M 336 147 C 380 130, 500 108, 612 145" stroke="#3fb950" stroke-width="1" stroke-opacity="0.4" fill="none" stroke-dasharray="4,3" marker-end="url(#arrow-green)"/>
|
||||
<text x="468" y="120" fill="#484f58" font-size="8" text-anchor="middle">proxy</text>
|
||||
<!-- CLIENT -> NODE direct SSE (data plane, below console) -->
|
||||
<!-- Browser -> Node A SSE (arc below console) -->
|
||||
<path d="M 180 205 C 240 370, 450 380, 588 330" stroke="#58a6ff" stroke-width="1" stroke-opacity="0.3" fill="none" stroke-dasharray="4,3" marker-end="url(#arrow-blue)"/>
|
||||
<text x="340" y="378" fill="#484f58" font-size="8" text-anchor="middle">SSE (data plane)</text>
|
||||
|
||||
<!-- NODE -> LLM connections -->
|
||||
<!-- Node A -> LLM providers -->
|
||||
<line x1="784" y1="168" x2="868" y2="155" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
<line x1="784" y1="176" x2="868" y2="219" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="784" y1="180" x2="868" y2="282" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
<line x1="810" y1="168" x2="928" y2="155" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
<line x1="810" y1="176" x2="928" y2="219" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="810" y1="180" x2="928" y2="282" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
|
||||
<!-- Node B -> LLM providers -->
|
||||
<line x1="784" y1="288" x2="868" y2="163" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
<line x1="784" y1="296" x2="868" y2="222" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="784" y1="300" x2="868" y2="288" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
<line x1="810" y1="288" x2="928" y2="163" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.2"/>
|
||||
<line x1="810" y1="296" x2="928" y2="222" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.3"/>
|
||||
<line x1="810" y1="300" x2="928" y2="288" stroke="#f47067" stroke-width="1.2" stroke-opacity="0.5" marker-end="url(#arrow-coral)"/>
|
||||
|
||||
<!-- NODE -> STORAGE connections -->
|
||||
<line x1="680" y1="338" x2="680" y2="448" stroke="#bc8cff" stroke-width="1.2" stroke-opacity="0.4" stroke-dasharray="4,3" marker-end="url(#arrow-muted)"/>
|
||||
<line x1="718" y1="218" x2="718" y2="236" stroke="#484f58" stroke-width="1" stroke-opacity="0.3" stroke-dasharray="2,2"/>
|
||||
|
||||
<!-- Extensibility hint -->
|
||||
<text x="699" y="392" text-anchor="middle" fill="#30363d" font-size="10">...</text>
|
||||
|
||||
<!-- Event flow: Bridges -> Redis (dashed, bidirectional feel) -->
|
||||
<line x1="624" y1="186" x2="542" y2="220" stroke="#f0883e" stroke-width="1" stroke-opacity="0.3" stroke-dasharray="3,3"/>
|
||||
<line x1="624" y1="286" x2="542" y2="250" stroke="#f0883e" stroke-width="1" stroke-opacity="0.3" stroke-dasharray="3,3"/>
|
||||
<text x="574" y="242" fill="#484f58" font-size="7" text-anchor="middle">events</text>
|
||||
<text x="700" y="392" text-anchor="middle" fill="#30363d" font-size="10">...</text>
|
||||
|
||||
<!-- ==================== FLOW LABELS ==================== -->
|
||||
<!-- Interactive flow label -->
|
||||
<!-- Direct / single-node flow label -->
|
||||
<rect x="30" y="395" width="10" height="10" rx="2" fill="none" stroke="#58a6ff" stroke-width="1.5" stroke-dasharray="3,2"/>
|
||||
<text x="46" y="404" fill="#8b949e" font-size="9">interactive (direct)</text>
|
||||
<text x="46" y="404" fill="#8b949e" font-size="9">direct (single-node / SSE)</text>
|
||||
|
||||
<!-- Queue flow label -->
|
||||
<rect x="160" y="395" width="10" height="10" rx="2" fill="none" stroke="#f0883e" stroke-width="1.5"/>
|
||||
<text x="176" y="404" fill="#8b949e" font-size="9">queue-driven</text>
|
||||
<!-- Control plane label -->
|
||||
<rect x="200" y="395" width="10" height="10" rx="2" fill="none" stroke="#58a6ff" stroke-width="1.5"/>
|
||||
<text x="216" y="404" fill="#8b949e" font-size="9">control plane (HTTP)</text>
|
||||
|
||||
<!-- Proxy/event label -->
|
||||
<rect x="275" y="395" width="10" height="10" rx="2" fill="none" stroke="#3fb950" stroke-width="1.5" stroke-dasharray="3,2"/>
|
||||
<text x="291" y="404" fill="#8b949e" font-size="9">proxy / events</text>
|
||||
<!-- Proxy label -->
|
||||
<rect x="340" y="395" width="10" height="10" rx="2" fill="none" stroke="#3fb950" stroke-width="1.5"/>
|
||||
<text x="356" y="404" fill="#8b949e" font-size="9">console proxy</text>
|
||||
|
||||
<!-- ==================== BOTTOM DETAILS ==================== -->
|
||||
<line x1="30" y1="430" x2="1170" y2="430" stroke="#21262d" stroke-width="1"/>
|
||||
|
||||
<!-- Routing rules at bottom, left-aligned -->
|
||||
<text x="44" y="456" fill="#30363d" font-size="9" font-weight="600" letter-spacing="1">ROUTING</text>
|
||||
<circle cx="44" cy="474" r="3" fill="#f47067" opacity="0.6"/>
|
||||
<text x="54" y="477" fill="#484f58" font-size="9">target_node set → route to specific node queue</text>
|
||||
<circle cx="44" cy="494" r="3" fill="#f0883e" opacity="0.6"/>
|
||||
<text x="54" y="497" fill="#484f58" font-size="9">ws_id set → route to owning node</text>
|
||||
<circle cx="44" cy="514" r="3" fill="#58a6ff" opacity="0.6"/>
|
||||
<text x="54" y="517" fill="#484f58" font-size="9">neither → shared queue, any node picks up</text></svg>
|
||||
<circle cx="44" cy="474" r="3" fill="#3fb950" opacity="0.6"/>
|
||||
<text x="54" y="477" fill="#484f58" font-size="9">control plane: client → console → server node (hash-ring bucket lookup)</text>
|
||||
<circle cx="44" cy="494" r="3" fill="#58a6ff" opacity="0.6"/>
|
||||
<text x="54" y="497" fill="#484f58" font-size="9">data plane: client → server node (direct SSE, node_url from create response)</text>
|
||||
<circle cx="44" cy="514" r="3" fill="#f47067" opacity="0.6"/>
|
||||
<text x="54" y="517" fill="#484f58" font-size="9">single-node: client → server (direct HTTP + SSE, no console needed)</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 16 KiB |
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d8ce6d2a43a991655c3f64a20b6e810fdb2f78eb767acc3d3d1b8d2c9f443181
|
||||
size 165011
|
||||
oid sha256:881a8b9bce67b5af9a52d5e50deaa72351cd99c76f18aad5caeb2b61131ca1af
|
||||
size 119798
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:efcc7cbe8161a54b5ec24bdfd47e8a142f70029e6e66c707e811b99369f85ebf
|
||||
size 310079
|
||||
oid sha256:75da80e6bd205e45b9fe48aa2f87197110c908f44e1d98c52eee69f956274711
|
||||
size 400402
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:72b3932ce99a860f5069544cd8423d3cdae3a51f6566db262b19ced19c780eb0
|
||||
size 274374
|
||||
oid sha256:674712a0563f51837383184652efeb28b7bec13378be636e89d2959bfba39d1e
|
||||
size 281519
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d831c5e10266f0232262b6a29b0ea8e45b3cc63df75f716a80f18462b4f85e66
|
||||
size 319125
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:09535722ba975e47cf0557a40b6c481f125ff2022c396f79715c3bba9f715871
|
||||
size 222032
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ed457b10b534b5fc2a5e190b281d7ded4dd1615da2229d67a373cf5dddccd059
|
||||
size 201601
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7bf27afa267d5b8d6da38e83213ed1b8e87639d5105a0a1ccc2e5a4bf4d3b67e
|
||||
size 185282
|
||||
oid sha256:04d2069a9b5155ad1e7d842147fd78535ad9106d6856520439c33a9868a47499
|
||||
size 156694
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:35cf3a6942f62dabcbbe012ac2f9e6f155332c894692981b076de5a25c1f3330
|
||||
size 374055
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e3f1ad0fcd55eaca3b8ad9c5abc07432803641c54ede9fc93c79df144cf77d1c
|
||||
size 407761
|
||||
oid sha256:040f7d9ec7d676da40b9487e0825caf2c1574cbdd9f16d0998d90e0c2e4f8861
|
||||
size 360309
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:09065fef028d05e6df425fd8abefaf5a2ca04b66802f2e3975f289fa597f63ed
|
||||
size 309656
|
||||
oid sha256:a872556d111185f4531d1b68ee892b4ce5042d7ccf277e2cad08beb6932c9803
|
||||
size 191144
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6fc99bb8d84d6e9f3dac9d5c12ac7f569a041b29431c57612c24b50f332982ed
|
||||
size 462992
|
||||
oid sha256:8a5957b71822656098cbe779ca619b7e56e9b11b0f1c4eac4fc880ed62b71a2a
|
||||
size 358670
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:83c0e6aad3eb19f6bc475a30a77215e801da3da5930f0462417fe7eb6eda6be2
|
||||
size 347144
|
||||
oid sha256:06fe076f0835a891e00afc804fd1805196ebde9fc0d34998c7873e87287f982b
|
||||
size 346887
|
||||
|
||||
+8
-46
@@ -1,6 +1,6 @@
|
||||
# Docker Deployment
|
||||
|
||||
Docker Compose stack for running the full turnstone platform or the simulator.
|
||||
Docker Compose stack for running the full turnstone platform.
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -10,9 +10,6 @@ cp .env.example .env
|
||||
|
||||
# Full stack (needs an LLM API on the host)
|
||||
docker compose up
|
||||
|
||||
# Simulator only (no LLM needed)
|
||||
docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
Console dashboard: http://localhost:8090
|
||||
@@ -23,18 +20,14 @@ Console dashboard: http://localhost:8090
|
||||
|
||||
| Service | Port | Profile | Description |
|
||||
|---------|------|---------|-------------|
|
||||
| `redis` | 6379 | default | Message broker, pub/sub, node registry |
|
||||
| `server` | 8080 | default | Web UI + chat workstreams + LLM |
|
||||
| `bridge` | — | default | Redis-to-HTTP bridge (multi-node routing) |
|
||||
| `console` | 8090 | default | Cluster dashboard |
|
||||
| `channel` | — | production | Channel gateway (Discord, Slack, etc.) |
|
||||
| `server-1`…`server-10` | — | cluster | 10-node server fleet (PostgreSQL required) |
|
||||
| `bridge-1`…`bridge-10` | — | cluster | Matching bridge fleet |
|
||||
| `sim` | — | sim | Multi-node cluster simulator |
|
||||
|
||||
## Profiles
|
||||
|
||||
**Default** (no flag) — starts `redis`, `server`, `bridge`, `console`. Requires an OpenAI-compatible LLM API running on the host (default: `http://localhost:8000/v1`).
|
||||
**Default** (no flag) — starts `server` and `console`. Requires an OpenAI-compatible LLM API running on the host (default: `http://localhost:8000/v1`).
|
||||
|
||||
```bash
|
||||
docker compose up
|
||||
@@ -46,22 +39,12 @@ docker compose up
|
||||
docker compose --profile production up
|
||||
```
|
||||
|
||||
**Cluster** — 10-node server/bridge fleet sharing PostgreSQL and Redis. Access all nodes via the console at `:8090`. Requires `POSTGRES_PASSWORD`:
|
||||
**Cluster** — 10-node server fleet sharing PostgreSQL. Access all nodes via the console at `:8090`. Requires `POSTGRES_PASSWORD`:
|
||||
|
||||
```bash
|
||||
docker compose --profile cluster up
|
||||
```
|
||||
|
||||
**Sim** — adds the simulator. Can run alongside the full stack or standalone with just Redis and the console:
|
||||
|
||||
```bash
|
||||
# Sim + console (no LLM needed)
|
||||
docker compose --profile sim up redis console sim
|
||||
|
||||
# Everything including sim
|
||||
docker compose --profile sim up
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration is via environment variables in `.env` (copy from `.env.example`):
|
||||
@@ -74,13 +57,6 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
| `OPENAI_API_KEY` | `dummy` | API key (`dummy` for local servers) |
|
||||
| `TAVILY_API_KEY` | — | Web search API key (only needed for local/vLLM models; Anthropic and OpenAI search models use native search) |
|
||||
|
||||
### Redis
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `REDIS_PASSWORD` | — | Redis auth password (empty = no auth) |
|
||||
| `REDIS_PORT` | `6379` | Host port mapping |
|
||||
|
||||
### Server
|
||||
|
||||
| Variable | Default | Description |
|
||||
@@ -93,14 +69,13 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `CONSOLE_PORT` | `8090` | Host port mapping |
|
||||
| `CONSOLE_POLL_INTERVAL` | `10` | Node polling interval (seconds) |
|
||||
|
||||
### Auth
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_AUTH_ENABLED` | — | Set to `1` to require authentication |
|
||||
| `TURNSTONE_AUTH_TOKEN` | — | Config-file token for server/bridge/console (backward compat, works alongside JWT) |
|
||||
| `TURNSTONE_AUTH_TOKEN` | — | Config-file token for server/console (backward compat, works alongside JWT) |
|
||||
| `TURNSTONE_JWT_SECRET` | — | Secret key for signing JWTs (required when using user identity / JWT auth) |
|
||||
|
||||
### Database
|
||||
@@ -130,29 +105,17 @@ The database stores workstream history, user accounts, and API tokens. When usin
|
||||
| `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord adapter) |
|
||||
| `TURNSTONE_DISCORD_GUILD` | `0` | Restrict to a single Discord guild (0 = all guilds) |
|
||||
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages through Redis MQ to the bridge and server. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
### Simulator
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `SIM_NODES` | `100` | Number of simulated nodes |
|
||||
| `SIM_SCENARIO` | `steady` | Scenario: `steady`, `burst`, `node_failure`, `directed`, `lifecycle` |
|
||||
| `SIM_DURATION` | `60` | Duration in seconds |
|
||||
| `SIM_MPS` | `5.0` | Messages per second (steady scenario) |
|
||||
| `SIM_LOG_LEVEL` | `INFO` | Log verbosity |
|
||||
| `SIM_SEED` | — | Random seed for reproducibility |
|
||||
| `SIM_METRICS_FILE` | — | Write JSON report to file |
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages to the server via HTTP. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
## Scaling
|
||||
|
||||
For multi-node testing, use the `cluster` profile which provides 10 dedicated server+bridge pairs with unique node IDs (`node-1` through `node-10`), resource limits, and shared PostgreSQL:
|
||||
For multi-node testing, use the `cluster` profile which provides 10 server instances with unique node IDs (`node-1` through `node-10`), resource limits, and shared PostgreSQL:
|
||||
|
||||
```bash
|
||||
POSTGRES_PASSWORD=secret docker compose --profile cluster up
|
||||
```
|
||||
|
||||
The default `server` and `bridge` also run alongside the cluster nodes (11 total). All nodes are accessible via the console dashboard at `:8090`.
|
||||
The default `server` also runs alongside the cluster nodes (11 total). All nodes are accessible via the console dashboard at `:8090`.
|
||||
|
||||
For production clusters beyond ~50 nodes, add PgBouncer between turnstone services and PostgreSQL. See [PgBouncer Connection Pooling](pgbouncer.md) for Docker Compose and Helm configuration.
|
||||
|
||||
@@ -160,7 +123,6 @@ For production clusters beyond ~50 nodes, add PgBouncer between turnstone servic
|
||||
|
||||
| Volume | Mount | Purpose |
|
||||
|--------|-------|---------|
|
||||
| `redis-data` | `/data` | Redis persistence |
|
||||
| `turnstone-data` | `/data` | SQLite database (`.turnstone.db`) |
|
||||
|
||||
## Building
|
||||
@@ -175,7 +137,7 @@ docker compose build
|
||||
docker compose build --no-cache
|
||||
```
|
||||
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-bridge`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-sim`, `turnstone-eval`.
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-eval`.
|
||||
|
||||
## Cleanup
|
||||
|
||||
|
||||
+1
-1
@@ -379,7 +379,7 @@ emitted to the frontend:
|
||||
```
|
||||
|
||||
The web UI renders this as an inline warning after the tool result. The CLI
|
||||
shows a colored terminal warning. The MQ bridge forwards it as an
|
||||
shows a colored terminal warning. The server forwards it as an
|
||||
`OutputWarningEvent` for console subscribers.
|
||||
|
||||
Assessments are persisted to the `output_assessments` table for v2
|
||||
|
||||
@@ -98,7 +98,6 @@ cannot bypass the proxy.
|
||||
| `skills_registry` | `skills.sh` | Skill discovery |
|
||||
| `github_api` | `api.github.com` (read-only L7), `raw.githubusercontent.com` | Skill fetch, GitHub API |
|
||||
| `mcp_registry` | `registry.modelcontextprotocol.io` (read-only L7) | MCP server discovery |
|
||||
| `redis` | `127.0.0.1:6379` | Message queue |
|
||||
| `web_fetch_common` | readthedocs, python docs, GitHub Pages, PyPI, npm, Stack Overflow, Wikipedia | Curated web_fetch domains |
|
||||
| `bash_network_tools` | Same as `web_fetch_common` | curl/wget from bash tool |
|
||||
| `package_registries` | `pypi.org`, `files.pythonhosted.org` | pip/uv package installs |
|
||||
@@ -283,6 +282,5 @@ For production deployments:
|
||||
- [ ] Review and trim `web_fetch_common` domains to your actual needs
|
||||
- [ ] Remove `package_registries` policy if pip/uv installs are not needed
|
||||
- [ ] Add your OIDC provider endpoint if using SSO
|
||||
- [ ] Set Redis `allowed_ips` to your actual Redis host if not localhost
|
||||
- [ ] Consider removing `bash_network_tools` entirely if bash should not have
|
||||
network access
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
# PgBouncer Connection Pooling
|
||||
|
||||
Turnstone cluster deployments share a single PostgreSQL instance across
|
||||
all server nodes, bridge processes, and the console. Each process
|
||||
all server nodes and the console. Each process
|
||||
maintains a small connection pool (2 base + 3 overflow = 5 max). At
|
||||
scale this adds up — a 100-node cluster opens up to 500 connections,
|
||||
and a 1000-node cluster up to 5,000.
|
||||
|
||||
+12
-17
@@ -339,10 +339,9 @@ secret. If no secret is configured, an ephemeral key is generated at
|
||||
startup and a warning is logged — JWTs will not survive restarts or work
|
||||
across nodes.
|
||||
|
||||
The bridge and console **require** `TURNSTONE_JWT_SECRET` when no
|
||||
`--auth-token` is provided. They exit with an error if the secret is
|
||||
missing, since ephemeral secrets would silently break inter-service
|
||||
communication.
|
||||
The console **requires** `TURNSTONE_JWT_SECRET` when no `--auth-token`
|
||||
is provided. It exits with an error if the secret is missing, since
|
||||
ephemeral secrets would silently break inter-service communication.
|
||||
|
||||
---
|
||||
|
||||
@@ -484,30 +483,26 @@ static token is used as a final fallback.
|
||||
|
||||
### Service-to-service authentication
|
||||
|
||||
The bridge and console collector use `ServiceTokenManager` for
|
||||
auto-rotating JWTs when communicating with server nodes:
|
||||
The console collector uses `ServiceTokenManager` for auto-rotating
|
||||
JWTs when communicating with server nodes:
|
||||
|
||||
| Service | Identity | Scope | Audience | Purpose |
|
||||
|---------|----------|-------|----------|---------|
|
||||
| Bridge | `bridge` | `approve` | `turnstone-server` | Tool approval proxy, message relay |
|
||||
| Console collector | `console-collector` | `read` | `turnstone-server` | Node health polling |
|
||||
| Console proxy (fallback) | `console-proxy` | `approve` | `turnstone-server` | Proxied API calls when no user context |
|
||||
| Channel notify | `system` | `write` | `turnstone-channel` | Notification delivery to channel gateway |
|
||||
|
||||
Service tokens use 1-hour expiry with automatic refresh via
|
||||
`ServiceTokenManager`. The bridge injects auth headers per-request via
|
||||
httpx event hooks to ensure rotated tokens are picked up on SSE
|
||||
reconnects.
|
||||
`ServiceTokenManager`.
|
||||
|
||||
### User identity in MQ-dispatched workstreams
|
||||
|
||||
When the console creates a workstream via MQ (the normal path), the
|
||||
authenticated user's `user_id` is embedded in the
|
||||
`CreateWorkstreamMessage`. The bridge forwards this `user_id` in the
|
||||
HTTP payload when calling the server's `POST /v1/api/workstreams/new`.
|
||||
The server accepts a `user_id` from the request body **only when the
|
||||
caller is a trusted service** — identified by `token_source` matching
|
||||
`bridge`, `console-proxy`, or `console`. Regular API callers cannot
|
||||
When the console creates a workstream (the normal path), the
|
||||
authenticated user's `user_id` is forwarded in the HTTP payload when
|
||||
calling the server's `POST /v1/api/workstreams/new`. The server
|
||||
accepts a `user_id` from the request body **only when the caller is a
|
||||
trusted service** — identified by `token_source` matching
|
||||
`console-proxy` or `console`. Regular API callers cannot
|
||||
override `user_id`; the server always uses their JWT identity.
|
||||
|
||||
Note that the channel gateway uses a distinct JWT audience
|
||||
|
||||
+1
-3
@@ -27,7 +27,7 @@ Settings resolution differs between entry points:
|
||||
|
||||
| Entry point | Chain |
|
||||
|-------------|-------|
|
||||
| **Server** (`turnstone-server`, `turnstone-bridge`) | CLI flag > ConfigStore > registry default |
|
||||
| **Server** (`turnstone-server`) | CLI flag > ConfigStore > registry default |
|
||||
| **CLI** (`turnstone`) | CLI flag > config.toml > argparse default |
|
||||
|
||||
The server's `apply_config()` ignores config.toml sections that overlap with
|
||||
@@ -46,9 +46,7 @@ connection, Redis, auth secrets, server bind address). These stay in
|
||||
|----------|---------|-------|
|
||||
| API credentials | `[api]` | config.toml / env |
|
||||
| Database | `[database]` | config.toml / env |
|
||||
| Redis | `[redis]` | config.toml / env |
|
||||
| Auth | `[auth]` | config.toml / env |
|
||||
| Bridge identity | `[bridge]` | config.toml / env |
|
||||
| Console bind | `[console]` | config.toml / env |
|
||||
|
||||
**ConfigStore settings** (48 settings) are loaded from the database after
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
# Cluster Simulator
|
||||
|
||||
The simulator (`turnstone-sim`) creates lightweight simulated nodes that talk to a real Redis instance using the standard turnstone protocol. External observers — `TurnstoneClient`, `turnstone-console`, real bridges — see identical behavior. No LLM backend is needed.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
pip install turnstone[sim]
|
||||
|
||||
# 10 nodes, steady load, 60 seconds
|
||||
turnstone-sim --nodes 10 --scenario steady --duration 60 --mps 5
|
||||
|
||||
# 100 nodes via Docker
|
||||
docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
Each simulated node is an asyncio coroutine (not a thread or process), so 1000 nodes run efficiently on a single event loop. The simulator:
|
||||
|
||||
1. Registers nodes via Redis heartbeats (same keys as real bridges)
|
||||
2. Accepts messages from per-node and shared inbound queues
|
||||
3. Simulates LLM responses with configurable latency and token generation
|
||||
4. Simulates tool execution with configurable latency and failure rates
|
||||
5. Publishes real protocol events (`ContentEvent`, `StateChangeEvent`, `TurnCompleteEvent`, etc.)
|
||||
6. Reports latency, throughput, and utilization metrics at completion
|
||||
|
||||
```
|
||||
TurnstoneClient → Redis Queue → SimNode → Redis Pub/Sub → TurnstoneClient
|
||||
↓
|
||||
turnstone-console (cluster dashboard)
|
||||
```
|
||||
|
||||
## Scenarios
|
||||
|
||||
| Scenario | Description |
|
||||
|----------|-------------|
|
||||
| `steady` | Inject messages at a constant rate (`--mps`) for `--duration` seconds |
|
||||
| `burst` | Push `--burst-size` messages instantly, then wait for completion |
|
||||
| `node_failure` | Steady load + periodically kill nodes to test redistribution |
|
||||
| `directed` | Send messages to specific nodes via `target_node` routing |
|
||||
| `lifecycle` | Create, use, and close workstreams across nodes |
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
turnstone-sim [options]
|
||||
```
|
||||
|
||||
### Cluster
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--nodes` | `10` | Number of simulated nodes |
|
||||
|
||||
### Scenario
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--scenario` | `steady` | Scenario name |
|
||||
| `--duration` | `60` | Duration in seconds |
|
||||
| `--mps` | `5.0` | Messages per second (steady) |
|
||||
| `--burst-size` | `100` | Messages to send (burst) |
|
||||
| `--node-kill-interval` | `15` | Seconds between kills (node_failure) |
|
||||
| `--node-kill-count` | `1` | Nodes per kill cycle |
|
||||
|
||||
### Simulation
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--llm-latency` | `2.0` | Mean LLM response latency (seconds) |
|
||||
| `--tool-latency` | `0.5` | Mean tool execution latency (seconds) |
|
||||
| `--tool-failure-rate` | `0.02` | Tool failure probability (0.0–1.0) |
|
||||
| `--seed` | — | Random seed for reproducibility |
|
||||
|
||||
### Redis
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--redis-host` | `localhost` | Redis host |
|
||||
| `--redis-port` | `6379` | Redis port |
|
||||
| `--redis-password` | — | Redis password |
|
||||
| `--prefix` | `turnstone` | Redis key prefix |
|
||||
|
||||
### Output
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--metrics-file` | — | Write JSON report to file |
|
||||
| `--log-level` | `INFO` | Log verbosity |
|
||||
|
||||
## Example: Load Testing
|
||||
|
||||
```bash
|
||||
# 100 nodes, high throughput, 2 minutes
|
||||
turnstone-sim --nodes 100 --scenario steady --duration 120 --mps 50
|
||||
|
||||
# Burst of 500 messages across 50 nodes
|
||||
turnstone-sim --nodes 50 --scenario burst --burst-size 500 --duration 60
|
||||
|
||||
# Node failure resilience (kill 2 nodes every 10 seconds)
|
||||
turnstone-sim --nodes 20 --scenario node_failure --duration 120 \
|
||||
--node-kill-interval 10 --node-kill-count 2
|
||||
|
||||
# Fast simulation (low latency, no failures)
|
||||
turnstone-sim --nodes 10 --scenario steady --duration 30 \
|
||||
--llm-latency 0.1 --tool-latency 0.05 --tool-failure-rate 0 --mps 10
|
||||
```
|
||||
|
||||
## Metrics Report
|
||||
|
||||
The simulator prints a summary at completion:
|
||||
|
||||
```
|
||||
============================================================
|
||||
SIMULATION REPORT
|
||||
============================================================
|
||||
Scenario: steady
|
||||
Nodes: 100
|
||||
Duration: 60.2s
|
||||
Total turns: 295
|
||||
Total errors: 5
|
||||
Node kills: 0
|
||||
------------------------------------------------------------
|
||||
THROUGHPUT
|
||||
Messages/sec: 4.97
|
||||
Turns/sec: 4.89
|
||||
------------------------------------------------------------
|
||||
LATENCY (seconds)
|
||||
p50: 3.21
|
||||
p90: 5.44
|
||||
p99: 8.12
|
||||
mean: 3.56
|
||||
max: 12.1
|
||||
------------------------------------------------------------
|
||||
UTILIZATION
|
||||
Mean ws/node: 2.3
|
||||
Max ws/node: 8
|
||||
Idle nodes: 12
|
||||
============================================================
|
||||
```
|
||||
|
||||
Use `--metrics-file report.json` to write the full report as JSON.
|
||||
|
||||
## Console Integration
|
||||
|
||||
The simulator's nodes appear in `turnstone-console` exactly like real nodes. Run them together to see the dashboard populate with simulated workstreams:
|
||||
|
||||
```bash
|
||||
# Terminal 1: start Redis and console
|
||||
docker compose up redis console
|
||||
|
||||
# Terminal 2: run simulator
|
||||
docker compose --profile sim up sim
|
||||
```
|
||||
|
||||
Or all at once:
|
||||
|
||||
```bash
|
||||
SIM_NODES=50 SIM_DURATION=120 docker compose --profile sim up redis console sim
|
||||
```
|
||||
|
||||
Open http://localhost:8090 to see simulated nodes, workstream states, token counts, and load bars updating in real time.
|
||||
|
||||
## Architecture
|
||||
|
||||
> See also: [Simulator Architecture diagram](diagrams/png/10-simulator-architecture.png)
|
||||
|
||||
```
|
||||
turnstone/sim/
|
||||
├── __init__.py # Public API: SimCluster, SimConfig
|
||||
├── config.py # SimConfig — all simulation parameters
|
||||
├── engine.py # SimEngine — LLM + tool execution simulation
|
||||
├── node.py # SimNode + SimWorkstream — protocol-compatible node
|
||||
├── cluster.py # SimCluster + InboundDispatcher + PooledBroker
|
||||
├── scenario.py # 5 scenario classes
|
||||
├── metrics.py # MetricsCollector — latency, throughput, utilization
|
||||
└── cli.py # CLI entry point
|
||||
```
|
||||
|
||||
**Key design:** The `InboundDispatcher` batches ~50 node queues into a single Redis `BLPOP` call, keeping connection count bounded at ~20 regardless of node count. All nodes share a single `ConnectionPool(max_connections=64)`.
|
||||
|
||||
## Programmatic Use
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from turnstone.sim import SimCluster, SimConfig
|
||||
|
||||
async def main():
|
||||
config = SimConfig(
|
||||
num_nodes=10,
|
||||
scenario="steady",
|
||||
duration=30,
|
||||
messages_per_second=2.0,
|
||||
llm_latency_mean=0.5,
|
||||
)
|
||||
cluster = SimCluster(config)
|
||||
await cluster.start()
|
||||
await cluster.run_scenario()
|
||||
print(cluster.report())
|
||||
await cluster.stop()
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
+6
-13
@@ -12,7 +12,7 @@ docker compose -f compose.yaml -f deploy/docker-compose.tls.yml up
|
||||
```
|
||||
|
||||
This:
|
||||
1. Bootstraps an internal CA and issues certs for Redis/PostgreSQL
|
||||
1. Bootstraps an internal CA and issues certs for PostgreSQL
|
||||
2. Starts the console with TLS enabled (internal CA + ACME server)
|
||||
3. Server nodes auto-provision certs via the console's ACME endpoint
|
||||
4. All inter-service communication uses mTLS
|
||||
@@ -30,9 +30,9 @@ Console (CA + ACME Server)
|
||||
| ACME protocol (auto-approve, no challenge validation)
|
||||
+-----------+-----------+
|
||||
| | |
|
||||
Server(s) Bridge Channel GW
|
||||
(auto-cert (mTLS (mTLS
|
||||
+ renewal) client) client)
|
||||
Server(s) Channel GW
|
||||
(auto-cert (mTLS
|
||||
+ renewal) client)
|
||||
```
|
||||
|
||||
**Two cert paths on the console:**
|
||||
@@ -57,12 +57,6 @@ Console (CA + ACME Server)
|
||||
These are needed before storage is available:
|
||||
|
||||
```toml
|
||||
[redis]
|
||||
tls = false
|
||||
tls_ca = "" # path to CA cert
|
||||
tls_cert = "" # path to client cert
|
||||
tls_key = "" # path to client key
|
||||
|
||||
[database]
|
||||
sslmode = "prefer" # disable, allow, prefer, require, verify-full
|
||||
sslrootcert = "" # path to CA cert
|
||||
@@ -89,12 +83,11 @@ sslkey = "" # path to client key
|
||||
Create a CA and infrastructure certs without a running console:
|
||||
|
||||
```bash
|
||||
# Bootstrap CA + Redis + PostgreSQL certs
|
||||
turnstone-admin tls-bootstrap --out /certs --issue redis --issue postgres
|
||||
# Bootstrap CA + PostgreSQL certs
|
||||
turnstone-admin tls-bootstrap --out /certs --issue postgres
|
||||
|
||||
# Output:
|
||||
# /certs/ca.pem (CA root certificate)
|
||||
# /certs/certs/redis/ (Redis cert + key)
|
||||
# /certs/certs/postgres/ (PostgreSQL cert + key)
|
||||
```
|
||||
|
||||
|
||||
+33
-11
@@ -1,6 +1,6 @@
|
||||
# Tools Reference
|
||||
|
||||
turnstone exposes 18 built-in tools plus any number of external MCP tools to the
|
||||
turnstone exposes 19 built-in tools plus any number of external MCP tools to the
|
||||
LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON
|
||||
files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`.
|
||||
MCP tools are discovered from configured MCP servers at startup by
|
||||
@@ -46,12 +46,12 @@ schema plus turnstone-specific metadata keys:
|
||||
|
||||
| Name | Description |
|
||||
|---------------------|-------------|
|
||||
| `TOOLS` | All 17 tool definitions (sent to the model). |
|
||||
| `TOOLS` | All 19 tool definitions (sent to the model). |
|
||||
| `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. |
|
||||
| `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. |
|
||||
| `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. |
|
||||
| `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 17 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 19 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. |
|
||||
|
||||
---
|
||||
@@ -69,7 +69,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools
|
||||
- Parses the JSON arguments (with fallback for malformed JSON).
|
||||
- If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string
|
||||
to the correct parameter.
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 19
|
||||
built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and
|
||||
the generic `_prepare_mcp_tool()` handler for MCP tools.
|
||||
- Validates arguments and builds a preview dict containing:
|
||||
@@ -188,8 +188,10 @@ Execute a bash command and return stdout + stderr.
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `command` | string | yes | The bash command to execute. |
|
||||
| `timeout` | integer | no | Timeout in seconds (1-600). Omit to use the global `tools.timeout` setting (typically 120s). |
|
||||
| `stop_on_error` | boolean | no | Enable `set -e` so the script exits on the first command failure. Default false. |
|
||||
|
||||
- **What it does**: Runs the command in a subprocess with a configurable timeout (default 120s). Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`). Environment variables containing secrets are scrubbed (`*_KEY`, `*_SECRET`, `*_TOKEN`, etc.).
|
||||
- **What it does**: Runs the command in a subprocess with a configurable timeout. Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`). Environment variables containing secrets are scrubbed (`*_KEY`, `*_SECRET`, `*_TOKEN`, etc.).
|
||||
- **Output format**: Stdout is returned directly. Stderr lines are prefixed with `[stderr]` so the model can distinguish them. When the command itself redirects stderr to stdout (`2>&1`), no prefix is added. Output exceeding 256KB is truncated (head + tail preserved, middle replaced with a truncation notice).
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only (not available to plan sub-agents).
|
||||
@@ -222,8 +224,9 @@ Write content to a file, creating it if needed.
|
||||
|-----------|--------|----------|-------------|
|
||||
| `path` | string | yes | Absolute or relative file path. |
|
||||
| `content` | string | yes | The full file content to write. |
|
||||
| `mode` | string | no | `"overwrite"` (default) replaces the file. `"append"` adds content to the end. |
|
||||
|
||||
- **What it does**: Creates or overwrites the file at the given path. Parent directories are created as needed.
|
||||
- **What it does**: Creates or overwrites (or appends to) the file at the given path. Parent directories are created as needed.
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only.
|
||||
|
||||
@@ -239,17 +242,36 @@ Replace exact strings in a file, or apply multiple replacements atomically.
|
||||
| `old_string` | string | no* | The exact text to find and replace. |
|
||||
| `new_string` | string | no* | The replacement text. |
|
||||
| `near_line` | integer | no | Disambiguate when `old_string` matches multiple locations. |
|
||||
| `edits` | array | no* | Multiple replacements to apply atomically (see below). |
|
||||
| `edits` | array | no* | Multiple replacements to apply atomically (see below). |
|
||||
| `replace_all` | boolean | no | Replace ALL occurrences of `old_string`. Cannot combine with `near_line` or `edits`. |
|
||||
|
||||
\* Provide either `old_string`+`new_string` (single edit) or `edits` array (batch), not both.
|
||||
|
||||
- **What it does**: Finds `old_string` in the file and replaces it with `new_string`. Fails if the string is not found or matches multiple locations (unless `near_line` is provided to pick the nearest match). Requires a prior `read_file` call on the same path.
|
||||
- **What it does**: Finds `old_string` in the file and replaces it with `new_string`. Fails if the string is not found or matches multiple locations (unless `near_line` or `replace_all` is provided). Requires a prior `read_file` or `diff_file` call on the same path.
|
||||
- **Batch mode**: The `edits` array accepts multiple `{old_string, new_string, near_line?}` entries applied atomically. All edits are validated before any are applied. Overlapping edits (two entries targeting the same text region) are rejected. Edits are applied in reverse file-position order so character offsets stay stable.
|
||||
- **Replace-all mode**: When `replace_all` is true, all occurrences are replaced via `str.replace()`. The approval preview shows the occurrence count.
|
||||
- **Auto-approve**: No -- requires user confirmation.
|
||||
- **Agent availability**: `task_agent` only.
|
||||
|
||||
---
|
||||
|
||||
### diff_file
|
||||
|
||||
Show a unified diff between two files, or between a file and a provided string.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------------|---------|----------|-------------|
|
||||
| `path_a` | string | yes | Path to the first file. |
|
||||
| `path_b` | string | no | Path to the second file. Mutually exclusive with `content_b`. |
|
||||
| `content_b` | string | no | String content to compare against `path_a`. Mutually exclusive with `path_b`. |
|
||||
| `context_lines` | integer | no | Number of context lines around changes (default 3, max 20). |
|
||||
|
||||
- **What it does**: Returns unified diff output using Python's `difflib`. Binary files (containing null bytes) are rejected with a clear error. Files read through `diff_file` satisfy `edit_file`'s read guard — you can diff then edit without a separate `read_file` call. Large diffs are streamed with early cutoff at the tool truncation limit.
|
||||
- **Auto-approve**: Yes (read-only).
|
||||
- **Agent availability**: `agent` and `task_agent`.
|
||||
|
||||
---
|
||||
|
||||
### search
|
||||
|
||||
Search file contents for a regex pattern.
|
||||
@@ -607,7 +629,7 @@ CLI flags override the config file:
|
||||
directly.
|
||||
|
||||
2. **Partitioning**: When active, tools are split into two sets:
|
||||
- **Always-on** -- the 17 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
- **Always-on** -- the 19 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
These are always visible to the model.
|
||||
- **Deferred** -- all MCP tools. These are not sent in the tool list unless
|
||||
the model searches for them.
|
||||
@@ -650,7 +672,7 @@ MCP-compatible service.
|
||||
3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI
|
||||
function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`.
|
||||
|
||||
4. **Merging**: MCP tools are appended after the 17 built-in tools via
|
||||
4. **Merging**: MCP tools are appended after the 19 built-in tools via
|
||||
`merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority.
|
||||
When dynamic tool search is active, MCP tools are deferred rather than directly
|
||||
visible -- the model discovers them via search as needed (see
|
||||
@@ -668,7 +690,7 @@ that external tools are read-only. However, global overrides such as
|
||||
`--skip-permissions` will auto-approve all tools, including MCP tools. The
|
||||
interactive "Always" button adds specific tool types to the per-tool auto-approve
|
||||
set. The web UI and server use `approval_label` for MCP tools, giving
|
||||
per-prompt/per-resource granularity. The CLI and bridge use `func_name`, which
|
||||
per-prompt/per-resource granularity. The CLI uses `func_name`, which
|
||||
gives per-tool-type granularity (e.g., all `use_prompt` calls).
|
||||
|
||||
### Sub-agent availability
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# MCP Cluster Ops
|
||||
|
||||
An MCP server that exposes tools for executing commands across a [Turnstone](https://github.com/turnstonelabs/turnstone) cluster. Serves as a reference implementation for both MCP server patterns and Turnstone MQ client SDK usage.
|
||||
An MCP server that exposes tools for executing commands across a [Turnstone](https://github.com/turnstonelabs/turnstone) cluster. Serves as a reference implementation for both MCP server patterns and Turnstone SDK usage.
|
||||
|
||||
## How it works
|
||||
|
||||
This server uses Turnstone's MQ client (`TurnstoneClient`) to dispatch shell commands to specific nodes via Redis. Remote agents execute the command and the raw bash output is captured directly from the `ToolResultEvent` stream — bypassing the costly "agent reads output → re-generates output as completion tokens" round-trip.
|
||||
This server uses Turnstone's SDK client (`TurnstoneServer`) to dispatch shell commands to specific nodes via HTTP. Remote agents execute the command and the raw bash output is captured directly from the `ToolResultEvent` stream — bypassing the costly "agent reads output → re-generates output as completion tokens" round-trip.
|
||||
|
||||
Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time is bounded by the slowest node rather than the sum.
|
||||
|
||||
@@ -19,8 +19,7 @@ Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time i
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A running Turnstone cluster (at least one `turnstone-server` + `turnstone-bridge`)
|
||||
- Redis accessible from wherever this MCP server runs
|
||||
- A running Turnstone cluster (at least one `turnstone-server`)
|
||||
- Python 3.11+
|
||||
|
||||
## Installation
|
||||
@@ -28,10 +27,6 @@ Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time i
|
||||
```bash
|
||||
# From the turnstone repo root:
|
||||
pip install -e ./examples/mcp-cluster-ops
|
||||
|
||||
# Or install turnstone with MQ support first, then the example:
|
||||
pip install -e ".[mq]"
|
||||
pip install -e ./examples/mcp-cluster-ops
|
||||
```
|
||||
|
||||
## Configuration
|
||||
@@ -40,9 +35,8 @@ pip install -e ./examples/mcp-cluster-ops
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `REDIS_HOST` | `localhost` | Redis host |
|
||||
| `REDIS_PORT` | `6379` | Redis port |
|
||||
| `REDIS_PASSWORD` | _(none)_ | Redis password (use env vars, not config files) |
|
||||
| `TURNSTONE_SERVER_URL` | `http://localhost:8080` | Server URL |
|
||||
| `TURNSTONE_API_TOKEN` | _(none)_ | API token for authentication |
|
||||
| `MCP_CLUSTER_OPS_TIMEOUT` | `120` | Default command timeout (seconds, clamped 5-3600) |
|
||||
| `MCP_CLUSTER_OPS_MAX_OUTPUT` | `8192` | Max output bytes per node (0 = unlimited) |
|
||||
| `MCP_CLUSTER_OPS_MAX_NODES` | `32` | Max concurrent node dispatches |
|
||||
@@ -57,7 +51,7 @@ pip install -e ./examples/mcp-cluster-ops
|
||||
command = "mcp-cluster-ops"
|
||||
|
||||
[mcp.servers.cluster-ops.env]
|
||||
REDIS_HOST = "redis.example.com"
|
||||
TURNSTONE_SERVER_URL = "http://turnstone.example.com:8080"
|
||||
```
|
||||
|
||||
**JSON** (via `--mcp-config`):
|
||||
@@ -68,7 +62,7 @@ REDIS_HOST = "redis.example.com"
|
||||
"cluster-ops": {
|
||||
"command": "mcp-cluster-ops",
|
||||
"env": {
|
||||
"REDIS_HOST": "redis.example.com"
|
||||
"TURNSTONE_SERVER_URL": "http://turnstone.example.com:8080"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -90,10 +84,6 @@ node-2: /dev/sda1 500G 410G 90G 82% /
|
||||
node-3: /dev/sda1 1.0T 200G 800G 20% /
|
||||
```
|
||||
|
||||
## Why MQ client instead of HTTP SDK?
|
||||
|
||||
The HTTP SDK (`TurnstoneServer`) talks to a single server instance. The MQ client (`TurnstoneClient`) routes through Redis with `target_node` support, which is the entire point of cross-node cluster operations.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
**This MCP server grants the calling agent shell access to cluster nodes.**
|
||||
@@ -104,8 +94,8 @@ The HTTP SDK (`TurnstoneServer`) talks to a single server instance. The MQ clien
|
||||
is returned through the MCP tool result and becomes part of the LLM context.
|
||||
- The security boundary is at the MCP host layer -- use Turnstone's tool
|
||||
policy system to restrict which agents can invoke these tools.
|
||||
- Set `REDIS_PASSWORD` via your environment or a secrets manager -- avoid
|
||||
hardcoding passwords in config files.
|
||||
- Set `TURNSTONE_API_TOKEN` via your environment or a secrets manager -- avoid
|
||||
hardcoding tokens in config files.
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""MCP server for Turnstone cluster operations.
|
||||
|
||||
Exposes tools to execute commands on specific nodes in a Turnstone cluster.
|
||||
Uses the MQ client (``TurnstoneClient``) for direct node targeting via Redis.
|
||||
Uses the SDK client (``TurnstoneServer``) for direct node targeting via HTTP.
|
||||
|
||||
Usage::
|
||||
|
||||
@@ -14,22 +14,20 @@ Configure in ``~/.config/turnstone/config.toml``::
|
||||
command = "mcp-cluster-ops"
|
||||
|
||||
[mcp.servers.cluster-ops.env]
|
||||
REDIS_HOST = "redis.example.com"
|
||||
TURNSTONE_SERVER_URL = "http://localhost:8080"
|
||||
|
||||
Environment variables
|
||||
---------------------
|
||||
REDIS_HOST Redis host (default: localhost)
|
||||
REDIS_PORT Redis port (default: 6379)
|
||||
REDIS_PASSWORD Redis password (default: none)
|
||||
TURNSTONE_SERVER_URL Server URL (default: http://localhost:8080)
|
||||
TURNSTONE_API_TOKEN API token for authentication (default: none)
|
||||
MCP_CLUSTER_OPS_TIMEOUT Default command timeout in seconds (default: 120)
|
||||
MCP_CLUSTER_OPS_MAX_OUTPUT Max output bytes per node (default: 8192, 0=unlimited)
|
||||
|
||||
Performance notes
|
||||
-----------------
|
||||
Remote agents are told to reply with only "ok" or "failed" — the raw bash
|
||||
output is captured directly from the ToolResultEvent that already flows
|
||||
through Redis, bypassing the costly "agent reads output then re-generates
|
||||
output as completion tokens" round-trip.
|
||||
output is captured directly from the ToolResultEvent, bypassing the costly
|
||||
"agent reads output then re-generates output as completion tokens" round-trip.
|
||||
|
||||
All multi-node dispatches run in parallel via ``asyncio.gather`` so total
|
||||
wall time is bounded by the slowest node, not the sum of all nodes.
|
||||
@@ -45,7 +43,7 @@ from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from mcp.server.fastmcp import Context, FastMCP
|
||||
from turnstone.mq.client import TurnResult, TurnstoneClient
|
||||
from turnstone.sdk import TurnResult, TurnstoneServer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
@@ -68,19 +66,14 @@ _MAX_TIMEOUT = 3600
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _redis_kwargs() -> dict[str, Any]:
|
||||
"""Build Redis connection kwargs from environment variables.
|
||||
|
||||
Follows the same env var convention as ``turnstone.mq.broker.add_redis_args``:
|
||||
``REDIS_HOST``, ``REDIS_PORT``, ``REDIS_PASSWORD``.
|
||||
"""
|
||||
kwargs: dict[str, Any] = {"host": os.environ.get("REDIS_HOST", "localhost")}
|
||||
port = os.environ.get("REDIS_PORT")
|
||||
if port is not None:
|
||||
kwargs["port"] = int(port)
|
||||
password = os.environ.get("REDIS_PASSWORD")
|
||||
if password:
|
||||
kwargs["password"] = password
|
||||
def _server_kwargs() -> dict[str, Any]:
|
||||
"""Build TurnstoneServer connection kwargs from environment variables."""
|
||||
kwargs: dict[str, Any] = {
|
||||
"base_url": os.environ.get("TURNSTONE_SERVER_URL", "http://localhost:8080"),
|
||||
}
|
||||
token = os.environ.get("TURNSTONE_API_TOKEN")
|
||||
if token:
|
||||
kwargs["token"] = token
|
||||
return kwargs
|
||||
|
||||
|
||||
@@ -172,12 +165,12 @@ def _format_node_result(
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core dispatch functions (testable with mocked TurnstoneClient)
|
||||
# Core dispatch functions (testable with mocked TurnstoneServer)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _exec_on_node_sync(
|
||||
redis_kw: dict[str, Any],
|
||||
server_kw: dict[str, Any],
|
||||
node_id: str,
|
||||
command: str,
|
||||
timeout: float,
|
||||
@@ -185,11 +178,11 @@ def _exec_on_node_sync(
|
||||
"""Dispatch *command* to *node_id* and block until complete.
|
||||
|
||||
Runs inside ``asyncio.to_thread`` so it does not block the event loop.
|
||||
Each call creates its own ``TurnstoneClient`` to avoid Redis pub/sub
|
||||
subscription conflicts between concurrent dispatches.
|
||||
Each call creates its own ``TurnstoneServer`` client to avoid state
|
||||
conflicts between concurrent dispatches.
|
||||
"""
|
||||
prompt = _exec_prompt(command)
|
||||
with TurnstoneClient(**redis_kw) as client:
|
||||
with TurnstoneServer(**server_kw) as client:
|
||||
result = client.send_and_wait(
|
||||
message=prompt,
|
||||
target_node=node_id,
|
||||
@@ -200,7 +193,7 @@ def _exec_on_node_sync(
|
||||
|
||||
|
||||
async def _dispatch_parallel(
|
||||
redis_kw: dict[str, Any],
|
||||
server_kw: dict[str, Any],
|
||||
node_ids: list[str],
|
||||
command: str,
|
||||
timeout: float,
|
||||
@@ -211,7 +204,7 @@ async def _dispatch_parallel(
|
||||
Total wall time is bounded by the slowest node.
|
||||
"""
|
||||
tasks = [
|
||||
asyncio.to_thread(_exec_on_node_sync, redis_kw, nid, command, timeout) for nid in node_ids
|
||||
asyncio.to_thread(_exec_on_node_sync, server_kw, nid, command, timeout) for nid in node_ids
|
||||
]
|
||||
outcomes = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
@@ -227,16 +220,16 @@ async def _dispatch_parallel(
|
||||
return results
|
||||
|
||||
|
||||
def _list_nodes_sync(redis_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _list_nodes_sync(server_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""List active cluster nodes (blocking)."""
|
||||
with TurnstoneClient(**redis_kw) as client:
|
||||
with TurnstoneServer(**server_kw) as client:
|
||||
nodes: list[dict[str, Any]] = client.list_nodes()
|
||||
return nodes
|
||||
|
||||
|
||||
async def _list_nodes_impl(redis_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
async def _list_nodes_impl(server_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""List active cluster nodes."""
|
||||
return await asyncio.to_thread(_list_nodes_sync, redis_kw)
|
||||
return await asyncio.to_thread(_list_nodes_sync, server_kw)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -246,9 +239,9 @@ async def _list_nodes_impl(redis_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(server: FastMCP[dict[str, Any]]) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Lifespan context — stores Redis kwargs for tool handlers."""
|
||||
kw = _redis_kwargs()
|
||||
yield {"redis_kwargs": kw}
|
||||
"""Lifespan context — stores server connection kwargs for tool handlers."""
|
||||
kw = _server_kwargs()
|
||||
yield {"server_kwargs": kw}
|
||||
|
||||
|
||||
mcp = FastMCP(
|
||||
@@ -270,8 +263,8 @@ async def list_nodes(ctx: Context[Any, Any, Any]) -> str:
|
||||
Call this before dispatching work to discover available node IDs.
|
||||
Returns a JSON array of node metadata objects.
|
||||
"""
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
nodes = await _list_nodes_impl(redis_kw)
|
||||
server_kw: dict[str, Any] = ctx.request_context.lifespan_context["server_kwargs"]
|
||||
nodes = await _list_nodes_impl(server_kw)
|
||||
return json.dumps(nodes, indent=2)
|
||||
|
||||
|
||||
@@ -298,12 +291,12 @@ async def run_on_node(
|
||||
if cmd_err:
|
||||
return json.dumps({"error": cmd_err})
|
||||
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
server_kw: dict[str, Any] = ctx.request_context.lifespan_context["server_kwargs"]
|
||||
max_output = _DEFAULT_MAX_OUTPUT
|
||||
|
||||
log.info("run_on_node node=%s cmd=%r", node_id, command)
|
||||
_, result = await asyncio.to_thread(
|
||||
_exec_on_node_sync, redis_kw, node_id, command, _clamp_timeout(timeout)
|
||||
_exec_on_node_sync, server_kw, node_id, command, _clamp_timeout(timeout)
|
||||
)
|
||||
formatted = _format_node_result(node_id, result, max_output)
|
||||
return json.dumps(formatted, indent=2)
|
||||
@@ -330,7 +323,7 @@ async def run_on_nodes(
|
||||
if cmd_err:
|
||||
return json.dumps({"error": cmd_err})
|
||||
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
server_kw: dict[str, Any] = ctx.request_context.lifespan_context["server_kwargs"]
|
||||
max_output = _DEFAULT_MAX_OUTPUT
|
||||
|
||||
clean_ids = list(dict.fromkeys(nid.strip() for nid in node_ids if nid.strip()))
|
||||
@@ -343,7 +336,7 @@ async def run_on_nodes(
|
||||
|
||||
log.info("run_on_nodes nodes=%s cmd=%r", clean_ids, command)
|
||||
results = await _dispatch_parallel(
|
||||
redis_kw, clean_ids, command, _clamp_timeout(timeout), max_output
|
||||
server_kw, clean_ids, command, _clamp_timeout(timeout), max_output
|
||||
)
|
||||
return json.dumps(results, indent=2)
|
||||
|
||||
@@ -368,10 +361,10 @@ async def run_on_all_nodes(
|
||||
if cmd_err:
|
||||
return json.dumps({"error": cmd_err})
|
||||
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
server_kw: dict[str, Any] = ctx.request_context.lifespan_context["server_kwargs"]
|
||||
max_output = _DEFAULT_MAX_OUTPUT
|
||||
|
||||
nodes = await _list_nodes_impl(redis_kw)
|
||||
nodes = await _list_nodes_impl(server_kw)
|
||||
if not nodes:
|
||||
return json.dumps({"error": "No active nodes found in cluster"})
|
||||
|
||||
@@ -388,7 +381,7 @@ async def run_on_all_nodes(
|
||||
)
|
||||
log.info("run_on_all_nodes nodes=%s cmd=%r", node_ids, command)
|
||||
results = await _dispatch_parallel(
|
||||
redis_kw, node_ids, command, _clamp_timeout(timeout), max_output
|
||||
server_kw, node_ids, command, _clamp_timeout(timeout), max_output
|
||||
)
|
||||
return json.dumps(results, indent=2)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ description = "MCP server for Turnstone cluster operations — reference impleme
|
||||
requires-python = ">=3.11"
|
||||
license = "BUSL-1.1"
|
||||
dependencies = [
|
||||
"turnstone[mq]",
|
||||
"turnstone",
|
||||
"mcp>=1.6",
|
||||
]
|
||||
|
||||
@@ -18,7 +18,7 @@ mcp-cluster-ops = "mcp_cluster_ops.server:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest>=9.0"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14", "types-redis>=4.6"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from turnstone.mq.client import TurnResult
|
||||
from turnstone.sdk import TurnResult
|
||||
|
||||
from mcp_cluster_ops.server import (
|
||||
_clamp_timeout,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for MCP tool handlers with mocked TurnstoneClient."""
|
||||
"""Tests for MCP tool handlers with mocked TurnstoneServer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -6,7 +6,7 @@ import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from turnstone.mq.client import TurnResult
|
||||
from turnstone.sdk import TurnResult
|
||||
|
||||
from mcp_cluster_ops.server import (
|
||||
_dispatch_parallel,
|
||||
@@ -22,7 +22,7 @@ from mcp_cluster_ops.server import (
|
||||
class TestListNodesImpl:
|
||||
def test_returns_nodes(self):
|
||||
nodes = [{"node_id": "a", "model": "gpt-5"}, {"node_id": "b", "model": "gpt-5"}]
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
with patch("mcp_cluster_ops.server.TurnstoneServer") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.list_nodes.return_value = nodes
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
@@ -32,7 +32,7 @@ class TestListNodesImpl:
|
||||
assert result == nodes
|
||||
|
||||
def test_empty_cluster(self):
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
with patch("mcp_cluster_ops.server.TurnstoneServer") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.list_nodes.return_value = []
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
@@ -52,7 +52,7 @@ class TestExecOnNodeSync:
|
||||
turn_result = TurnResult(
|
||||
tool_results=[("bash", "hello world")],
|
||||
)
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
with patch("mcp_cluster_ops.server.TurnstoneServer") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.send_and_wait.return_value = turn_result
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
@@ -70,7 +70,7 @@ class TestExecOnNodeSync:
|
||||
|
||||
def test_timeout(self):
|
||||
turn_result = TurnResult(timed_out=True)
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
with patch("mcp_cluster_ops.server.TurnstoneServer") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.send_and_wait.return_value = turn_result
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
@@ -88,7 +88,7 @@ class TestExecOnNodeSync:
|
||||
|
||||
class TestDispatchParallel:
|
||||
def test_parallel_success(self):
|
||||
def fake_exec(redis_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
def fake_exec(server_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
return (node_id, TurnResult(tool_results=[("bash", f"output-{node_id}")]))
|
||||
|
||||
with patch("mcp_cluster_ops.server._exec_on_node_sync", side_effect=fake_exec):
|
||||
@@ -108,9 +108,9 @@ class TestDispatchParallel:
|
||||
assert outputs["b"] == "output-b"
|
||||
|
||||
def test_partial_failure(self):
|
||||
def fake_exec(redis_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
def fake_exec(server_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
if node_id == "bad":
|
||||
raise ConnectionError("Redis down")
|
||||
raise ConnectionError("connection refused")
|
||||
return (node_id, TurnResult(tool_results=[("bash", "ok")]))
|
||||
|
||||
with patch("mcp_cluster_ops.server._exec_on_node_sync", side_effect=fake_exec):
|
||||
@@ -128,10 +128,10 @@ class TestDispatchParallel:
|
||||
bad = next(r for r in results if r["node"] == "bad")
|
||||
assert good["ok"] is True
|
||||
assert bad["ok"] is False
|
||||
assert "Redis down" in bad["error"]
|
||||
assert "connection refused" in bad["error"]
|
||||
|
||||
def test_all_fail(self):
|
||||
def fake_exec(redis_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
def fake_exec(server_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
raise RuntimeError(f"fail-{node_id}")
|
||||
|
||||
with patch("mcp_cluster_ops.server._exec_on_node_sync", side_effect=fake_exec):
|
||||
|
||||
+5
-9
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "0.9.3"
|
||||
version = "0.9.9"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
@@ -45,25 +45,21 @@ Issues = "https://github.com/turnstonelabs/turnstone/issues"
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest>=9.0", "pytest-cov>=6.0", "croniter>=3.0"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14", "types-redis>=4.6"]
|
||||
mq = ["redis>=7.2"]
|
||||
console = ["redis>=7.2", "croniter>=3.0"]
|
||||
sim = ["redis>=7.2"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14"]
|
||||
console = ["croniter>=3.0"]
|
||||
anthropic = ["anthropic>=0.39"]
|
||||
postgres = ["psycopg[binary]>=3.2"]
|
||||
ddg = ["ddgs>=9.0"]
|
||||
discord = ["discord.py>=2.4", "redis>=7.2"]
|
||||
discord = ["discord.py>=2.4"]
|
||||
tls = ["lacme>=1.0.4"]
|
||||
sandbox = ["sympy>=1.13", "numpy>=2.0", "scipy>=1.14", "pytest>=9.0"]
|
||||
all = ["turnstone[mq,console,sim,anthropic,postgres,discord,ddg,tls,sandbox]"]
|
||||
all = ["turnstone[console,anthropic,postgres,discord,ddg,tls,sandbox]"]
|
||||
|
||||
[project.scripts]
|
||||
turnstone = "turnstone.cli:main"
|
||||
turnstone-eval = "turnstone.eval:main"
|
||||
turnstone-server = "turnstone.server:main"
|
||||
turnstone-bridge = "turnstone.mq.bridge:main"
|
||||
turnstone-console = "turnstone.console.server:main"
|
||||
turnstone-sim = "turnstone.sim.cli:main"
|
||||
turnstone-admin = "turnstone.admin:main"
|
||||
turnstone-channel = "turnstone.channels.cli:main"
|
||||
turnstone-bootstrap = "turnstone.bootstrap:main"
|
||||
|
||||
@@ -83,6 +83,8 @@ export interface StatusEvent {
|
||||
effort: string;
|
||||
cache_creation_tokens?: number;
|
||||
cache_read_tokens?: number;
|
||||
tool_calls_this_turn?: number;
|
||||
turn_count?: number;
|
||||
}
|
||||
|
||||
export interface PlanReviewEvent {
|
||||
|
||||
@@ -84,7 +84,6 @@ class TestConsoleVersioning:
|
||||
}
|
||||
app = create_app(
|
||||
collector=collector,
|
||||
broker=MagicMock(),
|
||||
auth_config=AuthConfig(),
|
||||
)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
"""Tests for turnstone.mq.async_broker.AsyncRedisBroker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.mq.async_broker import AsyncRedisBroker
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def broker() -> AsyncRedisBroker:
|
||||
return AsyncRedisBroker(host="localhost", port=6379, db=0, prefix="test", response_ttl=120)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis() -> AsyncMock:
|
||||
"""Return a mock Redis client with common async methods."""
|
||||
r = AsyncMock()
|
||||
r.rpush = AsyncMock()
|
||||
r.publish = AsyncMock()
|
||||
r.expire = AsyncMock()
|
||||
r.get = AsyncMock(return_value=None)
|
||||
r.set = AsyncMock()
|
||||
r.delete = AsyncMock()
|
||||
r.blpop = AsyncMock(return_value=None)
|
||||
ps = AsyncMock()
|
||||
ps.subscribe = AsyncMock()
|
||||
ps.unsubscribe = AsyncMock()
|
||||
ps.close = AsyncMock()
|
||||
ps.get_message = AsyncMock(return_value=None)
|
||||
r.pubsub = MagicMock(return_value=ps)
|
||||
return r
|
||||
|
||||
|
||||
def _inject_redis(broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
"""Inject a mock Redis client into the broker, simulating connect()."""
|
||||
broker._redis = mock_redis
|
||||
broker._pubsub = mock_redis.pubsub()
|
||||
|
||||
|
||||
class TestConstructor:
|
||||
def test_stores_config(self) -> None:
|
||||
b = AsyncRedisBroker(host="h", port=1234, db=2, prefix="pfx", password="pw")
|
||||
assert b._host == "h"
|
||||
assert b._port == 1234
|
||||
assert b._db == 2
|
||||
assert b._prefix == "pfx"
|
||||
assert b._password == "pw"
|
||||
assert b._redis is None
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
b = AsyncRedisBroker()
|
||||
assert b._host == "localhost"
|
||||
assert b._port == 6379
|
||||
assert b._prefix == "turnstone"
|
||||
|
||||
|
||||
class TestConnect:
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_connection(self) -> None:
|
||||
b = AsyncRedisBroker()
|
||||
mock_r = AsyncMock()
|
||||
mock_r.pubsub = MagicMock(return_value=AsyncMock())
|
||||
with patch("redis.asyncio.Redis", return_value=mock_r):
|
||||
await b.connect()
|
||||
assert b._redis is mock_r
|
||||
assert b._pubsub is not None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_connect_idempotent(
|
||||
self, broker: AsyncRedisBroker, mock_redis: AsyncMock
|
||||
) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
old = broker._redis
|
||||
await broker.connect()
|
||||
assert broker._redis is old
|
||||
|
||||
|
||||
class TestPushInbound:
|
||||
@pytest.mark.anyio
|
||||
async def test_shared_queue(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.push_inbound('{"type":"send"}')
|
||||
mock_redis.rpush.assert_awaited_once_with("test:inbound", '{"type":"send"}')
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_per_node_queue(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.push_inbound('{"type":"send"}', node_id="node-1")
|
||||
mock_redis.rpush.assert_awaited_once_with("test:inbound:node-1", '{"type":"send"}')
|
||||
|
||||
|
||||
class TestPublishOutbound:
|
||||
@pytest.mark.anyio
|
||||
async def test_publishes(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.publish_outbound("test:events:global", '{"event":"data"}')
|
||||
mock_redis.publish.assert_awaited_once_with("test:events:global", '{"event":"data"}')
|
||||
|
||||
|
||||
class TestPushResponse:
|
||||
@pytest.mark.anyio
|
||||
async def test_rpush_and_expire(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.push_response("req-123", '{"ok":true}')
|
||||
mock_redis.rpush.assert_awaited_once_with("test:resp:req-123", '{"ok":true}')
|
||||
mock_redis.expire.assert_awaited_once_with("test:resp:req-123", 120)
|
||||
|
||||
|
||||
class TestSubscribe:
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_task(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.subscribe("test:events:global", lambda msg: None)
|
||||
assert "test:events:global" in broker._callbacks
|
||||
assert broker._listener_task is not None
|
||||
assert isinstance(broker._listener_task, asyncio.Task)
|
||||
# Clean up.
|
||||
broker._listener_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await broker._listener_task
|
||||
|
||||
|
||||
class TestUnsubscribe:
|
||||
@pytest.mark.anyio
|
||||
async def test_cancels_task(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.subscribe("test:events:ch", lambda msg: None)
|
||||
assert "test:events:ch" in broker._callbacks
|
||||
await broker.unsubscribe("test:events:ch")
|
||||
assert "test:events:ch" not in broker._callbacks
|
||||
|
||||
|
||||
class TestRoutingPrimitives:
|
||||
@pytest.mark.anyio
|
||||
async def test_get_ws_owner(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
mock_redis.get.return_value = "node-1"
|
||||
result = await broker.get_ws_owner("ws-abc")
|
||||
mock_redis.get.assert_awaited_once_with("test:ws:ws-abc")
|
||||
assert result == "node-1"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_set_ws_owner(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.set_ws_owner("ws-abc", "node-2")
|
||||
mock_redis.set.assert_awaited_once_with("test:ws:ws-abc", "node-2")
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_set_ws_owner_with_ttl(
|
||||
self, broker: AsyncRedisBroker, mock_redis: AsyncMock
|
||||
) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.set_ws_owner("ws-abc", "node-2", ttl=300)
|
||||
mock_redis.set.assert_awaited_once_with("test:ws:ws-abc", "node-2", ex=300)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_del_ws_owner(self, broker: AsyncRedisBroker, mock_redis: AsyncMock) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.del_ws_owner("ws-abc")
|
||||
mock_redis.delete.assert_awaited_once_with("test:ws:ws-abc")
|
||||
|
||||
|
||||
class TestClose:
|
||||
@pytest.mark.anyio
|
||||
async def test_cancels_tasks_and_closes(
|
||||
self, broker: AsyncRedisBroker, mock_redis: AsyncMock
|
||||
) -> None:
|
||||
_inject_redis(broker, mock_redis)
|
||||
await broker.subscribe("ch1", lambda m: None)
|
||||
assert len(broker._callbacks) == 1
|
||||
assert broker._listener_task is not None
|
||||
await broker.close()
|
||||
assert len(broker._callbacks) == 0
|
||||
assert broker._listener_task is None
|
||||
assert broker._redis is None
|
||||
assert broker._pubsub is None
|
||||
@@ -923,7 +923,6 @@ class TestConsoleAuth:
|
||||
|
||||
app = create_app(
|
||||
collector=mock_collector,
|
||||
broker=MagicMock(),
|
||||
auth_config=AuthConfig(
|
||||
enabled=True,
|
||||
tokens={"tok_full": "full", "tok_read": "read"},
|
||||
@@ -1095,7 +1094,6 @@ class TestConsoleLogin:
|
||||
|
||||
app = create_app(
|
||||
collector=mock_collector,
|
||||
broker=MagicMock(),
|
||||
auth_config=AuthConfig(
|
||||
enabled=True,
|
||||
tokens={"tok_full": "full", "tok_read": "read"},
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
"""Tests for bridge event publishing — TurnCompleteEvent on idle transitions."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from turnstone.mq.bridge import Bridge
|
||||
from turnstone.mq.protocol import ContentEvent, StateChangeEvent, TurnCompleteEvent
|
||||
|
||||
|
||||
def _make_bridge():
|
||||
"""Create a Bridge with a mock broker (no Redis or HTTP needed)."""
|
||||
broker = MagicMock()
|
||||
bridge = Bridge(server_url="http://localhost:8080", broker=broker, node_id="test-node")
|
||||
return bridge
|
||||
|
||||
|
||||
class TestIdleTurnComplete:
|
||||
"""TurnCompleteEvent should be emitted on every idle transition."""
|
||||
|
||||
def test_idle_emits_turn_complete_with_correlation_id(self):
|
||||
"""Bridge-initiated turn: TurnCompleteEvent has the correlation_id."""
|
||||
bridge = _make_bridge()
|
||||
bridge._active_sends["ws-1"] = "cid-abc"
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event({"type": "ws_state", "ws_id": "ws-1", "state": "idle"})
|
||||
|
||||
turn_completes = [(ws, ev) for ws, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(turn_completes) == 1
|
||||
ws, ev = turn_completes[0]
|
||||
assert ws == "ws-1"
|
||||
assert ev.correlation_id == "cid-abc"
|
||||
# correlation_id should be removed from _active_sends
|
||||
assert "ws-1" not in bridge._active_sends
|
||||
|
||||
def test_idle_emits_turn_complete_without_correlation_id(self):
|
||||
"""Server-UI-initiated turn: TurnCompleteEvent has empty correlation_id."""
|
||||
bridge = _make_bridge()
|
||||
# No entry in _active_sends for this workstream
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event({"type": "ws_state", "ws_id": "ws-2", "state": "idle"})
|
||||
|
||||
turn_completes = [(ws, ev) for ws, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(turn_completes) == 1
|
||||
ws, ev = turn_completes[0]
|
||||
assert ws == "ws-2"
|
||||
assert ev.correlation_id == ""
|
||||
|
||||
def test_non_idle_state_does_not_emit_turn_complete(self):
|
||||
"""Non-idle state transitions should emit StateChangeEvent but not TurnCompleteEvent."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event({"type": "ws_state", "ws_id": "ws-3", "state": "thinking"})
|
||||
|
||||
state_changes = [ev for _, ev in published if isinstance(ev, StateChangeEvent)]
|
||||
turn_completes = [ev for _, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(state_changes) == 1
|
||||
assert state_changes[0].state == "thinking"
|
||||
assert len(turn_completes) == 0
|
||||
|
||||
|
||||
class TestContentPassthrough:
|
||||
"""Bridge should pass through content from the server's idle SSE event."""
|
||||
|
||||
def test_content_passed_through_in_turn_complete(self):
|
||||
"""Content from idle event should be included in TurnCompleteEvent."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event(
|
||||
{"type": "ws_state", "ws_id": "ws-1", "state": "idle", "content": "Hello world"}
|
||||
)
|
||||
|
||||
turn_completes = [(ws, ev) for ws, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(turn_completes) == 1
|
||||
_, ev = turn_completes[0]
|
||||
assert ev.content == "Hello world"
|
||||
|
||||
def test_content_empty_when_not_in_event(self):
|
||||
"""TurnCompleteEvent.content should be empty when idle event has no content."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event({"type": "ws_state", "ws_id": "ws-1", "state": "idle"})
|
||||
|
||||
turn_completes = [(ws, ev) for ws, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(turn_completes) == 1
|
||||
_, ev = turn_completes[0]
|
||||
assert ev.content == ""
|
||||
|
||||
def test_content_event_still_published(self):
|
||||
"""Content events should still be published to per-ws channel."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_ws_event("ws-1", {"type": "content", "text": "hello"})
|
||||
|
||||
content_events = [(ws, ev) for ws, ev in published if isinstance(ev, ContentEvent)]
|
||||
assert len(content_events) == 1
|
||||
_, ev = content_events[0]
|
||||
assert ev.text == "hello"
|
||||
@@ -1,357 +0,0 @@
|
||||
"""Stress tests for bridge.py threading — race conditions in approval,
|
||||
plan review, and workstream lifecycle.
|
||||
|
||||
Each scenario is run many times (ITERATIONS) with threading.Barrier to
|
||||
maximize timing overlap. Uses mock broker (no Redis) and no HTTP calls.
|
||||
|
||||
Races tested:
|
||||
1. Duplicate approval on SSE reconnect (TOCTOU in _pending_approvals)
|
||||
2. Duplicate plan review on SSE reconnect (TOCTOU in _pending_plan_reviews)
|
||||
3. approve_set stale reference escape during concurrent update
|
||||
4. _running flag visibility across threads on shutdown
|
||||
5. Approval thread exits within bounded time after timeout
|
||||
6. Concurrent approval + workstream close leaves no orphaned state
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from turnstone.mq.bridge import Bridge
|
||||
|
||||
ITERATIONS = 100
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_bridge(**overrides) -> Bridge:
|
||||
"""Create a Bridge with a mock broker (no Redis or HTTP)."""
|
||||
broker = MagicMock()
|
||||
defaults = dict(
|
||||
server_url="http://localhost:8080",
|
||||
broker=broker,
|
||||
node_id="test-node",
|
||||
approval_timeout=1,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
bridge = Bridge(**defaults)
|
||||
# Replace real httpx client with a mock so daemon threads spawned by
|
||||
# _handle_approval / _handle_plan_review don't make real HTTP calls
|
||||
# after the test's patch context exits.
|
||||
bridge._http.close()
|
||||
bridge._http = MagicMock()
|
||||
return bridge
|
||||
|
||||
|
||||
def _approval_items(tool_name: str = "bash") -> list[dict]:
|
||||
return [{"func_name": tool_name, "needs_approval": True, "approval_label": tool_name}]
|
||||
|
||||
|
||||
def _wait_pending_resolved(bridge: Bridge, key: str, attr: str, deadline_s: float = 3.0) -> bool:
|
||||
"""Poll until the pending entry is resolved (tombstone) or absent."""
|
||||
deadline = time.monotonic() + deadline_s
|
||||
while time.monotonic() < deadline:
|
||||
with bridge._lock:
|
||||
entries = getattr(bridge, attr)
|
||||
if key not in entries:
|
||||
return True
|
||||
_, resolved_at = entries[key]
|
||||
if resolved_at > 0:
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 1: Duplicate approval on SSE reconnect
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDuplicateApproval:
|
||||
"""Two threads call _handle_approval for the same ws_id simultaneously.
|
||||
Only one should create a pending entry; the other should be skipped."""
|
||||
|
||||
def test_no_duplicate_approvals(self):
|
||||
sent_count = Counter()
|
||||
|
||||
for _ in range(ITERATIONS):
|
||||
bridge = _make_bridge()
|
||||
bridge._broker.pop_response.return_value = '{"type": "approve", "approved": true}'
|
||||
barrier = threading.Barrier(2, timeout=5)
|
||||
|
||||
def _call_approval(bridge=bridge, barrier=barrier):
|
||||
barrier.wait()
|
||||
bridge._handle_approval("ws-1", {"items": _approval_items()})
|
||||
|
||||
t1 = threading.Thread(target=_call_approval)
|
||||
t2 = threading.Thread(target=_call_approval)
|
||||
with (
|
||||
patch.object(bridge, "_api_approve") as mock_approve,
|
||||
patch.object(bridge, "_publish_ws"),
|
||||
):
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=5)
|
||||
t2.join(timeout=5)
|
||||
assert not t1.is_alive(), "Thread 1 hung"
|
||||
assert not t2.is_alive(), "Thread 2 hung"
|
||||
|
||||
# Wait for spawned _wait_approval threads to resolve
|
||||
_wait_pending_resolved(bridge, "ws-1", "_pending_approvals")
|
||||
|
||||
sent_count[mock_approve.call_count] += 1
|
||||
|
||||
# At most 1 approval should be forwarded per iteration
|
||||
assert sent_count.get(2, 0) == 0, (
|
||||
f"Duplicate approvals sent in {sent_count[2]}/{ITERATIONS} iterations"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 2: Duplicate plan review on SSE reconnect
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDuplicatePlanReview:
|
||||
"""Two threads call _handle_plan_review simultaneously.
|
||||
Only one should create a pending entry."""
|
||||
|
||||
def test_no_duplicate_plan_reviews(self):
|
||||
sent_count = Counter()
|
||||
|
||||
for _ in range(ITERATIONS):
|
||||
bridge = _make_bridge()
|
||||
bridge._broker.pop_response.return_value = (
|
||||
'{"type": "plan_feedback", "feedback": "looks good"}'
|
||||
)
|
||||
barrier = threading.Barrier(2, timeout=5)
|
||||
|
||||
def _call_plan(bridge=bridge, barrier=barrier):
|
||||
barrier.wait()
|
||||
bridge._handle_plan_review("ws-1", {"content": "plan text"})
|
||||
|
||||
t1 = threading.Thread(target=_call_plan)
|
||||
t2 = threading.Thread(target=_call_plan)
|
||||
with patch.object(bridge, "_publish_ws"), patch.object(bridge._http, "post"):
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=5)
|
||||
t2.join(timeout=5)
|
||||
assert not t1.is_alive(), "Thread 1 hung"
|
||||
assert not t2.is_alive(), "Thread 2 hung"
|
||||
|
||||
# Wait for spawned _wait_plan threads to resolve
|
||||
_wait_pending_resolved(bridge, "ws-1", "_pending_plan_reviews")
|
||||
|
||||
sent_count[bridge._http.post.call_count] += 1
|
||||
|
||||
assert sent_count.get(2, 0) == 0, (
|
||||
f"Duplicate plan reviews sent in {sent_count[2]}/{ITERATIONS} iterations"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 3: approve_set stale reference during concurrent update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApproveSetConsistency:
|
||||
"""One thread reads approve_set for auto-approve check while another
|
||||
updates it via _wait_approval 'always' path. The auto-approve
|
||||
decision should be consistent (either all-approved or not)."""
|
||||
|
||||
def test_approve_set_never_partially_visible(self):
|
||||
for _ in range(ITERATIONS):
|
||||
bridge = _make_bridge()
|
||||
with bridge._lock:
|
||||
bridge._ws_approve_tools["ws-1"] = {"read_file", "search"}
|
||||
|
||||
barrier = threading.Barrier(2, timeout=5)
|
||||
results = []
|
||||
|
||||
def _reader(bridge=bridge, barrier=barrier, results=results):
|
||||
barrier.wait()
|
||||
with bridge._lock:
|
||||
snap = bridge._ws_approve_tools.get("ws-1", set()).copy()
|
||||
results.append(snap)
|
||||
|
||||
def _writer(bridge=bridge, barrier=barrier):
|
||||
barrier.wait()
|
||||
with bridge._lock:
|
||||
existing = bridge._ws_approve_tools.get("ws-1", set())
|
||||
bridge._ws_approve_tools["ws-1"] = existing | {"bash", "write_file"}
|
||||
|
||||
t1 = threading.Thread(target=_reader)
|
||||
t2 = threading.Thread(target=_writer)
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=5)
|
||||
t2.join(timeout=5)
|
||||
assert not t1.is_alive(), "Reader hung"
|
||||
assert not t2.is_alive(), "Writer hung"
|
||||
|
||||
snap = results[0]
|
||||
assert snap in (
|
||||
{"read_file", "search"},
|
||||
{"read_file", "search", "bash", "write_file"},
|
||||
), f"Partial set observed: {snap}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 4: _running flag visibility across threads
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRunningFlagVisibility:
|
||||
"""All threads reading _running should see False within a bounded time
|
||||
after the main thread sets it."""
|
||||
|
||||
def test_all_threads_observe_shutdown(self):
|
||||
bridge = _make_bridge()
|
||||
observed_false = threading.Event()
|
||||
threads_running = []
|
||||
|
||||
def _spin_checker():
|
||||
while bridge._running:
|
||||
time.sleep(0.001)
|
||||
observed_false.set()
|
||||
|
||||
for _ in range(5):
|
||||
t = threading.Thread(target=_spin_checker, daemon=True)
|
||||
threads_running.append(t)
|
||||
t.start()
|
||||
|
||||
time.sleep(0.01)
|
||||
bridge._running = False
|
||||
|
||||
for t in threads_running:
|
||||
t.join(timeout=1)
|
||||
assert not t.is_alive(), "Thread did not observe _running=False"
|
||||
|
||||
assert observed_false.is_set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 5: Approval thread exits within bounded time
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApprovalThreadTimeout:
|
||||
"""An approval thread blocked on pop_response should exit within the
|
||||
configured approval_timeout, not hang indefinitely."""
|
||||
|
||||
def test_approval_thread_exits_within_timeout(self):
|
||||
for _ in range(10):
|
||||
bridge = _make_bridge(approval_timeout=0.5)
|
||||
|
||||
def _slow_pop(queue_name, timeout=300):
|
||||
time.sleep(min(timeout, 0.5))
|
||||
return None
|
||||
|
||||
bridge._broker.pop_response.side_effect = _slow_pop
|
||||
|
||||
with patch.object(bridge, "_publish_ws"), patch.object(bridge, "_api_approve"):
|
||||
bridge._handle_approval("ws-1", {"items": _approval_items()})
|
||||
|
||||
# The pending entry should be resolved within the timeout
|
||||
resolved = _wait_pending_resolved(bridge, "ws-1", "_pending_approvals", deadline_s=3.0)
|
||||
assert resolved, "Approval thread did not exit within expected timeout"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 6: Concurrent approval + workstream close
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApprovalDuringClose:
|
||||
"""An approval arriving at the exact same time as a ws_closed event
|
||||
should not leave orphaned state."""
|
||||
|
||||
def test_no_orphaned_pending_after_close(self):
|
||||
for _ in range(ITERATIONS):
|
||||
bridge = _make_bridge(approval_timeout=0.1)
|
||||
bridge._broker.pop_response.return_value = None # timeout
|
||||
|
||||
barrier = threading.Barrier(2, timeout=5)
|
||||
|
||||
def _send_approval(bridge=bridge, barrier=barrier):
|
||||
barrier.wait()
|
||||
with patch.object(bridge, "_publish_ws"), patch.object(bridge, "_api_approve"):
|
||||
bridge._handle_approval("ws-1", {"items": _approval_items()})
|
||||
|
||||
def _close_ws(bridge=bridge, barrier=barrier):
|
||||
barrier.wait()
|
||||
with (
|
||||
patch.object(bridge, "_publish_global"),
|
||||
patch.object(bridge, "_publish_cluster"),
|
||||
):
|
||||
bridge._handle_global_event({"type": "ws_closed", "ws_id": "ws-1"})
|
||||
|
||||
t1 = threading.Thread(target=_send_approval)
|
||||
t2 = threading.Thread(target=_close_ws)
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=5)
|
||||
t2.join(timeout=5)
|
||||
assert not t1.is_alive(), "Approval thread hung"
|
||||
assert not t2.is_alive(), "Close thread hung"
|
||||
|
||||
# Wait for spawned _wait_approval thread to resolve (if close
|
||||
# didn't remove the entry first)
|
||||
resolved = _wait_pending_resolved(bridge, "ws-1", "_pending_approvals")
|
||||
assert resolved, "Orphaned pending approval"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Race 7: Plan review refinement loop (tombstone → cleanup → re-entry)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPlanReviewRefinementLoop:
|
||||
"""After a plan review is resolved, a ws_state event should clean up the
|
||||
tombstone so the refinement-loop plan_review event is handled correctly."""
|
||||
|
||||
def test_refinement_loop_allows_reentry(self):
|
||||
for _ in range(ITERATIONS):
|
||||
bridge = _make_bridge()
|
||||
bridge._broker.pop_response.return_value = (
|
||||
'{"type": "plan_feedback", "feedback": "refine this"}'
|
||||
)
|
||||
|
||||
# Step 1: first plan review — creates pending entry, resolves it
|
||||
with patch.object(bridge, "_publish_ws"), patch.object(bridge._http, "post"):
|
||||
bridge._handle_plan_review("ws-1", {"content": "plan v1"})
|
||||
|
||||
_wait_pending_resolved(bridge, "ws-1", "_pending_plan_reviews")
|
||||
|
||||
# Verify tombstone is present (resolved_at > 0)
|
||||
with bridge._lock:
|
||||
assert "ws-1" in bridge._pending_plan_reviews
|
||||
assert bridge._pending_plan_reviews["ws-1"][1] > 0
|
||||
|
||||
# Step 2: ws_state event cleans up the resolved tombstone
|
||||
with (
|
||||
patch.object(bridge, "_publish_ws"),
|
||||
patch.object(bridge, "_publish_global"),
|
||||
patch.object(bridge, "_publish_cluster"),
|
||||
):
|
||||
bridge._handle_global_event(
|
||||
{"type": "ws_state", "ws_id": "ws-1", "state": "working"}
|
||||
)
|
||||
|
||||
with bridge._lock:
|
||||
assert "ws-1" not in bridge._pending_plan_reviews
|
||||
|
||||
# Step 3: refinement plan_review arrives — should create new entry
|
||||
with patch.object(bridge, "_publish_ws"), patch.object(bridge._http, "post"):
|
||||
bridge._handle_plan_review("ws-1", {"content": "plan v2"})
|
||||
|
||||
_wait_pending_resolved(bridge, "ws-1", "_pending_plan_reviews")
|
||||
|
||||
with bridge._lock:
|
||||
assert "ws-1" in bridge._pending_plan_reviews
|
||||
+10
-8
@@ -180,7 +180,7 @@ class TestCancelDuringToolExecution:
|
||||
"""Cancel while tools are being executed."""
|
||||
|
||||
def test_rollback_incomplete_tool_results(self, tmp_db):
|
||||
"""When cancelled during tool execution, incomplete results are rolled back."""
|
||||
"""When cancelled during tool execution, synthesized results replace missing tool outputs."""
|
||||
ui = NullUI()
|
||||
session = _make_session(ui=ui)
|
||||
|
||||
@@ -236,13 +236,15 @@ class TestCancelDuringToolExecution:
|
||||
|
||||
# Session should be idle
|
||||
assert ui.states[-1] == "idle"
|
||||
# No tool result messages should remain (rolled back)
|
||||
roles = [m["role"] for m in session.messages]
|
||||
assert "tool" not in roles
|
||||
# The assistant message with tool_calls should also be rolled back
|
||||
for m in session.messages:
|
||||
if m["role"] == "assistant":
|
||||
assert "tool_calls" not in m or not m["tool_calls"]
|
||||
# Cancelled tool calls should have synthesized results
|
||||
tool_msgs = [m for m in session.messages if m["role"] == "tool"]
|
||||
assert len(tool_msgs) == 1
|
||||
assert tool_msgs[0]["tool_call_id"] == "tc_1"
|
||||
assert "Cancelled by user" in tool_msgs[0]["content"]
|
||||
assert tool_msgs[0].get("is_error") is True
|
||||
# The assistant message with tool_calls should still be present
|
||||
assistant_msgs = [m for m in session.messages if m.get("tool_calls")]
|
||||
assert len(assistant_msgs) == 1
|
||||
|
||||
|
||||
class TestCancelWhenIdle:
|
||||
|
||||
+568
-85
@@ -76,8 +76,7 @@ class TestDiscordConfig:
|
||||
assert cfg.max_message_length == 2000
|
||||
assert cfg.streaming_edit_interval == 1.5
|
||||
# Inherited from ChannelConfig
|
||||
assert cfg.redis_host == "localhost"
|
||||
assert cfg.redis_port == 6379
|
||||
assert cfg.server_url == "http://localhost:8080"
|
||||
assert cfg.model == ""
|
||||
assert cfg.auto_approve is False
|
||||
|
||||
@@ -230,13 +229,15 @@ class TestMessageCog:
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_ignores_dms(self):
|
||||
def test_dm_without_reference_sends_guidance(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
msg = _make_message(guild=False)
|
||||
dm_channel = AsyncMock()
|
||||
msg = _make_message(guild=False, channel=dm_channel)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
dm_channel.send.assert_awaited_once()
|
||||
|
||||
def test_ignores_non_allowed_channels(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
@@ -316,12 +317,12 @@ class TestParseFooter:
|
||||
|
||||
|
||||
class TestWsEventFinalization:
|
||||
"""TurnCompleteEvent should finalize streaming messages in the Discord bot."""
|
||||
"""StreamEndEvent should finalize streaming messages in the Discord bot."""
|
||||
|
||||
def test_turn_complete_finalizes_streaming(self):
|
||||
"""ContentEvent + TurnCompleteEvent(correlation_id='') finalizes the message."""
|
||||
def test_stream_end_finalizes_streaming(self):
|
||||
"""ContentEvent + StreamEndEvent finalizes the message."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import ContentEvent, TurnCompleteEvent
|
||||
from turnstone.sdk.events import ContentEvent, StreamEndEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
@@ -330,6 +331,8 @@ class TestWsEventFinalization:
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
|
||||
@@ -339,34 +342,36 @@ class TestWsEventFinalization:
|
||||
thread = AsyncMock()
|
||||
|
||||
# Feed content event
|
||||
content_raw = ContentEvent(ws_id="ws-1", text="Hello world").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, content_raw))
|
||||
content_event = ContentEvent(ws_id="ws-1", text="Hello world")
|
||||
_run(bot._on_ws_event("ws-1", thread, content_event))
|
||||
|
||||
# StreamingMessage should exist
|
||||
assert "ws-1" in bot._streaming
|
||||
|
||||
# Feed turn complete with empty correlation_id (server-UI-initiated)
|
||||
complete_raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, complete_raw))
|
||||
# Feed stream end
|
||||
end_event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, end_event))
|
||||
|
||||
# StreamingMessage should be removed and finalized
|
||||
assert "ws-1" not in bot._streaming
|
||||
|
||||
def test_turn_complete_no_streaming_is_noop(self):
|
||||
"""TurnCompleteEvent without prior content should not error."""
|
||||
def test_stream_end_no_streaming_is_noop(self):
|
||||
"""StreamEndEvent without prior content should not error."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
complete_raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, complete_raw))
|
||||
end_event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, end_event))
|
||||
|
||||
# No error, no streaming message
|
||||
assert "ws-1" not in bot._streaming
|
||||
@@ -392,6 +397,8 @@ class TestApprovalVerdictDisplay:
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
@@ -399,8 +406,8 @@ class TestApprovalVerdictDisplay:
|
||||
return bot
|
||||
|
||||
def test_approval_with_heuristic_verdict(self):
|
||||
"""ApprovalRequestEvent items with verdict dicts add embed fields."""
|
||||
from turnstone.mq.protocol import ApprovalRequestEvent
|
||||
"""ApproveRequestEvent items with verdict dicts add embed fields."""
|
||||
from turnstone.sdk.events import ApproveRequestEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
@@ -421,8 +428,8 @@ class TestApprovalVerdictDisplay:
|
||||
},
|
||||
}
|
||||
]
|
||||
raw = ApprovalRequestEvent(ws_id="ws-1", correlation_id="corr-1", items=items).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
event = ApproveRequestEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# thread.send was called with an embed containing a verdict field
|
||||
thread.send.assert_awaited_once()
|
||||
@@ -439,8 +446,8 @@ class TestApprovalVerdictDisplay:
|
||||
assert "ws-1" in bot._pending_approval_msgs
|
||||
|
||||
def test_approval_without_verdict(self):
|
||||
"""ApprovalRequestEvent items without verdict still work normally."""
|
||||
from turnstone.mq.protocol import ApprovalRequestEvent
|
||||
"""ApproveRequestEvent items without verdict still work normally."""
|
||||
from turnstone.sdk.events import ApproveRequestEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
@@ -448,8 +455,8 @@ class TestApprovalVerdictDisplay:
|
||||
thread.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
items = [{"func_name": "read_file", "preview": "/etc/hosts", "needs_approval": True}]
|
||||
raw = ApprovalRequestEvent(ws_id="ws-1", correlation_id="corr-1", items=items).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
event = ApproveRequestEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once()
|
||||
call_kwargs = thread.send.call_args[1]
|
||||
@@ -459,7 +466,7 @@ class TestApprovalVerdictDisplay:
|
||||
|
||||
def test_intent_verdict_event_updates_embed(self):
|
||||
"""IntentVerdictEvent should update the pending approval embed."""
|
||||
from turnstone.mq.protocol import IntentVerdictEvent
|
||||
from turnstone.sdk.events import IntentVerdictEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
@@ -471,7 +478,7 @@ class TestApprovalVerdictDisplay:
|
||||
msg.edit = AsyncMock()
|
||||
bot._pending_approval_msgs["ws-1"] = msg
|
||||
|
||||
raw = IntentVerdictEvent(
|
||||
event = IntentVerdictEvent(
|
||||
ws_id="ws-1",
|
||||
func_name="bash",
|
||||
risk_level="high",
|
||||
@@ -479,8 +486,8 @@ class TestApprovalVerdictDisplay:
|
||||
confidence=0.9,
|
||||
intent_summary="Dangerous operation",
|
||||
tier="llm",
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Embed should be updated with the judge verdict field
|
||||
embed.add_field.assert_called_once()
|
||||
@@ -494,35 +501,37 @@ class TestApprovalVerdictDisplay:
|
||||
|
||||
def test_intent_verdict_without_pending_approval_is_noop(self):
|
||||
"""IntentVerdictEvent without a pending approval message should not error."""
|
||||
from turnstone.mq.protocol import IntentVerdictEvent
|
||||
from turnstone.sdk.events import IntentVerdictEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = IntentVerdictEvent(ws_id="ws-1", func_name="bash", risk_level="low").to_json()
|
||||
event = IntentVerdictEvent(ws_id="ws-1", func_name="bash", risk_level="low")
|
||||
# Should not raise
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
def test_turn_complete_clears_pending_approval(self):
|
||||
"""TurnCompleteEvent should clean up the pending approval message tracking."""
|
||||
def test_stream_end_clears_pending_approval(self):
|
||||
"""StreamEndEvent should clean up the pending approval message tracking."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {"ws-1": MagicMock()}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
assert "ws-1" not in bot._pending_approval_msgs
|
||||
|
||||
|
||||
class TestContentCatchup:
|
||||
"""TurnCompleteEvent with content field provides catch-up for missed ContentEvents."""
|
||||
class TestStreamEndBehavior:
|
||||
"""StreamEndEvent finalizes streaming and cleans up state."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
@@ -534,56 +543,42 @@ class TestContentCatchup:
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_catchup_sends_content_when_no_streaming(self):
|
||||
"""TurnCompleteEvent with content but no SM sends catch-up message."""
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
def test_stream_end_no_streaming_no_send(self):
|
||||
"""StreamEndEvent without prior content should not send anything."""
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(
|
||||
ws_id="ws-1", correlation_id="", content="Caught up response"
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once_with("Caught up response")
|
||||
thread.send.assert_not_awaited()
|
||||
|
||||
def test_catchup_skipped_when_streaming_exists(self):
|
||||
"""TurnCompleteEvent with content and existing SM uses SM finalize, not catch-up."""
|
||||
from turnstone.mq.protocol import ContentEvent, TurnCompleteEvent
|
||||
def test_stream_end_finalizes_existing_streaming(self):
|
||||
"""StreamEndEvent with an existing StreamingMessage should finalize it."""
|
||||
from turnstone.sdk.events import ContentEvent, StreamEndEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
# Feed content event to create SM
|
||||
content_raw = ContentEvent(ws_id="ws-1", text="Streamed").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, content_raw))
|
||||
content_event = ContentEvent(ws_id="ws-1", text="Streamed")
|
||||
_run(bot._on_ws_event("ws-1", thread, content_event))
|
||||
assert "ws-1" in bot._streaming
|
||||
|
||||
# Now TurnCompleteEvent with content — SM should be finalized, not catch-up
|
||||
complete_raw = TurnCompleteEvent(
|
||||
ws_id="ws-1", correlation_id="", content="Streamed"
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, complete_raw))
|
||||
# Now StreamEndEvent — SM should be finalized
|
||||
end_event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, end_event))
|
||||
assert "ws-1" not in bot._streaming
|
||||
|
||||
def test_catchup_empty_content_no_message(self):
|
||||
"""TurnCompleteEvent with empty content and no SM sends nothing."""
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="", content="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
|
||||
thread.send.assert_not_awaited()
|
||||
|
||||
|
||||
class TestNotificationTracking:
|
||||
"""Tests for notification message tracking and DM reply routing."""
|
||||
@@ -719,8 +714,8 @@ class TestNotificationTracking:
|
||||
# Should send feedback to the DM channel
|
||||
dm_channel.send.assert_awaited_once_with("*This notification is no longer active.*")
|
||||
|
||||
def test_dm_without_reference_ignored(self):
|
||||
"""DM without a message reference should be ignored."""
|
||||
def test_dm_without_reference_sends_guidance(self):
|
||||
"""DM without a message reference should reply with guidance."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
@@ -735,11 +730,15 @@ class TestNotificationTracking:
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
msg = _make_message(guild=False) # reference=None
|
||||
dm_channel = AsyncMock()
|
||||
msg = _make_message(guild=False, channel=dm_channel) # reference=None
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
dm_channel.send.assert_awaited_once()
|
||||
sent_text = dm_channel.send.call_args[0][0]
|
||||
assert "/ask" in sent_text
|
||||
|
||||
def test_dm_reply_unlinked_user_ignored(self):
|
||||
"""DM reply from an unlinked user should be ignored."""
|
||||
@@ -767,15 +766,18 @@ class TestNotificationTracking:
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_turn_complete_forwards_to_dm(self):
|
||||
"""TurnCompleteEvent should forward content to notification reply DM."""
|
||||
def test_stream_end_forwards_accumulated_content_to_dm(self):
|
||||
"""StreamEndEvent should forward accumulated content to notification reply DM."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
from turnstone.sdk.events import ContentEvent, StreamEndEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_ws_map = {}
|
||||
bot._MAX_NOTIFY_TRACKING = 100
|
||||
@@ -790,10 +792,13 @@ class TestNotificationTracking:
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(
|
||||
ws_id="ws-1", correlation_id="", content="Here's the response"
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
# Feed content events to accumulate buffer
|
||||
content_event = ContentEvent(ws_id="ws-1", text="Here's the response")
|
||||
_run(bot._on_ws_event("ws-1", thread, content_event))
|
||||
|
||||
# Feed stream end — should finalize and forward to DM
|
||||
end_event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, end_event))
|
||||
|
||||
# Should send to DM channel
|
||||
dm_channel.send.assert_awaited_once_with("Here's the response")
|
||||
@@ -803,13 +808,15 @@ class TestNotificationTracking:
|
||||
assert 88888 in bot._notify_ws_map
|
||||
assert bot._notify_ws_map[88888] == ("ws-1", "u123")
|
||||
|
||||
def test_turn_complete_cleans_up_dm_even_without_content(self):
|
||||
"""TurnCompleteEvent without content should still clean up DM tracking."""
|
||||
def test_stream_end_cleans_up_dm_even_without_content(self):
|
||||
"""StreamEndEvent without prior content should still clean up DM tracking."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_ws_map = {}
|
||||
|
||||
@@ -819,8 +826,8 @@ class TestNotificationTracking:
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="", content="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
end_event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, end_event))
|
||||
|
||||
# DM should not be sent to (no content)
|
||||
dm_channel.send.assert_not_awaited()
|
||||
@@ -830,6 +837,482 @@ class TestNotificationTracking:
|
||||
assert len(bot._notify_ws_map) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Formatter: format_tool_result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFormatToolResult:
|
||||
"""Tests for format_tool_result in _formatter.py."""
|
||||
|
||||
def test_basic_output(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
result = format_tool_result("hello world")
|
||||
assert "```" in result
|
||||
assert "hello world" in result
|
||||
|
||||
def test_wraps_in_code_block(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
result = format_tool_result("output text")
|
||||
assert result.startswith("```\n")
|
||||
assert result.endswith("\n```")
|
||||
|
||||
def test_truncates_long_output_by_lines(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
output = "\n".join(f"line {i}" for i in range(20))
|
||||
result = format_tool_result(output)
|
||||
# Should have at most 10 content lines + ellipsis
|
||||
inner = result.split("```")[1]
|
||||
assert inner.strip().count("\n") <= 11
|
||||
|
||||
def test_truncates_long_output_by_chars(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
output = "x" * 600
|
||||
result = format_tool_result(output)
|
||||
# Code block content should be <= 500 chars (497 + ellipsis)
|
||||
inner = result.split("```")[1].strip()
|
||||
assert len(inner) <= 501 # 497 + ellipsis char
|
||||
|
||||
def test_escapes_triple_backticks_in_output(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
output = "before ``` after"
|
||||
result = format_tool_result(output)
|
||||
# Only the opening and closing code fences should remain as ```.
|
||||
assert result.count("```") == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thinking indicator lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestThinkingIndicator:
|
||||
"""Tests for ThinkingStart/Stop event handling in the Discord bot."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_thinking_start_sends_message(self):
|
||||
from turnstone.sdk.events import ThinkingStartEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
sent_msg = MagicMock()
|
||||
thread.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
event = ThinkingStartEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once_with("*Thinking...*")
|
||||
assert bot._thinking_msgs["ws-1"] is sent_msg
|
||||
|
||||
def test_thinking_stop_preserves_message_for_reuse(self):
|
||||
from turnstone.sdk.events import ThinkingStopEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.delete = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
|
||||
event = ThinkingStopEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Message kept for next event to reuse via edit.
|
||||
thinking_msg.delete.assert_not_awaited()
|
||||
assert "ws-1" in bot._thinking_msgs
|
||||
|
||||
def test_thinking_stop_without_message_is_noop(self):
|
||||
from turnstone.sdk.events import ThinkingStopEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ThinkingStopEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
def test_content_event_reuses_thinking_message(self):
|
||||
from turnstone.sdk.events import ContentEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.edit = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
|
||||
event = ContentEvent(ws_id="ws-1", text="Hello")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Thinking message becomes the StreamingMessage base — no delete.
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
sm = bot._streaming["ws-1"]
|
||||
assert sm._message is thinking_msg
|
||||
|
||||
def test_stream_end_clears_thinking_message(self):
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.delete = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
bot._notify_reply_channels = {}
|
||||
|
||||
event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thinking_msg.delete.assert_awaited_once()
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool info / result embeds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestToolInfoEvent:
|
||||
"""Tests for ToolInfoEvent handling in the Discord bot."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_sends_per_item_embed(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
sent_msg = MagicMock()
|
||||
thread.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
items = [{"func_name": "bash", "preview": "ls -la", "needs_approval": False}]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once()
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.title == "bash"
|
||||
assert embed.description == "ls -la"
|
||||
# Message tracked for later editing by ToolResultEvent.
|
||||
assert bot._tool_info_msgs["ws-1"] == [("", "bash", "ls -la", sent_msg)]
|
||||
|
||||
def test_multiple_tools_send_multiple_embeds(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
items = [
|
||||
{"func_name": "bash", "preview": "ls", "needs_approval": False},
|
||||
{"func_name": "read_file", "preview": "/etc", "needs_approval": False},
|
||||
]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
assert thread.send.await_count == 2
|
||||
assert len(bot._tool_info_msgs["ws-1"]) == 2
|
||||
|
||||
def test_shows_all_items_regardless_of_approval(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
items = [
|
||||
{"func_name": "bash", "preview": "rm -rf /", "needs_approval": True},
|
||||
{"func_name": "read_file", "preview": "/etc/hosts", "needs_approval": False},
|
||||
]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Both items shown — running indicator is separate from approval dialog.
|
||||
assert thread.send.await_count == 2
|
||||
|
||||
def test_reuses_thinking_message_for_first_tool(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.edit = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
|
||||
items = [{"func_name": "bash", "preview": "ls -la", "needs_approval": False}]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Thinking message edited into tool embed, no new message sent.
|
||||
thinking_msg.edit.assert_awaited_once()
|
||||
thread.send.assert_not_awaited()
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
# The reused message is tracked for ToolResultEvent editing.
|
||||
assert bot._tool_info_msgs["ws-1"][0][3] is thinking_msg
|
||||
|
||||
|
||||
class TestToolResultEvent:
|
||||
"""Tests for ToolResultEvent handling in the Discord bot."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_marks_info_done_and_sends_result(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
# Pre-populate a tool info message (as ToolInfoEvent would).
|
||||
info_msg = MagicMock()
|
||||
info_msg.edit = AsyncMock()
|
||||
bot._tool_info_msgs["ws-1"] = [("", "bash", "ls -la", info_msg)]
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="file1\nfile2")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Info embed edited to "Done" status.
|
||||
info_msg.edit.assert_awaited_once()
|
||||
status_embed = info_msg.edit.call_args[1]["embed"]
|
||||
assert "Done" in status_embed.title
|
||||
assert status_embed.description == "ls -la" # preview preserved
|
||||
# Result sent as separate new message.
|
||||
thread.send.assert_awaited_once()
|
||||
result_embed = thread.send.call_args[1]["embed"]
|
||||
assert result_embed.title == "bash"
|
||||
assert "file1" in result_embed.description
|
||||
# Entry consumed from tracking list.
|
||||
assert bot._tool_info_msgs["ws-1"] == []
|
||||
|
||||
def test_result_sent_even_without_info_match(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="file1\nfile2")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once()
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.title == "bash"
|
||||
assert "file1" in embed.description
|
||||
|
||||
def test_error_result_uses_red_color(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ToolResultEvent(
|
||||
ws_id="ws-1", name="bash", output="command not found", is_error=True
|
||||
)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.color == discord.Color.red()
|
||||
|
||||
def test_success_result_uses_dark_grey_color(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="ok")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.color == discord.Color.dark_grey()
|
||||
|
||||
def test_call_id_matching(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
first_msg = MagicMock()
|
||||
first_msg.edit = AsyncMock()
|
||||
second_msg = MagicMock()
|
||||
second_msg.edit = AsyncMock()
|
||||
bot._tool_info_msgs["ws-1"] = [
|
||||
("call-1", "bash", "", first_msg),
|
||||
("call-2", "bash", "", second_msg),
|
||||
]
|
||||
|
||||
# Result with call_id matches the correct message regardless of order.
|
||||
event = ToolResultEvent(ws_id="ws-1", call_id="call-2", name="bash", output="result")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
second_msg.edit.assert_awaited_once()
|
||||
first_msg.edit.assert_not_awaited()
|
||||
|
||||
def test_fifo_fallback_when_no_call_id(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
first_msg = MagicMock()
|
||||
first_msg.edit = AsyncMock()
|
||||
second_msg = MagicMock()
|
||||
second_msg.edit = AsyncMock()
|
||||
bot._tool_info_msgs["ws-1"] = [("", "bash", "", first_msg), ("", "bash", "", second_msg)]
|
||||
|
||||
# No call_id — falls back to FIFO name match.
|
||||
event1 = ToolResultEvent(ws_id="ws-1", name="bash", output="result1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event1))
|
||||
first_msg.edit.assert_awaited_once()
|
||||
second_msg.edit.assert_not_awaited()
|
||||
|
||||
event2 = ToolResultEvent(ws_id="ws-1", name="bash", output="result2")
|
||||
_run(bot._on_ws_event("ws-1", thread, event2))
|
||||
second_msg.edit.assert_awaited_once()
|
||||
|
||||
def test_edit_failure_falls_back_to_send(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
info_msg = MagicMock()
|
||||
info_msg.edit = AsyncMock(side_effect=Exception("Discord API error"))
|
||||
bot._tool_info_msgs["ws-1"] = [("", "bash", "ls -la", info_msg)]
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="ok")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Edit failed, should fall back to send.
|
||||
info_msg.edit.assert_awaited_once()
|
||||
thread.send.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Approval resolved (timeout / external resolution)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApprovalResolved:
|
||||
"""ApprovalResolvedEvent should disable buttons on the pending approval embed."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_disables_buttons_on_timeout(self):
|
||||
from turnstone.sdk.events import ApprovalResolvedEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
# Set up a pending approval message with components.
|
||||
approval_msg = MagicMock()
|
||||
approval_msg.embeds = [MagicMock()]
|
||||
approval_msg.components = []
|
||||
approval_msg.edit = AsyncMock()
|
||||
bot._pending_approval_msgs["ws-1"] = approval_msg
|
||||
|
||||
event = ApprovalResolvedEvent(ws_id="ws-1", approved=False, feedback="timeout")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
approval_msg.edit.assert_awaited_once()
|
||||
# Pending approval message should be removed.
|
||||
assert "ws-1" not in bot._pending_approval_msgs
|
||||
|
||||
def test_disables_buttons_on_approved(self):
|
||||
from turnstone.sdk.events import ApprovalResolvedEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
approval_msg = MagicMock()
|
||||
approval_msg.embeds = [MagicMock()]
|
||||
approval_msg.components = []
|
||||
approval_msg.edit = AsyncMock()
|
||||
bot._pending_approval_msgs["ws-1"] = approval_msg
|
||||
|
||||
event = ApprovalResolvedEvent(ws_id="ws-1", approved=True)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
approval_msg.edit.assert_awaited_once()
|
||||
# Check the embed title was updated with "Approved".
|
||||
edited_embed = approval_msg.edit.call_args[1]["embed"]
|
||||
assert "Approved" in edited_embed.title
|
||||
|
||||
def test_no_pending_approval_is_noop(self):
|
||||
from turnstone.sdk.events import ApprovalResolvedEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ApprovalResolvedEvent(ws_id="ws-1", approved=False)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
# No error, no state change.
|
||||
|
||||
|
||||
class TestChannelCLI:
|
||||
"""Tests for the channel CLI entry point."""
|
||||
|
||||
|
||||
+265
-43
@@ -2,24 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.channels._routing import ChannelRouter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_broker() -> AsyncMock:
|
||||
"""Return a mock AsyncRedisBroker."""
|
||||
broker = AsyncMock()
|
||||
broker._prefix = "test"
|
||||
broker.push_inbound = AsyncMock()
|
||||
broker.push_response = AsyncMock()
|
||||
broker.subscribe = AsyncMock()
|
||||
broker.unsubscribe = AsyncMock()
|
||||
return broker
|
||||
from turnstone.sdk._types import TurnstoneAPIError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -35,8 +23,21 @@ def mock_storage() -> MagicMock:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def router(mock_broker: AsyncMock, mock_storage: MagicMock) -> ChannelRouter:
|
||||
return ChannelRouter(broker=mock_broker, storage=mock_storage)
|
||||
def router(mock_storage: MagicMock) -> ChannelRouter:
|
||||
return ChannelRouter(
|
||||
server_url="http://localhost:8080/v1",
|
||||
storage=mock_storage,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def console_router(mock_storage: MagicMock) -> ChannelRouter:
|
||||
return ChannelRouter(
|
||||
server_url="http://localhost:8080/v1",
|
||||
storage=mock_storage,
|
||||
console_url="http://localhost:8081/v1",
|
||||
api_token="tok-test",
|
||||
)
|
||||
|
||||
|
||||
class TestResolveUser:
|
||||
@@ -56,46 +57,82 @@ class TestResolveUser:
|
||||
|
||||
class TestSendMessage:
|
||||
@pytest.mark.anyio
|
||||
async def test_pushes_send_message(self, router: ChannelRouter, mock_broker: AsyncMock) -> None:
|
||||
cid = await router.send_message("ws-1", "hello world")
|
||||
assert isinstance(cid, str)
|
||||
assert len(cid) > 0
|
||||
mock_broker.push_inbound.assert_awaited_once()
|
||||
raw = mock_broker.push_inbound.call_args[0][0]
|
||||
payload = json.loads(raw)
|
||||
assert payload["type"] == "send"
|
||||
assert payload["ws_id"] == "ws-1"
|
||||
assert payload["message"] == "hello world"
|
||||
assert payload["correlation_id"] == cid
|
||||
async def test_calls_server_send(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_send = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "send", mock_send)
|
||||
await router.send_message("ws-1", "hello world")
|
||||
mock_send.assert_awaited_once_with("hello world", "ws-1")
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_calls_console_route_send(
|
||||
self, console_router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_send = AsyncMock()
|
||||
monkeypatch.setattr(console_router._console, "route_send", mock_send)
|
||||
await console_router.send_message("ws-1", "hello world")
|
||||
mock_send.assert_awaited_once_with("hello world", "ws-1")
|
||||
|
||||
|
||||
class TestSendApproval:
|
||||
@pytest.mark.anyio
|
||||
async def test_pushes_to_response_queue(
|
||||
self, router: ChannelRouter, mock_broker: AsyncMock
|
||||
async def test_calls_server_approve(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_approve = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "approve", mock_approve)
|
||||
await router.send_approval("ws-1", "corr-abc", approved=True, feedback="ok")
|
||||
mock_broker.push_response.assert_awaited_once()
|
||||
queue_name = mock_broker.push_response.call_args[0][0]
|
||||
assert queue_name == "corr-abc"
|
||||
raw = mock_broker.push_response.call_args[0][1]
|
||||
payload = json.loads(raw)
|
||||
assert payload["type"] == "approve"
|
||||
assert payload["approved"] is True
|
||||
assert payload["ws_id"] == "ws-1"
|
||||
mock_approve.assert_awaited_once_with(
|
||||
ws_id="ws-1", approved=True, feedback="ok", always=False
|
||||
)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_omits_empty_feedback(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_approve = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "approve", mock_approve)
|
||||
await router.send_approval("ws-1", "corr-abc", approved=False)
|
||||
mock_approve.assert_awaited_once_with(
|
||||
ws_id="ws-1", approved=False, feedback=None, always=False
|
||||
)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_calls_console_route_approve(
|
||||
self, console_router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_approve = AsyncMock()
|
||||
monkeypatch.setattr(console_router._console, "route_approve", mock_approve)
|
||||
await console_router.send_approval("ws-1", "corr-abc", approved=True, always=True)
|
||||
mock_approve.assert_awaited_once_with(ws_id="ws-1", approved=True, feedback="", always=True)
|
||||
|
||||
|
||||
class TestSendPlanFeedback:
|
||||
@pytest.mark.anyio
|
||||
async def test_pushes_to_response_queue(
|
||||
self, router: ChannelRouter, mock_broker: AsyncMock
|
||||
async def test_calls_server_plan_feedback(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_plan = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "plan_feedback", mock_plan)
|
||||
await router.send_plan_feedback("ws-2", "corr-xyz", "looks good")
|
||||
mock_broker.push_response.assert_awaited_once()
|
||||
raw = mock_broker.push_response.call_args[0][1]
|
||||
payload = json.loads(raw)
|
||||
assert payload["type"] == "plan_feedback"
|
||||
assert payload["feedback"] == "looks good"
|
||||
mock_plan.assert_awaited_once_with(ws_id="ws-2", feedback="looks good")
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_calls_console_route_plan_feedback(
|
||||
self, console_router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_plan = AsyncMock()
|
||||
monkeypatch.setattr(console_router._console, "route_plan_feedback", mock_plan)
|
||||
await console_router.send_plan_feedback("ws-2", "corr-xyz", "looks good")
|
||||
mock_plan.assert_awaited_once_with(ws_id="ws-2", feedback="looks good")
|
||||
|
||||
|
||||
class TestDeleteRoute:
|
||||
@@ -105,3 +142,188 @@ class TestDeleteRoute:
|
||||
) -> None:
|
||||
await router.delete_route("discord", "ch-123")
|
||||
mock_storage.delete_channel_route.assert_called_once_with("discord", "ch-123")
|
||||
|
||||
|
||||
class TestGetOrCreateWorkstream:
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_new_workstream_via_server(
|
||||
self,
|
||||
router: ChannelRouter,
|
||||
mock_storage: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_create = AsyncMock()
|
||||
mock_create.return_value = MagicMock(ws_id="ws-new", name="test")
|
||||
monkeypatch.setattr(router._server, "create_workstream", mock_create)
|
||||
ws_id, is_new = await router.get_or_create_workstream("discord", "ch-1", name="test")
|
||||
assert ws_id == "ws-new"
|
||||
assert is_new is True
|
||||
mock_storage.create_channel_route.assert_called_once_with("discord", "ch-1", "ws-new")
|
||||
mock_create.assert_awaited_once()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_new_workstream_via_console(
|
||||
self,
|
||||
console_router: ChannelRouter,
|
||||
mock_storage: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_create = AsyncMock(
|
||||
return_value={"ws_id": "ws-new", "name": "test", "node_url": "http://node1:8080/v1"}
|
||||
)
|
||||
monkeypatch.setattr(console_router._console, "route_create_workstream", mock_create)
|
||||
ws_id, is_new = await console_router.get_or_create_workstream(
|
||||
"discord", "ch-1", name="test"
|
||||
)
|
||||
assert ws_id == "ws-new"
|
||||
assert is_new is True
|
||||
mock_storage.create_channel_route.assert_called_once_with("discord", "ch-1", "ws-new")
|
||||
# Node URL should be cached.
|
||||
assert console_router._node_urls["ws-new"] == "http://node1:8080/v1"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_returns_existing_alive_workstream(
|
||||
self,
|
||||
router: ChannelRouter,
|
||||
mock_storage: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_storage.get_channel_route.return_value = {
|
||||
"ws_id": "ws-old",
|
||||
"channel_type": "discord",
|
||||
"channel_id": "ch-1",
|
||||
}
|
||||
monkeypatch.setattr(router, "_is_ws_alive", AsyncMock(return_value=True))
|
||||
ws_id, is_new = await router.get_or_create_workstream("discord", "ch-1")
|
||||
assert ws_id == "ws-old"
|
||||
assert is_new is False
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_resumes_stale_workstream(
|
||||
self,
|
||||
router: ChannelRouter,
|
||||
mock_storage: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_storage.get_channel_route.return_value = {
|
||||
"ws_id": "ws-stale",
|
||||
"channel_type": "discord",
|
||||
"channel_id": "ch-1",
|
||||
}
|
||||
# Alive check returns False — ws is not alive.
|
||||
monkeypatch.setattr(router, "_is_ws_alive", AsyncMock(return_value=False))
|
||||
# Server create returns a resumed workstream.
|
||||
assert router._server is not None
|
||||
mock_create = AsyncMock()
|
||||
mock_create.return_value = MagicMock(ws_id="ws-resumed", name="test")
|
||||
monkeypatch.setattr(router._server, "create_workstream", mock_create)
|
||||
|
||||
ws_id, is_new = await router.get_or_create_workstream("discord", "ch-1", name="test")
|
||||
assert ws_id == "ws-resumed"
|
||||
assert is_new is True
|
||||
# Should have deleted the stale route and created a new one.
|
||||
mock_storage.delete_channel_route.assert_called_once_with("discord", "ch-1")
|
||||
mock_storage.create_channel_route.assert_called_once_with("discord", "ch-1", "ws-resumed")
|
||||
# The create call should include resume_ws pointing at the old ws.
|
||||
mock_create.assert_awaited_once()
|
||||
call_kwargs = mock_create.call_args[1]
|
||||
assert call_kwargs["resume_ws"] == "ws-stale"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_sends_initial_message_for_new_workstream(
|
||||
self,
|
||||
router: ChannelRouter,
|
||||
mock_storage: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_create = AsyncMock()
|
||||
mock_create.return_value = MagicMock(ws_id="ws-new", name="test")
|
||||
monkeypatch.setattr(router._server, "create_workstream", mock_create)
|
||||
mock_send = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "send", mock_send)
|
||||
|
||||
await router.get_or_create_workstream("discord", "ch-1", name="test", initial_message="hi")
|
||||
mock_send.assert_awaited_once_with("hi", "ws-new")
|
||||
|
||||
|
||||
class TestCloseWorkstream:
|
||||
@pytest.mark.anyio
|
||||
async def test_calls_server_close(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_close = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "close_workstream", mock_close)
|
||||
await router.close_workstream("ws-1")
|
||||
mock_close.assert_awaited_once_with("ws-1")
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_catches_api_error(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_close = AsyncMock(side_effect=TurnstoneAPIError(404, "not found"))
|
||||
monkeypatch.setattr(router._server, "close_workstream", mock_close)
|
||||
# Should not raise.
|
||||
await router.close_workstream("ws-1")
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_calls_console_route_close(
|
||||
self, console_router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_close = AsyncMock()
|
||||
monkeypatch.setattr(console_router._console, "route_close", mock_close)
|
||||
await console_router.close_workstream("ws-1")
|
||||
mock_close.assert_awaited_once_with("ws-1")
|
||||
|
||||
|
||||
class TestAclose:
|
||||
@pytest.mark.anyio
|
||||
async def test_closes_server_client(
|
||||
self, router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert router._server is not None
|
||||
mock_close = AsyncMock()
|
||||
monkeypatch.setattr(router._server, "aclose", mock_close)
|
||||
await router.aclose()
|
||||
mock_close.assert_awaited_once()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_closes_console_client(
|
||||
self, console_router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_close = AsyncMock()
|
||||
monkeypatch.setattr(console_router._console, "aclose", mock_close)
|
||||
await console_router.aclose()
|
||||
mock_close.assert_awaited_once()
|
||||
|
||||
|
||||
class TestGetNodeUrl:
|
||||
@pytest.mark.anyio
|
||||
async def test_returns_cached_url(self, router: ChannelRouter) -> None:
|
||||
router._node_urls["ws-1"] = "http://node1:8080/v1"
|
||||
url = await router.get_node_url("ws-1")
|
||||
assert url == "http://node1:8080/v1"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_falls_back_to_server_url(self, router: ChannelRouter) -> None:
|
||||
url = await router.get_node_url("ws-unknown")
|
||||
assert url == "http://localhost:8080/v1"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_queries_console_route_lookup(
|
||||
self, console_router: ChannelRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
assert console_router._console is not None
|
||||
mock_lookup = AsyncMock(return_value={"node_url": "http://node2:8080/v1", "node_id": "n2"})
|
||||
monkeypatch.setattr(console_router._console, "route_lookup", mock_lookup)
|
||||
url = await console_router.get_node_url("ws-1")
|
||||
assert url == "http://node2:8080/v1"
|
||||
mock_lookup.assert_awaited_once_with("ws-1")
|
||||
# Should be cached now.
|
||||
assert console_router._node_urls["ws-1"] == "http://node2:8080/v1"
|
||||
|
||||
+30
-35
@@ -21,20 +21,20 @@ def test_load_config_missing_file(tmp_path):
|
||||
def test_load_config_valid_toml(tmp_path):
|
||||
_reset_cache()
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text('[redis]\nhost = "10.0.0.1"\nport = 6380\npassword = "secret"\n')
|
||||
cfg.write_text('[database]\nhost = "10.0.0.1"\nport = 5432\nname = "turnstone"\n')
|
||||
set_config_path(str(cfg))
|
||||
result = load_config()
|
||||
assert result["redis"]["host"] == "10.0.0.1"
|
||||
assert result["redis"]["port"] == 6380
|
||||
assert result["redis"]["password"] == "secret"
|
||||
assert result["database"]["host"] == "10.0.0.1"
|
||||
assert result["database"]["port"] == 5432
|
||||
assert result["database"]["name"] == "turnstone"
|
||||
|
||||
|
||||
def test_load_config_section(tmp_path):
|
||||
_reset_cache()
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text('[api]\nbase_url = "http://x:8000/v1"\n[redis]\nhost = "y"\n')
|
||||
cfg.write_text('[api]\nbase_url = "http://x:8000/v1"\n[database]\nhost = "y"\n')
|
||||
set_config_path(str(cfg))
|
||||
assert load_config("redis") == {"host": "y"}
|
||||
assert load_config("database") == {"host": "y"}
|
||||
assert load_config("api") == {"base_url": "http://x:8000/v1"}
|
||||
assert load_config("nonexistent") == {}
|
||||
|
||||
@@ -65,61 +65,56 @@ def test_apply_config_sets_defaults(tmp_path):
|
||||
_reset_cache()
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text(
|
||||
'[redis]\nhost = "redis.local"\nport = 7777\npassword = "pw"\n'
|
||||
'[bridge]\nserver_url = "http://bridge:9090"\n'
|
||||
'[server]\nhost = "0.0.0.0"\nport = 9090\n[api]\nbase_url = "http://custom/v1"\n'
|
||||
)
|
||||
set_config_path(str(cfg))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--redis-host", default="localhost")
|
||||
parser.add_argument("--redis-port", type=int, default=6379)
|
||||
parser.add_argument("--redis-password", default=None)
|
||||
parser.add_argument("--server-url", default="http://localhost:8080")
|
||||
parser.add_argument("--host", default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
parser.add_argument("--base-url", default="http://localhost:11434/v1")
|
||||
|
||||
apply_config(parser, ["redis", "bridge"])
|
||||
apply_config(parser, ["server", "api"])
|
||||
args = parser.parse_args([])
|
||||
|
||||
assert args.redis_host == "redis.local"
|
||||
assert args.redis_port == 7777
|
||||
assert args.redis_password == "pw"
|
||||
assert args.server_url == "http://bridge:9090"
|
||||
assert args.host == "0.0.0.0"
|
||||
assert args.port == 9090
|
||||
assert args.base_url == "http://custom/v1"
|
||||
|
||||
|
||||
def test_apply_config_cli_overrides(tmp_path):
|
||||
_reset_cache()
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text('[redis]\nhost = "config-host"\nport = 7777\n')
|
||||
cfg.write_text('[server]\nhost = "config-host"\nport = 7777\n')
|
||||
set_config_path(str(cfg))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--redis-host", default="localhost")
|
||||
parser.add_argument("--redis-port", type=int, default=6379)
|
||||
parser.add_argument("--host", default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
|
||||
apply_config(parser, ["redis"])
|
||||
apply_config(parser, ["server"])
|
||||
# CLI flag overrides config
|
||||
args = parser.parse_args(["--redis-host", "cli-host"])
|
||||
args = parser.parse_args(["--host", "cli-host"])
|
||||
|
||||
assert args.redis_host == "cli-host" # CLI wins
|
||||
assert args.redis_port == 7777 # config wins (no CLI override)
|
||||
assert args.host == "cli-host" # CLI wins
|
||||
assert args.port == 7777 # config wins (no CLI override)
|
||||
|
||||
|
||||
def test_apply_config_missing_keys_keep_defaults(tmp_path):
|
||||
_reset_cache()
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text('[redis]\nhost = "only-host"\n') # no port, no password
|
||||
cfg.write_text('[server]\nhost = "only-host"\n') # no port
|
||||
set_config_path(str(cfg))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--redis-host", default="localhost")
|
||||
parser.add_argument("--redis-port", type=int, default=6379)
|
||||
parser.add_argument("--redis-password", default=None)
|
||||
parser.add_argument("--host", default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
|
||||
apply_config(parser, ["redis"])
|
||||
apply_config(parser, ["server"])
|
||||
args = parser.parse_args([])
|
||||
|
||||
assert args.redis_host == "only-host"
|
||||
assert args.redis_port == 6379 # original default kept
|
||||
assert args.redis_password is None # original default kept
|
||||
assert args.host == "only-host"
|
||||
assert args.port == 8080 # original default kept
|
||||
|
||||
|
||||
def test_apply_config_no_file(tmp_path):
|
||||
@@ -127,11 +122,11 @@ def test_apply_config_no_file(tmp_path):
|
||||
set_config_path(str(tmp_path / "nope.toml"))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--redis-host", default="localhost")
|
||||
parser.add_argument("--host", default="localhost")
|
||||
|
||||
apply_config(parser, ["redis"])
|
||||
apply_config(parser, ["server"])
|
||||
args = parser.parse_args([])
|
||||
assert args.redis_host == "localhost"
|
||||
assert args.host == "localhost"
|
||||
|
||||
|
||||
def test_apply_config_model_section(tmp_path):
|
||||
|
||||
+344
-367
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,148 @@
|
||||
"""Tests for turnstone.console.metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from turnstone.console.metrics import ConsoleMetrics
|
||||
|
||||
|
||||
class TestRecordRoute:
|
||||
"""Recording routed requests."""
|
||||
|
||||
def test_single_request(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_route("send", 200, 0.05)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
||||
|
||||
def test_multiple_methods(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_route("send", 200, 0.01)
|
||||
m.record_route("create", 200, 0.02)
|
||||
m.record_route("send", 502, 0.5)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
||||
assert 'turnstone_router_requests_total{method="create",status="2xx"} 1' in text
|
||||
assert 'turnstone_router_requests_total{method="send",status="5xx"} 1' in text
|
||||
|
||||
def test_duration_recorded(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_route("send", 200, 0.123)
|
||||
m.record_route("send", 200, 0.456)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_router_request_duration_seconds_count{method="send"} 2' in text
|
||||
# Sum should be 0.579
|
||||
assert "turnstone_router_request_duration_seconds_sum" in text
|
||||
|
||||
|
||||
class TestRingInfo:
|
||||
"""Ring membership and version gauges."""
|
||||
|
||||
def test_defaults_zero(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert "turnstone_ring_membership_size 0" in text
|
||||
assert "turnstone_ring_version 0" in text
|
||||
|
||||
def test_set_ring_info(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.set_ring_info(3, 7)
|
||||
|
||||
text = m.generate_text()
|
||||
assert "turnstone_ring_membership_size 3" in text
|
||||
assert "turnstone_ring_version 7" in text
|
||||
|
||||
|
||||
class TestRebalance:
|
||||
"""Rebalance and migration counters."""
|
||||
|
||||
def test_noop(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_rebalance("noop")
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_ring_rebalance_total{result="noop"} 1' in text
|
||||
|
||||
def test_seeded(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_rebalance("seeded")
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_ring_rebalance_total{result="seeded"} 1' in text
|
||||
|
||||
def test_rebalanced(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_rebalance("rebalanced")
|
||||
m.record_rebalance("rebalanced")
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_ring_rebalance_total{result="rebalanced"} 2' in text
|
||||
|
||||
def test_migrations(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_migrations(5)
|
||||
m.record_migrations(3)
|
||||
|
||||
text = m.generate_text()
|
||||
assert "turnstone_ring_migrations_total 8" in text
|
||||
|
||||
def test_migrations_default_zero(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert "turnstone_ring_migrations_total 0" in text
|
||||
|
||||
|
||||
class TestGenerateText:
|
||||
"""Output format validation."""
|
||||
|
||||
def test_contains_all_metric_names(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
expected = [
|
||||
"turnstone_router_requests_total",
|
||||
"turnstone_router_request_duration_seconds",
|
||||
"turnstone_ring_membership_size",
|
||||
"turnstone_ring_version",
|
||||
"turnstone_ring_rebalance_total",
|
||||
"turnstone_ring_migrations_total",
|
||||
]
|
||||
for name in expected:
|
||||
assert name in text, f"Missing metric: {name}"
|
||||
|
||||
def test_has_help_and_type(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert "# HELP turnstone_router_requests_total" in text
|
||||
assert "# TYPE turnstone_router_requests_total counter" in text
|
||||
assert "# HELP turnstone_ring_membership_size" in text
|
||||
assert "# TYPE turnstone_ring_membership_size gauge" in text
|
||||
|
||||
def test_ends_with_newline(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert text.endswith("\n")
|
||||
|
||||
def test_combined_scenario(self) -> None:
|
||||
"""Full scenario: routes, ring info, rebalances, migrations."""
|
||||
m = ConsoleMetrics()
|
||||
m.record_route("create", 200, 0.1)
|
||||
m.record_route("send", 200, 0.05)
|
||||
m.record_route("send", 502, 1.2)
|
||||
m.set_ring_info(3, 12)
|
||||
m.record_rebalance("seeded")
|
||||
m.record_rebalance("noop")
|
||||
m.record_rebalance("rebalanced")
|
||||
m.record_migrations(4)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_router_requests_total{method="create",status="2xx"} 1' in text
|
||||
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
||||
assert 'turnstone_router_requests_total{method="send",status="5xx"} 1' in text
|
||||
assert "turnstone_ring_membership_size 3" in text
|
||||
assert "turnstone_ring_version 12" in text
|
||||
assert 'turnstone_ring_rebalance_total{result="noop"} 1' in text
|
||||
assert 'turnstone_ring_rebalance_total{result="rebalanced"} 1' in text
|
||||
assert 'turnstone_ring_rebalance_total{result="seeded"} 1' in text
|
||||
assert "turnstone_ring_migrations_total 4" in text
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Tests for turnstone.console.router."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.console.router import ConsoleRouter, NodeRef
|
||||
from turnstone.core.hash_ring import RING_SIZE, NoAvailableNodeError
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake storage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
"""Minimal storage mock for router tests."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.services: list[dict[str, str]] = []
|
||||
self.buckets: list[dict[str, Any]] = []
|
||||
self.overrides: list[dict[str, str]] = []
|
||||
self.settings: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
return list(self.services)
|
||||
|
||||
def list_ring_buckets(self) -> list[dict[str, Any]]:
|
||||
return list(self.buckets)
|
||||
|
||||
def list_workstream_overrides(self) -> list[dict[str, str]]:
|
||||
return list(self.overrides)
|
||||
|
||||
def get_system_setting(self, key: str, node_id: str = "") -> dict[str, Any] | None:
|
||||
return self.settings.get(key)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NODE_A = {"service_id": "node-a", "url": "http://a:8080", "metadata": "{}"}
|
||||
NODE_B = {"service_id": "node-b", "url": "http://b:8080", "metadata": "{}"}
|
||||
NODE_C = {"service_id": "node-c", "url": "http://c:8080", "metadata": "{}"}
|
||||
|
||||
|
||||
def _make_router(storage: FakeStorage | None = None) -> tuple[ConsoleRouter, FakeStorage]:
|
||||
s = storage or FakeStorage()
|
||||
return ConsoleRouter(s), s # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _ws_id_for_bucket(bucket: int) -> str:
|
||||
"""Build a 32-char hex ws_id whose first 4 chars encode *bucket*."""
|
||||
return f"{bucket:04x}" + "0" * 28
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestRouteBasic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteBasic:
|
||||
"""Basic routing through the bucket cache."""
|
||||
|
||||
def test_route_returns_correct_node(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
storage.buckets = [
|
||||
{"bucket": 0x0000, "node_id": "node-a"},
|
||||
{"bucket": 0x0001, "node_id": "node-b"},
|
||||
{"bucket": 0x0002, "node_id": "node-c"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.route(_ws_id_for_bucket(0x0000)) == NodeRef("node-a", "http://a:8080")
|
||||
assert router.route(_ws_id_for_bucket(0x0001)) == NodeRef("node-b", "http://b:8080")
|
||||
assert router.route(_ws_id_for_bucket(0x0002)) == NodeRef("node-c", "http://c:8080")
|
||||
|
||||
def test_route_override_priority(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B]
|
||||
storage.buckets = [{"bucket": 0x0000, "node_id": "node-a"}]
|
||||
ws_id = _ws_id_for_bucket(0x0000)
|
||||
storage.overrides = [{"ws_id": ws_id, "node_id": "node-b"}]
|
||||
router.refresh_cache()
|
||||
|
||||
# Override wins over bucket assignment
|
||||
assert router.route(ws_id) == NodeRef("node-b", "http://b:8080")
|
||||
|
||||
def test_route_empty_cache_raises(self) -> None:
|
||||
router, _ = _make_router()
|
||||
|
||||
with pytest.raises(NoAvailableNodeError, match="not assigned"):
|
||||
router.route(_ws_id_for_bucket(0x0000))
|
||||
|
||||
def test_route_url_convenience(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0x0010, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.route_url(_ws_id_for_bucket(0x0010)) == "http://a:8080"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestRefreshCache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRefreshCache:
|
||||
"""Cache loading from storage."""
|
||||
|
||||
def test_refresh_loads_from_storage(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 100, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
ref = router.route(_ws_id_for_bucket(100))
|
||||
assert ref.node_id == "node-a"
|
||||
|
||||
def test_refresh_handles_dead_nodes(self) -> None:
|
||||
router, storage = _make_router()
|
||||
# node-b is in buckets but not in services (dead/expired)
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [
|
||||
{"bucket": 0x0000, "node_id": "node-a"},
|
||||
{"bucket": 0x0001, "node_id": "node-b"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.route(_ws_id_for_bucket(0x0000)).node_id == "node-a"
|
||||
with pytest.raises(NoAvailableNodeError):
|
||||
router.route(_ws_id_for_bucket(0x0001))
|
||||
|
||||
def test_refresh_returns_true_on_change(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
|
||||
assert router.refresh_cache() is True
|
||||
|
||||
def test_refresh_returns_false_on_no_change(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
|
||||
router.refresh_cache()
|
||||
assert router.refresh_cache() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestCheckVersion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckVersion:
|
||||
"""Version-gated refresh."""
|
||||
|
||||
def test_version_change_triggers_refresh(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
storage.settings["rebalancer_version"] = {"value": "1"}
|
||||
|
||||
assert router.check_version() is True
|
||||
assert router.is_ready()
|
||||
|
||||
def test_same_version_skips(self) -> None:
|
||||
router, storage = _make_router()
|
||||
# Default version is 0; setting absent also means 0
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
|
||||
# First call: version=0 matches self._version=0 -> no refresh
|
||||
assert router.check_version() is False
|
||||
assert not router.is_ready() # cache was never loaded
|
||||
|
||||
def test_version_none_treated_as_zero(self) -> None:
|
||||
router, storage = _make_router()
|
||||
# settings dict is empty -> get_system_setting returns None
|
||||
assert router.check_version() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestGenerateWsId
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerateWsId:
|
||||
"""Workstream ID generation targeting a specific node."""
|
||||
|
||||
def test_generates_routable_id(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B]
|
||||
storage.buckets = [
|
||||
{"bucket": 0x00FF, "node_id": "node-a"},
|
||||
{"bucket": 0x0100, "node_id": "node-b"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
ws_id = router.generate_ws_id_for_node("node-a")
|
||||
assert len(ws_id) == 32
|
||||
assert router.route(ws_id).node_id == "node-a"
|
||||
|
||||
def test_unknown_node_raises(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
with pytest.raises(NoAvailableNodeError, match="node-z"):
|
||||
router.generate_ws_id_for_node("node-z")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestIsReady
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsReady:
|
||||
"""Readiness checks."""
|
||||
|
||||
def test_false_when_empty(self) -> None:
|
||||
router, _ = _make_router()
|
||||
assert router.is_ready() is False
|
||||
|
||||
def test_true_after_refresh(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.is_ready() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestNodeCount
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNodeCount:
|
||||
"""Distinct node counting."""
|
||||
|
||||
def test_count_distinct_nodes(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
# Spread all 65536 buckets across 3 nodes
|
||||
storage.buckets = [
|
||||
{"bucket": b, "node_id": f"node-{['a', 'b', 'c'][b % 3]}"} for b in range(RING_SIZE)
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.node_count() == 3
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Tests for console routing proxy endpoints (route_create, route_proxy, route_lookup)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.router import ConsoleRouter, NodeRef
|
||||
from turnstone.core.hash_ring import NoAvailableNodeError
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_mock_collector() -> MagicMock:
|
||||
collector = MagicMock(spec=ClusterCollector)
|
||||
collector.get_overview.return_value = {
|
||||
"nodes": 1,
|
||||
"workstreams": 0,
|
||||
"states": {"running": 0, "thinking": 0, "attention": 0, "idle": 0, "error": 0},
|
||||
"aggregate": {"total_tokens": 0, "total_tool_calls": 0},
|
||||
}
|
||||
return collector
|
||||
|
||||
|
||||
def _make_mock_router(ready: bool = True) -> MagicMock:
|
||||
router = MagicMock(spec=ConsoleRouter)
|
||||
router.is_ready.return_value = ready
|
||||
router.route.return_value = NodeRef("node-a", "http://a:8080")
|
||||
router.generate_ws_id_for_node.return_value = "00ff" + "0" * 28
|
||||
return router
|
||||
|
||||
|
||||
def _make_app(
|
||||
collector: Any = None,
|
||||
router: Any = None,
|
||||
) -> Any:
|
||||
from turnstone.console.server import _load_static, create_app
|
||||
from turnstone.core.auth import AuthConfig
|
||||
|
||||
_load_static()
|
||||
return create_app(
|
||||
collector=collector or _make_mock_collector(),
|
||||
auth_config=AuthConfig(),
|
||||
router=router,
|
||||
)
|
||||
|
||||
|
||||
def _make_proxy_post(
|
||||
status_code: int = 200,
|
||||
json_data: dict[str, Any] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Create a mock for httpx.AsyncClient.post that returns a fixed response."""
|
||||
data = json_data or {"ws_id": "abc123", "name": "test"}
|
||||
|
||||
async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code,
|
||||
json=data,
|
||||
request=httpx.Request("POST", args[0] if args else "http://test"),
|
||||
)
|
||||
|
||||
mock_post = MagicMock(side_effect=_mock_post)
|
||||
return mock_post
|
||||
|
||||
|
||||
def _wire_proxy(app: Any, mock_post: MagicMock | None = None) -> None:
|
||||
"""Attach a mock proxy_client to the app (lifespan doesn't run in TestClient)."""
|
||||
if mock_post is None:
|
||||
mock_post = _make_proxy_post()
|
||||
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
||||
mock_proxy.post = mock_post
|
||||
app.state.proxy_client = mock_proxy
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests — route_create
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteCreate:
|
||||
"""POST /v1/api/route/workstreams/new — create via hash-ring routing."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
router = _make_mock_router()
|
||||
app = _make_app(router=router)
|
||||
_wire_proxy(app, _make_proxy_post(json_data={"ws_id": "abc123", "name": "test"}))
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_route_create_proxies_to_node(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test-ws"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ws_id"] == "abc123"
|
||||
|
||||
def test_route_create_injects_node_url(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test-ws"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["node_url"] == "http://a:8080"
|
||||
assert data["node_id"] == "node-a"
|
||||
|
||||
def test_route_create_resume_ws(self):
|
||||
"""resume_ws should route to the node that owns the old workstream."""
|
||||
router = _make_mock_router()
|
||||
router.route.return_value = NodeRef("node-b", "http://b:8080")
|
||||
app = _make_app(router=router)
|
||||
_wire_proxy(app, _make_proxy_post(json_data={"ws_id": "old_ws_resumed", "name": "resumed"}))
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"resume_ws": "old_ws_id"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["node_url"] == "http://b:8080"
|
||||
assert data["node_id"] == "node-b"
|
||||
# route() should have been called with the old ws_id
|
||||
router.route.assert_called_with("old_ws_id")
|
||||
client.close()
|
||||
|
||||
def test_route_create_target_node(self):
|
||||
"""target_node should generate a ws_id that hashes to that node."""
|
||||
router = _make_mock_router()
|
||||
router.generate_ws_id_for_node.return_value = "00ff" + "0" * 28
|
||||
router.route.return_value = NodeRef("node-c", "http://c:8080")
|
||||
app = _make_app(router=router)
|
||||
_wire_proxy(
|
||||
app,
|
||||
_make_proxy_post(json_data={"ws_id": "00ff" + "0" * 28, "name": "pinned"}),
|
||||
)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"target_node": "node-c"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["node_id"] == "node-c"
|
||||
router.generate_ws_id_for_node.assert_called_with("node-c")
|
||||
client.close()
|
||||
|
||||
|
||||
class TestRouteCreate503Retry:
|
||||
"""503 retry logic in route_create."""
|
||||
|
||||
def test_route_create_503_retries_on_different_node(self):
|
||||
"""If the first node returns 503, retry with a new ws_id targeting a different node."""
|
||||
router = _make_mock_router()
|
||||
call_count = 0
|
||||
|
||||
def side_effect_route(ws_id: str) -> NodeRef:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count <= 1:
|
||||
# First call returns node-a
|
||||
return NodeRef("node-a", "http://a:8080")
|
||||
# Subsequent calls return node-b (different node for retry)
|
||||
return NodeRef("node-b", "http://b:8080")
|
||||
|
||||
router.route.side_effect = side_effect_route
|
||||
app = _make_app(router=router)
|
||||
|
||||
post_count = 0
|
||||
|
||||
async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response:
|
||||
nonlocal post_count
|
||||
post_count += 1
|
||||
if post_count == 1:
|
||||
return httpx.Response(
|
||||
503,
|
||||
json={"error": "overloaded"},
|
||||
request=httpx.Request("POST", args[0] if args else "http://test"),
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"ws_id": "retry_ws", "name": "retry"},
|
||||
request=httpx.Request("POST", args[0] if args else "http://test"),
|
||||
)
|
||||
|
||||
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
||||
mock_proxy.post = MagicMock(side_effect=_mock_post)
|
||||
app.state.proxy_client = mock_proxy
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test-ws"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ws_id"] == "retry_ws"
|
||||
assert data["node_id"] == "node-b"
|
||||
assert post_count == 2
|
||||
client.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests — route_proxy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteProxy:
|
||||
"""POST /v1/api/route/send (and other routed endpoints)."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
router = _make_mock_router()
|
||||
app = _make_app(router=router)
|
||||
_wire_proxy(app, _make_proxy_post(json_data={"status": "ok"}))
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_route_proxy_send(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc123", "message": "hello"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# Verify upstream URL was /v1/api/send (not /v1/api/route/send)
|
||||
mock_post = client.app.state.proxy_client.post
|
||||
call_args = mock_post.call_args
|
||||
assert "/v1/api/send" in call_args[0][0]
|
||||
assert "/route/" not in call_args[0][0]
|
||||
|
||||
def test_route_proxy_approve(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/approve",
|
||||
json={"ws_id": "abc123", "approved": True},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_route_proxy_cancel(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/cancel",
|
||||
json={"ws_id": "abc123"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_route_proxy_command(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/command",
|
||||
json={"ws_id": "abc123", "command": "status"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_route_proxy_close(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/close",
|
||||
json={"ws_id": "abc123"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests — route_lookup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteLookup:
|
||||
"""GET /v1/api/route — look up which node owns a workstream."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
router = _make_mock_router()
|
||||
app = _make_app(router=router)
|
||||
app.state.proxy_client = MagicMock(spec=httpx.AsyncClient)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_route_lookup(self, client):
|
||||
resp = client.get("/v1/api/route?ws_id=abc123")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["node_url"] == "http://a:8080"
|
||||
assert data["node_id"] == "node-a"
|
||||
|
||||
def test_route_lookup_missing_ws_id(self, client):
|
||||
resp = client.get("/v1/api/route")
|
||||
assert resp.status_code == 400
|
||||
assert "ws_id" in resp.json()["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests — not ready / no router -> 503
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteNotReady:
|
||||
"""When router is None or empty cache, all routing endpoints return 503."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client_no_router(self):
|
||||
app = _make_app(router=None)
|
||||
app.state.proxy_client = MagicMock(spec=httpx.AsyncClient)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
@pytest.fixture()
|
||||
def client_empty_cache(self):
|
||||
router = _make_mock_router(ready=False)
|
||||
app = _make_app(router=router)
|
||||
app.state.proxy_client = MagicMock(spec=httpx.AsyncClient)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_route_create_no_router_503(self, client_no_router):
|
||||
resp = client_no_router.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test"},
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_route_create_empty_cache_503(self, client_empty_cache):
|
||||
resp = client_empty_cache.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test"},
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_route_proxy_no_router_503(self, client_no_router):
|
||||
resp = client_no_router.post(
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc", "message": "hello"},
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_route_lookup_no_router_503(self, client_no_router):
|
||||
resp = client_no_router.get("/v1/api/route?ws_id=abc")
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_route_proxy_empty_cache_503(self, client_empty_cache):
|
||||
resp = client_empty_cache.post(
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc", "message": "hello"},
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_route_lookup_empty_cache_503(self, client_empty_cache):
|
||||
resp = client_empty_cache.get("/v1/api/route?ws_id=abc")
|
||||
assert resp.status_code == 503
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests — NoAvailableNodeError handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteNoNode:
|
||||
"""When router.route() raises NoAvailableNodeError, endpoints return 503."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
router = _make_mock_router()
|
||||
router.route.side_effect = NoAvailableNodeError("bucket 0 not assigned")
|
||||
app = _make_app(router=router)
|
||||
app.state.proxy_client = MagicMock(spec=httpx.AsyncClient)
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
def test_route_create_no_node_503(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test"},
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
assert "No available node" in resp.json()["error"]
|
||||
|
||||
def test_route_proxy_no_node_503(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc", "message": "hello"},
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_route_lookup_no_node_503(self, client):
|
||||
resp = client.get("/v1/api/route?ws_id=abc")
|
||||
assert resp.status_code == 503
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Tests for turnstone.core.hash_ring."""
|
||||
|
||||
from turnstone.core.hash_ring import bucket_of
|
||||
|
||||
|
||||
class TestBucketOf:
|
||||
def test_known_vectors(self):
|
||||
assert bucket_of("a3f1" + "0" * 28) == 0xA3F1
|
||||
assert bucket_of("0000" + "a" * 28) == 0
|
||||
assert bucket_of("ffff" + "b" * 28) == 65535
|
||||
|
||||
def test_hex_prefix(self):
|
||||
# Only the first 4 hex chars matter — the rest is ignored.
|
||||
assert bucket_of("abcd0000") == bucket_of("abcdffff")
|
||||
assert bucket_of("abcd0000") == 0xABCD
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Tests for the hash ring routing storage methods."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class TestHashRingBuckets:
|
||||
def test_list_empty(self, storage):
|
||||
assert storage.list_ring_buckets() == []
|
||||
|
||||
def test_seed_and_list(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-b"), (2, "node-a")])
|
||||
rows = storage.list_ring_buckets()
|
||||
assert len(rows) == 3
|
||||
assert rows[0] == {"bucket": 0, "node_id": "node-a"}
|
||||
assert rows[1] == {"bucket": 1, "node_id": "node-b"}
|
||||
assert rows[2] == {"bucket": 2, "node_id": "node-a"}
|
||||
|
||||
def test_seed_idempotent(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-b")])
|
||||
# Re-seed with conflicting assignment: should keep original
|
||||
storage.seed_ring_buckets([(0, "node-x"), (2, "node-c")])
|
||||
rows = storage.list_ring_buckets()
|
||||
by_bucket = {r["bucket"]: r["node_id"] for r in rows}
|
||||
assert by_bucket[0] == "node-a" # original preserved
|
||||
assert by_bucket[1] == "node-b"
|
||||
assert by_bucket[2] == "node-c" # new bucket added
|
||||
|
||||
def test_assign_buckets(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-a"), (2, "node-b")])
|
||||
storage.assign_buckets([0, 1], "node-c")
|
||||
rows = storage.list_ring_buckets()
|
||||
by_bucket = {r["bucket"]: r["node_id"] for r in rows}
|
||||
assert by_bucket[0] == "node-c"
|
||||
assert by_bucket[1] == "node-c"
|
||||
assert by_bucket[2] == "node-b"
|
||||
|
||||
def test_assign_returns_count(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-a")])
|
||||
count = storage.assign_buckets([0, 1], "node-b")
|
||||
assert count == 2
|
||||
# Empty list returns 0
|
||||
assert storage.assign_buckets([], "node-x") == 0
|
||||
|
||||
|
||||
class TestBucketStats:
|
||||
def test_increment_creates_row(self, storage):
|
||||
storage.increment_bucket_count(42)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert len(stats) == 1
|
||||
assert stats[0]["bucket"] == 42
|
||||
assert stats[0]["ws_count"] == 1
|
||||
assert stats[0]["active_count"] == 0
|
||||
|
||||
def test_increment_active(self, storage):
|
||||
storage.increment_bucket_count(10, active=True)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 1
|
||||
assert stats[0]["active_count"] == 1
|
||||
# Increment again without active
|
||||
storage.increment_bucket_count(10)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 2
|
||||
assert stats[0]["active_count"] == 1
|
||||
|
||||
def test_decrement(self, storage):
|
||||
storage.increment_bucket_count(5, active=True)
|
||||
storage.increment_bucket_count(5, active=True)
|
||||
storage.decrement_bucket_count(5, active=True)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 1
|
||||
assert stats[0]["active_count"] == 1
|
||||
|
||||
def test_decrement_clamps_at_zero(self, storage):
|
||||
storage.increment_bucket_count(7)
|
||||
storage.decrement_bucket_count(7)
|
||||
storage.decrement_bucket_count(7) # already at 0
|
||||
stats = storage.list_bucket_stats()
|
||||
# ws_count is 0, so should not appear (filter ws_count > 0)
|
||||
assert len(stats) == 0
|
||||
|
||||
def test_adjust_active_only(self, storage):
|
||||
storage.increment_bucket_count(20, active=True)
|
||||
storage.increment_bucket_count(20, active=True)
|
||||
# Decrease active without changing ws_count
|
||||
storage.adjust_bucket_active(20, -1)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 2
|
||||
assert stats[0]["active_count"] == 1
|
||||
# Clamp at zero
|
||||
storage.adjust_bucket_active(20, -5)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["active_count"] == 0
|
||||
|
||||
def test_list_sparse(self, storage):
|
||||
storage.increment_bucket_count(100)
|
||||
storage.increment_bucket_count(200)
|
||||
storage.increment_bucket_count(300)
|
||||
# Decrement 200 to zero
|
||||
storage.decrement_bucket_count(200)
|
||||
stats = storage.list_bucket_stats()
|
||||
buckets = [s["bucket"] for s in stats]
|
||||
assert 100 in buckets
|
||||
assert 200 not in buckets
|
||||
assert 300 in buckets
|
||||
|
||||
def test_set_bucket_stat_creates(self, storage):
|
||||
"""set_bucket_stat upserts a new row."""
|
||||
storage.set_bucket_stat(42, 5, 2)
|
||||
stats = storage.list_bucket_stats()
|
||||
row = next(s for s in stats if s["bucket"] == 42)
|
||||
assert row["ws_count"] == 5
|
||||
assert row["active_count"] == 2
|
||||
|
||||
def test_set_bucket_stat_overwrites(self, storage):
|
||||
"""set_bucket_stat overwrites existing values."""
|
||||
storage.set_bucket_stat(42, 10, 3)
|
||||
storage.set_bucket_stat(42, 2, 0)
|
||||
stats = storage.list_bucket_stats()
|
||||
row = next(s for s in stats if s["bucket"] == 42)
|
||||
assert row["ws_count"] == 2
|
||||
assert row["active_count"] == 0
|
||||
|
||||
def test_set_bucket_stat_zero_removes_from_sparse(self, storage):
|
||||
"""Setting ws_count=0 means list_bucket_stats excludes it (sparse)."""
|
||||
storage.set_bucket_stat(42, 5, 1)
|
||||
storage.set_bucket_stat(42, 0, 0)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert not any(s["bucket"] == 42 for s in stats)
|
||||
|
||||
|
||||
class TestWorkstreamOverrides:
|
||||
def test_set_and_list(self, storage):
|
||||
storage.set_workstream_override("ws-001", "node-a", reason="affinity")
|
||||
overrides = storage.list_workstream_overrides()
|
||||
assert len(overrides) == 1
|
||||
assert overrides[0]["ws_id"] == "ws-001"
|
||||
assert overrides[0]["node_id"] == "node-a"
|
||||
assert overrides[0]["reason"] == "affinity"
|
||||
|
||||
def test_upsert(self, storage):
|
||||
storage.set_workstream_override("ws-002", "node-a")
|
||||
storage.set_workstream_override("ws-002", "node-b", reason="migration")
|
||||
overrides = storage.list_workstream_overrides()
|
||||
assert len(overrides) == 1
|
||||
assert overrides[0]["node_id"] == "node-b"
|
||||
assert overrides[0]["reason"] == "migration"
|
||||
|
||||
def test_delete(self, storage):
|
||||
storage.set_workstream_override("ws-003", "node-a")
|
||||
result = storage.delete_workstream_override("ws-003")
|
||||
assert result is True
|
||||
assert storage.list_workstream_overrides() == []
|
||||
|
||||
def test_delete_nonexistent(self, storage):
|
||||
result = storage.delete_workstream_override("ws-nope")
|
||||
assert result is False
|
||||
|
||||
def test_list_empty(self, storage):
|
||||
assert storage.list_workstream_overrides() == []
|
||||
@@ -425,18 +425,23 @@ class TestSkillCatalogDisclosure:
|
||||
session._mcp_client = None
|
||||
session._notify_on_complete = "{}"
|
||||
session._tool_error_flags = {}
|
||||
from turnstone.prompts import ClientType
|
||||
|
||||
session._tools = []
|
||||
session._client_type = ClientType.CLI
|
||||
session._username = ""
|
||||
|
||||
# Memory stubs
|
||||
session._memory_config = MagicMock()
|
||||
session._memory_config.fetch_limit = 0
|
||||
session._user_id = ""
|
||||
session._user_id = "test-user"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"turnstone.core.session.list_skills_by_activation",
|
||||
return_value=search_skills or [],
|
||||
),
|
||||
patch.object(session, "_get_visible_memories", return_value=[]),
|
||||
patch.object(session, "_list_visible_memories", return_value=[]),
|
||||
):
|
||||
session._init_system_messages()
|
||||
|
||||
|
||||
@@ -20,12 +20,15 @@ def _mock_model(
|
||||
*,
|
||||
owned_by: str = "test",
|
||||
meta: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> MagicMock:
|
||||
m = MagicMock()
|
||||
m.id = model_id
|
||||
dumped: dict[str, Any] = {"owned_by": owned_by}
|
||||
if meta is not None:
|
||||
dumped["meta"] = meta
|
||||
if kwargs.get("max_model_len") is not None:
|
||||
dumped["max_model_len"] = kwargs["max_model_len"]
|
||||
m.model_dump.return_value = dumped
|
||||
return m
|
||||
|
||||
@@ -109,6 +112,27 @@ class TestProbeModelEndpoint:
|
||||
result = probe_model_endpoint("openai", "http://localhost:30000/v1", "key")
|
||||
assert result["server_type"] == "sglang"
|
||||
|
||||
@patch("turnstone.core.providers.create_client")
|
||||
def test_vllm_max_model_len(self, mock_cc: MagicMock) -> None:
|
||||
m = _mock_model("/models/nemotron", max_model_len=262144, owned_by="vllm")
|
||||
mock_cc.return_value = _mock_client(m)
|
||||
|
||||
result = probe_model_endpoint("openai", "http://localhost:8000/v1", "key")
|
||||
assert result["context_window"] == 262144
|
||||
assert result["server_type"] == "vllm"
|
||||
|
||||
@patch("turnstone.core.providers.create_client")
|
||||
def test_vllm_max_model_len_preferred_over_meta(self, mock_cc: MagicMock) -> None:
|
||||
m = _mock_model(
|
||||
"/models/test",
|
||||
meta={"n_ctx_train": 8192},
|
||||
max_model_len=131072,
|
||||
)
|
||||
mock_cc.return_value = _mock_client(m)
|
||||
|
||||
result = probe_model_endpoint("openai", "http://localhost:8000/v1", "key")
|
||||
assert result["context_window"] == 131072
|
||||
|
||||
@patch("turnstone.core.providers.create_client")
|
||||
def test_server_type_vllm(self, mock_cc: MagicMock) -> None:
|
||||
m = _mock_model("org/model-name")
|
||||
@@ -138,7 +162,7 @@ class TestProbeModelEndpoint:
|
||||
)
|
||||
assert result["reachable"] is True
|
||||
assert result["server_type"] == "anthropic"
|
||||
assert result["context_window"] == 200000
|
||||
assert result["context_window"] == 1000000
|
||||
|
||||
@patch("turnstone.core.providers.create_client")
|
||||
def test_connection_failure(self, mock_cc: MagicMock) -> None:
|
||||
@@ -184,7 +208,7 @@ class TestLookupModelCapabilities:
|
||||
def test_known_anthropic_model(self) -> None:
|
||||
caps = lookup_model_capabilities("anthropic", "claude-opus-4-6")
|
||||
assert caps is not None
|
||||
assert caps["context_window"] == 200000
|
||||
assert caps["context_window"] == 1000000
|
||||
assert caps["thinking_mode"] == "adaptive"
|
||||
|
||||
def test_unknown_model_returns_none(self) -> None:
|
||||
|
||||
+136
-19
@@ -840,32 +840,27 @@ class TestWorkstreamModelParam:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol
|
||||
# CreateWorkstreamRequest model field
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProtocolModel:
|
||||
def test_create_workstream_message_has_model(self) -> None:
|
||||
from turnstone.mq.protocol import CreateWorkstreamMessage
|
||||
class TestCreateWorkstreamRequestModel:
|
||||
def test_request_has_model(self) -> None:
|
||||
from turnstone.api.server_schemas import CreateWorkstreamRequest
|
||||
|
||||
msg = CreateWorkstreamMessage(name="test", model="openai")
|
||||
assert msg.model == "openai"
|
||||
req = CreateWorkstreamRequest(name="test", model="openai")
|
||||
assert req.model == "openai"
|
||||
|
||||
def test_create_workstream_message_default(self) -> None:
|
||||
from turnstone.mq.protocol import CreateWorkstreamMessage
|
||||
def test_request_model_default(self) -> None:
|
||||
from turnstone.api.server_schemas import CreateWorkstreamRequest
|
||||
|
||||
msg = CreateWorkstreamMessage(name="test")
|
||||
assert msg.model == ""
|
||||
req = CreateWorkstreamRequest(name="test")
|
||||
assert req.model == ""
|
||||
|
||||
def test_round_trip(self) -> None:
|
||||
from turnstone.mq.protocol import CreateWorkstreamMessage, InboundMessage
|
||||
|
||||
msg = CreateWorkstreamMessage(name="ws1", model="local")
|
||||
raw = msg.to_json()
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert isinstance(restored, CreateWorkstreamMessage)
|
||||
assert restored.model == "local"
|
||||
assert restored.name == "ws1"
|
||||
def test_json_payload_carries_model(self) -> None:
|
||||
body = {"name": "ws1", "model": "local"}
|
||||
assert body["model"] == "local"
|
||||
assert body["name"] == "ws1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -899,3 +894,125 @@ class TestDetectModelTimeout:
|
||||
|
||||
result = detect_model(client, provider="openai", fatal=False)
|
||||
assert result == (None, None)
|
||||
|
||||
def test_vllm_max_model_len_detected(self) -> None:
|
||||
"""detect_model() reads max_model_len from vLLM model objects."""
|
||||
mock_model = MagicMock()
|
||||
mock_model.id = "/models/nemotron"
|
||||
mock_model.model_dump.return_value = {
|
||||
"owned_by": "vllm",
|
||||
"max_model_len": 262144,
|
||||
}
|
||||
|
||||
fast_client = MagicMock()
|
||||
fast_client.models.list.return_value = MagicMock(data=[mock_model])
|
||||
|
||||
client = MagicMock()
|
||||
client.with_options.return_value = fast_client
|
||||
|
||||
model_id, ctx = detect_model(client, provider="openai")
|
||||
assert model_id == "/models/nemotron"
|
||||
assert ctx == 262144
|
||||
|
||||
|
||||
class TestExtractContextWindow:
|
||||
def test_vllm_max_model_len(self) -> None:
|
||||
from turnstone.core.model_registry import _extract_context_window
|
||||
|
||||
m = MagicMock()
|
||||
m.id = "/models/test"
|
||||
m.model_dump.return_value = {"max_model_len": 131072}
|
||||
assert _extract_context_window(m, "openai") == 131072
|
||||
|
||||
def test_llama_cpp_meta(self) -> None:
|
||||
from turnstone.core.model_registry import _extract_context_window
|
||||
|
||||
m = MagicMock()
|
||||
m.id = "test"
|
||||
m.model_dump.return_value = {"meta": {"n_ctx_train": 8192}}
|
||||
assert _extract_context_window(m, "openai") == 8192
|
||||
|
||||
def test_vllm_preferred_over_meta(self) -> None:
|
||||
from turnstone.core.model_registry import _extract_context_window
|
||||
|
||||
m = MagicMock()
|
||||
m.id = "test"
|
||||
m.model_dump.return_value = {"max_model_len": 262144, "meta": {"n_ctx_train": 4096}}
|
||||
assert _extract_context_window(m, "openai") == 262144
|
||||
|
||||
def test_no_metadata_returns_none(self) -> None:
|
||||
from turnstone.core.model_registry import _extract_context_window
|
||||
|
||||
m = MagicMock()
|
||||
m.id = "test"
|
||||
m.model_dump.return_value = {}
|
||||
assert _extract_context_window(m, "openai") is None
|
||||
|
||||
|
||||
class TestHealthMonitorModelChange:
|
||||
def test_model_change_fires_callback(self) -> None:
|
||||
from turnstone.core.healthcheck import BackendHealthMonitor
|
||||
|
||||
changes: list[tuple[str, int | None]] = []
|
||||
|
||||
def on_change(model_id: str, ctx: int | None) -> None:
|
||||
changes.append((model_id, ctx))
|
||||
|
||||
client = MagicMock()
|
||||
monitor = BackendHealthMonitor(
|
||||
client=client,
|
||||
provider="openai",
|
||||
initial_model="model-a",
|
||||
on_model_changed=on_change,
|
||||
)
|
||||
|
||||
# Simulate probe returning a different model
|
||||
resp = MagicMock()
|
||||
m = MagicMock()
|
||||
m.id = "model-b"
|
||||
m.model_dump.return_value = {"max_model_len": 131072}
|
||||
resp.data = [m]
|
||||
|
||||
monitor._check_model_change(resp)
|
||||
assert len(changes) == 1
|
||||
assert changes[0] == ("model-b", 131072)
|
||||
assert monitor._last_detected_model == "model-b"
|
||||
|
||||
def test_same_model_no_callback(self) -> None:
|
||||
from turnstone.core.healthcheck import BackendHealthMonitor
|
||||
|
||||
changes: list[tuple[str, int | None]] = []
|
||||
|
||||
def on_change(model_id: str, ctx: int | None) -> None:
|
||||
changes.append((model_id, ctx))
|
||||
|
||||
client = MagicMock()
|
||||
monitor = BackendHealthMonitor(
|
||||
client=client,
|
||||
provider="openai",
|
||||
initial_model="model-a",
|
||||
on_model_changed=on_change,
|
||||
)
|
||||
|
||||
resp = MagicMock()
|
||||
m = MagicMock()
|
||||
m.id = "model-a"
|
||||
m.model_dump.return_value = {}
|
||||
resp.data = [m]
|
||||
|
||||
monitor._check_model_change(resp)
|
||||
assert len(changes) == 0
|
||||
|
||||
def test_no_callback_configured(self) -> None:
|
||||
from turnstone.core.healthcheck import BackendHealthMonitor
|
||||
|
||||
client = MagicMock()
|
||||
monitor = BackendHealthMonitor(client=client, initial_model="model-a")
|
||||
|
||||
resp = MagicMock()
|
||||
m = MagicMock()
|
||||
m.id = "model-b"
|
||||
resp.data = [m]
|
||||
|
||||
# Should not raise
|
||||
monitor._check_model_change(resp)
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Tests for the system message composition harness (turnstone.prompts)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.prompts import (
|
||||
ClientType,
|
||||
SessionContext,
|
||||
compose_system_message,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALID_CTX = SessionContext(
|
||||
current_datetime="2026-03-31T14:22:00-07:00",
|
||||
timezone="PDT",
|
||||
username="sarah.chen",
|
||||
)
|
||||
|
||||
_ALL_TOOLS: frozenset[str] = frozenset({"web_search", "read_file", "bash"})
|
||||
_NO_TOOLS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Assembly smoke test per client type
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ct", [ClientType.WEB, ClientType.CLI, ClientType.CHAT])
|
||||
def test_smoke_all_client_types(ct: ClientType) -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ct,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
# BASE content present
|
||||
assert "resident engineer" in result
|
||||
# CONTEXT present
|
||||
assert "sarah.chen" in result
|
||||
assert "2026-03-31" in result
|
||||
|
||||
|
||||
def test_smoke_web_has_mermaid() -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ClientType.WEB,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
assert "Mermaid" in result
|
||||
assert "KaTeX" in result
|
||||
|
||||
|
||||
def test_smoke_cli_no_mermaid() -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ClientType.CLI,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
assert "Mermaid" not in result or "Do not use" in result
|
||||
|
||||
|
||||
def test_smoke_chat_no_tables() -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ClientType.CHAT,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
assert "Do not use them" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Required field validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_current_datetime() -> None:
|
||||
ctx = SessionContext(current_datetime="", timezone="PDT", username="alice")
|
||||
with pytest.raises(ValueError, match="current_datetime"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
def test_missing_timezone() -> None:
|
||||
ctx = SessionContext(
|
||||
current_datetime="2026-03-31T14:22:00-07:00",
|
||||
timezone="",
|
||||
username="alice",
|
||||
)
|
||||
with pytest.raises(ValueError, match="timezone"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
def test_missing_username() -> None:
|
||||
ctx = SessionContext(
|
||||
current_datetime="2026-03-31T14:22:00-07:00",
|
||||
timezone="PDT",
|
||||
username="",
|
||||
)
|
||||
with pytest.raises(ValueError, match="username"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Unknown client type rejection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unknown_client_type() -> None:
|
||||
with pytest.raises(ValueError, match="Unknown client_type"):
|
||||
compose_system_message("tablet", _VALID_CTX, _NO_TOOLS) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Missing policy file
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_policy_file() -> None:
|
||||
with pytest.raises(FileNotFoundError, match="nonexistent"):
|
||||
compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
policies=["nonexistent"],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Module isolation — BASE must be environment-agnostic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_base_module_isolation() -> None:
|
||||
from turnstone.prompts import _load
|
||||
|
||||
base = _load("base.md")
|
||||
for forbidden in ("Mermaid", "KaTeX", "terminal", "monospace", "Slack", "Discord"):
|
||||
assert forbidden not in base, f"BASE must not contain '{forbidden}'"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. ENV mutual exclusion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_env_mutual_exclusion() -> None:
|
||||
web = compose_system_message(ClientType.WEB, _VALID_CTX, _ALL_TOOLS)
|
||||
# Web should have Mermaid.js but not "No diagram rendering" from CLI
|
||||
assert "Mermaid" in web
|
||||
assert "No diagram rendering" not in web
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. Policy tool gating — file-based (negative case)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_policy_gated_out() -> None:
|
||||
"""web_search policy excluded when web_search tool is not available."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
frozenset({"read_file"}), # no web_search
|
||||
policies=["web_search"],
|
||||
)
|
||||
assert "Web Search Policy" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 8. Policy tool gating — positive case
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_policy_gated_in() -> None:
|
||||
"""web_search policy included when web_search tool is available."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
frozenset({"web_search"}),
|
||||
policies=["web_search"],
|
||||
)
|
||||
assert "Web Search Policy" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. Unconditional policy not gated
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unconditional_policy() -> None:
|
||||
"""A DB policy with no tool_gate is always included."""
|
||||
db = [
|
||||
{
|
||||
"name": "custom_rule",
|
||||
"content": "## Custom Rule\nAlways be polite.",
|
||||
"tool_gate": "",
|
||||
"priority": 0,
|
||||
"enabled": True,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
assert "Always be polite" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. DB policy override
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_db_policy_overrides_file() -> None:
|
||||
"""DB policy with same name as file policy wins."""
|
||||
db = [
|
||||
{
|
||||
"name": "web_search",
|
||||
"content": "## DB Web Search Override\nCustom content.",
|
||||
"tool_gate": "web_search",
|
||||
"priority": 0,
|
||||
"enabled": True,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
policies=["web_search"],
|
||||
db_policies=db,
|
||||
)
|
||||
assert "DB Web Search Override" in result
|
||||
assert "Use local tools" not in result # original file content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 11. DB-only policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_db_only_policy() -> None:
|
||||
"""DB policy not in explicit list is still included."""
|
||||
db = [
|
||||
{
|
||||
"name": "extra_rule",
|
||||
"content": "## Extra\nDo not share secrets.",
|
||||
"tool_gate": "",
|
||||
"priority": 5,
|
||||
"enabled": True,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
assert "Do not share secrets" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 12. Disabled DB policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_disabled_db_policy() -> None:
|
||||
"""DB policy with enabled=False is skipped."""
|
||||
db = [
|
||||
{
|
||||
"name": "disabled_rule",
|
||||
"content": "## Disabled\nThis should not appear.",
|
||||
"tool_gate": "",
|
||||
"priority": 0,
|
||||
"enabled": False,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
assert "This should not appear" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 13. ISO 8601 validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_invalid_iso_datetime() -> None:
|
||||
ctx = SessionContext(
|
||||
current_datetime="not-a-date",
|
||||
timezone="PDT",
|
||||
username="alice",
|
||||
)
|
||||
with pytest.raises(ValueError, match="not valid ISO 8601"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 14. DB policy priority ordering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_db_policy_priority_ordering() -> None:
|
||||
"""DB-only policies are assembled in priority order (ascending)."""
|
||||
db = [
|
||||
{
|
||||
"name": "second",
|
||||
"content": "SECOND_MARKER",
|
||||
"tool_gate": "",
|
||||
"priority": 10,
|
||||
"enabled": True,
|
||||
},
|
||||
{
|
||||
"name": "first",
|
||||
"content": "FIRST_MARKER",
|
||||
"tool_gate": "",
|
||||
"priority": 1,
|
||||
"enabled": True,
|
||||
},
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
first_pos = result.index("FIRST_MARKER")
|
||||
second_pos = result.index("SECOND_MARKER")
|
||||
assert first_pos < second_pos
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 15. TOOLS module excluded when no tools available
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tools_excluded_when_no_tools() -> None:
|
||||
"""TOOLS module is not included when available_tools is empty."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
)
|
||||
assert "TOOL PATTERNS" not in result
|
||||
|
||||
|
||||
def test_tools_included_when_tools_available() -> None:
|
||||
"""TOOLS module is included when available_tools is non-empty."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
)
|
||||
assert "TOOL PATTERNS" in result
|
||||
@@ -1,239 +0,0 @@
|
||||
"""Tests for turnstone.mq.protocol message serialization."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.mq.protocol import (
|
||||
AckEvent,
|
||||
ApprovalRequestEvent,
|
||||
ApproveMessage,
|
||||
CancelMessage,
|
||||
CloseWorkstreamMessage,
|
||||
CommandMessage,
|
||||
ContentEvent,
|
||||
CreateWorkstreamMessage,
|
||||
ErrorEvent,
|
||||
HealthMessage,
|
||||
HealthResponseEvent,
|
||||
InboundMessage,
|
||||
InfoEvent,
|
||||
ListNodesMessage,
|
||||
ListWorkstreamsMessage,
|
||||
NodeListEvent,
|
||||
OutboundEvent,
|
||||
PlanFeedbackMessage,
|
||||
PlanReviewEvent,
|
||||
ReasoningEvent,
|
||||
SendMessage,
|
||||
StateChangeEvent,
|
||||
StatusEvent,
|
||||
StreamEndEvent,
|
||||
ToolInfoEvent,
|
||||
ToolResultEvent,
|
||||
TurnCompleteEvent,
|
||||
WorkstreamClosedEvent,
|
||||
WorkstreamCreatedEvent,
|
||||
WorkstreamListEvent,
|
||||
WorkstreamRenameEvent,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inbound message round-trip tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
INBOUND_TYPES = [
|
||||
(
|
||||
SendMessage,
|
||||
{
|
||||
"message": "hello",
|
||||
"ws_id": "abc",
|
||||
"auto_approve": True,
|
||||
"auto_approve_tools": ["bash"],
|
||||
},
|
||||
),
|
||||
(
|
||||
ApproveMessage,
|
||||
{"ws_id": "abc", "request_id": "r1", "approved": True, "feedback": "ok"},
|
||||
),
|
||||
(
|
||||
PlanFeedbackMessage,
|
||||
{"ws_id": "abc", "request_id": "r2", "feedback": "looks good"},
|
||||
),
|
||||
(CommandMessage, {"ws_id": "abc", "command": "/clear"}),
|
||||
(
|
||||
CreateWorkstreamMessage,
|
||||
{"name": "test-ws", "auto_approve": False, "auto_approve_tools": ["read_file"]},
|
||||
),
|
||||
(CloseWorkstreamMessage, {"ws_id": "abc"}),
|
||||
(ListWorkstreamsMessage, {}),
|
||||
(HealthMessage, {}),
|
||||
(ListNodesMessage, {}),
|
||||
(CancelMessage, {"ws_id": "abc"}),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls,kwargs", INBOUND_TYPES)
|
||||
def test_inbound_round_trip(cls, kwargs):
|
||||
msg = cls(**kwargs)
|
||||
raw = msg.to_json()
|
||||
parsed = json.loads(raw)
|
||||
|
||||
# type field matches
|
||||
assert parsed["type"] == msg.type
|
||||
|
||||
# correlation_id auto-generated
|
||||
assert len(msg.correlation_id) == 12
|
||||
assert parsed["correlation_id"] == msg.correlation_id
|
||||
|
||||
# timestamp present
|
||||
assert msg.timestamp > 0
|
||||
|
||||
# Deserialize back
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert type(restored) is cls
|
||||
assert restored.type == msg.type
|
||||
assert restored.correlation_id == msg.correlation_id
|
||||
|
||||
# Check custom fields
|
||||
for k, v in kwargs.items():
|
||||
assert getattr(restored, k) == v
|
||||
|
||||
|
||||
def test_inbound_unknown_type():
|
||||
with pytest.raises(ValueError, match="Unknown inbound"):
|
||||
InboundMessage.from_json('{"type": "nonexistent"}')
|
||||
|
||||
|
||||
def test_inbound_extra_fields_ignored():
|
||||
raw = json.dumps({"type": "send", "message": "hi", "extra_field": 42})
|
||||
msg = InboundMessage.from_json(raw)
|
||||
assert isinstance(msg, SendMessage)
|
||||
assert msg.message == "hi"
|
||||
assert not hasattr(msg, "extra_field")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Outbound event round-trip tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
OUTBOUND_TYPES = [
|
||||
(AckEvent, {"status": "ok", "detail": "done"}),
|
||||
(ContentEvent, {"text": "hello world"}),
|
||||
(ReasoningEvent, {"text": "thinking..."}),
|
||||
(ToolInfoEvent, {"items": [{"name": "bash", "preview": "ls"}]}),
|
||||
(ApprovalRequestEvent, {"items": [{"name": "bash", "needs_approval": True}]}),
|
||||
(ToolResultEvent, {"call_id": "call_123", "name": "bash", "output": "file.txt"}),
|
||||
(PlanReviewEvent, {"content": "# Plan\n\nStep 1: ..."}),
|
||||
(StatusEvent, {"prompt_tokens": 100, "completion_tokens": 50, "pct": 0.42}),
|
||||
(StateChangeEvent, {"state": "thinking"}),
|
||||
(TurnCompleteEvent, {}),
|
||||
(StreamEndEvent, {}),
|
||||
(WorkstreamCreatedEvent, {"name": "test-ws"}),
|
||||
(WorkstreamClosedEvent, {}),
|
||||
(WorkstreamListEvent, {"workstreams": [{"id": "abc", "name": "ws"}]}),
|
||||
(WorkstreamRenameEvent, {"name": "renamed"}),
|
||||
(HealthResponseEvent, {"data": {"status": "ok"}}),
|
||||
(ErrorEvent, {"message": "something broke"}),
|
||||
(InfoEvent, {"message": "heads up"}),
|
||||
(
|
||||
NodeListEvent,
|
||||
{"nodes": [{"node_id": "server-12", "server_url": "http://x:8080"}]},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls,kwargs", OUTBOUND_TYPES)
|
||||
def test_outbound_round_trip(cls, kwargs):
|
||||
event = cls(ws_id="ws1", correlation_id="c1", **kwargs)
|
||||
raw = event.to_json()
|
||||
parsed = json.loads(raw)
|
||||
|
||||
assert parsed["type"] == event.type
|
||||
assert parsed["ws_id"] == "ws1"
|
||||
assert parsed["correlation_id"] == "c1"
|
||||
|
||||
restored = OutboundEvent.from_json(raw)
|
||||
assert type(restored) is cls
|
||||
assert restored.ws_id == "ws1"
|
||||
assert restored.correlation_id == "c1"
|
||||
|
||||
for k, v in kwargs.items():
|
||||
assert getattr(restored, k) == v
|
||||
|
||||
|
||||
def test_outbound_unknown_type_falls_back():
|
||||
raw = json.dumps({"type": "future_event", "ws_id": "x"})
|
||||
event = OutboundEvent.from_json(raw)
|
||||
assert isinstance(event, OutboundEvent)
|
||||
assert event.ws_id == "x"
|
||||
|
||||
|
||||
def test_send_message_defaults():
|
||||
msg = SendMessage(message="hello")
|
||||
assert msg.ws_id == ""
|
||||
assert msg.auto_approve is False
|
||||
assert msg.auto_approve_tools == []
|
||||
assert msg.name == ""
|
||||
assert msg.target_node == ""
|
||||
assert len(msg.correlation_id) == 12
|
||||
|
||||
|
||||
def test_create_workstream_with_tools():
|
||||
msg = CreateWorkstreamMessage(
|
||||
name="ci-runner",
|
||||
auto_approve=False,
|
||||
auto_approve_tools=["bash", "read_file", "search"],
|
||||
)
|
||||
raw = msg.to_json()
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert restored.auto_approve_tools == ["bash", "read_file", "search"]
|
||||
assert restored.name == "ci-runner"
|
||||
|
||||
|
||||
def test_send_message_target_node():
|
||||
msg = SendMessage(message="check disk", target_node="server-12")
|
||||
raw = msg.to_json()
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert isinstance(restored, SendMessage)
|
||||
assert restored.target_node == "server-12"
|
||||
assert restored.message == "check disk"
|
||||
|
||||
|
||||
def test_create_workstream_target_node():
|
||||
msg = CreateWorkstreamMessage(name="debug-ws", target_node="gpu-node-3")
|
||||
raw = msg.to_json()
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert isinstance(restored, CreateWorkstreamMessage)
|
||||
assert restored.target_node == "gpu-node-3"
|
||||
assert restored.name == "debug-ws"
|
||||
|
||||
|
||||
def test_create_workstream_skill_field():
|
||||
msg = CreateWorkstreamMessage(name="ws", skill="code-review")
|
||||
assert msg.skill == "code-review"
|
||||
raw = msg.to_json()
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert isinstance(restored, CreateWorkstreamMessage)
|
||||
assert restored.skill == "code-review"
|
||||
|
||||
|
||||
def test_create_workstream_skill_default_empty():
|
||||
msg = CreateWorkstreamMessage(name="ws")
|
||||
assert msg.skill == ""
|
||||
|
||||
|
||||
def test_list_nodes_round_trip():
|
||||
msg = ListNodesMessage()
|
||||
raw = msg.to_json()
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert isinstance(restored, ListNodesMessage)
|
||||
|
||||
|
||||
def test_node_list_event_round_trip():
|
||||
nodes = [{"node_id": "a", "server_url": "http://a:8080"}]
|
||||
event = NodeListEvent(nodes=nodes, correlation_id="c1")
|
||||
raw = event.to_json()
|
||||
restored = OutboundEvent.from_json(raw)
|
||||
assert isinstance(restored, NodeListEvent)
|
||||
assert restored.nodes == nodes
|
||||
@@ -957,7 +957,7 @@ class TestAnthropicHelpers:
|
||||
|
||||
provider = AnthropicProvider()
|
||||
caps = provider.get_capabilities("claude-opus-4-6")
|
||||
assert caps.context_window == 200000
|
||||
assert caps.context_window == 1000000
|
||||
assert caps.max_output_tokens == 128000
|
||||
assert caps.thinking_mode == "adaptive"
|
||||
assert caps.supports_effort is True
|
||||
@@ -966,11 +966,11 @@ class TestAnthropicHelpers:
|
||||
from turnstone.core.providers._anthropic import AnthropicProvider
|
||||
|
||||
provider = AnthropicProvider()
|
||||
# Prefix match: "claude-sonnet-4" matches dated variants
|
||||
caps = provider.get_capabilities("claude-sonnet-4-20260101")
|
||||
assert caps.context_window == 200000
|
||||
# Prefix match: "claude-sonnet-4-6" matches dated variants
|
||||
caps = provider.get_capabilities("claude-sonnet-4-6-20260101")
|
||||
assert caps.context_window == 1000000
|
||||
assert caps.token_param == "max_tokens"
|
||||
assert caps.thinking_mode == "manual"
|
||||
assert caps.thinking_mode == "adaptive"
|
||||
|
||||
def test_capabilities_lookup_unknown(self) -> None:
|
||||
from turnstone.core.providers._anthropic import AnthropicProvider
|
||||
@@ -1407,7 +1407,7 @@ class TestAnthropicWebSearch:
|
||||
"""All Anthropic models should support native web search."""
|
||||
caps = self.provider.get_capabilities("claude-opus-4-6")
|
||||
assert caps.supports_web_search is True
|
||||
caps = self.provider.get_capabilities("claude-sonnet-4")
|
||||
caps = self.provider.get_capabilities("claude-sonnet-4-6")
|
||||
assert caps.supports_web_search is True
|
||||
# Unknown models use default which also has web search
|
||||
caps = self.provider.get_capabilities("claude-unknown-99")
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
"""Tests for turnstone.console.rebalancer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.console.rebalancer import Rebalancer
|
||||
from turnstone.core.hash_ring import RING_SIZE
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
"""Fresh SQLite backend for each test."""
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
def _register_nodes(storage: SQLiteBackend, count: int, *, weight: int = 1) -> None:
|
||||
"""Register *count* server nodes in the services table."""
|
||||
for i in range(count):
|
||||
meta = json.dumps({"weight": weight, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", f"node-{i}", f"http://node-{i}:8080", metadata=meta)
|
||||
|
||||
|
||||
def _register_weighted_nodes(storage: SQLiteBackend, weights: dict[str, int]) -> None:
|
||||
"""Register nodes with specific weights."""
|
||||
for node_id, w in weights.items():
|
||||
meta = json.dumps({"weight": w, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", node_id, f"http://{node_id}:8080", metadata=meta)
|
||||
|
||||
|
||||
def _get_version(storage: SQLiteBackend) -> int:
|
||||
"""Read the rebalancer_version from system_settings."""
|
||||
raw = storage.get_system_setting("rebalancer_version", node_id="")
|
||||
if raw is None:
|
||||
return 0
|
||||
try:
|
||||
return int(json.loads(raw.get("value", "0")))
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
class TestFirstRunSeed:
|
||||
def test_first_run_seeds_ring(self, storage):
|
||||
"""Empty assignment table + 2 nodes -> seed all 65536 rows."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
result = rb.rebalance_once()
|
||||
|
||||
assert result.seeded is True
|
||||
assert result.noop is False
|
||||
assert result.nodes == 2
|
||||
|
||||
buckets = storage.list_ring_buckets()
|
||||
assert len(buckets) == RING_SIZE
|
||||
|
||||
# All buckets should be assigned to one of the two nodes
|
||||
node_ids = {b["node_id"] for b in buckets}
|
||||
assert node_ids == {"node-0", "node-1"}
|
||||
|
||||
|
||||
class TestIdempotent:
|
||||
def test_second_run_is_noop(self, storage):
|
||||
"""Running rebalance twice with same membership produces noop on second pass."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
|
||||
r1 = rb.rebalance_once()
|
||||
assert r1.seeded is True
|
||||
|
||||
r2 = rb.rebalance_once()
|
||||
assert r2.noop is True
|
||||
assert r2.moves == 0
|
||||
|
||||
|
||||
class TestNewNodeRebalances:
|
||||
def test_adding_node_moves_buckets(self, storage):
|
||||
"""Seed with 2 nodes, add 3rd -> some buckets move to the new node."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Verify only 2 nodes initially
|
||||
buckets_before = storage.list_ring_buckets()
|
||||
nodes_before = {b["node_id"] for b in buckets_before}
|
||||
assert nodes_before == {"node-0", "node-1"}
|
||||
|
||||
# Add a third node
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is False
|
||||
assert result.moves > 0
|
||||
assert result.nodes == 3
|
||||
|
||||
# Verify all three nodes have buckets
|
||||
buckets_after = storage.list_ring_buckets()
|
||||
nodes_after = {b["node_id"] for b in buckets_after}
|
||||
assert "node-2" in nodes_after
|
||||
|
||||
|
||||
class TestDeadNodeReassigned:
|
||||
def test_dead_node_buckets_move_to_survivors(self, storage):
|
||||
"""Seed with 3 nodes, deregister one -> its buckets move to survivors."""
|
||||
_register_nodes(storage, 3)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Verify node-2 has some buckets
|
||||
buckets = storage.list_ring_buckets()
|
||||
node2_count = sum(1 for b in buckets if b["node_id"] == "node-2")
|
||||
assert node2_count > 0
|
||||
|
||||
# Deregister node-2
|
||||
storage.deregister_service("server", "node-2")
|
||||
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is False
|
||||
assert result.moves > 0
|
||||
|
||||
# Verify no buckets assigned to dead node
|
||||
buckets_after = storage.list_ring_buckets()
|
||||
nodes_after = {b["node_id"] for b in buckets_after}
|
||||
assert "node-2" not in nodes_after
|
||||
|
||||
|
||||
class TestSingleNodeNoop:
|
||||
def test_single_node_already_assigned_is_noop(self, storage):
|
||||
"""1 node with all buckets assigned -> noop."""
|
||||
_register_nodes(storage, 1)
|
||||
rb = Rebalancer(storage=storage)
|
||||
|
||||
# Seed with single node
|
||||
rb.rebalance_once()
|
||||
|
||||
# Second run should be noop
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is True
|
||||
|
||||
|
||||
class TestWeightedDistribution:
|
||||
def test_weight_2_gets_more_buckets(self, storage):
|
||||
"""Node with weight=2 gets roughly 2x the buckets of weight=1."""
|
||||
_register_weighted_nodes(storage, {"heavy": 2, "light": 1})
|
||||
rb = Rebalancer(storage=storage, vnodes_per_unit=150)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
buckets = storage.list_ring_buckets()
|
||||
heavy_count = sum(1 for b in buckets if b["node_id"] == "heavy")
|
||||
light_count = sum(1 for b in buckets if b["node_id"] == "light")
|
||||
|
||||
# heavy should have roughly 2/3 of total, light roughly 1/3
|
||||
# Allow 10% tolerance
|
||||
expected_heavy = RING_SIZE * 2 // 3
|
||||
assert abs(heavy_count - expected_heavy) < RING_SIZE * 0.10
|
||||
assert heavy_count > light_count
|
||||
|
||||
|
||||
class TestVersionIncremented:
|
||||
def test_version_bumps_on_seed(self, storage):
|
||||
"""Verify rebalancer_version increments after seed."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
|
||||
v0 = _get_version(storage)
|
||||
assert v0 == 0
|
||||
|
||||
rb.rebalance_once()
|
||||
v1 = _get_version(storage)
|
||||
assert v1 == 1
|
||||
|
||||
def test_version_bumps_on_rebalance(self, storage):
|
||||
"""Version bumps on actual moves, not on noops."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed: version -> 1
|
||||
|
||||
# Noop: version stays at 1
|
||||
rb.rebalance_once()
|
||||
assert _get_version(storage) == 1
|
||||
|
||||
# Add node: version -> 2
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
rb.rebalance_once()
|
||||
assert _get_version(storage) == 2
|
||||
|
||||
|
||||
class TestReconcileStats:
|
||||
def test_bucket_stats_corrected(self, storage):
|
||||
"""Create workstreams in DB, verify bucket_stats are reconciled."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Create some workstreams — ws_id starts with hex bucket
|
||||
# Bucket 0x0000 = 0, bucket 0x0001 = 1
|
||||
storage.register_workstream("0000" + "a" * 28, state="idle")
|
||||
storage.register_workstream("0000" + "b" * 28, state="running")
|
||||
storage.register_workstream("0001" + "c" * 28, state="idle")
|
||||
|
||||
# Set bogus stats that will be corrected
|
||||
storage.increment_bucket_count(0) # says 1, should be 2
|
||||
storage.increment_bucket_count(5) # says 1, should be 0
|
||||
|
||||
rb._reconcile_bucket_stats()
|
||||
|
||||
stats = storage.list_bucket_stats()
|
||||
stats_map = {s["bucket"]: s for s in stats}
|
||||
|
||||
# Bucket 0 should have 2 ws, 1 active (running)
|
||||
assert stats_map[0]["ws_count"] == 2
|
||||
assert stats_map[0]["active_count"] == 1
|
||||
|
||||
# Bucket 1 should have 1 ws, 0 active
|
||||
assert stats_map[1]["ws_count"] == 1
|
||||
assert stats_map[1]["active_count"] == 0
|
||||
|
||||
# Bucket 5 should have been removed (ws_count=0)
|
||||
assert 5 not in stats_map
|
||||
|
||||
|
||||
class TestTransferPriorityEmptyFirst:
|
||||
def test_empty_buckets_moved_before_occupied(self, storage):
|
||||
"""Verify the sort key puts empty buckets before occupied ones.
|
||||
|
||||
Rather than asserting specific bucket assignments (which depend on
|
||||
hash ring placement), we verify the sorting invariant directly by
|
||||
checking that moves with zero occupancy come before occupied ones
|
||||
in the internal ordering.
|
||||
"""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Create workstreams in a few buckets owned by node-0
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_buckets = [b["bucket"] for b in buckets if b["node_id"] == "node-0"]
|
||||
|
||||
occupied = set()
|
||||
for b in node0_buckets[:3]:
|
||||
ws_id = f"{b:04x}" + "d" * 28
|
||||
storage.register_workstream(ws_id, state="running")
|
||||
storage.increment_bucket_count(b, active=True)
|
||||
occupied.add(b)
|
||||
|
||||
# Reconcile stats so the rebalancer sees them
|
||||
rb._reconcile_bucket_stats()
|
||||
|
||||
# Read stats to verify ordering assumptions
|
||||
stats = storage.list_bucket_stats()
|
||||
stats_map = {s["bucket"]: (s["ws_count"], s["active_count"]) for s in stats}
|
||||
|
||||
# The sort key is (active_count, ws_count) — occupied buckets
|
||||
# must sort AFTER empty buckets
|
||||
for b in occupied:
|
||||
assert stats_map[b][0] > 0 # ws_count > 0
|
||||
assert stats_map[b][1] > 0 # active_count > 0
|
||||
|
||||
# Empty buckets have (0, 0) which sorts before (1, 1)
|
||||
assert (0, 0) < (1, 1)
|
||||
|
||||
|
||||
class TestLeaderElection:
|
||||
def test_two_rebalancers_one_runs(self, storage):
|
||||
"""Two rebalancers compete — only one acquires the lock."""
|
||||
_register_nodes(storage, 2)
|
||||
rb1 = Rebalancer(storage=storage)
|
||||
rb2 = Rebalancer(storage=storage)
|
||||
|
||||
# rb1 acquires the lock
|
||||
assert rb1._try_acquire_lock() is True
|
||||
|
||||
# rb2 cannot acquire (lock is fresh)
|
||||
assert rb2._try_acquire_lock() is False
|
||||
|
||||
# rb1 releases
|
||||
rb1._release_lock()
|
||||
|
||||
# Now rb2 can acquire
|
||||
assert rb2._try_acquire_lock() is True
|
||||
rb2._release_lock()
|
||||
|
||||
|
||||
class TestZeroNodes:
|
||||
def test_no_nodes_returns_noop(self, storage):
|
||||
"""Zero live nodes -> noop result."""
|
||||
rb = Rebalancer(storage=storage)
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is True
|
||||
assert result.nodes == 0
|
||||
|
||||
|
||||
class TestStartStop:
|
||||
def test_start_stop_lifecycle(self, storage):
|
||||
"""Verify start/stop lifecycle doesn't hang or crash."""
|
||||
_register_nodes(storage, 1)
|
||||
rb = Rebalancer(storage=storage, interval=1)
|
||||
rb.start()
|
||||
assert rb._thread is not None
|
||||
assert rb._thread.is_alive()
|
||||
rb.stop()
|
||||
assert not rb._thread.is_alive()
|
||||
|
||||
def test_trigger_wakes_thread(self, storage):
|
||||
"""Verify trigger() causes an immediate pass."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, interval=3600) # long interval
|
||||
rb.start()
|
||||
try:
|
||||
rb.trigger()
|
||||
# Give it a moment to process
|
||||
rb._stop_event.wait(timeout=2)
|
||||
finally:
|
||||
rb.stop()
|
||||
# After trigger, the ring should be seeded
|
||||
assert len(storage.list_ring_buckets()) == RING_SIZE
|
||||
|
||||
|
||||
class TestGetStatus:
|
||||
def test_status_before_any_run(self, storage):
|
||||
"""Status returns version=0 and no last_result before any run."""
|
||||
rb = Rebalancer(storage=storage)
|
||||
status = rb.get_status()
|
||||
assert status["version"] == 0
|
||||
assert status["last_result"] is None
|
||||
|
||||
def test_status_after_seed(self, storage):
|
||||
"""Status reflects the seed run when result is stored."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
result = rb.rebalance_once()
|
||||
# The loop normally sets _last_result; simulate that here
|
||||
rb._last_result = result
|
||||
status = rb.get_status()
|
||||
assert status["version"] == 1
|
||||
assert status["last_result"] is not None
|
||||
assert status["last_result"]["seeded"] is True
|
||||
|
||||
|
||||
class TestEagerMigration:
|
||||
def test_eager_migrate_posts_to_source_nodes(self, storage):
|
||||
"""When eager_migrate=True, rebalancer POSTs /_internal/migrate for idle workstreams."""
|
||||
import httpx
|
||||
|
||||
# Seed ring with 2 nodes
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, eager_migrate=True)
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
# Create a workstream on node-0's bucket range
|
||||
# Find a bucket assigned to node-0
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_bucket = None
|
||||
for b in buckets:
|
||||
if b["node_id"] == "node-0":
|
||||
node0_bucket = b["bucket"]
|
||||
break
|
||||
assert node0_bucket is not None
|
||||
|
||||
ws_id = f"{node0_bucket:04x}" + "a" * 28
|
||||
storage.register_workstream(ws_id, node_id="node-0", name="test")
|
||||
storage.increment_bucket_count(node0_bucket)
|
||||
|
||||
# Add a 3rd node — this will trigger rebalance
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
# Track migrate calls
|
||||
migrate_calls: list[tuple[str, str]] = [] # (url, ws_id)
|
||||
|
||||
class FakeTransport(httpx.BaseTransport):
|
||||
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
migrate_calls.append((str(request.url), body.get("ws_id", "")))
|
||||
return httpx.Response(200, json={"status": "ok", "ws_id": body["ws_id"]})
|
||||
|
||||
# Monkey-patch httpx.Client to use our fake transport
|
||||
original_init = httpx.Client.__init__
|
||||
|
||||
def patched_init(self_client, **kwargs):
|
||||
kwargs["transport"] = FakeTransport()
|
||||
original_init(self_client, **kwargs)
|
||||
|
||||
import unittest.mock
|
||||
|
||||
with unittest.mock.patch.object(httpx.Client, "__init__", patched_init):
|
||||
result = rb.rebalance_once(trigger="test")
|
||||
|
||||
# If the bucket moved to a different node, the workstream should be migrated
|
||||
new_buckets = storage.list_ring_buckets()
|
||||
new_owner = None
|
||||
for b in new_buckets:
|
||||
if b["bucket"] == node0_bucket:
|
||||
new_owner = b["node_id"]
|
||||
break
|
||||
|
||||
if new_owner != "node-0":
|
||||
# Bucket moved — migration should have happened
|
||||
assert result.migrations > 0
|
||||
assert any(ws_id in call[1] for call in migrate_calls)
|
||||
else:
|
||||
# Bucket stayed — no migration needed for this ws
|
||||
assert result.migrations >= 0 # other workstreams might have been migrated
|
||||
|
||||
def test_eager_migrate_skips_active_workstreams(self, storage):
|
||||
"""Active workstreams are not eagerly migrated (would disrupt in-flight work)."""
|
||||
import httpx
|
||||
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, eager_migrate=True)
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
# Find a bucket on node-0
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_bucket = None
|
||||
for b in buckets:
|
||||
if b["node_id"] == "node-0":
|
||||
node0_bucket = b["bucket"]
|
||||
break
|
||||
assert node0_bucket is not None
|
||||
|
||||
# Create an ACTIVE workstream (state="running")
|
||||
ws_id = f"{node0_bucket:04x}" + "b" * 28
|
||||
storage.register_workstream(ws_id, node_id="node-0", name="active-ws")
|
||||
storage.update_workstream_state(ws_id, "running")
|
||||
storage.increment_bucket_count(node0_bucket, active=True)
|
||||
|
||||
# Add 3rd node to trigger rebalance
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
migrate_calls: list[str] = []
|
||||
|
||||
class FakeTransport(httpx.BaseTransport):
|
||||
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
migrate_calls.append(body.get("ws_id", ""))
|
||||
return httpx.Response(200, json={"status": "ok"})
|
||||
|
||||
original_init = httpx.Client.__init__
|
||||
|
||||
def patched_init(self_client, **kwargs):
|
||||
kwargs["transport"] = FakeTransport()
|
||||
original_init(self_client, **kwargs)
|
||||
|
||||
import unittest.mock
|
||||
|
||||
with unittest.mock.patch.object(httpx.Client, "__init__", patched_init):
|
||||
rb.rebalance_once(trigger="test")
|
||||
|
||||
# The active workstream should NOT have been migrated
|
||||
assert ws_id not in migrate_calls
|
||||
|
||||
def test_eager_migrate_disabled_by_default(self, storage):
|
||||
"""When eager_migrate=False (default), no migrate calls happen."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage) # eager_migrate defaults to False
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
# Create workstream and trigger rebalance
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_bucket = next(b["bucket"] for b in buckets if b["node_id"] == "node-0")
|
||||
ws_id = f"{node0_bucket:04x}" + "c" * 28
|
||||
storage.register_workstream(ws_id, node_id="node-0", name="test")
|
||||
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
result = rb.rebalance_once(trigger="test")
|
||||
assert result.migrations == 0 # no eager migration when disabled
|
||||
|
||||
|
||||
class TestMinimalTransfer:
|
||||
def test_new_node_only_receives_never_shuffles(self, storage):
|
||||
"""Adding a 3rd node moves buckets TO it, never between existing nodes.
|
||||
|
||||
This is the key property of the minimal-transfer algorithm: nodes A
|
||||
and B should not exchange buckets with each other — only donate to C.
|
||||
"""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.05)
|
||||
rb.rebalance_once() # seeds: node-0 gets 32768, node-1 gets 32768
|
||||
|
||||
# Record which node owns each bucket before adding node-2
|
||||
before = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Add a third node
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
result = rb.rebalance_once()
|
||||
|
||||
after = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Verify: every bucket that moved went TO node-2
|
||||
for bucket in range(RING_SIZE):
|
||||
old = before[bucket]
|
||||
new = after[bucket]
|
||||
if old != new:
|
||||
assert new == "node-2", (
|
||||
f"bucket {bucket} moved {old} -> {new}, expected all moves to target node-2"
|
||||
)
|
||||
|
||||
# Verify: node-2 got roughly 1/3 of all buckets
|
||||
node2_count = sum(1 for nid in after.values() if nid == "node-2")
|
||||
assert 19000 < node2_count < 24000, f"node-2 got {node2_count} buckets"
|
||||
assert result.moves > 0
|
||||
|
||||
def test_remove_node_distributes_proportionally(self, storage):
|
||||
"""Removing a node distributes its buckets to remaining nodes
|
||||
proportionally — doesn't shuffle between survivors."""
|
||||
_register_nodes(storage, 3)
|
||||
rb = Rebalancer(storage=storage, threshold=0.05)
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
before = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Remove node-2
|
||||
storage.deregister_service("server", "node-2")
|
||||
result = rb.rebalance_once()
|
||||
|
||||
after = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Every moved bucket should have been owned by node-2 (the dead node)
|
||||
for bucket in range(RING_SIZE):
|
||||
old = before[bucket]
|
||||
new = after[bucket]
|
||||
if old != new:
|
||||
assert old == "node-2", (
|
||||
f"bucket {bucket} moved {old} -> {new}, but only node-2's buckets should move"
|
||||
)
|
||||
|
||||
# node-2 should have zero buckets now
|
||||
node2_count = sum(1 for nid in after.values() if nid == "node-2")
|
||||
assert node2_count == 0
|
||||
assert result.moves > 0
|
||||
+22
-80
@@ -1,97 +1,39 @@
|
||||
"""Tests for the atomic workstream resumption flow.
|
||||
"""Tests for the workstream resume request schema.
|
||||
|
||||
Covers CreateWorkstreamMessage resume_ws field, WorkstreamResumedEvent,
|
||||
WorkstreamCreatedEvent resumed fields, and server endpoint handling.
|
||||
Verifies that the create-workstream JSON payload carries the resume_ws field
|
||||
correctly, matching the server's ``CreateWorkstreamRequest`` schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from turnstone.mq.protocol import (
|
||||
CreateWorkstreamMessage,
|
||||
WorkstreamCreatedEvent,
|
||||
WorkstreamResumedEvent,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol tests
|
||||
# CreateWorkstreamRequest resume_ws field
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCreateWorkstreamMessageResumeField:
|
||||
class TestCreateWorkstreamResumeField:
|
||||
def test_resume_ws_defaults_empty(self) -> None:
|
||||
msg = CreateWorkstreamMessage(name="test")
|
||||
assert msg.resume_ws == ""
|
||||
body: dict[str, str] = {"name": "test"}
|
||||
assert body.get("resume_ws", "") == ""
|
||||
|
||||
def test_resume_ws_set(self) -> None:
|
||||
msg = CreateWorkstreamMessage(name="test", resume_ws="ws-abc")
|
||||
assert msg.resume_ws == "ws-abc"
|
||||
body = {"name": "test", "resume_ws": "ws-abc"}
|
||||
assert body["resume_ws"] == "ws-abc"
|
||||
|
||||
def test_resume_ws_serializes(self) -> None:
|
||||
msg = CreateWorkstreamMessage(resume_ws="ws-xyz")
|
||||
data = json.loads(msg.to_json())
|
||||
assert data["resume_ws"] == "ws-xyz"
|
||||
def test_resume_ws_present_in_payload(self) -> None:
|
||||
body = {"name": "test", "resume_ws": "ws-xyz"}
|
||||
assert "resume_ws" in body
|
||||
assert body["resume_ws"] == "ws-xyz"
|
||||
|
||||
def test_resume_ws_deserializes(self) -> None:
|
||||
msg = CreateWorkstreamMessage(resume_ws="ws-123")
|
||||
raw = msg.to_json()
|
||||
from turnstone.mq.protocol import InboundMessage
|
||||
def test_pydantic_schema_has_resume_ws(self) -> None:
|
||||
"""CreateWorkstreamRequest schema includes resume_ws."""
|
||||
from turnstone.api.server_schemas import CreateWorkstreamRequest
|
||||
|
||||
restored = InboundMessage.from_json(raw)
|
||||
assert getattr(restored, "resume_ws", "") == "ws-123"
|
||||
req = CreateWorkstreamRequest(name="test", resume_ws="ws-123")
|
||||
assert req.resume_ws == "ws-123"
|
||||
|
||||
def test_pydantic_schema_default_empty(self) -> None:
|
||||
from turnstone.api.server_schemas import CreateWorkstreamRequest
|
||||
|
||||
class TestWorkstreamCreatedEventResumeFields:
|
||||
def test_default_not_resumed(self) -> None:
|
||||
event = WorkstreamCreatedEvent(ws_id="ws-1", name="test")
|
||||
assert event.resumed is False
|
||||
assert event.message_count == 0
|
||||
|
||||
def test_resumed_fields(self) -> None:
|
||||
event = WorkstreamCreatedEvent(ws_id="ws-1", name="test", resumed=True, message_count=42)
|
||||
assert event.resumed is True
|
||||
assert event.message_count == 42
|
||||
|
||||
def test_serializes_resumed_fields(self) -> None:
|
||||
event = WorkstreamCreatedEvent(ws_id="ws-1", resumed=True, message_count=10)
|
||||
data = json.loads(event.to_json())
|
||||
assert data["resumed"] is True
|
||||
assert data["message_count"] == 10
|
||||
|
||||
def test_deserializes_resumed_fields(self) -> None:
|
||||
event = WorkstreamCreatedEvent(ws_id="ws-1", resumed=True, message_count=5)
|
||||
from turnstone.mq.protocol import OutboundEvent
|
||||
|
||||
restored = OutboundEvent.from_json(event.to_json())
|
||||
assert isinstance(restored, WorkstreamCreatedEvent)
|
||||
assert restored.resumed is True
|
||||
assert restored.message_count == 5
|
||||
|
||||
|
||||
class TestWorkstreamResumedEvent:
|
||||
def test_defaults(self) -> None:
|
||||
event = WorkstreamResumedEvent(ws_id="ws-1")
|
||||
assert event.type == "ws_resumed"
|
||||
assert event.message_count == 0
|
||||
assert event.name == ""
|
||||
|
||||
def test_with_values(self) -> None:
|
||||
event = WorkstreamResumedEvent(ws_id="ws-1", message_count=25, name="My Chat")
|
||||
assert event.message_count == 25
|
||||
assert event.name == "My Chat"
|
||||
|
||||
def test_round_trip(self) -> None:
|
||||
event = WorkstreamResumedEvent(ws_id="ws-1", message_count=10, name="Chat")
|
||||
from turnstone.mq.protocol import OutboundEvent
|
||||
|
||||
restored = OutboundEvent.from_json(event.to_json())
|
||||
assert isinstance(restored, WorkstreamResumedEvent)
|
||||
assert restored.message_count == 10
|
||||
assert restored.name == "Chat"
|
||||
|
||||
def test_registered_in_outbound_registry(self) -> None:
|
||||
from turnstone.mq.protocol import _OUTBOUND_REGISTRY
|
||||
|
||||
assert "ws_resumed" in _OUTBOUND_REGISTRY
|
||||
assert _OUTBOUND_REGISTRY["ws_resumed"] is WorkstreamResumedEvent
|
||||
req = CreateWorkstreamRequest(name="test")
|
||||
assert req.resume_ws == ""
|
||||
|
||||
+201
-85
@@ -2,21 +2,46 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.console.scheduler import TaskScheduler
|
||||
from turnstone.sdk._types import TurnstoneAPIError
|
||||
|
||||
|
||||
def _wire_lock_storage(storage: MagicMock, initial: dict[str, str] | None = None) -> None:
|
||||
"""Configure *storage* mock so upsert/get track scheduler_lock state.
|
||||
|
||||
The scheduler's ``_try_acquire_lock`` now writes then reads back to
|
||||
verify ownership. The mock must reflect what was most recently
|
||||
upserted so the read-back succeeds.
|
||||
"""
|
||||
state: dict[str, dict[str, str] | None] = {"scheduler_lock": initial}
|
||||
|
||||
def _get(key: str, **_kw: object) -> dict[str, str] | None:
|
||||
return state.get(key)
|
||||
|
||||
def _upsert(key: str, value: str, **_kw: object) -> None:
|
||||
state[key] = {"value": value}
|
||||
|
||||
def _delete(key: str, **_kw: object) -> None:
|
||||
state.pop(key, None)
|
||||
|
||||
storage.get_system_setting.side_effect = _get
|
||||
storage.upsert_system_setting.side_effect = _upsert
|
||||
storage.delete_system_setting.side_effect = _delete
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocks():
|
||||
"""Broker, collector, and storage mocks for scheduler tests."""
|
||||
broker = MagicMock()
|
||||
broker._redis = MagicMock()
|
||||
"""Collector and storage mocks for scheduler tests."""
|
||||
collector = MagicMock()
|
||||
storage = MagicMock()
|
||||
return broker, collector, storage
|
||||
# Default: no existing lock
|
||||
_wire_lock_storage(storage, initial=None)
|
||||
return collector, storage
|
||||
|
||||
|
||||
def _make_task(**overrides):
|
||||
@@ -54,74 +79,105 @@ def _make_node(node_id="node-001", reachable=True, ws_total=2, max_ws=10):
|
||||
}
|
||||
|
||||
|
||||
def _mock_create_response(ws_id: str = "ws_abc123") -> MagicMock:
|
||||
"""Build a mock CreateWorkstreamResponse with the given ws_id."""
|
||||
resp = MagicMock()
|
||||
resp.ws_id = ws_id
|
||||
return resp
|
||||
|
||||
|
||||
class TestSchedulerTick:
|
||||
"""Tests for _tick() lock acquisition and dispatch logic."""
|
||||
|
||||
def test_tick_acquires_lock(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
storage.list_due_tasks.return_value = []
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
scheduler._tick()
|
||||
|
||||
broker._redis.set.assert_called_once()
|
||||
storage.get_system_setting.assert_called()
|
||||
storage.upsert_system_setting.assert_called()
|
||||
storage.list_due_tasks.assert_called_once()
|
||||
# Lock released via Lua eval (conditional delete)
|
||||
broker._redis.eval.assert_called_once()
|
||||
|
||||
def test_tick_skips_when_locked(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = None # lock held by another console
|
||||
collector, storage = mocks
|
||||
# Another instance holds the lock (recent timestamp)
|
||||
from datetime import UTC, datetime
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
now_str = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
_wire_lock_storage(
|
||||
storage,
|
||||
initial={"value": json.dumps({"owner": "other-instance", "acquired": now_str})},
|
||||
)
|
||||
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
scheduler._tick()
|
||||
|
||||
storage.list_due_tasks.assert_not_called()
|
||||
|
||||
def test_tick_takes_expired_lock(self, mocks):
|
||||
"""An expired lock from another instance should be taken over."""
|
||||
collector, storage = mocks
|
||||
_wire_lock_storage(
|
||||
storage,
|
||||
initial={
|
||||
"value": json.dumps({"owner": "other-instance", "acquired": "2020-01-01T00:00:00"})
|
||||
},
|
||||
)
|
||||
storage.list_due_tasks.return_value = []
|
||||
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
scheduler._tick()
|
||||
|
||||
storage.list_due_tasks.assert_called_once()
|
||||
|
||||
def test_dispatch_auto_mode(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="auto")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([_make_node()], 1)
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
broker.push_inbound.assert_called_once()
|
||||
_, kwargs = broker.push_inbound.call_args
|
||||
assert (
|
||||
kwargs.get("node_id") == "node-001"
|
||||
or broker.push_inbound.call_args[1].get("node_id") == "node-001"
|
||||
)
|
||||
mock_create.assert_called_once()
|
||||
storage.record_task_run.assert_called_once()
|
||||
run_kwargs = storage.record_task_run.call_args[1]
|
||||
assert run_kwargs["node_id"] == "node-001"
|
||||
assert run_kwargs["status"] == "dispatched"
|
||||
assert run_kwargs["ws_id"] == "ws_abc123"
|
||||
|
||||
def test_dispatch_pool_mode(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="pool")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([_make_node("node-001")], 1)
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
broker.push_inbound.assert_called_once()
|
||||
# Pool dispatch calls push_inbound without node_id kwarg
|
||||
args, kwargs = broker.push_inbound.call_args
|
||||
assert kwargs.get("node_id") is None or "node_id" not in kwargs
|
||||
mock_create.assert_called_once()
|
||||
storage.record_task_run.assert_called_once()
|
||||
run_kwargs = storage.record_task_run.call_args[1]
|
||||
assert run_kwargs["node_id"] == "pool"
|
||||
|
||||
def test_dispatch_all_mode(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="all")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
@@ -129,37 +185,56 @@ class TestSchedulerTick:
|
||||
[_make_node("node-001"), _make_node("node-002")],
|
||||
2,
|
||||
)
|
||||
collector.get_node_detail.side_effect = lambda nid: {
|
||||
"server_url": f"http://{nid}:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
assert broker.push_inbound.call_count == 2
|
||||
assert mock_create.call_count == 2
|
||||
assert storage.record_task_run.call_count == 2
|
||||
|
||||
def test_dispatch_specific_node(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="node-001")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
broker.push_inbound.assert_called_once()
|
||||
_, kwargs = broker.push_inbound.call_args
|
||||
assert kwargs["node_id"] == "node-001"
|
||||
mock_create.assert_called_once()
|
||||
run_kwargs = storage.record_task_run.call_args[1]
|
||||
assert run_kwargs["node_id"] == "node-001"
|
||||
|
||||
def test_at_task_disables_after_dispatch(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(schedule_type="at", cron_expr="", at_time="2099-01-01T00:00:00")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([_make_node()], 1)
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
):
|
||||
scheduler._tick()
|
||||
|
||||
# At-task should be disabled after dispatch
|
||||
update_calls = storage.update_scheduled_task.call_args_list
|
||||
@@ -170,15 +245,21 @@ class TestSchedulerTick:
|
||||
assert kwargs["next_run"] == ""
|
||||
|
||||
def test_cron_task_updates_next_run(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(schedule_type="cron", cron_expr="0 9 * * *")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([_make_node()], 1)
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
):
|
||||
scheduler._tick()
|
||||
|
||||
update_calls = storage.update_scheduled_task.call_args_list
|
||||
assert len(update_calls) == 1
|
||||
@@ -187,8 +268,7 @@ class TestSchedulerTick:
|
||||
assert "enabled" not in kwargs # cron tasks stay enabled
|
||||
|
||||
def test_no_reachable_nodes_records_failure(self, mocks):
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="auto")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
@@ -198,10 +278,9 @@ class TestSchedulerTick:
|
||||
1,
|
||||
)
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
scheduler._tick()
|
||||
|
||||
broker.push_inbound.assert_not_called()
|
||||
storage.record_task_run.assert_called_once()
|
||||
run_kwargs = storage.record_task_run.call_args[1]
|
||||
assert run_kwargs["status"] == "failed"
|
||||
@@ -209,14 +288,13 @@ class TestSchedulerTick:
|
||||
|
||||
def test_failure_does_not_advance_schedule(self, mocks):
|
||||
"""When dispatch fails, last_run/next_run should not be updated."""
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="auto")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([], 0) # no nodes at all
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
scheduler._tick()
|
||||
|
||||
# update_scheduled_task should NOT be called (no last_run/next_run advance)
|
||||
@@ -224,49 +302,87 @@ class TestSchedulerTick:
|
||||
|
||||
def test_fan_out_capped(self, mocks):
|
||||
"""Fan-out 'all' mode should respect max_fan_out limit."""
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="all")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
# 10 reachable nodes but max_fan_out=3
|
||||
nodes = [_make_node(f"node-{i:03d}") for i in range(10)]
|
||||
collector.get_nodes.return_value = (nodes, 10)
|
||||
collector.get_node_detail.side_effect = lambda nid: {
|
||||
"server_url": f"http://{nid}:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage, max_fan_out=3)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage, max_fan_out=3)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
assert broker.push_inbound.call_count == 3
|
||||
assert mock_create.call_count == 3
|
||||
assert storage.record_task_run.call_count == 3
|
||||
|
||||
def test_specific_node_target(self, mocks):
|
||||
"""Non-enum target_mode is treated as a specific node_id."""
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="node-custom-123")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-custom-123:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
broker.push_inbound.assert_called_once()
|
||||
call_kwargs = broker.push_inbound.call_args
|
||||
assert call_kwargs[1]["node_id"] == "node-custom-123"
|
||||
mock_create.assert_called_once()
|
||||
run_kwargs = storage.record_task_run.call_args[1]
|
||||
assert run_kwargs["node_id"] == "node-custom-123"
|
||||
|
||||
def test_user_id_in_dispatched_message(self, mocks):
|
||||
"""Dispatched message should include created_by as user_id."""
|
||||
import json
|
||||
def test_user_id_in_dispatched_call(self, mocks):
|
||||
"""Dispatched SDK call should include created_by as user_id."""
|
||||
collector, storage = mocks
|
||||
|
||||
broker, collector, storage = mocks
|
||||
broker._redis.set.return_value = True
|
||||
|
||||
task = _make_task(target_mode="pool", created_by="u_scheduler_admin")
|
||||
task = _make_task(target_mode="auto", created_by="u_scheduler_admin")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([_make_node()], 1)
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(broker, collector, storage)
|
||||
scheduler._tick()
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
return_value=_mock_create_response(),
|
||||
) as mock_create:
|
||||
scheduler._tick()
|
||||
|
||||
msg_json = broker.push_inbound.call_args[0][0]
|
||||
msg_data = json.loads(msg_json)
|
||||
assert msg_data["user_id"] == "u_scheduler_admin"
|
||||
_, kwargs = mock_create.call_args
|
||||
assert kwargs["user_id"] == "u_scheduler_admin"
|
||||
|
||||
def test_sdk_failure_records_failure(self, mocks):
|
||||
"""SDK errors during dispatch should record a failure."""
|
||||
collector, storage = mocks
|
||||
|
||||
task = _make_task(target_mode="auto")
|
||||
storage.list_due_tasks.return_value = [task]
|
||||
collector.get_nodes.return_value = ([_make_node()], 1)
|
||||
collector.get_node_detail.return_value = {
|
||||
"server_url": "http://node-001:8080",
|
||||
}
|
||||
|
||||
scheduler = TaskScheduler(collector, storage)
|
||||
with patch(
|
||||
"turnstone.console.scheduler.TurnstoneServer.create_workstream",
|
||||
side_effect=TurnstoneAPIError(502, "Bad Gateway"),
|
||||
):
|
||||
scheduler._tick()
|
||||
|
||||
storage.record_task_run.assert_called_once()
|
||||
run_kwargs = storage.record_task_run.call_args[1]
|
||||
assert run_kwargs["status"] == "failed"
|
||||
|
||||
@@ -379,3 +379,259 @@ async def test_list_schedule_runs():
|
||||
assert len(resp.runs) == 1
|
||||
assert resp.runs[0].run_id == "r1"
|
||||
assert resp.runs[0].status == "dispatched"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# create_workstream extended params
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_workstream_extended_params():
|
||||
"""New optional params appear in JSON body only when non-empty."""
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response(
|
||||
{"status": "dispatched", "correlation_id": "abc", "target_node": "n1"}
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.create_workstream(
|
||||
node_id="n1",
|
||||
name="ext",
|
||||
auto_approve=True,
|
||||
auto_approve_tools="read_file",
|
||||
user_id="u42",
|
||||
)
|
||||
assert captured_body["node_id"] == "n1"
|
||||
assert captured_body["name"] == "ext"
|
||||
assert captured_body["auto_approve"] is True
|
||||
assert captured_body["auto_approve_tools"] == "read_file"
|
||||
assert captured_body["user_id"] == "u42"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_workstream_omits_empty_new_params():
|
||||
"""Default-valued new params should not appear in JSON body."""
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response(
|
||||
{"status": "dispatched", "correlation_id": "abc", "target_node": "n1"}
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.create_workstream(name="min")
|
||||
assert captured_body == {"name": "min"}
|
||||
assert "auto_approve" not in captured_body
|
||||
assert "auto_approve_tools" not in captured_body
|
||||
assert "user_id" not in captured_body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Route methods
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_create_workstream():
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response({"ws_id": "ws1", "node_url": "http://n1:8080", "node_id": "n1"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
resp = await client.route_create_workstream(
|
||||
name="routed",
|
||||
model="gpt-5",
|
||||
auto_approve=True,
|
||||
target_node="n1",
|
||||
user_id="u1",
|
||||
)
|
||||
assert resp["ws_id"] == "ws1"
|
||||
assert resp["node_url"] == "http://n1:8080"
|
||||
assert captured_body["name"] == "routed"
|
||||
assert captured_body["model"] == "gpt-5"
|
||||
assert captured_body["auto_approve"] is True
|
||||
assert captured_body["target_node"] == "n1"
|
||||
assert captured_body["user_id"] == "u1"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_create_workstream_omits_defaults():
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response({"ws_id": "ws1", "node_url": "http://n1:8080"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_create_workstream(name="bare")
|
||||
assert captured_body == {"name": "bare"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_send():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["path"] = request.url.path
|
||||
captured["body"] = json.loads(request.content)
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
resp = await client.route_send("Hello", "ws1")
|
||||
assert resp["status"] == "ok"
|
||||
assert captured["path"] == "/v1/api/route/send"
|
||||
assert captured["body"] == {"message": "Hello", "ws_id": "ws1"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_approve():
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_approve(ws_id="ws1", approved=False, feedback="no", always=True)
|
||||
assert captured_body["ws_id"] == "ws1"
|
||||
assert captured_body["approved"] is False
|
||||
assert captured_body["feedback"] == "no"
|
||||
assert captured_body["always"] is True
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_approve_omits_defaults():
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_approve(ws_id="ws1", approved=True)
|
||||
assert captured_body == {"ws_id": "ws1", "approved": True}
|
||||
assert "feedback" not in captured_body
|
||||
assert "always" not in captured_body
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_plan_feedback():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["path"] = request.url.path
|
||||
captured["body"] = json.loads(request.content)
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_plan_feedback(ws_id="ws1", feedback="approved")
|
||||
assert captured["path"] == "/v1/api/route/plan"
|
||||
assert captured["body"] == {"ws_id": "ws1", "feedback": "approved"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_close():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["path"] = request.url.path
|
||||
captured["body"] = json.loads(request.content)
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
resp = await client.route_close("ws1")
|
||||
assert resp["status"] == "ok"
|
||||
assert captured["path"] == "/v1/api/route/workstreams/close"
|
||||
assert captured["body"] == {"ws_id": "ws1"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_cancel():
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_cancel("ws1", force=True)
|
||||
assert captured_body == {"ws_id": "ws1", "force": True}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_cancel_omits_force_when_false():
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_cancel("ws1")
|
||||
assert captured_body == {"ws_id": "ws1"}
|
||||
assert "force" not in captured_body
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_command():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["path"] = request.url.path
|
||||
captured["body"] = json.loads(request.content)
|
||||
return _json_response({"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
await client.route_command(ws_id="ws1", command="/clear")
|
||||
assert captured["path"] == "/v1/api/route/command"
|
||||
assert captured["body"] == {"ws_id": "ws1", "command": "/clear"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_route_lookup():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["path"] = request.url.path
|
||||
captured["url"] = str(request.url)
|
||||
return _json_response({"node_url": "http://n1:8080", "node_id": "n1"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneConsole(httpx_client=hc)
|
||||
resp = await client.route_lookup("ws1")
|
||||
assert resp["node_url"] == "http://n1:8080"
|
||||
assert resp["node_id"] == "n1"
|
||||
assert captured["path"] == "/v1/api/route"
|
||||
assert "ws_id=ws1" in captured["url"]
|
||||
|
||||
@@ -279,3 +279,54 @@ async def test_request_body_correct():
|
||||
client = AsyncTurnstoneServer(httpx_client=hc)
|
||||
await client.send("Hello world", "ws_123")
|
||||
assert captured_body == {"message": "Hello world", "ws_id": "ws_123"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# create_workstream extended params
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_workstream_extended_params():
|
||||
"""New optional params appear in JSON body only when non-empty."""
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return httpx.Response(200, json={"ws_id": "ws_ext", "name": "ext"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneServer(httpx_client=hc)
|
||||
await client.create_workstream(
|
||||
name="ext",
|
||||
initial_message="hi",
|
||||
auto_approve_tools="read_file,write_file",
|
||||
user_id="u42",
|
||||
ws_id="ws_custom",
|
||||
)
|
||||
assert captured_body["name"] == "ext"
|
||||
assert captured_body["initial_message"] == "hi"
|
||||
assert captured_body["auto_approve_tools"] == "read_file,write_file"
|
||||
assert captured_body["user_id"] == "u42"
|
||||
assert captured_body["ws_id"] == "ws_custom"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_workstream_omits_empty_params():
|
||||
"""Empty-string params should NOT appear in the JSON body."""
|
||||
captured_body: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
return httpx.Response(200, json={"ws_id": "ws_min", "name": "min"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as hc:
|
||||
client = AsyncTurnstoneServer(httpx_client=hc)
|
||||
await client.create_workstream(name="min")
|
||||
assert captured_body == {"name": "min"}
|
||||
assert "initial_message" not in captured_body
|
||||
assert "auto_approve_tools" not in captured_body
|
||||
assert "user_id" not in captured_body
|
||||
assert "ws_id" not in captured_body
|
||||
|
||||
@@ -52,11 +52,11 @@ class TestServiceRegistry:
|
||||
|
||||
def test_list_filters_by_type(self, storage):
|
||||
storage.register_service("channel", "ch-1", "http://localhost:8091")
|
||||
storage.register_service("bridge", "br-1", "http://localhost:8080")
|
||||
storage.register_service("worker", "wk-1", "http://localhost:8080")
|
||||
channels = storage.list_services("channel", max_age_seconds=120)
|
||||
bridges = storage.list_services("bridge", max_age_seconds=120)
|
||||
workers = storage.list_services("worker", max_age_seconds=120)
|
||||
assert len(channels) == 1
|
||||
assert len(bridges) == 1
|
||||
assert len(workers) == 1
|
||||
|
||||
def test_deregister(self, storage):
|
||||
storage.register_service("channel", "ch-1", "http://localhost:8091")
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
"""Tests for the turnstone cluster simulator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.mq.protocol import (
|
||||
OutboundEvent,
|
||||
SendMessage,
|
||||
StateChangeEvent,
|
||||
)
|
||||
from turnstone.sim.config import SimConfig
|
||||
from turnstone.sim.engine import SimEngine, ToolSimulationError
|
||||
from turnstone.sim.metrics import MetricsCollector
|
||||
from turnstone.sim.node import SimNode, SimWorkstream
|
||||
|
||||
|
||||
def _run(coro):
|
||||
"""Run an async coroutine synchronously."""
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimConfig
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSimConfig:
|
||||
def test_defaults(self):
|
||||
cfg = SimConfig()
|
||||
assert cfg.num_nodes == 10
|
||||
assert cfg.scenario == "steady"
|
||||
assert cfg.llm_latency_mean == 2.0
|
||||
assert cfg.tool_failure_rate == 0.02
|
||||
|
||||
def test_frozen(self):
|
||||
cfg = SimConfig()
|
||||
with pytest.raises(AttributeError):
|
||||
cfg.num_nodes = 5 # type: ignore[misc]
|
||||
|
||||
def test_custom_values(self):
|
||||
cfg = SimConfig(num_nodes=100, scenario="burst", seed=42)
|
||||
assert cfg.num_nodes == 100
|
||||
assert cfg.scenario == "burst"
|
||||
assert cfg.seed == 42
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimEngine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSimEngine:
|
||||
@pytest.fixture
|
||||
def fast_config(self):
|
||||
return SimConfig(
|
||||
llm_latency_mean=0.01,
|
||||
llm_latency_stddev=0.001,
|
||||
llm_tokens_mean=20,
|
||||
llm_tokens_stddev=5,
|
||||
tool_latency_mean=0.01,
|
||||
tool_latency_stddev=0.001,
|
||||
tool_failure_rate=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def engine(self, fast_config):
|
||||
return SimEngine(fast_config)
|
||||
|
||||
def test_llm_response_returns_content(self, engine):
|
||||
async def _test():
|
||||
content, tool_calls = await engine.simulate_llm_response(True)
|
||||
assert isinstance(content, str)
|
||||
assert len(content) > 0
|
||||
assert isinstance(tool_calls, list)
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_llm_response_reproducible_with_seed(self, fast_config):
|
||||
async def _test():
|
||||
e1 = SimEngine(fast_config, rng=random.Random(123))
|
||||
e2 = SimEngine(fast_config, rng=random.Random(123))
|
||||
c1, t1 = await e1.simulate_llm_response(True)
|
||||
c2, t2 = await e2.simulate_llm_response(True)
|
||||
assert c1 == c2
|
||||
assert len(t1) == len(t2)
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_tool_execution_success(self, engine):
|
||||
async def _test():
|
||||
result = await engine.simulate_tool_execution("bash")
|
||||
assert "bash" in result
|
||||
assert "completed" in result
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_tool_execution_failure(self, fast_config):
|
||||
cfg = SimConfig(
|
||||
llm_latency_mean=0.01,
|
||||
tool_latency_mean=0.01,
|
||||
tool_latency_stddev=0.001,
|
||||
tool_failure_rate=1.0, # always fail
|
||||
seed=42,
|
||||
)
|
||||
engine = SimEngine(cfg)
|
||||
|
||||
async def _test():
|
||||
with pytest.raises(ToolSimulationError, match="Simulated bash failure"):
|
||||
await engine.simulate_tool_execution("bash")
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_generate_content(self, engine):
|
||||
content = engine._generate_content(10)
|
||||
words = content.split()
|
||||
assert len(words) == 10
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MetricsCollector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMetricsCollector:
|
||||
def test_record_and_summary(self):
|
||||
m = MetricsCollector()
|
||||
m.record_inject()
|
||||
m.record_turn("ws1", "node-0", 1.5)
|
||||
m.record_turn("ws2", "node-0", 2.5)
|
||||
m.record_turn("ws3", "node-1", 3.0)
|
||||
m.record_error("node-0", "test error")
|
||||
|
||||
report = m.summary()
|
||||
assert report["total_turns"] == 3
|
||||
assert report["total_errors"] == 1
|
||||
assert report["latency"]["p50"] == 2.5
|
||||
assert report["latency"]["max"] == 3.0
|
||||
assert report["turns_per_node"]["node-0"] == 2
|
||||
assert report["turns_per_node"]["node-1"] == 1
|
||||
|
||||
def test_empty_summary(self):
|
||||
m = MetricsCollector()
|
||||
report = m.summary()
|
||||
assert report["total_turns"] == 0
|
||||
assert report["latency"]["p50"] == 0
|
||||
|
||||
def test_node_kill_tracking(self):
|
||||
m = MetricsCollector()
|
||||
m.record_node_kill("node-0")
|
||||
m.record_node_kill("node-1")
|
||||
report = m.summary()
|
||||
assert report["node_kills"] == 2
|
||||
|
||||
def test_utilization_snapshot(self):
|
||||
m = MetricsCollector()
|
||||
m.snapshot_utilization({"node-0": 3, "node-1": 5, "node-2": 0})
|
||||
report = m.summary()
|
||||
assert report["utilization"]["mean_ws_per_node"] == pytest.approx(8 / 3)
|
||||
assert report["utilization"]["max_ws_per_node"] == 5
|
||||
assert report["utilization"]["nodes_with_zero_ws"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimNode — message dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSimNode:
|
||||
@pytest.fixture
|
||||
def fast_config(self):
|
||||
return SimConfig(
|
||||
llm_latency_mean=0.01,
|
||||
llm_latency_stddev=0.001,
|
||||
llm_tokens_mean=10,
|
||||
llm_tokens_stddev=2,
|
||||
llm_token_rate=1000,
|
||||
tool_latency_mean=0.01,
|
||||
tool_latency_stddev=0.001,
|
||||
tool_failure_rate=0.0,
|
||||
max_tool_rounds=0, # no tool calls — fast turn
|
||||
seed=42,
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_broker(self):
|
||||
broker = MagicMock()
|
||||
broker.list_nodes.return_value = []
|
||||
return broker
|
||||
|
||||
@pytest.fixture
|
||||
def node(self, fast_config, mock_broker):
|
||||
metrics = MetricsCollector()
|
||||
return SimNode("test-node", mock_broker, fast_config, metrics)
|
||||
|
||||
def test_handle_send_creates_workstream(self, node, mock_broker):
|
||||
async def _test():
|
||||
msg = SendMessage(message="hello", auto_approve=True)
|
||||
await node.handle_message(msg.to_json())
|
||||
|
||||
assert node.workstream_count == 1
|
||||
mock_broker.set_ws_owner.assert_called_once()
|
||||
assert mock_broker.publish_outbound.call_count > 0
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_handle_send_reuses_existing_ws(self, node, mock_broker):
|
||||
async def _test():
|
||||
msg1 = SendMessage(message="hello", auto_approve=True)
|
||||
await node.handle_message(msg1.to_json())
|
||||
assert node.workstream_count == 1
|
||||
|
||||
ws_id = list(node._workstreams.keys())[0]
|
||||
|
||||
msg2 = SendMessage(message="world", ws_id=ws_id, auto_approve=True)
|
||||
await node.handle_message(msg2.to_json())
|
||||
assert node.workstream_count == 1
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_published_events_are_valid_protocol(self, node, mock_broker):
|
||||
async def _test():
|
||||
msg = SendMessage(message="test", auto_approve=True)
|
||||
await node.handle_message(msg.to_json())
|
||||
|
||||
for c in mock_broker.publish_outbound.call_args_list:
|
||||
_channel, event_json = c[0]
|
||||
event = OutboundEvent.from_json(event_json)
|
||||
assert event.type != ""
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_state_transitions(self, node, mock_broker):
|
||||
async def _test():
|
||||
msg = SendMessage(message="test", auto_approve=True)
|
||||
await node.handle_message(msg.to_json())
|
||||
|
||||
states = []
|
||||
for c in mock_broker.publish_outbound.call_args_list:
|
||||
channel, event_json = c[0]
|
||||
event = OutboundEvent.from_json(event_json)
|
||||
if isinstance(event, StateChangeEvent):
|
||||
states.append(event.state)
|
||||
|
||||
assert "thinking" in states
|
||||
assert "idle" in states
|
||||
assert states.index("thinking") < states.index("idle")
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_turn_complete_published(self, node, mock_broker):
|
||||
async def _test():
|
||||
msg = SendMessage(message="test", auto_approve=True)
|
||||
await node.handle_message(msg.to_json())
|
||||
|
||||
turn_completes = [
|
||||
OutboundEvent.from_json(c[0][1])
|
||||
for c in mock_broker.publish_outbound.call_args_list
|
||||
if '"turn_complete"' in c[0][1]
|
||||
]
|
||||
assert len(turn_completes) >= 1
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_close_workstream(self, node, mock_broker):
|
||||
async def _test():
|
||||
msg = SendMessage(message="hello", auto_approve=True)
|
||||
await node.handle_message(msg.to_json())
|
||||
ws_id = list(node._workstreams.keys())[0]
|
||||
|
||||
from turnstone.mq.protocol import CloseWorkstreamMessage
|
||||
|
||||
close_msg = CloseWorkstreamMessage(ws_id=ws_id)
|
||||
await node.handle_message(close_msg.to_json())
|
||||
|
||||
assert node.workstream_count == 0
|
||||
mock_broker.del_ws_owner.assert_called_with(ws_id)
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_stop_cleans_up(self, node, mock_broker):
|
||||
# Add a fake workstream
|
||||
node._workstreams["fake"] = MagicMock()
|
||||
mock_broker.set_ws_owner("fake", "test-node")
|
||||
|
||||
node.stop()
|
||||
assert not node._running
|
||||
assert node.workstream_count == 0
|
||||
mock_broker.del_ws_owner.assert_called()
|
||||
|
||||
def test_heartbeat_once(self, node, mock_broker):
|
||||
node.heartbeat_once()
|
||||
mock_broker.register_node.assert_called_once()
|
||||
args = mock_broker.register_node.call_args
|
||||
assert args[0][0] == "test-node"
|
||||
assert args[0][1]["sim"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimWorkstream — state machine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSimWorkstream:
|
||||
@pytest.fixture
|
||||
def fast_config(self):
|
||||
return SimConfig(
|
||||
llm_latency_mean=0.01,
|
||||
llm_latency_stddev=0.001,
|
||||
llm_tokens_mean=10,
|
||||
llm_tokens_stddev=2,
|
||||
llm_token_rate=1000,
|
||||
tool_latency_mean=0.01,
|
||||
tool_latency_stddev=0.001,
|
||||
tool_failure_rate=0.0,
|
||||
max_tool_rounds=0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
def test_turn_ends_in_idle(self, fast_config):
|
||||
async def _test():
|
||||
broker = MagicMock()
|
||||
metrics = MetricsCollector()
|
||||
node = SimNode("test", broker, fast_config, metrics)
|
||||
engine = SimEngine(fast_config)
|
||||
ws = SimWorkstream("ws1", "test-ws", node, engine, fast_config)
|
||||
|
||||
await ws.process_turn("hello", "cid-123")
|
||||
assert ws.state == "idle"
|
||||
|
||||
_run(_test())
|
||||
|
||||
def test_turn_records_metrics(self, fast_config):
|
||||
async def _test():
|
||||
broker = MagicMock()
|
||||
metrics = MetricsCollector()
|
||||
node = SimNode("test", broker, fast_config, metrics)
|
||||
engine = SimEngine(fast_config)
|
||||
ws = SimWorkstream("ws1", "test-ws", node, engine, fast_config)
|
||||
|
||||
await ws.process_turn("hello", "cid-123")
|
||||
report = metrics.summary()
|
||||
assert report["total_turns"] == 1
|
||||
assert report["turns_per_node"]["test"] == 1
|
||||
|
||||
_run(_test())
|
||||
@@ -3,6 +3,7 @@
|
||||
from turnstone.core.memory import (
|
||||
count_structured_memories,
|
||||
delete_structured_memory,
|
||||
get_structured_memory_by_name,
|
||||
list_structured_memories,
|
||||
normalize_key,
|
||||
save_structured_memory,
|
||||
@@ -67,6 +68,29 @@ class TestSearchStructuredMemories:
|
||||
assert any(r["name"] == "db_host" for r in results)
|
||||
|
||||
|
||||
class TestGetStructuredMemoryByName:
|
||||
def test_get_existing(self, tmp_db):
|
||||
save_structured_memory("my_mem", "full content here that is quite long")
|
||||
mem = get_structured_memory_by_name("my_mem", "global", "")
|
||||
assert mem is not None
|
||||
assert mem["content"] == "full content here that is quite long"
|
||||
assert mem["name"] == "my_mem"
|
||||
|
||||
def test_get_nonexistent(self, tmp_db):
|
||||
assert get_structured_memory_by_name("nope", "global", "") is None
|
||||
|
||||
def test_get_wrong_scope(self, tmp_db):
|
||||
save_structured_memory("ws_mem", "data", scope="workstream", scope_id="ws1")
|
||||
assert get_structured_memory_by_name("ws_mem", "global", "") is None
|
||||
assert get_structured_memory_by_name("ws_mem", "workstream", "ws1") is not None
|
||||
|
||||
def test_get_normalizes_key(self, tmp_db):
|
||||
save_structured_memory("My-Key", "value")
|
||||
mem = get_structured_memory_by_name("My-Key", "global", "")
|
||||
assert mem is not None
|
||||
assert mem["name"] == "my_key"
|
||||
|
||||
|
||||
class TestCountStructuredMemories:
|
||||
def test_count_zero(self, tmp_db):
|
||||
assert count_structured_memories() == 0
|
||||
@@ -80,3 +104,109 @@ class TestCountStructuredMemories:
|
||||
class TestNormalizeKey:
|
||||
def test_basic(self):
|
||||
assert normalize_key("My-Key Name") == "my_key_name"
|
||||
|
||||
|
||||
class TestScopeIsolation:
|
||||
"""Verify that list/search without scope only returns visible memories.
|
||||
|
||||
Reproduces the cross-workstream leak: unscoped list/search must not
|
||||
return workstream-scoped memories from other workstreams or
|
||||
user-scoped memories from other users.
|
||||
"""
|
||||
|
||||
def _seed(self):
|
||||
"""Create memories across multiple scopes."""
|
||||
save_structured_memory("global_note", "visible to all", scope="global")
|
||||
save_structured_memory("ws1_note", "belongs to ws1", scope="workstream", scope_id="ws1")
|
||||
save_structured_memory("ws2_note", "belongs to ws2", scope="workstream", scope_id="ws2")
|
||||
save_structured_memory("u1_note", "belongs to user1", scope="user", scope_id="u1")
|
||||
save_structured_memory("u2_note", "belongs to user2", scope="user", scope_id="u2")
|
||||
|
||||
@staticmethod
|
||||
def _list_visible(ws_id: str, user_id: str, mem_type: str = "", limit: int = 50):
|
||||
"""Replicate the scope-filtered list logic from ChatSession."""
|
||||
global_mems = list_structured_memories(mem_type=mem_type, scope="global", limit=limit)
|
||||
ws_mems = list_structured_memories(
|
||||
mem_type=mem_type, scope="workstream", scope_id=ws_id, limit=limit
|
||||
)
|
||||
user_mems = (
|
||||
list_structured_memories(mem_type=mem_type, scope="user", scope_id=user_id, limit=limit)
|
||||
if user_id
|
||||
else []
|
||||
)
|
||||
combined = global_mems + ws_mems + user_mems
|
||||
combined.sort(key=lambda m: m.get("updated", ""), reverse=True)
|
||||
return combined[:limit]
|
||||
|
||||
@staticmethod
|
||||
def _search_visible(query: str, ws_id: str, user_id: str, mem_type: str = "", limit: int = 20):
|
||||
"""Replicate the scope-filtered search logic from ChatSession."""
|
||||
global_mems = search_structured_memories(
|
||||
query, mem_type=mem_type, scope="global", limit=limit
|
||||
)
|
||||
ws_mems = search_structured_memories(
|
||||
query, mem_type=mem_type, scope="workstream", scope_id=ws_id, limit=limit
|
||||
)
|
||||
user_mems = (
|
||||
search_structured_memories(
|
||||
query, mem_type=mem_type, scope="user", scope_id=user_id, limit=limit
|
||||
)
|
||||
if user_id
|
||||
else []
|
||||
)
|
||||
combined = global_mems + ws_mems + user_mems
|
||||
combined.sort(key=lambda m: m.get("updated", ""), reverse=True)
|
||||
return combined[:limit]
|
||||
|
||||
def test_unscoped_list_returns_all_scopes(self, tmp_db):
|
||||
"""Demonstrate the leak: unscoped list returns everything."""
|
||||
self._seed()
|
||||
all_mems = list_structured_memories()
|
||||
assert len(all_mems) == 5 # no scope filter → all memories
|
||||
|
||||
def test_visible_list_excludes_other_workstreams(self, tmp_db):
|
||||
"""Scope-filtered list for ws1/u1 excludes ws2 and u2 memories."""
|
||||
self._seed()
|
||||
visible = self._list_visible("ws1", "u1")
|
||||
names = {m["name"] for m in visible}
|
||||
assert "global_note" in names
|
||||
assert "ws1_note" in names
|
||||
assert "u1_note" in names
|
||||
assert "ws2_note" not in names
|
||||
assert "u2_note" not in names
|
||||
|
||||
def test_visible_list_no_user(self, tmp_db):
|
||||
"""Scope-filtered list with no user_id excludes all user memories."""
|
||||
self._seed()
|
||||
visible = self._list_visible("ws1", "")
|
||||
names = {m["name"] for m in visible}
|
||||
assert "global_note" in names
|
||||
assert "ws1_note" in names
|
||||
assert "u1_note" not in names
|
||||
assert "u2_note" not in names
|
||||
|
||||
def test_visible_search_no_user(self, tmp_db):
|
||||
"""Scope-filtered search with no user_id excludes all user memories."""
|
||||
self._seed()
|
||||
visible = self._search_visible("belongs", "ws1", "")
|
||||
names = {m["name"] for m in visible}
|
||||
assert "ws1_note" in names
|
||||
assert "u1_note" not in names
|
||||
assert "u2_note" not in names
|
||||
|
||||
def test_visible_search_excludes_other_workstreams(self, tmp_db):
|
||||
"""Scope-filtered search for ws1/u1 excludes ws2 and u2 memories."""
|
||||
self._seed()
|
||||
visible = self._search_visible("belongs", "ws1", "u1")
|
||||
names = {m["name"] for m in visible}
|
||||
assert "ws1_note" in names
|
||||
assert "u1_note" in names
|
||||
assert "ws2_note" not in names
|
||||
assert "u2_note" not in names
|
||||
|
||||
def test_explicit_scope_still_works(self, tmp_db):
|
||||
"""Explicit scope filter continues to work as before."""
|
||||
self._seed()
|
||||
ws2_only = list_structured_memories(scope="workstream", scope_id="ws2")
|
||||
assert len(ws2_only) == 1
|
||||
assert ws2_only[0]["name"] == "ws2_note"
|
||||
|
||||
+2
-13
@@ -134,13 +134,13 @@ def test_cli_bootstrap(tmp_path):
|
||||
from turnstone.admin import _cmd_tls_bootstrap
|
||||
|
||||
out = tmp_path / "certs"
|
||||
args = argparse.Namespace(out=str(out), issue=["redis.internal", "pg.internal"])
|
||||
args = argparse.Namespace(out=str(out), issue=["app.internal", "pg.internal"])
|
||||
_cmd_tls_bootstrap(args)
|
||||
|
||||
assert (out / "ca.pem").exists()
|
||||
assert b"BEGIN CERTIFICATE" in (out / "ca.pem").read_bytes()
|
||||
# Check certs were issued
|
||||
assert (out / "certs" / "redis.internal").exists()
|
||||
assert (out / "certs" / "app.internal").exists()
|
||||
assert (out / "certs" / "pg.internal").exists()
|
||||
|
||||
|
||||
@@ -164,17 +164,6 @@ def test_cli_bootstrap_no_issue(tmp_path):
|
||||
# ── Config parsing ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_redis_tls_config_map():
|
||||
"""Redis TLS keys are in the config map."""
|
||||
from turnstone.core.config import _CONFIG_MAP
|
||||
|
||||
redis_map = _CONFIG_MAP["redis"]
|
||||
assert "tls" in redis_map
|
||||
assert "tls_ca" in redis_map
|
||||
assert "tls_cert" in redis_map
|
||||
assert "tls_key" in redis_map
|
||||
|
||||
|
||||
def test_database_ssl_config_map():
|
||||
"""Database SSL keys are in the config map."""
|
||||
from turnstone.core.config import _CONFIG_MAP
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user