mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 15:32:24 -06:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 364a49c7fb |
@@ -43,13 +43,6 @@ jobs:
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
# Node is required by tests/test_renderer_js.py — without
|
||||
# explicit setup, that suite silently skips if the runner
|
||||
# image happens not to ship Node, masking regressions in
|
||||
# the browser-side renderer.
|
||||
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: "20"
|
||||
- run: pip install -e ".[test]"
|
||||
- run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -q
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
@@ -79,9 +72,6 @@ jobs:
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: "20"
|
||||
- run: pip install -e ".[test,postgres]"
|
||||
- run: pytest tests/ -m "not live" --storage-backend=postgresql -q
|
||||
env:
|
||||
@@ -138,7 +128,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- run: uv lock --check
|
||||
@@ -147,7 +137,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
@@ -165,7 +155,7 @@ jobs:
|
||||
working-directory: sdk/typescript
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
|
||||
with:
|
||||
node-version: "24"
|
||||
- run: npm ci
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Docker Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, "stable/*"]
|
||||
schedule:
|
||||
- cron: "0 6 * * 1" # Weekly Monday 06:00 UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- run: docker build -t turnstone:scan .
|
||||
- uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0
|
||||
with:
|
||||
image-ref: "turnstone:scan"
|
||||
severity: "HIGH,CRITICAL"
|
||||
exit-code: "1"
|
||||
@@ -21,5 +21,3 @@ PROGRESS.md
|
||||
.coverage
|
||||
tools/skill_audit_analysis/data/
|
||||
tools/skill_audit_analysis/output/
|
||||
design_ideas/
|
||||
.claude/
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# libexpat integer overflow — no fix available in Debian repos yet
|
||||
# https://avd.aquasec.com/nvd/cve-2026-25210
|
||||
# Review: remove this entry once a patched libexpat1 is published
|
||||
CVE-2026-25210
|
||||
|
||||
# ncurses buffer overflow — no fix in Debian 13 repos yet
|
||||
# Affects libncursesw6, libtinfo6, ncurses-base, ncurses-bin
|
||||
# https://avd.aquasec.com/nvd/cve-2025-69720
|
||||
CVE-2025-69720
|
||||
|
||||
# nghttp2 DoS via malformed HTTP/2 frames — no fix in Debian 13 repos yet
|
||||
# Affects libnghttp2-14
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27135
|
||||
CVE-2026-27135
|
||||
|
||||
# systemd arbitrary code execution via spurious IPC — no fix in Debian 13 repos yet
|
||||
# Affects libsystemd0, libudev1
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29111
|
||||
CVE-2026-29111
|
||||
|
||||
# glibc iconv() DoS — fix_deferred, no patched libc in Debian 13 yet
|
||||
# Affects libc-bin, libc6
|
||||
# https://avd.aquasec.com/nvd/cve-2026-4046
|
||||
CVE-2026-4046
|
||||
|
||||
# minimatch ReDoS — transitive npm dep (MCP server), no direct exposure
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27903
|
||||
CVE-2026-27903
|
||||
# https://avd.aquasec.com/nvd/cve-2026-27904
|
||||
CVE-2026-27904
|
||||
|
||||
# picomatch ReDoS — transitive npm dep, no direct exposure
|
||||
# https://avd.aquasec.com/nvd/cve-2026-33671
|
||||
CVE-2026-33671
|
||||
|
||||
# node-tar path traversal — transitive npm dep, not used to extract untrusted archives
|
||||
# https://avd.aquasec.com/nvd/cve-2026-29786
|
||||
CVE-2026-29786
|
||||
# https://avd.aquasec.com/nvd/cve-2026-31802
|
||||
CVE-2026-31802
|
||||
|
||||
# jq out-of-bounds read on non-NUL-terminated buffers — no fix in Debian 13 repos yet.
|
||||
# Affects jq + libjq1 (1.7.1-6+deb13u1). jq is invoked only on trusted
|
||||
# CLI/admin paths against process-controlled JSON input, never on untrusted
|
||||
# network bytes, so the NUL-terminated invariant holds in our usage.
|
||||
# https://avd.aquasec.com/nvd/cve-2026-39979
|
||||
CVE-2026-39979
|
||||
|
||||
# jq DoS via crafted JSON object causing hash collisions — no fix in Debian 13 repos yet.
|
||||
# Affects jq + libjq1 (1.7.1-6+deb13u1). Same trust boundary as above:
|
||||
# jq is not exposed to attacker-controlled JSON in turnstone.
|
||||
# https://avd.aquasec.com/nvd/cve-2026-40164
|
||||
CVE-2026-40164
|
||||
+18
-1070
File diff suppressed because it is too large
Load Diff
+1
-9
@@ -26,15 +26,7 @@ transferring ownership.
|
||||
```
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -e ".[test,dev]"
|
||||
```
|
||||
|
||||
The `dev` extra installs `ruff` and `mypy`. Before pushing, run:
|
||||
|
||||
```
|
||||
ruff check turnstone tests
|
||||
mypy turnstone
|
||||
pytest
|
||||
pip install -e ".[test]"
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
|
||||
+4
-4
@@ -55,7 +55,7 @@ The wizard supports two deployment modes:
|
||||
```
|
||||
$ turnstone-bootstrap
|
||||
|
||||
Turnstone Bootstrap Wizard v1.5.0
|
||||
Turnstone Bootstrap Wizard v0.5.4
|
||||
────────────────────────────────────────────────
|
||||
|
||||
Which provider for this wizard?
|
||||
@@ -87,6 +87,6 @@ $ turnstone-bootstrap
|
||||
|
||||
## See Also
|
||||
|
||||
- [Docker Deployment](docs/docker.md) — manual compose setup and profiles
|
||||
- [Security](docs/security.md) — auth architecture and token types
|
||||
- [Governance](docs/governance.md) — roles, policies, and templates
|
||||
- [Docker Deployment](docker.md) — manual compose setup and profiles
|
||||
- [Security](security.md) — auth architecture and token types
|
||||
- [Governance](governance.md) — roles, policies, and templates
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
Multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers with direct HTTP routing, interactive interfaces, and enterprise governance.
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/assets/hero.png" alt="Turnstone coordinator — parallel tool batches with judge-graded approval and child workstream tracking" width="960"/>
|
||||
<img src="docs/assets/hero.png" alt="Turnstone console — multi-workstream AI orchestration with mermaid diagrams" width="960"/>
|
||||
</p>
|
||||
|
||||
Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) (*Arenaria interpres*) — a shorebird that flips stones to discover what's hiding underneath.
|
||||
@@ -84,20 +84,20 @@ with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
|
||||
## Tools
|
||||
|
||||
Built-in tools for shell, files, search, web, memory, notifications, and autonomous sub-agents — plus external tools via [MCP](https://modelcontextprotocol.io/) with native deferred loading. See [docs/tools.md](docs/tools.md) for the full reference and [docs/mcp-registry.md](docs/mcp-registry.md) for MCP configuration.
|
||||
Built-in tools for shell, files, search, web, memory, notifications, and autonomous sub-agents — plus external tools via [MCP](https://modelcontextprotocol.io/) with native deferred loading. See [docs/tools.md](docs/tools.md) for the full reference and [docs/mcp.md](docs/mcp.md) for MCP configuration.
|
||||
|
||||
## Architecture
|
||||
|
||||
**Single-node**: Client → Server (direct HTTP + SSE). No external dependencies beyond the database.
|
||||
|
||||
**Multi-node**: Client → Console (rendezvous routing proxy) → Server nodes. The console picks the target node for each workstream via rendezvous (HRW) hashing over the live service registry — pure function of `(ws_id, live_nodes)`, no stored bucket state, deterministic across readers. A node join or drop only re-routes the keys that score highest on the affected node.
|
||||
**Multi-node**: Client → Console (hash ring routing proxy) → Server nodes. The console maintains a 65536-entry bucket cache for O(1) workstream routing. A rebalancer daemon redistributes buckets when nodes join or leave.
|
||||
|
||||
| Component | Purpose |
|
||||
|-----------|---------|
|
||||
| `turnstone` | Terminal CLI (REPL) |
|
||||
| `turnstone-server` | Web UI + REST API + SSE events |
|
||||
| `turnstone-console` | Cluster dashboard + routing proxy + admin panel |
|
||||
| `turnstone-channel` | Channel gateway (Discord and Slack adapters) |
|
||||
| `turnstone-channel` | Channel gateway (Discord, with adapters for Slack/Teams planned) |
|
||||
| `turnstone-admin` | User/token management CLI |
|
||||
| `turnstone-eval` | Eval harness for prompt/tool optimization |
|
||||
| `turnstone-bootstrap` | LLM-guided setup wizard |
|
||||
@@ -117,7 +117,7 @@ UML diagrams in [`docs/diagrams/`](docs/diagrams/):
|
||||
| [Console Data Flow](docs/diagrams/png/11-console-data-flow.png) | Dashboard data collection |
|
||||
| [Deployment](docs/diagrams/png/12-deployment.png) | Docker Compose topology |
|
||||
| [Auth](docs/diagrams/png/15-auth-architecture.png) | JWT, scopes, login flows |
|
||||
| [Channels](docs/diagrams/png/16-channel-architecture.png) | Discord / Slack adapters + routing |
|
||||
| [Channels](docs/diagrams/png/16-channel-architecture.png) | Discord adapter + routing |
|
||||
| [Judge](docs/diagrams/png/22-judge-architecture.png) | Intent validation pipeline |
|
||||
| [OIDC](docs/diagrams/png/25-oidc-architecture.png) | SSO authorization code flow |
|
||||
|
||||
@@ -136,7 +136,7 @@ UML diagrams in [`docs/diagrams/`](docs/diagrams/):
|
||||
| Console dashboard | [docs/console.md](docs/console.md) |
|
||||
| Eval harness | [docs/eval.md](docs/eval.md) |
|
||||
| Tools reference | [docs/tools.md](docs/tools.md) |
|
||||
| MCP integration | [docs/mcp-registry.md](docs/mcp-registry.md) |
|
||||
| MCP integration | [docs/mcp.md](docs/mcp.md) |
|
||||
|
||||
## Requirements
|
||||
|
||||
|
||||
+3
-3
@@ -57,8 +57,8 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
cpus: '4.0'
|
||||
memory: 1G
|
||||
cpus: '1.0'
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
@@ -231,7 +231,7 @@ services:
|
||||
start_period: 60s
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 4G, cpus: '4' }
|
||||
limits: { memory: 384M, cpus: '0.5' }
|
||||
restart: unless-stopped
|
||||
|
||||
server-2:
|
||||
|
||||
@@ -7,6 +7,6 @@ appVersion: "0.3.0"
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: ~18.6.0
|
||||
version: ~18.5.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
|
||||
+36
-165
@@ -229,12 +229,12 @@ below.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/workstreams/{ws_id}/events`
|
||||
### `GET /v1/api/events?ws_id=<id>`
|
||||
|
||||
Opens a Server-Sent Events stream scoped to a single workstream. The connection
|
||||
remains open indefinitely; the server pushes events as they occur.
|
||||
|
||||
**Path parameters:**
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------------|
|
||||
@@ -346,7 +346,7 @@ action required).
|
||||
```
|
||||
|
||||
**`approve_request`** -- one or more tool calls that require user approval. The
|
||||
client must respond via `POST /v1/api/workstreams/{ws_id}/approve`.
|
||||
client must respond via `POST /v1/api/approve`.
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -450,7 +450,7 @@ after `/clear` or `/new` commands).
|
||||
```
|
||||
|
||||
**`cancelled`** -- a cancel request was acknowledged (via the Stop button or
|
||||
`POST /v1/api/workstreams/{ws_id}/cancel`). This signals that cancellation is in progress, not
|
||||
`POST /v1/api/cancel`). This signals that cancellation is in progress, not
|
||||
that it is complete. The worker thread may still be finishing — wait for
|
||||
`stream_end` before transitioning to a ready state. The client should clear
|
||||
any in-progress assistant rendering but not re-enable the send button until
|
||||
@@ -558,7 +558,7 @@ Possible `state` values:
|
||||
and copies each event to every client queue. If a client queue is full, the
|
||||
event is silently dropped for that client.
|
||||
|
||||
**Keepalive:** Same as `/v1/api/workstreams/{ws_id}/events` -- an SSE comment every 5 seconds.
|
||||
**Keepalive:** Same as `/v1/api/events` -- an SSE comment every 5 seconds.
|
||||
|
||||
---
|
||||
|
||||
@@ -571,8 +571,8 @@ Returns a list of all active workstreams.
|
||||
```json
|
||||
{
|
||||
"workstreams": [
|
||||
{"ws_id": "abc123", "name": "default", "state": "idle"},
|
||||
{"ws_id": "def456", "name": "hacker-news", "state": "thinking"}
|
||||
{"id": "abc123", "name": "default", "state": "idle"},
|
||||
{"id": "def456", "name": "hacker-news", "state": "thinking"}
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -581,7 +581,7 @@ Each workstream object:
|
||||
|
||||
| Field | Type | Description |
|
||||
|--------------|-------------|--------------------------------------------------------|
|
||||
| `ws_id` | string | Unique workstream routing identifier |
|
||||
| `id` | string | Unique workstream routing identifier |
|
||||
| `name` | string | Display name (alias if set, otherwise `ws-xxxx`) |
|
||||
| `state` | string | Current state (see state values above) |
|
||||
|
||||
@@ -654,26 +654,21 @@ Each skill summary:
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/send`
|
||||
### `POST /v1/api/send`
|
||||
|
||||
Sends a user message to a workstream. Spawns a daemon worker thread that calls
|
||||
`session.send()` and streams results back via the SSE channel.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------|
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"message": "Explain how the server works"}
|
||||
{"message": "Explain how the server works", "ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------------------|
|
||||
| `message` | string | yes | The user's message text |
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Response (success):**
|
||||
|
||||
@@ -697,21 +692,15 @@ from a previous request. Also pushes a `busy_error` event to the SSE stream.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/approve`
|
||||
### `POST /v1/api/approve`
|
||||
|
||||
Responds to a tool approval request. The SSE stream must have previously sent
|
||||
an `approve_request` event for the given workstream.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------|
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"approved": true, "feedback": null, "always": false}
|
||||
{"approved": true, "feedback": null, "always": false, "ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
@@ -719,6 +708,7 @@ an `approve_request` event for the given workstream.
|
||||
| `approved` | bool | yes | `true` to approve, `false` to deny |
|
||||
| `feedback` | string/null | no | Optional feedback text (sent as denial reason) |
|
||||
| `always` | bool | no | If `true` and `approved`, enables auto-approve |
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
When `always` is `true` and `approved` is `true`, the workstream's WebUI
|
||||
instance sets `auto_approve = True`, causing all subsequent tool calls to be
|
||||
@@ -799,7 +789,7 @@ containing the resumed session's messages.
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/cancel`
|
||||
### `POST /v1/api/cancel`
|
||||
|
||||
Cancels the active generation in a workstream. Sets a cooperative cancellation
|
||||
flag that is checked at multiple points in the generation loop (per streaming
|
||||
@@ -822,20 +812,15 @@ for the orphaned thread. Use force cancel when cooperative cancel has not
|
||||
resolved within a few seconds — the web UI offers this as a "Force Stop"
|
||||
button automatically.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|----------------------|
|
||||
| `ws_id` | string | yes | Target workstream ID |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{"force": false}
|
||||
{"ws_id": "abc123", "force": false}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|--------|--------|----------|----------------------|
|
||||
| `ws_id`| string | yes | Target workstream ID |
|
||||
| `force`| bool | no | Abandon stuck worker immediately (default: `false`) |
|
||||
|
||||
**Response:**
|
||||
@@ -857,15 +842,6 @@ button automatically.
|
||||
|
||||
Creates a new workstream. The server supports up to 10 concurrent workstreams.
|
||||
|
||||
The endpoint accepts **either** `application/json` (legacy shape) **or**
|
||||
`multipart/form-data` when you want to upload attachments at creation
|
||||
time. Multipart requests carry one `meta` field containing the JSON body
|
||||
shown below plus zero-or-more `file` parts; each file is validated and
|
||||
reserved onto the new workstream's first turn before the dispatch worker
|
||||
runs, so queued multimodal turns cannot lose files to racing sends. If
|
||||
validation fails the fresh workstream is rolled back so no orphan rows
|
||||
leak.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
@@ -908,32 +884,20 @@ Status code: `400`
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/close`
|
||||
### `POST /v1/api/workstreams/close`
|
||||
|
||||
Closes and removes a workstream. The last remaining workstream cannot be
|
||||
closed.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|------------------------|
|
||||
| `ws_id` | string | yes | Workstream ID to close |
|
||||
|
||||
**Request body:**
|
||||
|
||||
The body must be valid JSON. If you are not supplying any optional
|
||||
fields, send `{}` — an empty / non-JSON body is rejected with a
|
||||
`400`.
|
||||
```json
|
||||
{"ws_id": "abc123"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|----------|--------|----------|----------------------------------------------------------|
|
||||
| `reason` | string | no | Optional close reason persisted to `workstream_config`. |
|
||||
|
||||
The `reason` is capped at **512 UTF-8 bytes** (multibyte-safe — the
|
||||
cap holds for CJK and emoji payloads), and the output guard's
|
||||
credential-redaction pass strips secrets before the value is
|
||||
persisted. A non-string `reason` is silently coerced to empty and
|
||||
the close proceeds without writing the field.
|
||||
| Field | Type | Required | Description |
|
||||
|---------|--------|----------|---------------------------|
|
||||
| `ws_id` | string | yes | Workstream ID to close |
|
||||
|
||||
**Response (success):**
|
||||
|
||||
@@ -951,100 +915,6 @@ Status code: `400`
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/attachments`
|
||||
|
||||
Upload an image or text document and attach it to the caller's next user
|
||||
turn on this workstream.
|
||||
|
||||
- Images (png/jpeg/gif/webp) are capped at **4 MiB** and validated via
|
||||
magic-byte sniff on upload.
|
||||
- Text documents (any `text/*` MIME, allow-listed application MIMEs, or
|
||||
known text extensions) are capped at **512 KiB** and must be UTF-8.
|
||||
- Per-(workstream, user) pending cap is **10** attachments.
|
||||
|
||||
The attachment moves through three states: `pending → reserved →
|
||||
consumed`. Reservation tokens are threaded through
|
||||
`POST /v1/api/workstreams/{ws_id}/send` so a queued multimodal turn cannot lose its file to
|
||||
an overlapping send.
|
||||
|
||||
Ownership failures are masked as `404` so non-owners cannot enumerate
|
||||
workstream existence.
|
||||
|
||||
**Content-Type:** `multipart/form-data` with a single `file` field.
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"attachment_id": "att_abc123",
|
||||
"kind": "image",
|
||||
"mime_type": "image/png",
|
||||
"size_bytes": 73240,
|
||||
"filename": "screenshot.png",
|
||||
"state": "pending"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
| Code | Meaning |
|
||||
|------|---------------------------------------------------------|
|
||||
| 400 | Missing/invalid form, unsupported MIME, not UTF-8, etc. |
|
||||
| 403 | Auth/scope failure |
|
||||
| 404 | Workstream not found / not owned by caller |
|
||||
| 409 | Pending-cap reached |
|
||||
| 413 | Payload exceeds size cap |
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/workstreams/{ws_id}/attachments`
|
||||
|
||||
List the caller's **pending** (unconsumed) attachments for this
|
||||
workstream. Ownership failures are masked as `404`.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"attachments": [
|
||||
{
|
||||
"attachment_id": "att_abc123",
|
||||
"kind": "image",
|
||||
"mime_type": "image/png",
|
||||
"size_bytes": 73240,
|
||||
"filename": "screenshot.png",
|
||||
"state": "pending"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/workstreams/{ws_id}/attachments/{attachment_id}/content`
|
||||
|
||||
Returns the raw bytes of an attachment with its stored `Content-Type`.
|
||||
Useful for previewing an image or replaying a document. Ownership
|
||||
failures are masked as `404`.
|
||||
|
||||
**Response:** `200` — binary body, original `Content-Type`.
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/workstreams/{ws_id}/attachments/{attachment_id}`
|
||||
|
||||
Remove a pending attachment. Consumed attachments return `404` (they
|
||||
are part of a committed conversation turn). Ownership failures are also
|
||||
masked as `404`.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{"deleted": "att_abc123"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/workstreams/{ws_id}/delete`
|
||||
|
||||
Permanently delete a saved workstream and all its messages from storage.
|
||||
@@ -1638,7 +1508,7 @@ version. Requires the `admin.skills` permission.
|
||||
|
||||
```json
|
||||
{
|
||||
"risk_level": "medium",
|
||||
"scan_status": "medium",
|
||||
"scan_report": "{\"composite\": 1.75, \"details\": {...}}",
|
||||
"scan_version": "1"
|
||||
}
|
||||
@@ -1979,7 +1849,7 @@ Status code: `200` with an empty body.
|
||||
| Malformed or unparseable JSON body | Treated as an empty dict `{}`; missing fields use defaults |
|
||||
| Unknown `ws_id` | `404` with `{"error": "Unknown workstream"}` |
|
||||
| Unknown path (GET or POST) | `404` with plain-text body `Not found` |
|
||||
| Empty `message` on `/v1/api/workstreams/{ws_id}/send` | `400` with `{"error": "Empty message"}` |
|
||||
| Empty `message` on `/v1/api/send` | `400` with `{"error": "Empty message"}` |
|
||||
| Empty `command` on `/v1/api/command` | `400` with `{"error": "Empty command"}` |
|
||||
| Rate limit exceeded | `429` with `Retry-After` header (see below) |
|
||||
|
||||
@@ -2022,7 +1892,7 @@ reconnection:
|
||||
On reconnect, the server replays the full conversation history via the
|
||||
`history` event, so the client can rebuild its UI state without data loss. The
|
||||
same reconnection strategy applies to both the per-workstream SSE stream
|
||||
(`/v1/api/workstreams/{ws_id}/events`) and the global state stream (`/v1/api/events/global`).
|
||||
(`/v1/api/events`) and the global state stream (`/v1/api/events/global`).
|
||||
|
||||
---
|
||||
|
||||
@@ -2132,7 +2002,7 @@ turnstone_workstreams_active_total 1
|
||||
# TYPE turnstone_http_requests_total counter
|
||||
turnstone_http_requests_total{method="GET",endpoint="/health",status_code="200"} 42
|
||||
turnstone_http_requests_total{method="GET",endpoint="/metrics",status_code="200"} 7
|
||||
turnstone_http_requests_total{method="POST",endpoint="/v1/api/workstreams/{ws_id}/send",status_code="200"} 18
|
||||
turnstone_http_requests_total{method="POST",endpoint="/v1/api/send",status_code="200"} 18
|
||||
# HELP turnstone_tokens_total Total tokens consumed
|
||||
# TYPE turnstone_tokens_total counter
|
||||
turnstone_tokens_total{type="prompt"} 84320
|
||||
@@ -2148,15 +2018,15 @@ turnstone_tool_calls_total{tool="read_file"} 3
|
||||
## Console Routing Proxy Endpoints
|
||||
|
||||
These endpoints are served by the console (`turnstone-console`) and proxy
|
||||
requests to the correct server node via rendezvous (HRW) hashing over the
|
||||
live service registry. In multi-node deployments, clients (SDK, channel
|
||||
gateway) talk to the console instead of individual server nodes.
|
||||
requests to the correct server node via the hash ring bucket cache. In
|
||||
multi-node deployments, clients (SDK, channel gateway) talk to the console
|
||||
instead of individual server nodes.
|
||||
|
||||
### `POST /v1/api/route/workstreams/new`
|
||||
|
||||
Create a workstream via rendezvous routing. The console generates the `ws_id`,
|
||||
routes to the rendezvous-selected node, and includes `node_url` in the
|
||||
response for direct SSE connections.
|
||||
Create a workstream via hash-ring routing. The console generates the `ws_id`,
|
||||
routes to the assigned node, and includes `node_url` in the response for
|
||||
direct SSE connections.
|
||||
|
||||
### `POST /v1/api/route/send`
|
||||
|
||||
@@ -2191,4 +2061,5 @@ Used by channel adapters to open direct SSE connections to the correct server no
|
||||
|
||||
Prometheus metrics for the console routing layer. Includes:
|
||||
`turnstone_router_requests_total`, `turnstone_router_request_duration_seconds`,
|
||||
`turnstone_router_membership_size`, `turnstone_router_refresh_total`.
|
||||
`turnstone_ring_membership_size`, `turnstone_ring_version`,
|
||||
`turnstone_ring_rebalance_total`, `turnstone_ring_migrations_total`.
|
||||
|
||||
+45
-61
@@ -21,8 +21,7 @@ plugs in.
|
||||
| `turnstone-console` | `turnstone.console.server` | ClusterCollector | Cluster dashboard (aggregates all nodes) |
|
||||
| `turnstone-eval` | `turnstone.eval` | `NullUI` | Headless evaluation and prompt optimization |
|
||||
| `turnstone-channel` | `turnstone.channels.cli` | ChannelAdapter | Channel gateway (Discord, Slack, etc.) |
|
||||
| `turnstone-admin` | `turnstone.admin` | — | Offline user and API token management |
|
||||
| `turnstone-bootstrap` | `turnstone.bootstrap` | — | LLM-guided setup wizard |
|
||||
| `turnstone-admin` | `turnstone.core.admin_cli` | — | Offline user and API token management |
|
||||
|
||||
---
|
||||
|
||||
@@ -37,10 +36,7 @@ turnstone/
|
||||
session.py ChatSession engine, SessionUI protocol, tool dispatch
|
||||
providers/ LLM provider adapters (pluggable backend layer)
|
||||
_protocol.py LLMProvider protocol, ModelCapabilities, StreamChunk, CompletionResult
|
||||
_openai.py OpenAIProvider facade (re-exports Chat/Responses providers)
|
||||
_openai_chat.py OpenAIChatCompletionsProvider — vLLM, llama.cpp, local compatible APIs
|
||||
_openai_responses.py OpenAIResponsesProvider — commercial OpenAI Responses API
|
||||
_openai_common.py Shared ModelCapabilities table + helpers
|
||||
_openai.py OpenAIProvider — OpenAI, vLLM, llama.cpp, any compatible API
|
||||
_anthropic.py AnthropicProvider — Anthropic Messages API, native streaming, thinking
|
||||
_google.py GoogleProvider — Google Gemini via OpenAI-compat endpoint
|
||||
__init__.py create_provider() + create_client() factory functions
|
||||
@@ -85,11 +81,10 @@ turnstone/
|
||||
static/ Cluster dashboard web UI (page-specific HTML, CSS, JS)
|
||||
channels/
|
||||
cli.py Unified channel gateway entry point (turnstone-channel)
|
||||
_protocol.py ChannelAdapter protocol
|
||||
_protocol.py ChannelAdapter protocol, ChannelEvent dataclass
|
||||
_routing.py ChannelRouter — channel/thread ↔ workstream mapping via HTTP
|
||||
_config.py Base ChannelConfig dataclass
|
||||
discord/ Discord adapter (bot, cog, views, streaming, config)
|
||||
slack/ Slack adapter (Socket Mode bot, DM routing, approval buttons)
|
||||
shared_static/ Shared design system (base.css, auth.js, theme.js, toast.js, utils.js, kb.js)
|
||||
katex-0.16.45/ Vendored KaTeX math rendering library (MIT, woff2 fonts)
|
||||
ui/
|
||||
@@ -102,7 +97,7 @@ turnstone/
|
||||
renderer.js Markdown + LaTeX renderer (tables, nested lists, blockquotes, KaTeX math)
|
||||
app.js Split-pane UI (Pane class, binary layout tree, SSE, tool approval)
|
||||
tools/
|
||||
*.json 19 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
*.json 15 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
```
|
||||
|
||||
Both UIs share a common design system extracted into `turnstone/shared_static/`: design tokens, login overlay, toast notifications, theme toggle, keyboard shortcuts, and utility functions. Each UI imports `base.css` and the shared JS modules at `/shared/`, then adds only page-specific code at `/static/`.
|
||||
@@ -384,11 +379,11 @@ non-idle background workstreams above the input prompt.
|
||||
(`Ctrl+\`, `Ctrl+Shift+\`). Max 6 panes; no duplicate workstreams across panes.
|
||||
Layout persisted to `localStorage`.
|
||||
- **Per-pane SSE**: `Pane.connectSSE(wsId)` opens
|
||||
`/v1/api/workstreams/{ws_id}/events` for each pane's event stream independently.
|
||||
`/v1/api/events?ws_id=<id>` for each pane's event stream independently.
|
||||
- **Global SSE**: `connectGlobalSSE()` opens `/v1/api/events/global` which
|
||||
receives `ws_state` broadcasts from all workstreams, used to update tab
|
||||
indicators and pane headers without switching.
|
||||
- **New tab / close**: POST `/v1/api/workstreams/new`, POST `/v1/api/workstreams/{ws_id}/close`.
|
||||
- **New tab / close**: POST `/v1/api/workstreams/new`, POST `/v1/api/workstreams/close`.
|
||||
|
||||
### Thread Safety
|
||||
|
||||
@@ -448,15 +443,13 @@ from each schema and builds:
|
||||
- `PRIMARY_KEY_MAP` -- `{name: primary_key}` for JSON fallback recovery
|
||||
- `merge_mcp_tools(builtin, mcp_tools)` -- merges built-in + MCP tools at session init
|
||||
|
||||
### 19 Tools by Category
|
||||
### 13 Tools by Category
|
||||
|
||||
**Read-only (auto-approve)**:
|
||||
- `read_file` -- read file contents with optional offset/limit
|
||||
- `diff_file` -- show diff between two files / versions
|
||||
- `search` -- ripgrep-based codebase search
|
||||
- `man` -- read man pages
|
||||
- `recall` -- search conversation history
|
||||
- `read_resource` -- read an MCP resource by URI
|
||||
|
||||
**Write (requires approval)**:
|
||||
- `bash` -- execute shell commands (with safety checks via `turnstone.core.safety`)
|
||||
@@ -465,20 +458,13 @@ from each schema and builds:
|
||||
- `math` -- execute Python in sandboxed subprocess (via `turnstone.core.sandbox`)
|
||||
- `web_fetch` -- fetch a URL (with SSRF protection via `turnstone.core.web`)
|
||||
- `web_search` -- search the web (provider-native for Anthropic/OpenAI, Tavily fallback for local models)
|
||||
- `notify` -- send a user-facing notification (Discord/Slack, optional reply routing)
|
||||
- `watch` -- schedule a recurring poll with condition DSL
|
||||
|
||||
**Agent (delegated sub-sessions)**:
|
||||
- `task_agent` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`)
|
||||
- `plan_agent` -- explore codebase and write a structured plan (`AGENT_TOOLS`)
|
||||
- `task` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`)
|
||||
- `plan` -- explore codebase and write a structured plan (`AGENT_TOOLS`)
|
||||
|
||||
**Memory / skills / prompts**:
|
||||
**Memory (structured persistent store)**:
|
||||
- `memory` -- save, search, delete, or list memories (typed and scoped)
|
||||
- `skill` -- invoke a skill (governed, versioned procedure)
|
||||
- `use_prompt` -- fetch and apply a prompt template
|
||||
|
||||
Tool names are `plan_agent` / `task_agent` (not `plan` / `task`); bare words
|
||||
collide with chat-template channels on some local models.
|
||||
|
||||
### Prepare / Execute Pattern
|
||||
|
||||
@@ -497,14 +483,14 @@ separation allows the UI to show previews before any side effects occur.
|
||||
|
||||
### Agent Tools
|
||||
|
||||
`task_agent` and `plan_agent` invoke `_run_agent()`, which runs a multi-turn
|
||||
loop with a subset of tools and its own system prompt. The sub-agent runs
|
||||
`task` and `plan` invoke `_run_agent()`, which runs a multi-turn loop with
|
||||
a subset of tools and its own system prompt. The sub-agent runs
|
||||
independently, then returns the final content as the tool result.
|
||||
|
||||
- **task_agent**: uses `self._task_tools` (`TASK_AGENT_TOOLS` + MCP tools)
|
||||
- **plan_agent**: uses `self._agent_tools` (`AGENT_TOOLS` + MCP tools). Writes output
|
||||
- **task**: uses `self._task_tools` (`TASK_AGENT_TOOLS` + MCP tools)
|
||||
- **plan**: uses `self._agent_tools` (`AGENT_TOOLS` + MCP tools). Writes output
|
||||
to `.plan-<ws_id>.md` — unique per `ChatSession` so concurrent workstreams
|
||||
don't collide. On repeat invocations the prior `plan_agent` tool call and its result
|
||||
don't collide. On repeat invocations the prior `plan` tool call and its result
|
||||
are forwarded from `self.messages` so the agent refines the existing plan rather
|
||||
than starting over. Planning instructions are injected as a developer message
|
||||
prepended to the agent's conversation.
|
||||
@@ -884,7 +870,7 @@ and are the single source of truth for both backends and Alembic migrations.
|
||||
| `update_workstream_title(ws_id, title)` | Set/update LLM-generated title |
|
||||
| `update_workstream_state(ws_id, state)` | Update workstream state and bump timestamp |
|
||||
| `update_workstream_name(ws_id, name)` | Update workstream display name |
|
||||
| `list_workstreams(node_id, limit, *, parent_ws_id, kind, user_id)` | List workstreams, optionally filtered by node, parent, kind, or owning user |
|
||||
| `list_workstreams(node_id, limit)` | List workstreams, optionally by node |
|
||||
| `kv_get(key)` / `kv_set(key, value)` / `kv_delete(key)` | Generic key-value store (backs memories table) |
|
||||
| `kv_list()` / `kv_search(query)` | List or search key-value pairs |
|
||||
| `search_history(query, limit)` | Full-text search (FTS5 on SQLite, tsvector on PostgreSQL) |
|
||||
@@ -1100,8 +1086,8 @@ Three hierarchical scopes control endpoint access:
|
||||
| Scope | Grants | Endpoints |
|
||||
|-------|--------|-----------|
|
||||
| `read` | SSE streams, workstream listing, history | GET endpoints |
|
||||
| `write` | `read` + send, command, workstream create/close | POST to `/api/workstreams/{ws_id}/send`, `/api/command`, etc. |
|
||||
| `approve` | `write` + tool approval, admin operations | POST to `/api/workstreams/{ws_id}/approve`, `/api/admin/*` |
|
||||
| `write` | `read` + send, command, workstream create/close | POST to `/api/send`, `/api/command`, etc. |
|
||||
| `approve` | `write` + tool approval, admin operations | POST to `/api/approve`, `/api/admin/*` |
|
||||
|
||||
### Middleware Flow
|
||||
|
||||
@@ -1125,9 +1111,8 @@ Three hierarchical scopes control endpoint access:
|
||||
- **Console** is the auth management hub — it hosts the admin endpoints for
|
||||
creating users, issuing API tokens, and managing channel mappings. User
|
||||
records and token hashes live in the shared storage backend. The console
|
||||
dashboard includes an **admin panel** (18 tabs) for managing
|
||||
credentials, governance, MCP servers, models, node metadata, and runtime
|
||||
settings through the browser.
|
||||
dashboard includes an **admin panel** (14 tabs) for managing
|
||||
credentials, governance, MCP servers, and runtime settings through the browser.
|
||||
- **Server** is a JWT validator only — it validates tokens on each request but
|
||||
never creates users or tokens. Both processes share the same `jwt_secret`
|
||||
(via `TURNSTONE_JWT_SECRET` env var or `[auth].jwt_secret` config).
|
||||
@@ -1198,12 +1183,12 @@ stderr so it does not interfere with readline. Tool execution may use a
|
||||
Starlette ASGI app (served by uvicorn)
|
||||
|
|
||||
+-- Async request handlers (all under /v1/ prefix)
|
||||
| POST /v1/api/workstreams/{ws_id}/send -> starts worker thread per workstream
|
||||
| POST /v1/api/workstreams/{ws_id}/approve -> unblocks WebUI._approval_event
|
||||
| POST /v1/api/plan -> unblocks WebUI._plan_event
|
||||
| POST /v1/api/workstreams/new -> creates workstream + worker
|
||||
| GET /v1/api/workstreams/{ws_id}/events -> SSE via EventSourceResponse (per workstream)
|
||||
| GET /v1/api/events/global -> SSE via EventSourceResponse (fan-out)
|
||||
| POST /v1/api/send -> starts worker thread per workstream
|
||||
| POST /v1/api/approve -> unblocks WebUI._approval_event
|
||||
| POST /v1/api/plan -> unblocks WebUI._plan_event
|
||||
| POST /v1/api/workstreams/new -> creates workstream + worker
|
||||
| GET /v1/api/events -> SSE via EventSourceResponse (per workstream)
|
||||
| GET /v1/api/events/global -> SSE via EventSourceResponse (fan-out)
|
||||
|
|
||||
+-- ASGI middleware stack
|
||||
| MetricsMiddleware -> CORSMiddleware -> AuthMiddleware -> RateLimitMiddleware
|
||||
@@ -1277,10 +1262,10 @@ Monitoring (2 daemon threads) Control + Proxy (async Starlette)
|
||||
| SSE manager | | GET /node/{node_id}/ |
|
||||
| asyncio loop | | → httpx.AsyncClient |
|
||||
| 1 task per node | | proxy to server_url |
|
||||
| /events/global | | GET /node/{id}/v1/api/workstreams/{ws_id}/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/workstreams/{ws_id}/send |
|
||||
| → forwarded to server |
|
||||
| /events/global | | GET /node/{id}/v1/api/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/send |
|
||||
| → forwarded to server |
|
||||
+----------------------------+
|
||||
```
|
||||
|
||||
@@ -1362,10 +1347,9 @@ setup, auth headers, `_request()` (REST) and `_stream_sse()` (SSE). Sync
|
||||
clients delegate through `_SyncRunner` which maintains a persistent background
|
||||
event loop on a daemon thread.
|
||||
|
||||
**Event types**: 38 standalone dataclasses in `events.py` with a type-registry
|
||||
dispatch (`from_json()` on each event). Events are decoupled from server
|
||||
internals — the SDK parses SSE frames directly from the `/v1/api/events`
|
||||
streams.
|
||||
**Event types**: 27 standalone dataclasses in `events.py` with a type-registry
|
||||
pattern matching `OutboundEvent.from_json()` from `mq/protocol.py`. Events are
|
||||
decoupled from server internals.
|
||||
|
||||
**TypeScript SDK**: `sdk/typescript/` — separate npm package with the same API
|
||||
surface. Zero browser dependencies, SSE via `fetch` + `ReadableStream` parsing.
|
||||
@@ -1387,8 +1371,7 @@ with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
> See also: [Channel Integrations guide](channels.md)
|
||||
|
||||
The `turnstone-channel` gateway connects external messaging platforms
|
||||
(Discord and Slack today, with an adapter protocol for future platforms) to
|
||||
the turnstone cluster via HTTP. Each
|
||||
(Discord, Slack, Teams) to the turnstone cluster via HTTP. Each
|
||||
platform adapter implements the `ChannelAdapter` protocol and translates
|
||||
between platform-native events and turnstone server API calls.
|
||||
|
||||
@@ -1401,7 +1384,7 @@ workstream is reactivated, the router uses atomic resume via the
|
||||
the old workstream's conversation during creation in a single HTTP
|
||||
request, eliminating ordering fragility.
|
||||
|
||||
Discord and Slack adapters ship today. See [channels.md](channels.md) for
|
||||
Discord ships as the first adapter. See [channels.md](channels.md) for
|
||||
setup instructions, configuration reference, and the adapter development
|
||||
guide.
|
||||
|
||||
@@ -1422,11 +1405,11 @@ retries up to 3 times with backoff, re-querying the service registry on
|
||||
each attempt. See [Notification Flow diagram](diagrams/png/17-notify-flow.png).
|
||||
|
||||
**Bidirectional replies:** When a user replies to a notification DM, the
|
||||
channel adapter (Discord or Slack) looks up the originating `ws_id` from the
|
||||
tracked message ID, verifies the replying user matches the notification
|
||||
recipient, and routes the reply to the workstream via `router.send_message()`.
|
||||
The workstream's response is forwarded back to the DM via a temporary entry
|
||||
in `_notify_reply_channels`. On `TurnCompleteEvent`, the response message is
|
||||
Discord bot looks up the originating `ws_id` from the tracked message ID,
|
||||
verifies the replying user matches the notification recipient, and routes
|
||||
the reply to the workstream via `router.send_message()`. The workstream's
|
||||
response is forwarded back to the DM via a temporary entry in
|
||||
`_notify_reply_channels`. On `TurnCompleteEvent`, the response message is
|
||||
itself tracked for further replies, enabling multi-turn DM conversations
|
||||
without requiring the user to open the web UI. Tracking entries are capped
|
||||
at 100 (FIFO eviction) and cleaned up on workstream close.
|
||||
@@ -1459,10 +1442,11 @@ and workstreams record which skill and version spawned them. Token budget
|
||||
enforcement tracks consumption in `session.send()` with 80% warning and
|
||||
100% approval gate via the `__budget_override__` synthetic tool name.
|
||||
|
||||
The console admin panel exposes these capabilities as 18 permission-gated
|
||||
tabs: Users, API Tokens, Channels, Schedules, Watches, Roles, Policies,
|
||||
Prompts, Judge, Skills, MCP Servers, Usage, Audit, Memories, Models, Nodes,
|
||||
Settings, and TLS.
|
||||
The console admin panel adds 5 governance tabs (Roles, Policies, Skills,
|
||||
Usage, Audit), a Memories tab, a Settings tab (form-based editor for all
|
||||
ConfigStore settings), and an MCP Servers tab (database-backed server
|
||||
definitions with live connection status and cluster-wide reload) for a
|
||||
total of 13 tabs, all permission-gated.
|
||||
Both Python and TypeScript SDKs expose governance methods on the console
|
||||
client.
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5d500479d3be2363d4f594042a27e2ef5e2974750f580f6c4037a1fe85868ed9
|
||||
size 251904
|
||||
oid sha256:75c1832b6079e8628f4bbf4ce98d37880c4de133636b7555e3869990b046ddc6
|
||||
size 567704
|
||||
|
||||
@@ -1,237 +0,0 @@
|
||||
# Bulk endpoint shape contract
|
||||
|
||||
Turnstone exposes several endpoints and tool calls that take multiple
|
||||
ids and return a per-id outcome. Over the last few phases two
|
||||
**distinct** response shapes have settled, one per semantic category.
|
||||
This doc codifies both so a future endpoint author can pick the right
|
||||
shape by semantics instead of by coin-flip.
|
||||
|
||||
Existing bulk endpoints at time of writing:
|
||||
|
||||
| Endpoint / tool | Category | Response shape |
|
||||
|---------------------------------------------------------|--------------------------|------------------------------------------|
|
||||
| `GET /v1/api/cluster/ws/live?ids=a,b,c` | bulk read | `{results, denied, truncated}` |
|
||||
| model tool `spawn_batch` | bulk create (per-item) | `{results, denied}` |
|
||||
| `POST /v1/api/workstreams/{ws_id}/stop_cascade` | cascade mutation | `{cancelled, failed, skipped}` |
|
||||
| `POST /v1/api/workstreams/{ws_id}/close_all_children` | cascade mutation | `{closed, failed, skipped}` |
|
||||
|
||||
---
|
||||
|
||||
## Why two shapes
|
||||
|
||||
The ask-to-outcome mapping is fundamentally different between the
|
||||
two categories, and a one-size-fits-all envelope ends up papering
|
||||
over distinctions the caller genuinely needs to branch on.
|
||||
|
||||
**Bulk read / bulk create-with-payload.** Each input id (or batch
|
||||
index) carries a *request-side* concept — "give me the live block
|
||||
for this ws_id" or "spawn a child with this spec" — and each
|
||||
successful output carries a *payload* — the live block, or the new
|
||||
workstream's identifying triple. The interesting distinction on
|
||||
failure is *ownership / validation* (caller can't see that id, spec
|
||||
was malformed) — independent of the storage state.
|
||||
|
||||
**Cascade mutation.** The action is uniform across every id (cancel
|
||||
this subtree, close this child). The interesting distinctions on
|
||||
outcome are *did it reach the terminal state?* (succeeded / already
|
||||
was there / the dispatch itself failed) — driven by the storage
|
||||
state plus transport reliability, not by the caller's input.
|
||||
|
||||
Trying to unify these forces either:
|
||||
|
||||
- a stateless `denied` bucket that has to carry "already gone"
|
||||
*and* "you don't have permission" *and* "transport failed" with a
|
||||
separate reason string — reviewers end up string-matching to branch.
|
||||
- or a per-item-payload map for cascade mutations where every
|
||||
successful value is the same sentinel — carrier with no payload.
|
||||
|
||||
So: two shapes, one per category. The rest of this doc spells out
|
||||
each.
|
||||
|
||||
---
|
||||
|
||||
## Shape A — bulk read / bulk create-with-payload
|
||||
|
||||
```json
|
||||
{
|
||||
"results": { "<key>": <value-or-null>, ... },
|
||||
"denied": [ "<key>", ... ],
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
**`results`** is a key-indexed map of the positive-path payload.
|
||||
The key is the input id for read endpoints (`cluster/ws/live` uses
|
||||
the ws_id), or the input-array index (stringified) for create
|
||||
endpoints that want ordering preserved (`spawn_batch` uses `"0"`,
|
||||
`"1"`, ...). The value is whatever the endpoint produces per
|
||||
success — a live block, a `{ws_id, name, node_id, status}` triple,
|
||||
etc. A `null` value (read endpoints only) means "the id existed and
|
||||
you own it, but the live block wasn't available" — distinct from
|
||||
"denied".
|
||||
|
||||
**`denied`** is the negative-path list. For read endpoints it's a
|
||||
flat list of ids (preserves input order so callers can re-zip
|
||||
against their input). For create endpoints with per-item payloads
|
||||
it's a list of `{idx, reason}` objects (`spawn_batch`'s validation
|
||||
and spawn-error rows; also the operator-reject surface when per-item
|
||||
selective-deny ships). Include every reason that's *not* the
|
||||
positive path — authz, ownership, validation, already-consumed,
|
||||
spawn failure — so callers don't branch on status codes.
|
||||
|
||||
**`truncated`** is a boolean set to `true` when the server's
|
||||
per-endpoint input cap was exceeded and the tail was dropped. The
|
||||
endpoint docs each spell out the cap (50 for `cluster/ws/live`).
|
||||
`spawn_batch` hard-errors on overflow instead of silently
|
||||
truncating — it omits the field entirely rather than carry a
|
||||
permanently-false flag.
|
||||
|
||||
### Example — `cluster/ws/live`
|
||||
|
||||
```http
|
||||
GET /v1/api/cluster/ws/live?ids=a1b2,c3d4,nonexistent,foreign HTTP/1.1
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"results": {
|
||||
"a1b2": {"state": "running", "tokens": 12843, "activity": "..."},
|
||||
"c3d4": null
|
||||
},
|
||||
"denied": ["nonexistent", "foreign"],
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
Callers that need ordered output zip their original id list against
|
||||
this map; ids in `denied` drop out of the zip cleanly. A live-block
|
||||
`null` doesn't route to `denied` — the row exists and the caller
|
||||
owns it; the node is just currently unreachable.
|
||||
|
||||
### Example — `spawn_batch`
|
||||
|
||||
```json
|
||||
{
|
||||
"results": {
|
||||
"0": {"ws_id": "d4e5f6...", "name": "csrf-audit", "node_id": "gpu-3"},
|
||||
"2": {"ws_id": "f1a2b3...", "name": "xss-audit", "node_id": "gpu-1"}
|
||||
},
|
||||
"denied": [
|
||||
{"idx": 1, "reason": "skill not found: nonexistent-skill"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Indexes are stringified to keep the envelope JSON-safe and
|
||||
consistently-typed across the read and create cases.
|
||||
|
||||
---
|
||||
|
||||
## Shape B — cascade mutation
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"<bucket>": [ "<ws_id>", ... ],
|
||||
"failed": [ "<ws_id>", ... ],
|
||||
"skipped": [ "<ws_id>", ... ]
|
||||
}
|
||||
```
|
||||
|
||||
Where `<bucket>` is the endpoint-specific name for "succeeded" —
|
||||
`cancelled` for `stop_cascade`, `closed` for `close_all_children`.
|
||||
The three buckets partition the input set exactly once:
|
||||
|
||||
| Bucket | Meaning |
|
||||
|---------------|-------------------------------------------------------------------------------|
|
||||
| `<bucket>` | Action dispatch accepted; target reached the intended terminal state. |
|
||||
| `failed` | Dispatch returned a non-404 error (transport issue, upstream 5xx, exception). |
|
||||
| `skipped` | Upstream 404 — stale registry entry, row already deleted, or peer gone. |
|
||||
|
||||
The split between `failed` and `skipped` is load-bearing. `failed`
|
||||
is actionable — the operator may want to retry, or the cascade may
|
||||
be partial. `skipped` is pre-resolved — the target is already in
|
||||
the terminal state the cascade was aiming at, so it's neither a
|
||||
win to report nor a fault to fix.
|
||||
|
||||
### Example — `stop_cascade`
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"cancelled": ["child-1", "child-3"],
|
||||
"failed": [],
|
||||
"skipped": ["child-2"]
|
||||
}
|
||||
```
|
||||
|
||||
A subsequent retry would target only `failed` ids, not `skipped`
|
||||
ones — the latter are already done.
|
||||
|
||||
### Example — `close_all_children`
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"closed": ["child-1", "child-3"],
|
||||
"failed": ["child-2"],
|
||||
"skipped": []
|
||||
}
|
||||
```
|
||||
|
||||
Same partition, different success-bucket name. When `coord_client`
|
||||
is unavailable (session loaded but no HTTP client attached — a
|
||||
construction bug) every id goes to `failed` so the operator notices
|
||||
rather than getting a silent all-skipped response.
|
||||
|
||||
---
|
||||
|
||||
## Guidance for future bulk endpoints
|
||||
|
||||
1. **Pick by semantics, not by "what shape is nearby."**
|
||||
- Mutation that's uniform across ids + terminal-state outcome? →
|
||||
**Shape B** (cascade mutation).
|
||||
- Read or create where the input id carries payload, or where the
|
||||
denial axis is independent of storage state? → **Shape A**
|
||||
(bulk read / bulk create-with-payload).
|
||||
|
||||
2. **Cap the input.** Both shapes assume a bounded input — the
|
||||
server rejects or silently truncates past the cap. Document the
|
||||
cap in the endpoint's OpenAPI description. Shape A uses
|
||||
`truncated: true` on quiet truncation; Shape B hard-errors on
|
||||
overflow.
|
||||
|
||||
3. **Match existing bucket names for the same semantic.** Use
|
||||
`failed` and `skipped` verbatim in Shape B — the per-endpoint
|
||||
success bucket is the only slot that varies. Use `results` and
|
||||
`denied` verbatim in Shape A; the per-endpoint `<key>` /
|
||||
`<value>` types vary.
|
||||
|
||||
4. **Audit the verbose shape.** Both endpoints emit a corresponding
|
||||
audit event with the full before/after bucket lists — the SSE
|
||||
stream and the in-process response give live feedback, but a
|
||||
postmortem operator will read the audit row. Use
|
||||
`_emit_coord_audit` (coordinator-scoped) or `record_audit`
|
||||
directly; don't inline.
|
||||
|
||||
5. **Don't mix shapes within one endpoint.** If a bulk endpoint
|
||||
wants both partial-success creation AND per-item failure reasons
|
||||
(like `spawn_batch` with its `{idx, reason}` denial rows), that's
|
||||
Shape A with a richer denial element — not a blend with Shape B.
|
||||
|
||||
---
|
||||
|
||||
## History
|
||||
|
||||
- **Phase 6** shipped `cluster/ws/live` as the first Shape A endpoint
|
||||
(`{results, denied, truncated}`).
|
||||
- **Phase 7** shipped `stop_cascade` as the first Shape B endpoint
|
||||
(`{cancelled, failed, skipped}`).
|
||||
- **Phase 8 PR A** shipped `spawn_batch` (Shape A, keyed by idx) and
|
||||
`close_all_children` (Shape B, twin of `stop_cascade`), which
|
||||
crystallised the two-shape-per-semantic-category policy codified
|
||||
here.
|
||||
|
||||
Before adding a third shape, read this doc and argue for why the
|
||||
new surface doesn't fit either A or B. Two idioms in the cluster
|
||||
API is a finite operator tax; three is one too many.
|
||||
+21
-95
@@ -7,35 +7,31 @@ platform-native events (messages, button clicks, slash commands) into
|
||||
turnstone API calls, and renders workstream output back into the
|
||||
platform's UI.
|
||||
|
||||
Discord and Slack adapters ship today. The adapter protocol is designed
|
||||
so new platforms can be added with only a new package under
|
||||
`turnstone/channels/<platform>/`.
|
||||
Discord ships as the first adapter. The adapter protocol is designed for
|
||||
future Slack and Teams integrations.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Discord Gateway Slack (Socket Mode WebSocket)
|
||||
\ /
|
||||
v v
|
||||
turnstone-channel (one or more adapters)
|
||||
|
|
||||
v
|
||||
turnstone-server (direct HTTP)
|
||||
or
|
||||
turnstone-console (routing proxy, multi-node)
|
||||
Discord Gateway
|
||||
|
|
||||
v
|
||||
turnstone-channel (Discord adapter)
|
||||
|
|
||||
v
|
||||
turnstone-server (direct HTTP)
|
||||
or
|
||||
turnstone-console (routing proxy, multi-node)
|
||||
```
|
||||
|
||||
A single `turnstone-channel` process can run multiple adapters
|
||||
simultaneously (e.g. Discord + Slack) — pass the tokens for each
|
||||
platform you want to enable.
|
||||
|
||||
Key components:
|
||||
|
||||
- **ChannelAdapter protocol** (`turnstone/channels/_protocol.py`) — generic
|
||||
interface for any messaging platform. Defines `start()`, `stop()`,
|
||||
`send()`, and `send_notification()`.
|
||||
`send()`, `send_notification()`, `edit_message()`,
|
||||
`send_approval_request()`, `send_plan_review()`, and `create_thread()`.
|
||||
- **ChannelRouter** (`turnstone/channels/_routing.py`) — maps
|
||||
channel/thread IDs to turnstone workstream IDs. Handles workstream
|
||||
creation via HTTP, stale route detection, and user identity resolution.
|
||||
@@ -124,68 +120,6 @@ An admin can also force-link or unlink users via the console admin panel
|
||||
|
||||
---
|
||||
|
||||
## Slack Setup
|
||||
|
||||
Slack uses **Socket Mode**, so no public URL or API Gateway is required — Slack
|
||||
connects outbound to the bot via a WebSocket. Install with:
|
||||
|
||||
```bash
|
||||
pip install 'turnstone[slack]'
|
||||
```
|
||||
|
||||
### 1. Create a Slack App
|
||||
|
||||
1. Go to https://api.slack.com/apps and click **Create New App**
|
||||
2. Under **Settings > Socket Mode**, enable Socket Mode. This generates an
|
||||
**App-Level Token** (prefix `xapp-`) — copy it.
|
||||
3. Under **OAuth & Permissions**, add these **Bot Token Scopes**:
|
||||
`chat:write`, `chat:write.public`, `channels:history`, `im:history`,
|
||||
`groups:history`, `mpim:history`, `reactions:write`, `commands`
|
||||
4. Under **Event Subscriptions** (Socket Mode delivers events), subscribe
|
||||
to bot events: `message.channels`, `message.im`, `message.groups`
|
||||
5. Under **Slash Commands**, create a command (default `/turnstone`)
|
||||
6. Install the app to your workspace to generate the **Bot User OAuth
|
||||
Token** (prefix `xoxb-`).
|
||||
|
||||
### 2. Configure Turnstone
|
||||
|
||||
**Environment variables** (recommended for Docker):
|
||||
|
||||
```bash
|
||||
TURNSTONE_SLACK_TOKEN=xoxb-... # Bot User OAuth Token
|
||||
TURNSTONE_SLACK_APP_TOKEN=xapp-... # App-Level Token (Socket Mode)
|
||||
TURNSTONE_SLACK_CHANNELS= # optional, comma-separated channel IDs
|
||||
TURNSTONE_SLACK_SLASH_COMMAND=/turnstone
|
||||
```
|
||||
|
||||
**CLI flags** (bare-metal):
|
||||
|
||||
```bash
|
||||
turnstone-channel \
|
||||
--slack-token "xoxb-..." \
|
||||
--slack-app-token "xapp-..." \
|
||||
--slack-slash-command /turnstone \
|
||||
--server-url http://localhost:8080
|
||||
```
|
||||
|
||||
The Slack and Discord adapters can be enabled together — pass tokens for
|
||||
both and the gateway hosts both adapters in one process.
|
||||
|
||||
### 3. Usage
|
||||
|
||||
- **DM the bot**: messages sent directly to the bot create a workstream
|
||||
scoped to that DM; the slash command is not required.
|
||||
- **Slash command**: `/turnstone <message>` in any channel the bot can
|
||||
see starts a per-user channel session.
|
||||
- Tool approvals render as Slack **Block Kit** buttons; only the user
|
||||
who owns the workstream can approve/reject.
|
||||
- Plan reviews render as a modal with approve / request-changes actions.
|
||||
- Notifications and reply routing work identically to Discord.
|
||||
- Session recovery: persisted channel routes are re-subscribed when the
|
||||
bot restarts, so existing Slack conversations keep flowing.
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
### Conversations
|
||||
@@ -250,13 +184,9 @@ Plan review requests are displayed as a blue embed with:
|
||||
|
||||
| CLI Flag | Env Var | Default | Description |
|
||||
|----------|---------|---------|-------------|
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord) |
|
||||
| `--discord-token` | `TURNSTONE_DISCORD_TOKEN` | — | Bot token (required to enable Discord) |
|
||||
| `--discord-guild` | — | `0` (all guilds) | Restrict to a single Discord guild |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated Discord channel IDs to allow |
|
||||
| `--slack-token` | `TURNSTONE_SLACK_TOKEN` | — | Slack Bot User OAuth token (`xoxb-…`, required to enable Slack) |
|
||||
| `--slack-app-token` | `TURNSTONE_SLACK_APP_TOKEN` | — | Slack App-Level token (`xapp-…`, required with `--slack-token`) |
|
||||
| `--slack-channels` | `TURNSTONE_SLACK_CHANNELS` | empty (all) | Comma-separated Slack channel IDs to allow |
|
||||
| `--slack-slash-command` | `TURNSTONE_SLACK_SLASH_COMMAND` | `/turnstone` | Slash command name registered in the Slack app |
|
||||
| `--discord-channels` | — | empty (all) | Comma-separated channel IDs to allow |
|
||||
| `--server-url` | `TURNSTONE_SERVER_URL` | `http://localhost:8080` | Server URL (single-node) |
|
||||
| `--console-url` | `TURNSTONE_CONSOLE_URL` | — | Console URL (multi-node routing proxy) |
|
||||
| `--model` | — | server default | Default model for new workstreams |
|
||||
@@ -266,9 +196,6 @@ Plan review requests are displayed as a blue embed with:
|
||||
| `--log-level` | `TURNSTONE_LOG_LEVEL` | `INFO` | Log level |
|
||||
| `--log-format` | `TURNSTONE_LOG_FORMAT` | `auto` | Log format (`auto`/`json`/`text`) |
|
||||
|
||||
At least one of `--discord-token` or `--slack-token` must be supplied.
|
||||
Passing both starts both adapters in the same process.
|
||||
|
||||
---
|
||||
|
||||
## User Identity
|
||||
@@ -322,8 +249,8 @@ waiting for them to check in.
|
||||
Two modes:
|
||||
|
||||
- **Username** — provide a turnstone `username`. The gateway resolves
|
||||
it via the `channel_users` table and sends to every linked platform
|
||||
the user has (e.g. Discord + Slack).
|
||||
it via the `channel_users` table and sends to all linked channels
|
||||
(e.g. Discord + future Slack).
|
||||
- **Direct** — provide `channel_type` + `channel_id` to target a
|
||||
specific platform channel or user DM.
|
||||
|
||||
@@ -424,6 +351,10 @@ class ChannelAdapter(Protocol):
|
||||
async def stop(self) -> None: ...
|
||||
async def send(self, channel_id: str, content: str) -> str: ...
|
||||
async def send_notification(self, channel_id: str, content: str, ws_id: str) -> str: ...
|
||||
async def edit_message(self, channel_id: str, message_id: str, content: str) -> None: ...
|
||||
async def send_approval_request(self, channel_id: str, ws_id: str, correlation_id: str, items: list[dict]) -> None: ...
|
||||
async def send_plan_review(self, channel_id: str, ws_id: str, correlation_id: str, content: str) -> None: ...
|
||||
async def create_thread(self, parent_channel_id: str, name: str, message_id: str = "") -> str: ...
|
||||
```
|
||||
|
||||
`send_notification()` is like `send()` but associates the outgoing
|
||||
@@ -431,11 +362,6 @@ message with a `ws_id` so that user replies can be routed back to the
|
||||
originating workstream. Adapters must track the mapping from outgoing
|
||||
message ID to `(ws_id, target_user_id)` and handle DM replies.
|
||||
|
||||
Platform-specific concerns — approval prompts, plan reviews, message
|
||||
edits, thread creation — live inside the adapter implementation and are
|
||||
not part of the protocol surface. Each adapter drives those via its
|
||||
own `_on_ws_event` dispatcher using SDK-native APIs.
|
||||
|
||||
To add a new platform:
|
||||
|
||||
1. Create `turnstone/channels/<platform>/` package
|
||||
|
||||
+6
-15
@@ -334,7 +334,7 @@ The console reverse-proxies each node's server UI at `/node/{node_id}/`. This al
|
||||
|
||||
### URL Rewriting
|
||||
|
||||
The server UI uses root-relative URLs (`/v1/api/workstreams/{ws_id}/send`, `/static/app.js`, `/shared/base.css`, etc.). Since `<base>` tags cannot rewrite root-relative URLs, the console uses a JS shim approach:
|
||||
The server UI uses root-relative URLs (`/v1/api/send`, `/static/app.js`, `/shared/base.css`, etc.). Since `<base>` tags cannot rewrite root-relative URLs, the console uses a JS shim approach:
|
||||
|
||||
1. **HTML rewriting** — when serving `index.html`, replaces `href=` and `src=` references to both `/static/` and `/shared/` with the proxy prefix (`/node/{node_id}/static/` and `/node/{node_id}/shared/` respectively).
|
||||
|
||||
@@ -344,7 +344,7 @@ The server UI uses root-relative URLs (`/v1/api/workstreams/{ws_id}/send`, `/sta
|
||||
|
||||
### SSE Proxy
|
||||
|
||||
SSE streams (`/v1/api/workstreams/{ws_id}/events`, `/v1/api/events/global`) are proxied as raw byte passthrough — the console opens an `httpx.AsyncClient.stream()` to the upstream server (with `read=None` and `pool=None` timeouts since SSE connections are long-lived) and relays every byte via `StreamingResponse`. This preserves server-side ping comments, event framing, and keepalives verbatim without parsing or re-encoding.
|
||||
SSE streams (`/v1/api/events`, `/v1/api/events/global`) are proxied as raw byte passthrough — the console opens an `httpx.AsyncClient.stream()` to the upstream server (with `read=None` and `pool=None` timeouts since SSE connections are long-lived) and relays every byte via `StreamingResponse`. This preserves server-side ping comments, event framing, and keepalives verbatim without parsing or re-encoding.
|
||||
|
||||
### Authentication
|
||||
|
||||
@@ -396,19 +396,10 @@ The browser maintains a local `clusterState` object that mirrors the cluster sna
|
||||
|
||||
Accessed via the "admin" button in the header (visible when authenticated
|
||||
with `approve` scope). Provides user, API token, channel link, MCP server,
|
||||
and skill management with 18 tabs (Users, API Tokens, Channels, Schedules,
|
||||
Watches, Roles, Policies, Prompts, Judge, Skills, MCP Servers, Usage,
|
||||
Audit, Memories, Models, Nodes, Settings, TLS). See also
|
||||
[Governance](governance.md) for the Roles, Policies, Skills, Usage, and
|
||||
Audit tabs, and [Settings](settings.md) for the database-backed
|
||||
configuration editor.
|
||||
|
||||
The **Channels** tab links users to either a Discord or Slack account
|
||||
via a per-row channel-type selector. The **Models** tab is a CRUD
|
||||
editor for `model_definitions`, the **Nodes** tab edits per-node
|
||||
metadata, and the **TLS** tab manages CA and leaf certificates for the
|
||||
internal mTLS fabric. The **Settings** tab edits ConfigStore values
|
||||
live; edits apply without restart.
|
||||
and skill management with 13 tabs (see also
|
||||
[Governance](governance.md) for
|
||||
the Roles, Policies, Skills, Usage, and Audit tabs, and
|
||||
[Settings](settings.md) for the database-backed configuration editor):
|
||||
|
||||
**Users tab:**
|
||||
|
||||
|
||||
@@ -1,387 +0,0 @@
|
||||
# Coordinator API tour
|
||||
|
||||
Turnstone's **coordinator workstream** is a session hosted on the
|
||||
console whose job is to orchestrate other workstreams. It runs an LLM
|
||||
that can spawn child workstreams on any node, watch their progress,
|
||||
wait for them to finish, steer them mid-flight, and tear them down.
|
||||
This doc walks the full lifecycle — one request, one response, and the
|
||||
relevant SSE events at each step.
|
||||
|
||||
Aimed at integrators driving a coordinator from a custom UI or SDK
|
||||
without reverse-engineering the built-in console page. The shapes
|
||||
here match the live OpenAPI spec served at `/openapi.json` and
|
||||
rendered at `/docs` on every `turnstone-console` process. Every
|
||||
step references the operation id from that spec so doc updates track
|
||||
schema changes.
|
||||
|
||||
> **Auth throughout.** Every endpoint below sits behind bearer-token
|
||||
> auth and the `admin.coordinator` permission. A session-scoped JWT
|
||||
> is minted per login (see [docs/oidc.md](oidc.md) / [docs/security.md](security.md));
|
||||
> a service token may call the read paths but destructive governance
|
||||
> paths (`/restrict`, `/stop_cascade`, `/close_all_children`) require
|
||||
> the explicit `admin.coordinator` grant — a service-token owner
|
||||
> match isn't enough.
|
||||
|
||||
---
|
||||
|
||||
## The 9 steps
|
||||
|
||||
> **URL convergence (1.5.0).** Pre-1.5 coord-only endpoints lived
|
||||
> under `/v1/api/coordinator/...`. The Stage 2 verb-shape lift
|
||||
> consolidated coord and interactive onto the unified
|
||||
> `/v1/api/workstreams/{ws_id}/<verb>` tree; coord still distinguishes
|
||||
> itself via the `kind=coordinator` row classifier rather than a
|
||||
> separate URL space. The endpoints below reflect the post-lift
|
||||
> surface served by `turnstone-console`.
|
||||
|
||||
| # | Action | Operation |
|
||||
|---|------------------------------|-------------------------------------------------------------|
|
||||
| 1 | Create | `POST /v1/api/workstreams/new` |
|
||||
| 2 | Subscribe to events | `GET /v1/api/workstreams/{ws_id}/events` (SSE) |
|
||||
| 3 | Send a user message | `POST /v1/api/workstreams/{ws_id}/send` |
|
||||
| 4 | Inspect children | `GET /v1/api/workstreams/{ws_id}/children` |
|
||||
| 5 | Inspect one workstream | `GET /v1/api/cluster/ws/{ws_id}/detail` |
|
||||
| 6 | Wait for fan-out | model-side tool `wait_for_workstream` |
|
||||
| 7 | Govern | `POST /v1/api/workstreams/{ws_id}/trust` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/restrict` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/stop_cascade` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/close_all_children` |
|
||||
| 8 | Approve / cancel | `POST /v1/api/workstreams/{ws_id}/approve` |
|
||||
| | | `POST /v1/api/workstreams/{ws_id}/cancel` |
|
||||
| 9 | Close | `POST /v1/api/workstreams/{ws_id}/close` |
|
||||
|
||||
Refer to `/openapi.json` (Swagger UI at `/docs`) on any
|
||||
`turnstone-console` process for the authoritative operation ids and
|
||||
schemas. Coordinator-only verbs (`/children`, `/trust`, `/restrict`,
|
||||
`/stop_cascade`, `/close_all_children`) 404 against `kind=interactive`
|
||||
rows; the shared verbs (`/send`, `/approve`, `/cancel`, `/events`,
|
||||
`/history`, `/open`, `/close`, etc.) work on both kinds.
|
||||
|
||||
---
|
||||
|
||||
## 1. Create a coordinator
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/new
|
||||
Content-Type: application/json
|
||||
Authorization: Bearer <token>
|
||||
|
||||
{
|
||||
"name": "release-coord",
|
||||
"skill": "engineer-orchestrator",
|
||||
"initial_message": "audit /auth for CSRF handling across all active routes"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 201 Created
|
||||
Content-Type: application/json
|
||||
|
||||
```
|
||||
|
||||
All three body fields are optional — an empty body still creates a
|
||||
coordinator with an auto-generated name and no initial message.
|
||||
Returns **503** with a remediation message when the cluster isn't
|
||||
configured with a coordinator model; see
|
||||
[`coordinator.model_alias`](settings.md) to set one.
|
||||
|
||||
**SSE implication:** the `ws_created` event fires on the cluster-wide
|
||||
stream (`/v1/api/cluster/events`) once the row is committed. Per-ws
|
||||
subscribers (step 2) see the session warm up as token traffic starts.
|
||||
|
||||
---
|
||||
|
||||
## 2. Subscribe to the per-coordinator event stream
|
||||
|
||||
```http
|
||||
GET /v1/api/workstreams/{ws_id}/events HTTP/1.1
|
||||
Accept: text/event-stream
|
||||
Authorization: Bearer <token>
|
||||
```
|
||||
|
||||
One persistent SSE connection per browser tab / SDK caller — the
|
||||
console fans each event out to every listener queue (cap 500 events
|
||||
per queue, put_nowait drop on overflow). Events come in flat JSON
|
||||
with a `type` field. The recurring shapes a UI has to handle:
|
||||
|
||||
| `type` | Emitted when | Payload highlights |
|
||||
|---------------------|--------------------------------------------------------------------------------------------|--------------------|
|
||||
| `thinking_start` / `thinking_stop` | Model has entered / exited a reasoning block | — |
|
||||
| `reasoning` | Reasoning-token stream chunk (when the model exposes it) | `text` |
|
||||
| `content` | Assistant-content stream chunk | `text` |
|
||||
| `stream_end` | End of a single provider stream | — |
|
||||
| `tool_result` | A tool call completed (success or error) | `call_id`, `name`, `output`, `is_error?` |
|
||||
| `tool_output_chunk` | Streaming tool output (e.g. long bash command) | `call_id`, `chunk` |
|
||||
| `approve_request` | One or more tool calls need operator approval | `items: [{call_id, header, preview, func_name, approval_label, needs_approval}]` |
|
||||
| `approval_resolved` | Operator answered the approval prompt | `approved`, `feedback` |
|
||||
| `state_change` | Worker-thread state transition | `state` ∈ `running`, `thinking`, `attention`, `idle`, `error` |
|
||||
| `status` | Token usage + context-window snapshot (fires on every streaming tick) | `prompt_tokens`, `completion_tokens`, `total_tokens`, `context_window`, `pct`, `effort`, `cache_creation_tokens`, `cache_read_tokens` |
|
||||
| `rename` | Session's display name changed | `name` |
|
||||
| `intent_verdict` | Intent judge produced a verdict on a pending tool call | `risk_level`, `recommendation`, `reasons` |
|
||||
| `output_warning` | Output guard flagged a tool result | `call_id`, `risk_level`, `flags` |
|
||||
| `child_ws_created` | A direct child of this coord was just created (fan-out from the cluster bus) | `child_ws_id`, `node_id`, `name`, `parent_ws_id` (`ws_id` in the envelope is always the coord's own id) |
|
||||
| `child_ws_state` | A direct child transitioned state | `child_ws_id`, `state` |
|
||||
| `child_ws_closed` | A direct child closed | `child_ws_id` |
|
||||
| `child_ws_rename` | A direct child's name changed | `child_ws_id`, `name` |
|
||||
| `wait_started` / `wait_progress` / `wait_ended` | `wait_for_workstream` tool lifecycle (see §6) | `call_id`, `ws_ids`, `elapsed`, `results`, `complete` |
|
||||
| `batch_started` / `batch_ended` | `spawn_batch` / `close_all_children` tool lifecycle | `call_id`, `op`, `total`/`succeeded`/`denied`/`closed`/`failed`/`skipped` |
|
||||
| `info` / `error` | Operational messages | `message` |
|
||||
|
||||
**Reconnection contract:** a freshly-opened SSE connection receives
|
||||
the current snapshot of any pending tool approval (`approve_request`
|
||||
is re-sent if unresolved) and any in-flight `wait_*` / `batch_*`
|
||||
indicator — so a tab refresh mid-approval doesn't strand the
|
||||
operator.
|
||||
|
||||
---
|
||||
|
||||
## 3. Send the first user message
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/send
|
||||
Content-Type: application/json
|
||||
|
||||
{"message": "audit /auth for CSRF handling across all active routes"}
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 200 OK
|
||||
{"status": "ok"}
|
||||
```
|
||||
|
||||
The message is queued for the worker thread at its next tool-result
|
||||
seam (so you can send follow-ups mid-conversation without corrupting
|
||||
the in-progress turn). On the SSE stream you'll see `state_change`
|
||||
→ `thinking_start` → streaming `reasoning` / `content` / `tool_result`
|
||||
events, finishing with `state_change → idle` or an
|
||||
`approve_request` when the model invokes a gated tool.
|
||||
|
||||
---
|
||||
|
||||
## 4. Inspect direct children
|
||||
|
||||
```http
|
||||
GET /v1/api/workstreams/{ws_id}/children HTTP/1.1
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{"ws_id": "d4e5f6...", "name": "csrf-audit", "state": "running", "node_id": "gpu-3"},
|
||||
{"ws_id": "e1f2a3...", "name": "xss-audit", "state": "idle", "node_id": "gpu-1"}
|
||||
],
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
The response key is `items`, not `children` — the endpoint shape
|
||||
follows the cluster-wide workstream-list idiom rather than the
|
||||
coordinator `list_workstreams` tool's (which uses `children`).
|
||||
Rows include every state stored for the parent (`running`, `idle`,
|
||||
`closed`, ...); the endpoint does not accept a state query param,
|
||||
so clients should inspect each row's `state` field and filter
|
||||
locally if they want to hide closed/deleted children. Nested
|
||||
coordinator rows are dropped server-side so only interactive
|
||||
descendants appear.
|
||||
|
||||
---
|
||||
|
||||
## 5. Inspect one workstream (storage + live block + tail)
|
||||
|
||||
```http
|
||||
GET /v1/api/cluster/ws/{ws_id}/detail?message_limit=20 HTTP/1.1
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"persisted": { "ws_id": "...", "state": "running", "parent_ws_id": "...", "kind": "interactive", ... },
|
||||
"live": { "state": "thinking", "tokens": 12843, "activity": "...", "pending_approval": null },
|
||||
"tail": [ {"role": "assistant", "content": "...", "tokens": 128}, ... ]
|
||||
}
|
||||
```
|
||||
|
||||
Works for any workstream the caller has `admin.cluster.inspect` on,
|
||||
not just children of a single coordinator — useful for a cluster
|
||||
admin panel watching multiple coordinators at once. `live` is
|
||||
`null` when the owning node is unreachable or has dropped the row
|
||||
from its dashboard cache; callers should degrade gracefully, not
|
||||
treat it as an error.
|
||||
|
||||
For fan-out views, prefer
|
||||
[`GET /v1/api/cluster/ws/live?ids=a,b,c`](bulk-endpoints.md) — it
|
||||
collapses N per-row round-trips into one, returning the live block
|
||||
for every id in a `{results, denied, truncated}` envelope.
|
||||
|
||||
---
|
||||
|
||||
## 6. Wait for fan-out (`wait_for_workstream`)
|
||||
|
||||
`wait_for_workstream` is a **model-side tool**, not an HTTP endpoint
|
||||
— the coordinator's LLM invokes it with a list of child ws_ids, the
|
||||
session's worker thread blocks inside the tool, and a sequence of
|
||||
`wait_started` / `wait_progress` / `wait_ended` SSE events is emitted
|
||||
for the UI to drive a "waiting on N children" indicator.
|
||||
|
||||

|
||||
|
||||
Key properties:
|
||||
|
||||
- **Caps** — up to 32 ws_ids per call, up to 600 seconds per call.
|
||||
A coordinator that needs to wait on more children re-invokes the
|
||||
tool with a fresh timeout.
|
||||
- **Modes** — `mode="any"` returns as soon as one child reaches a
|
||||
real terminal state (`idle` / `error` / `closed` / `deleted`);
|
||||
`mode="all"` waits for every polled child.
|
||||
- **Progress throttling** — the poll loop runs every 500 ms but the
|
||||
SSE emission is diff-on-state-change plus a 5-second heartbeat. A
|
||||
600 s wait generates O(dozens) of progress events, not 1200.
|
||||
- **Denied rows** — an id the caller doesn't own (cross-tenant) or a
|
||||
missing row is reported as a `denied` state in the results dict;
|
||||
`mode="any"` won't satisfy on a pure-denied list (the LLM should
|
||||
treat it as a config error, not a completion).
|
||||
|
||||
Prefer `wait_for_workstream` over polling `inspect_workstream` in a
|
||||
loop — a wait consumes one assistant turn regardless of how long the
|
||||
children take, whereas each `inspect_workstream` poll costs a full
|
||||
turn (plus judge, plus tokens). On a fan-out of 3+ children this
|
||||
rounds to a 10× token-efficiency win.
|
||||
|
||||
---
|
||||
|
||||
## 7. Governance — trust, restrict, stop_cascade, close_all_children
|
||||
|
||||
These four endpoints let an operator steer a live coordinator session
|
||||
mid-flight. All four emit an audit event tagged
|
||||
`coordinator.<action>` via the dedicated audit executor so a cascade
|
||||
burst can't starve audit writes.
|
||||
|
||||
### `POST /trust` — auto-approve own-subtree sends
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/trust
|
||||
{"send": true}
|
||||
```
|
||||
|
||||
Flips `trust_send=true` on the live session. Subsequent
|
||||
`send_to_workstream` calls that target a ws_id in the coordinator's
|
||||
own subtree skip the approval prompt; foreign ws_ids and other tool
|
||||
calls still go through the normal flow. Requires both
|
||||
`admin.coordinator` AND `coordinator.trust.send` permissions (the
|
||||
second grants a service token the opt-in it otherwise wouldn't get).
|
||||
|
||||
### `POST /restrict` — revoke tool access mid-session
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/restrict
|
||||
{"revoke": ["spawn_workstream", "delete_workstream"]}
|
||||
```
|
||||
|
||||
Unions the names into the session's revoked-tools set. Additive and
|
||||
idempotent — calling twice with overlapping lists converges to the
|
||||
union. Revocations don't survive a session close/reopen; operators
|
||||
opt in per session. Cap 256 tool names per request, 128 chars each.
|
||||
|
||||
### `POST /stop_cascade` — cancel the subtree
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/stop_cascade
|
||||
{}
|
||||
```
|
||||
|
||||
Cancels the coordinator's in-flight generation AND dispatches
|
||||
`cancel_workstream` through the routing proxy for every direct
|
||||
child in the in-memory registry. Returns:
|
||||
|
||||
```json
|
||||
{"status": "ok", "cancelled": ["child-1", "child-3"], "failed": [], "skipped": ["child-2"]}
|
||||
```
|
||||
|
||||
Response uses the [cascade-mutation bulk shape](bulk-endpoints.md):
|
||||
`cancelled` = accepted, `failed` = dispatch error worth retrying,
|
||||
`skipped` = upstream 404 (already gone — stale registry entry or
|
||||
the row was deleted between snapshot and dispatch). Grandchildren
|
||||
aren't touched directly; they sit behind their parent's cancel and
|
||||
propagate via the child's SSE stream.
|
||||
|
||||
### `POST /close_all_children` — soft-close the direct fan-out
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/close_all_children
|
||||
{"reason": "audit round complete"}
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{"status": "ok", "closed": ["c-1", "c-2"], "failed": [], "skipped": []}
|
||||
```
|
||||
|
||||
Soft-close cascade bounded by the same semaphore as `stop_cascade`.
|
||||
The `reason` (up to 512 chars) propagates into each closed child's
|
||||
audit + `workstream_config` for postmortem. Unlike `stop_cascade`
|
||||
this does NOT recurse into grandchildren — the model-facing tool
|
||||
that pairs with this endpoint asks for a bounded teardown of the
|
||||
coordinator's own fan-out. For a full-subtree teardown, use
|
||||
`stop_cascade`.
|
||||
|
||||
See [bulk-endpoints.md](bulk-endpoints.md) for why both endpoints
|
||||
share the cascade-mutation shape and how it differs from the
|
||||
`spawn_batch` / `cluster/ws/live` shape.
|
||||
|
||||
---
|
||||
|
||||
## 8. Approve / cancel
|
||||
|
||||
The `approve` endpoint is what resolves an `approve_request` SSE
|
||||
event. The coordinator's worker thread is blocked inside
|
||||
`ui.approve_tools` waiting for this POST.
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/approve
|
||||
{"approved": true, "feedback": null, "always": false}
|
||||
{"approved": false, "feedback": "spawn count looks too high — try 3 not 10"}
|
||||
{"approved": true, "feedback": null, "always": true} // always-approve this tool name
|
||||
```
|
||||
|
||||
`cancel` drops the in-flight generation but leaves the coordinator
|
||||
idle and open for a fresh `send`:
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/cancel
|
||||
{}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Close
|
||||
|
||||
```http
|
||||
POST /v1/api/workstreams/{ws_id}/close
|
||||
{}
|
||||
```
|
||||
|
||||
Soft-closes the session — state persists, children keep running (use
|
||||
`close_all_children` or `stop_cascade` first to wind them down), the
|
||||
worker thread exits, SSE streams send a final `stream_end` and
|
||||
disconnect. The row is reopenable via
|
||||
`POST /v1/api/workstreams/{ws_id}/open` so long as it hasn't been
|
||||
deleted.
|
||||
|
||||
---
|
||||
|
||||
## Further reading
|
||||
|
||||
- [coordinator-skills.md](coordinator-skills.md) — writing a skill
|
||||
that runs on a coordinator session (orchestrator persona,
|
||||
workflow patterns, `SkillKind` classifier).
|
||||
- [bulk-endpoints.md](bulk-endpoints.md) — the two bulk-shape
|
||||
idioms (`{results, denied, truncated}` vs
|
||||
`{<bucket>, failed, skipped}`) used by `cluster/ws/live`,
|
||||
`spawn_batch`, `stop_cascade`, and `close_all_children`.
|
||||
- [architecture.md](architecture.md) — cluster-wide architecture
|
||||
including how coordinator sessions fit next to node-hosted
|
||||
interactive workstreams.
|
||||
- The live OpenAPI spec (`/openapi.json` on any console process)
|
||||
and Swagger UI (`/docs`) — authoritative schemas for every
|
||||
endpoint above.
|
||||
endpoint above.
|
||||
@@ -1,323 +0,0 @@
|
||||
# Writing a coordinator-specific skill
|
||||
|
||||
Skills are prompt-level personas that steer a Turnstone session
|
||||
toward a narrow task. Most skills target **interactive** sessions —
|
||||
the single-workstream "do this thing" surface where the model wields
|
||||
`bash`, `edit_file`, `web_fetch`, and the rest of the maker toolset.
|
||||
|
||||
A **coordinator skill** is different. It runs on a session whose job
|
||||
is to orchestrate other sessions. The toolset is smaller and
|
||||
narrower, the persona is an orchestrator instead of a maker, and the
|
||||
success metric is "did the plan resolve" instead of "did the code
|
||||
compile". This doc covers the differences a skill author has to
|
||||
care about.
|
||||
|
||||
---
|
||||
|
||||
## The two-surface model
|
||||
|
||||
A row in `prompt_templates` carries a `kind` column (see
|
||||
[`turnstone/core/skill_kind.py`](../turnstone/core/skill_kind.py);
|
||||
migration 044 added the column). Three values:
|
||||
|
||||
| `SkillKind` enum | Stored as | Visible in |
|
||||
|----------------------|-----------------------------|---------------------------------------------------------------------------|
|
||||
| `SkillKind.INTERACTIVE` | `"interactive"` | Only the interactive-session activation path. `list_skills` on a coord won't show it. |
|
||||
| `SkillKind.COORDINATOR` | `"coordinator"` | Only the coordinator's `list_skills` tool. Hidden from interactive activation pickers. |
|
||||
| `SkillKind.ANY` | `"any"` | Both surfaces. Default for legacy rows predating the classifier. |
|
||||
|
||||
The `kind` field is a `StrEnum` — drop-in ``str`` compatible — so
|
||||
DB rows, JSON payloads, and `==` comparisons all work without
|
||||
translation at the edge.
|
||||
|
||||
When a coordinator calls `list_skills`, the SQL filter narrows to
|
||||
`kind IN ('coordinator', 'any')`. When an interactive session picks
|
||||
a skill at activation, the filter narrows to
|
||||
`kind IN ('interactive', 'any')`. A skill author tags once at
|
||||
creation; the two surfaces stay partitioned without any
|
||||
per-call filtering on the LLM side.
|
||||
|
||||
**Tagging a new skill as coordinator-only** — set `kind` to
|
||||
`SkillKind.COORDINATOR` (or the literal string `"coordinator"`) when
|
||||
you POST to `/v1/api/admin/skills`. Existing rows default to
|
||||
`SkillKind.ANY`; bump them to `COORDINATOR` if you've rewritten the
|
||||
prompt around the orchestrator toolset.
|
||||
|
||||
---
|
||||
|
||||
## Tool surface differences
|
||||
|
||||
Coordinator sessions receive a **fixed** tool set, defined in
|
||||
`turnstone/core/tools.py` as `COORDINATOR_TOOLS`. Nothing a skill
|
||||
or MCP config can do adds to it. Current members:
|
||||
|
||||
| Tool | Category | Notes |
|
||||
|---------------------------|-----------------|---------------------------------------------------------------------|
|
||||
| `spawn_workstream` | delegate | Create one child. Requires approval. |
|
||||
| `spawn_batch` | delegate | Create up to 10 children in one approval. Partial-success shape. |
|
||||
| `inspect_workstream` | observe | Read state + tail of one child. Auto-approved (no mutation). |
|
||||
| `list_workstreams` | observe | List the direct children (same shape as `/children` endpoint). |
|
||||
| `wait_for_workstream` | block | Block until one/all listed children hit a terminal state. |
|
||||
| `send_to_workstream` | steer | Queue a follow-up message to a running child. |
|
||||
| `close_workstream` | wind-down | Soft-close one child. Requires approval. |
|
||||
| `close_all_children` | wind-down | Soft-close every direct child in one approval. Partial-success shape. |
|
||||
| `cancel_workstream` | wind-down | Drop the in-flight generation; leaves child idle for a fresh send. |
|
||||
| `delete_workstream` | wind-down | Hard-delete one child. Requires approval. |
|
||||
| `list_nodes` | discover | Enumerate live cluster nodes + capabilities. |
|
||||
| `list_skills` | discover | Coordinator-visible skills only (SkillKind filter above). |
|
||||
| `tasks` | plan | Orchestrator-only scratchpad. Children don't see it. |
|
||||
|
||||
Explicitly **not** in the coordinator set:
|
||||
|
||||
- `bash` / `edit_file` / `write_file` / `append_file` / `diff_file` — no local FS.
|
||||
- `read_file` / `search` — no local FS reads.
|
||||
- `web_fetch` / `web_search` — no direct web access.
|
||||
- `task_agent` / `plan_agent` — sub-agent tools are zeroed on coord sessions.
|
||||
- `memory` / `recall` / `notify` / `watch` / `read_resource` / `use_prompt` / `skill` — the orchestrator's "memory" is its children's outputs; these UX / persistence tools belong to interactive sessions.
|
||||
|
||||
If your skill needs a coordinator to "run a command" or "read a
|
||||
file", write the delegate pattern instead: spawn a child with an
|
||||
appropriate skill, `wait_for_workstream`, then `inspect_workstream`
|
||||
for the output. The coordinator stays the orchestrator.
|
||||
|
||||
---
|
||||
|
||||
## Persona differences
|
||||
|
||||
Interactive skills compose on top of `base_interactive.md` — a
|
||||
"maker" persona: get the work done, use the tools, edit the code,
|
||||
close the loop.
|
||||
|
||||
Coordinator skills compose on top of
|
||||
[`base_coordinator.md`](../turnstone/prompts/base_coordinator.md) —
|
||||
an "orchestrator" persona: decompose, delegate, monitor, synthesise.
|
||||
The base text is short but sets the tone every coordinator skill
|
||||
inherits:
|
||||
|
||||
> You are a coordinator on a small, focused infrastructure team.
|
||||
> Your role is to orchestrate work across the cluster... You do
|
||||
> not edit files, run shell commands, browse the web, or manipulate
|
||||
> the codebase directly. Children do that.
|
||||
|
||||
Write your skill's system prompt to *add* task-specific orchestration
|
||||
hints on top — don't re-explain the role, don't paste tool JSON,
|
||||
don't try to override the "no direct action" contract. Keep the
|
||||
additions to: (a) the specific kind of work this skill delegates;
|
||||
(b) the preferred skill tags for children; (c) the synthesis shape
|
||||
the skill should end on.
|
||||
|
||||
---
|
||||
|
||||
## `tasks` integration
|
||||
|
||||
`tasks` is the coordinator's scratchpad — a persisted, ordered
|
||||
list of rows with fields `{id, title, status, child_ws_id, created,
|
||||
updated}` that only this coordinator sees. Children don't see it;
|
||||
the user does via the sidebar. Five actions: `add`, `update`,
|
||||
`remove`, `reorder`, `list` (only `list` is auto-approved; the
|
||||
mutators go through the approval flow).
|
||||
|
||||
The input schema refers to rows by `task_id`; the persisted row
|
||||
object exposes the same id as `id`. The `child_ws_id` field is a
|
||||
free-form label the skill sets to link a task to a spawned
|
||||
workstream — it is NOT validated against the workstreams table, so
|
||||
a skill can set it to a placeholder before `spawn_workstream`
|
||||
returns or keep it pointing at a closed child for later audit.
|
||||
|
||||
A skill's initial prompt can seed the task list by calling
|
||||
`tasks(action="add", title=...)` as its very first tool calls —
|
||||
the user gets a visible plan before any child is spawned, and the
|
||||
coordinator's future self has something concrete to iterate on.
|
||||
Status transitions (`pending` → `in_progress` → `done` / `blocked`)
|
||||
are the skill's main feedback loop: mutate the task when the child
|
||||
covering it finishes, not when the child starts. Use
|
||||
`tasks(action="update", task_id=..., child_ws_id=<ws_id>)` to
|
||||
link a task to the child that owns it once spawn returns.
|
||||
|
||||
A final gotcha: parallel tool dispatch does NOT serialise reads
|
||||
after writes in the same batch. If a skill issues an `update` and
|
||||
a `list` in one parallel tool batch, the `list` response may reflect
|
||||
the pre-update state. Dispatch mutate and list serially (one
|
||||
tool_use turn each) when the list must observe the mutation.
|
||||
|
||||
Keep the tasks coarse-grained — one per child, roughly. A 20-task
|
||||
list for a 3-child fan-out is noise; a 1-task list for a 5-child
|
||||
fan-out loses the plan. The sidebar renders tasks as the operator's
|
||||
mental model of "what the coord thinks it's doing".
|
||||
|
||||
---
|
||||
|
||||
## Referencing children by `ws_id`
|
||||
|
||||
Every ws_id returned by `spawn_workstream` / `spawn_batch` is a
|
||||
**full 32-char hex string**. The skill's system prompt must not
|
||||
invent ws_ids — a model that hallucinates `"child-1"` or `"ws-abc"`
|
||||
hits the tenant guard in `CoordinatorClient._is_own_subtree`, which
|
||||
validates ws_id against `parent_ws_id=coord_ws_id` AND
|
||||
`user_id=owner` in storage. The rejection shape varies by tool:
|
||||
|
||||
- **Mutating ops** (`send_to_workstream`, `close_workstream`,
|
||||
`cancel_workstream`, `delete_workstream`) return
|
||||
`{"error": "workstream not in coordinator subtree: <ws_id>", "status": 404}`
|
||||
— the skill should treat this as a tool error, not an empty result.
|
||||
- **`inspect_workstream`** returns `{"error": "workstream not found", "ws_id": "<ws_id>"}`
|
||||
(same shape as a genuinely missing row, so the guard can't be
|
||||
used as an existence oracle).
|
||||
- **`wait_for_workstream`** reports the offending id with
|
||||
`state="denied"` in its `results` dict; `mode="any"` won't
|
||||
satisfy on a pure-denied list, so a hallucinated id won't trick
|
||||
the wait into reporting "complete".
|
||||
|
||||
Pattern: capture each spawn result in the next tool call's input.
|
||||
The JSON tool-result carries `{"ws_id": "...", "name": "...",
|
||||
"node_id": "...", "routing_strategy": "..."}`; the model should
|
||||
extract the ws_id and pass it to `inspect_workstream` /
|
||||
`wait_for_workstream` / `send_to_workstream` / `close_workstream`
|
||||
verbatim.
|
||||
|
||||
A UI that wants human-readable identifiers should render the `name`
|
||||
field and keep the ws_id as the click-through key.
|
||||
|
||||
---
|
||||
|
||||
## `wait_for_workstream` vs `inspect_workstream`
|
||||
|
||||
Two distinct semantics, different cost profiles:
|
||||
|
||||
- **`wait_for_workstream(ws_ids=[...], timeout=60, mode="any")`** —
|
||||
blocks inside a single tool call until one (or all, for `mode="all"`)
|
||||
of the listed children reaches a terminal state (`idle`, `error`,
|
||||
`closed`, `deleted`). The worker thread blocks up to `timeout`
|
||||
seconds; the assistant turn remains a single round-trip regardless
|
||||
of how long the wait actually takes. Prefer this for "the plan
|
||||
needs child X to finish before the next step."
|
||||
- **`inspect_workstream(ws_id=...)`** — single read of the child's
|
||||
state + tail. Costs a full assistant turn (judge, tokens, stream).
|
||||
Prefer this for "what does the final message say?" after the child
|
||||
has already resolved (via `wait_for_workstream` or a known
|
||||
transition).
|
||||
|
||||
Rule of thumb: wait once for a fan-out, then inspect once per
|
||||
child for the content. A loop of inspect-every-few-seconds is a
|
||||
token-burning antipattern — on 3+ children it rounds to a 10×
|
||||
efficiency hit over a wait+inspect pair.
|
||||
|
||||
---
|
||||
|
||||
## Common coordinator patterns
|
||||
|
||||
Three patterns cover most coordinator skills. Pick the one that
|
||||
matches the task, or combine them deliberately.
|
||||
|
||||
### Pattern 1 — delegate-and-summarise
|
||||
|
||||
One specialist child, one focused brief, one synthesis message back
|
||||
to the user. Appropriate when the user's request is "run the thing
|
||||
and tell me what happened" and the work fits in one workstream.
|
||||
|
||||
```
|
||||
tasks(action='add', title='audit /auth for CSRF')
|
||||
spawn_workstream(skill='engineer', initial_message='audit /auth ...')
|
||||
wait_for_workstream(ws_ids=[<child>], timeout=300)
|
||||
inspect_workstream(ws_id=<child>)
|
||||
→ synthesise the final message into a user-facing response
|
||||
tasks(action='update', task_id='t_01', status='done')
|
||||
close_workstream(ws_id=<child>, reason='audit complete')
|
||||
```
|
||||
|
||||
### Pattern 2 — fan-out-and-synthesise
|
||||
|
||||
N children running in parallel, each with a distinct brief, all
|
||||
waited-on together, then synthesised. Appropriate when the user's
|
||||
request naturally decomposes into independent subtasks.
|
||||
|
||||
```
|
||||
tasks seeds:
|
||||
t_01 benchmark Anthropic 4.7 latency on summarisation
|
||||
t_02 benchmark OpenAI GPT-5.2 latency on summarisation
|
||||
t_03 benchmark Gemini 2.5 latency on summarisation
|
||||
spawn_batch(children=[...3 briefs...])
|
||||
wait_for_workstream(ws_ids=[c1, c2, c3], mode='all', timeout=600)
|
||||
inspect_workstream(ws_id=c1); ...(c2); ...(c3)
|
||||
→ synthesise head-to-head comparison
|
||||
tasks → all done
|
||||
close_all_children(reason='benchmark complete')
|
||||
```
|
||||
|
||||
Prefer `spawn_batch` over 3 individual `spawn_workstream` calls —
|
||||
one approval instead of three, one audit trail, deterministic
|
||||
sibling ordering. Pair with `wait_for_workstream(mode='all')` and
|
||||
`close_all_children(reason=...)` to wind the fan-out down in one
|
||||
approval each.
|
||||
|
||||
### Pattern 3 — plan-then-delegate
|
||||
|
||||
The coordinator first uses its own reasoning to carve the plan,
|
||||
records it in `tasks`, then spawns children that each own one
|
||||
task. Appropriate when the user's request is "figure out how to X"
|
||||
and the coordinator's planning step is itself valuable.
|
||||
|
||||
```
|
||||
→ coord reasons about the shape of the work
|
||||
tasks(action='add', title='...') × N # the plan, visible in the sidebar
|
||||
for task in tasks:
|
||||
spawn_workstream(skill=..., initial_message=task.brief)
|
||||
tasks(action='update', task_id=task.id, notes='ws=<child_ws_id>')
|
||||
wait_for_workstream(ws_ids=[...], mode='all', timeout=...)
|
||||
for child in children:
|
||||
inspect_workstream(ws_id=child)
|
||||
tasks(action='update', task_id=..., status='done', notes='result summary')
|
||||
→ synthesise
|
||||
```
|
||||
|
||||
The key distinction from Pattern 2: the plan is an artifact the user
|
||||
can see and interact with (via the sidebar). If the coordinator's
|
||||
reasoning-pass was wrong about the decomposition, the user can
|
||||
course-correct before any child runs.
|
||||
|
||||
---
|
||||
|
||||
## Testing a coordinator skill
|
||||
|
||||
Coordinator sessions are hosted on the console, not on a node.
|
||||
Integration tests that drive a real coord session live under
|
||||
`tests/test_coordinator_end_to_end.py` — they spin a console with
|
||||
an in-memory SQLite backend and a fake upstream node, then drive
|
||||
the session through its HTTP surface.
|
||||
|
||||
For a new coordinator skill:
|
||||
|
||||
1. Write the skill prompt as a string and pass it to the
|
||||
`coord_session` fixture's `skill=` kwarg (see
|
||||
`tests/test_coordinator_tools.py` for the pattern).
|
||||
2. Build a small fake cluster: one node + two children via
|
||||
the `_seed_children` helper in `tests/_coord_test_helpers.py`
|
||||
(``_seed_children(mgr._adapter, coord.id, ["child-1", "child-2"])``).
|
||||
3. Drive the session with seeded tool_call dicts matching the
|
||||
provider layer's shape. The unit-level tests in
|
||||
`tests/test_coordinator_tools.py` show the helper (`_tc(name,
|
||||
args, call_id)`).
|
||||
4. Assert the skill's decision shape — which tools fire in what
|
||||
order, what the tasks looks like at the end, which
|
||||
`_error` reasons appear on the denied-path.
|
||||
|
||||
A full end-to-end test isn't required for every skill; a
|
||||
prepare-step unit test that asserts "given this initial message, the
|
||||
first tool call is X with Y args" is usually sufficient to catch
|
||||
persona drift without a real LLM in the loop.
|
||||
|
||||
---
|
||||
|
||||
## Further reading
|
||||
|
||||
- [coordinator-api-tour.md](coordinator-api-tour.md) — the HTTP
|
||||
surface every coordinator skill indirectly drives.
|
||||
- [bulk-endpoints.md](bulk-endpoints.md) — the response shape
|
||||
`spawn_batch` and `close_all_children` use, so your skill can
|
||||
parse results / denied arrays correctly.
|
||||
- [governance.md](governance.md) — the broader governance surface
|
||||
(`/trust`, `/restrict`, `/stop_cascade`, role-based permissions)
|
||||
that wraps every coord session.
|
||||
- [settings.md](settings.md) — `coordinator.model_alias` and
|
||||
`coordinator.reasoning_effort` settings that gate which LLM runs
|
||||
the coordinator session at all.
|
||||
@@ -1,27 +1,34 @@
|
||||
# Consistent Hash Ring — Reference Design
|
||||
|
||||
**Status**: Reference — alternative routing strategy
|
||||
**Status**: Reference (not currently in the hot path)
|
||||
**Date**: 2026-03-30
|
||||
|
||||
Live routing uses **rendezvous (HRW) hashing** in
|
||||
`turnstone/core/rendezvous.py` and `turnstone/console/router.py`. This
|
||||
document captures a vnode-ring approach as a reference for future
|
||||
evaluation if the cluster outgrows rendezvous's O(N)-per-route
|
||||
characteristic.
|
||||
## Overview
|
||||
|
||||
The FNV-1a-32 hash function specified below is bit-identical to the
|
||||
hash used by the live rendezvous implementation; cross-language clients
|
||||
can rely on these test vectors.
|
||||
This document describes a consistent hash ring algorithm evaluated during
|
||||
the design of the direct HTTP transport routing system. The current
|
||||
implementation uses weight-proportional bucket assignment with a
|
||||
donor/recipient rebalancing algorithm (see `direct-http-transport.md`).
|
||||
The consistent hash ring is documented here as a reference for future
|
||||
scalability work — if the cluster grows beyond the point where the
|
||||
weight-proportional approach is sufficient, the ring provides a
|
||||
proven alternative with stronger stability guarantees.
|
||||
|
||||
## When the ring approach becomes interesting
|
||||
## When to consider the ring approach
|
||||
|
||||
The vnode ring becomes preferable to rendezvous hashing when:
|
||||
The current weight-proportional seeding + donor/recipient rebalancer works
|
||||
well when:
|
||||
- Cluster size is moderate (< 50 nodes)
|
||||
- Nodes join/leave infrequently
|
||||
- The rebalancer runs centrally (in the console)
|
||||
|
||||
- Cluster size grows large (50+ nodes) and the per-route O(N) hash
|
||||
computation becomes visible against downstream HTTP cost.
|
||||
- Decentralised routing is needed (each node computes the ring locally,
|
||||
no central console required).
|
||||
- A precomputed flat-array lookup is desired so the routing hot path
|
||||
avoids hashing entirely.
|
||||
The consistent hash ring becomes advantageous when:
|
||||
- Cluster size grows large (50+ nodes) and frequent membership changes
|
||||
cause the donor/recipient algorithm to churn
|
||||
- Decentralized routing is needed (each node computes the ring locally,
|
||||
no central console required)
|
||||
- Cross-language determinism is important (multiple implementations must
|
||||
agree on the same assignment without sharing state)
|
||||
|
||||
## Algorithm
|
||||
|
||||
@@ -126,17 +133,16 @@ class HashRing:
|
||||
# Precompute all 65536 bucket assignments
|
||||
```
|
||||
|
||||
## Comparison with rendezvous (HRW) hashing
|
||||
## Comparison with current approach
|
||||
|
||||
| Aspect | Rendezvous (live) | Consistent hash ring (this doc) |
|
||||
|--------|-------------------|---------------------------------|
|
||||
| Per-route cost | O(N) hash computes | O(log V) bisect against precomputed array |
|
||||
| Seeding | None — pure function | Build vnode array on every membership change |
|
||||
| Node addition | Pure function moves ~1/N keys | Ring moves ~1/N buckets |
|
||||
| Node removal | Surviving nodes' keys unchanged | Surviving nodes' buckets unchanged |
|
||||
| Decentralised | Yes — pure function over services | Yes — each node computes locally |
|
||||
| Persistent state | None | None on the hot path; precomputed array in memory |
|
||||
| Complexity | ~20 LOC | Virtual-node construction + bisect |
|
||||
| Aspect | Weight-proportional (current) | Consistent hash ring |
|
||||
|--------|------------------------------|---------------------|
|
||||
| Seeding | Exact weight split, deterministic | Hash-based, ~3% variance |
|
||||
| Node addition | Donor/recipient moves only excess | Ring moves ~1/N buckets |
|
||||
| Node removal | Dead buckets → most underloaded | Ring redistributes to clockwise neighbors |
|
||||
| Cross-node churn | Zero (only donor→recipient) | Zero (ring stability guarantee) |
|
||||
| Decentralized | No (needs central rebalancer) | Yes (each node computes locally) |
|
||||
| Complexity | Simple weight arithmetic | Virtual node construction + bisect |
|
||||
|
||||
## Test vectors
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ eval --> sqlite : SQLite
|
||||
|
||||
console --> server : HTTP proxy\n(hash-ring bucket lookup,\nproxy /node/{id}/* traffic)
|
||||
|
||||
channel --> server : HTTP + SSE\n(POST /v1/api/workstreams/{ws_id}/send,\nGET /v1/api/workstreams/{ws_id}/events)
|
||||
channel --> server : HTTP + SSE\n(POST /v1/api/send,\nGET /v1/api/events)
|
||||
|
||||
' Notes
|
||||
note right of console
|
||||
|
||||
@@ -19,8 +19,7 @@ package "Entry Points" <<Rectangle>> {
|
||||
component [cli.py\nturnstone] as cli <<entry>>
|
||||
component [server.py\nturnstone-server] as server <<entry>>
|
||||
component [eval.py\nturnstone-eval] as eval <<entry>>
|
||||
component [admin.py\nturnstone-admin] as admin <<entry>>
|
||||
component [bootstrap.py\nturnstone-bootstrap] as bootstrap <<entry>>
|
||||
component [chat.py\n(re-exports)] as chat <<entry>>
|
||||
}
|
||||
|
||||
' Core engine
|
||||
@@ -49,8 +48,7 @@ package "turnstone/core/" <<Rectangle>> {
|
||||
package "turnstone/channels/" <<Rectangle>> {
|
||||
component [_routing.py\nChannelRouter] as router <<channel>>
|
||||
component [discord/bot.py\nDiscordBot] as discordbot <<channel>>
|
||||
component [slack/bot.py\nSlackBot (Socket Mode)] as slackbot <<channel>>
|
||||
component [cli.py\nturnstone-channel] as gateway <<channel>>
|
||||
component [gateway.py\nturnstone-channel] as gateway <<channel>>
|
||||
}
|
||||
|
||||
' Console
|
||||
@@ -114,8 +112,7 @@ eval --> memory
|
||||
eval --> config
|
||||
eval --> tools
|
||||
|
||||
admin --> auth
|
||||
bootstrap --> providers
|
||||
chat --> session
|
||||
|
||||
' Core internal deps
|
||||
session --> providers
|
||||
@@ -138,10 +135,8 @@ tools --> schemas
|
||||
|
||||
' Channel dependencies
|
||||
gateway --> discordbot
|
||||
gateway --> slackbot
|
||||
gateway --> router
|
||||
discordbot --> sdkserver : HTTP + SSE
|
||||
slackbot --> sdkserver : HTTP + SSE
|
||||
router --> storage : channel_routes
|
||||
|
||||
' Console dependencies
|
||||
|
||||
@@ -170,15 +170,15 @@ Server --> Browser : Shimmed app.js
|
||||
deactivate Server
|
||||
|
||||
note right of Browser
|
||||
All fetch("/v1/api/workstreams/{ws_id}/send") calls in the
|
||||
server UI now become fetch("/node/nodeA/v1/api/workstreams/{ws_id}/send"),
|
||||
All fetch("/v1/api/send") calls in the
|
||||
server UI now become fetch("/node/nodeA/v1/api/send"),
|
||||
routed through the console proxy.
|
||||
end note
|
||||
|
||||
Browser -> Server : GET /node/nodeA/v1/api/workstreams/ws789/events
|
||||
Browser -> Server : GET /node/nodeA/v1/api/events?ws_id=ws789
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/v1/api/workstreams/ws789/events\n(SSE stream via httpx.AsyncClient timeout=None)
|
||||
Server -> NodeA : GET http://10.0.1.1:8080/v1/api/events?ws_id=ws789\n(SSE stream via httpx.AsyncClient timeout=None)
|
||||
activate NodeA
|
||||
|
||||
loop SSE streaming
|
||||
@@ -189,10 +189,10 @@ end
|
||||
deactivate NodeA
|
||||
deactivate Server
|
||||
|
||||
Browser -> Server : POST /node/nodeA/v1/api/workstreams/ws789/send\n{message:"hello"}
|
||||
Browser -> Server : POST /node/nodeA/v1/api/send\n{message:"hello", ws_id:"ws789"}
|
||||
activate Server #FFF9C4
|
||||
|
||||
Server -> NodeA : POST http://10.0.1.1:8080/v1/api/workstreams/ws789/send\n(body forwarded)
|
||||
Server -> NodeA : POST http://10.0.1.1:8080/v1/api/send\n(body forwarded)
|
||||
activate NodeA
|
||||
NodeA --> Server : {status:"ok"}
|
||||
deactivate NodeA
|
||||
|
||||
@@ -23,7 +23,7 @@ interface "StorageBackend" as SB <<protocol>> {
|
||||
+resolve_workstream(alias_or_id) → str | None
|
||||
+delete_workstream(ws_id) → bool
|
||||
+prune_workstreams(retention_days) → (int, int)
|
||||
+list_workstreams(node_id, limit, *, parent_ws_id, kind, user_id) → list
|
||||
+list_workstreams(node_id, limit) → list
|
||||
+save_workstream_config(ws_id, config)
|
||||
+load_workstream_config(ws_id) → dict
|
||||
+kv_get(key) → str | None
|
||||
|
||||
@@ -79,7 +79,7 @@ class "Scope Hierarchy" as SH <<scope>> {
|
||||
--
|
||||
GET → read
|
||||
POST write paths → write
|
||||
POST /api/workstreams/{ws_id}/approve → approve
|
||||
POST /api/approve → approve
|
||||
/api/admin/* → approve
|
||||
}
|
||||
|
||||
|
||||
@@ -20,14 +20,11 @@ class "Discord" as Discord <<platform>> {
|
||||
asyncio event loop
|
||||
}
|
||||
|
||||
class "Slack" as Slack <<platform>> {
|
||||
Socket Mode WebSocket
|
||||
class "Slack (future)" as Slack <<platform>> {
|
||||
Socket Mode / Events API
|
||||
Block Kit messages
|
||||
Slash command (default /turnstone)
|
||||
DM + channel events
|
||||
--
|
||||
slack-bolt (Python)
|
||||
asyncio event loop
|
||||
Planned integration
|
||||
}
|
||||
|
||||
class "Teams (future)" as Teams <<platform>> {
|
||||
@@ -41,7 +38,7 @@ class "Teams (future)" as Teams <<platform>> {
|
||||
class "turnstone-channel" as ChannelService <<service>> {
|
||||
entry point: turnstone-channel
|
||||
--
|
||||
One process — hosts one or more adapters
|
||||
One process per platform
|
||||
asyncio event loop
|
||||
Structured logging (structlog)
|
||||
--log-level, --log-format
|
||||
@@ -50,19 +47,6 @@ class "turnstone-channel" as ChannelService <<service>> {
|
||||
GET /health
|
||||
}
|
||||
|
||||
class "SlackBot" as SlackBot <<service>> {
|
||||
+on_message(event)
|
||||
+on_action(action) (Block Kit buttons)
|
||||
+send(channel_id, content)
|
||||
+send_notification(channel_id, content, ws_id)
|
||||
+run(bot_token, app_token)
|
||||
--
|
||||
slack-bolt AsyncApp
|
||||
Socket Mode client
|
||||
Per-user channel sessions via slash command
|
||||
DM routing without slash command
|
||||
}
|
||||
|
||||
class "DiscordBot" as Bot <<service>> {
|
||||
+on_message(msg)
|
||||
+on_interaction(interaction)
|
||||
@@ -95,10 +79,10 @@ class "ChannelRouter" as Router <<service>> {
|
||||
|
||||
' -- Server --
|
||||
class "turnstone-server" as Server <<server>> {
|
||||
POST /v1/api/workstreams/{ws_id}/send
|
||||
POST /v1/api/workstreams/{ws_id}/approve
|
||||
POST /v1/api/send
|
||||
POST /v1/api/approve
|
||||
POST /v1/api/workstreams/new
|
||||
GET /v1/api/workstreams/{ws_id}/events
|
||||
GET /v1/api/events?ws_id=
|
||||
--
|
||||
LLM execution + tool use
|
||||
SSE event stream
|
||||
@@ -148,21 +132,16 @@ Bot --> Router : on_message\non_interaction
|
||||
Router --> CU : resolve identity
|
||||
Router --> CR : resolve / register route
|
||||
|
||||
Router --> Server : POST /v1/api/workstreams/{ws_id}/send\nPOST /v1/api/workstreams/{ws_id}/approve\nPOST /v1/api/workstreams/new
|
||||
Bot --> Server : GET /v1/api/workstreams/{ws_id}/events\n(SSE via httpx-sse)
|
||||
Router --> Server : POST /v1/api/send\nPOST /v1/api/approve\nPOST /v1/api/workstreams/new
|
||||
Bot --> Server : GET /v1/api/events?ws_id=\n(SSE via httpx-sse)
|
||||
Server --> Bot : SSE event stream
|
||||
|
||||
Bot --> Discord : reply / embed\nbutton callback
|
||||
|
||||
Slack --> SlackBot : socket-mode\nevents
|
||||
SlackBot --> Router : on_message / on_action
|
||||
SlackBot --> Server : POST /v1/api/workstreams/{ws_id}/send\nGET /v1/api/workstreams/{ws_id}/events
|
||||
SlackBot --> Slack : post / update\nBlock Kit button callbacks
|
||||
|
||||
Slack .[hidden]. Discord
|
||||
Teams .[hidden]. Slack
|
||||
|
||||
ChannelService --> Bot : creates + runs
|
||||
ChannelService --> SlackBot : creates + runs
|
||||
ChannelService --> Router : creates
|
||||
ChannelService --> SVC : register / heartbeat /\nderegister
|
||||
|
||||
@@ -179,7 +158,7 @@ note right of Bot
|
||||
(or creates new workstream)
|
||||
4. ChannelRouter resolves platform user -> user_id
|
||||
via channel_users table
|
||||
5. Router sends POST /v1/api/workstreams/{ws_id}/send to server
|
||||
5. Router sends POST /v1/api/send to server
|
||||
|
||||
**Workstream Resume (evicted workstreams)**
|
||||
1. Stale route detected (no active SSE listener)
|
||||
@@ -193,7 +172,7 @@ end note
|
||||
note right of Server
|
||||
**Outbound Flow**
|
||||
1. Server emits SSE events on
|
||||
GET /v1/api/workstreams/{ws_id}/events
|
||||
GET /v1/api/events?ws_id=
|
||||
2. Bot subscribes via httpx-sse
|
||||
3. Bot formats and sends to Discord thread
|
||||
end note
|
||||
@@ -204,7 +183,7 @@ note bottom of CR
|
||||
2. Bot renders Discord buttons (Approve / Deny)
|
||||
3. User clicks button -> on_interaction()
|
||||
4. Router builds ApproveMessage
|
||||
5. Router sends POST /v1/api/workstreams/{ws_id}/approve to server
|
||||
5. Router sends POST /v1/api/approve to server
|
||||
end note
|
||||
|
||||
note bottom of CU
|
||||
|
||||
@@ -211,7 +211,7 @@ note over Session, Judge
|
||||
**Storage:**
|
||||
intent_verdicts table (migration 012), output_assessments table
|
||||
(migration 022). Both queryable via admin API endpoints
|
||||
(requires admin.judge permission). Skills store risk_level,
|
||||
(requires admin.judge permission). Skills store scan_status,
|
||||
scan_report, scan_version for install-time risk assessment.
|
||||
end note
|
||||
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
@startuml
|
||||
title Turnstone - coordinator wait_for_workstream lifecycle
|
||||
|
||||
skinparam sequenceArrowThickness 1.5
|
||||
skinparam noteBackgroundColor #FDF6E3
|
||||
|
||||
participant "Coordinator\nLLM" as LLM
|
||||
participant "ChatSession\n(worker thread)" as CS
|
||||
participant "CoordinatorClient" as CC
|
||||
participant "SessionUI\n(SSE fanout)" as UI
|
||||
participant "Console routing\nproxy" as RP
|
||||
database "Storage\n(workstreams row)" as DB
|
||||
participant "Child\nnode" as NODE
|
||||
|
||||
== Spawn ==
|
||||
|
||||
LLM -> CS : tool_call spawn_workstream(...)
|
||||
activate CS
|
||||
CS -> CC : spawn(initial_message=...,\nparent_ws_id=coord, user_id=...)
|
||||
CC -> RP : POST /v1/api/route/workstreams/new
|
||||
RP -> NODE : dispatch (rendezvous)
|
||||
NODE -> DB : insert workstreams row\nstate='running'
|
||||
RP --> CC : {ws_id, node_id, name, status: 200}
|
||||
CC --> CS : {ws_id, ...}
|
||||
CS -> UI : on_tool_result\n("spawn_workstream", ws_id)
|
||||
deactivate CS
|
||||
|
||||
note right of LLM
|
||||
Model now knows the child ws_id.
|
||||
It can inspect / send / wait, and
|
||||
the parent registry tracks it.
|
||||
end note
|
||||
|
||||
== Wait (blocking) ==
|
||||
|
||||
LLM -> CS : tool_call wait_for_workstream\n(ws_ids=[child], mode="any", timeout=60)
|
||||
activate CS
|
||||
CS -> CS : _prepare_wait_for_workstream\n(validate ws_ids, timeout, mode)
|
||||
CS -> UI : emit wait_started\n{call_id, ws_ids, mode, timeout}
|
||||
|
||||
CS -> CC : wait_for_workstream(ws_ids, timeout,\nmode, progress_callback)
|
||||
activate CC
|
||||
|
||||
loop every 500ms up to timeout
|
||||
CC -> DB : read workstreams row(s)
|
||||
DB --> CC : {state, updated, tokens, ...}
|
||||
alt state in {idle, error, closed, deleted}
|
||||
note over CC
|
||||
real-terminal state ->
|
||||
completion condition met
|
||||
end note
|
||||
else still running / thinking / attention
|
||||
CC -> CS : progress_callback(snap)\n(diff-on-change or 5s heartbeat)
|
||||
CS -> UI : emit wait_progress\n{call_id, elapsed, results?}
|
||||
end
|
||||
end
|
||||
|
||||
CC --> CS : {complete, elapsed,\nresults: {ws_id: snap}}
|
||||
deactivate CC
|
||||
|
||||
CS -> UI : emit wait_ended\n{call_id, complete, elapsed, results}
|
||||
CS -> UI : on_tool_result\n("wait_for_workstream",\n"complete after Ns (R/N resolved)")
|
||||
CS --> LLM : tool_result (full results dict)
|
||||
deactivate CS
|
||||
|
||||
note left of UI
|
||||
Sidebar "waiting on N children" indicator
|
||||
keys on call_id - started / progress / ended
|
||||
scope to a single wait invocation so
|
||||
nested waits render independent badges.
|
||||
end note
|
||||
|
||||
== After wait: inspect + close ==
|
||||
|
||||
LLM -> CS : tool_call inspect_workstream(ws_id=child)
|
||||
CS -> CC : inspect(ws_id)
|
||||
CC -> DB : read row + tail
|
||||
CC --> CS : {state, messages, tokens, ...}
|
||||
CS --> LLM : tool_result (serialised)
|
||||
|
||||
LLM -> CS : tool_call close_workstream\n(ws_id=child, reason="...")
|
||||
CS -> CC : close_workstream(ws_id, reason)
|
||||
CC -> RP : POST /v1/api/route/workstreams/close
|
||||
RP -> NODE : dispatch
|
||||
NODE -> DB : state='closed',\nclose_reason='...'
|
||||
RP --> CC : {status: 200}
|
||||
CC --> CS : {closed: true, status: 200, reason: ...}
|
||||
CS --> LLM : tool_result
|
||||
@enduml
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a3b5c59403a6febd81667fc8fd2a7d22bc59da6130eba0dea5449c42668d0ede
|
||||
size 387044
|
||||
oid sha256:75da80e6bd205e45b9fe48aa2f87197110c908f44e1d98c52eee69f956274711
|
||||
size 400402
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ae4f79fb22600106f8cb0af4ba5586bb26ea5d57e27ef382fdc59b6549fdbd21
|
||||
size 415473
|
||||
oid sha256:8a5957b71822656098cbe779ca619b7e56e9b11b0f1c4eac4fc880ed62b71a2a
|
||||
size 358670
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:aa12d81dc578f7e65bf4df3152b3de1736289c422f83d0b0cd32107726722357
|
||||
size 172028
|
||||
+3
-13
@@ -22,7 +22,7 @@ Console dashboard: http://localhost:8090
|
||||
|---------|------|---------|-------------|
|
||||
| `server` | 8080 | default | Web UI + chat workstreams + LLM |
|
||||
| `console` | 8090 | default | Cluster dashboard |
|
||||
| `channel` | — | production | Channel gateway (Discord and/or Slack adapters) |
|
||||
| `channel` | — | production | Channel gateway (Discord, Slack, etc.) |
|
||||
| `server-1`…`server-10` | — | cluster | 10-node server fleet (PostgreSQL required) |
|
||||
|
||||
## Profiles
|
||||
@@ -108,16 +108,8 @@ The database stores workstream history, user accounts, and API tokens. When usin
|
||||
|----------|---------|-------------|
|
||||
| `TURNSTONE_DISCORD_TOKEN` | — | Discord bot token (required to enable Discord adapter) |
|
||||
| `TURNSTONE_DISCORD_GUILD` | `0` | Restrict to a single Discord guild (0 = all guilds) |
|
||||
| `TURNSTONE_SLACK_TOKEN` | — | Slack Bot User OAuth token `xoxb-…` (required to enable Slack adapter) |
|
||||
| `TURNSTONE_SLACK_APP_TOKEN` | — | Slack App-Level token `xapp-…` (required with `TURNSTONE_SLACK_TOKEN`) |
|
||||
| `TURNSTONE_SLACK_CHANNELS` | — | Comma-separated Slack channel IDs to allow (empty = all) |
|
||||
| `TURNSTONE_SLACK_SLASH_COMMAND` | `/turnstone` | Slash command registered in the Slack app |
|
||||
|
||||
The channel service runs in the `production` profile. When
|
||||
`TURNSTONE_DISCORD_TOKEN` or the Slack pair is set the gateway starts the
|
||||
corresponding adapter; both can run in one process. See
|
||||
[Channel Integrations](channels.md) for platform app setup and user
|
||||
account linking.
|
||||
The channel service runs in the `production` profile. When `TURNSTONE_DISCORD_TOKEN` is set, the Discord adapter connects to the Discord Gateway and routes messages to the server via HTTP. See [Channel Integrations](channels.md) for full setup instructions including Discord application creation and user account linking.
|
||||
|
||||
## Scaling
|
||||
|
||||
@@ -149,9 +141,7 @@ docker compose build
|
||||
docker compose build --no-cache
|
||||
```
|
||||
|
||||
All entry points are installed in a single image: `turnstone`,
|
||||
`turnstone-server`, `turnstone-console`, `turnstone-channel`,
|
||||
`turnstone-admin`, `turnstone-eval`, and `turnstone-bootstrap`.
|
||||
All entry points are installed in a single image: `turnstone-server`, `turnstone-console`, `turnstone-channel`, `turnstone-admin`, `turnstone-eval`.
|
||||
|
||||
## Cleanup
|
||||
|
||||
|
||||
+6
-12
@@ -62,14 +62,14 @@ etc.) since workstream templates were merged into the skills system in v0.8.0.
|
||||
- **Default skills**: All `is_default=true` skills auto-apply to new
|
||||
workstreams, concatenated in alphabetical order by name. Use name prefixes
|
||||
(e.g. `01-safety`, `02-style`) to control ordering.
|
||||
- **Explicit selection**: `--skill <name>` CLI flag, `skill` field on
|
||||
- **Explicit selection**: `--template <name>` CLI flag, `template` field on
|
||||
`POST /v1/api/workstreams/new`, console creation modal dropdown, scheduled task
|
||||
config, and channel adapter config. An explicit skill *replaces* defaults.
|
||||
- **Variables**: Three built-in placeholders resolved at load time:
|
||||
`{{model}}` (active model name), `{{ws_id}}` (workstream ID),
|
||||
`{{node_id}}` (server node ID). Unrecognized placeholders are kept as-is.
|
||||
- **Runtime switching**: `/skill <name>` to switch, `/skill clear` to revert
|
||||
to defaults, `/skill` to show current. Persisted across resume.
|
||||
- **Runtime switching**: `/template <name>` to switch, `/template clear` to revert
|
||||
to defaults, `/template` to show current. Persisted across resume.
|
||||
- **Model-driven loading**: The `skill` built-in tool lets the model
|
||||
discover and activate skills mid-conversation. `search` action finds skills
|
||||
by query (auto-approved); `load` action activates by name (requires user
|
||||
@@ -91,7 +91,7 @@ etc.) since workstream templates were merged into the skills system in v0.8.0.
|
||||
time. The scanner evaluates four risk axes: content risk (command execution,
|
||||
data exfiltration), supply chain risk (pipe-to-shell, transitive installs),
|
||||
vulnerability risk (prompt injection, insecure credentials), and declared
|
||||
capability risk (from `allowed-tools` in SKILL.md). Results populate the `risk_level`
|
||||
capability risk (from `allowed-tools` in SKILL.md). Results populate the `scan_status`
|
||||
(safe/low/medium/high/critical) and `scan_report` (JSON breakdown) columns.
|
||||
These fields are system-managed and cannot be overwritten via the admin API.
|
||||
- **Discovery**: External skills can be discovered and installed from registries:
|
||||
@@ -186,21 +186,15 @@ Full OpenAPI spec at `/openapi.json` and Swagger UI at `/docs`.
|
||||
|
||||
## Admin Console UI
|
||||
|
||||
Governance-related tabs within the 18-tab admin panel:
|
||||
6 new tabs added to the admin panel (11 total):
|
||||
|
||||
- **Roles** — CRUD roles, permission checkbox grid, user role assignment modal
|
||||
- **Policies** — CRUD tool policies with colored action badges (green/red/amber)
|
||||
- **Prompts** — Prompt-policy editor (heuristics for admin guardrails)
|
||||
- **Skills** — CRUD skills with wide modal, textarea editor; Discover pill for
|
||||
installing from skills.sh / GitHub; per-row scan badges (safe/low/med/high/critical)
|
||||
- **Judge** — Intent validation configuration and verdict history
|
||||
- **Skills** — CRUD skills with wide modal, textarea editor
|
||||
- **Usage** — Summary readouts + CSS bar chart, time range + group-by selectors
|
||||
- **Audit** — Filterable log with relative timestamps, load-more pagination
|
||||
|
||||
Tabs are permission-gated: hidden if the user lacks the required permission.
|
||||
See [docs/console.md](console.md) for the full tab list and
|
||||
[docs/settings.md](settings.md) for the Settings tab that edits live
|
||||
ConfigStore values.
|
||||
|
||||
## SDK
|
||||
|
||||
|
||||
+4
-4
@@ -315,7 +315,7 @@ four independent risk axes:
|
||||
`Bash(*)` (unrestricted shell) is high risk. `Bash(git:*)` is low.
|
||||
Read-only tools are safe.
|
||||
|
||||
Results are stored in `risk_level` (tier: safe/low/medium/high/critical) and
|
||||
Results are stored in `scan_status` (tier: safe/low/medium/high/critical) and
|
||||
`scan_report` (JSON breakdown) on the `prompt_templates` table. These fields are
|
||||
system-managed and not editable via the admin API.
|
||||
|
||||
@@ -400,11 +400,11 @@ level, annotations, output length, redaction status).
|
||||
|
||||
### Session-level skill scan warning
|
||||
|
||||
When a skill with `risk_level` of `high` or `critical` is loaded into a
|
||||
When a skill with `scan_status` of `high` or `critical` is loaded into a
|
||||
session, a warning is emitted via `on_info`:
|
||||
|
||||
```
|
||||
⚠ Skill 'my-skill' has risk level: high.
|
||||
⚠ Skill 'my-skill' has scan status: high.
|
||||
Review scan report in admin panel before enabling in production.
|
||||
```
|
||||
|
||||
@@ -421,7 +421,7 @@ All three evaluation systems persist their assessments for future calibration:
|
||||
|-------|--------|-------------|
|
||||
| `intent_verdicts` | Intent judge (heuristic + LLM) | `func_name`, `risk_level`, `confidence`, `user_decision` |
|
||||
| `output_assessments` | Output guard | `func_name`, `risk_level`, `flags`, `redacted` |
|
||||
| `prompt_templates` | Skill scanner | `risk_level`, `scan_report`, `scan_version` |
|
||||
| `prompt_templates` | Skill scanner | `scan_status`, `scan_report`, `scan_version` |
|
||||
|
||||
Run v1 with all tools requiring manual approval to build a local dataset.
|
||||
In v2, calibration tooling will analyze this data to:
|
||||
|
||||
@@ -146,7 +146,7 @@ with TurnstoneConsole("http://localhost:8081", token="...") as client:
|
||||
### TypeScript
|
||||
|
||||
```typescript
|
||||
import { TurnstoneConsole } from "@turnstone/sdk";
|
||||
import { TurnstoneConsole } from "@anthropic/turnstone-sdk";
|
||||
|
||||
const client = new TurnstoneConsole({
|
||||
baseUrl: "http://localhost:8081",
|
||||
|
||||
+15
-24
@@ -1,24 +1,17 @@
|
||||
# Release Process
|
||||
|
||||
Turnstone ships several parallel release tracks from a single PyPI package.
|
||||
Turnstone uses two parallel release tracks published from a single PyPI package.
|
||||
|
||||
## Release Tracks
|
||||
|
||||
| Track | Versions | Branch | Docker tags | PyPI install |
|
||||
|-------|----------|--------|-------------|--------------|
|
||||
| **Legacy 1.0** | `1.0.x` | `stable/1.0` | `:1.0.x`, `:1.0` | `pip install 'turnstone==1.0.*'` |
|
||||
| **Stable 1.3** | `1.3.x` | `stable/1.3` | `:1.3.x`, `:1.3` | `pip install 'turnstone==1.3.*'` |
|
||||
| **Stable 1.4** | `1.4.x` | `stable/1.4` | `:1.4.x`, `:1.4`, `:stable`, `:latest` | `pip install turnstone` |
|
||||
| **Experimental** | `1.5.0aN` | `main` | `:1.5.0aN`, `:experimental` | `pip install turnstone --pre` |
|
||||
| **Stable** | `1.1.0`, `1.1.1` | `stable/1.1` | `:1.1.0`, `:1.1`, `:stable`, `:latest` | `pip install turnstone` |
|
||||
| **Experimental** | `1.2.0a1`, `1.2.0a2` | `main` | `:1.2.0a1`, `:experimental` | `pip install turnstone --pre` |
|
||||
|
||||
- **Stable** tracks receive bugfixes only. The most-recent stable minor
|
||||
owns the `:stable` / `:latest` Docker tags and the default PyPI
|
||||
install.
|
||||
- **Experimental** (always on `main`) receives new features. May be
|
||||
rough around the edges.
|
||||
- When experimental matures, it is promoted to a new stable minor via
|
||||
a `stable/X.Y` branch; older stable branches continue to receive
|
||||
security fixes until explicitly retired.
|
||||
- **Stable** receives bugfixes only. Production-grade.
|
||||
- **Experimental** receives new features. May be rough around the edges.
|
||||
- When experimental matures, it is promoted to stable. The previous stable branch stops receiving patches.
|
||||
|
||||
## Version Scheme
|
||||
|
||||
@@ -33,17 +26,17 @@ Turnstone ships several parallel release tracks from a single PyPI package.
|
||||
## Releasing an Experimental Version (from main)
|
||||
|
||||
```bash
|
||||
scripts/release.sh 1.5.0a2 --push
|
||||
scripts/release.sh 1.1.0a2 --push
|
||||
```
|
||||
|
||||
This bumps `pyproject.toml` + `turnstone/__init__.py`, regenerates `uv.lock`, commits, tags `v1.5.0a2`, and pushes. CI runs, then publish + Docker workflows fire automatically.
|
||||
This bumps `pyproject.toml` + `turnstone/__init__.py`, regenerates `uv.lock`, commits, tags `v1.1.0a2`, and pushes. CI runs, then publish + Docker workflows fire automatically.
|
||||
|
||||
## Releasing a Stable Patch (from stable/X.Y)
|
||||
|
||||
```bash
|
||||
git checkout stable/1.4
|
||||
git checkout stable/1.0
|
||||
git cherry-pick <commit-hash> # bugfix from main
|
||||
scripts/release.sh 1.4.1 --push
|
||||
scripts/release.sh 1.0.2 --push
|
||||
```
|
||||
|
||||
## Promoting Experimental to Stable
|
||||
@@ -52,19 +45,17 @@ When `main` is ready for a stable release:
|
||||
|
||||
```bash
|
||||
# 1. Tag the stable release on main
|
||||
scripts/release.sh 1.5.0 --push
|
||||
scripts/release.sh 1.1.0 --push
|
||||
|
||||
# 2. Create the stable maintenance branch from that tag
|
||||
git branch stable/1.5 v1.5.0
|
||||
git push origin stable/1.5
|
||||
git branch stable/1.1 v1.1.0
|
||||
git push origin stable/1.1
|
||||
|
||||
# 3. Start the next experimental cycle on main
|
||||
scripts/release.sh 1.6.0a1 --push
|
||||
scripts/release.sh 1.2.0a1 --push
|
||||
```
|
||||
|
||||
The previous stable branch (`stable/1.4`) continues to receive
|
||||
security-only patches; older tracks (`stable/1.0`, `stable/1.3`) are
|
||||
retired when they fall out of support.
|
||||
The previous `stable/1.0` branch stops receiving patches at this point.
|
||||
|
||||
## CI/CD Pipeline
|
||||
|
||||
|
||||
+2
-36
@@ -69,12 +69,8 @@ Both `TurnstoneServer` (sync) and `AsyncTurnstoneServer` (async) expose:
|
||||
|----------|--------|---------|
|
||||
| **Workstreams** | `list_workstreams()` | `ListWorkstreamsResponse` |
|
||||
| | `dashboard()` | `DashboardResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve, skill, initial_message, attachments)` | `CreateWorkstreamResponse` |
|
||||
| | `create_workstream(*, name, model, auto_approve, skill)` | `CreateWorkstreamResponse` |
|
||||
| | `close_workstream(ws_id)` | `StatusResponse` |
|
||||
| **Attachments** | `upload_attachment(ws_id, filename, data, *, mime_type=...)` | `UploadAttachmentResponse` |
|
||||
| | `list_attachments(ws_id)` | `ListAttachmentsResponse` |
|
||||
| | `get_attachment_content(ws_id, attachment_id)` | `bytes` |
|
||||
| | `delete_attachment(ws_id, attachment_id)` | `StatusResponse` |
|
||||
| **Chat** | `send(message, ws_id)` | `SendResponse` |
|
||||
| | `approve(*, ws_id, approved, feedback, always)` | `StatusResponse` |
|
||||
| | `plan_feedback(*, ws_id, feedback)` | `StatusResponse` |
|
||||
@@ -175,36 +171,6 @@ result.ok # True if no errors and not timed out
|
||||
result.timed_out # True if timeout expired
|
||||
```
|
||||
|
||||
### Attachments
|
||||
|
||||
Upload files to a workstream and attach them to the next user turn:
|
||||
|
||||
```python
|
||||
# Upload separately, then send a message — attachments auto-attach
|
||||
with open("screenshot.png", "rb") as f:
|
||||
att = client.upload_attachment(ws.ws_id, "screenshot.png",
|
||||
f.read(),
|
||||
mime_type="image/png")
|
||||
client.send("What's wrong in this screenshot?", ws.ws_id)
|
||||
|
||||
# Or attach at workstream-creation time (multipart upload)
|
||||
from turnstone.sdk import AttachmentUpload
|
||||
|
||||
with open("notes.txt", "rb") as f:
|
||||
ws = client.create_workstream(
|
||||
name="triage",
|
||||
initial_message="Summarize the notes",
|
||||
attachments=[AttachmentUpload(data=f.read(),
|
||||
filename="notes.txt",
|
||||
mime_type="text/plain")],
|
||||
)
|
||||
```
|
||||
|
||||
Limits: images ≤ 4 MiB (png/jpeg/gif/webp), text ≤ 512 KiB (UTF-8),
|
||||
10 pending per (workstream, user). The SDK auto-generates `ws_id` on the
|
||||
client so cluster-routed callers bind attachments to the owning node
|
||||
before the request lands.
|
||||
|
||||
### Error Handling
|
||||
|
||||
Non-2xx responses raise `TurnstoneAPIError`:
|
||||
@@ -318,7 +284,7 @@ turnstone/sdk/ Python SDK (sub-package)
|
||||
_base.py Shared httpx async client, auth, error handling
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
events.py 38 SSE event dataclasses with type registry
|
||||
events.py 27 SSE event dataclasses with type registry
|
||||
server.py AsyncTurnstoneServer + TurnstoneServer
|
||||
console.py AsyncTurnstoneConsole + TurnstoneConsole
|
||||
|
||||
|
||||
+8
-11
@@ -1,10 +1,8 @@
|
||||
# Security and Authentication
|
||||
|
||||
Turnstone uses a layered authentication system with two token types
|
||||
(database-backed API tokens + HMAC-SHA256 JWTs), hierarchical scopes,
|
||||
and a split architecture where the console manages credentials while
|
||||
individual server nodes validate JWTs locally. Inter-service traffic
|
||||
uses short-lived service JWTs minted by `ServiceTokenManager`.
|
||||
Turnstone uses a layered authentication system with three token types,
|
||||
hierarchical scopes, and a split architecture where the console manages
|
||||
credentials while individual server nodes validate JWTs locally.
|
||||
|
||||
---
|
||||
|
||||
@@ -39,7 +37,7 @@ Claims:
|
||||
|-------|-------------|
|
||||
| `sub` | User ID |
|
||||
| `scopes` | Comma-separated scope list (`read,write,approve`) |
|
||||
| `src` | Token source (`password`, `database`, `oidc`, or a service origin like `console`, `cli`, or `channel`) |
|
||||
| `src` | Token source (`password`, `api_token`, `config`, `oidc`) |
|
||||
| `iss` | Issuer — always `turnstone` |
|
||||
| `aud` | Audience — `turnstone-server` or `turnstone-console` |
|
||||
| `iat` | Issued-at timestamp |
|
||||
@@ -67,11 +65,10 @@ Scopes are hierarchical — higher scopes imply all lower ones.
|
||||
| Method | Path pattern | Required scope |
|
||||
|--------|-------------|----------------|
|
||||
| GET | Any protected path | `read` |
|
||||
| POST | `/api/plan`, `/api/command` | `write` |
|
||||
| POST | `/api/workstreams/new`, `/api/cluster/workstreams/new` | `write` |
|
||||
| POST | `/api/workstreams/{ws_id}/{send,cancel,close,delete,open,refresh-title,title,attachments}` | `write` |
|
||||
| DELETE | `/api/workstreams/{ws_id}/send` (dequeue), `/api/workstreams/{ws_id}/attachments/{attachment_id}` | `write` |
|
||||
| POST | `/api/workstreams/{ws_id}/approve` | `approve` |
|
||||
| POST | `/api/send`, `/api/plan`, `/api/command` | `write` |
|
||||
| POST | `/api/workstreams/new`, `/api/workstreams/close` | `write` |
|
||||
| POST | `/api/cluster/workstreams/new` | `write` |
|
||||
| POST | `/api/approve` | `approve` |
|
||||
| Any | `/api/admin/*` | `approve` |
|
||||
|
||||
Public paths bypass authentication entirely: `/`, `/health`, `/metrics`,
|
||||
|
||||
+1
-17
@@ -59,22 +59,6 @@ from ConfigStore. Model names and context windows are now configured per-model
|
||||
in the Models tab. A startup warning is logged if these keys appear in
|
||||
`config.toml`.
|
||||
|
||||
### Plan / task agent overrides
|
||||
|
||||
`plan_agent` and `task_agent` sub-sessions resolve independently from the
|
||||
conversation model so operators can pick a cheaper/faster model for
|
||||
autonomous loops:
|
||||
|
||||
| Setting | Purpose |
|
||||
|---------|---------|
|
||||
| `model.plan_alias` | Alias used for `plan_agent` sub-sessions. Falls back to `[model].plan_model` in config.toml, then `[model].agent_model`, then the session's active model. |
|
||||
| `model.task_alias` | Alias used for `task_agent` sub-sessions. Same fallback chain as `plan_alias`. |
|
||||
| `model.plan_effort` | Reasoning effort for `plan_agent` (`none` / `minimal` / `low` / `medium` / `high` / `xhigh` / `max`). Defaults to `high`. |
|
||||
| `model.task_effort` | Reasoning effort for `task_agent`. Empty string means "inherit from the session". |
|
||||
|
||||
All four are live-editable from the Settings tab and take effect on the
|
||||
next sub-agent invocation — no restart required.
|
||||
|
||||
---
|
||||
|
||||
## Bootstrap vs ConfigStore
|
||||
@@ -95,7 +79,7 @@ initialization:
|
||||
|
||||
| Section | Settings |
|
||||
|---------|----------|
|
||||
| `model` | default_alias, temperature, max_tokens, reasoning_effort, plan_alias, task_alias, plan_effort, task_effort |
|
||||
| `model` | default_alias, temperature, max_tokens, reasoning_effort |
|
||||
| `session` | instructions, retention_days, compact_max_tokens, auto_compact_pct |
|
||||
| `tools` | timeout, truncation, agent_max_turns, skip_permissions, search, search_threshold, search_max_results |
|
||||
| `server` | workstream_idle_timeout, max_workstreams |
|
||||
|
||||
+8
-11
@@ -169,8 +169,8 @@ Every tool defines a `primary_key`. The mapping is:
|
||||
| `man` | `page` |
|
||||
| `web_fetch` | `url` |
|
||||
| `web_search` | `query` |
|
||||
| `task_agent` | `prompt` |
|
||||
| `plan_agent` | `goal` |
|
||||
| `task` | `prompt` |
|
||||
| `plan` | `prompt` |
|
||||
| `memory` | `name` |
|
||||
| `recall` | `query` |
|
||||
| `notify` | `message` |
|
||||
@@ -357,10 +357,7 @@ Search the web using a text query.
|
||||
|
||||
## Agent
|
||||
|
||||
Tool names use the `_agent` suffix — bare `plan` / `task` collide with
|
||||
chat-template channel names on some local models.
|
||||
|
||||
### task_agent
|
||||
### task
|
||||
|
||||
Delegate a general-purpose task to an autonomous sub-agent.
|
||||
|
||||
@@ -374,7 +371,7 @@ Delegate a general-purpose task to an autonomous sub-agent.
|
||||
|
||||
---
|
||||
|
||||
### plan_agent
|
||||
### plan
|
||||
|
||||
Plan before implementing -- an autonomous agent explores the codebase and writes a structured plan.
|
||||
|
||||
@@ -546,11 +543,11 @@ pre-configure skills at workstream creation.
|
||||
- `load` — Activate a skill by name. Calls `set_skill()` which handles content
|
||||
rendering with `{{model}}`/`{{ws_id}}`/`{{node_id}}` variables, system message
|
||||
reinitialization, and config persistence. Returns the skill name, description,
|
||||
and security risk level. Warns on high/critical risk level.
|
||||
and security scan tier. Warns on high/critical scan status.
|
||||
- `search` — Find available skills by query. Uses BM25 relevance ranking over
|
||||
name, description, tags, and category (same `BM25Index` used by memory
|
||||
relevance and tool search). Returns up to 10 results with name, description,
|
||||
category, risk level, and activation type.
|
||||
category, scan status, and activation type.
|
||||
|
||||
- **Auto-approve**: `load` requires approval (changes session behavior); `search`
|
||||
is auto-approved (read-only).
|
||||
@@ -571,8 +568,8 @@ pre-configure skills at workstream creation.
|
||||
| `man` | Info | Yes | Yes | Yes | `page` |
|
||||
| `web_fetch` | Info | No | Yes | Yes | `url` |
|
||||
| `web_search` | Info | No | Yes | Yes | `query` |
|
||||
| `task_agent` | Agent | No | No | No | `prompt` |
|
||||
| `plan_agent` | Agent | No | No | No | `goal` |
|
||||
| `task` | Agent | No | No | No | `prompt` |
|
||||
| `plan` | Agent | No | No | No | `prompt` |
|
||||
| `memory` | Memory | Yes | No | No | `name` |
|
||||
| `recall` | Memory | Yes | No | No | `query` |
|
||||
| `notify` | Notify | Yes | Yes | Yes | `message` |
|
||||
|
||||
@@ -2,48 +2,11 @@
|
||||
|
||||
An MCP server that exposes tools for executing commands across a Turnstone cluster. Serves as a reference implementation for both MCP server patterns and Turnstone SDK usage.
|
||||
|
||||
> [!NOTE]
|
||||
> **Superseded by the built-in coordinator workstream in Turnstone 1.5.**
|
||||
>
|
||||
> This MCP side-car is the pre-1.5 pattern for cluster-wide orchestration.
|
||||
> Turnstone 1.5 promotes coordinator behaviour to a first-class workstream
|
||||
> kind hosted inside `turnstone-console` — no external MCP server to
|
||||
> install or operate, proper per-user audit attribution, and a dedicated
|
||||
> UI at `/coordinator/{ws_id}`.
|
||||
>
|
||||
> The extension continues to work for 1.4-and-earlier clusters. On 1.5+:
|
||||
> grant the `admin.coordinator` permission, set `coordinator.model_alias`
|
||||
> in the admin Settings tab, and create sessions via the dashboard's
|
||||
> "new coordinator" button or `POST /v1/api/coordinator/new`. Full
|
||||
> removal of this example (including docker / compose references) is
|
||||
> planned once 1.5 is confirmed in production.
|
||||
>
|
||||
> | Concern | Built-in coordinator (1.5+) | This MCP extension (1.4-and-earlier) |
|
||||
> |---|---|---|
|
||||
> | Install | None — shipped in-tree | `pip install -e examples/mcp-cluster-ops` + MCP client config |
|
||||
> | Auth | Real creator's `user_id` + `admin.coordinator` permission | Shared service token |
|
||||
> | Audit | `coordinator.create` / `close` / `cancel` events on the console; `src="coordinator"` preserved on upstream hops | Service identity only |
|
||||
> | UI | `/coordinator/{ws_id}` one-pane HTML | No UI — model-only |
|
||||
> | Tool approvals | Inline approval bar in the coordinator pane | MCP approval flow |
|
||||
> | Configuration | `coordinator.model_alias`, `coordinator.max_active`, `coordinator.reasoning_effort`, `coordinator.session_jwt_ttl_seconds` | MCP server config file |
|
||||
>
|
||||
> Minimal 1.5 migration:
|
||||
>
|
||||
> ```bash
|
||||
> curl -X POST https://console.example/v1/api/coordinator/new \
|
||||
> -H "Authorization: Bearer $TOKEN" \
|
||||
> -H "Content-Type: application/json" \
|
||||
> -d '{"name":"planner","initial_message":"Spawn a worker to check the build"}'
|
||||
> ```
|
||||
>
|
||||
> The response carries `ws_id`; open
|
||||
> `https://console.example/coordinator/{ws_id}` to watch the session.
|
||||
|
||||
## How it works
|
||||
|
||||
This server uses the Turnstone console SDK (`TurnstoneConsole`) for node discovery and routing, and `TurnstoneServer` for per-node SSE streaming. The dispatch flow for each command is:
|
||||
|
||||
1. **Route** — `TurnstoneConsole.route_create_workstream(target_node=..., auto_approve=True)` creates a workstream pinned to the target node via the console's rendezvous routing proxy, returning `ws_id` and `node_url`.
|
||||
1. **Route** — `TurnstoneConsole.route_create_workstream(target_node=..., auto_approve=True)` creates a workstream pinned to the target node via the console's hash-ring routing proxy, returning `ws_id` and `node_url`.
|
||||
2. **Execute** — `TurnstoneServer(node_url, token=...)` connects directly to the node's SSE stream using the same `TURNSTONE_API_TOKEN`. `send_and_wait(prompt, ws_id)` runs the command and the raw bash output is captured from the `ToolResultEvent` — bypassing the costly "agent reads output then re-generates output as completion tokens" round-trip.
|
||||
3. **Cleanup** — `TurnstoneConsole.route_close(ws_id)` closes the workstream.
|
||||
|
||||
|
||||
+1
-4
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "1.5.3"
|
||||
version = "1.4.0"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
@@ -76,9 +76,6 @@ include = [
|
||||
"turnstone/console/static/*.html",
|
||||
"turnstone/console/static/*.css",
|
||||
"turnstone/console/static/*.js",
|
||||
"turnstone/console/static/coordinator/*.html",
|
||||
"turnstone/console/static/coordinator/*.css",
|
||||
"turnstone/console/static/coordinator/*.js",
|
||||
"turnstone/shared_static/*.css",
|
||||
"turnstone/shared_static/*.js",
|
||||
"turnstone/shared_static/katex-0.16.45/**/*",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+63
-1358
File diff suppressed because it is too large
Load Diff
Generated
+142
-142
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@turnstone/sdk",
|
||||
"version": "0.4.0",
|
||||
"version": "0.3.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@turnstone/sdk",
|
||||
"version": "0.4.0",
|
||||
"version": "0.3.0",
|
||||
"license": "BUSL-1.1",
|
||||
"devDependencies": {
|
||||
"typescript": "^6.0.0",
|
||||
@@ -14,9 +14,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/core": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz",
|
||||
"integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==",
|
||||
"version": "1.9.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz",
|
||||
"integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
@@ -26,9 +26,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/runtime": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz",
|
||||
"integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==",
|
||||
"version": "1.9.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz",
|
||||
"integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
@@ -74,9 +74,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-project/types": {
|
||||
"version": "0.127.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz",
|
||||
"integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==",
|
||||
"version": "0.124.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz",
|
||||
"integrity": "sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
@@ -84,9 +84,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-android-arm64": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-s70pVGhw4zqGeFnXWvAzJDlvxhlRollagdCCKRgOsgUOH3N1l0LIxf83AtGzmb5SiVM4Hjl5HyarMRfdfj3DaQ==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -101,9 +101,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-darwin-arm64": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-4ksWc9n0mhlZpZ9PMZgTGjeOPRu8MB1Z3Tz0Mo02eWfWCHMW1zN82Qz/pL/rC+yQa+8ZnutMF0JjJe7PjwasYw==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -118,9 +118,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-darwin-x64": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-SUSDOI6WwUVNcWxd02QEBjLdY1VPHvlEkw6T/8nYG322iYWCTxRb1vzk4E+mWWYehTp7ERibq54LSJGjmouOsw==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -135,9 +135,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-freebsd-x64": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-hwnz3nw9dbJ05EDO/PvcjaaewqqDy7Y1rn1UO81l8iIK1GjenME75dl16ajbvSSMfv66WXSRCYKIqfgq2KCfxw==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -152,9 +152,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-arm-gnueabihf": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-IS+W7epTcwANmFSQFrS1SivEXHtl1JtuQA9wlxrZTcNi6mx+FDOYrakGevvvTwgj2JvWiK8B29/qD9BELZPyXQ==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -169,9 +169,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-arm64-gnu": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-e6usGaHKW5BMNZOymS1UcEYGowQMWcgZ71Z17Sl/h2+ZziNJ1a9n3Zvcz6LdRyIW5572wBCTH/Z+bKuZouGk9Q==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -189,9 +189,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-arm64-musl": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-b/CgbwAJpmrRLp02RPfhbudf5tZnN9nsPWK82znefso832etkem8H7FSZwxrOI9djcdTP7U6YfNhbRnh7djErg==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -209,9 +209,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-ppc64-gnu": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-4EII1iNGRUN5WwGbF/kOh/EIkoDN9HsupgLQoXfY+D1oyJm7/F4t5PYU5n8SWZgG0FEwakyM8pGgwcBYruGTlA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -229,9 +229,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-s390x-gnu": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-AH8oq3XqQo4IibpVXvPeLDI5pzkpYn0WiZAfT05kFzoJ6tQNzwRdDYQ45M8I/gslbodRZwW8uxLhbSBbkv96rA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -249,9 +249,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-x64-gnu": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-cLnjV3xfo7KslbU41Z7z8BH/E1y5mzUYzAqih1d1MDaIGZRCMqTijqLv76/P7fyHuvUcfGsIpqCdddbxLLK9rA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -269,9 +269,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-linux-x64-musl": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-0phclDw1spsL7dUB37sIARuis2tAgomCJXAHZlpt8PXZ4Ba0dRP1e+66lsRqrfhISeN9bEGNjQs+T/Fbd7oYGw==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -289,9 +289,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-openharmony-arm64": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-0ag/hEgXOwgw4t8QyQvUCxvEg+V0KBcA6YuOx9g0r02MprutRF5dyljgm3EmR02O292UX7UeS6HzWHAl6KgyhA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -306,9 +306,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-wasm32-wasi": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-LEXei6vo0E5wTGwpkJ4KoT3OZJRnglwldt5ziLzOlc6qqb55z4tWNq2A+PFqCJuvWWdP53CVhG1Z9NtToDPJrA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==",
|
||||
"cpu": [
|
||||
"wasm32"
|
||||
],
|
||||
@@ -316,18 +316,18 @@
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/core": "1.10.0",
|
||||
"@emnapi/runtime": "1.10.0",
|
||||
"@napi-rs/wasm-runtime": "^1.1.4"
|
||||
"@emnapi/core": "1.9.2",
|
||||
"@emnapi/runtime": "1.9.2",
|
||||
"@napi-rs/wasm-runtime": "^1.1.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-win32-arm64-msvc": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-gUmyzBl3SPMa6hrqFUth9sVfcLBlYsbMzBx5PlexMroZStgzGqlZ26pYG89rBb45Mnia+oil6YAIFeEWGWhoZA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -342,9 +342,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/binding-win32-x64-msvc": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-3hkiolcUAvPB9FLb3UZdfjVVNWherN1f/skkGWJP/fgSQhYUZpSIRr0/I8ZK9TkF3F7kxvJAk0+IcKvPHk9qQg==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -359,9 +359,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/pluginutils": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-n8iosDOt6Ig1UhJ2AYqoIhHWh/isz0xpicHTzpKBeotdVsTEcxsSA/i3EVM7gQAj0rU27OLAxCjzlj15IWY7bg==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
@@ -409,16 +409,16 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@vitest/expect": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz",
|
||||
"integrity": "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.4.tgz",
|
||||
"integrity": "sha512-iPBpra+VDuXmBFI3FMKHSFXp3Gx5HfmSCE8X67Dn+bwephCnQCaB7qWK2ldHa+8ncN8hJU8VTMcxjPpyMkUjww==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@standard-schema/spec": "^1.1.0",
|
||||
"@types/chai": "^5.2.2",
|
||||
"@vitest/spy": "4.1.5",
|
||||
"@vitest/utils": "4.1.5",
|
||||
"@vitest/spy": "4.1.4",
|
||||
"@vitest/utils": "4.1.4",
|
||||
"chai": "^6.2.2",
|
||||
"tinyrainbow": "^3.1.0"
|
||||
},
|
||||
@@ -427,13 +427,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/mocker": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.5.tgz",
|
||||
"integrity": "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.4.tgz",
|
||||
"integrity": "sha512-R9HTZBhW6yCSGbGQnDnH3QHfJxokKN4KB+Yvk9Q1le7eQNYwiCyKxmLmurSpFy6BzJanSLuEUDrD+j97Q+ZLPg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/spy": "4.1.5",
|
||||
"@vitest/spy": "4.1.4",
|
||||
"estree-walker": "^3.0.3",
|
||||
"magic-string": "^0.30.21"
|
||||
},
|
||||
@@ -454,9 +454,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/pretty-format": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.5.tgz",
|
||||
"integrity": "sha512-7I3q6l5qr03dVfMX2wCo9FxwSJbPdwKjy2uu/YPpU3wfHvIL4QHwVRp57OfGrDFeUJ8/8QdfBKIV12FTtLn00g==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.4.tgz",
|
||||
"integrity": "sha512-ddmDHU0gjEUyEVLxtZa7xamrpIefdEETu3nZjWtHeZX4QxqJ7tRxSteHVXJOcr8jhiLoGAhkK4WJ3WqBpjx42A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -467,13 +467,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/runner": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.5.tgz",
|
||||
"integrity": "sha512-2D+o7Pr82IEO46YPpoA/YU0neeyr6FTerQb5Ro7BUnBuv6NQtT/kmVnczngiMEBhzgqz2UZYl5gArejsyERDSQ==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.4.tgz",
|
||||
"integrity": "sha512-xTp7VZ5aXP5ZJrn15UtJUWlx6qXLnGtF6jNxHepdPHpMfz/aVPx+htHtgcAL2mDXJgKhpoo2e9/hVJsIeFbytQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/utils": "4.1.5",
|
||||
"@vitest/utils": "4.1.4",
|
||||
"pathe": "^2.0.3"
|
||||
},
|
||||
"funding": {
|
||||
@@ -481,14 +481,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/snapshot": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.5.tgz",
|
||||
"integrity": "sha512-zypXEt4KH/XgKGPUz4eC2AvErYx0My5hfL8oDb1HzGFpEk1P62bxSohdyOmvz+d9UJwanI68MKwr2EquOaOgMQ==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.4.tgz",
|
||||
"integrity": "sha512-MCjCFgaS8aZz+m5nTcEcgk/xhWv0rEH4Yl53PPlMXOZ1/Ka2VcZU6CJ+MgYCZbcJvzGhQRjVrGQNZqkGPttIKw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/pretty-format": "4.1.5",
|
||||
"@vitest/utils": "4.1.5",
|
||||
"@vitest/pretty-format": "4.1.4",
|
||||
"@vitest/utils": "4.1.4",
|
||||
"magic-string": "^0.30.21",
|
||||
"pathe": "^2.0.3"
|
||||
},
|
||||
@@ -497,9 +497,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/spy": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.5.tgz",
|
||||
"integrity": "sha512-2lNOsh6+R2Idnf1TCZqSwYlKN2E/iDlD8sgU59kYVl+OMDmvldO1VDk39smRfpUNwYpNRVn3w4YfuC7KfbBnkQ==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.4.tgz",
|
||||
"integrity": "sha512-XxNdAsKW7C+FLydqFJLb5KhJtl3PGCMmYwFRfhvIgxJvLSXhhVI1zM8f1qD3Zg7RCjTSzDVyct6sghs9UEgBEQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
@@ -507,13 +507,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/utils": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.5.tgz",
|
||||
"integrity": "sha512-76wdkrmfXfqGjueGgnb45ITPyUi1ycZ4IHgC2bhPDUfWHklY/q3MdLOAB+TF1e6xfl8NxNY0ZYaPCFNWSsw3Ug==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.4.tgz",
|
||||
"integrity": "sha512-13QMT+eysM5uVGa1rG4kegGYNp6cnQcsTc67ELFbhNLQO+vgsygtYJx2khvdt4gVQqSSpC/KT5FZZxUpP3Oatw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/pretty-format": "4.1.5",
|
||||
"@vitest/pretty-format": "4.1.4",
|
||||
"convert-source-map": "^2.0.0",
|
||||
"tinyrainbow": "^3.1.0"
|
||||
},
|
||||
@@ -559,9 +559,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/es-module-lexer": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz",
|
||||
"integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==",
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz",
|
||||
"integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
@@ -959,9 +959,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.12",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.12.tgz",
|
||||
"integrity": "sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==",
|
||||
"version": "8.5.10",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz",
|
||||
"integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -988,14 +988,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/rolldown": {
|
||||
"version": "1.0.0-rc.17",
|
||||
"resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.17.tgz",
|
||||
"integrity": "sha512-ZrT53oAKrtA4+YtBWPQbtPOxIbVDbxT0orcYERKd63VJTF13zPcgXTvD4843L8pcsI7M6MErt8QtON6lrB9tyA==",
|
||||
"version": "1.0.0-rc.15",
|
||||
"resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz",
|
||||
"integrity": "sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@oxc-project/types": "=0.127.0",
|
||||
"@rolldown/pluginutils": "1.0.0-rc.17"
|
||||
"@oxc-project/types": "=0.124.0",
|
||||
"@rolldown/pluginutils": "1.0.0-rc.15"
|
||||
},
|
||||
"bin": {
|
||||
"rolldown": "bin/cli.mjs"
|
||||
@@ -1004,21 +1004,21 @@
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@rolldown/binding-android-arm64": "1.0.0-rc.17",
|
||||
"@rolldown/binding-darwin-arm64": "1.0.0-rc.17",
|
||||
"@rolldown/binding-darwin-x64": "1.0.0-rc.17",
|
||||
"@rolldown/binding-freebsd-x64": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-arm64-musl": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-x64-gnu": "1.0.0-rc.17",
|
||||
"@rolldown/binding-linux-x64-musl": "1.0.0-rc.17",
|
||||
"@rolldown/binding-openharmony-arm64": "1.0.0-rc.17",
|
||||
"@rolldown/binding-wasm32-wasi": "1.0.0-rc.17",
|
||||
"@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.17",
|
||||
"@rolldown/binding-win32-x64-msvc": "1.0.0-rc.17"
|
||||
"@rolldown/binding-android-arm64": "1.0.0-rc.15",
|
||||
"@rolldown/binding-darwin-arm64": "1.0.0-rc.15",
|
||||
"@rolldown/binding-darwin-x64": "1.0.0-rc.15",
|
||||
"@rolldown/binding-freebsd-x64": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-arm64-musl": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-x64-gnu": "1.0.0-rc.15",
|
||||
"@rolldown/binding-linux-x64-musl": "1.0.0-rc.15",
|
||||
"@rolldown/binding-openharmony-arm64": "1.0.0-rc.15",
|
||||
"@rolldown/binding-wasm32-wasi": "1.0.0-rc.15",
|
||||
"@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.15",
|
||||
"@rolldown/binding-win32-x64-msvc": "1.0.0-rc.15"
|
||||
}
|
||||
},
|
||||
"node_modules/siginfo": {
|
||||
@@ -1105,9 +1105,9 @@
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz",
|
||||
"integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==",
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.2.tgz",
|
||||
"integrity": "sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
@@ -1119,17 +1119,17 @@
|
||||
}
|
||||
},
|
||||
"node_modules/vite": {
|
||||
"version": "8.0.10",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-8.0.10.tgz",
|
||||
"integrity": "sha512-rZuUu9j6J5uotLDs+cAA4O5H4K1SfPliUlQwqa6YEwSrWDZzP4rhm00oJR5snMewjxF5V/K3D4kctsUTsIU9Mw==",
|
||||
"version": "8.0.8",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-8.0.8.tgz",
|
||||
"integrity": "sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"lightningcss": "^1.32.0",
|
||||
"picomatch": "^4.0.4",
|
||||
"postcss": "^8.5.10",
|
||||
"rolldown": "1.0.0-rc.17",
|
||||
"tinyglobby": "^0.2.16"
|
||||
"postcss": "^8.5.8",
|
||||
"rolldown": "1.0.0-rc.15",
|
||||
"tinyglobby": "^0.2.15"
|
||||
},
|
||||
"bin": {
|
||||
"vite": "bin/vite.js"
|
||||
@@ -1197,19 +1197,19 @@
|
||||
}
|
||||
},
|
||||
"node_modules/vitest": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.5.tgz",
|
||||
"integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==",
|
||||
"version": "4.1.4",
|
||||
"resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.4.tgz",
|
||||
"integrity": "sha512-tFuJqTxKb8AvfyqMfnavXdzfy3h3sWZRWwfluGbkeR7n0HUev+FmNgZ8SDrRBTVrVCjgH5cA21qGbCffMNtWvg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/expect": "4.1.5",
|
||||
"@vitest/mocker": "4.1.5",
|
||||
"@vitest/pretty-format": "4.1.5",
|
||||
"@vitest/runner": "4.1.5",
|
||||
"@vitest/snapshot": "4.1.5",
|
||||
"@vitest/spy": "4.1.5",
|
||||
"@vitest/utils": "4.1.5",
|
||||
"@vitest/expect": "4.1.4",
|
||||
"@vitest/mocker": "4.1.4",
|
||||
"@vitest/pretty-format": "4.1.4",
|
||||
"@vitest/runner": "4.1.4",
|
||||
"@vitest/snapshot": "4.1.4",
|
||||
"@vitest/spy": "4.1.4",
|
||||
"@vitest/utils": "4.1.4",
|
||||
"es-module-lexer": "^2.0.0",
|
||||
"expect-type": "^1.3.0",
|
||||
"magic-string": "^0.30.21",
|
||||
@@ -1237,12 +1237,12 @@
|
||||
"@edge-runtime/vm": "*",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0",
|
||||
"@vitest/browser-playwright": "4.1.5",
|
||||
"@vitest/browser-preview": "4.1.5",
|
||||
"@vitest/browser-webdriverio": "4.1.5",
|
||||
"@vitest/coverage-istanbul": "4.1.5",
|
||||
"@vitest/coverage-v8": "4.1.5",
|
||||
"@vitest/ui": "4.1.5",
|
||||
"@vitest/browser-playwright": "4.1.4",
|
||||
"@vitest/browser-preview": "4.1.4",
|
||||
"@vitest/browser-webdriverio": "4.1.4",
|
||||
"@vitest/coverage-istanbul": "4.1.4",
|
||||
"@vitest/coverage-v8": "4.1.4",
|
||||
"@vitest/ui": "4.1.4",
|
||||
"happy-dom": "*",
|
||||
"jsdom": "*",
|
||||
"vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@turnstone/sdk",
|
||||
"version": "0.4.0",
|
||||
"version": "0.3.0",
|
||||
"description": "TypeScript client SDK for the turnstone AI orchestration platform",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -146,7 +146,7 @@ export class TurnstoneConsole extends BaseClient {
|
||||
// -- Routing proxy --------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Create a workstream via the console rendezvous router.
|
||||
* Create a workstream via the console hash-ring router.
|
||||
*
|
||||
* When `attachments` is non-empty the request is sent as
|
||||
* multipart/form-data and the console routes via `?ws_id=<hex>`
|
||||
|
||||
@@ -93,17 +93,10 @@ export class TurnstoneServer extends BaseClient {
|
||||
});
|
||||
}
|
||||
|
||||
async closeWorkstream(
|
||||
wsId: string,
|
||||
opts?: { reason?: string },
|
||||
): Promise<StatusResponse> {
|
||||
const body: Record<string, unknown> = {};
|
||||
if (opts?.reason !== undefined) body.reason = opts.reason;
|
||||
return this.request(
|
||||
"POST",
|
||||
`/v1/api/workstreams/${encodeURIComponent(wsId)}/close`,
|
||||
{ json: body },
|
||||
);
|
||||
async closeWorkstream(wsId: string): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/workstreams/close", {
|
||||
json: { ws_id: wsId },
|
||||
});
|
||||
}
|
||||
|
||||
// -- Chat interaction -----------------------------------------------------
|
||||
@@ -113,15 +106,11 @@ export class TurnstoneServer extends BaseClient {
|
||||
wsId: string,
|
||||
opts?: { attachmentIds?: string[] },
|
||||
): Promise<SendResponse> {
|
||||
const body: Record<string, unknown> = { message };
|
||||
const body: Record<string, unknown> = { message, ws_id: wsId };
|
||||
if (opts?.attachmentIds !== undefined) {
|
||||
body.attachment_ids = opts.attachmentIds;
|
||||
}
|
||||
return this.request(
|
||||
"POST",
|
||||
`/v1/api/workstreams/${encodeURIComponent(wsId)}/send`,
|
||||
{ json: body },
|
||||
);
|
||||
return this.request("POST", "/v1/api/send", { json: body });
|
||||
}
|
||||
|
||||
// -- Attachments ----------------------------------------------------------
|
||||
@@ -167,17 +156,14 @@ export class TurnstoneServer extends BaseClient {
|
||||
feedback?: string | null;
|
||||
always?: boolean;
|
||||
}): Promise<StatusResponse> {
|
||||
return this.request(
|
||||
"POST",
|
||||
`/v1/api/workstreams/${encodeURIComponent(opts.wsId)}/approve`,
|
||||
{
|
||||
json: {
|
||||
approved: opts.approved ?? true,
|
||||
feedback: opts.feedback,
|
||||
always: opts.always,
|
||||
},
|
||||
return this.request("POST", "/v1/api/approve", {
|
||||
json: {
|
||||
ws_id: opts.wsId,
|
||||
approved: opts.approved ?? true,
|
||||
feedback: opts.feedback,
|
||||
always: opts.always,
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async planFeedback(opts: {
|
||||
@@ -202,21 +188,15 @@ export class TurnstoneServer extends BaseClient {
|
||||
wsId: string,
|
||||
opts?: { force?: boolean },
|
||||
): Promise<StatusResponse> {
|
||||
const body: Record<string, unknown> = {};
|
||||
const body: Record<string, unknown> = { ws_id: wsId };
|
||||
if (opts?.force) body.force = true;
|
||||
return this.request(
|
||||
"POST",
|
||||
`/v1/api/workstreams/${encodeURIComponent(wsId)}/cancel`,
|
||||
{ json: body },
|
||||
);
|
||||
return this.request("POST", "/v1/api/cancel", { json: body });
|
||||
}
|
||||
|
||||
// -- Streaming ------------------------------------------------------------
|
||||
|
||||
async *streamEvents(wsId: string): AsyncIterableIterator<ServerEvent> {
|
||||
yield* this.streamSSE<ServerEvent>(
|
||||
`/v1/api/workstreams/${encodeURIComponent(wsId)}/events`,
|
||||
);
|
||||
yield* this.streamSSE<ServerEvent>("/v1/api/events", { ws_id: wsId });
|
||||
}
|
||||
|
||||
async *streamGlobalEvents(): AsyncIterableIterator<ServerEvent> {
|
||||
@@ -256,8 +236,8 @@ export class TurnstoneServer extends BaseClient {
|
||||
try {
|
||||
// Start consuming the per-workstream SSE stream first
|
||||
const events = this.streamSSE<ServerEvent>(
|
||||
`/v1/api/workstreams/${encodeURIComponent(wsId)}/events`,
|
||||
undefined,
|
||||
"/v1/api/events",
|
||||
{ ws_id: wsId },
|
||||
controller.signal,
|
||||
);
|
||||
|
||||
|
||||
@@ -162,51 +162,21 @@ export interface CreateWorkstreamResponse {
|
||||
}
|
||||
|
||||
export interface CloseWorkstreamRequest {
|
||||
/**
|
||||
* Optional close reason persisted to `workstream_config` for
|
||||
* postmortem. Capped at 512 UTF-8 bytes server-side; credential
|
||||
* redaction is applied via the output guard.
|
||||
*/
|
||||
reason?: string;
|
||||
ws_id: string;
|
||||
}
|
||||
|
||||
export interface WorkstreamInfo {
|
||||
// Renamed `id` → `ws_id` and added kind/parent_ws_id/user_id in
|
||||
// the Stage 2 list-verb lift. Pre-1.5 readers branching on
|
||||
// `row.id` should swap to `row.ws_id`.
|
||||
ws_id: string;
|
||||
id: string;
|
||||
name: string;
|
||||
state: string;
|
||||
kind: string;
|
||||
parent_ws_id: string | null;
|
||||
user_id: string;
|
||||
}
|
||||
|
||||
export interface ListWorkstreamsResponse {
|
||||
workstreams: WorkstreamInfo[];
|
||||
}
|
||||
|
||||
export interface WorkstreamDetailResponse {
|
||||
// Lifted from coord-only into a shared verb in the Stage 2
|
||||
// history/detail verb lift. Both kinds populate every field; SDK
|
||||
// consumers don't branch on kind.
|
||||
ws_id: string;
|
||||
name: string;
|
||||
state: string;
|
||||
user_id: string;
|
||||
kind: string;
|
||||
}
|
||||
|
||||
export interface WorkstreamHistoryResponse {
|
||||
ws_id: string;
|
||||
// Tail of the workstream's reconstructed message history
|
||||
// (provider-fidelity OpenAI-like shape). Bounded by the ?limit=
|
||||
// query param (default 100, max 500).
|
||||
messages: Record<string, unknown>[];
|
||||
}
|
||||
|
||||
export interface DashboardWorkstream {
|
||||
ws_id: string;
|
||||
id: string;
|
||||
name: string;
|
||||
state: string;
|
||||
title?: string;
|
||||
@@ -982,7 +952,7 @@ export interface SkillDiscoverListing {
|
||||
install_count: number;
|
||||
tags: string[];
|
||||
installed: boolean;
|
||||
risk_level?: string;
|
||||
scan_status?: string;
|
||||
template_id?: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,16 +26,7 @@ function mockFetchError(
|
||||
describe("TurnstoneServer", () => {
|
||||
it("listWorkstreams returns parsed response", async () => {
|
||||
const fetchFn = mockFetch({
|
||||
workstreams: [
|
||||
{
|
||||
ws_id: "ws1",
|
||||
name: "test",
|
||||
state: "idle",
|
||||
kind: "interactive",
|
||||
parent_ws_id: null,
|
||||
user_id: "u1",
|
||||
},
|
||||
],
|
||||
workstreams: [{ id: "ws1", name: "test", state: "idle" }],
|
||||
});
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://test",
|
||||
@@ -43,9 +34,7 @@ describe("TurnstoneServer", () => {
|
||||
});
|
||||
const resp = await client.listWorkstreams();
|
||||
expect(resp.workstreams).toHaveLength(1);
|
||||
// Row key renamed id → ws_id in the Stage 2 list-verb lift.
|
||||
expect(resp.workstreams[0].ws_id).toBe("ws1");
|
||||
expect(resp.workstreams[0].kind).toBe("interactive");
|
||||
expect(resp.workstreams[0].id).toBe("ws1");
|
||||
expect(fetchFn).toHaveBeenCalledWith(
|
||||
"http://test/v1/api/workstreams",
|
||||
expect.objectContaining({ method: "GET" }),
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
"""Shared builders for the coordinator-endpoint test files.
|
||||
|
||||
The four coordinator test modules each ship a copy of the same
|
||||
``_AuthMiddleware`` / ``_FakeConfigStore`` / ``_fake_registry`` /
|
||||
``_build_mgr`` helpers — this module is the single home for them so
|
||||
future edits land once. Named with a leading underscore so pytest
|
||||
does not collect it.
|
||||
|
||||
``_make_client`` stays local to each test module because the route
|
||||
list differs per file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.coordinator_adapter import CoordinatorAdapter
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
from turnstone.core.auth import AuthResult
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
def _seed_children(
|
||||
adapter: CoordinatorAdapter, coord_ws_id: str, child_ws_ids: Iterable[str]
|
||||
) -> None:
|
||||
"""Seed the coordinator adapter's children registry directly.
|
||||
|
||||
The production path populates the registry via the cluster-event
|
||||
fan-out thread observing ``ws_created`` events. These tests just
|
||||
need a known-children set for the endpoint handlers to iterate —
|
||||
inject directly under ``_children_lock`` rather than spinning up
|
||||
the collector + fan-out plumbing.
|
||||
"""
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked(coord_ws_id, child_ws_ids)
|
||||
|
||||
|
||||
class _AuthMiddleware(BaseHTTPMiddleware):
|
||||
"""Inject a configurable AuthResult from a header-based contract.
|
||||
|
||||
Tests set ``X-Test-Perms`` to a comma-separated permission list, and
|
||||
``X-Test-User`` to the user id. Empty or missing → no auth.
|
||||
"""
|
||||
|
||||
async def dispatch(self, request, call_next): # type: ignore[no-untyped-def]
|
||||
perms = request.headers.get("X-Test-Perms", "")
|
||||
user_id = request.headers.get("X-Test-User", "")
|
||||
if perms or user_id:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id=user_id,
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="test",
|
||||
permissions=frozenset(p for p in perms.split(",") if p),
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
class _FakeConfigStore:
|
||||
"""Minimal ConfigStore stub — returns values from a dict."""
|
||||
|
||||
def __init__(self, values: dict[str, Any]) -> None:
|
||||
self._values = values
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
return self._values.get(key, default)
|
||||
|
||||
|
||||
def _fake_registry() -> MagicMock:
|
||||
"""MagicMock whose ``.resolve()`` succeeds so the 503 gate passes."""
|
||||
reg = MagicMock()
|
||||
reg.resolve.return_value = (MagicMock(), "gpt-4", MagicMock())
|
||||
return reg
|
||||
|
||||
|
||||
def _build_mgr_with_factory(storage: Any, session_factory: Any) -> SessionManager:
|
||||
"""Build a SessionManager(CoordinatorAdapter) with a caller-supplied factory.
|
||||
|
||||
Used by tests that need to capture or assert factory kwargs (e.g.
|
||||
per-call ``model`` / ``judge_model`` overrides). Plain :func:`_build_mgr`
|
||||
is the right entry point when the test doesn't care about the
|
||||
factory.
|
||||
"""
|
||||
adapter = CoordinatorAdapter(
|
||||
collector=MagicMock(),
|
||||
ui_factory=lambda ws: ConsoleCoordinatorUI(ws_id=ws.id, user_id=ws.user_id or ""),
|
||||
session_factory=session_factory,
|
||||
)
|
||||
mgr = SessionManager(
|
||||
adapter,
|
||||
storage=storage,
|
||||
max_active=3,
|
||||
node_id=ClusterCollector.CONSOLE_PSEUDO_NODE_ID,
|
||||
event_emitter=adapter,
|
||||
)
|
||||
adapter.attach(mgr)
|
||||
return mgr
|
||||
|
||||
|
||||
def _build_mgr(storage: Any) -> SessionManager:
|
||||
"""Build a SessionManager(CoordinatorAdapter) with stub factories (test default)."""
|
||||
|
||||
def _sf(ui, model_alias=None, ws_id=None, **kw): # type: ignore[no-untyped-def]
|
||||
s = MagicMock()
|
||||
s.send.return_value = None
|
||||
return s
|
||||
|
||||
return _build_mgr_with_factory(storage, _sf)
|
||||
|
||||
|
||||
class MockStorage:
|
||||
"""Minimal storage mock that implements ``list_services``.
|
||||
|
||||
Used by the collector tests + the console route-walk tests. The
|
||||
collector calls ``list_services("turnstone-server", ...)`` to
|
||||
discover nodes; tests that don't care about discovery push an
|
||||
empty list (the default).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.services: list[dict[str, str]] = []
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
return list(self.services)
|
||||
@@ -1,58 +0,0 @@
|
||||
"""Shared mock factory for ``events_replay`` tests.
|
||||
|
||||
Both interactive (:func:`turnstone.server._interactive_events_replay`)
|
||||
and coord (:func:`turnstone.console.server._coord_events_replay`) drive
|
||||
the same shared preamble at
|
||||
:func:`turnstone.core.session_replay.session_replay_preamble`. Their
|
||||
test suites share the underlying mock surface (session.model,
|
||||
session.model_alias, session._last_usage, ui._pending_*, ui._ws_lock,
|
||||
counters); this module is the single home for that shape so a future
|
||||
field add lands once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
||||
def make_replay_mocks(
|
||||
*,
|
||||
last_usage: dict[str, Any] | None = None,
|
||||
**ui_overrides: Any,
|
||||
) -> tuple[Any, Any, Any]:
|
||||
"""Build ``(ws, ui, request)`` MagicMocks for events-replay tests.
|
||||
|
||||
Defaults match a fresh workstream that hasn't completed a turn
|
||||
(no ``last_usage``, no pending prompts).
|
||||
|
||||
Args:
|
||||
last_usage: Sets ``ws.session._last_usage`` directly so tests
|
||||
don't have to reach into the nested mock; when ``None``
|
||||
(default), the status replay branch stays inert.
|
||||
**ui_overrides: Additional attributes set directly on the ``ui``
|
||||
mock (e.g. ``_pending_approval``, ``_pending_plan_review``,
|
||||
``_llm_verdicts``, ``_ws_turn_tool_calls``, ``_ws_messages``).
|
||||
"""
|
||||
session = MagicMock()
|
||||
session.model = "gpt-5"
|
||||
session.model_alias = "default"
|
||||
session._last_usage = last_usage
|
||||
session.context_window = 100000
|
||||
session.reasoning_effort = "medium"
|
||||
session.messages = []
|
||||
ui = MagicMock()
|
||||
ui.auto_approve = False
|
||||
ui._pending_approval = None
|
||||
ui._pending_plan_review = None
|
||||
ui._llm_verdicts = {}
|
||||
ui._ws_lock = threading.Lock()
|
||||
ui._ws_turn_tool_calls = 0
|
||||
ui._ws_messages = 0
|
||||
for key, value in ui_overrides.items():
|
||||
setattr(ui, key, value)
|
||||
ws = MagicMock()
|
||||
ws.session = session
|
||||
request = MagicMock()
|
||||
return ws, ui, request
|
||||
@@ -95,21 +95,3 @@ def mock_openai_client():
|
||||
client = MagicMock()
|
||||
client.models.list.return_value.data = [MagicMock(id="test-model")]
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_policy_cache():
|
||||
"""Drop the in-process tool-policy cache between tests.
|
||||
|
||||
The cache is keyed by org_id (default ``""``), so without this
|
||||
autouse hook a policy created in test A would leak into test B's
|
||||
``evaluate_tool_policy`` call — distinct storage instances, same
|
||||
cache slot. Production singleton storage doesn't see the leak
|
||||
because there's only one storage instance for the process lifetime;
|
||||
the test isolation requirement is what motivates the autouse.
|
||||
"""
|
||||
from turnstone.core.policy import invalidate_policy_cache
|
||||
|
||||
invalidate_policy_cache()
|
||||
yield
|
||||
invalidate_policy_cache()
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
"""Console-side coord_registry auto-refresh on model-definition CRUD + reload.
|
||||
|
||||
The console builds ``app.state.coord_registry`` once at lifespan startup
|
||||
and the coordinator session factory closes over that exact instance.
|
||||
Without these refresh hooks, an admin who edits a model definition
|
||||
through the UI sees the DB change immediately but coordinator sessions
|
||||
keep calling the prior model name — the on-disk truth diverges from the
|
||||
in-process registry until the console is restarted.
|
||||
|
||||
These tests cover both the helper (``_refresh_coord_registry``)
|
||||
and the four wired endpoints (create / update / delete / explicit reload)
|
||||
to lock in:
|
||||
|
||||
- in-place mutation: ``coord_registry`` object identity is preserved
|
||||
across refreshes (factory closure must not be invalidated);
|
||||
- failure isolation: a load or reload failure leaves the existing
|
||||
registry intact rather than tearing down a working coordinator;
|
||||
- no-op safety: the helper short-circuits when ``coord_registry`` is
|
||||
``None`` so a coord-less console (no model rows at boot) doesn't
|
||||
500 on routine model-definition CRUD.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from tests._coord_test_helpers import _AuthMiddleware
|
||||
from turnstone.console.server import (
|
||||
_refresh_coord_registry,
|
||||
admin_create_model_definition,
|
||||
admin_delete_model_definition,
|
||||
admin_model_reload,
|
||||
admin_update_model_definition,
|
||||
)
|
||||
from turnstone.core.model_registry import ModelConfig, ModelRegistry
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path: Any) -> SQLiteBackend:
|
||||
return SQLiteBackend(str(tmp_path / "models.db"))
|
||||
|
||||
|
||||
def _seed_model_def(
|
||||
storage: SQLiteBackend,
|
||||
*,
|
||||
definition_id: str,
|
||||
alias: str,
|
||||
model: str,
|
||||
base_url: str = "http://localhost:8000/v1",
|
||||
enabled: bool = True,
|
||||
) -> None:
|
||||
"""Insert a model definition row directly via the storage API."""
|
||||
storage.create_model_definition(
|
||||
definition_id=definition_id,
|
||||
alias=alias,
|
||||
model=model,
|
||||
provider="openai-compatible",
|
||||
base_url=base_url,
|
||||
api_key="sk-test",
|
||||
context_window=8192,
|
||||
capabilities="{}",
|
||||
enabled=enabled,
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
|
||||
def _make_config(alias: str, model: str) -> ModelConfig:
|
||||
return ModelConfig(
|
||||
alias=alias,
|
||||
base_url="http://localhost:8000/v1",
|
||||
api_key="sk-test",
|
||||
model=model,
|
||||
context_window=8192,
|
||||
provider="openai-compatible",
|
||||
source="db",
|
||||
)
|
||||
|
||||
|
||||
def _make_registry(
|
||||
*,
|
||||
alias: str = "local",
|
||||
model: str = "old-model",
|
||||
extras: dict[str, str] | None = None,
|
||||
) -> ModelRegistry:
|
||||
"""Build a real ModelRegistry seeded with ``alias`` (the default) plus
|
||||
any ``extras`` (alias → model). ``ModelRegistry.__init__`` rejects an
|
||||
empty model dict so tests that exercise the helper need at least one
|
||||
entry; pass ``extras`` for multi-alias scenarios (e.g. delete-by-alias).
|
||||
"""
|
||||
configs = {alias: _make_config(alias, model)}
|
||||
for extra_alias, extra_model in (extras or {}).items():
|
||||
configs[extra_alias] = _make_config(extra_alias, extra_model)
|
||||
return ModelRegistry(configs, default=alias)
|
||||
|
||||
|
||||
class _AppState:
|
||||
"""Shim mirroring Starlette's ``app.state`` for direct helper tests."""
|
||||
|
||||
coord_registry: ModelRegistry | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper-level tests — ``_refresh_coord_registry`` semantics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_helper_rebuilds_registry_from_db(storage: SQLiteBackend) -> None:
|
||||
"""Helper pulls the latest DB rows into the existing registry."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="new-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="old-model")
|
||||
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "new-model"
|
||||
|
||||
|
||||
def test_helper_preserves_object_identity(storage: SQLiteBackend) -> None:
|
||||
"""The factory closes over the registry object — refresh must mutate
|
||||
in place rather than swap the attribute."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="new-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry()
|
||||
before = id(state.coord_registry)
|
||||
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert id(state.coord_registry) == before
|
||||
|
||||
|
||||
def test_helper_noop_when_coord_registry_none(storage: SQLiteBackend) -> None:
|
||||
"""Console boot with no model rows leaves coord_registry = None.
|
||||
The helper must not 500 in that state — CRUD that lands the FIRST
|
||||
row would otherwise fail before the operator can recover."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m")
|
||||
state = _AppState()
|
||||
state.coord_registry = None
|
||||
|
||||
_refresh_coord_registry(state, storage) # must not raise
|
||||
|
||||
assert state.coord_registry is None
|
||||
|
||||
|
||||
def test_helper_preserves_registry_when_load_fails(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An unexpected error from ``load_model_registry`` (e.g. config.toml
|
||||
parse failure, programming bug) must not tear down a working
|
||||
registry — log + leave the existing instance intact."""
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="old-model")
|
||||
|
||||
def _boom(**_kw: Any) -> ModelRegistry:
|
||||
raise RuntimeError("simulated loader failure")
|
||||
|
||||
monkeypatch.setattr("turnstone.core.model_registry.load_model_registry", _boom)
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "old-model"
|
||||
|
||||
|
||||
def test_helper_preserves_registry_when_strict_load_fails(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""``load_model_registry`` normally swallows storage read errors and
|
||||
would return a config.toml-only registry on a transient DB outage —
|
||||
applying that via ``reload()`` would silently drop every DB-sourced
|
||||
alias. The helper passes ``strict=True`` so the loader re-raises
|
||||
instead, the helper's outer except catches it, and the existing
|
||||
registry survives intact."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="db-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="db-model")
|
||||
|
||||
def _broken(**_kw: Any) -> Any:
|
||||
raise RuntimeError("simulated transient DB outage")
|
||||
|
||||
monkeypatch.setattr(storage, "list_model_definitions", _broken)
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
# Existing registry untouched — strict=True surfaced the storage
|
||||
# error to the helper before the loader's silent fallback could
|
||||
# produce a truncated registry for reload().
|
||||
assert state.coord_registry.get_config("local").model == "db-model"
|
||||
|
||||
|
||||
def test_helper_preserves_registry_when_no_enabled_rows(storage: SQLiteBackend) -> None:
|
||||
"""All rows disabled/deleted: ModelRegistry.__init__ rejects an empty
|
||||
model dict (raises ValueError). Helper must catch and preserve the
|
||||
existing registry so coord stays usable while admin restores rows."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m", enabled=False)
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="cached-model")
|
||||
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "cached-model"
|
||||
|
||||
|
||||
def test_helper_preserves_registry_on_reload_validation_error(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A reload that raises mid-mutation (e.g. validation guard) must
|
||||
leave the existing registry instance functional."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="new-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="old-model")
|
||||
|
||||
def _broken_reload(*_a: Any, **_kw: Any) -> None:
|
||||
raise ValueError("simulated reload validation failure")
|
||||
|
||||
monkeypatch.setattr(state.coord_registry, "reload", _broken_reload)
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
# Existing registry still reachable; the broken reload was a no-op
|
||||
# at the public-facing level.
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "old-model"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoint-level integration tests — verify wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_client(storage: SQLiteBackend, registry: ModelRegistry | None) -> TestClient:
|
||||
"""Build a TestClient wired to the four model-definition endpoints.
|
||||
|
||||
Uses the shared header-driven ``_AuthMiddleware`` from
|
||||
``tests/_coord_test_helpers``; default headers below grant
|
||||
``admin.models`` permission so the endpoint gate passes.
|
||||
"""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions",
|
||||
admin_create_model_definition,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions/reload",
|
||||
admin_model_reload,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions/{definition_id}",
|
||||
admin_update_model_definition,
|
||||
methods=["PUT"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions/{definition_id}",
|
||||
admin_delete_model_definition,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_AuthMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
app.state.coord_registry = registry
|
||||
# Reload endpoint also touches these — stub them so the test focuses
|
||||
# on the registry-refresh behaviour without dragging in a full
|
||||
# collector / proxy_client wiring.
|
||||
app.state.collector = MagicMock()
|
||||
app.state.collector.get_all_nodes.return_value = []
|
||||
app.state.proxy_client = MagicMock()
|
||||
app.state.config_store = MagicMock()
|
||||
client = TestClient(app)
|
||||
client.headers.update({"X-Test-User": "admin", "X-Test-Perms": "admin.models"})
|
||||
return client
|
||||
|
||||
|
||||
def test_create_endpoint_refreshes_registry(storage: SQLiteBackend) -> None:
|
||||
"""POST /api/admin/model-definitions bumps the in-process registry
|
||||
so newly-spawned coord sessions see the new alias immediately."""
|
||||
# Pre-existing alias (registry needs at least one row)
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m")
|
||||
registry = _make_registry(alias="local", model="m")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
resp = client.post(
|
||||
"/v1/api/admin/model-definitions",
|
||||
json={
|
||||
"alias": "fast",
|
||||
"model": "fast-model",
|
||||
"provider": "openai-compatible",
|
||||
"base_url": "http://localhost:9000/v1",
|
||||
"api_key": "sk-x",
|
||||
"context_window": 4096,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert registry.has_alias("fast")
|
||||
assert registry.get_config("fast").model == "fast-model"
|
||||
|
||||
|
||||
def test_update_endpoint_refreshes_registry(storage: SQLiteBackend) -> None:
|
||||
"""PUT swaps the underlying model name behind a stable alias — the
|
||||
user's reported regression."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="old-model")
|
||||
registry = _make_registry(alias="local", model="old-model")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
resp = client.put(
|
||||
"/v1/api/admin/model-definitions/m1",
|
||||
json={"model": "new-model"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert registry.get_config("local").model == "new-model"
|
||||
|
||||
|
||||
def test_update_endpoint_skips_refresh_on_empty_body(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An empty PUT body must skip the registry refresh — the
|
||||
``if updates:`` gate exists because ``load_model_registry`` is
|
||||
non-trivial and a no-op refresh on every PUT would burn cycles
|
||||
rebuilding state that hasn't changed. Spy on the helper to lock
|
||||
the gate down: a regression that drops the conditional would
|
||||
register a call here and trip the assertion.
|
||||
"""
|
||||
from turnstone.console import server as server_module
|
||||
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="locked-in")
|
||||
registry = _make_registry(alias="local", model="locked-in")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
calls: list[tuple[Any, Any]] = []
|
||||
|
||||
def _spy(app_state: Any, storage: Any) -> None:
|
||||
calls.append((app_state, storage))
|
||||
|
||||
monkeypatch.setattr(server_module, "_refresh_coord_registry", _spy)
|
||||
|
||||
resp = client.put("/v1/api/admin/model-definitions/m1", json={})
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert calls == [] # gate held: empty body did not trigger a refresh
|
||||
|
||||
|
||||
def test_delete_endpoint_refreshes_registry(storage: SQLiteBackend) -> None:
|
||||
"""DELETE drops the alias from the in-process registry too — a
|
||||
coord session that tried to resolve the deleted alias would
|
||||
otherwise hit a stale cached client."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m")
|
||||
_seed_model_def(storage, definition_id="m2", alias="extra", model="x")
|
||||
registry = _make_registry(alias="local", model="m", extras={"extra": "x"})
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
resp = client.delete("/v1/api/admin/model-definitions/m2")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert not registry.has_alias("extra")
|
||||
assert registry.has_alias("local") # default alias unaffected
|
||||
|
||||
|
||||
def test_reload_endpoint_refreshes_registry(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The explicit reload button must refresh the console's own
|
||||
registry — until this PR it only fanned out to nodes."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="initial")
|
||||
registry = _make_registry(alias="local", model="initial")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
# Bypass the CRUD endpoints to mimic an out-of-band DB change (e.g.
|
||||
# an operator psql session) and verify the explicit reload path
|
||||
# still pulls the change in.
|
||||
storage.update_model_definition("m1", model="reloaded-model")
|
||||
|
||||
# Stub the async cluster fan-out helpers — they require a fully-wired
|
||||
# collector / proxy_client which is orthogonal to the helper under test.
|
||||
async def _noop_publish(_request: Any) -> None:
|
||||
return None
|
||||
|
||||
async def _noop_notify(_request: Any) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr("turnstone.console.server._publish_config_change", _noop_publish)
|
||||
monkeypatch.setattr("turnstone.console.server._notify_nodes_model_reload", _noop_notify)
|
||||
|
||||
resp = client.post("/v1/api/admin/model-definitions/reload")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert registry.get_config("local").model == "reloaded-model"
|
||||
@@ -49,7 +49,7 @@ class TestServerVersioning:
|
||||
|
||||
mock_mgr = MagicMock()
|
||||
mock_mgr.list_all.return_value = []
|
||||
mock_mgr.max_active = 10
|
||||
mock_mgr.max_workstreams = 10
|
||||
app = create_app(
|
||||
workstreams=mock_mgr,
|
||||
global_queue=queue.Queue(),
|
||||
@@ -76,7 +76,7 @@ class TestServerVersioning:
|
||||
assert resp.status_code == 200
|
||||
spec = resp.json()
|
||||
assert spec["openapi"] == "3.1.0"
|
||||
assert "/v1/api/workstreams/{ws_id}/send" in spec["paths"]
|
||||
assert "/v1/api/send" in spec["paths"]
|
||||
|
||||
def test_docs_page(self, client):
|
||||
resp = client.get("/docs")
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
"""Static smoke guards for ``turnstone/ui/static/app.js``.
|
||||
|
||||
The interactive WebUI's app.js has no JS test framework on the
|
||||
project side. This file holds Python-side string-presence assertions
|
||||
that catch regressions on critical paths — the kind of one-line
|
||||
deletion or rename that breaks the UI silently and only surfaces in
|
||||
manual testing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_APP_JS = Path(__file__).resolve().parent.parent / "turnstone/ui/static/app.js"
|
||||
|
||||
|
||||
def test_switch_tab_bootstraps_pane_when_none_exists() -> None:
|
||||
"""``switchTab`` must create a pane when none exists. A fresh-
|
||||
loaded interactive UI with no workstreams shows the dashboard
|
||||
and creates no panes (per ``initWorkstreams``); the user's first
|
||||
``create`` or ``open`` then calls ``switchTab(newWsId)``. Pre-fix,
|
||||
the early ``if (!pane) return;`` left switchTab with nowhere to
|
||||
attach — the chat UI never connected SSE for the freshly-created
|
||||
workstream, and only a page refresh fixed it. This test guards
|
||||
against accidentally re-introducing the early-return."""
|
||||
body = _APP_JS.read_text(encoding="utf-8")
|
||||
start = body.index("function switchTab(wsId) {")
|
||||
# Bound the search to the function body — switchTab is short.
|
||||
fn = body[start : start + 2000]
|
||||
assert "if (!pane) return;" not in fn, (
|
||||
"switchTab must not early-return when no pane exists — that's "
|
||||
"the no-chat-after-first-create bug. Bootstrap a pane instead."
|
||||
)
|
||||
# Affirmatively check the bootstrap path exists.
|
||||
assert "createPane(wsId)" in fn, (
|
||||
"switchTab must call createPane(wsId) to bootstrap the first "
|
||||
"pane when getFocusedPane returns null"
|
||||
)
|
||||
|
||||
|
||||
def test_tool_error_does_not_overwrite_approval_badge() -> None:
|
||||
"""When an approved tool subsequently errors, the existing
|
||||
``✓ approved`` (or ``✓ auto-approved``) pill must remain visible —
|
||||
the error indicator is appended as a sibling pill, not by mutating
|
||||
the approval pill in place. Pre-fix, both ``appendToolOutput``
|
||||
(live) and ``replayHistory`` (history reconstruction) located the
|
||||
existing approval badge via ``querySelector(".ts-approval-badge")``
|
||||
and overwrote its className + textContent with the ``--error``
|
||||
state, so the user lost the record that they had approved the
|
||||
call. This test pins the new append-sibling behaviour."""
|
||||
body = _APP_JS.read_text(encoding="utf-8")
|
||||
# Affirmatively check that an idempotency guard exists somewhere:
|
||||
# a ``querySelector(".ts-approval-badge--error")`` lookup is the
|
||||
# structural marker of the fix. Pre-fix the modifier never appeared
|
||||
# in app.js at all. Loose on quote style and surrounding form (the
|
||||
# guard might be a negated ``if (!q) {build...}`` block at a call
|
||||
# site, or a positive ``if (q) return;`` early-exit inside an
|
||||
# extracted helper) so a later refactor doesn't trip CI on
|
||||
# cosmetics.
|
||||
error_guard_re = re.compile(
|
||||
r"""querySelector\(\s*['"]\.ts-approval-badge--error['"]\s*\)""",
|
||||
)
|
||||
assert error_guard_re.search(body), (
|
||||
"The error-badge code path must guard creation with a "
|
||||
"querySelector for .ts-approval-badge--error so duplicate fires "
|
||||
"(live + history re-render) do not stack badges."
|
||||
)
|
||||
# Forbid the mutate-existing-badge sequence: a generic
|
||||
# ``.ts-approval-badge`` lookup followed within a handful of lines
|
||||
# by mutating that same handle into the ``--error`` state. Two
|
||||
# unrelated call sites (history rendering + live tool-output
|
||||
# insertion) legitimately query ``.ts-approval-badge`` to position
|
||||
# output above it, so the bare query alone is not the anti-pattern;
|
||||
# the close pairing with an ``--error`` class mutation is. Accept
|
||||
# either quote style and catch both ``className = "..."`` and
|
||||
# ``classList.add("ts-approval-badge--error")`` forms.
|
||||
overwrite_re = re.compile(
|
||||
r"""(\w+)\s*=\s*\w+\.querySelector\(\s*(["'])\.ts-approval-badge\2\s*\)\s*;"""
|
||||
r""".{0,200}?"""
|
||||
r"""(?:"""
|
||||
r"""\1\.className\s*=\s*(["'])[^"']*\bts-approval-badge--error\b[^"']*\3"""
|
||||
r"""|"""
|
||||
r"""\1\.classList\.add\([^)]*(["'])ts-approval-badge--error\4[^)]*\)"""
|
||||
r""")""",
|
||||
re.DOTALL,
|
||||
)
|
||||
assert not overwrite_re.search(body), (
|
||||
"Found the badge-overwrite anti-pattern: a queried "
|
||||
".ts-approval-badge handle is mutated into the --error variant "
|
||||
"(via className overwrite or classList.add). Append a sibling "
|
||||
"badge instead so the approval verdict stays visible alongside "
|
||||
"the error."
|
||||
)
|
||||
@@ -56,155 +56,3 @@ def test_record_audit_generates_unique_ids(storage):
|
||||
events = storage.list_audit_events()
|
||||
assert len(events) == 2
|
||||
assert events[0]["event_id"] != events[1]["event_id"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Credential redaction at the audit boundary
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_record_audit_redacts_passwords_by_default(storage):
|
||||
"""Detail strings go through redact_credentials by default."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"coordinator.spawn",
|
||||
detail={
|
||||
"initial_message": "connect via postgresql://alice:s3cret@db.example.com/app",
|
||||
},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
# Exact redaction text comes from output_guard._redact_credentials —
|
||||
# assert the token is stripped rather than the exact marker so
|
||||
# this test doesn't break if the marker format evolves.
|
||||
assert "s3cret" not in detail["initial_message"]
|
||||
assert "REDACTED" in detail["initial_message"]
|
||||
|
||||
|
||||
def test_record_audit_redacts_nested_strings(storage):
|
||||
"""Walker descends into lists / nested dicts."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"tasks.update",
|
||||
detail={
|
||||
"tasks": [
|
||||
{"title": "normal task"},
|
||||
{"title": "pull secret from AWS_SECRET_ACCESS_KEY=AKIAEXAMPLE123"},
|
||||
],
|
||||
},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
assert detail["tasks"][0]["title"] == "normal task"
|
||||
assert "AKIAEXAMPLE123" not in detail["tasks"][1]["title"]
|
||||
|
||||
|
||||
def test_record_audit_raw_detail_preserves_payload(storage):
|
||||
"""`raw_detail=True` bypasses the scrub — operator-originated detail only."""
|
||||
secret_like = "postgresql://alice:s3cret@db.example.com/app"
|
||||
record_audit(
|
||||
storage,
|
||||
"admin-1",
|
||||
"investigation.note",
|
||||
detail={"note": secret_like},
|
||||
raw_detail=True,
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
assert detail["note"] == secret_like
|
||||
|
||||
|
||||
def test_record_audit_strips_control_chars(storage):
|
||||
"""CR/LF/NUL/DEL and C0 controls are replaced with spaces so a
|
||||
downstream exporter that prints raw detail strings can't re-surface
|
||||
log-injection. Tab/newline are deliberately preserved."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"coordinator.note",
|
||||
detail={
|
||||
"msg": "hello\r\nInjected: bad\x00 escape \x1b[31mred\x1b[0m\x7f",
|
||||
"ok_tab": "a\tb\nc",
|
||||
},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
# CR / NUL / ESC / DEL scrubbed to spaces; tab + newline kept.
|
||||
assert "\r" not in detail["msg"]
|
||||
assert "\x00" not in detail["msg"]
|
||||
assert "\x1b" not in detail["msg"]
|
||||
assert "\x7f" not in detail["msg"]
|
||||
assert "hello" in detail["msg"]
|
||||
assert detail["ok_tab"] == "a\tb\nc"
|
||||
|
||||
|
||||
def test_record_audit_clean_strings_roundtrip_unchanged(storage):
|
||||
"""Detail strings with no credential patterns and no control chars
|
||||
pass through unchanged — the fast-path / scrub must not corrupt the
|
||||
common case."""
|
||||
clean = {"note": "hello world", "code": "import foo", "state": "ok"}
|
||||
record_audit(storage, "u1", "coordinator.note", detail=clean)
|
||||
event = storage.list_audit_events()[0]
|
||||
assert json.loads(event["detail"]) == clean
|
||||
|
||||
|
||||
def test_record_audit_fast_path_skips_no_string_detail(storage):
|
||||
"""A detail carrying only scalars (no strings anywhere) must persist
|
||||
identically — exercises the ``_has_any_string`` fast path."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"coordinator.metric",
|
||||
detail={"spawned": 5, "ok": True, "parent": None, "tail": [1, 2, 3]},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
assert json.loads(event["detail"]) == {
|
||||
"spawned": 5,
|
||||
"ok": True,
|
||||
"parent": None,
|
||||
"tail": [1, 2, 3],
|
||||
}
|
||||
|
||||
|
||||
def test_record_audit_redacts_dict_keys(storage):
|
||||
"""Walker descends into dict keys too — a caller using
|
||||
model-controlled text as a key can't leak it verbatim."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"coordinator.note",
|
||||
detail={"postgresql://alice:s3cret@db.example.com/app": True},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
assert all("s3cret" not in k for k in detail)
|
||||
|
||||
|
||||
def test_record_audit_walks_set_and_frozenset(storage):
|
||||
"""Walker handles set/frozenset values (docstring promise)."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"coordinator.note",
|
||||
detail={"tags": frozenset({"ak_" + "x" * 40, "plain"})},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
# The credential-looking AK token gets scrubbed; the plain one survives.
|
||||
tags = detail["tags"]
|
||||
assert "plain" in tags
|
||||
|
||||
|
||||
def test_record_audit_leaves_non_string_scalars_alone(storage):
|
||||
"""Non-string scalars (int / bool / None) pass through unchanged."""
|
||||
record_audit(
|
||||
storage,
|
||||
"u1",
|
||||
"coordinator.spawn",
|
||||
detail={"budget_ok": True, "spawned": 5, "parent": None},
|
||||
)
|
||||
event = storage.list_audit_events()[0]
|
||||
detail = json.loads(event["detail"])
|
||||
assert detail == {"budget_ok": True, "spawned": 5, "parent": None}
|
||||
|
||||
+37
-280
@@ -53,8 +53,8 @@ class TestIsPublicPath:
|
||||
def test_api_workstreams_not_public(self):
|
||||
assert is_public_path("/api/workstreams") is False
|
||||
|
||||
def test_api_workstreams_send_not_public(self):
|
||||
assert is_public_path("/api/workstreams/abc/send") is False
|
||||
def test_api_send_not_public(self):
|
||||
assert is_public_path("/api/send") is False
|
||||
|
||||
def test_api_cluster_overview_not_public(self):
|
||||
assert is_public_path("/api/cluster/overview") is False
|
||||
@@ -71,8 +71,8 @@ class TestIsPublicPath:
|
||||
def test_v1_api_workstreams_not_public(self):
|
||||
assert is_public_path("/v1/api/workstreams") is False
|
||||
|
||||
def test_v1_api_workstreams_send_not_public(self):
|
||||
assert is_public_path("/v1/api/workstreams/abc/send") is False
|
||||
def test_v1_api_send_not_public(self):
|
||||
assert is_public_path("/v1/api/send") is False
|
||||
|
||||
def test_openapi_json_public(self):
|
||||
assert is_public_path("/openapi.json") is True
|
||||
@@ -97,22 +97,10 @@ class TestRequiredScope:
|
||||
assert required_scope("GET", "/api/events") == "read"
|
||||
|
||||
def test_post_send_needs_write(self):
|
||||
assert required_scope("POST", "/api/workstreams/abc/send") == "write"
|
||||
|
||||
def test_delete_send_needs_write(self):
|
||||
assert required_scope("DELETE", "/api/workstreams/abc/send") == "write"
|
||||
assert required_scope("POST", "/api/send") == "write"
|
||||
|
||||
def test_post_approve_needs_approve(self):
|
||||
assert required_scope("POST", "/api/workstreams/abc/approve") == "approve"
|
||||
|
||||
def test_post_cancel_needs_write(self):
|
||||
assert required_scope("POST", "/api/workstreams/abc/cancel") == "write"
|
||||
|
||||
def test_post_close_needs_write(self):
|
||||
assert required_scope("POST", "/api/workstreams/abc/close") == "write"
|
||||
|
||||
def test_get_events_per_ws_needs_read(self):
|
||||
assert required_scope("GET", "/api/workstreams/abc/events") == "read"
|
||||
assert required_scope("POST", "/api/approve") == "approve"
|
||||
|
||||
def test_post_plan_needs_write(self):
|
||||
assert required_scope("POST", "/api/plan") == "write"
|
||||
@@ -123,6 +111,9 @@ class TestRequiredScope:
|
||||
def test_post_workstreams_new_needs_write(self):
|
||||
assert required_scope("POST", "/api/workstreams/new") == "write"
|
||||
|
||||
def test_post_workstreams_close_needs_write(self):
|
||||
assert required_scope("POST", "/api/workstreams/close") == "write"
|
||||
|
||||
def test_all_write_paths_need_write(self):
|
||||
for path in WRITE_PATHS:
|
||||
scope = required_scope("POST", path)
|
||||
@@ -132,10 +123,10 @@ class TestRequiredScope:
|
||||
assert required_scope("POST", "/api/unknown") == "read"
|
||||
|
||||
def test_v1_post_send_needs_write(self):
|
||||
assert required_scope("POST", "/v1/api/workstreams/abc/send") == "write"
|
||||
assert required_scope("POST", "/v1/api/send") == "write"
|
||||
|
||||
def test_v1_post_approve_needs_approve(self):
|
||||
assert required_scope("POST", "/v1/api/workstreams/abc/approve") == "approve"
|
||||
assert required_scope("POST", "/v1/api/approve") == "approve"
|
||||
|
||||
def test_v1_get_workstreams_needs_read(self):
|
||||
assert required_scope("GET", "/v1/api/workstreams") == "read"
|
||||
@@ -144,10 +135,10 @@ class TestRequiredScope:
|
||||
assert required_scope("POST", "/v1/api/cluster/workstreams/new") == "write"
|
||||
|
||||
def test_proxy_v1_send_needs_write(self):
|
||||
assert required_scope("POST", "/node/node-a/v1/api/workstreams/abc/send") == "write"
|
||||
assert required_scope("POST", "/node/node-a/v1/api/send") == "write"
|
||||
|
||||
def test_proxy_v1_approve_needs_approve(self):
|
||||
assert required_scope("POST", "/node/node-a/v1/api/workstreams/abc/approve") == "approve"
|
||||
assert required_scope("POST", "/node/node-a/v1/api/approve") == "approve"
|
||||
|
||||
def test_proxy_v1_read_endpoint_needs_read(self):
|
||||
assert required_scope("GET", "/node/node-a/v1/api/workstreams") == "read"
|
||||
@@ -411,7 +402,7 @@ class TestCheckRequest:
|
||||
|
||||
def test_write_read_token_403(self, read_jwt):
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST", "/api/workstreams/abc/send", read_jwt, jwt_secret=self._SECRET
|
||||
"POST", "/api/send", read_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is False
|
||||
assert status == 403
|
||||
@@ -419,14 +410,14 @@ class TestCheckRequest:
|
||||
|
||||
def test_write_full_token_ok(self, full_jwt):
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST", "/api/workstreams/abc/send", full_jwt, jwt_secret=self._SECRET
|
||||
"POST", "/api/send", full_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is True
|
||||
assert status == 200
|
||||
|
||||
def test_approve_read_token_403(self, read_jwt):
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST", "/api/workstreams/abc/approve", read_jwt, jwt_secret=self._SECRET
|
||||
"POST", "/api/approve", read_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is False
|
||||
assert status == 403
|
||||
@@ -434,10 +425,7 @@ class TestCheckRequest:
|
||||
def test_proxy_write_read_token_403(self, read_jwt):
|
||||
"""Read tokens cannot escalate to write ops via proxy routes."""
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST",
|
||||
"/node/node-a/api/workstreams/abc/send",
|
||||
read_jwt,
|
||||
jwt_secret=self._SECRET,
|
||||
"POST", "/node/node-a/api/send", read_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is False
|
||||
assert status == 403
|
||||
@@ -445,10 +433,7 @@ class TestCheckRequest:
|
||||
def test_proxy_write_trailing_slash_read_token_403(self, read_jwt):
|
||||
"""Trailing slash must not bypass write-role check on proxy routes."""
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST",
|
||||
"/node/node-a/api/workstreams/abc/send/",
|
||||
read_jwt,
|
||||
jwt_secret=self._SECRET,
|
||||
"POST", "/node/node-a/api/send/", read_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is False
|
||||
assert status == 403
|
||||
@@ -456,7 +441,7 @@ class TestCheckRequest:
|
||||
def test_direct_write_trailing_slash_read_token_403(self, read_jwt):
|
||||
"""Trailing slash must not bypass write-role check on direct routes."""
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST", "/api/workstreams/abc/send/", read_jwt, jwt_secret=self._SECRET
|
||||
"POST", "/api/send/", read_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is False
|
||||
assert status == 403
|
||||
@@ -464,20 +449,14 @@ class TestCheckRequest:
|
||||
def test_proxy_write_full_token_ok(self, full_jwt):
|
||||
"""Full tokens pass through proxy write routes."""
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST",
|
||||
"/node/node-a/api/workstreams/abc/send",
|
||||
full_jwt,
|
||||
jwt_secret=self._SECRET,
|
||||
"POST", "/node/node-a/api/send", full_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is True
|
||||
|
||||
def test_proxy_v1_write_read_token_403(self, read_jwt):
|
||||
"""Read tokens cannot escalate to write ops via v1 proxy routes."""
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST",
|
||||
"/node/node-a/v1/api/workstreams/abc/send",
|
||||
read_jwt,
|
||||
jwt_secret=self._SECRET,
|
||||
"POST", "/node/node-a/v1/api/send", read_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is False
|
||||
assert status == 403
|
||||
@@ -485,10 +464,7 @@ class TestCheckRequest:
|
||||
def test_proxy_v1_write_full_token_ok(self, full_jwt):
|
||||
"""Full tokens pass through v1 proxy write routes."""
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST",
|
||||
"/node/node-a/v1/api/workstreams/abc/send",
|
||||
full_jwt,
|
||||
jwt_secret=self._SECRET,
|
||||
"POST", "/node/node-a/v1/api/send", full_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is True
|
||||
|
||||
@@ -520,7 +496,7 @@ class TestCheckRequest:
|
||||
|
||||
def test_approve_full_token_ok(self, full_jwt):
|
||||
allowed, status, msg, _result = check_request(
|
||||
"POST", "/api/workstreams/abc/approve", full_jwt, jwt_secret=self._SECRET
|
||||
"POST", "/api/approve", full_jwt, jwt_secret=self._SECRET
|
||||
)
|
||||
assert allowed is True
|
||||
|
||||
@@ -562,7 +538,7 @@ class TestCheckRequestWithCookie:
|
||||
def test_bearer_takes_precedence_over_cookie(self, read_jwt, full_jwt):
|
||||
allowed, status, _, _r = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/send",
|
||||
"/api/send",
|
||||
f"Bearer {full_jwt}",
|
||||
cookie_header=f"turnstone_auth={read_jwt}",
|
||||
jwt_secret=self._SECRET,
|
||||
@@ -583,7 +559,7 @@ class TestCheckRequestWithCookie:
|
||||
def test_cookie_read_on_write_403(self, read_jwt):
|
||||
allowed, status, _, _r = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/send",
|
||||
"/api/send",
|
||||
None,
|
||||
cookie_header=f"turnstone_auth={read_jwt}",
|
||||
jwt_secret=self._SECRET,
|
||||
@@ -594,7 +570,7 @@ class TestCheckRequestWithCookie:
|
||||
def test_cookie_full_on_write_ok(self, full_jwt):
|
||||
allowed, status, _, _r = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/send",
|
||||
"/api/send",
|
||||
None,
|
||||
cookie_header=f"turnstone_auth={full_jwt}",
|
||||
jwt_secret=self._SECRET,
|
||||
@@ -659,15 +635,9 @@ class TestServerAuth:
|
||||
mock_ws.name = "test"
|
||||
mock_ws.state = WorkstreamState.IDLE
|
||||
mock_ws.session = mock_session
|
||||
# Set kind / parent_ws_id / user_id explicitly so list_workstreams
|
||||
# JSON-serializes them — a bare MagicMock attribute returns another
|
||||
# MagicMock that fails json.dumps and surfaces as 500.
|
||||
mock_ws.kind = "interactive"
|
||||
mock_ws.parent_ws_id = None
|
||||
mock_ws.user_id = "u1"
|
||||
mock_mgr = MagicMock()
|
||||
mock_mgr.list_all.return_value = [mock_ws]
|
||||
mock_mgr.max_active = 10
|
||||
mock_mgr.max_workstreams = 10
|
||||
|
||||
from turnstone.core.auth import JWT_AUD_SERVER
|
||||
|
||||
@@ -724,25 +694,25 @@ class TestServerAuth:
|
||||
|
||||
def test_api_send_read_token_403(self):
|
||||
resp = self.client.post(
|
||||
"/v1/api/workstreams/x/send",
|
||||
"/v1/api/send",
|
||||
headers=self._read_hdr,
|
||||
json={"message": "hello"},
|
||||
json={"message": "hello", "ws_id": "x"},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
assert "Forbidden" in resp.json().get("error", "")
|
||||
|
||||
def test_api_send_full_token_passes_auth(self):
|
||||
resp = self.client.post(
|
||||
"/v1/api/workstreams/nonexistent/send",
|
||||
"/v1/api/send",
|
||||
headers=self._full_hdr,
|
||||
json={"message": "hello"},
|
||||
json={"message": "hello", "ws_id": "nonexistent"},
|
||||
)
|
||||
assert resp.status_code not in (401, 403)
|
||||
|
||||
def test_api_send_no_token_401(self):
|
||||
resp = self.client.post(
|
||||
"/v1/api/workstreams/x/send",
|
||||
json={"message": "hello"},
|
||||
"/v1/api/send",
|
||||
json={"message": "hello", "ws_id": "x"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
@@ -755,7 +725,7 @@ class TestServerAuth:
|
||||
|
||||
def test_options_no_auth_required(self):
|
||||
resp = self.client.options(
|
||||
"/v1/api/workstreams/x/send",
|
||||
"/v1/api/send",
|
||||
headers={
|
||||
"Origin": "http://example.com",
|
||||
"Access-Control-Request-Method": "POST",
|
||||
@@ -881,15 +851,9 @@ class TestServerLogin:
|
||||
mock_ws.name = "test"
|
||||
mock_ws.state = WorkstreamState.IDLE
|
||||
mock_ws.session = mock_session
|
||||
# Set kind / parent_ws_id / user_id explicitly so list_workstreams
|
||||
# JSON-serializes them — a bare MagicMock attribute returns another
|
||||
# MagicMock that fails json.dumps and surfaces as 500.
|
||||
mock_ws.kind = "interactive"
|
||||
mock_ws.parent_ws_id = None
|
||||
mock_ws.user_id = "u1"
|
||||
mock_mgr = MagicMock()
|
||||
mock_mgr.list_all.return_value = [mock_ws]
|
||||
mock_mgr.max_active = 10
|
||||
mock_mgr.max_workstreams = 10
|
||||
|
||||
# Mock storage with a test user for password login
|
||||
from turnstone.core.auth import hash_password
|
||||
@@ -977,163 +941,6 @@ class TestServerLogin:
|
||||
resp = self.test_client.get("/v1/api/workstreams")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_whoami_includes_exp(self):
|
||||
"""whoami exposes the JWT exp so the frontend can schedule refresh."""
|
||||
import time
|
||||
|
||||
self.test_client.post(
|
||||
"/v1/api/auth/login",
|
||||
json={"username": "testuser", "password": "testpass"},
|
||||
)
|
||||
resp = self.test_client.get("/v1/api/auth/whoami")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "exp" in data
|
||||
# Default JWT TTL is 24h; exp should be > now and < now + 25h.
|
||||
now = int(time.time())
|
||||
assert now < data["exp"] < now + 25 * 3600
|
||||
|
||||
def test_refresh_returns_new_jwt_and_cookie(self):
|
||||
"""POST /api/auth/refresh re-mints the cookie with a fresh exp."""
|
||||
from turnstone.core.auth import AUTH_COOKIE
|
||||
|
||||
# Storage needs get_user_permissions for the refresh re-resolve path.
|
||||
# Mock is shared across tests in the class — re-arm here in case a
|
||||
# prior test left it default.
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = {
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
|
||||
login = self.test_client.post(
|
||||
"/v1/api/auth/login",
|
||||
json={"username": "testuser", "password": "testpass"},
|
||||
)
|
||||
assert login.status_code == 200
|
||||
|
||||
refresh = self.test_client.post("/v1/api/auth/refresh")
|
||||
assert refresh.status_code == 200
|
||||
body = refresh.json()
|
||||
assert body["status"] == "ok"
|
||||
assert body["user_id"] == "uid_test"
|
||||
assert "jwt" in body
|
||||
# Set-Cookie header must be present so the browser updates. Don't
|
||||
# assert the new JWT differs from the original — sub-second login
|
||||
# and refresh produce identical iat/exp claims and therefore an
|
||||
# identical token, which is fine: the cookie still gets re-set.
|
||||
cookie_hdr = refresh.headers.get("set-cookie", "")
|
||||
assert AUTH_COOKIE in cookie_hdr
|
||||
assert "HttpOnly" in cookie_hdr
|
||||
|
||||
# The refreshed cookie must keep working.
|
||||
resp = self.test_client.get("/v1/api/workstreams")
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_refresh_response_includes_exp_and_permissions(self):
|
||||
"""Refresh response shape must match whoami so the frontend can
|
||||
populate sessionStorage + reschedule the next refresh off the
|
||||
single round-trip without a follow-up /whoami call."""
|
||||
import time
|
||||
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = {
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
login = self.test_client.post(
|
||||
"/v1/api/auth/login",
|
||||
json={"username": "testuser", "password": "testpass"},
|
||||
)
|
||||
assert login.status_code == 200
|
||||
|
||||
refresh = self.test_client.post("/v1/api/auth/refresh")
|
||||
assert refresh.status_code == 200
|
||||
body = refresh.json()
|
||||
# exp present + within the expected default JWT TTL window
|
||||
assert "exp" in body, body
|
||||
now = int(time.time())
|
||||
assert now < body["exp"] < now + 25 * 3600, body
|
||||
# permissions present + non-empty (matches the seeded role set)
|
||||
assert body.get("permissions"), body
|
||||
assert "write" in body["permissions"].split(",")
|
||||
|
||||
def test_refresh_unauthenticated_401(self):
|
||||
"""Refresh requires a currently-valid cookie — no cookie → 401."""
|
||||
# Clear cookies on the test client
|
||||
self.test_client.cookies.clear()
|
||||
resp = self.test_client.post("/v1/api/auth/refresh")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_refresh_storage_failure_falls_back(self):
|
||||
"""Transient storage error → fall back to in-token claims, not 403.
|
||||
|
||||
The earlier implementation called _load_user_permissions() which
|
||||
swallows exceptions and returns set(); that path was
|
||||
indistinguishable from a deleted user (legitimate 403). The
|
||||
handler now calls storage.get_user_permissions() directly so
|
||||
DB hiccups fall through to in-token perms.
|
||||
"""
|
||||
# Re-arm the storage so login works first
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = {
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
login = self.test_client.post(
|
||||
"/v1/api/auth/login",
|
||||
json={"username": "testuser", "password": "testpass"},
|
||||
)
|
||||
assert login.status_code == 200
|
||||
|
||||
# Now make storage raise on the refresh re-resolve
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.side_effect = RuntimeError(
|
||||
"db down"
|
||||
)
|
||||
try:
|
||||
resp = self.test_client.post("/v1/api/auth/refresh")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
# Permissions should still be present (fell back to in-token claims)
|
||||
assert body.get("permissions"), body
|
||||
finally:
|
||||
# Restore for any subsequent tests
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.side_effect = None
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = {
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
|
||||
def test_refresh_user_with_no_perms_403(self):
|
||||
"""Storage returns empty (user deleted/role-stripped) → 403.
|
||||
|
||||
Distinguished from the storage-failure case above because
|
||||
get_user_permissions returned a value (the empty set) without
|
||||
raising — that's an authoritative "no roles", not a hiccup.
|
||||
"""
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = {
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
login = self.test_client.post(
|
||||
"/v1/api/auth/login",
|
||||
json={"username": "testuser", "password": "testpass"},
|
||||
)
|
||||
assert login.status_code == 200
|
||||
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = set()
|
||||
try:
|
||||
resp = self.test_client.post("/v1/api/auth/refresh")
|
||||
assert resp.status_code == 403
|
||||
finally:
|
||||
self.test_client.app.state.auth_storage.get_user_permissions.return_value = {
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
|
||||
|
||||
class TestConsoleLogin:
|
||||
"""Test login/logout cookie flow on turnstone-console."""
|
||||
@@ -1321,56 +1128,6 @@ class TestJWTAudienceIssuer:
|
||||
result = validate_jwt(token, self.SECRET, audience="")
|
||||
assert result is not None
|
||||
|
||||
def test_validate_jwt_accepts_within_leeway_after_expiry(self):
|
||||
"""validate_jwt has 30s leeway for clock skew across hosts/processes."""
|
||||
import time
|
||||
|
||||
import jwt as pyjwt
|
||||
|
||||
from turnstone.core.auth import JWT_ISSUER, validate_jwt
|
||||
|
||||
# Mint a token that "expired" 10 seconds ago — still within 30s leeway.
|
||||
now = int(time.time())
|
||||
token = pyjwt.encode(
|
||||
{
|
||||
"sub": "user1",
|
||||
"scopes": "read",
|
||||
"src": "test",
|
||||
"iss": JWT_ISSUER,
|
||||
"iat": now - 100,
|
||||
"exp": now - 10,
|
||||
},
|
||||
self.SECRET,
|
||||
algorithm="HS256",
|
||||
)
|
||||
result = validate_jwt(token, self.SECRET, audience="")
|
||||
assert result is not None
|
||||
assert result.user_id == "user1"
|
||||
|
||||
def test_validate_jwt_rejects_past_leeway(self):
|
||||
"""Tokens expired beyond the 30s leeway must still be rejected."""
|
||||
import time
|
||||
|
||||
import jwt as pyjwt
|
||||
|
||||
from turnstone.core.auth import JWT_ISSUER, validate_jwt
|
||||
|
||||
now = int(time.time())
|
||||
token = pyjwt.encode(
|
||||
{
|
||||
"sub": "user1",
|
||||
"scopes": "read",
|
||||
"src": "test",
|
||||
"iss": JWT_ISSUER,
|
||||
"iat": now - 200,
|
||||
"exp": now - 60,
|
||||
},
|
||||
self.SECRET,
|
||||
algorithm="HS256",
|
||||
)
|
||||
result = validate_jwt(token, self.SECRET, audience="")
|
||||
assert result is None
|
||||
|
||||
def test_create_jwt_expiry_seconds(self):
|
||||
import jwt as pyjwt
|
||||
|
||||
@@ -1695,7 +1452,7 @@ class TestCorsConfigurable:
|
||||
|
||||
mgr = MagicMock()
|
||||
mgr.list_all.return_value = []
|
||||
mgr.max_active = 10
|
||||
mgr.max_workstreams = 10
|
||||
app = srv_mod.create_app(
|
||||
workstreams=mgr,
|
||||
global_queue=queue.Queue(),
|
||||
@@ -1716,7 +1473,7 @@ class TestCorsConfigurable:
|
||||
|
||||
mgr = MagicMock()
|
||||
mgr.list_all.return_value = []
|
||||
mgr.max_active = 10
|
||||
mgr.max_workstreams = 10
|
||||
app = srv_mod.create_app(
|
||||
workstreams=mgr,
|
||||
global_queue=queue.Queue(),
|
||||
|
||||
+11
-11
@@ -175,10 +175,10 @@ class TestRequiredScope:
|
||||
assert required_scope("GET", "/api/workstreams") == "read"
|
||||
|
||||
def test_post_write(self):
|
||||
assert required_scope("POST", "/api/workstreams/abc/send") == "write"
|
||||
assert required_scope("POST", "/api/send") == "write"
|
||||
|
||||
def test_post_approve(self):
|
||||
assert required_scope("POST", "/api/workstreams/abc/approve") == "approve"
|
||||
assert required_scope("POST", "/api/approve") == "approve"
|
||||
|
||||
def test_admin_prefix(self):
|
||||
assert required_scope("GET", "/api/admin/users") == "approve"
|
||||
@@ -186,14 +186,14 @@ class TestRequiredScope:
|
||||
assert required_scope("DELETE", "/api/admin/users/abc") == "approve"
|
||||
|
||||
def test_versioned_path(self):
|
||||
assert required_scope("POST", "/v1/api/workstreams/abc/send") == "write"
|
||||
assert required_scope("POST", "/v1/api/workstreams/abc/approve") == "approve"
|
||||
assert required_scope("POST", "/v1/api/send") == "write"
|
||||
assert required_scope("POST", "/v1/api/approve") == "approve"
|
||||
|
||||
def test_proxy_write(self):
|
||||
assert required_scope("POST", "/node/n1/api/workstreams/abc/send") == "write"
|
||||
assert required_scope("POST", "/node/n1/api/send") == "write"
|
||||
|
||||
def test_proxy_approve(self):
|
||||
assert required_scope("POST", "/node/n1/api/workstreams/abc/approve") == "approve"
|
||||
assert required_scope("POST", "/node/n1/api/approve") == "approve"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -270,7 +270,7 @@ class TestCheckRequestScopes:
|
||||
jwt_tok = create_jwt("u1", frozenset({"read"}), "test", self._SECRET)
|
||||
allowed, status, msg, _ = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/send",
|
||||
"/api/send",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=self._SECRET,
|
||||
)
|
||||
@@ -282,7 +282,7 @@ class TestCheckRequestScopes:
|
||||
jwt_tok = create_jwt("u1", frozenset({"read"}), "test", self._SECRET)
|
||||
allowed, status, msg, _ = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/approve",
|
||||
"/api/approve",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=self._SECRET,
|
||||
)
|
||||
@@ -294,7 +294,7 @@ class TestCheckRequestScopes:
|
||||
jwt_tok = create_jwt("u1", frozenset({"read", "write", "approve"}), "test", self._SECRET)
|
||||
allowed, status, msg, result = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/approve",
|
||||
"/api/approve",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=self._SECRET,
|
||||
)
|
||||
@@ -306,7 +306,7 @@ class TestCheckRequestScopes:
|
||||
jwt_tok = create_jwt("u1", frozenset({"read", "write"}), "db", self._SECRET)
|
||||
allowed, status, msg, result = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/send",
|
||||
"/api/send",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=self._SECRET,
|
||||
)
|
||||
@@ -318,7 +318,7 @@ class TestCheckRequestScopes:
|
||||
jwt_tok = create_jwt("u1", frozenset({"read"}), "db", self._SECRET)
|
||||
allowed, status, msg, _ = check_request(
|
||||
"POST",
|
||||
"/api/workstreams/abc/send",
|
||||
"/api/send",
|
||||
f"Bearer {jwt_tok}",
|
||||
jwt_secret=self._SECRET,
|
||||
)
|
||||
|
||||
+4
-15
@@ -168,18 +168,10 @@ class TestCancelDuringStreaming:
|
||||
assert ui.states[-1] == "idle"
|
||||
# Check that "[Generation cancelled]" was emitted
|
||||
assert any("cancelled" in i.lower() for i in ui.infos)
|
||||
# The partial content should be preserved as an assistant
|
||||
# message AND annotated with a marker that downstream readers
|
||||
# (inspect_workstream, the next coord turn) can use to
|
||||
# distinguish a cancelled fragment from a completed turn — the
|
||||
# raw "Hello world" without a marker would look like the
|
||||
# final assistant answer to a coord LLM reading the child's
|
||||
# transcript.
|
||||
# The partial content should be preserved as an assistant message
|
||||
assistant_msgs = [m for m in session.messages if m["role"] == "assistant"]
|
||||
assert len(assistant_msgs) == 1
|
||||
content = assistant_msgs[0]["content"]
|
||||
assert content.startswith("Hello world")
|
||||
assert "[generation cancelled before completion]" in content
|
||||
assert assistant_msgs[0]["content"] == "Hello world"
|
||||
# No tool_calls in the partial message
|
||||
assert "tool_calls" not in assistant_msgs[0]
|
||||
|
||||
@@ -519,13 +511,10 @@ class TestStreamAbort:
|
||||
# Should complete as cancelled, not error
|
||||
assert "idle" in ui.states
|
||||
assert any("cancelled" in i.lower() for i in ui.infos)
|
||||
# Partial content preserved AND annotated with the
|
||||
# cancelled-before-completion marker.
|
||||
# Partial content preserved
|
||||
assistant_msgs = [m for m in session.messages if m["role"] == "assistant"]
|
||||
assert len(assistant_msgs) == 1
|
||||
content = assistant_msgs[0]["content"]
|
||||
assert content.startswith("Hello")
|
||||
assert "[generation cancelled before completion]" in content
|
||||
assert assistant_msgs[0]["content"] == "Hello"
|
||||
|
||||
def test_non_cancel_exception_not_swallowed(self, tmp_db):
|
||||
"""Exceptions during streaming that aren't caused by cancel
|
||||
|
||||
+47
-381
@@ -28,21 +28,6 @@ def _run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _bind_ws_event_handlers(bot, cls):
|
||||
"""Bind ``_on_ws_event`` + every ``_handle_*`` method from *cls* to *bot*.
|
||||
|
||||
``MagicMock(spec=cls)`` stubs async methods as ``AsyncMock`` no-ops,
|
||||
so dispatcher tests that invoke the real ``_on_ws_event`` must also
|
||||
bind the per-event handlers it delegates to.
|
||||
"""
|
||||
bot._on_ws_event = cls._on_ws_event.__get__(bot, cls)
|
||||
for name in dir(cls):
|
||||
if name.startswith("_handle_"):
|
||||
attr = getattr(cls, name)
|
||||
if callable(attr):
|
||||
setattr(bot, name, attr.__get__(bot, cls))
|
||||
|
||||
|
||||
def _make_message(*, bot=False, guild=True, content="hello", channel=None, reference=None):
|
||||
"""Build a mock ``discord.Message``."""
|
||||
msg = MagicMock(spec=discord.Message)
|
||||
@@ -143,7 +128,7 @@ class TestStreamingMessage:
|
||||
_run(sm.append("hello "))
|
||||
_run(sm.append("world"))
|
||||
|
||||
assert sm.accumulated_text == "hello world"
|
||||
assert "".join(sm._buffer) == "hello world"
|
||||
|
||||
def test_finalize_sends_when_no_prior_message(self):
|
||||
from turnstone.channels.discord.bot import StreamingMessage
|
||||
@@ -168,7 +153,7 @@ class TestStreamingMessage:
|
||||
|
||||
# First append triggers flush (interval=0) which creates the message.
|
||||
_run(sm.append("hi"))
|
||||
assert sm.message is sent_msg
|
||||
assert sm._message is sent_msg
|
||||
|
||||
_run(sm.append(" there"))
|
||||
_run(sm.finalize())
|
||||
@@ -367,20 +352,20 @@ class TestAskModelSelection:
|
||||
class TestParseFooter:
|
||||
"""Tests for _parse_footer in views.py."""
|
||||
|
||||
def test_valid_footer_with_owner(self):
|
||||
def test_valid_footer(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = _make_interaction(footer_text="ws_abc|corr_123|12345")
|
||||
result = _parse_footer(interaction)
|
||||
assert result == ("ws_abc", "corr_123", "12345")
|
||||
|
||||
def test_footer_without_owner_returns_empty_owner(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
# Legacy footer without an owner field (pre-upgrade posts).
|
||||
interaction = _make_interaction(footer_text="ws_abc|corr_123")
|
||||
result = _parse_footer(interaction)
|
||||
assert result == ("ws_abc", "corr_123", "")
|
||||
assert result == ("ws_abc", "corr_123")
|
||||
|
||||
def test_footer_with_pipe_in_correlation(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
|
||||
interaction = _make_interaction(footer_text="ws_abc|corr|extra")
|
||||
result = _parse_footer(interaction)
|
||||
# split("|", 1) means the second part includes everything after first pipe.
|
||||
assert result == ("ws_abc", "corr|extra")
|
||||
|
||||
def test_no_message_returns_none(self):
|
||||
from turnstone.channels.discord.views import _parse_footer
|
||||
@@ -443,7 +428,7 @@ class TestWsEventFinalization:
|
||||
bot._notify_reply_channels = {}
|
||||
|
||||
# Use the real _on_ws_event method
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
@@ -472,7 +457,7 @@ class TestWsEventFinalization:
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
@@ -493,7 +478,6 @@ class TestApprovalVerdictDisplay:
|
||||
|
||||
def _make_bot(self):
|
||||
"""Build a mock TurnstoneBot with _on_ws_event bound."""
|
||||
from turnstone.channels._routing import PolicyVerdict
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
@@ -509,9 +493,7 @@ class TestApprovalVerdictDisplay:
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot.router = MagicMock()
|
||||
bot.router.evaluate_tool_policies = AsyncMock(return_value=PolicyVerdict(kind="none"))
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_approval_with_heuristic_verdict(self):
|
||||
@@ -630,7 +612,7 @@ class TestApprovalVerdictDisplay:
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {"ws-1": MagicMock()}
|
||||
bot._notify_reply_channels = {}
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
event = StreamEndEvent(ws_id="ws-1")
|
||||
@@ -656,7 +638,7 @@ class TestStreamEndBehavior:
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_stream_end_no_streaming_no_send(self):
|
||||
@@ -692,88 +674,38 @@ class TestStreamEndBehavior:
|
||||
class TestNotificationTracking:
|
||||
"""Tests for notification message tracking and DM reply routing."""
|
||||
|
||||
def _make_dm_bot(self, *, sent_message_id: int):
|
||||
"""Build a MagicMock bot whose notification target resolves to a DM."""
|
||||
def test_send_notification_tracks_message(self):
|
||||
"""send_notification should store message_id -> (ws_id, target_user) mapping."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = sent_message_id
|
||||
|
||||
dm_channel = MagicMock()
|
||||
dm_channel.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
user = MagicMock()
|
||||
user.id = 7777
|
||||
user.create_dm = AsyncMock(return_value=dm_channel)
|
||||
|
||||
inner_bot = MagicMock()
|
||||
inner_bot.get_channel = MagicMock(return_value=None)
|
||||
inner_bot.fetch_user = AsyncMock(return_value=user)
|
||||
bot._bot = inner_bot
|
||||
|
||||
bot.send_notification = TurnstoneBot.send_notification.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_send_notification_tracks_dm_with_user_id(self):
|
||||
"""send_notification for a DM records (ws_id, resolved_user_id)."""
|
||||
bot = self._make_dm_bot(sent_message_id=12345)
|
||||
bot._notify_ws_map = {}
|
||||
bot._MAX_NOTIFY_TRACKING = 100
|
||||
bot.send = AsyncMock(return_value="12345")
|
||||
bot.send_notification = TurnstoneBot.send_notification.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
_run(bot.send_notification("7777", "Hello", "ws-abc"))
|
||||
_run(bot.send_notification("chan-1", "Hello", "ws-abc"))
|
||||
|
||||
# Tracked under the resolved Discord user ID, not the raw argument.
|
||||
assert 12345 in bot._notify_ws_map
|
||||
assert bot._notify_ws_map[12345] == ("ws-abc", "7777")
|
||||
|
||||
def test_send_notification_to_guild_channel_is_not_tracked(self):
|
||||
"""Notifications delivered to a guild channel must not register reply tracking.
|
||||
|
||||
The reply-channel_id check treats the stored value as a Discord
|
||||
user ID, so storing a channel ID would reject every legitimate
|
||||
reply.
|
||||
"""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot._notify_ws_map = {}
|
||||
bot._MAX_NOTIFY_TRACKING = 100
|
||||
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = 99999
|
||||
|
||||
channel = MagicMock()
|
||||
channel.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
inner_bot = MagicMock()
|
||||
inner_bot.get_channel = MagicMock(return_value=channel)
|
||||
bot._bot = inner_bot
|
||||
|
||||
bot.send_notification = TurnstoneBot.send_notification.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
_run(bot.send_notification("888888", "Hello", "ws-abc"))
|
||||
|
||||
assert bot._notify_ws_map == {}
|
||||
assert bot._notify_ws_map[12345] == ("ws-abc", "chan-1")
|
||||
|
||||
def test_send_notification_evicts_old_entries(self):
|
||||
"""Oldest notification tracking entries are evicted when cap is reached."""
|
||||
bot = self._make_dm_bot(sent_message_id=4)
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._MAX_NOTIFY_TRACKING = 3
|
||||
bot._notify_ws_map = {
|
||||
1: ("ws-1", "u1"),
|
||||
2: ("ws-2", "u2"),
|
||||
3: ("ws-3", "u3"),
|
||||
}
|
||||
bot.send = AsyncMock(return_value="4")
|
||||
bot.send_notification = TurnstoneBot.send_notification.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
_run(bot.send_notification("7777", "Hello", "ws-4"))
|
||||
_run(bot.send_notification("chan-1", "Hello", "ws-4"))
|
||||
|
||||
assert 4 in bot._notify_ws_map
|
||||
assert 1 not in bot._notify_ws_map # oldest evicted
|
||||
@@ -946,7 +878,7 @@ class TestNotificationTracking:
|
||||
sent_msg.id = 88888
|
||||
dm_channel.send = AsyncMock(return_value=sent_msg)
|
||||
bot._notify_reply_channels = {"ws-1": (dm_channel, "u123")}
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
@@ -981,7 +913,7 @@ class TestNotificationTracking:
|
||||
|
||||
dm_channel = AsyncMock()
|
||||
bot._notify_reply_channels = {"ws-1": (dm_channel, "u123")}
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
@@ -1120,83 +1052,40 @@ class TestTryParseMedia:
|
||||
class TestIsSafeImageUrl:
|
||||
"""Tests for _is_safe_image_url in _formatter.py."""
|
||||
|
||||
@staticmethod
|
||||
def _patch_resolver(monkeypatch, ips):
|
||||
"""Replace socket.getaddrinfo with a stub returning *ips*."""
|
||||
import socket
|
||||
|
||||
def fake(host, port, family=0, *args, **kwargs): # noqa: ARG001
|
||||
return [(family, 0, 0, "", (ip, 0)) for ip in ips]
|
||||
|
||||
monkeypatch.setattr(socket, "getaddrinfo", fake)
|
||||
|
||||
def test_http_url(self, monkeypatch):
|
||||
def test_http_url(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
self._patch_resolver(monkeypatch, ["203.0.113.5"])
|
||||
assert _run(_is_safe_image_url("http://jellyfin:8096/Items/abc/Images/Primary")) is True
|
||||
assert _is_safe_image_url("http://jellyfin:8096/Items/abc/Images/Primary") is True
|
||||
|
||||
def test_https_url(self, monkeypatch):
|
||||
def test_https_url(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
self._patch_resolver(monkeypatch, ["203.0.113.5"])
|
||||
assert (
|
||||
_run(_is_safe_image_url("https://jellyfin.example.com/Items/abc/Images/Primary"))
|
||||
is True
|
||||
)
|
||||
assert _is_safe_image_url("https://jellyfin.example.com/Items/abc/Images/Primary") is True
|
||||
|
||||
def test_ftp_rejected(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
assert _run(_is_safe_image_url("ftp://evil.com/image.jpg")) is False
|
||||
assert _is_safe_image_url("ftp://evil.com/image.jpg") is False
|
||||
|
||||
def test_file_rejected(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
assert _run(_is_safe_image_url("file:///etc/passwd")) is False
|
||||
assert _is_safe_image_url("file:///etc/passwd") is False
|
||||
|
||||
def test_userinfo_rejected(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
assert _run(_is_safe_image_url("http://user:pass@jellyfin:8096/image")) is False
|
||||
assert _is_safe_image_url("http://user:pass@jellyfin:8096/image") is False
|
||||
|
||||
def test_empty_rejected(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
assert _run(_is_safe_image_url("")) is False
|
||||
assert _is_safe_image_url("") is False
|
||||
|
||||
def test_private_ip_allowed(self):
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
assert _run(_is_safe_image_url("http://192.168.0.6:8096/Items/abc/Images/Primary")) is True
|
||||
|
||||
def test_dns_rebinding_rejected(self, monkeypatch):
|
||||
"""Hostname that resolves to a loopback IP must be rejected."""
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
self._patch_resolver(monkeypatch, ["127.0.0.1"])
|
||||
assert _run(_is_safe_image_url("http://rebind.example.com/image")) is False
|
||||
|
||||
def test_metadata_endpoint_rejected(self):
|
||||
"""AWS/GCP metadata IP is link-local → rejected."""
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
assert _run(_is_safe_image_url("http://169.254.169.254/latest/meta-data/")) is False
|
||||
|
||||
def test_ipv6_aws_nitro_metadata_rejected(self, monkeypatch):
|
||||
"""fd00:ec2::254 is IPv6 ULA (is_private) but must be blocked —
|
||||
the IPv4 169.254.169.254 check left this analogue open."""
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
self._patch_resolver(monkeypatch, ["fd00:ec2::254"])
|
||||
assert _run(_is_safe_image_url("http://nitro.example.com/")) is False
|
||||
|
||||
def test_ipv6_ecs_task_metadata_rejected(self, monkeypatch):
|
||||
"""ECS Task Metadata lives in the same fd00:ec2::/32 prefix."""
|
||||
from turnstone.channels._formatter import _is_safe_image_url
|
||||
|
||||
self._patch_resolver(monkeypatch, ["fd00:ec2::23"])
|
||||
assert _run(_is_safe_image_url("http://ecs-meta.example.com/")) is False
|
||||
assert _is_safe_image_url("http://192.168.0.6:8096/Items/abc/Images/Primary") is True
|
||||
|
||||
|
||||
class TestBuildMediaEmbed:
|
||||
@@ -1298,7 +1187,7 @@ class TestThinkingIndicator:
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_thinking_start_sends_message(self):
|
||||
@@ -1355,7 +1244,7 @@ class TestThinkingIndicator:
|
||||
# Thinking message becomes the StreamingMessage base — no delete.
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
sm = bot._streaming["ws-1"]
|
||||
assert sm.message is thinking_msg
|
||||
assert sm._message is thinking_msg
|
||||
|
||||
def test_stream_end_clears_thinking_message(self):
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
@@ -1398,7 +1287,7 @@ class TestToolInfoEvent:
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_sends_per_item_embed(self):
|
||||
@@ -1493,7 +1382,7 @@ class TestToolResultEvent:
|
||||
bot._notify_reply_channels = {}
|
||||
bot._http_client = MagicMock()
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_marks_info_done_and_sends_result(self):
|
||||
@@ -1648,7 +1537,7 @@ class TestApprovalResolved:
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
_bind_ws_event_handlers(bot, TurnstoneBot)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_disables_buttons_on_timeout(self):
|
||||
@@ -1716,226 +1605,3 @@ class TestChannelCLI:
|
||||
main()
|
||||
|
||||
assert exc_info.value.code == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Approval / plan-review interaction views — owner-check regression tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_view_interaction(user_id: int, footer: str | None) -> MagicMock:
|
||||
"""Build a minimal interaction for ApprovalView / PlanReviewView tests."""
|
||||
interaction = MagicMock(spec=discord.Interaction)
|
||||
interaction.user = MagicMock()
|
||||
interaction.user.id = user_id
|
||||
interaction.response = MagicMock()
|
||||
interaction.response.send_message = AsyncMock()
|
||||
interaction.response.defer = AsyncMock()
|
||||
interaction.response.send_modal = AsyncMock()
|
||||
interaction.followup = MagicMock()
|
||||
interaction.followup.send = AsyncMock()
|
||||
interaction.message = MagicMock()
|
||||
if footer is None:
|
||||
interaction.message.embeds = []
|
||||
else:
|
||||
embed = MagicMock()
|
||||
embed.footer.text = footer
|
||||
interaction.message.embeds = [embed]
|
||||
return interaction
|
||||
|
||||
|
||||
def _make_view_bot() -> MagicMock:
|
||||
"""Build a TurnstoneBot double with just the surface the views read."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.router = MagicMock()
|
||||
bot.router.resolve_user = AsyncMock(return_value="turnstone-user-1")
|
||||
bot.router.send_approval = AsyncMock()
|
||||
bot.router.send_plan_feedback = AsyncMock()
|
||||
bot._pending_approval_msgs = {}
|
||||
return bot
|
||||
|
||||
|
||||
class TestApprovalViewOwnerCheck:
|
||||
"""ApprovalView rejects clicks from anyone other than the session owner."""
|
||||
|
||||
def test_owner_approve_allowed(self, monkeypatch):
|
||||
from turnstone.channels.discord.views import ApprovalView
|
||||
|
||||
# Avoid real disable_message_buttons (touches discord.ui internals).
|
||||
monkeypatch.setattr(
|
||||
"turnstone.channels.discord.views._disable_buttons",
|
||||
AsyncMock(),
|
||||
)
|
||||
view = ApprovalView(_make_view_bot())
|
||||
interaction = _make_view_interaction(user_id=42, footer="ws-1|corr-1|42")
|
||||
|
||||
_run(view._handle(interaction, approved=True, always=False))
|
||||
|
||||
view.bot.router.send_approval.assert_awaited_once_with(
|
||||
ws_id="ws-1",
|
||||
correlation_id="corr-1",
|
||||
approved=True,
|
||||
always=False,
|
||||
)
|
||||
|
||||
def test_non_owner_rejected(self):
|
||||
from turnstone.channels.discord.views import ApprovalView
|
||||
|
||||
view = ApprovalView(_make_view_bot())
|
||||
interaction = _make_view_interaction(user_id=999, footer="ws-1|corr-1|42")
|
||||
|
||||
_run(view._handle(interaction, approved=True, always=False))
|
||||
|
||||
view.bot.router.send_approval.assert_not_awaited()
|
||||
interaction.response.send_message.assert_awaited_once()
|
||||
msg_kwargs = interaction.response.send_message.call_args
|
||||
assert "Only the session owner" in msg_kwargs.args[0]
|
||||
assert msg_kwargs.kwargs.get("ephemeral") is True
|
||||
|
||||
def test_legacy_footer_without_owner_rejected(self):
|
||||
from turnstone.channels.discord.views import ApprovalView
|
||||
|
||||
view = ApprovalView(_make_view_bot())
|
||||
# Pre-upgrade footer with only ws_id|correlation_id — fail closed.
|
||||
interaction = _make_view_interaction(user_id=42, footer="ws-1|corr-1")
|
||||
|
||||
_run(view._handle(interaction, approved=True, always=False))
|
||||
|
||||
view.bot.router.send_approval.assert_not_awaited()
|
||||
|
||||
|
||||
class TestPlanReviewViewOwnerCheck:
|
||||
"""PlanReviewView rejects clicks from anyone other than the session owner."""
|
||||
|
||||
def test_owner_approve_allowed(self, monkeypatch):
|
||||
from turnstone.channels.discord.views import PlanReviewView
|
||||
|
||||
monkeypatch.setattr(
|
||||
"turnstone.channels.discord.views._disable_buttons",
|
||||
AsyncMock(),
|
||||
)
|
||||
view = PlanReviewView(_make_view_bot())
|
||||
interaction = _make_view_interaction(user_id=42, footer="ws-1|corr-1|42")
|
||||
|
||||
_run(view._handle_approve(interaction))
|
||||
|
||||
view.bot.router.send_plan_feedback.assert_awaited_once_with(
|
||||
ws_id="ws-1",
|
||||
correlation_id="corr-1",
|
||||
feedback="",
|
||||
)
|
||||
|
||||
def test_non_owner_approve_rejected(self):
|
||||
from turnstone.channels.discord.views import PlanReviewView
|
||||
|
||||
view = PlanReviewView(_make_view_bot())
|
||||
interaction = _make_view_interaction(user_id=999, footer="ws-1|corr-1|42")
|
||||
|
||||
_run(view._handle_approve(interaction))
|
||||
|
||||
view.bot.router.send_plan_feedback.assert_not_awaited()
|
||||
interaction.response.send_message.assert_awaited_once()
|
||||
|
||||
def test_non_owner_changes_modal_rejected(self):
|
||||
from turnstone.channels.discord.views import PlanReviewView
|
||||
|
||||
view = PlanReviewView(_make_view_bot())
|
||||
interaction = _make_view_interaction(user_id=999, footer="ws-1|corr-1|42")
|
||||
|
||||
_run(view._handle_changes(interaction))
|
||||
|
||||
interaction.response.send_modal.assert_not_awaited()
|
||||
interaction.response.send_message.assert_awaited_once()
|
||||
|
||||
|
||||
class TestDiscordThreadOwnerCheck:
|
||||
"""Sec-3 gate: only the thread creator can send messages into the workstream."""
|
||||
|
||||
@staticmethod
|
||||
def _make_cog_and_ts():
|
||||
"""Build a MessageCog wired to a minimal TurnstoneBot double."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
bot.user.mentioned_in = MagicMock(return_value=False)
|
||||
|
||||
ts = MagicMock()
|
||||
ts._is_allowed_channel = MagicMock(return_value=True)
|
||||
ts.storage = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.lookup_ws_id = AsyncMock(return_value="ws-1")
|
||||
ts.router.resolve_user = AsyncMock(return_value="turnstone-user-1")
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts.router.get_or_create_workstream = AsyncMock(return_value=("ws-1", False))
|
||||
ts.config = MagicMock()
|
||||
ts._ws_tasks = {}
|
||||
ts._subscribed_ws = {"ws-1"}
|
||||
ts._notify_ws_map = {}
|
||||
ts._notify_reply_channels = {}
|
||||
ts.get_thread_invoker = MagicMock(return_value=None)
|
||||
ts.subscribe_ws = AsyncMock()
|
||||
bot.turnstone = ts
|
||||
|
||||
return MessageCog(bot), ts
|
||||
|
||||
def test_non_owner_thread_message_dropped(self):
|
||||
"""A linked user who is NOT the thread creator gets their message
|
||||
silently dropped — router.send_message must not fire."""
|
||||
cog, ts = self._make_cog_and_ts()
|
||||
|
||||
# Build a thread whose owner_id is different from the message author.
|
||||
thread = MagicMock(spec=discord.Thread)
|
||||
thread.id = 555
|
||||
thread.parent_id = 111
|
||||
thread.owner_id = 42 # thread creator
|
||||
thread.name = "some-thread"
|
||||
|
||||
msg = _make_message(guild=True, channel=thread)
|
||||
msg.author.id = 999 # non-owner trying to inject
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
ts.router.get_or_create_workstream.assert_not_awaited()
|
||||
|
||||
def test_ask_thread_followup_allowed_when_invoker_registered(self):
|
||||
"""/ask creates threads with owner_id=bot; follow-ups from the
|
||||
registered invoker must still reach the workstream."""
|
||||
cog, ts = self._make_cog_and_ts()
|
||||
# Simulate what _cmd_ask does after channel.create_thread().
|
||||
ts.get_thread_invoker = MagicMock(return_value=111)
|
||||
|
||||
thread = MagicMock(spec=discord.Thread)
|
||||
thread.id = 555
|
||||
thread.parent_id = 222
|
||||
thread.owner_id = 99999 # bot owns the thread after channel.create_thread
|
||||
thread.name = "ask-thread"
|
||||
|
||||
msg = _make_message(guild=True, channel=thread)
|
||||
msg.author.id = 111 # the human who ran /ask
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_awaited_once_with("ws-1", msg.content)
|
||||
|
||||
def test_ask_thread_rejects_other_user_even_when_invoker_registered(self):
|
||||
"""Registered invoker lock: only that user's follow-ups pass."""
|
||||
cog, ts = self._make_cog_and_ts()
|
||||
ts.get_thread_invoker = MagicMock(return_value=111)
|
||||
|
||||
thread = MagicMock(spec=discord.Thread)
|
||||
thread.id = 555
|
||||
thread.parent_id = 222
|
||||
thread.owner_id = 99999 # bot-owned
|
||||
thread.name = "ask-thread"
|
||||
|
||||
msg = _make_message(guild=True, channel=thread)
|
||||
msg.author.id = 222 # someone other than the recorded invoker
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
@@ -1,13 +1,55 @@
|
||||
"""Tests for turnstone.channels._formatter."""
|
||||
"""Tests for turnstone.channels._protocol and turnstone.channels._formatter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from turnstone.channels._formatter import (
|
||||
chunk_message,
|
||||
format_approval_request,
|
||||
format_plan_review,
|
||||
format_verdict,
|
||||
truncate,
|
||||
)
|
||||
from turnstone.channels._protocol import ChannelEvent
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ChannelEvent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChannelEvent:
|
||||
def test_construction(self) -> None:
|
||||
evt = ChannelEvent(
|
||||
channel_type="discord",
|
||||
channel_id="ch-1",
|
||||
channel_user_id="u-42",
|
||||
message="hello",
|
||||
parent_channel_id="parent",
|
||||
metadata={"key": "val"},
|
||||
)
|
||||
assert evt.channel_type == "discord"
|
||||
assert evt.channel_id == "ch-1"
|
||||
assert evt.channel_user_id == "u-42"
|
||||
assert evt.message == "hello"
|
||||
assert evt.parent_channel_id == "parent"
|
||||
assert evt.metadata == {"key": "val"}
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
evt = ChannelEvent(
|
||||
channel_type="slack",
|
||||
channel_id="ch-2",
|
||||
channel_user_id="u-7",
|
||||
message="hi",
|
||||
)
|
||||
assert evt.parent_channel_id == ""
|
||||
assert evt.metadata == {}
|
||||
|
||||
def test_metadata_independence(self) -> None:
|
||||
"""Default metadata dicts are independent across instances."""
|
||||
a = ChannelEvent(channel_type="x", channel_id="1", channel_user_id="u", message="m")
|
||||
b = ChannelEvent(channel_type="x", channel_id="2", channel_user_id="u", message="m")
|
||||
a.metadata["key"] = "val"
|
||||
assert "key" not in b.metadata
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# chunk_message
|
||||
@@ -130,6 +172,18 @@ class TestFormatApprovalRequest:
|
||||
assert "/etc/hosts" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# format_plan_review
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFormatPlanReview:
|
||||
def test_format(self) -> None:
|
||||
result = format_plan_review("Step 1: do stuff")
|
||||
assert result.startswith("**Plan review requested:**")
|
||||
assert "Step 1: do stuff" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# format_verdict
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+9
-184
@@ -62,20 +62,12 @@ def _make_bot() -> tuple[object, MagicMock, MagicMock]:
|
||||
storage = MagicMock()
|
||||
storage.list_channel_routes_by_type = MagicMock(return_value=[])
|
||||
|
||||
from turnstone.channels._routing import PolicyVerdict
|
||||
|
||||
router = MagicMock()
|
||||
router.get_or_create_workstream = AsyncMock(return_value=("ws-1", True))
|
||||
router.send_message = AsyncMock()
|
||||
router.send_approval = AsyncMock()
|
||||
router.send_plan_feedback = AsyncMock()
|
||||
router.get_node_url = AsyncMock(return_value="http://localhost:8080")
|
||||
router.evaluate_tool_policies = AsyncMock(return_value=PolicyVerdict(kind="none"))
|
||||
router.delete_route = AsyncMock()
|
||||
router.close_workstream = AsyncMock()
|
||||
# Default: every test Slack user is already linked. Tests that
|
||||
# exercise the unlinked path override this per-instance.
|
||||
router.resolve_user = AsyncMock(return_value="turnstone-user-1")
|
||||
router.aclose = AsyncMock()
|
||||
|
||||
client = AsyncMock()
|
||||
@@ -193,98 +185,6 @@ class TestSlackRoute:
|
||||
== "C123:U456:111.222"
|
||||
)
|
||||
|
||||
def test_round_trip(self) -> None:
|
||||
"""Every shape emitted by to_channel_id must round-trip through parse."""
|
||||
from turnstone.channels.slack.routes import SlackRoute
|
||||
|
||||
shapes = [
|
||||
SlackRoute(channel="C123"),
|
||||
SlackRoute(channel="C123", user_id="U456"),
|
||||
SlackRoute(channel="C123", user_id="U456", thread_ts="111.222"),
|
||||
]
|
||||
for route in shapes:
|
||||
assert SlackRoute.parse(route.to_channel_id()) == route
|
||||
|
||||
def test_parse_trailing_colon_normalizes(self) -> None:
|
||||
"""``"C123:"`` should normalize to ``SlackRoute("C123")``."""
|
||||
from turnstone.channels.slack.routes import SlackRoute
|
||||
|
||||
assert SlackRoute.parse("C123:") == SlackRoute(channel="C123")
|
||||
assert SlackRoute.parse("C123:U456:") == SlackRoute(channel="C123", user_id="U456")
|
||||
|
||||
def test_parse_extra_colons_folded_into_thread_ts(self) -> None:
|
||||
"""Extra ``:`` past the third field fold into ``thread_ts`` verbatim.
|
||||
|
||||
Slack IDs and timestamps never contain ``:`` so this is safe in
|
||||
practice; the test locks the documented behaviour.
|
||||
"""
|
||||
from turnstone.channels.slack.routes import SlackRoute
|
||||
|
||||
route = SlackRoute.parse("C1:U1:ts:extra")
|
||||
assert route.channel == "C1"
|
||||
assert route.user_id == "U1"
|
||||
assert route.thread_ts == "ts:extra"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Route recovery + session archival
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRecoverRoutes:
|
||||
"""Tests for TurnstoneSlackBot._recover_routes on bot startup."""
|
||||
|
||||
def test_latest_ts_per_user_wins(self) -> None:
|
||||
"""When multiple routes exist for the same (channel, user), the
|
||||
newest thread_ts populates _channel_sessions."""
|
||||
bot, _router, _client = _make_bot()
|
||||
bot.storage.list_channel_routes_by_type = MagicMock( # type: ignore[attr-defined]
|
||||
return_value=[
|
||||
{"ws_id": "ws-old", "channel_id": "C1:U1:1000000.000001"},
|
||||
{"ws_id": "ws-new", "channel_id": "C1:U1:2000000.000001"},
|
||||
]
|
||||
)
|
||||
bot.subscribe_ws = AsyncMock() # type: ignore[attr-defined]
|
||||
|
||||
_run(bot._recover_routes()) # type: ignore[attr-defined]
|
||||
|
||||
assert bot._channel_sessions == {("C1", "U1"): ("ws-new", "2000000.000001")} # type: ignore[attr-defined]
|
||||
# Both routes get resubscribed so their SSE streams stay active.
|
||||
assert bot.subscribe_ws.await_count == 2 # type: ignore[attr-defined]
|
||||
|
||||
def test_non_threaded_routes_skip_session_table(self) -> None:
|
||||
"""A route without a thread_ts still gets subscribed but never
|
||||
populates _channel_sessions (DMs fall into this shape)."""
|
||||
bot, _router, _client = _make_bot()
|
||||
bot.storage.list_channel_routes_by_type = MagicMock( # type: ignore[attr-defined]
|
||||
return_value=[{"ws_id": "ws-dm", "channel_id": "D1:U9"}]
|
||||
)
|
||||
bot.subscribe_ws = AsyncMock() # type: ignore[attr-defined]
|
||||
|
||||
_run(bot._recover_routes()) # type: ignore[attr-defined]
|
||||
|
||||
assert bot._channel_sessions == {} # type: ignore[attr-defined]
|
||||
bot.subscribe_ws.assert_awaited_once_with("ws-dm", "D1:U9") # type: ignore[attr-defined]
|
||||
|
||||
|
||||
class TestArchiveSession:
|
||||
"""Tests for TurnstoneSlackBot._archive_session cleanup."""
|
||||
|
||||
def test_archive_drops_route_and_closes_workstream(self) -> None:
|
||||
bot, router, client = _make_bot()
|
||||
bot._channel_sessions[("C1", "U1")] = ("ws-old", "1000000.000001") # type: ignore[attr-defined]
|
||||
bot._subscribed_ws.add("ws-old") # type: ignore[attr-defined]
|
||||
|
||||
_run(bot._archive_session("C1", "U1", "ws-old", "1000000.000001")) # type: ignore[attr-defined]
|
||||
|
||||
router.delete_route.assert_awaited_once_with( # type: ignore[attr-defined]
|
||||
"slack", "C1:U1:1000000.000001"
|
||||
)
|
||||
router.close_workstream.assert_awaited_once_with("ws-old") # type: ignore[attr-defined]
|
||||
assert ("C1", "U1") not in bot._channel_sessions # type: ignore[attr-defined]
|
||||
# archive notice posted in the old thread
|
||||
client.chat_postMessage.assert_awaited() # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preview sanitization
|
||||
@@ -341,7 +241,7 @@ class TestStreamingMessage:
|
||||
_run(sm.append("hello "))
|
||||
_run(sm.append("world"))
|
||||
|
||||
assert sm.accumulated_text == "hello world"
|
||||
assert "".join(sm._buffer) == "hello world"
|
||||
|
||||
def test_finalize_sends_when_no_prior_message(self) -> None:
|
||||
from turnstone.channels.slack.bot import StreamingMessage
|
||||
@@ -364,7 +264,7 @@ class TestStreamingMessage:
|
||||
sm = StreamingMessage(client=client, channel="C1", edit_interval=0.0)
|
||||
|
||||
_run(sm.append("hi"))
|
||||
assert sm.message_ts == "123"
|
||||
assert sm._ts == "123"
|
||||
|
||||
_run(sm.finalize())
|
||||
client.chat_update.assert_awaited()
|
||||
@@ -589,7 +489,7 @@ class TestApprovalOwnership:
|
||||
"container": {"channel_id": "C01SAPU5414", "message_ts": "111.222"},
|
||||
}
|
||||
|
||||
_run(bot._resolve_approval(AsyncMock(), body, approved=True)) # type: ignore[attr-defined]
|
||||
_run(bot._on_approve(AsyncMock(), body)) # type: ignore[attr-defined]
|
||||
|
||||
client.chat_postEphemeral.assert_awaited_once()
|
||||
router.send_approval.assert_not_awaited()
|
||||
@@ -611,7 +511,7 @@ class TestApprovalOwnership:
|
||||
"container": {"channel_id": "C01SAPU5414", "message_ts": "111.222"},
|
||||
}
|
||||
|
||||
_run(bot._resolve_approval(AsyncMock(), body, approved=False)) # type: ignore[attr-defined]
|
||||
_run(bot._on_deny(AsyncMock(), body)) # type: ignore[attr-defined]
|
||||
|
||||
client.chat_postEphemeral.assert_awaited_once()
|
||||
router.send_approval.assert_not_awaited()
|
||||
@@ -633,7 +533,7 @@ class TestApprovalOwnership:
|
||||
"container": {"channel_id": "C01SAPU5414", "message_ts": "111.222"},
|
||||
}
|
||||
|
||||
_run(bot._resolve_approval(AsyncMock(), body, approved=True)) # type: ignore[attr-defined]
|
||||
_run(bot._on_approve(AsyncMock(), body)) # type: ignore[attr-defined]
|
||||
|
||||
router.send_approval.assert_awaited_once_with(ws_id, "corr-1", approved=True)
|
||||
client.chat_update.assert_awaited_once()
|
||||
@@ -648,7 +548,6 @@ class TestWsEventDispatch:
|
||||
"""Tests for SSE event handling in the Slack bot."""
|
||||
|
||||
def _make_ws_bot(self) -> tuple[object, MagicMock]:
|
||||
from turnstone.channels._routing import PolicyVerdict
|
||||
from turnstone.channels.slack.bot import TurnstoneSlackBot
|
||||
from turnstone.channels.slack.config import SlackConfig
|
||||
|
||||
@@ -661,8 +560,6 @@ class TestWsEventDispatch:
|
||||
router = MagicMock()
|
||||
router.send_approval = AsyncMock()
|
||||
router.send_plan_feedback = AsyncMock()
|
||||
router.evaluate_tool_policies = AsyncMock(return_value=PolicyVerdict(kind="none"))
|
||||
router.resolve_user = AsyncMock(return_value="turnstone-user-1")
|
||||
client = AsyncMock()
|
||||
client.chat_postMessage = AsyncMock(return_value={"ok": True, "ts": "123"})
|
||||
client.chat_update = AsyncMock(return_value={"ok": True})
|
||||
@@ -736,7 +633,6 @@ class TestWsEventDispatch:
|
||||
assert "Something went wrong" in text
|
||||
|
||||
def test_approve_request_auto_approve(self) -> None:
|
||||
from turnstone.channels._routing import PolicyVerdict
|
||||
from turnstone.channels.slack.bot import TurnstoneSlackBot
|
||||
from turnstone.channels.slack.config import SlackConfig
|
||||
from turnstone.channels.slack.routes import SlackRoute
|
||||
@@ -746,7 +642,6 @@ class TestWsEventDispatch:
|
||||
storage = MagicMock()
|
||||
router = MagicMock()
|
||||
router.send_approval = AsyncMock()
|
||||
router.evaluate_tool_policies = AsyncMock(return_value=PolicyVerdict(kind="none"))
|
||||
client = AsyncMock()
|
||||
client.chat_postMessage = AsyncMock(return_value={"ok": True, "ts": "123"})
|
||||
|
||||
@@ -854,11 +749,8 @@ class TestWsEventDispatch:
|
||||
|
||||
def test_plan_approve_sends_feedback_and_updates_message(self) -> None:
|
||||
bot, client = self._make_ws_bot()
|
||||
# Register pending review with an owner so the new sec-2 gate passes.
|
||||
bot._pending_plan_review_ts["ws-1"] = ("C1", "111.222", "U_OWNER") # type: ignore[attr-defined]
|
||||
body = {
|
||||
"actions": [{"value": "ws-1"}],
|
||||
"user": {"id": "U_OWNER"},
|
||||
"container": {"channel_id": "C1", "message_ts": "111.222"},
|
||||
}
|
||||
|
||||
@@ -867,23 +759,9 @@ class TestWsEventDispatch:
|
||||
bot.router.send_plan_feedback.assert_awaited_once_with("ws-1", "", "") # type: ignore[attr-defined]
|
||||
client.chat_update.assert_awaited_once()
|
||||
|
||||
def test_plan_approve_rejects_non_owner(self) -> None:
|
||||
bot, client = self._make_ws_bot()
|
||||
bot._pending_plan_review_ts["ws-1"] = ("C1", "111.222", "U_OWNER") # type: ignore[attr-defined]
|
||||
body = {
|
||||
"actions": [{"value": "ws-1"}],
|
||||
"user": {"id": "U_OTHER"},
|
||||
"container": {"channel_id": "C1", "message_ts": "111.222"},
|
||||
}
|
||||
|
||||
_run(bot._on_plan_approve(AsyncMock(), body)) # type: ignore[attr-defined]
|
||||
|
||||
bot.router.send_plan_feedback.assert_not_awaited() # type: ignore[attr-defined]
|
||||
client.chat_postEphemeral.assert_awaited_once()
|
||||
|
||||
def test_plan_feedback_modal_sends_feedback_and_updates_message(self) -> None:
|
||||
bot, client = self._make_ws_bot()
|
||||
bot._pending_plan_review_ts["ws-1"] = ("C1", "111.222", "U_OWNER") # type: ignore[attr-defined]
|
||||
bot._pending_plan_review_ts["ws-1"] = ("C1", "111.222") # type: ignore[attr-defined]
|
||||
|
||||
view = {
|
||||
"private_metadata": "ws-1",
|
||||
@@ -891,9 +769,8 @@ class TestWsEventDispatch:
|
||||
"values": {"feedback_block": {"feedback_input": {"value": "please revise step 2"}}}
|
||||
},
|
||||
}
|
||||
body = {"user": {"id": "U_OWNER"}}
|
||||
|
||||
_run(bot._on_plan_feedback_modal(AsyncMock(), body, view)) # type: ignore[attr-defined]
|
||||
_run(bot._on_plan_feedback_modal(AsyncMock(), {}, view)) # type: ignore[attr-defined]
|
||||
|
||||
bot.router.send_plan_feedback.assert_awaited_once_with( # type: ignore[attr-defined]
|
||||
"ws-1",
|
||||
@@ -902,49 +779,6 @@ class TestWsEventDispatch:
|
||||
)
|
||||
client.chat_update.assert_awaited_once()
|
||||
|
||||
def test_link_prefix_does_not_hijack_regular_prompt(self) -> None:
|
||||
"""`/turnstone linking up the docs` must not misroute into
|
||||
_handle_link with `"ing up the docs"` as the token."""
|
||||
bot, router, client = _make_bot()
|
||||
bot._handle_link = AsyncMock() # type: ignore[attr-defined]
|
||||
# Force the linked-user gate to pass so the natural-language
|
||||
# prompt can flow through to the session-start branch.
|
||||
router.get_or_create_workstream = AsyncMock(return_value=("ws-new", True))
|
||||
body = {
|
||||
"channel_id": "C01SAPU5414",
|
||||
"user_id": "U111",
|
||||
"text": "linking up the docs",
|
||||
}
|
||||
_run(bot._on_slash_command(AsyncMock(), body)) # type: ignore[attr-defined]
|
||||
bot._handle_link.assert_not_awaited() # type: ignore[attr-defined]
|
||||
|
||||
def test_link_rate_limit_blocks_after_cap(self) -> None:
|
||||
"""Sec-3: /turnstone link must throttle at _LINK_RATE_LIMIT/hour."""
|
||||
from turnstone.channels.slack.bot import _LINK_RATE_LIMIT
|
||||
|
||||
bot, _router, client = _make_bot()
|
||||
# Make the user already linked so _handle_link skips past the
|
||||
# rate limit check would otherwise take a slot on a successful
|
||||
# storage hit; we still want to exercise the throttle directly.
|
||||
for _ in range(_LINK_RATE_LIMIT):
|
||||
assert bot._allow_link_attempt("U111") # type: ignore[attr-defined]
|
||||
# Next attempt is blocked.
|
||||
assert not bot._allow_link_attempt("U111") # type: ignore[attr-defined]
|
||||
|
||||
def test_plan_feedback_modal_rejects_non_owner(self) -> None:
|
||||
bot, _client = self._make_ws_bot()
|
||||
bot._pending_plan_review_ts["ws-1"] = ("C1", "111.222", "U_OWNER") # type: ignore[attr-defined]
|
||||
|
||||
view = {
|
||||
"private_metadata": "ws-1",
|
||||
"state": {"values": {"feedback_block": {"feedback_input": {"value": "please revise"}}}},
|
||||
}
|
||||
body = {"user": {"id": "U_OTHER"}}
|
||||
|
||||
_run(bot._on_plan_feedback_modal(AsyncMock(), body, view)) # type: ignore[attr-defined]
|
||||
|
||||
bot.router.send_plan_feedback.assert_not_awaited() # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notification tracking
|
||||
@@ -1135,9 +969,6 @@ class TestChannelCLI:
|
||||
async def start(self) -> None:
|
||||
return None
|
||||
|
||||
async def stop(self) -> None:
|
||||
return None
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, _config) -> None:
|
||||
pass
|
||||
@@ -1149,7 +980,7 @@ class TestChannelCLI:
|
||||
created_adapters.update(adapters)
|
||||
return MagicMock()
|
||||
|
||||
async def _fake_gather(*aws, return_exceptions=False): # type: ignore[no-untyped-def]
|
||||
async def _fake_gather(*aws): # type: ignore[no-untyped-def]
|
||||
for aw in aws:
|
||||
await aw
|
||||
return []
|
||||
@@ -1204,9 +1035,6 @@ class TestChannelCLI:
|
||||
async def start(self) -> None:
|
||||
return None
|
||||
|
||||
async def stop(self) -> None:
|
||||
return None
|
||||
|
||||
class FakeDiscordBot:
|
||||
channel_type = "discord"
|
||||
|
||||
@@ -1216,9 +1044,6 @@ class TestChannelCLI:
|
||||
async def start(self) -> None:
|
||||
return None
|
||||
|
||||
async def stop(self) -> None:
|
||||
return None
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, _config) -> None:
|
||||
pass
|
||||
@@ -1230,7 +1055,7 @@ class TestChannelCLI:
|
||||
created_adapters.update(adapters)
|
||||
return MagicMock()
|
||||
|
||||
async def _fake_gather(*aws, return_exceptions=False): # type: ignore[no-untyped-def]
|
||||
async def _fake_gather(*aws): # type: ignore[no-untyped-def]
|
||||
for aw in aws:
|
||||
await aw
|
||||
return []
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
"""Tests for the shared SSE reconnect helper in turnstone.channels._sse."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
def _run(coro): # type: ignore[no-untyped-def]
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
class _FakeSSEEvent:
|
||||
"""A fake ``httpx_sse.ServerSentEvent`` with the subset we read."""
|
||||
|
||||
def __init__(self, event: str, data: str) -> None:
|
||||
self.event = event
|
||||
self.data = data
|
||||
|
||||
|
||||
class _FakeEventSource:
|
||||
"""Context manager returned by our fake ``aconnect_sse``.
|
||||
|
||||
Captures the (status_code, events) the test wants to deliver.
|
||||
``aiter_sse`` yields the events then returns; the caller then hits
|
||||
the outer ``while True`` loop again, which will pick up the next
|
||||
queued response via the shared iterator state on _FakeConnect.
|
||||
"""
|
||||
|
||||
def __init__(self, *, status_code: int, events: list[_FakeSSEEvent]) -> None:
|
||||
self.response = SimpleNamespace(
|
||||
status_code=status_code,
|
||||
request=MagicMock(),
|
||||
)
|
||||
self._events = events
|
||||
|
||||
async def __aenter__(self) -> _FakeEventSource:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None: # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def aiter_sse(self): # type: ignore[no-untyped-def]
|
||||
for event in self._events:
|
||||
yield event
|
||||
|
||||
|
||||
class _FakeConnect:
|
||||
"""Drop-in replacement for ``httpx_sse.aconnect_sse``.
|
||||
|
||||
On each call, pops the next ``_FakeEventSource`` from *queue*. When
|
||||
the queue is empty, raises ``asyncio.CancelledError`` so the loop
|
||||
terminates cleanly in tests.
|
||||
"""
|
||||
|
||||
def __init__(self, queue: list[_FakeEventSource]) -> None:
|
||||
self._queue = queue
|
||||
self.call_count = 0
|
||||
|
||||
def __call__(self, *args, **kwargs): # noqa: ANN001, ANN204
|
||||
self.call_count += 1
|
||||
if not self._queue:
|
||||
raise asyncio.CancelledError
|
||||
return self._queue.pop(0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fast_sleep(monkeypatch):
|
||||
"""Patch asyncio.sleep so backoff doesn't actually wait; record calls."""
|
||||
sleeps: list[float] = []
|
||||
|
||||
async def fake_sleep(delay: float) -> None:
|
||||
sleeps.append(delay)
|
||||
|
||||
monkeypatch.setattr("turnstone.channels._sse.asyncio.sleep", fake_sleep)
|
||||
return sleeps
|
||||
|
||||
|
||||
def _valid_event_data(ws_id: str = "ws-1") -> str:
|
||||
"""A payload ``ServerEvent.from_dict`` will accept (a ContentEvent)."""
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "content",
|
||||
"ws_id": ws_id,
|
||||
"text": "hello",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 404 → on_stale + exit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStaleRoute:
|
||||
def test_404_calls_on_stale_and_returns(self, monkeypatch, _fast_sleep):
|
||||
from turnstone.channels import _sse
|
||||
|
||||
queue = [_FakeEventSource(status_code=404, events=[])]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
on_stale = AsyncMock()
|
||||
on_event = AsyncMock()
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=on_event,
|
||||
on_stale=on_stale,
|
||||
)
|
||||
)
|
||||
|
||||
on_stale.assert_awaited_once()
|
||||
on_event.assert_not_awaited()
|
||||
# No reconnect after 404.
|
||||
assert fake_connect.call_count == 1
|
||||
assert _fast_sleep == []
|
||||
|
||||
def test_on_stale_exception_still_exits(self, monkeypatch, _fast_sleep):
|
||||
"""If on_stale raises, the loop must not reconnect."""
|
||||
from turnstone.channels import _sse
|
||||
|
||||
queue = [_FakeEventSource(status_code=404, events=[])]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
on_stale = AsyncMock(side_effect=RuntimeError("storage down"))
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=AsyncMock(),
|
||||
on_stale=on_stale,
|
||||
)
|
||||
)
|
||||
|
||||
on_stale.assert_awaited_once()
|
||||
# Still a single connect — no livelock.
|
||||
assert fake_connect.call_count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 500+ → exponential backoff
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBackoff:
|
||||
def test_500_triggers_backoff_and_retries(self, monkeypatch, _fast_sleep):
|
||||
from turnstone.channels import _sse
|
||||
|
||||
queue = [
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=AsyncMock(),
|
||||
on_stale=AsyncMock(),
|
||||
)
|
||||
)
|
||||
|
||||
assert fake_connect.call_count >= 3
|
||||
# First three recorded sleeps are 2s, 4s, 8s (starts at
|
||||
# SSE_RECONNECT_DELAY, doubles each time, capped at
|
||||
# SSE_MAX_RECONNECT_DELAY).
|
||||
assert _fast_sleep[0] == _sse.SSE_RECONNECT_DELAY
|
||||
assert _fast_sleep[1] == _sse.SSE_RECONNECT_DELAY * 2
|
||||
assert _fast_sleep[2] == _sse.SSE_RECONNECT_DELAY * 4
|
||||
|
||||
def test_backoff_resets_after_successful_dispatch(self, monkeypatch, _fast_sleep):
|
||||
"""After a 200 + successful event dispatch, the next error
|
||||
restarts backoff at the initial delay."""
|
||||
from turnstone.channels import _sse
|
||||
|
||||
good_event = _FakeSSEEvent(event="message", data=_valid_event_data())
|
||||
queue = [
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
_FakeEventSource(status_code=200, events=[good_event]),
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
on_event = AsyncMock()
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=on_event,
|
||||
on_stale=AsyncMock(),
|
||||
)
|
||||
)
|
||||
|
||||
on_event.assert_awaited()
|
||||
# Sleep sequence: 2 (after first 503), 2 (reset after 200/event),
|
||||
# then CancelledError exits. First two sleeps are both the base
|
||||
# delay — the reset did its job.
|
||||
assert len(_fast_sleep) >= 2
|
||||
assert _fast_sleep[0] == _sse.SSE_RECONNECT_DELAY
|
||||
assert _fast_sleep[1] == _sse.SSE_RECONNECT_DELAY
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Event dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEventDispatch:
|
||||
def test_invalid_json_is_skipped(self, monkeypatch, _fast_sleep):
|
||||
from turnstone.channels import _sse
|
||||
|
||||
bad = _FakeSSEEvent(event="message", data="{not json")
|
||||
good = _FakeSSEEvent(event="message", data=_valid_event_data())
|
||||
queue = [_FakeEventSource(status_code=200, events=[bad, good])]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
on_event = AsyncMock()
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=on_event,
|
||||
on_stale=AsyncMock(),
|
||||
)
|
||||
)
|
||||
|
||||
# Good event delivered, bad one silently dropped.
|
||||
assert on_event.await_count == 1
|
||||
|
||||
def test_on_event_exception_does_not_kill_stream(self, monkeypatch, _fast_sleep):
|
||||
from turnstone.channels import _sse
|
||||
|
||||
e1 = _FakeSSEEvent(event="message", data=_valid_event_data())
|
||||
e2 = _FakeSSEEvent(event="message", data=_valid_event_data())
|
||||
queue = [_FakeEventSource(status_code=200, events=[e1, e2])]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
on_event = AsyncMock(side_effect=[RuntimeError("boom"), None])
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=on_event,
|
||||
on_stale=AsyncMock(),
|
||||
)
|
||||
)
|
||||
|
||||
# Both events attempted — first raised but second still delivered.
|
||||
assert on_event.await_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTokenFactory:
|
||||
def test_header_refreshed_per_connection(self, monkeypatch, _fast_sleep):
|
||||
"""token_factory is called once per reconnect so rotating service
|
||||
JWTs stay fresh."""
|
||||
from turnstone.channels import _sse
|
||||
|
||||
# Two reconnects followed by CancelledError to exit.
|
||||
queue = [
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
_FakeEventSource(status_code=503, events=[]),
|
||||
]
|
||||
fake_connect = _FakeConnect(queue)
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
tokens: list[str] = []
|
||||
|
||||
def factory() -> str:
|
||||
tok = f"tok-{len(tokens)}"
|
||||
tokens.append(tok)
|
||||
return tok
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=factory,
|
||||
on_event=AsyncMock(),
|
||||
on_stale=AsyncMock(),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(tokens) >= 2
|
||||
assert tokens[0] != tokens[1]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# httpx errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTransportErrors:
|
||||
def test_connect_error_falls_through_to_backoff(self, monkeypatch, _fast_sleep):
|
||||
"""ConnectError is caught and treated as retryable."""
|
||||
from turnstone.channels import _sse
|
||||
|
||||
call_order = {"n": 0}
|
||||
|
||||
def fake_connect(*args, **kwargs): # noqa: ANN001, ANN003
|
||||
call_order["n"] += 1
|
||||
if call_order["n"] == 1:
|
||||
raise httpx.ConnectError("boom")
|
||||
# Second attempt: signal the loop to exit.
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(_sse.httpx_sse, "aconnect_sse", fake_connect)
|
||||
|
||||
async def node_url_fn(ws_id: str) -> str:
|
||||
return "http://node"
|
||||
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
_run(
|
||||
_sse.run_sse_stream(
|
||||
http_client=MagicMock(),
|
||||
log_prefix="test",
|
||||
ws_id="ws-1",
|
||||
node_url_fn=node_url_fn,
|
||||
token_factory=None,
|
||||
on_event=AsyncMock(),
|
||||
on_stale=AsyncMock(),
|
||||
)
|
||||
)
|
||||
|
||||
assert call_order["n"] == 2
|
||||
# Backoff ran once after the ConnectError.
|
||||
assert _fast_sleep == [_sse.SSE_RECONNECT_DELAY]
|
||||
@@ -1,184 +0,0 @@
|
||||
"""Server-side tests for the close_workstream handler's close_reason
|
||||
persistence — guards the seam that lets coordinator inspect surface
|
||||
why a workstream was retired without scraping the audit log.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
import turnstone.server as srv_mod
|
||||
from turnstone.core.auth import JWT_AUD_SERVER, create_jwt
|
||||
from turnstone.core.metrics import MetricsCollector
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
_JWT_SECRET = "test-jwt-secret-minimum-32-chars!"
|
||||
|
||||
|
||||
def _full_hdr() -> dict[str, str]:
|
||||
return {
|
||||
"Authorization": (
|
||||
f"Bearer {create_jwt('u1', frozenset({'read', 'write', 'approve'}), 'test', _JWT_SECRET, audience=JWT_AUD_SERVER)}"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _make_app(storage: Any) -> TestClient:
|
||||
srv_mod._metrics = MetricsCollector()
|
||||
srv_mod._metrics.model = "test-model"
|
||||
mock_session = MagicMock()
|
||||
mock_ws = MagicMock()
|
||||
mock_ws.id = "ws-target"
|
||||
mock_ws.name = "test"
|
||||
mock_ws.state = WorkstreamState.IDLE
|
||||
mock_ws.session = mock_session
|
||||
# Tenant gate (#375) checks ws.user_id == JWT subject; explicit set
|
||||
# so MagicMock's auto-generated truthy attribute doesn't reject the
|
||||
# request before the persistence path runs. kind / parent_ws_id
|
||||
# land in the audit_detail dict alongside ``reason``.
|
||||
mock_ws.user_id = "u1"
|
||||
mock_ws.kind = "interactive"
|
||||
mock_ws.parent_ws_id = None
|
||||
mock_mgr = MagicMock()
|
||||
mock_mgr.get.return_value = mock_ws
|
||||
mock_mgr.close.return_value = True
|
||||
mock_mgr.list_all.return_value = [mock_ws]
|
||||
mock_mgr.max_active = 10
|
||||
|
||||
app = srv_mod.create_app(
|
||||
workstreams=mock_mgr,
|
||||
global_queue=queue.Queue(),
|
||||
global_listeners=[],
|
||||
global_listeners_lock=threading.Lock(),
|
||||
skip_permissions=False,
|
||||
jwt_secret=_JWT_SECRET,
|
||||
auth_storage=storage,
|
||||
cors_origins=["*"],
|
||||
)
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "close.db"))
|
||||
|
||||
|
||||
def test_close_with_reason_persists_to_workstream_config(storage):
|
||||
client = _make_app(storage)
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={"reason": "task complete"},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
cfg = storage.load_workstream_config("ws-target")
|
||||
assert cfg.get("close_reason") == "task complete"
|
||||
|
||||
|
||||
def test_close_without_reason_does_not_touch_config(storage):
|
||||
client = _make_app(storage)
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
cfg = storage.load_workstream_config("ws-target")
|
||||
assert "close_reason" not in cfg
|
||||
|
||||
|
||||
def test_close_reason_capped_at_512_bytes(storage):
|
||||
"""A model that dumps a multi-KB blob (or a captured secret) into the
|
||||
close reason must not be able to grow the workstream_config row
|
||||
without bound — the handler enforces a 512-byte ceiling. Tested
|
||||
with ASCII (1B/char) so the byte cap and char count coincide."""
|
||||
huge = "x" * 5000
|
||||
client = _make_app(storage)
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={"reason": huge},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
cfg = storage.load_workstream_config("ws-target")
|
||||
stored = cfg.get("close_reason")
|
||||
assert stored is not None
|
||||
assert len(stored.encode("utf-8")) <= 512
|
||||
|
||||
|
||||
def test_close_reason_byte_cap_holds_for_multibyte_utf8(storage):
|
||||
"""Repro for the char-cap-vs-byte-cap mismatch: a CJK-only payload
|
||||
of 600 chars would have leaked through a code-point slice at
|
||||
600*3=1800 bytes. The byte-aware cap holds it at <=512 bytes."""
|
||||
huge = "\u6f22" * 600 # 3 bytes/char in UTF-8
|
||||
client = _make_app(storage)
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={"reason": huge},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
cfg = storage.load_workstream_config("ws-target")
|
||||
stored = cfg.get("close_reason")
|
||||
assert stored is not None
|
||||
assert len(stored.encode("utf-8")) <= 512
|
||||
|
||||
|
||||
def test_close_with_non_string_reason_drops_silently(storage):
|
||||
"""A malformed body (reason=dict / list / int) should not crash the
|
||||
handler — non-string reasons are coerced to empty and the close
|
||||
proceeds without writing to workstream_config."""
|
||||
client = _make_app(storage)
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={"reason": {"unexpected": "shape"}},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
cfg = storage.load_workstream_config("ws-target")
|
||||
assert "close_reason" not in cfg
|
||||
|
||||
|
||||
def test_close_reason_redacts_credentials(storage):
|
||||
"""A model under prompt injection that captures a secret and stuffs
|
||||
it into ``reason`` must not get to plant the plaintext secret in
|
||||
audit logs / workstream_config. The output guard's credential-
|
||||
redaction pass runs at the close handler boundary."""
|
||||
client = _make_app(storage)
|
||||
secret = "AKIAIOSFODNN7EXAMPLE" # AWS access key — output guard catches.
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={"reason": f"task done; key={secret}"},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
cfg = storage.load_workstream_config("ws-target")
|
||||
stored = cfg.get("close_reason")
|
||||
assert stored is not None
|
||||
assert secret not in stored
|
||||
assert "[REDACTED:" in stored
|
||||
|
||||
|
||||
def test_close_reason_persistence_failure_does_not_block_close(storage):
|
||||
"""If the storage save raises, the close still succeeds — persistence
|
||||
is best-effort; a transient storage error must not block the user
|
||||
from closing a workstream."""
|
||||
client = _make_app(storage)
|
||||
|
||||
def _boom(*args, **kwargs):
|
||||
raise RuntimeError("storage down")
|
||||
|
||||
storage.save_workstream_config = _boom # type: ignore[method-assign]
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/ws-target/close",
|
||||
json={"reason": "task complete"},
|
||||
headers=_full_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
+18
-180
@@ -31,7 +31,16 @@ _TEST_AUTH_HEADERS = {"Authorization": f"Bearer {_test_jwt()}"}
|
||||
# Mock storage for collector tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from tests._coord_test_helpers import MockStorage # noqa: E402, F401
|
||||
|
||||
class MockStorage:
|
||||
"""Minimal storage mock that implements list_services for collector tests."""
|
||||
|
||||
def __init__(self):
|
||||
self.services: list[dict[str, str]] = []
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
return list(self.services)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -314,48 +323,6 @@ class TestCollectorSnapshot:
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["state"] == "running"
|
||||
|
||||
def test_apply_snapshot_state_change_forwards_pending_approval_detail(self):
|
||||
"""Reconnect-via-snapshot is the resync path after every console
|
||||
restart or network blip. Without forwarding the field here,
|
||||
a child sitting in approval-pending across the gap renders as
|
||||
``activity_state=approval`` with no buttons until the next
|
||||
state change — broken UX during the most common re-sync event."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "same", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [
|
||||
{
|
||||
"id": "ws1",
|
||||
"name": "same",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
}
|
||||
],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["pending_approval_detail"] == detail
|
||||
|
||||
def test_apply_snapshot_skips_empty_id_workstream(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
@@ -401,40 +368,6 @@ class TestCollectorDelta:
|
||||
# Verify in-memory state was updated
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["state"] == "running"
|
||||
|
||||
def test_apply_delta_ws_state_forwards_pending_approval_detail(self):
|
||||
"""The rich approval payload now travels on the cluster bus so
|
||||
coord tabs can render inline approve/deny buttons in lockstep
|
||||
with the activity_state transition. Collector must forward
|
||||
the field verbatim — the adapter does the child-routing on
|
||||
top, but the bus carries the data."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
"type": "ws_state",
|
||||
"ws_id": "ws1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["pending_approval_detail"] == detail
|
||||
|
||||
def test_apply_delta_ws_created(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
@@ -844,7 +777,6 @@ class TestConsoleHTTPEndpoints:
|
||||
sort_by="state",
|
||||
page=1,
|
||||
per_page=25,
|
||||
extra_rows=[],
|
||||
)
|
||||
|
||||
def test_get_workstreams_per_page_capped(self, client, mock_collector):
|
||||
@@ -901,18 +833,16 @@ class TestConsoleHTTPEndpoints:
|
||||
resp = client.get("/nonexistent")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_index_landing_surfaces(self, client):
|
||||
def test_index_has_new_ws_button(self, client):
|
||||
status, body, ct = self._get_raw(client, "/")
|
||||
assert status == 200
|
||||
# Coordinator-first landing keeps the node list always-visible.
|
||||
assert 'id="view-overview"' in body
|
||||
assert 'id="node-table"' in body
|
||||
# Removed in the 1.5.0 landing-page cleanup — guard against
|
||||
# accidental reintroduction.
|
||||
assert 'id="new-ws-overlay"' not in body
|
||||
assert 'id="new-ws-btn"' not in body
|
||||
assert 'id="cluster-summary-compact"' not in body
|
||||
assert 'id="view-node"' not in body
|
||||
assert 'id="new-ws-btn"' in body
|
||||
assert "showNewWsModal" in body
|
||||
|
||||
def test_index_has_new_ws_modal(self, client):
|
||||
status, body, ct = self._get_raw(client, "/")
|
||||
assert 'id="new-ws-overlay"' in body
|
||||
assert 'id="new-ws-node"' in body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1270,98 +1200,6 @@ class TestConsoleProxy:
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_proxy_api_per_ws_events_routes_to_sse_handler(self, client, mock_collector):
|
||||
"""``/node/{node_id}/v1/api/workstreams/{ws_id}/events`` is the
|
||||
per-workstream SSE stream the interactive WebUI subscribes to.
|
||||
Without explicit detection, the path falls through to the
|
||||
regular GET branch and the EventSource API can't consume the
|
||||
one-shot response — Firefox surfaces it as "can't establish a
|
||||
connection". Regression guard for the legacy URL surface
|
||||
removal (#422) that moved per-ws SSE under
|
||||
``/workstreams/{ws_id}/events`` without updating the proxy."""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from starlette.responses import Response
|
||||
|
||||
mock_collector.get_node_detail.return_value = {
|
||||
"node_id": "node-a",
|
||||
"server_url": "http://a:8080",
|
||||
"reachable": True,
|
||||
}
|
||||
ws_id = "a" * 32
|
||||
with (
|
||||
patch(
|
||||
"turnstone.console.server._proxy_sse",
|
||||
new_callable=AsyncMock,
|
||||
return_value=Response("ok", status_code=200),
|
||||
) as sse_mock,
|
||||
patch(
|
||||
"turnstone.console.server._proxy_get",
|
||||
new_callable=AsyncMock,
|
||||
return_value=Response("ok", status_code=200),
|
||||
) as get_mock,
|
||||
):
|
||||
client.get(f"/node/node-a/v1/api/workstreams/{ws_id}/events")
|
||||
assert sse_mock.await_count == 1, (
|
||||
"per-ws events path must route to _proxy_sse, not _proxy_get"
|
||||
)
|
||||
assert get_mock.await_count == 0
|
||||
# Path passed to _proxy_sse must be the workstreams-prefixed
|
||||
# form so the upstream URL is reconstructed correctly.
|
||||
sse_args = sse_mock.await_args
|
||||
assert sse_args.args[2] == f"workstreams/{ws_id}/events"
|
||||
|
||||
def test_proxy_api_global_events_still_routes_to_sse(self, client, mock_collector):
|
||||
"""The bare ``events/global`` path was the only SSE path the
|
||||
proxy recognized before the per-ws fix. Verify it still routes
|
||||
correctly so the new branch didn't regress the existing case."""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from starlette.responses import Response
|
||||
|
||||
mock_collector.get_node_detail.return_value = {
|
||||
"node_id": "node-a",
|
||||
"server_url": "http://a:8080",
|
||||
"reachable": True,
|
||||
}
|
||||
with patch(
|
||||
"turnstone.console.server._proxy_sse",
|
||||
new_callable=AsyncMock,
|
||||
return_value=Response("ok", status_code=200),
|
||||
) as sse_mock:
|
||||
client.get("/node/node-a/v1/api/events/global")
|
||||
assert sse_mock.await_count == 1
|
||||
# events/global must use the console's service token —
|
||||
# the upstream gates this path on `service` scope and
|
||||
# end-user JWTs don't carry it. Without this, the
|
||||
# browser's interactive UI 403-loops on every retry.
|
||||
assert sse_mock.await_args.kwargs.get("use_service_auth") is True
|
||||
|
||||
def test_proxy_api_per_ws_events_uses_user_auth_not_service(self, client, mock_collector):
|
||||
"""Per-ws events route uses the user's re-minted JWT, not the
|
||||
service token — the upstream per-ws SSE handler scopes by
|
||||
user identity for tenant filtering, and a service-scoped
|
||||
call would bypass that gate. Only ``events/global``
|
||||
(cross-tenant inventory by design) opts into service auth."""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from starlette.responses import Response
|
||||
|
||||
mock_collector.get_node_detail.return_value = {
|
||||
"node_id": "node-a",
|
||||
"server_url": "http://a:8080",
|
||||
"reachable": True,
|
||||
}
|
||||
ws_id = "b" * 32
|
||||
with patch(
|
||||
"turnstone.console.server._proxy_sse",
|
||||
new_callable=AsyncMock,
|
||||
return_value=Response("ok", status_code=200),
|
||||
) as sse_mock:
|
||||
client.get(f"/node/node-a/v1/api/workstreams/{ws_id}/events")
|
||||
assert sse_mock.await_count == 1
|
||||
assert sse_mock.await_args.kwargs.get("use_service_auth") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Proxy URL rewriting unit tests (no HTTP needed)
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
"""Tests for the console's coordinator idle-cleanup thread helper.
|
||||
|
||||
The helper itself is a tiny loop wrapping ``mgr.close_idle``; the heavy
|
||||
lifting is in ``SessionManager.close_idle`` (covered in
|
||||
``test_session_manager.py``) and ``bulk_close_stale_orphans`` (covered
|
||||
in ``test_storage_sqlite.py``). These tests verify the glue:
|
||||
|
||||
- the helper runs an initial sweep BEFORE its first sleep (cold-start
|
||||
cleanup without blocking the lifespan),
|
||||
- the helper swallows exceptions so a transient DB blip can't kill the
|
||||
daemon thread,
|
||||
- the helper exits cleanly when ``stop_event`` is set.
|
||||
|
||||
The ``stop_event`` parameter is exclusively for tests — production
|
||||
callers pass ``None`` and the daemon runs for process lifetime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
from turnstone.console.server import _coord_idle_cleanup_thread
|
||||
|
||||
|
||||
class _StubMgr:
|
||||
def __init__(
|
||||
self, *, stop_event: threading.Event, expected_calls: int, raise_after: int = -1
|
||||
) -> None:
|
||||
self.calls: list[float] = []
|
||||
self.sleep_calls_at_each_close: list[int] = []
|
||||
self._stop_event = stop_event
|
||||
self._expected = expected_calls
|
||||
self._raise_after = raise_after
|
||||
self._sleep_count = 0
|
||||
|
||||
def close_idle(self, timeout_sec: float) -> list[str]:
|
||||
# Snapshot how many sleeps preceded this close — lets the
|
||||
# "initial sweep" test verify the first close_idle ran with
|
||||
# zero preceding sleeps.
|
||||
self.sleep_calls_at_each_close.append(self._sleep_count)
|
||||
self.calls.append(timeout_sec)
|
||||
try:
|
||||
if 0 <= self._raise_after < len(self.calls):
|
||||
raise RuntimeError("simulated DB blip")
|
||||
finally:
|
||||
# Set stop after the helper has been exercised enough,
|
||||
# regardless of whether this call raised.
|
||||
if len(self.calls) >= self._expected:
|
||||
self._stop_event.set()
|
||||
return []
|
||||
|
||||
def record_sleep(self, _seconds: float) -> None:
|
||||
self._sleep_count += 1
|
||||
|
||||
|
||||
def _run_until_done(mgr: _StubMgr, stop_event: threading.Event, timeout_sec: float) -> None:
|
||||
with patch("turnstone.console.server.time.sleep", mgr.record_sleep):
|
||||
thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(mgr, timeout_sec, stop_event),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
thread.join(timeout=2.0)
|
||||
assert not thread.is_alive(), "helper failed to exit on stop_event"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_runs_initial_sweep_before_sleep() -> None:
|
||||
"""The first close_idle call must happen BEFORE the first time.sleep —
|
||||
otherwise cold-start orphans wait one ``check_every`` interval (~30 min
|
||||
on default 2h timeout) for the first reap. Crucial because the
|
||||
lifespan no longer does a synchronous initial sweep."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert mgr.sleep_calls_at_each_close == [0], "first close_idle should run before any sleep"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_calls_close_idle_each_tick() -> None:
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=3)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert len(mgr.calls) == 3
|
||||
assert all(t == 120.0 for t in mgr.calls)
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_survives_close_idle_exceptions() -> None:
|
||||
"""A transient DB error must not kill the daemon thread — the next
|
||||
tick should still fire close_idle. Without the try/except, a single
|
||||
blip would silently leak orphans forever."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=4, raise_after=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
# All four calls must have fired despite calls 2-4 raising.
|
||||
assert len(mgr.calls) == 4
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_exits_cleanly_on_stop_event() -> None:
|
||||
"""The stop_event mechanism is the test contract; verify the thread
|
||||
actually exits when the event is set, without needing exceptions or
|
||||
daemon-process termination."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=2)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert stop_event.is_set()
|
||||
@@ -37,49 +37,61 @@ class TestRecordRoute:
|
||||
assert "turnstone_router_request_duration_seconds_sum" in text
|
||||
|
||||
|
||||
class TestRecordJudgeVerdict:
|
||||
"""Coord-side intent-judge verdict counter."""
|
||||
|
||||
def test_single_verdict(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_judge_verdict("heuristic", "high", 12)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_judge_verdicts_total{tier="heuristic",risk_level="high"} 1' in text
|
||||
|
||||
def test_aggregates_by_tier_and_risk(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_judge_verdict("heuristic", "low", 5)
|
||||
m.record_judge_verdict("heuristic", "low", 7)
|
||||
m.record_judge_verdict("llm", "high", 250)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_judge_verdicts_total{tier="heuristic",risk_level="low"} 2' in text
|
||||
assert 'turnstone_judge_verdicts_total{tier="llm",risk_level="high"} 1' in text
|
||||
|
||||
def test_section_omitted_when_empty(self) -> None:
|
||||
"""No verdicts recorded → don't emit the empty header block."""
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert "turnstone_judge_verdicts_total" not in text
|
||||
|
||||
|
||||
class TestRouterInfo:
|
||||
"""Live-membership gauge + refresh counter."""
|
||||
class TestRingInfo:
|
||||
"""Ring membership and version gauges."""
|
||||
|
||||
def test_defaults_zero(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert "turnstone_router_membership_size 0" in text
|
||||
assert "turnstone_router_refresh_total 0" in text
|
||||
assert "turnstone_ring_membership_size 0" in text
|
||||
assert "turnstone_ring_version 0" in text
|
||||
|
||||
def test_set_router_info(self) -> None:
|
||||
def test_set_ring_info(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.set_router_info(3, 7)
|
||||
m.set_ring_info(3, 7)
|
||||
|
||||
text = m.generate_text()
|
||||
assert "turnstone_router_membership_size 3" in text
|
||||
assert "turnstone_router_refresh_total 7" in text
|
||||
assert "turnstone_ring_membership_size 3" in text
|
||||
assert "turnstone_ring_version 7" in text
|
||||
|
||||
|
||||
class TestRebalance:
|
||||
"""Rebalance and migration counters."""
|
||||
|
||||
def test_noop(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_rebalance("noop")
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_ring_rebalance_total{result="noop"} 1' in text
|
||||
|
||||
def test_seeded(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_rebalance("seeded")
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_ring_rebalance_total{result="seeded"} 1' in text
|
||||
|
||||
def test_rebalanced(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_rebalance("rebalanced")
|
||||
m.record_rebalance("rebalanced")
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_ring_rebalance_total{result="rebalanced"} 2' in text
|
||||
|
||||
def test_migrations(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
m.record_migrations(5)
|
||||
m.record_migrations(3)
|
||||
|
||||
text = m.generate_text()
|
||||
assert "turnstone_ring_migrations_total 8" in text
|
||||
|
||||
def test_migrations_default_zero(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
text = m.generate_text()
|
||||
assert "turnstone_ring_migrations_total 0" in text
|
||||
|
||||
|
||||
class TestGenerateText:
|
||||
@@ -91,8 +103,10 @@ class TestGenerateText:
|
||||
expected = [
|
||||
"turnstone_router_requests_total",
|
||||
"turnstone_router_request_duration_seconds",
|
||||
"turnstone_router_membership_size",
|
||||
"turnstone_router_refresh_total",
|
||||
"turnstone_ring_membership_size",
|
||||
"turnstone_ring_version",
|
||||
"turnstone_ring_rebalance_total",
|
||||
"turnstone_ring_migrations_total",
|
||||
]
|
||||
for name in expected:
|
||||
assert name in text, f"Missing metric: {name}"
|
||||
@@ -102,8 +116,8 @@ class TestGenerateText:
|
||||
text = m.generate_text()
|
||||
assert "# HELP turnstone_router_requests_total" in text
|
||||
assert "# TYPE turnstone_router_requests_total counter" in text
|
||||
assert "# HELP turnstone_router_membership_size" in text
|
||||
assert "# TYPE turnstone_router_membership_size gauge" in text
|
||||
assert "# HELP turnstone_ring_membership_size" in text
|
||||
assert "# TYPE turnstone_ring_membership_size gauge" in text
|
||||
|
||||
def test_ends_with_newline(self) -> None:
|
||||
m = ConsoleMetrics()
|
||||
@@ -111,16 +125,24 @@ class TestGenerateText:
|
||||
assert text.endswith("\n")
|
||||
|
||||
def test_combined_scenario(self) -> None:
|
||||
"""Full scenario: routes + router info."""
|
||||
"""Full scenario: routes, ring info, rebalances, migrations."""
|
||||
m = ConsoleMetrics()
|
||||
m.record_route("create", 200, 0.1)
|
||||
m.record_route("send", 200, 0.05)
|
||||
m.record_route("send", 502, 1.2)
|
||||
m.set_router_info(3, 12)
|
||||
m.set_ring_info(3, 12)
|
||||
m.record_rebalance("seeded")
|
||||
m.record_rebalance("noop")
|
||||
m.record_rebalance("rebalanced")
|
||||
m.record_migrations(4)
|
||||
|
||||
text = m.generate_text()
|
||||
assert 'turnstone_router_requests_total{method="create",status="2xx"} 1' in text
|
||||
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
||||
assert 'turnstone_router_requests_total{method="send",status="5xx"} 1' in text
|
||||
assert "turnstone_router_membership_size 3" in text
|
||||
assert "turnstone_router_refresh_total 12" in text
|
||||
assert "turnstone_ring_membership_size 3" in text
|
||||
assert "turnstone_ring_version 12" in text
|
||||
assert 'turnstone_ring_rebalance_total{result="noop"} 1' in text
|
||||
assert 'turnstone_ring_rebalance_total{result="rebalanced"} 1' in text
|
||||
assert 'turnstone_ring_rebalance_total{result="seeded"} 1' in text
|
||||
assert "turnstone_ring_migrations_total 4" in text
|
||||
|
||||
+200
-186
@@ -1,13 +1,17 @@
|
||||
"""Tests for turnstone.console.router (rendezvous routing)."""
|
||||
"""Tests for turnstone.console.router."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.console.router import ConsoleRouter, NodeRef
|
||||
from turnstone.core.rendezvous import NoAvailableNodeError
|
||||
from turnstone.core.hash_ring import RING_SIZE, NoAvailableNodeError
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake storage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
@@ -15,14 +19,26 @@ class FakeStorage:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.services: list[dict[str, str]] = []
|
||||
self.buckets: list[dict[str, Any]] = []
|
||||
self.overrides: list[dict[str, str]] = []
|
||||
self.settings: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
return list(self.services)
|
||||
|
||||
def list_ring_buckets(self) -> list[dict[str, Any]]:
|
||||
return list(self.buckets)
|
||||
|
||||
def list_workstream_overrides(self) -> list[dict[str, str]]:
|
||||
return list(self.overrides)
|
||||
|
||||
def get_system_setting(self, key: str, node_id: str = "") -> dict[str, Any] | None:
|
||||
return self.settings.get(key)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NODE_A = {"service_id": "node-a", "url": "http://a:8080", "metadata": "{}"}
|
||||
NODE_B = {"service_id": "node-b", "url": "http://b:8080", "metadata": "{}"}
|
||||
@@ -34,262 +50,260 @@ def _make_router(storage: FakeStorage | None = None) -> tuple[ConsoleRouter, Fak
|
||||
return ConsoleRouter(s), s # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _random_ws_id() -> str:
|
||||
return secrets.token_hex(16)
|
||||
def _ws_id_for_bucket(bucket: int) -> str:
|
||||
"""Build a 32-char hex ws_id whose first 4 chars encode *bucket*."""
|
||||
return f"{bucket:04x}" + "0" * 28
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestRouteBasic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRouteBasic:
|
||||
def test_route_returns_a_live_node(self) -> None:
|
||||
"""Basic routing through the bucket cache."""
|
||||
|
||||
def test_route_returns_correct_node(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
storage.buckets = [
|
||||
{"bucket": 0x0000, "node_id": "node-a"},
|
||||
{"bucket": 0x0001, "node_id": "node-b"},
|
||||
{"bucket": 0x0002, "node_id": "node-c"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
ref = router.route(_random_ws_id())
|
||||
assert ref.node_id in {"node-a", "node-b", "node-c"}
|
||||
|
||||
def test_route_is_deterministic_for_same_ws_id(self) -> None:
|
||||
"""Same ws_id + same membership → same target every time."""
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
router.refresh_cache()
|
||||
|
||||
ws_id = _random_ws_id()
|
||||
first = router.route(ws_id)
|
||||
for _ in range(50):
|
||||
assert router.route(ws_id) == first
|
||||
assert router.route(_ws_id_for_bucket(0x0000)) == NodeRef("node-a", "http://a:8080")
|
||||
assert router.route(_ws_id_for_bucket(0x0001)) == NodeRef("node-b", "http://b:8080")
|
||||
assert router.route(_ws_id_for_bucket(0x0002)) == NodeRef("node-c", "http://c:8080")
|
||||
|
||||
def test_route_override_priority(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B]
|
||||
ws_id = _random_ws_id()
|
||||
storage.buckets = [{"bucket": 0x0000, "node_id": "node-a"}]
|
||||
ws_id = _ws_id_for_bucket(0x0000)
|
||||
storage.overrides = [{"ws_id": ws_id, "node_id": "node-b"}]
|
||||
router.refresh_cache()
|
||||
|
||||
# Override wins regardless of HRW score.
|
||||
# Override wins over bucket assignment
|
||||
assert router.route(ws_id) == NodeRef("node-b", "http://b:8080")
|
||||
|
||||
def test_route_empty_membership_raises(self) -> None:
|
||||
def test_route_empty_cache_raises(self) -> None:
|
||||
router, _ = _make_router()
|
||||
with pytest.raises(NoAvailableNodeError):
|
||||
router.route(_random_ws_id())
|
||||
|
||||
def test_route_empty_ws_id_raises(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
router.refresh_cache()
|
||||
with pytest.raises(NoAvailableNodeError, match="empty"):
|
||||
router.route("")
|
||||
with pytest.raises(NoAvailableNodeError, match="not assigned"):
|
||||
router.route(_ws_id_for_bucket(0x0000))
|
||||
|
||||
def test_route_url_convenience(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0x0010, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
assert router.route_url(_random_ws_id()) == "http://a:8080"
|
||||
|
||||
assert router.route_url(_ws_id_for_bucket(0x0010)) == "http://a:8080"
|
||||
|
||||
|
||||
class TestMembershipConvergence:
|
||||
"""Rendezvous gives the minimal-moves property; pin it."""
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestRefreshCache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_node_join_only_steals_some_keys(self) -> None:
|
||||
"""Adding a 4th node moves ~1/4 of keys to it; the other 3
|
||||
nodes' kept keys are unchanged."""
|
||||
|
||||
class TestRefreshCache:
|
||||
"""Cache loading from storage."""
|
||||
|
||||
def test_refresh_loads_from_storage(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 100, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
sample = [_random_ws_id() for _ in range(2000)]
|
||||
before = {ws: router.route(ws).node_id for ws in sample}
|
||||
ref = router.route(_ws_id_for_bucket(100))
|
||||
assert ref.node_id == "node-a"
|
||||
|
||||
storage.services = [
|
||||
NODE_A,
|
||||
NODE_B,
|
||||
NODE_C,
|
||||
{"service_id": "node-d", "url": "http://d:8080", "metadata": "{}"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
after = {ws: router.route(ws).node_id for ws in sample}
|
||||
|
||||
moved = sum(1 for ws in sample if before[ws] != after[ws])
|
||||
moved_to_new = sum(1 for ws in sample if after[ws] == "node-d")
|
||||
# Every move must be onto the new node — no churn between
|
||||
# existing nodes.
|
||||
assert moved == moved_to_new
|
||||
# Should be roughly 1/4 of keys; allow a wide band for variance.
|
||||
assert 0.15 < moved / len(sample) < 0.35
|
||||
|
||||
def test_node_leave_only_redistributes_dead_node_keys(self) -> None:
|
||||
"""Removing node-a sends node-a's keys to b/c only; keys that
|
||||
were on b/c stay put."""
|
||||
def test_refresh_handles_dead_nodes(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
router.refresh_cache()
|
||||
|
||||
sample = [_random_ws_id() for _ in range(2000)]
|
||||
before = {ws: router.route(ws).node_id for ws in sample}
|
||||
|
||||
storage.services = [NODE_B, NODE_C]
|
||||
router.refresh_cache()
|
||||
after = {ws: router.route(ws).node_id for ws in sample}
|
||||
|
||||
for ws in sample:
|
||||
if before[ws] in ("node-b", "node-c"):
|
||||
assert after[ws] == before[ws], (
|
||||
f"key {ws} moved from {before[ws]} to {after[ws]} "
|
||||
"even though its old owner is still live"
|
||||
)
|
||||
else: # was on node-a
|
||||
assert after[ws] in ("node-b", "node-c")
|
||||
|
||||
|
||||
class TestWeights:
|
||||
def test_weight_2_node_gets_more_keys_than_weight_1(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [
|
||||
{"service_id": "node-a", "url": "http://a:8080", "metadata": '{"weight": 2}'},
|
||||
{"service_id": "node-b", "url": "http://b:8080", "metadata": '{"weight": 1}'},
|
||||
# node-b is in buckets but not in services (dead/expired)
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [
|
||||
{"bucket": 0x0000, "node_id": "node-a"},
|
||||
{"bucket": 0x0001, "node_id": "node-b"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
sample = [_random_ws_id() for _ in range(5000)]
|
||||
on_a = sum(1 for ws in sample if router.route(ws).node_id == "node-a")
|
||||
# Heavier node should win clearly more than half; exact ratio
|
||||
# depends on the simple hash×weight formulation but a/b > 1.4
|
||||
# for weight 2:1 across 5k samples is reliable.
|
||||
assert on_a / len(sample) > 0.55
|
||||
assert router.route(_ws_id_for_bucket(0x0000)).node_id == "node-a"
|
||||
with pytest.raises(NoAvailableNodeError):
|
||||
router.route(_ws_id_for_bucket(0x0001))
|
||||
|
||||
def test_invalid_metadata_falls_back_to_weight_1(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [
|
||||
{"service_id": "node-a", "url": "http://a:8080", "metadata": "not json"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
# Just confirms it doesn't blow up.
|
||||
router.route(_random_ws_id())
|
||||
|
||||
|
||||
class TestRefreshLifecycle:
|
||||
def test_refresh_cache_publishes_new_membership_immediately(self) -> None:
|
||||
"""refresh_cache() reloads on the calling thread — the next
|
||||
route() sees the new membership without any further trigger."""
|
||||
def test_refresh_returns_true_on_change(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
router.refresh_cache()
|
||||
assert router.node_count() == 1
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
|
||||
storage.services = [NODE_A, NODE_B]
|
||||
router.refresh_cache()
|
||||
assert router.node_count() == 2
|
||||
|
||||
def test_concurrent_refresh_returns_false_on_lock_contention(self) -> None:
|
||||
"""refresh_cache uses a non-blocking lock acquire — if another
|
||||
thread is already refreshing, the second caller bails so the
|
||||
in-flight refresh's result is the one that publishes."""
|
||||
assert router.refresh_cache() is True
|
||||
|
||||
def test_refresh_returns_false_on_no_change(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
|
||||
with router._refresh_lock:
|
||||
# Lock held by this thread → the call below can't acquire.
|
||||
assert router.refresh_cache() is False
|
||||
|
||||
def test_force_refresh_blocks_until_in_flight_refresh_releases(self) -> None:
|
||||
"""force_refresh acquires the refresh lock blocking — used by the
|
||||
404-retry path to guarantee a fresh view even under contention."""
|
||||
import threading
|
||||
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
|
||||
# Hold the refresh lock from another thread.
|
||||
lock_held = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def hold_lock() -> None:
|
||||
with router._refresh_lock:
|
||||
lock_held.set()
|
||||
release.wait(timeout=2)
|
||||
|
||||
holder = threading.Thread(target=hold_lock, daemon=True)
|
||||
holder.start()
|
||||
assert lock_held.wait(timeout=1)
|
||||
|
||||
# force_refresh should block, not bail.
|
||||
result_box: list[bool] = []
|
||||
|
||||
def call_force() -> None:
|
||||
result_box.append(router.force_refresh())
|
||||
|
||||
caller = threading.Thread(target=call_force, daemon=True)
|
||||
caller.start()
|
||||
caller.join(timeout=0.2)
|
||||
assert caller.is_alive(), "force_refresh returned without acquiring lock"
|
||||
|
||||
release.set()
|
||||
holder.join(timeout=1)
|
||||
caller.join(timeout=1)
|
||||
assert not caller.is_alive()
|
||||
# Membership changed from empty → 1 live node.
|
||||
assert result_box == [True]
|
||||
assert router.node_count() == 1
|
||||
|
||||
def test_force_refresh_always_reloads(self) -> None:
|
||||
"""force_refresh skips the non-blocking-lock bail and always
|
||||
publishes a fresh view — back-to-back calls each pick up the
|
||||
latest storage state."""
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
router.force_refresh()
|
||||
assert router.node_count() == 1
|
||||
|
||||
storage.services = [NODE_A, NODE_B]
|
||||
router.force_refresh()
|
||||
assert router.node_count() == 2
|
||||
|
||||
def test_version_is_monotonic_across_refreshes(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
router.refresh_cache()
|
||||
v1 = router.version
|
||||
router.refresh_cache()
|
||||
v2 = router.version
|
||||
assert v2 > v1
|
||||
router.force_refresh()
|
||||
assert router.version > v2
|
||||
assert router.refresh_cache() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestCheckVersion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckVersion:
|
||||
"""Version-gated refresh."""
|
||||
|
||||
def test_version_change_triggers_refresh(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
storage.settings["rebalancer_version"] = {"value": "1"}
|
||||
|
||||
assert router.check_version() is True
|
||||
assert router.is_ready()
|
||||
|
||||
def test_same_version_skips(self) -> None:
|
||||
router, storage = _make_router()
|
||||
# Default version is 0; setting absent also means 0
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
|
||||
# First call: version=0 matches self._version=0 -> no refresh
|
||||
assert router.check_version() is False
|
||||
assert not router.is_ready() # cache was never loaded
|
||||
|
||||
def test_version_none_treated_as_zero(self) -> None:
|
||||
router, storage = _make_router()
|
||||
# settings dict is empty -> get_system_setting returns None
|
||||
assert router.check_version() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestGenerateWsId
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerateWsId:
|
||||
"""Workstream ID generation targeting a specific node."""
|
||||
|
||||
def test_generates_routable_id(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
storage.services = [NODE_A, NODE_B]
|
||||
storage.buckets = [
|
||||
{"bucket": 0x00FF, "node_id": "node-a"},
|
||||
{"bucket": 0x0100, "node_id": "node-b"},
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
ws_id = router.generate_ws_id_for_node("node-b")
|
||||
ws_id = router.generate_ws_id_for_node("node-a")
|
||||
assert len(ws_id) == 32
|
||||
assert router.route(ws_id).node_id == "node-b"
|
||||
assert router.route(ws_id).node_id == "node-a"
|
||||
|
||||
def test_unknown_node_raises(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
with pytest.raises(NoAvailableNodeError, match="node-z"):
|
||||
router.generate_ws_id_for_node("node-z")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestIsReady
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsReady:
|
||||
"""Readiness checks."""
|
||||
|
||||
def test_false_when_empty(self) -> None:
|
||||
router, _ = _make_router()
|
||||
assert router.is_ready() is False
|
||||
|
||||
def test_true_after_membership_loads(self) -> None:
|
||||
def test_true_after_refresh(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A]
|
||||
storage.buckets = [{"bucket": 0, "node_id": "node-a"}]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.is_ready() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestPopulateFromAssignments
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPopulateFromAssignments:
|
||||
"""Direct cache population without DB round-trip."""
|
||||
|
||||
def test_populate_makes_router_ready(self) -> None:
|
||||
router, _ = _make_router()
|
||||
assignments = [(b, "node-a") for b in range(RING_SIZE)]
|
||||
nodes = {"node-a": NodeRef("node-a", "http://a:8080")}
|
||||
router.populate_from_assignments(assignments, nodes)
|
||||
|
||||
assert router.is_ready()
|
||||
assert router.node_count() == 1
|
||||
assert router.route(_ws_id_for_bucket(0)).node_id == "node-a"
|
||||
|
||||
def test_populate_multi_node(self) -> None:
|
||||
router, _ = _make_router()
|
||||
assignments = [(0, "node-a"), (1, "node-b"), (2, "node-a")]
|
||||
nodes = {
|
||||
"node-a": NodeRef("node-a", "http://a:8080"),
|
||||
"node-b": NodeRef("node-b", "http://b:8080"),
|
||||
}
|
||||
router.populate_from_assignments(assignments, nodes)
|
||||
|
||||
assert router.route(_ws_id_for_bucket(0)).node_id == "node-a"
|
||||
assert router.route(_ws_id_for_bucket(1)).node_id == "node-b"
|
||||
assert router.route(_ws_id_for_bucket(2)).node_id == "node-a"
|
||||
|
||||
def test_populate_loads_overrides_from_db(self) -> None:
|
||||
router, storage = _make_router()
|
||||
ws_id = _ws_id_for_bucket(0)
|
||||
storage.overrides = [{"ws_id": ws_id, "node_id": "node-b"}]
|
||||
nodes = {
|
||||
"node-a": NodeRef("node-a", "http://a:8080"),
|
||||
"node-b": NodeRef("node-b", "http://b:8080"),
|
||||
}
|
||||
router.populate_from_assignments([(0, "node-a")], nodes)
|
||||
|
||||
# Override should route bucket 0 to node-b despite assignment to node-a
|
||||
assert router.route(ws_id) == NodeRef("node-b", "http://b:8080")
|
||||
|
||||
def test_populate_no_overrides_when_table_empty(self) -> None:
|
||||
router, storage = _make_router()
|
||||
# No overrides in storage
|
||||
router.populate_from_assignments(
|
||||
[(0, "node-a")],
|
||||
{"node-a": NodeRef("node-a", "http://a:8080")},
|
||||
)
|
||||
assert len(router._overrides) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestNodeCount
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNodeCount:
|
||||
def test_count_matches_live_services(self) -> None:
|
||||
"""Distinct node counting."""
|
||||
|
||||
def test_count_distinct_nodes(self) -> None:
|
||||
router, storage = _make_router()
|
||||
storage.services = [NODE_A, NODE_B, NODE_C]
|
||||
# Spread all 65536 buckets across 3 nodes
|
||||
storage.buckets = [
|
||||
{"bucket": b, "node_id": f"node-{['a', 'b', 'c'][b % 3]}"} for b in range(RING_SIZE)
|
||||
]
|
||||
router.refresh_cache()
|
||||
|
||||
assert router.node_count() == 3
|
||||
|
||||
@@ -11,7 +11,7 @@ from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.router import ConsoleRouter, NodeRef
|
||||
from turnstone.core.rendezvous import NoAvailableNodeError
|
||||
from turnstone.core.hash_ring import NoAvailableNodeError
|
||||
|
||||
# Shared test auth — JWT-based
|
||||
_TEST_JWT_SECRET = "test-jwt-secret-minimum-32-chars!"
|
||||
@@ -93,15 +93,6 @@ def _wire_proxy(app: Any, mock_post: MagicMock | None = None) -> None:
|
||||
mock_post = _make_proxy_post()
|
||||
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
||||
mock_proxy.post = mock_post
|
||||
|
||||
# route_proxy uses ``client.request(method, url, ...)`` for path-keyed
|
||||
# routes (so DELETE on /send proxies through correctly). Wire a
|
||||
# request-shim that drops the leading method positional and forwards
|
||||
# to the same mock_post for compatibility.
|
||||
async def _request_shim(method: str, *args: Any, **kwargs: Any) -> httpx.Response:
|
||||
return await mock_post(*args, **kwargs)
|
||||
|
||||
mock_proxy.request = MagicMock(side_effect=_request_shim)
|
||||
app.state.proxy_client = mock_proxy
|
||||
|
||||
|
||||
@@ -111,7 +102,7 @@ def _wire_proxy(app: Any, mock_post: MagicMock | None = None) -> None:
|
||||
|
||||
|
||||
class TestRouteCreate:
|
||||
"""POST /v1/api/route/workstreams/new — create via rendezvous routing."""
|
||||
"""POST /v1/api/route/workstreams/new — create via hash-ring routing."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
@@ -187,49 +178,6 @@ class TestRouteCreate:
|
||||
router.generate_ws_id_for_node.assert_called_with("node-c")
|
||||
client.close()
|
||||
|
||||
def test_route_create_routing_strategy_rendezvous(self, client):
|
||||
"""Default fan-out (no resume_ws / no target_node) reports
|
||||
routing_strategy='rendezvous' so the coordinator's spawn tool
|
||||
can explain why the node was chosen."""
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"name": "test-ws"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["routing_strategy"] == "rendezvous"
|
||||
|
||||
def test_route_create_routing_strategy_target_node(self):
|
||||
router = _make_mock_router()
|
||||
router.generate_ws_id_for_node.return_value = "00ff" + "0" * 28
|
||||
router.route.return_value = NodeRef("node-c", "http://c:8080")
|
||||
app = _make_app(router=router)
|
||||
_wire_proxy(app, _make_proxy_post(json_data={"ws_id": "00ff" + "0" * 28, "name": "pinned"}))
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"target_node": "node-c"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["routing_strategy"] == "target_node"
|
||||
client.close()
|
||||
|
||||
def test_route_create_routing_strategy_resume(self):
|
||||
router = _make_mock_router()
|
||||
router.route.return_value = NodeRef("node-b", "http://b:8080")
|
||||
app = _make_app(router=router)
|
||||
_wire_proxy(app, _make_proxy_post(json_data={"ws_id": "old_ws_resumed", "name": "resumed"}))
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/new",
|
||||
json={"resume_ws": "old_ws_id"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["routing_strategy"] == "resume"
|
||||
client.close()
|
||||
|
||||
|
||||
class TestRouteCreate503Retry:
|
||||
"""503 retry logic in route_create."""
|
||||
@@ -292,8 +240,7 @@ class TestRouteCreate503Retry:
|
||||
|
||||
|
||||
class TestRouteProxy:
|
||||
"""POST /v1/api/route/workstreams/{ws_id}/<verb> (and the surviving
|
||||
body-keyed plan/command routes)."""
|
||||
"""POST /v1/api/route/send (and other routed endpoints)."""
|
||||
|
||||
@pytest.fixture()
|
||||
def client(self):
|
||||
@@ -306,33 +253,29 @@ class TestRouteProxy:
|
||||
|
||||
def test_route_proxy_send(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/abc123/send",
|
||||
json={"message": "hello"},
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc123", "message": "hello"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# Verify upstream URL was /v1/api/workstreams/abc123/send
|
||||
# (not /v1/api/route/workstreams/abc123/send).
|
||||
mock_request = client.app.state.proxy_client.request
|
||||
call_args = mock_request.call_args
|
||||
# request is called as ``request(method, url, ...)`` — url is the
|
||||
# second positional arg.
|
||||
upstream_url = call_args[0][1]
|
||||
assert "/v1/api/workstreams/abc123/send" in upstream_url
|
||||
assert "/route/" not in upstream_url
|
||||
# Verify upstream URL was /v1/api/send (not /v1/api/route/send)
|
||||
mock_post = client.app.state.proxy_client.post
|
||||
call_args = mock_post.call_args
|
||||
assert "/v1/api/send" in call_args[0][0]
|
||||
assert "/route/" not in call_args[0][0]
|
||||
|
||||
def test_route_proxy_approve(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/abc123/approve",
|
||||
json={"approved": True},
|
||||
"/v1/api/route/approve",
|
||||
json={"ws_id": "abc123", "approved": True},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_route_proxy_cancel(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/abc123/cancel",
|
||||
json={},
|
||||
"/v1/api/route/cancel",
|
||||
json={"ws_id": "abc123"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
@@ -347,8 +290,8 @@ class TestRouteProxy:
|
||||
|
||||
def test_route_proxy_close(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/abc123/close",
|
||||
json={},
|
||||
"/v1/api/route/workstreams/close",
|
||||
json={"ws_id": "abc123"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
@@ -427,8 +370,8 @@ class TestRouteNotReady:
|
||||
|
||||
def test_route_proxy_no_router_503(self, client_no_router):
|
||||
resp = client_no_router.post(
|
||||
"/v1/api/route/workstreams/abc/send",
|
||||
json={"message": "hello"},
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc", "message": "hello"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
@@ -439,8 +382,8 @@ class TestRouteNotReady:
|
||||
|
||||
def test_route_proxy_empty_cache_503(self, client_empty_cache):
|
||||
resp = client_empty_cache.post(
|
||||
"/v1/api/route/workstreams/abc/send",
|
||||
json={"message": "hello"},
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc", "message": "hello"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
@@ -479,8 +422,8 @@ class TestRouteNoNode:
|
||||
|
||||
def test_route_proxy_no_node_503(self, client):
|
||||
resp = client.post(
|
||||
"/v1/api/route/workstreams/abc/send",
|
||||
json={"message": "hello"},
|
||||
"/v1/api/route/send",
|
||||
json={"ws_id": "abc", "message": "hello"},
|
||||
headers=_TEST_AUTH_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 503
|
||||
|
||||
@@ -1,599 +0,0 @@
|
||||
"""Tests for the rich ``ws_state`` payload on coord (Stage 2 follow-up).
|
||||
|
||||
Pre-lift coord's ``ConsoleCoordinatorUI`` populated none of the per-ws
|
||||
metric fields ``SessionUIBase`` defines (``_ws_prompt_tokens`` /
|
||||
``_ws_context_ratio`` / ``_ws_current_activity`` / ``_ws_turn_content``)
|
||||
and the ``coord_adapter.emit_state`` broadcast was state-only —
|
||||
``tokens=0`` / ``content=""`` were hardcoded into
|
||||
``collector.emit_console_ws_state``. The lift turned ``on_status`` /
|
||||
``on_content_token`` / ``on_thinking_*`` / ``on_tool_result`` into
|
||||
shared bodies on :class:`SessionUIBase` so coord populates the same
|
||||
fields, then enriched ``coord_adapter.emit_state`` to read them under
|
||||
lock and pass through to the cluster collector with the rich kwargs.
|
||||
The cluster dashboard's coord rows now render with the same
|
||||
tokens / activity / content / context_ratio fields interactive rows do.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
from turnstone.core.session_ui_base import _MAX_TURN_CONTENT_CHARS
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-ws metric writes — lifted to SessionUIBase, both subclasses inherit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _patch_get_storage(storage: Any):
|
||||
return patch("turnstone.core.storage._registry.get_storage", return_value=storage)
|
||||
|
||||
|
||||
def test_coord_on_status_writes_per_ws_metrics() -> None:
|
||||
"""Pre-lift coord ``on_status`` was an enqueue-only stub — ``_ws_*``
|
||||
fields stayed at their initial zero values regardless of token usage.
|
||||
Post-lift coord inherits SessionUIBase's body, so token counters and
|
||||
context ratio populate just like interactive."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
with _patch_get_storage(MagicMock()):
|
||||
ui.on_status(
|
||||
{"prompt_tokens": 100, "completion_tokens": 50},
|
||||
context_window=1000,
|
||||
effort="medium",
|
||||
)
|
||||
assert ui._ws_prompt_tokens == 100
|
||||
assert ui._ws_completion_tokens == 50
|
||||
assert ui._ws_context_ratio == pytest.approx(0.15)
|
||||
|
||||
|
||||
def test_coord_on_status_persists_usage_event() -> None:
|
||||
"""Pre-lift coord didn't persist usage_event rows — only WebUI did.
|
||||
Lift extends usage tracking to coord so governance dashboards see
|
||||
coordinator token consumption alongside interactive."""
|
||||
storage = MagicMock()
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
with _patch_get_storage(storage):
|
||||
ui.on_status(
|
||||
{"prompt_tokens": 7, "completion_tokens": 3, "model": "gpt-x"},
|
||||
context_window=200,
|
||||
effort="low",
|
||||
)
|
||||
storage.record_usage_event.assert_called_once()
|
||||
kwargs = storage.record_usage_event.call_args.kwargs
|
||||
assert kwargs["ws_id"] == "coord-ws"
|
||||
assert kwargs["user_id"] == "u1"
|
||||
assert kwargs["model"] == "gpt-x"
|
||||
assert kwargs["prompt_tokens"] == 7
|
||||
assert kwargs["completion_tokens"] == 3
|
||||
|
||||
|
||||
def test_coord_on_content_token_accumulates() -> None:
|
||||
"""Pre-lift coord ``on_content_token`` only enqueued; lift turns it
|
||||
into the same per-ws accumulator WebUI uses so the collector
|
||||
broadcast can piggyback the joined turn content on the IDLE
|
||||
state-change event."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui.on_content_token("Hello ")
|
||||
ui.on_content_token("world")
|
||||
assert ui._ws_turn_content == ["Hello ", "world"]
|
||||
assert ui._ws_turn_content_size == len("Hello world")
|
||||
|
||||
|
||||
def test_coord_on_content_token_caps_at_ceiling() -> None:
|
||||
"""Same content cap interactive enforces — keeps a runaway turn from
|
||||
ballooning the cluster broadcast event past listener queue size."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
chunk = "x" * 1024
|
||||
rounds = (_MAX_TURN_CONTENT_CHARS // 1024) + 50
|
||||
for _ in range(rounds):
|
||||
ui.on_content_token(chunk)
|
||||
# Cap is enforced at the size check; one over-cap chunk still
|
||||
# gets in (per the original ``< _MAX``-not-``<=`` semantics) but
|
||||
# nothing past that lands.
|
||||
assert ui._ws_turn_content_size <= _MAX_TURN_CONTENT_CHARS + 1024
|
||||
|
||||
|
||||
def test_coord_on_thinking_start_sets_activity() -> None:
|
||||
"""Live activity tracking — coord's dashboard row now flips
|
||||
``activity_state`` to ``"thinking"`` when the model starts."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui.on_thinking_start()
|
||||
assert ui._ws_current_activity == "Thinking…"
|
||||
assert ui._ws_activity_state == "thinking"
|
||||
|
||||
|
||||
def test_coord_on_tool_result_clears_activity_and_increments_counters() -> None:
|
||||
"""Lifted ``on_tool_result`` body increments ``_ws_tool_calls`` /
|
||||
``_ws_turn_tool_calls`` and clears the activity. Pre-lift coord
|
||||
just enqueued without touching counters."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui._ws_current_activity = "⚙ bash: ls -la"
|
||||
ui._ws_activity_state = "tool"
|
||||
ui.on_tool_result("call-1", "bash", "output")
|
||||
assert ui._ws_tool_calls == {"bash": 1}
|
||||
assert ui._ws_turn_tool_calls == 1
|
||||
assert ui._ws_current_activity == ""
|
||||
assert ui._ws_activity_state == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Snapshot helper — drains turn content on IDLE/ERROR
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_snapshot_idle_returns_content_and_clears_accumulator() -> None:
|
||||
"""IDLE snapshot piggybacks the joined assistant content onto the
|
||||
state-change broadcast (so the dashboard renders the turn without
|
||||
a storage round-trip), then clears the accumulator for the next
|
||||
turn."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui.on_content_token("Here's ")
|
||||
ui.on_content_token("the result.")
|
||||
payload = ui.snapshot_and_consume_state_payload("idle")
|
||||
assert payload["content"] == "Here's the result."
|
||||
assert ui._ws_turn_content == []
|
||||
assert ui._ws_turn_content_size == 0
|
||||
|
||||
|
||||
def test_snapshot_error_clears_accumulator_without_emitting_content() -> None:
|
||||
"""ERROR clears the partial content (the turn's broken; nothing to
|
||||
render) but the broadcast itself doesn't carry it — the state
|
||||
transition is what matters."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui.on_content_token("partial...")
|
||||
payload = ui.snapshot_and_consume_state_payload("error")
|
||||
assert payload["content"] == ""
|
||||
assert ui._ws_turn_content == []
|
||||
|
||||
|
||||
def test_snapshot_thinking_does_not_touch_accumulator() -> None:
|
||||
"""Mid-turn state transitions (running / thinking / attention)
|
||||
don't drain the accumulator — only IDLE / ERROR are terminal."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui.on_content_token("partial mid-turn")
|
||||
payload = ui.snapshot_and_consume_state_payload("thinking")
|
||||
assert payload["content"] == ""
|
||||
# Accumulator preserved.
|
||||
assert ui._ws_turn_content == ["partial mid-turn"]
|
||||
|
||||
|
||||
def test_snapshot_carries_token_and_activity_snapshot() -> None:
|
||||
"""Snapshot reads tokens / context_ratio / activity under one lock
|
||||
acquisition so concurrent on_status / on_thinking_start writes
|
||||
don't tear the snapshot."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
with _patch_get_storage(MagicMock()):
|
||||
ui.on_status(
|
||||
{"prompt_tokens": 80, "completion_tokens": 20},
|
||||
context_window=400,
|
||||
effort="medium",
|
||||
)
|
||||
ui.on_thinking_start() # sets activity = "Thinking…"
|
||||
payload = ui.snapshot_and_consume_state_payload("running")
|
||||
assert payload["tokens"] == 100
|
||||
assert payload["context_ratio"] == pytest.approx(0.25)
|
||||
assert payload["activity"] == "Thinking…"
|
||||
assert payload["activity_state"] == "thinking"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coord adapter — passes rich payload to collector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeCollectorRecorder:
|
||||
"""Captures emit_console_ws_state calls so we can assert on the
|
||||
rich kwargs the lifted coord_adapter.emit_state passes through."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.state_calls: list[dict[str, Any]] = []
|
||||
self.activity_calls: list[dict[str, Any]] = []
|
||||
|
||||
def emit_console_ws_state(
|
||||
self,
|
||||
ws_id: str,
|
||||
state: str,
|
||||
*,
|
||||
tokens: int = 0,
|
||||
context_ratio: float = 0.0,
|
||||
activity: str = "",
|
||||
activity_state: str = "",
|
||||
content: str = "",
|
||||
) -> None:
|
||||
self.state_calls.append(
|
||||
{
|
||||
"ws_id": ws_id,
|
||||
"state": state,
|
||||
"tokens": tokens,
|
||||
"context_ratio": context_ratio,
|
||||
"activity": activity,
|
||||
"activity_state": activity_state,
|
||||
"content": content,
|
||||
}
|
||||
)
|
||||
|
||||
def update_console_ws_activity(self, ws_id: str, *, activity: str, activity_state: str) -> None:
|
||||
self.activity_calls.append(
|
||||
{"ws_id": ws_id, "activity": activity, "activity_state": activity_state}
|
||||
)
|
||||
|
||||
def emit_console_ws_created(self, *_a: Any, **_kw: Any) -> None:
|
||||
pass
|
||||
|
||||
def emit_console_ws_closed(self, *_a: Any, **_kw: Any) -> None:
|
||||
pass
|
||||
|
||||
def emit_console_ws_rename(self, *_a: Any, **_kw: Any) -> None:
|
||||
pass
|
||||
|
||||
def ensure_console_pseudo_node(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _build_adapter_and_ws(ws_id: str = "coord-ws-1") -> tuple[Any, Any, _FakeCollectorRecorder]:
|
||||
"""Construct a minimal adapter + Workstream + UI for emit_state tests.
|
||||
|
||||
Skips the full SessionManager wire-up — the adapter's ``emit_state``
|
||||
only reads ``ws.id`` and ``ws.ui``, so a real ``Workstream`` with
|
||||
a populated ``ConsoleCoordinatorUI`` is enough.
|
||||
"""
|
||||
from turnstone.console.coordinator_adapter import CoordinatorAdapter
|
||||
from turnstone.core.workstream import Workstream
|
||||
|
||||
recorder = _FakeCollectorRecorder()
|
||||
adapter = CoordinatorAdapter(
|
||||
collector=recorder, # type: ignore[arg-type]
|
||||
ui_factory=lambda ws: ConsoleCoordinatorUI(ws_id=ws.id, user_id=ws.user_id),
|
||||
session_factory=lambda ws: MagicMock(),
|
||||
)
|
||||
ws = Workstream(id=ws_id, user_id="u1", name="my-coord")
|
||||
ws.ui = ConsoleCoordinatorUI(ws_id=ws_id, user_id="u1")
|
||||
return adapter, ws, recorder
|
||||
|
||||
|
||||
def test_coord_adapter_emit_state_passes_rich_payload_to_collector() -> None:
|
||||
"""Pre-lift coord_adapter.emit_state called collector with state-only;
|
||||
post-lift it reads the UI's per-ws snapshot under lock and passes
|
||||
tokens / context_ratio / activity / content kwargs through."""
|
||||
adapter, ws, recorder = _build_adapter_and_ws()
|
||||
with _patch_get_storage(MagicMock()):
|
||||
ws.ui.on_status(
|
||||
{"prompt_tokens": 60, "completion_tokens": 40},
|
||||
context_window=400,
|
||||
effort="medium",
|
||||
)
|
||||
ws.ui.on_content_token("partial answer")
|
||||
ws.ui.on_thinking_start()
|
||||
adapter.emit_state(ws, WorkstreamState.RUNNING)
|
||||
assert len(recorder.state_calls) == 1
|
||||
call = recorder.state_calls[0]
|
||||
assert call["ws_id"] == ws.id
|
||||
assert call["state"] == "running"
|
||||
assert call["tokens"] == 100
|
||||
assert call["context_ratio"] == pytest.approx(0.25)
|
||||
assert call["activity"] == "Thinking…"
|
||||
assert call["activity_state"] == "thinking"
|
||||
# Mid-turn (RUNNING) — content stays accumulated for the eventual IDLE drain.
|
||||
assert call["content"] == ""
|
||||
|
||||
|
||||
def test_coord_adapter_emit_state_idle_drains_content() -> None:
|
||||
"""IDLE state-change drains the turn-content accumulator and
|
||||
piggybacks the joined content on the broadcast — same shape WebUI
|
||||
uses on global_queue. Subsequent emit_state must see the
|
||||
accumulator cleared."""
|
||||
adapter, ws, recorder = _build_adapter_and_ws()
|
||||
ws.ui.on_content_token("Here's ")
|
||||
ws.ui.on_content_token("the result.")
|
||||
adapter.emit_state(ws, WorkstreamState.IDLE)
|
||||
assert len(recorder.state_calls) == 1
|
||||
assert recorder.state_calls[0]["content"] == "Here's the result."
|
||||
# Accumulator drained — next emit_state sees nothing carried over.
|
||||
adapter.emit_state(ws, WorkstreamState.IDLE)
|
||||
assert recorder.state_calls[1]["content"] == ""
|
||||
|
||||
|
||||
def test_coord_adapter_emit_state_handles_missing_ui_defensively() -> None:
|
||||
"""``ws.ui`` can be ``None`` mid-eviction; emit_state still
|
||||
broadcasts the state-change with empty rich fields so the
|
||||
dashboard's coord row still flips state instead of going stale."""
|
||||
adapter, ws, recorder = _build_adapter_and_ws()
|
||||
ws.ui = None # simulate teardown race
|
||||
adapter.emit_state(ws, WorkstreamState.RUNNING)
|
||||
assert len(recorder.state_calls) == 1
|
||||
call = recorder.state_calls[0]
|
||||
assert call["state"] == "running"
|
||||
assert call["tokens"] == 0
|
||||
assert call["content"] == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coord activity broadcast — UI fans out directly to the collector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_ui_broadcast_activity_calls_collector() -> None:
|
||||
"""Live activity transitions on coord (between state changes) reach
|
||||
the cluster collector via the new ``update_console_ws_activity``
|
||||
method. WebUI's analog goes via the global SSE queue; coord's
|
||||
UI calls the collector directly since the console isn't a node."""
|
||||
recorder = _FakeCollectorRecorder()
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = recorder # type: ignore[assignment]
|
||||
try:
|
||||
ui.on_thinking_start() # base impl calls _broadcast_activity
|
||||
assert len(recorder.activity_calls) == 1
|
||||
call = recorder.activity_calls[0]
|
||||
assert call["ws_id"] == "coord-ws"
|
||||
assert call["activity"] == "Thinking…"
|
||||
assert call["activity_state"] == "thinking"
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
def test_coord_ui_broadcast_activity_swallows_collector_failure() -> None:
|
||||
"""A flaky collector must NOT block the worker thread — activity
|
||||
fan-out is observational, the worker keeps running on collector
|
||||
failure."""
|
||||
recorder = MagicMock()
|
||||
recorder.update_console_ws_activity.side_effect = RuntimeError("collector dead")
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = recorder
|
||||
try:
|
||||
ui.on_thinking_start() # must not raise
|
||||
recorder.update_console_ws_activity.assert_called_once()
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
def test_coord_ui_broadcast_activity_no_op_when_collector_unset() -> None:
|
||||
"""Tests / tooling that don't wire a collector shouldn't crash."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
ui.on_thinking_start() # must not raise
|
||||
|
||||
|
||||
def test_coord_ui_broadcast_activity_failure_does_not_strand_dedup() -> None:
|
||||
"""Regression for the Copilot finding on PR #420: post-fix the
|
||||
dedup state ``_last_broadcast_activity`` is updated **only after**
|
||||
a successful collector call. If the collector raises mid-broadcast
|
||||
on tick #1, tick #2 with the same activity tuple must still
|
||||
attempt the broadcast (otherwise a transient collector failure
|
||||
would strand the dashboard's coord row at the pre-failure
|
||||
activity until the activity actually changes). Pre-fix the
|
||||
dedup state was assigned inside the lock before the collector
|
||||
call, so the failed broadcast still updated it and tick #2
|
||||
silently no-op'd."""
|
||||
recorder = MagicMock()
|
||||
# First call fails (transient collector outage); second call succeeds.
|
||||
recorder.update_console_ws_activity.side_effect = [
|
||||
RuntimeError("collector dead"),
|
||||
None,
|
||||
]
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = recorder
|
||||
try:
|
||||
# Tick #1 — collector raises; dedup state must NOT update.
|
||||
ui.on_thinking_start()
|
||||
assert ui._last_broadcast_activity is None, (
|
||||
"dedup state was updated despite a failed collector call — "
|
||||
"next identical tick would be silently suppressed"
|
||||
)
|
||||
# Tick #2 — same activity tuple. Pre-fix this would no-op
|
||||
# (because dedup state was already (Thinking…, thinking)).
|
||||
# Post-fix it retries; collector succeeds; dedup state lands.
|
||||
ui.on_thinking_start()
|
||||
assert recorder.update_console_ws_activity.call_count == 2, (
|
||||
"second tick was deduped despite the first call failing"
|
||||
)
|
||||
assert ui._last_broadcast_activity == ("Thinking…", "thinking")
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
def test_coord_ui_broadcast_activity_dedup_skips_identical_after_success() -> None:
|
||||
"""Happy-path dedup: after a successful broadcast, the next identical
|
||||
tick is deduped — the cluster collector lock is not re-acquired
|
||||
for a no-op write. This is the perf optimization the dedup is
|
||||
there for; the regression test above checks the failure-recovery
|
||||
invariant doesn't break it."""
|
||||
recorder = MagicMock()
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = recorder
|
||||
try:
|
||||
ui.on_thinking_start() # tick 1 — fires
|
||||
ui.on_thinking_start() # tick 2 — same tuple, deduped
|
||||
ui.on_thinking_start() # tick 3 — same tuple, deduped
|
||||
assert recorder.update_console_ws_activity.call_count == 1
|
||||
assert ui._last_broadcast_activity == ("Thinking…", "thinking")
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spawn metrics — coord wires its own hook
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_spawn_metrics_increments_messages_and_resets_tool_count() -> None:
|
||||
"""Coord's ``_coord_spawn_metrics`` mirrors interactive's per-spawn
|
||||
counter writes (sans the Prometheus call) so the rich ``ws_state``
|
||||
broadcast renders the same per-turn shape."""
|
||||
from turnstone.console.server import _coord_spawn_metrics
|
||||
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui._ws_messages = 5
|
||||
ui._ws_turn_tool_calls = 3
|
||||
_coord_spawn_metrics(MagicMock(), ui)
|
||||
assert ui._ws_messages == 6
|
||||
assert ui._ws_turn_tool_calls == 0
|
||||
|
||||
|
||||
def test_coord_spawn_metrics_tolerates_ui_without_counters() -> None:
|
||||
"""A SessionUI subclass without the per-ws counters shouldn't trip
|
||||
the hook — defensive guard mirrors the interactive analog."""
|
||||
from turnstone.console.server import _coord_spawn_metrics
|
||||
|
||||
class _StubUI:
|
||||
pass
|
||||
|
||||
_coord_spawn_metrics(MagicMock(), _StubUI()) # must not raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Snapshot lock — single-acquisition guarantee
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_snapshot_acquires_ws_lock_exactly_once() -> None:
|
||||
"""Snapshot must read all four fields under a single lock acquisition
|
||||
so concurrent on_status / on_thinking_start writes can't tear the
|
||||
payload."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
acquire_count = 0
|
||||
inner = ui._ws_lock
|
||||
|
||||
class _CountingLock:
|
||||
def __enter__(self) -> None:
|
||||
nonlocal acquire_count
|
||||
acquire_count += 1
|
||||
inner.acquire()
|
||||
|
||||
def __exit__(self, *a: Any) -> None:
|
||||
inner.release()
|
||||
|
||||
def acquire(self, *a: Any, **kw: Any) -> bool:
|
||||
return inner.acquire(*a, **kw)
|
||||
|
||||
def release(self) -> None:
|
||||
inner.release()
|
||||
|
||||
ui._ws_lock = _CountingLock() # type: ignore[assignment]
|
||||
ui.snapshot_and_consume_state_payload("idle")
|
||||
assert acquire_count == 1, (
|
||||
f"snapshot acquired _ws_lock {acquire_count} times; concurrent "
|
||||
"writes could tear the rich payload"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concurrency — snapshot under load
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_snapshot_under_concurrent_writes_does_not_crash() -> None:
|
||||
"""Sanity stress: snapshot reads while on_status / on_thinking_start /
|
||||
on_content_token write concurrently. Reader cycles through
|
||||
``("running", "idle", "error")`` so the IDLE/ERROR drain branches
|
||||
that mutate ``_ws_turn_content`` actually get exercised against
|
||||
concurrent appends — running-only would only hit the read-only
|
||||
snapshot path. Each thread's exception (if any) is captured + raised
|
||||
on join so a silent worker crash can't slip through as a bare
|
||||
deadlock-check pass."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
writer_exc: list[Exception] = []
|
||||
reader_exc: list[Exception] = []
|
||||
|
||||
def _writer() -> None:
|
||||
try:
|
||||
with _patch_get_storage(MagicMock()):
|
||||
for i in range(50):
|
||||
ui.on_status(
|
||||
{"prompt_tokens": i, "completion_tokens": i},
|
||||
context_window=1000,
|
||||
effort="low",
|
||||
)
|
||||
ui.on_content_token(f"chunk-{i}")
|
||||
ui.on_thinking_start()
|
||||
except Exception as exc: # noqa: BLE001 — surface to main thread
|
||||
writer_exc.append(exc)
|
||||
|
||||
def _reader() -> None:
|
||||
try:
|
||||
states = ("running", "idle", "error")
|
||||
for i in range(50):
|
||||
ui.snapshot_and_consume_state_payload(states[i % len(states)])
|
||||
except Exception as exc: # noqa: BLE001 — surface to main thread
|
||||
reader_exc.append(exc)
|
||||
|
||||
writer = threading.Thread(target=_writer)
|
||||
reader = threading.Thread(target=_reader)
|
||||
writer.start()
|
||||
reader.start()
|
||||
writer.join(timeout=5)
|
||||
reader.join(timeout=5)
|
||||
assert not writer.is_alive(), "writer thread deadlocked"
|
||||
assert not reader.is_alive(), "reader thread deadlocked"
|
||||
assert not writer_exc, f"writer raised: {writer_exc[0]!r}"
|
||||
assert not reader_exc, f"reader raised: {reader_exc[0]!r}"
|
||||
|
||||
|
||||
def test_coord_on_stream_end_clears_activity() -> None:
|
||||
"""Lifted ``on_stream_end`` body clears ``_ws_current_activity``
|
||||
and ``_ws_activity_state`` so the dashboard's coord row stops
|
||||
showing the stale 'Thinking…' indicator after the stream
|
||||
finishes. Pre-lift coord just enqueued ``stream_end`` without
|
||||
touching activity — this test pins the new clear path so a
|
||||
future re-stub doesn't silently re-introduce a stuck activity
|
||||
indicator."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-ws", user_id="u1")
|
||||
ui._ws_current_activity = "Thinking…"
|
||||
ui._ws_activity_state = "thinking"
|
||||
ui.on_stream_end()
|
||||
assert ui._ws_current_activity == ""
|
||||
assert ui._ws_activity_state == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WebUI override semantics still preserved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_webui_on_status_still_records_prometheus_metrics() -> None:
|
||||
"""The lift moves the per-ws writes to SessionUIBase but WebUI's
|
||||
override must still fire ``_metrics.record_*`` (Prometheus on the
|
||||
node /metrics endpoint). Regression guard against a future refactor
|
||||
accidentally dropping the override."""
|
||||
import queue
|
||||
|
||||
from turnstone.server import WebUI
|
||||
|
||||
WebUI._global_queue = queue.Queue()
|
||||
try:
|
||||
ui = WebUI(ws_id="ws-int", user_id="u1")
|
||||
with patch("turnstone.server._metrics") as mock_metrics, _patch_get_storage(MagicMock()):
|
||||
ui.on_status(
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
context_window=200,
|
||||
effort="low",
|
||||
)
|
||||
mock_metrics.record_tokens.assert_called_once_with(10, 5)
|
||||
mock_metrics.record_cache_tokens.assert_called_once()
|
||||
mock_metrics.record_context_ratio.assert_called_once()
|
||||
finally:
|
||||
WebUI._global_queue = None
|
||||
|
||||
|
||||
def test_webui_on_tool_result_still_records_prometheus_tool_call() -> None:
|
||||
"""Same as above for ``on_tool_result``."""
|
||||
import queue
|
||||
|
||||
from turnstone.server import WebUI
|
||||
|
||||
WebUI._global_queue = queue.Queue()
|
||||
try:
|
||||
ui = WebUI(ws_id="ws-int", user_id="u1")
|
||||
with patch("turnstone.server._metrics") as mock_metrics:
|
||||
ui.on_tool_result("call-1", "bash", "output")
|
||||
mock_metrics.record_tool_call.assert_called_once_with("bash")
|
||||
# Per-ws counter writes happened too (inherited from base).
|
||||
assert ui._ws_tool_calls == {"bash": 1}
|
||||
assert ui._ws_turn_tool_calls == 1
|
||||
finally:
|
||||
WebUI._global_queue = None
|
||||
@@ -1,466 +0,0 @@
|
||||
"""Tests for the unified ``approve_tools`` body, viewed from the coord side.
|
||||
|
||||
The body itself is exercised by ``test_webui_auto_approve_visibility``;
|
||||
this file pins down the coord-specific contracts that lifting the body
|
||||
to ``SessionUIBase`` automatically enables:
|
||||
|
||||
- Tool-policy gating now applies to coord tool calls (was interactive-only).
|
||||
- Heuristic verdicts persist on coord (was interactive-only).
|
||||
- The activity tag fields populate on coord during pending approval.
|
||||
- ``judge_pending`` is dynamic on the coord ``approve_request``
|
||||
(was hardcoded ``False``).
|
||||
- The auto-approve fall-through emits ``tool_info`` (was
|
||||
``tools_auto_approved``).
|
||||
- ``_record_judge_metric`` is a no-op on coord (no Prometheus on console).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
|
||||
|
||||
def _make_items(*specs: tuple[str, str], needs_approval: bool = True) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"call_id": call_id,
|
||||
"header": f"Tool: {func}",
|
||||
"preview": "preview text",
|
||||
"func_name": func,
|
||||
"approval_label": func,
|
||||
"needs_approval": needs_approval,
|
||||
}
|
||||
for call_id, func in specs
|
||||
]
|
||||
|
||||
|
||||
def _patch_storage(storage: Any):
|
||||
return patch("turnstone.core.storage._registry.get_storage", return_value=storage)
|
||||
|
||||
|
||||
def _patch_policies(verdicts: dict[str, str]):
|
||||
return patch(
|
||||
"turnstone.core.policy.evaluate_tool_policies_batch",
|
||||
return_value=verdicts,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inheritance regression — the unification itself
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_inherits_approve_tools_from_base() -> None:
|
||||
"""``ConsoleCoordinatorUI`` must NOT define its own ``approve_tools``;
|
||||
the shared body lives on :class:`SessionUIBase`. A future drift —
|
||||
adding a coord-only override — is exactly the kind of bug this
|
||||
unification is meant to prevent, so guard it explicitly."""
|
||||
assert "approve_tools" not in ConsoleCoordinatorUI.__dict__, (
|
||||
"ConsoleCoordinatorUI shouldn't redefine approve_tools — "
|
||||
"the shared body on SessionUIBase covers both kinds."
|
||||
)
|
||||
assert ConsoleCoordinatorUI.approve_tools.__qualname__ == "SessionUIBase.approve_tools"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool-policy gating now applies to coord
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_tool_policy_deny_blocks_coord_tool() -> None:
|
||||
"""Admin-defined ``deny`` policies now fire on coord tool calls.
|
||||
Pre-lift this was interactive-only; an admin who wanted to block
|
||||
e.g. ``delete_workstream`` on the coord couldn't."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
items = _make_items(("c1", "delete_workstream"))
|
||||
|
||||
storage = MagicMock()
|
||||
with _patch_storage(storage), _patch_policies({"delete_workstream": "deny"}):
|
||||
approved, err = ui.approve_tools(items)
|
||||
|
||||
assert approved is False
|
||||
assert err == "Blocked by tool policy"
|
||||
assert items[0].get("denied") is True
|
||||
|
||||
|
||||
def test_coord_tool_policy_allow_tags_with_policy_source() -> None:
|
||||
"""Admin ``allow`` rule auto-approves the item with
|
||||
``AutoApproveReason.POLICY``. This was a no-op on coord pre-lift."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
|
||||
storage = MagicMock()
|
||||
with _patch_storage(storage), _patch_policies({"spawn_workstream": "allow"}):
|
||||
approved, _err = ui.approve_tools(items)
|
||||
|
||||
assert approved is True
|
||||
snapshot = ui.serialize_recent_auto_approvals()
|
||||
assert len(snapshot) == 1
|
||||
assert snapshot[0]["func_name"] == "spawn_workstream"
|
||||
assert snapshot[0]["auto_approve_reason"] == "policy"
|
||||
|
||||
|
||||
def test_coord_tool_policy_mixed_allow_deny_records_allowed_sibling() -> None:
|
||||
"""Same ``mixed-policy`` audit-leak fix that
|
||||
``test_webui_auto_approve_visibility`` validates for interactive,
|
||||
now auto-applies to coord via the lifted body."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
items = _make_items(("c1", "delete_workstream"), ("c2", "list_workstreams"))
|
||||
|
||||
storage = MagicMock()
|
||||
with (
|
||||
_patch_storage(storage),
|
||||
_patch_policies({"delete_workstream": "deny", "list_workstreams": "allow"}),
|
||||
):
|
||||
approved, _err = ui.approve_tools(items)
|
||||
|
||||
assert approved is False
|
||||
snapshot = ui.serialize_recent_auto_approvals()
|
||||
assert len(snapshot) == 1
|
||||
assert snapshot[0]["func_name"] == "list_workstreams"
|
||||
assert snapshot[0]["auto_approve_reason"] == "policy"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Heuristic-verdict persistence + metric hook
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_heuristic_verdict_persists_to_storage() -> None:
|
||||
"""Heuristic verdicts attached to items now flow through to
|
||||
``storage.create_intent_verdicts_bulk`` on coord. Pre-lift coord
|
||||
silently dropped them; only LLM-tier verdicts (from the daemon
|
||||
judge thread via ``on_intent_verdict``) reached storage. Post
|
||||
perf-2 the path uses bulk INSERT so a fan-out turn pays one commit
|
||||
instead of N."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
hv = {
|
||||
"verdict_id": "v1",
|
||||
"call_id": "c1",
|
||||
"func_name": "spawn_workstream",
|
||||
"tier": "heuristic",
|
||||
"risk_level": "high",
|
||||
"confidence": 0.75,
|
||||
"recommendation": "review",
|
||||
"reasoning": "spawning child with bash skill",
|
||||
"evidence": ["bash"],
|
||||
"latency_ms": 12,
|
||||
}
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
items[0]["_heuristic_verdict"] = hv
|
||||
|
||||
storage = MagicMock()
|
||||
timer = threading.Timer(0.05, lambda: ui.resolve_approval(False))
|
||||
timer.start()
|
||||
try:
|
||||
with _patch_storage(storage):
|
||||
ui.approve_tools(items)
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
storage.create_intent_verdicts_bulk.assert_called_once()
|
||||
rows = storage.create_intent_verdicts_bulk.call_args.args[0]
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["verdict_id"] == "v1"
|
||||
assert rows[0]["tier"] == "heuristic"
|
||||
assert rows[0]["ws_id"] == "coord-1"
|
||||
|
||||
|
||||
def test_coord_record_judge_metric_fires_console_metrics() -> None:
|
||||
"""``_record_judge_metric`` increments the console's
|
||||
``ConsoleMetrics`` judge counter when the class attribute is wired,
|
||||
so coord verdicts surface on the console's /metrics endpoint
|
||||
alongside the per-node series."""
|
||||
from turnstone.console.metrics import ConsoleMetrics
|
||||
|
||||
cm = ConsoleMetrics()
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
try:
|
||||
ConsoleCoordinatorUI._console_metrics = cm
|
||||
ui._record_judge_metric({"tier": "heuristic", "risk_level": "high", "latency_ms": 12})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._console_metrics = None
|
||||
|
||||
text = cm.generate_text()
|
||||
assert 'turnstone_judge_verdicts_total{tier="heuristic",risk_level="high"} 1' in text
|
||||
|
||||
|
||||
def test_coord_record_judge_metric_safe_when_unwired() -> None:
|
||||
"""No /metrics instance set → silent no-op. Test fixtures that
|
||||
don't spin up a full console app must not crash on judge
|
||||
verdicts during the shared ``approve_tools`` body."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
# Sanity: class attribute is None at module import time outside
|
||||
# the lifespan — exactly the test-fixture state.
|
||||
assert ConsoleCoordinatorUI._console_metrics is None
|
||||
# Should not raise.
|
||||
ui._record_judge_metric({"tier": "heuristic", "risk_level": "low"})
|
||||
|
||||
|
||||
def test_coord_on_intent_verdict_fires_metric_for_llm_tier() -> None:
|
||||
"""Async LLM verdicts from the daemon judge thread land at
|
||||
``on_intent_verdict``. Coord overrides it to fire the same
|
||||
``record_judge_verdict`` call WebUI does — different tier label,
|
||||
same cluster-wide histogram."""
|
||||
from turnstone.console.metrics import ConsoleMetrics
|
||||
|
||||
cm = ConsoleMetrics()
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
try:
|
||||
ConsoleCoordinatorUI._console_metrics = cm
|
||||
with _patch_storage(MagicMock()):
|
||||
ui.on_intent_verdict(
|
||||
{
|
||||
"verdict_id": "v1",
|
||||
"call_id": "c1",
|
||||
"tier": "llm",
|
||||
"risk_level": "medium",
|
||||
"latency_ms": 250,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._console_metrics = None
|
||||
|
||||
text = cm.generate_text()
|
||||
assert 'turnstone_judge_verdicts_total{tier="llm",risk_level="medium"} 1' in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Activity tagging during pending approval
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_pending_approval_sets_activity_tag() -> None:
|
||||
"""The shared body tags ``_ws_current_activity`` /
|
||||
``_ws_activity_state`` so the cluster collector's coord-row
|
||||
snapshot reflects the approval wait. Pre-lift coord left these
|
||||
fields empty during pending approval."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
items = _make_items(("c1", "delete_workstream"))
|
||||
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
def _capture_activity() -> None:
|
||||
captured["activity"] = ui._ws_current_activity
|
||||
captured["state"] = ui._ws_activity_state
|
||||
ui.resolve_approval(False)
|
||||
|
||||
timer = threading.Timer(0.05, _capture_activity)
|
||||
timer.start()
|
||||
try:
|
||||
with _patch_storage(MagicMock()):
|
||||
ui.approve_tools(items)
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
assert "Awaiting approval" in captured["activity"]
|
||||
assert "delete_workstream" in captured["activity"]
|
||||
assert captured["state"] == "approval"
|
||||
|
||||
|
||||
def test_coord_auto_approve_sets_tool_activity_tag() -> None:
|
||||
"""Blanket auto-approve flips activity to the ``⚙ {tool}: {preview}``
|
||||
shape WebUI has used; coord row now mirrors it."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
ui.auto_approve = True
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
|
||||
with _patch_storage(MagicMock()):
|
||||
approved, _err = ui.approve_tools(items)
|
||||
|
||||
assert approved is True
|
||||
assert "spawn_workstream" in ui._ws_current_activity
|
||||
assert ui._ws_activity_state == "tool"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# judge_pending flag + event-name parity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_judge_pending_flag_dynamic_when_heuristic_present() -> None:
|
||||
"""Pre-lift coord hardcoded ``judge_pending=False`` on every
|
||||
``approve_request``; the unified body computes the bool from the
|
||||
items, matching WebUI."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
items[0]["_heuristic_verdict"] = {"verdict_id": "v1", "tier": "heuristic"}
|
||||
|
||||
captured_events: list[dict[str, Any]] = []
|
||||
ui._enqueue = captured_events.append # type: ignore[method-assign]
|
||||
|
||||
timer = threading.Timer(0.05, lambda: ui.resolve_approval(False))
|
||||
timer.start()
|
||||
try:
|
||||
with _patch_storage(MagicMock()):
|
||||
ui.approve_tools(items)
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
approve_requests = [e for e in captured_events if e.get("type") == "approve_request"]
|
||||
assert len(approve_requests) == 1
|
||||
assert approve_requests[0]["judge_pending"] is True
|
||||
|
||||
|
||||
def test_coord_blanket_auto_approve_emits_tool_info() -> None:
|
||||
"""Event-name parity: the auto-approve fall-through emits
|
||||
``tool_info`` for both kinds. Pre-lift coord emitted
|
||||
``tools_auto_approved`` — the rename happens implicitly via
|
||||
inheritance."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
ui.auto_approve = True
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
|
||||
captured_events: list[dict[str, Any]] = []
|
||||
ui._enqueue = captured_events.append # type: ignore[method-assign]
|
||||
|
||||
with _patch_storage(MagicMock()):
|
||||
ui.approve_tools(items)
|
||||
|
||||
types = [e.get("type") for e in captured_events]
|
||||
assert "tool_info" in types
|
||||
assert "tools_auto_approved" not in types
|
||||
|
||||
|
||||
def test_coord_judge_pending_false_when_no_heuristic_verdict() -> None:
|
||||
"""Counterpart to ``test_coord_judge_pending_flag_dynamic_when_heuristic_present``:
|
||||
items with no ``_heuristic_verdict`` produce ``approve_request`` with
|
||||
``judge_pending=False``. Without this case pinned, a regression that
|
||||
hardcodes ``judge_pending=True`` (the inverse of the pre-lift coord
|
||||
bug) would slip through unnoticed."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
# Deliberately no _heuristic_verdict on any item.
|
||||
|
||||
captured_events: list[dict[str, Any]] = []
|
||||
ui._enqueue = captured_events.append # type: ignore[method-assign]
|
||||
|
||||
timer = threading.Timer(0.05, lambda: ui.resolve_approval(False))
|
||||
timer.start()
|
||||
try:
|
||||
with _patch_storage(MagicMock()):
|
||||
ui.approve_tools(items)
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
approve_requests = [e for e in captured_events if e.get("type") == "approve_request"]
|
||||
assert len(approve_requests) == 1
|
||||
assert approve_requests[0]["judge_pending"] is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-tool auto-approve via auto_approve_tools (set membership)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_per_tool_auto_approve_tags_with_source() -> None:
|
||||
"""When a coord tool name lands in ``auto_approve_tools`` (e.g. via a
|
||||
skill template's ``allowed_tools``), the lifted body short-circuits
|
||||
the prompt and tags the item with ``AutoApproveReason.AUTO_APPROVE_TOOLS``
|
||||
(or the per-tool source from ``_auto_approve_tools_source``).
|
||||
Mirrors the WebUI test ``test_auto_approve_tools_skill_source_renders_as_skill``
|
||||
on the coord side so the unified body gains parity coverage."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
ui.auto_approve_tools = {"spawn_workstream"}
|
||||
ui._auto_approve_tools_source = {"spawn_workstream": "skill"}
|
||||
items = _make_items(("c1", "spawn_workstream"))
|
||||
|
||||
storage = MagicMock()
|
||||
with _patch_storage(storage):
|
||||
approved, _err = ui.approve_tools(items)
|
||||
|
||||
assert approved is True
|
||||
snapshot = ui.serialize_recent_auto_approvals()
|
||||
assert len(snapshot) == 1
|
||||
assert snapshot[0]["func_name"] == "spawn_workstream"
|
||||
assert snapshot[0]["auto_approve_reason"] == "skill"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# __budget_override__ carve-out — sec-2 hardening
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coord_budget_override_prompts_even_under_blanket_auto_approve() -> None:
|
||||
"""The carve-out promises ``__budget_override__`` always prompts the
|
||||
operator. Pin that behavior on the coord side so a future regression
|
||||
of the post-filter / pre-filter check (sec-2) gets caught.
|
||||
|
||||
``__budget_override__`` is interactive-only today (coord workstreams
|
||||
don't have token budgets), but the synthetic item can be threaded
|
||||
through ``approve_tools`` directly the same way ``ChatSession.send``
|
||||
does on the interactive side. The carve-out fires uniformly across
|
||||
both kinds."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
ui.auto_approve = True # blanket flag — should NOT bypass the carve-out
|
||||
items = [
|
||||
{
|
||||
"call_id": "c1",
|
||||
"header": "Token budget exhausted",
|
||||
"preview": "Token budget (200,000) exhausted. Approve to continue.",
|
||||
"func_name": "__budget_override__",
|
||||
"approval_label": "__budget_override__",
|
||||
"needs_approval": True,
|
||||
}
|
||||
]
|
||||
|
||||
captured_events: list[dict[str, Any]] = []
|
||||
ui._enqueue = captured_events.append # type: ignore[method-assign]
|
||||
|
||||
timer = threading.Timer(0.05, lambda: ui.resolve_approval(True))
|
||||
timer.start()
|
||||
try:
|
||||
with _patch_storage(MagicMock()):
|
||||
approved, _err = ui.approve_tools(items)
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
assert approved is True
|
||||
# The carve-out forces the prompt path, NOT the auto-approve fall-through.
|
||||
types = [e.get("type") for e in captured_events]
|
||||
assert "approve_request" in types, (
|
||||
"Budget override must produce an approve_request even under blanket auto_approve"
|
||||
)
|
||||
assert "tool_info" not in types, (
|
||||
"Auto-approve fall-through must not fire when a budget override is present"
|
||||
)
|
||||
|
||||
|
||||
def test_coord_budget_override_survives_wildcard_allow_policy() -> None:
|
||||
"""A wildcard ``*: allow`` policy must not strip ``__budget_override__``
|
||||
from the gate. Pre-sec-2, the policy block could mark the item
|
||||
``needs_approval=False`` and remove it from ``pending``, after which
|
||||
the carve-out (which read ``pending``) would see no override and
|
||||
blanket auto-approve would silently fire. Post-fix the carve-out
|
||||
reads from the pre-filter ``items`` list AND the policy block skips
|
||||
matching the synthetic name entirely."""
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-1", user_id="u1")
|
||||
ui.auto_approve = True
|
||||
items = [
|
||||
{
|
||||
"call_id": "c1",
|
||||
"header": "Token budget exhausted",
|
||||
"preview": "Token budget exhausted. Approve to continue.",
|
||||
"func_name": "__budget_override__",
|
||||
"approval_label": "__budget_override__",
|
||||
"needs_approval": True,
|
||||
}
|
||||
]
|
||||
|
||||
captured_events: list[dict[str, Any]] = []
|
||||
ui._enqueue = captured_events.append # type: ignore[method-assign]
|
||||
|
||||
timer = threading.Timer(0.05, lambda: ui.resolve_approval(True))
|
||||
timer.start()
|
||||
try:
|
||||
with _patch_storage(MagicMock()), _patch_policies({"__budget_override__": "allow"}):
|
||||
approved, _err = ui.approve_tools(items)
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
assert approved is True
|
||||
types = [e.get("type") for e in captured_events]
|
||||
assert "approve_request" in types, "Wildcard allow must not strip the budget-override prompt"
|
||||
@@ -1,611 +0,0 @@
|
||||
"""Tests for CoordinatorAdapter.
|
||||
|
||||
Mirrors test_interactive_adapter.py: focuses on the transport contract
|
||||
(what gets sent to the ClusterCollector) and cleanup_ui behavior
|
||||
(unblock listener queues, cancel session). The SessionManager-level
|
||||
tests in test_session_manager.py cover the lifecycle path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from turnstone.console.coordinator_adapter import CoordinatorAdapter
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
|
||||
|
||||
class _StubCoordUI:
|
||||
"""Stub matching the subset of ConsoleCoordinatorUI the adapter touches."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._approval_event = threading.Event()
|
||||
self._approval_result: tuple[bool, str | None] = (True, "initial")
|
||||
self._plan_event = threading.Event()
|
||||
self._plan_result: str = "accept"
|
||||
self._fg_event = threading.Event()
|
||||
self._listeners_lock = threading.Lock()
|
||||
self._listeners: list[queue.Queue[dict[str, Any]]] = []
|
||||
|
||||
|
||||
class _StubSession:
|
||||
def __init__(self) -> None:
|
||||
self.cancelled = False
|
||||
self.closed = False
|
||||
|
||||
def cancel(self) -> None:
|
||||
self.cancelled = True
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _make_adapter(
|
||||
collector: Any = None,
|
||||
*,
|
||||
ui_factory: Any = None,
|
||||
session_factory: Any = None,
|
||||
) -> tuple[CoordinatorAdapter, MagicMock]:
|
||||
collector = collector or MagicMock()
|
||||
adapter = CoordinatorAdapter(
|
||||
collector=collector,
|
||||
ui_factory=ui_factory or (lambda ws: _StubCoordUI()),
|
||||
session_factory=session_factory or (lambda *a, **kw: _StubSession()),
|
||||
)
|
||||
return adapter, collector
|
||||
|
||||
|
||||
def _make_ws(**overrides: Any) -> Workstream:
|
||||
ws = Workstream(id="coord-1", name="my-coord")
|
||||
ws.kind = WorkstreamKind.COORDINATOR
|
||||
ws.user_id = "u1"
|
||||
ws.ui = _StubCoordUI()
|
||||
ws.session = _StubSession()
|
||||
for k, v in overrides.items():
|
||||
setattr(ws, k, v)
|
||||
return ws
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Transport — emit_created / emit_state / emit_closed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_emit_created_calls_collector_with_coord_fields() -> None:
|
||||
adapter, collector = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.emit_created(ws)
|
||||
collector.emit_console_ws_created.assert_called_once_with(
|
||||
"coord-1",
|
||||
name="my-coord",
|
||||
user_id="u1",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=WorkstreamState.IDLE.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
|
||||
|
||||
def test_emit_state_calls_collector_state() -> None:
|
||||
"""Post-rich-payload, emit_state passes tokens / context_ratio /
|
||||
activity / activity_state / content kwargs read from ws.ui's
|
||||
snapshot. Default values (zeros / empty strings) when the UI
|
||||
hasn't recorded any per-ws metrics yet."""
|
||||
adapter, collector = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.emit_state(ws, WorkstreamState.RUNNING)
|
||||
collector.emit_console_ws_state.assert_called_once_with(
|
||||
"coord-1",
|
||||
WorkstreamState.RUNNING.value,
|
||||
tokens=0,
|
||||
context_ratio=0.0,
|
||||
activity="",
|
||||
activity_state="",
|
||||
content="",
|
||||
)
|
||||
|
||||
|
||||
def test_emit_closed_calls_collector_closed() -> None:
|
||||
adapter, collector = _make_adapter()
|
||||
adapter.emit_closed("coord-1")
|
||||
collector.emit_console_ws_closed.assert_called_once_with("coord-1")
|
||||
|
||||
|
||||
def test_emit_closed_swallows_reason_kwarg() -> None:
|
||||
"""The console collector doesn't propagate a 'reason' — the console
|
||||
frontend's evicted special-case only fires for real-node
|
||||
workstreams. Protocol compatibility only."""
|
||||
adapter, collector = _make_adapter()
|
||||
adapter.emit_closed("coord-1", reason="evicted")
|
||||
collector.emit_console_ws_closed.assert_called_once_with("coord-1")
|
||||
|
||||
|
||||
def test_emit_tolerates_collector_exception() -> None:
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_created.side_effect = RuntimeError("collector dead")
|
||||
collector.emit_console_ws_state.side_effect = RuntimeError("collector dead")
|
||||
collector.emit_console_ws_closed.side_effect = RuntimeError("collector dead")
|
||||
adapter, _ = _make_adapter(collector=collector)
|
||||
ws = _make_ws()
|
||||
# All three must swallow — the session lifecycle must not break
|
||||
# because the collector had a transient failure.
|
||||
adapter.emit_created(ws)
|
||||
adapter.emit_state(ws, WorkstreamState.RUNNING)
|
||||
adapter.emit_closed("coord-1")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cleanup_ui
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cleanup_ui_unblocks_events_and_broadcasts_to_listeners() -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
ws.ui._approval_event.clear() # type: ignore[attr-defined]
|
||||
ws.ui._plan_event.clear() # type: ignore[attr-defined]
|
||||
ws.ui._fg_event.clear() # type: ignore[attr-defined]
|
||||
lq: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=5)
|
||||
ws.ui._listeners.append(lq) # type: ignore[attr-defined]
|
||||
|
||||
adapter.cleanup_ui(ws)
|
||||
|
||||
assert ws.ui._approval_event.is_set() # type: ignore[attr-defined]
|
||||
assert ws.ui._plan_event.is_set() # type: ignore[attr-defined]
|
||||
assert ws.ui._fg_event.is_set() # type: ignore[attr-defined]
|
||||
assert ws.ui._approval_result == (False, None) # type: ignore[attr-defined]
|
||||
assert ws.ui._plan_result == "reject" # type: ignore[attr-defined]
|
||||
assert lq.get_nowait() == {"type": "ws_closed"}
|
||||
assert ws.ui._listeners == [] # type: ignore[attr-defined]
|
||||
assert ws.session.cancelled is True # type: ignore[attr-defined]
|
||||
assert ws.session.closed is True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_cleanup_ui_listener_full_queue_evicts_head() -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
lq: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1)
|
||||
lq.put_nowait({"type": "stale"})
|
||||
ws.ui._listeners.append(lq) # type: ignore[attr-defined]
|
||||
adapter.cleanup_ui(ws)
|
||||
assert lq.get_nowait() == {"type": "ws_closed"}
|
||||
|
||||
|
||||
def test_cleanup_ui_tolerates_missing_session_and_ui() -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
ws.session = None
|
||||
ws.ui = None
|
||||
adapter.cleanup_ui(ws) # no crash
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction passthrough
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_session_forwards_skill_model_kind_parent() -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _sf(ui: Any, model: str | None, ws_id: str, **kwargs: Any) -> Any:
|
||||
captured["ui"] = ui
|
||||
captured["model"] = model
|
||||
captured["ws_id"] = ws_id
|
||||
captured.update(kwargs)
|
||||
return _StubSession()
|
||||
|
||||
adapter, _ = _make_adapter(session_factory=_sf)
|
||||
ws = _make_ws()
|
||||
ws.parent_ws_id = None
|
||||
adapter.build_session(ws, skill="coordinator", model="gpt-5")
|
||||
assert captured["ui"] is ws.ui
|
||||
assert captured["model"] == "gpt-5"
|
||||
assert captured["skill"] == "coordinator"
|
||||
assert captured["kind"] == WorkstreamKind.COORDINATOR
|
||||
assert captured["parent_ws_id"] is None
|
||||
# client_type intentionally NOT forwarded — coord session_factory
|
||||
# doesn't accept it (fixed as 'console').
|
||||
assert "client_type" not in captured
|
||||
|
||||
|
||||
def test_build_ui_delegates_to_ui_factory() -> None:
|
||||
captured_ws: list[Workstream] = []
|
||||
|
||||
def _ui_factory(ws: Workstream) -> Any:
|
||||
captured_ws.append(ws)
|
||||
return _StubCoordUI()
|
||||
|
||||
adapter, _ = _make_adapter(ui_factory=_ui_factory)
|
||||
ws = _make_ws()
|
||||
result = adapter.build_ui(ws)
|
||||
assert captured_ws == [ws]
|
||||
assert isinstance(result, _StubCoordUI)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker dispatch — _spawn_worker / send
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _SendSession:
|
||||
"""ChatSession stub with send / queue_message accounting."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
queue_full: bool = False,
|
||||
send_gate: threading.Event | None = None,
|
||||
) -> None:
|
||||
self.send_calls: list[str] = []
|
||||
self.queue_calls: list[str] = []
|
||||
self._queue_full = queue_full
|
||||
# When set, ``send`` blocks on this event — lets the test pin a
|
||||
# worker inside session.send while a second thread races through
|
||||
# _spawn_worker, proving the lock gate (not Thread.is_alive) is
|
||||
# what serialises them.
|
||||
self._send_gate = send_gate
|
||||
self._send_lock = threading.Lock()
|
||||
self.cancelled = False
|
||||
self.closed = False
|
||||
|
||||
def send(
|
||||
self,
|
||||
message: str,
|
||||
attachments: Any = None,
|
||||
send_id: str | None = None,
|
||||
) -> None:
|
||||
if self._send_gate is not None:
|
||||
self._send_gate.wait(timeout=2.0)
|
||||
with self._send_lock:
|
||||
self.send_calls.append(message)
|
||||
|
||||
def queue_message(
|
||||
self,
|
||||
message: str,
|
||||
attachment_ids: Any = None,
|
||||
queue_msg_id: str | None = None,
|
||||
) -> None:
|
||||
if self._queue_full:
|
||||
raise queue.Full
|
||||
self.queue_calls.append(message)
|
||||
|
||||
def cancel(self) -> None:
|
||||
self.cancelled = True
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _StubManager:
|
||||
"""Minimal SessionManager stub exposing ``get`` for adapter.send."""
|
||||
|
||||
def __init__(self, ws: Workstream | None = None) -> None:
|
||||
self._ws = ws
|
||||
|
||||
def get(self, ws_id: str) -> Workstream | None:
|
||||
if self._ws is not None and self._ws.id == ws_id:
|
||||
return self._ws
|
||||
return None
|
||||
|
||||
|
||||
class TestCoordinatorAdapterWorkerDispatch:
|
||||
def test_spawn_worker_reuses_when_worker_running(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
session = _SendSession()
|
||||
ws.session = session # type: ignore[assignment]
|
||||
ws._worker_running = True # pre-existing worker
|
||||
adapter.attach(_StubManager(ws)) # type: ignore[arg-type]
|
||||
|
||||
assert adapter.send(ws.id, "hello") is True
|
||||
assert session.queue_calls == ["hello"]
|
||||
assert session.send_calls == []
|
||||
# worker_thread not replaced
|
||||
assert ws.worker_thread is None
|
||||
|
||||
def test_spawn_worker_returns_false_on_queue_full(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
session = _SendSession(queue_full=True)
|
||||
ws.session = session # type: ignore[assignment]
|
||||
ws._worker_running = True
|
||||
adapter.attach(_StubManager(ws)) # type: ignore[arg-type]
|
||||
|
||||
assert adapter.send(ws.id, "hello") is False
|
||||
assert session.send_calls == []
|
||||
|
||||
def test_spawn_worker_concurrent_calls_produce_one_worker(self) -> None:
|
||||
"""Bug-1 reproducer: two simultaneous send() calls under ws._lock
|
||||
must land as exactly one ChatSession.send and one queued message,
|
||||
not two parallel workers on the same ChatSession."""
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
send_gate = threading.Event()
|
||||
session = _SendSession(send_gate=send_gate)
|
||||
ws.session = session # type: ignore[assignment]
|
||||
adapter.attach(_StubManager(ws)) # type: ignore[arg-type]
|
||||
|
||||
results: list[bool] = []
|
||||
start_barrier = threading.Barrier(2)
|
||||
results_lock = threading.Lock()
|
||||
|
||||
def _caller(msg: str) -> None:
|
||||
start_barrier.wait(timeout=1.0)
|
||||
r = adapter.send(ws.id, msg)
|
||||
with results_lock:
|
||||
results.append(r)
|
||||
|
||||
t1 = threading.Thread(target=_caller, args=("first",))
|
||||
t2 = threading.Thread(target=_caller, args=("second",))
|
||||
t1.start()
|
||||
t2.start()
|
||||
# Both callers return quickly: the winner spawns the worker
|
||||
# (returns True immediately) and the loser queues (returns True).
|
||||
t1.join(timeout=3.0)
|
||||
t2.join(timeout=3.0)
|
||||
assert not t1.is_alive() and not t2.is_alive()
|
||||
# At this point session.send is still blocked on send_gate —
|
||||
# the second caller MUST have taken the queue path.
|
||||
assert len(session.queue_calls) == 1
|
||||
# Release the worker and let it finish.
|
||||
send_gate.set()
|
||||
if ws.worker_thread is not None:
|
||||
ws.worker_thread.join(timeout=3.0)
|
||||
|
||||
assert results == [True, True]
|
||||
assert len(session.send_calls) == 1
|
||||
assert set(session.send_calls + session.queue_calls) == {"first", "second"}
|
||||
assert ws._worker_running is False
|
||||
|
||||
def test_worker_finally_clears_running_flag(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
session = _SendSession()
|
||||
ws.session = session # type: ignore[assignment]
|
||||
adapter.attach(_StubManager(ws)) # type: ignore[arg-type]
|
||||
|
||||
assert adapter.send(ws.id, "hello") is True
|
||||
assert ws.worker_thread is not None
|
||||
ws.worker_thread.join(timeout=2.0)
|
||||
assert ws._worker_running is False
|
||||
assert session.send_calls == ["hello"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Children registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCoordinatorAdapterChildrenRegistry:
|
||||
def test_emit_created_seeds_empty_children_set(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.emit_created(ws)
|
||||
assert ws.id in adapter._children
|
||||
assert adapter._children[ws.id] == set()
|
||||
assert adapter._active_coords[ws.id] is ws.ui
|
||||
|
||||
def test_emit_rehydrated_calls_rebuild(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
calls: list[str] = []
|
||||
# Monkeypatch the rebuild hook to count invocations without
|
||||
# requiring a real storage backend.
|
||||
adapter._rebuild_children_registry = calls.append # type: ignore[method-assign, assignment]
|
||||
ws = _make_ws()
|
||||
adapter.emit_created(ws)
|
||||
assert calls == []
|
||||
adapter.emit_rehydrated(ws)
|
||||
assert calls == [ws.id]
|
||||
|
||||
def test_emit_closed_clears_forward_and_reverse_indexes(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._merge_child_ids_locked("coord-b", ["child-b1"])
|
||||
adapter._active_coords["coord-a"] = object()
|
||||
adapter._active_coords["coord-b"] = object()
|
||||
|
||||
adapter.emit_closed("coord-a")
|
||||
|
||||
assert "coord-a" not in adapter._children
|
||||
assert "coord-a" not in adapter._active_coords
|
||||
assert "child-a1" not in adapter._child_to_coord
|
||||
assert "child-a2" not in adapter._child_to_coord
|
||||
# coord-b untouched
|
||||
assert adapter._child_to_coord["child-b1"] == "coord-b"
|
||||
assert "coord-b" in adapter._children
|
||||
|
||||
def test_merge_child_ids_locked_is_idempotent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
assert adapter._children["coord-a"] == {"child-1"}
|
||||
assert adapter._child_to_coord == {"child-1": "coord-a"}
|
||||
|
||||
def test_prime_children_from_snapshot_merges_without_overwriting(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
# Seed one in-memory coord + one existing child
|
||||
coord_ws = _make_ws()
|
||||
coord_ws.id = "coord-a"
|
||||
mgr = MagicMock()
|
||||
mgr.list_all.return_value = [coord_ws]
|
||||
adapter.attach(mgr)
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
{
|
||||
"workstreams": [
|
||||
{"id": "child-a2", "parent_ws_id": "coord-a"},
|
||||
# Unknown parent — skipped
|
||||
{"id": "child-x", "parent_ws_id": "coord-unknown"},
|
||||
# Missing fields — skipped
|
||||
{"id": "", "parent_ws_id": "coord-a"},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
adapter._prime_children_from_snapshot(snapshot)
|
||||
assert adapter._children["coord-a"] == {"child-a1", "child-a2"}
|
||||
assert adapter._child_to_coord["child-a2"] == "coord-a"
|
||||
assert "child-x" not in adapter._child_to_coord
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch — _dispatch_child_event
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _UIRecorder:
|
||||
"""UI stub capturing _enqueue payloads for dispatch assertions."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.enqueued: list[dict[str, Any]] = []
|
||||
|
||||
def _enqueue(self, payload: dict[str, Any]) -> None:
|
||||
self.enqueued.append(payload)
|
||||
|
||||
|
||||
class TestCoordinatorAdapterDispatchChildEvent:
|
||||
def _setup(
|
||||
self, coord_id: str = "coord-a"
|
||||
) -> tuple[CoordinatorAdapter, _UIRecorder, Workstream]:
|
||||
adapter, _ = _make_adapter()
|
||||
coord_ws = _make_ws()
|
||||
coord_ws.id = coord_id
|
||||
recorder = _UIRecorder()
|
||||
coord_ws.ui = recorder # type: ignore[assignment]
|
||||
with adapter._children_lock:
|
||||
adapter._children.setdefault(coord_id, set())
|
||||
adapter._active_coords[coord_id] = recorder
|
||||
adapter.attach(_StubManager(coord_ws)) # type: ignore[arg-type]
|
||||
return adapter, recorder, coord_ws
|
||||
|
||||
def test_dispatch_unknown_parent_drops_event(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{"type": "ws_created", "ws_id": "orphan", "parent_ws_id": "coord-unknown"}
|
||||
)
|
||||
adapter._dispatch_child_event({"type": "cluster_state", "ws_id": "orphan"})
|
||||
adapter._dispatch_child_event({"type": "ws_closed", "ws_id": "orphan"})
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_ws_created_routes_to_parent_coord_ui(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "ws_created",
|
||||
"ws_id": "child-a1",
|
||||
"parent_ws_id": "coord-a",
|
||||
"name": "kid",
|
||||
"node_id": "node-1",
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_created"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
# Reverse index updated for subsequent cluster_state events.
|
||||
assert adapter._child_to_coord["child-a1"] == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_routes_via_reverse_index(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"tokens": 42,
|
||||
"node_id": "node-1",
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_state"
|
||||
assert payload["state"] == "running"
|
||||
assert payload["tokens"] == 42
|
||||
|
||||
def test_dispatch_ws_closed_routes_to_parent_coord(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{"type": "ws_closed", "ws_id": "child-a1", "reason": "evicted"}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_closed"
|
||||
assert payload["reason"] == "evicted"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
|
||||
def test_dispatch_adds_ws_id_in_place(self) -> None:
|
||||
"""perf-6: _enqueue_on_ui mutates the payload dict in place with
|
||||
the coord's ws_id so the browser can discriminate child events."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
}
|
||||
)
|
||||
assert recorder.enqueued[0]["ws_id"] == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_forwards_pending_approval_detail(self) -> None:
|
||||
"""The rich approval payload now rides on child_ws_state directly so
|
||||
the browser can mutate liveBadgeCache without a separate live-bulk
|
||||
fetch. Drift here means the inline approve/deny buttons would
|
||||
regress to chasing the dashboard cache (the load-storm pattern
|
||||
Shape A is unwinding)."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_state"
|
||||
assert payload["activity_state"] == "approval"
|
||||
assert payload["pending_approval_detail"] == detail
|
||||
|
||||
def test_dispatch_cluster_state_pending_approval_detail_none_passes_through(
|
||||
self,
|
||||
) -> None:
|
||||
"""Missing pending_approval_detail (no approval pending, or pre-fix
|
||||
node mid-rolling-upgrade) must forward as None — not raise, not
|
||||
omit — so the browser's handleChildState treats it as "no SSE-
|
||||
supplied detail, fall back to cached value"."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "tool",
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert "pending_approval_detail" in payload
|
||||
assert payload["pending_approval_detail"] is None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,236 +0,0 @@
|
||||
"""Tests for the coordinator ``close_all_children`` endpoint.
|
||||
|
||||
Near-twin of the ``stop_cascade`` tests in
|
||||
``test_coordinator_governance.py``. Keeps the close-cascade surface in
|
||||
its own file so PR A's review surface stays tight.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from tests._coord_test_helpers import (
|
||||
_AuthMiddleware,
|
||||
_build_mgr,
|
||||
_fake_registry,
|
||||
_FakeConfigStore,
|
||||
_seed_children,
|
||||
)
|
||||
from turnstone.console.server import coordinator_close_all_children
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "coord.db"))
|
||||
|
||||
|
||||
_COORD_HEADERS = {"X-Test-User": "user-1", "X-Test-Perms": "admin.coordinator"}
|
||||
|
||||
|
||||
def _make_client(storage, *, coord_mgr, alias="my-model", registry=None) -> TestClient:
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/close_all_children",
|
||||
coordinator_close_all_children,
|
||||
methods=["POST"],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_AuthMiddleware)],
|
||||
)
|
||||
app.state.coord_mgr = coord_mgr
|
||||
app.state.coord_adapter = coord_mgr._adapter if coord_mgr is not None else None
|
||||
app.state.config_store = _FakeConfigStore({"coordinator.model_alias": alias})
|
||||
app.state.coord_registry = registry
|
||||
app.state.coord_registry_error = "" if coord_mgr else "registry missing"
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "x" * 64
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_close_all_children_closes_each_child_and_audits(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["child-1", "child-2", "child-3"])
|
||||
|
||||
def _close(wid, reason):
|
||||
if wid == "child-2":
|
||||
return {"error": "gateway_timeout", "status": 502}
|
||||
return {"status": "ok"}
|
||||
|
||||
coord_client = MagicMock()
|
||||
coord_client.close_workstream.side_effect = _close
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = coord_client
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={"reason": "tests done"},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert set(body["closed"] + body["failed"] + body["skipped"]) == {
|
||||
"child-1",
|
||||
"child-2",
|
||||
"child-3",
|
||||
}
|
||||
assert body["failed"] == ["child-2"]
|
||||
assert set(body["closed"]) == {"child-1", "child-3"}
|
||||
assert body["skipped"] == []
|
||||
assert coord_client.close_workstream.call_count == 3
|
||||
# Reason must propagate to each per-child close call.
|
||||
for call in coord_client.close_workstream.call_args_list:
|
||||
assert call.args[1] == "tests done"
|
||||
|
||||
events = [
|
||||
e for e in storage.list_audit_events() if e["action"] == "coordinator.closed_all_children"
|
||||
]
|
||||
assert len(events) == 1
|
||||
detail = json.loads(events[0]["detail"])
|
||||
assert detail["reason"] == "tests done"
|
||||
assert set(detail["closed"] + detail["failed"] + detail["skipped"]) == {
|
||||
"child-1",
|
||||
"child-2",
|
||||
"child-3",
|
||||
}
|
||||
|
||||
|
||||
def test_close_all_children_routes_404_to_skipped_bucket(storage):
|
||||
"""An upstream 404 (child row already deleted, stale registry entry)
|
||||
is 'already gone', not a dispatch failure. Route to skipped."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["stale-child"])
|
||||
|
||||
coord_client = MagicMock()
|
||||
coord_client.close_workstream.return_value = {
|
||||
"error": "workstream not in coordinator subtree: stale-child",
|
||||
"status": 404,
|
||||
}
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = coord_client
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["closed"] == []
|
||||
assert body["failed"] == []
|
||||
assert body["skipped"] == ["stale-child"]
|
||||
|
||||
|
||||
def test_close_all_children_empty_children_still_audits(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = MagicMock()
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body == {"status": "ok", "closed": [], "failed": [], "skipped": []}
|
||||
assert [
|
||||
e for e in storage.list_audit_events() if e["action"] == "coordinator.closed_all_children"
|
||||
]
|
||||
|
||||
|
||||
def test_close_all_children_without_coord_client_marks_all_failed(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["child-a", "child-b"])
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = None
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["closed"] == []
|
||||
assert body["skipped"] == []
|
||||
assert set(body["failed"]) == {"child-a", "child-b"}
|
||||
|
||||
|
||||
def test_close_all_children_rejects_non_string_reason(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = MagicMock()
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={"reason": 123},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_close_all_children_rejects_overlong_reason(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = MagicMock()
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={"reason": "x" * 600},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_close_all_children_404_when_session_not_loaded(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = None
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_close_all_children_service_token_cannot_bypass_admin_coordinator(storage):
|
||||
"""Destructive endpoint — a service token matching the coord owner
|
||||
still needs the explicit ``admin.coordinator`` grant. Mirrors the
|
||||
stop_cascade treatment."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = MagicMock()
|
||||
|
||||
# Service token without admin.coordinator should be rejected.
|
||||
headers = {"X-Test-User": "user-1", "X-Test-Perms": ""}
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/close_all_children",
|
||||
json={},
|
||||
headers=headers,
|
||||
)
|
||||
assert resp.status_code in (401, 403)
|
||||
@@ -1,484 +0,0 @@
|
||||
"""End-to-end integration tests for the coordinator workstream feature.
|
||||
|
||||
Tests cover the full create → inspect → list → close lifecycle using
|
||||
real in-process components:
|
||||
|
||||
1. Create + list + detail round-trip via the Starlette TestClient.
|
||||
2. CoordinatorClient against a MockTransport "server node" stub.
|
||||
3. list_children storage read flow (kind filtering, parent scoping).
|
||||
4. Lazy rehydration via GET /v1/api/workstreams/{ws_id}.
|
||||
|
||||
Intentionally no real LLM infrastructure — session factories return
|
||||
MagicMock-backed stubs. All four tests run in < 2 s total.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.coordinator_adapter import CoordinatorAdapter
|
||||
from turnstone.console.coordinator_client import CoordinatorClient
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
from turnstone.console.server import (
|
||||
_audit_close_coordinator,
|
||||
_audit_coordinator_create,
|
||||
_coord_create_build_kwargs,
|
||||
_coord_create_post_install,
|
||||
_coord_create_validate_request,
|
||||
_require_admin_coordinator,
|
||||
_require_coord_mgr,
|
||||
)
|
||||
from turnstone.core.auth import AuthResult
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
from turnstone.core.session_routes import (
|
||||
SessionEndpointConfig,
|
||||
make_close_handler,
|
||||
make_create_handler,
|
||||
make_detail_handler,
|
||||
make_list_handler,
|
||||
)
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# Per-kind config the lifted handler factories capture by closure.
|
||||
_coord_endpoint_config = SessionEndpointConfig(
|
||||
permission_gate=_require_admin_coordinator,
|
||||
manager_lookup=_require_coord_mgr,
|
||||
tenant_check=None,
|
||||
not_found_label="coordinator not found",
|
||||
audit_action_prefix="coordinator",
|
||||
create_supports_attachments=True,
|
||||
create_supports_user_id_override=False,
|
||||
create_validate_request=_coord_create_validate_request,
|
||||
create_build_kwargs=_coord_create_build_kwargs,
|
||||
create_post_install=_coord_create_post_install,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared auth-injection middleware (mirrors test_coordinator_endpoints.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _AuthMiddleware(BaseHTTPMiddleware):
|
||||
"""Inject an ``AuthResult`` from ``X-Test-Perms`` / ``X-Test-User``."""
|
||||
|
||||
async def dispatch(self, request, call_next):
|
||||
perms = request.headers.get("X-Test-Perms", "")
|
||||
user_id = request.headers.get("X-Test-User", "")
|
||||
if perms or user_id:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id=user_id,
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="test",
|
||||
permissions=frozenset(p for p in perms.split(",") if p),
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared stubs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeConfigStore:
|
||||
"""Minimal ConfigStore stub returning values from a dict."""
|
||||
|
||||
def __init__(self, values: dict[str, Any]) -> None:
|
||||
self._values = values
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
return self._values.get(key, default)
|
||||
|
||||
|
||||
def _fake_registry() -> MagicMock:
|
||||
"""Registry stub that always succeeds on .resolve() so the 503 gate passes."""
|
||||
reg = MagicMock()
|
||||
reg.resolve.return_value = (MagicMock(), "gpt-test", MagicMock())
|
||||
return reg
|
||||
|
||||
|
||||
def _build_mgr(storage: SQLiteBackend) -> SessionManager:
|
||||
"""Build a SessionManager(CoordinatorAdapter) backed by stub factories."""
|
||||
|
||||
def _sf(ui, model_alias=None, ws_id=None, **kw):
|
||||
s = MagicMock()
|
||||
s.ws_id = ws_id
|
||||
s.send.return_value = None
|
||||
return s
|
||||
|
||||
adapter = CoordinatorAdapter(
|
||||
collector=MagicMock(),
|
||||
ui_factory=lambda ws: ConsoleCoordinatorUI(ws_id=ws.id, user_id=ws.user_id or ""),
|
||||
session_factory=_sf,
|
||||
)
|
||||
mgr = SessionManager(
|
||||
adapter,
|
||||
storage=storage,
|
||||
max_active=5,
|
||||
node_id=ClusterCollector.CONSOLE_PSEUDO_NODE_ID,
|
||||
event_emitter=adapter,
|
||||
)
|
||||
adapter.attach(mgr)
|
||||
return mgr
|
||||
|
||||
|
||||
def _make_client(
|
||||
storage: SQLiteBackend,
|
||||
*,
|
||||
coord_mgr: SessionManager | None = None,
|
||||
alias: str = "my-model",
|
||||
registry: Any = None,
|
||||
) -> TestClient:
|
||||
"""Build a Starlette TestClient exposing the coordinator routes."""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/v1/api/workstreams/new",
|
||||
make_create_handler(_coord_endpoint_config, audit_emit=_audit_coordinator_create),
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams",
|
||||
make_list_handler(_coord_endpoint_config),
|
||||
methods=["GET"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/close",
|
||||
make_close_handler(
|
||||
_coord_endpoint_config,
|
||||
audit_emit=_audit_close_coordinator,
|
||||
supports_close_reason=False,
|
||||
),
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}",
|
||||
make_detail_handler(_coord_endpoint_config),
|
||||
methods=["GET"],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_AuthMiddleware)],
|
||||
)
|
||||
app.state.coord_mgr = coord_mgr
|
||||
app.state.coord_adapter = coord_mgr._adapter if coord_mgr is not None else None
|
||||
app.state.config_store = _FakeConfigStore({"coordinator.model_alias": alias})
|
||||
app.state.coord_registry = registry
|
||||
app.state.coord_registry_error = "" if coord_mgr else "registry missing"
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "x" * 64
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1 — Create + list + detail round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COORD_HEADERS = {"X-Test-User": "user-1", "X-Test-Perms": "admin.coordinator"}
|
||||
|
||||
|
||||
def test_create_list_detail_lifecycle(tmp_path):
|
||||
"""POST /new → appears in GET / → GET /{ws_id} returns correct detail."""
|
||||
storage = SQLiteBackend(str(tmp_path / "coord.db"))
|
||||
mgr = _build_mgr(storage)
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
|
||||
# --- Create ---
|
||||
resp = client.post(
|
||||
"/v1/api/workstreams/new",
|
||||
json={"name": "e2e-coord"},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
ws_id = body["ws_id"]
|
||||
assert ws_id
|
||||
assert "e2e-coord" in body["name"]
|
||||
|
||||
# --- List: caller sees their own coordinator ---
|
||||
resp = client.get("/v1/api/workstreams", headers=_COORD_HEADERS)
|
||||
assert resp.status_code == 200, resp.text
|
||||
coordinators = resp.json()["workstreams"]
|
||||
ids = {c["ws_id"] for c in coordinators}
|
||||
assert ws_id in ids
|
||||
|
||||
# Trusted-team visibility: every ``admin.coordinator`` caller sees
|
||||
# every active coordinator regardless of owner.
|
||||
mgr.create(user_id="other-user", name="not-mine")
|
||||
resp = client.get("/v1/api/workstreams", headers=_COORD_HEADERS)
|
||||
assert resp.status_code == 200
|
||||
names = {c["name"] for c in resp.json()["workstreams"]}
|
||||
assert "not-mine" in names
|
||||
|
||||
# --- Detail ---
|
||||
resp = client.get(f"/v1/api/workstreams/{ws_id}", headers=_COORD_HEADERS)
|
||||
assert resp.status_code == 200, resp.text
|
||||
detail = resp.json()
|
||||
assert detail["ws_id"] == ws_id
|
||||
assert detail["kind"] == "coordinator"
|
||||
assert detail["user_id"] == "user-1"
|
||||
|
||||
# --- Close ---
|
||||
resp = client.post(f"/v1/api/workstreams/{ws_id}/close", headers=_COORD_HEADERS)
|
||||
assert resp.status_code == 200
|
||||
|
||||
# Manager no longer tracks it after close.
|
||||
assert mgr.get(ws_id) is None
|
||||
|
||||
# Storage row reflects closed state.
|
||||
row = storage.get_workstream(ws_id)
|
||||
assert row is not None
|
||||
assert row["state"] == "closed"
|
||||
|
||||
# Detail endpoint returns 404 after close (not in memory, not rehydratable
|
||||
# from a "closed" row — well, the manager would rehydrate it but let's verify
|
||||
# the row is gone from the in-memory index).
|
||||
assert mgr.get(ws_id) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 2 — CoordinatorClient against a MockTransport "server node" stub
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_coordinator_client_spawn_close_delete(tmp_path):
|
||||
"""CoordinatorClient.spawn / close_workstream / delete produce correct
|
||||
upstream HTTP requests to the mocked server node."""
|
||||
storage = SQLiteBackend(str(tmp_path / "client.db"))
|
||||
# Register the coordinator + the soon-to-be-spawned child so the
|
||||
# client-side tenant guard on close/delete passes. In production
|
||||
# the spawn route adds the child row before the model can call
|
||||
# close on it; the test stub doesn't run that side-effect, so we
|
||||
# set it up here.
|
||||
storage.register_workstream("coord-42", kind="coordinator", user_id="user-1")
|
||||
storage.register_workstream(
|
||||
"child-99", kind="interactive", parent_ws_id="coord-42", user_id="user-1"
|
||||
)
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
def _handler(req: httpx.Request) -> httpx.Response:
|
||||
captured.append(req)
|
||||
path = req.url.path
|
||||
if path == "/v1/api/route/workstreams/new":
|
||||
return httpx.Response(
|
||||
201,
|
||||
json={"ws_id": "child-99", "name": "spawned", "node_id": "node-a"},
|
||||
)
|
||||
# close and delete both return a generic ok
|
||||
return httpx.Response(200, json={"status": "ok"})
|
||||
|
||||
transport = httpx.MockTransport(_handler)
|
||||
http = httpx.Client(transport=transport)
|
||||
coord_client = CoordinatorClient(
|
||||
console_base_url="http://console",
|
||||
storage=storage,
|
||||
token_factory=lambda: "bearer-test-token",
|
||||
coord_ws_id="coord-42",
|
||||
user_id="user-1",
|
||||
http_client=http,
|
||||
)
|
||||
|
||||
# spawn ---------------------------------------------------------------
|
||||
result = coord_client.spawn(
|
||||
initial_message="analyse data",
|
||||
parent_ws_id="coord-42",
|
||||
user_id="user-1",
|
||||
skill="data-skill",
|
||||
target_node="node-a",
|
||||
)
|
||||
assert result["ws_id"] == "child-99"
|
||||
|
||||
spawn_req = captured[0]
|
||||
assert spawn_req.method == "POST"
|
||||
assert spawn_req.url.path == "/v1/api/route/workstreams/new"
|
||||
assert spawn_req.headers["Authorization"] == "Bearer bearer-test-token"
|
||||
|
||||
spawn_body = json.loads(spawn_req.content)
|
||||
assert spawn_body["kind"] == "interactive"
|
||||
assert spawn_body["parent_ws_id"] == "coord-42"
|
||||
assert spawn_body["user_id"] == "user-1"
|
||||
assert spawn_body["initial_message"] == "analyse data"
|
||||
assert spawn_body["skill"] == "data-skill"
|
||||
assert spawn_body["target_node"] == "node-a"
|
||||
|
||||
# close_workstream ----------------------------------------------------
|
||||
captured.clear()
|
||||
close_result = coord_client.close_workstream("child-99")
|
||||
assert close_result.get("status") in (200, "ok"), close_result
|
||||
|
||||
close_req = captured[0]
|
||||
# Path-keyed shape post-#422: ws_id rides in the URL.
|
||||
assert close_req.url.path == "/v1/api/route/workstreams/child-99/close"
|
||||
close_body = json.loads(close_req.content)
|
||||
# Body no longer carries ws_id — the path is authoritative.
|
||||
assert "ws_id" not in close_body
|
||||
|
||||
# delete --------------------------------------------------------------
|
||||
captured.clear()
|
||||
del_result = coord_client.delete("child-99")
|
||||
assert del_result.get("status") in (200, "ok"), del_result
|
||||
|
||||
del_req = captured[0]
|
||||
assert del_req.url.path == "/v1/api/route/workstreams/delete"
|
||||
del_body = json.loads(del_req.content)
|
||||
assert del_body["ws_id"] == "child-99"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 3 — list_children storage read: kind filtering + parent scoping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def seeded_storage(tmp_path):
|
||||
"""SQLiteBackend with a coordinator + 2 interactive children + extras."""
|
||||
st = SQLiteBackend(str(tmp_path / "seed.db"))
|
||||
# Parent coordinator.
|
||||
st.register_workstream("coord-root", kind="coordinator", user_id="user-1")
|
||||
# Two interactive children — one idle, one running. Children inherit
|
||||
# the coord's user_id by construction (server-side create gate), which
|
||||
# the list_children SQL filter now enforces.
|
||||
st.register_workstream(
|
||||
"child-idle",
|
||||
kind="interactive",
|
||||
parent_ws_id="coord-root",
|
||||
state="idle",
|
||||
skill_id="skill-alpha",
|
||||
user_id="user-1",
|
||||
)
|
||||
st.register_workstream(
|
||||
"child-running",
|
||||
kind="interactive",
|
||||
parent_ws_id="coord-root",
|
||||
state="running",
|
||||
skill_id="skill-beta",
|
||||
user_id="user-1",
|
||||
)
|
||||
# Coordinator child — MUST be excluded from list_children results.
|
||||
st.register_workstream(
|
||||
"child-coord",
|
||||
kind="coordinator",
|
||||
parent_ws_id="coord-root",
|
||||
user_id="user-1",
|
||||
)
|
||||
# Unrelated workstream with no parent — MUST be excluded.
|
||||
st.register_workstream("unrelated-ws", kind="interactive", user_id="user-1")
|
||||
return st
|
||||
|
||||
|
||||
def _read_client(storage: SQLiteBackend) -> CoordinatorClient:
|
||||
"""Build a CoordinatorClient whose HTTP transport is a no-op stub."""
|
||||
transport = httpx.MockTransport(lambda r: httpx.Response(200))
|
||||
http = httpx.Client(transport=transport)
|
||||
return CoordinatorClient(
|
||||
console_base_url="http://x",
|
||||
storage=storage,
|
||||
token_factory=lambda: "t",
|
||||
coord_ws_id="coord-root",
|
||||
user_id="user-1",
|
||||
http_client=http,
|
||||
)
|
||||
|
||||
|
||||
def test_list_children_excludes_coordinator_and_unrelated_rows(seeded_storage):
|
||||
"""list_children returns only interactive children of the given parent."""
|
||||
client = _read_client(seeded_storage)
|
||||
result = client.list_children("coord-root")
|
||||
rows = result["children"]
|
||||
|
||||
ws_ids = {r["ws_id"] for r in rows}
|
||||
# The two interactive children are present.
|
||||
assert ws_ids == {"child-idle", "child-running"}
|
||||
# Every returned row must be interactive and linked to coord-root.
|
||||
for r in rows:
|
||||
assert r["kind"] == "interactive"
|
||||
assert r["parent_ws_id"] == "coord-root"
|
||||
|
||||
# Coordinator child and unrelated ws are absent.
|
||||
assert "child-coord" not in ws_ids
|
||||
assert "unrelated-ws" not in ws_ids
|
||||
assert result["truncated"] is False
|
||||
|
||||
|
||||
def test_list_children_state_filter(seeded_storage):
|
||||
"""list_children(state='running') filters to only running children."""
|
||||
client = _read_client(seeded_storage)
|
||||
result = client.list_children("coord-root", state="running")
|
||||
assert {r["ws_id"] for r in result["children"]} == {"child-running"}
|
||||
|
||||
|
||||
def test_list_children_skill_filter(seeded_storage):
|
||||
"""list_children(skill='skill-alpha') returns the matching child only."""
|
||||
client = _read_client(seeded_storage)
|
||||
result = client.list_children("coord-root", skill="skill-alpha")
|
||||
rows = result["children"]
|
||||
assert {r["ws_id"] for r in rows} == {"child-idle"}
|
||||
assert rows[0].get("skill_id") == "skill-alpha"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 4 — Lazy rehydration via GET /v1/api/workstreams/{ws_id}
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_lazy_rehydration_on_detail_get(tmp_path):
|
||||
"""A persisted coordinator row rehydrates into the manager on GET /{ws_id}.
|
||||
|
||||
Sequence:
|
||||
1. Pre-seed storage with a coordinator row (simulating a previous process).
|
||||
2. Build a SessionManager (coordinator kind) that doesn't know about it yet.
|
||||
3. Hit GET /v1/api/workstreams/{ws_id} — expect 200.
|
||||
4. Manager now tracks the rehydrated session.
|
||||
5. The response body carries the correct kind / user_id metadata.
|
||||
"""
|
||||
storage = SQLiteBackend(str(tmp_path / "rehydrate.db"))
|
||||
|
||||
# Seed the row directly — the manager has never seen it.
|
||||
storage.register_workstream(
|
||||
"persisted-coord",
|
||||
node_id="console",
|
||||
user_id="user-1",
|
||||
name="old-coord",
|
||||
kind="coordinator",
|
||||
)
|
||||
|
||||
mgr = _build_mgr(storage)
|
||||
# Confirm: not tracked in memory yet.
|
||||
assert mgr.get("persisted-coord") is None
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.get("/v1/api/workstreams/persisted-coord", headers=_COORD_HEADERS)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
body = resp.json()
|
||||
assert body["ws_id"] == "persisted-coord"
|
||||
assert body["kind"] == "coordinator"
|
||||
assert body["user_id"] == "user-1"
|
||||
|
||||
# The endpoint triggers lazy rehydration — manager now tracks it.
|
||||
assert mgr.get("persisted-coord") is not None
|
||||
|
||||
# Trusted-team visibility: any admin.coordinator caller can read
|
||||
# the coordinator's detail, regardless of ``user_id``.
|
||||
resp_stranger = client.get(
|
||||
"/v1/api/workstreams/persisted-coord",
|
||||
headers={"X-Test-User": "stranger", "X-Test-Perms": "admin.coordinator"},
|
||||
)
|
||||
assert resp_stranger.status_code == 200
|
||||
assert resp_stranger.json()["user_id"] == "user-1"
|
||||
|
||||
# A workstream with kind='interactive' is not reachable via the coordinator
|
||||
# endpoint even when it exists in storage.
|
||||
storage.register_workstream("interactive-ws", kind="interactive", user_id="user-1")
|
||||
resp_int = client.get("/v1/api/workstreams/interactive-ws", headers=_COORD_HEADERS)
|
||||
assert resp_int.status_code == 404
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,816 +0,0 @@
|
||||
"""Tests for the coordinator governance endpoints and session hooks.
|
||||
|
||||
Covers the three console endpoints that let an operator steer a live
|
||||
coordinator session mid-flight (``/trust``, ``/restrict``,
|
||||
``/stop_cascade``), the two ``ChatSession`` methods the endpoints
|
||||
toggle (``set_trust_send`` / ``revoke_tools``), the audit rows the
|
||||
handlers emit, and the ``_prepare_tool`` revocation gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from tests._coord_test_helpers import (
|
||||
_AuthMiddleware,
|
||||
_build_mgr,
|
||||
_fake_registry,
|
||||
_FakeConfigStore,
|
||||
_seed_children,
|
||||
)
|
||||
from turnstone.console.server import (
|
||||
coordinator_restrict,
|
||||
coordinator_stop_cascade,
|
||||
coordinator_trust,
|
||||
)
|
||||
from turnstone.core.auth import AuthResult
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "coord.db"))
|
||||
|
||||
|
||||
def _make_client(storage, *, coord_mgr, alias="my-model", registry=None) -> TestClient:
|
||||
"""Starlette app exposing only the three governance endpoints."""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/trust",
|
||||
coordinator_trust,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/restrict",
|
||||
coordinator_restrict,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/stop_cascade",
|
||||
coordinator_stop_cascade,
|
||||
methods=["POST"],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_AuthMiddleware)],
|
||||
)
|
||||
app.state.coord_mgr = coord_mgr
|
||||
app.state.coord_adapter = coord_mgr._adapter if coord_mgr is not None else None
|
||||
app.state.config_store = _FakeConfigStore({"coordinator.model_alias": alias})
|
||||
app.state.coord_registry = registry
|
||||
app.state.coord_registry_error = "" if coord_mgr else "registry missing"
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "x" * 64
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _make_session_mock(*, trust_send: bool = False, revoked: frozenset[str] = frozenset()):
|
||||
"""Build a MagicMock ``session`` that honours the new ChatSession
|
||||
governance surface (``set_trust_send`` / ``get_trust_send`` /
|
||||
``revoke_tools`` / ``get_revoked_tools``) so handler tests exercise
|
||||
the real method calls rather than reaching into attributes."""
|
||||
|
||||
state: dict[str, Any] = {"trust_send": trust_send, "revoked": revoked}
|
||||
|
||||
def _set_trust_send(value: bool) -> None:
|
||||
state["trust_send"] = bool(value)
|
||||
|
||||
def _get_trust_send() -> bool:
|
||||
return bool(state["trust_send"])
|
||||
|
||||
def _revoke_tools(names):
|
||||
state["revoked"] = state["revoked"] | frozenset(names)
|
||||
return state["revoked"]
|
||||
|
||||
def _get_revoked_tools():
|
||||
return state["revoked"]
|
||||
|
||||
session = MagicMock()
|
||||
session.set_trust_send.side_effect = _set_trust_send
|
||||
session.get_trust_send.side_effect = _get_trust_send
|
||||
session.revoke_tools.side_effect = _revoke_tools
|
||||
session.get_revoked_tools.side_effect = _get_revoked_tools
|
||||
return session, state
|
||||
|
||||
|
||||
_COORD_HEADERS = {"X-Test-User": "user-1", "X-Test-Perms": "admin.coordinator"}
|
||||
_TRUST_HEADERS = {
|
||||
"X-Test-User": "user-1",
|
||||
"X-Test-Perms": "admin.coordinator,coordinator.trust.send",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /trust endpoint — trusted-session mode (item 1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_trust_toggle_requires_trust_send_permission(storage):
|
||||
"""Double-gated: admin.coordinator alone is insufficient — the
|
||||
trust-send perm is an explicit opt-in."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": True},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
def test_trust_toggle_flips_session_flag_and_audits(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
session, state = _make_session_mock()
|
||||
coord.session = session
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": True},
|
||||
headers=_TRUST_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"status": "ok", "trust_send": True}
|
||||
assert state["trust_send"] is True
|
||||
|
||||
events = [e for e in storage.list_audit_events() if e["action"] == "coordinator.trust.toggled"]
|
||||
assert len(events) == 1
|
||||
detail = json.loads(events[0]["detail"])
|
||||
assert detail["send_before"] is False
|
||||
assert detail["send_after"] is True
|
||||
|
||||
|
||||
def _service_token_client(
|
||||
storage,
|
||||
coord_mgr,
|
||||
*,
|
||||
user_id: str,
|
||||
permissions: frozenset[str],
|
||||
) -> TestClient:
|
||||
"""Build a TestClient whose middleware injects a service-scoped token.
|
||||
|
||||
Used to verify that the capability-escalating endpoints (``/trust``,
|
||||
``/restrict``, ``/stop_cascade``) do NOT honor the normal
|
||||
``require_permission`` service-scope bypass when the caller lacks
|
||||
the specific grant they need.
|
||||
"""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/trust",
|
||||
coordinator_trust,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/restrict",
|
||||
coordinator_restrict,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/stop_cascade",
|
||||
coordinator_stop_cascade,
|
||||
methods=["POST"],
|
||||
),
|
||||
],
|
||||
)
|
||||
app.state.coord_mgr = coord_mgr
|
||||
app.state.coord_adapter = coord_mgr._adapter if coord_mgr is not None else None
|
||||
app.state.config_store = _FakeConfigStore({"coordinator.model_alias": "my-model"})
|
||||
app.state.coord_registry = _fake_registry()
|
||||
app.state.coord_registry_error = ""
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "x" * 64
|
||||
|
||||
captured_perms = permissions
|
||||
|
||||
class _ServiceAuth(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request, call_next):
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id=user_id,
|
||||
scopes=frozenset({"read", "write", "approve", "service"}),
|
||||
token_source="test",
|
||||
permissions=captured_perms,
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
app.user_middleware = [Middleware(_ServiceAuth)]
|
||||
app.middleware_stack = app.build_middleware_stack()
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_trust_toggle_service_token_cannot_bypass_permission(storage):
|
||||
"""Service token without coordinator.trust.send is 403'd even when
|
||||
its user_id matches the coord owner."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="svc-user", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
|
||||
client = _service_token_client(
|
||||
storage,
|
||||
mgr,
|
||||
user_id="svc-user",
|
||||
permissions=frozenset({"admin.coordinator"}),
|
||||
)
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": True},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
assert "coordinator.trust.send" in resp.json()["error"]
|
||||
|
||||
|
||||
def test_trust_toggle_service_token_with_permission_succeeds(storage):
|
||||
"""Service token WITH the explicit coordinator.trust.send grant IS
|
||||
allowed through — locks the intended invariant: bypass is off, but
|
||||
an explicit perm still works."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="svc-user", name="coord-a")
|
||||
session, state = _make_session_mock()
|
||||
coord.session = session
|
||||
|
||||
client = _service_token_client(
|
||||
storage,
|
||||
mgr,
|
||||
user_id="svc-user",
|
||||
permissions=frozenset({"admin.coordinator", "coordinator.trust.send"}),
|
||||
)
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": True},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"status": "ok", "trust_send": True}
|
||||
assert state["trust_send"] is True
|
||||
|
||||
|
||||
def test_restrict_service_token_cannot_bypass_admin_coordinator(storage):
|
||||
"""/restrict is destructive — a service token WITHOUT explicit
|
||||
admin.coordinator grant must be 403'd rather than letting the
|
||||
service-scope bypass open the endpoint up."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="svc-user", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
|
||||
client = _service_token_client(
|
||||
storage,
|
||||
mgr,
|
||||
user_id="svc-user",
|
||||
permissions=frozenset(), # no admin.coordinator
|
||||
)
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": ["bash"]},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
def test_stop_cascade_service_token_cannot_bypass_admin_coordinator(storage):
|
||||
"""/stop_cascade mirrors /restrict — same destructive treatment."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="svc-user", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
|
||||
client = _service_token_client(
|
||||
storage,
|
||||
mgr,
|
||||
user_id="svc-user",
|
||||
permissions=frozenset(),
|
||||
)
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/stop_cascade",
|
||||
json={},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
def test_trust_toggle_rejects_non_bool(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": "yes"},
|
||||
headers=_TRUST_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_trust_toggle_rejects_non_object_body(storage):
|
||||
"""A valid-JSON-but-non-object body (null / list / scalar) must
|
||||
400 cleanly rather than AttributeError → 500."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
# Non-dict JSON values — all must 400. Different bodies may hit
|
||||
# `read_json_or_400`'s own parse error ("Invalid JSON body") or the
|
||||
# downstream dict-shape guard ("body must be a JSON object"); we
|
||||
# only care that none 500.
|
||||
for body in ([], 42, "string"):
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json=body,
|
||||
headers=_TRUST_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400, body
|
||||
assert "JSON object" in resp.json()["error"], resp.json()
|
||||
|
||||
|
||||
def test_restrict_rejects_non_object_body(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json=[],
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_trust_toggle_cluster_wide_access(storage):
|
||||
# Trusted-team model: the trust toggle is gated on the scope
|
||||
# permission, not on row-level ownership. A caller holding
|
||||
# ``coordinator.trust.send`` may toggle any coord's trust state.
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-owner", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": True},
|
||||
headers={
|
||||
"X-Test-User": "user-other",
|
||||
"X-Test-Perms": "admin.coordinator,coordinator.trust.send",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
def test_trust_toggle_404_when_session_not_loaded(storage):
|
||||
"""Persisted-but-not-loaded coordinator: runtime session state can't
|
||||
be mutated, so the endpoint 404s. Matches the tenant-miss shape
|
||||
so non-admins can't probe for closed rows via this endpoint."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = None # simulate a closed / lazy-rehydrate coord
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/trust",
|
||||
json={"send": True},
|
||||
headers=_TRUST_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _prepare_send_to_workstream — trust gate (item 1, unit-level)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prepare_send_to_workstream_trust_skips_approval_for_own_child():
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
session._coord_client = MagicMock()
|
||||
session._trust_send = True
|
||||
session._coord_client._is_own_subtree.return_value = True
|
||||
|
||||
item = session._prepare_send_to_workstream(call_id="c1", args={"ws_id": "abc", "message": "hi"})
|
||||
assert item["needs_approval"] is False
|
||||
assert item["trust_auto_approved"] is True
|
||||
|
||||
|
||||
def test_prepare_send_to_workstream_trust_holds_for_foreign_ws():
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
session._coord_client = MagicMock()
|
||||
session._trust_send = True
|
||||
session._coord_client._is_own_subtree.return_value = False
|
||||
|
||||
item = session._prepare_send_to_workstream(
|
||||
call_id="c2", args={"ws_id": "foreign-ws", "message": "hi"}
|
||||
)
|
||||
assert item["needs_approval"] is True
|
||||
assert item["trust_auto_approved"] is False
|
||||
|
||||
|
||||
def test_prepare_send_to_workstream_without_trust_always_requires_approval():
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
session._coord_client = MagicMock()
|
||||
session._trust_send = False
|
||||
session._coord_client._is_own_subtree.return_value = True
|
||||
|
||||
item = session._prepare_send_to_workstream(call_id="c3", args={"ws_id": "abc", "message": "hi"})
|
||||
assert item["needs_approval"] is True
|
||||
assert item["trust_auto_approved"] is False
|
||||
|
||||
|
||||
def test_exec_send_to_workstream_records_trust_audit(storage):
|
||||
"""The audit row fires before the HTTP send so a downstream failure
|
||||
can't suppress the trail."""
|
||||
from turnstone.console.coordinator_client import CoordinatorClient
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
client = CoordinatorClient.__new__(CoordinatorClient)
|
||||
client._storage = storage
|
||||
client._user_id = "user-1"
|
||||
client._coord_ws_id = "coord-1"
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
session._coord_client = client
|
||||
session.ui = MagicMock()
|
||||
send_mock = MagicMock(return_value={"status": "ok"})
|
||||
client.send = send_mock # type: ignore[method-assign]
|
||||
|
||||
session._exec_send_to_workstream(
|
||||
{
|
||||
"call_id": "c1",
|
||||
"ws_id": "child-ws-1",
|
||||
"message": "please summarise",
|
||||
"trust_auto_approved": True,
|
||||
}
|
||||
)
|
||||
|
||||
events = [
|
||||
e for e in storage.list_audit_events() if e["action"] == "coordinator.send.auto_approved"
|
||||
]
|
||||
assert len(events) == 1
|
||||
detail = json.loads(events[0]["detail"])
|
||||
assert detail["src"] == "coordinator"
|
||||
assert detail["trust"] is True
|
||||
assert detail["ws_id"] == "child-ws-1"
|
||||
assert "please summarise" in detail["message_preview"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /restrict endpoint + _prepare_tool revocation gate (item 5a)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_restrict_adds_to_revoked_tools_and_audits(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
session, state = _make_session_mock()
|
||||
coord.session = session
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": ["spawn_workstream", "delete_workstream"]},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert set(body["revoked_tools"]) == {"spawn_workstream", "delete_workstream"}
|
||||
assert state["revoked"] == frozenset({"spawn_workstream", "delete_workstream"})
|
||||
|
||||
events = [e for e in storage.list_audit_events() if e["action"] == "coordinator.restricted"]
|
||||
assert len(events) == 1
|
||||
detail = json.loads(events[0]["detail"])
|
||||
assert set(detail["revoked"]) == {"spawn_workstream", "delete_workstream"}
|
||||
|
||||
|
||||
def test_restrict_is_additive_across_calls(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
|
||||
client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": ["spawn_workstream"]},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": ["delete_workstream"]},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert set(resp.json()["revoked_tools"]) == {
|
||||
"spawn_workstream",
|
||||
"delete_workstream",
|
||||
}
|
||||
|
||||
|
||||
def test_restrict_empty_revoke_is_noop_but_audits(storage):
|
||||
"""Empty list is accepted as a no-op write — still emits the audit
|
||||
row so operators can see 'operator poked the restrict endpoint but
|
||||
didn't actually revoke anything' events."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _state = _make_session_mock(revoked=frozenset({"spawn_workstream"}))
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": []},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# Pre-existing revocations are preserved; no new entries were added.
|
||||
assert set(resp.json()["revoked_tools"]) == {"spawn_workstream"}
|
||||
events = [e for e in storage.list_audit_events() if e["action"] == "coordinator.restricted"]
|
||||
assert len(events) == 1
|
||||
detail = json.loads(events[0]["detail"])
|
||||
assert detail["revoked"] == []
|
||||
|
||||
|
||||
def test_restrict_rejects_non_list_body(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": "spawn_workstream"},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_restrict_rejects_oversize_list(storage):
|
||||
"""Defense-in-depth cap — an admin-sized list can't blow up the
|
||||
session frozenset or the audit row's detail column."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": [f"tool_{i}" for i in range(500)]},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_restrict_rejects_oversize_name(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session, _ = _make_session_mock()
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": ["x" * 1000]},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_restrict_404_when_session_not_loaded(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = None
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/restrict",
|
||||
json={"revoke": ["bash"]},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_prepare_tool_blocks_revoked_tool():
|
||||
"""Revocation short-circuits BEFORE the preparer dispatch so the
|
||||
model sees a clear 'revoked' error rather than a preparer-level
|
||||
validation message."""
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
session._revoked_tools = frozenset({"spawn_workstream"})
|
||||
session._mcp_client = None
|
||||
session.ui = MagicMock()
|
||||
|
||||
tc = {
|
||||
"id": "call-1",
|
||||
"function": {
|
||||
"name": "spawn_workstream",
|
||||
"arguments": '{"initial_message": "x"}',
|
||||
},
|
||||
}
|
||||
item = session._prepare_tool(tc)
|
||||
assert item["needs_approval"] is False
|
||||
assert "revoked" in item["header"].lower()
|
||||
assert "revoked" in item["error"].lower()
|
||||
|
||||
|
||||
def test_prepare_tool_allows_non_revoked_tool():
|
||||
"""The revocation gate must not fire on a tool name that isn't in
|
||||
the revoked set. We pick a name that's also not in the preparers
|
||||
dict so we can assert the 'unknown tool' result shape without
|
||||
exercising a real preparer."""
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
session._revoked_tools = frozenset({"spawn_workstream"})
|
||||
session._mcp_client = None
|
||||
session.ui = MagicMock()
|
||||
|
||||
tc = {
|
||||
"id": "call-2",
|
||||
"function": {"name": "this_tool_is_not_registered", "arguments": "{}"},
|
||||
}
|
||||
item = session._prepare_tool(tc)
|
||||
# Unknown tool path — not the revocation error path.
|
||||
err = str(item.get("error") or "")
|
||||
assert "revoked" not in err.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /stop_cascade endpoint (item 5b)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_cascade_cancels_coord_and_each_child(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["child-1", "child-2", "child-3"])
|
||||
|
||||
def _cancel(wid: str) -> dict:
|
||||
if wid == "child-2":
|
||||
return {"error": "gateway_timeout", "status": 502}
|
||||
return {"status": "ok"}
|
||||
|
||||
coord_client = MagicMock()
|
||||
coord_client.cancel.side_effect = _cancel
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = coord_client
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/stop_cascade",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert set(body["cancelled"] + body["failed"] + body["skipped"]) == {
|
||||
"child-1",
|
||||
"child-2",
|
||||
"child-3",
|
||||
}
|
||||
assert body["failed"] == ["child-2"]
|
||||
assert set(body["cancelled"]) == {"child-1", "child-3"}
|
||||
assert body["skipped"] == []
|
||||
assert coord_client.cancel.call_count == 3
|
||||
|
||||
events = [
|
||||
e for e in storage.list_audit_events() if e["action"] == "coordinator.stopped_cascade"
|
||||
]
|
||||
assert len(events) == 1
|
||||
detail = json.loads(events[0]["detail"])
|
||||
assert set(detail["cancelled"] + detail["failed"] + detail["skipped"]) == {
|
||||
"child-1",
|
||||
"child-2",
|
||||
"child-3",
|
||||
}
|
||||
|
||||
|
||||
def test_stop_cascade_routes_404_to_skipped_bucket(storage):
|
||||
"""A stale registry entry (child row already deleted from storage)
|
||||
or an upstream-404 on cancel is semantically 'already gone', not a
|
||||
dispatch failure. Report it in ``skipped`` so operators can tell
|
||||
them apart."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["stale-child"])
|
||||
|
||||
coord_client = MagicMock()
|
||||
coord_client.cancel.return_value = {
|
||||
"error": "workstream not in coordinator subtree: stale-child",
|
||||
"status": 404,
|
||||
}
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = coord_client
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/stop_cascade",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["cancelled"] == []
|
||||
assert body["failed"] == []
|
||||
assert body["skipped"] == ["stale-child"]
|
||||
|
||||
|
||||
def test_stop_cascade_empty_children_still_audits(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = MagicMock()
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/stop_cascade",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body == {"status": "ok", "cancelled": [], "failed": [], "skipped": []}
|
||||
assert [e for e in storage.list_audit_events() if e["action"] == "coordinator.stopped_cascade"]
|
||||
|
||||
|
||||
def test_stop_cascade_without_coord_client_marks_all_failed(storage):
|
||||
"""If the coord session has no attached coord_client (unexpected
|
||||
state for a loaded session), every child routes to ``failed`` so
|
||||
the operator can investigate."""
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["child-a", "child-b"])
|
||||
coord.session = MagicMock()
|
||||
coord.session._coord_client = None
|
||||
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/stop_cascade",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["cancelled"] == []
|
||||
assert body["skipped"] == []
|
||||
assert set(body["failed"]) == {"child-a", "child-b"}
|
||||
|
||||
|
||||
def test_stop_cascade_404_when_session_not_loaded(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
coord.session = None
|
||||
client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry())
|
||||
resp = client.post(
|
||||
f"/v1/api/workstreams/{coord.id}/stop_cascade",
|
||||
json={},
|
||||
headers=_COORD_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_children_snapshot_returns_copy_not_live_set(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
coord = mgr.create(user_id="user-1", name="coord-a")
|
||||
_seed_children(mgr._adapter, coord.id, ["a", "b", "c"])
|
||||
snap = mgr._adapter.children_snapshot(coord.id)
|
||||
assert set(snap) == {"a", "b", "c"}
|
||||
_seed_children(mgr._adapter, coord.id, ["d"])
|
||||
assert set(snap) == {"a", "b", "c"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ChatSession governance methods (q-14) — unit-level
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_set_and_get_trust_send_round_trip():
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
import threading as _t
|
||||
|
||||
session._trust_send = False
|
||||
session._governance_lock = _t.Lock()
|
||||
|
||||
assert session.get_trust_send() is False
|
||||
session.set_trust_send(True)
|
||||
assert session.get_trust_send() is True
|
||||
session.set_trust_send(False)
|
||||
assert session.get_trust_send() is False
|
||||
|
||||
|
||||
def test_revoke_tools_is_additive_and_returns_post_state():
|
||||
from turnstone.core.session import ChatSession
|
||||
|
||||
session = ChatSession.__new__(ChatSession)
|
||||
import threading as _t
|
||||
|
||||
session._revoked_tools = frozenset()
|
||||
session._governance_lock = _t.Lock()
|
||||
|
||||
after = session.revoke_tools(["bash", "read_file"])
|
||||
assert after == frozenset({"bash", "read_file"})
|
||||
after2 = session.revoke_tools(["write_file"])
|
||||
assert after2 == frozenset({"bash", "read_file", "write_file"})
|
||||
# Re-revoking is a no-op (idempotent).
|
||||
after3 = session.revoke_tools(["bash"])
|
||||
assert after3 == after2
|
||||
assert session.get_revoked_tools() == after3
|
||||
@@ -1,246 +0,0 @@
|
||||
"""Tests for the /coordinator/{ws_id} HTML page handler.
|
||||
|
||||
The handler serves the shared template with the ws_id injected as a
|
||||
``data-ws-id`` attribute. It does NOT enforce auth on the page itself —
|
||||
auth gating happens on the API endpoints the page calls (an unauthenticated
|
||||
visitor lands on the page but all API calls fail).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.console.server import coordinator_page
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
app = Starlette(routes=[Route("/coordinator/{ws_id}", coordinator_page, methods=["GET"])])
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_valid_ws_id_injects_data_attr(client):
|
||||
ws_id = "a" * 32
|
||||
resp = client.get(f"/coordinator/{ws_id}")
|
||||
assert resp.status_code == 200
|
||||
assert "text/html" in resp.headers["content-type"]
|
||||
body = resp.text
|
||||
# ws_id is injected into the html data-ws-id attribute.
|
||||
assert f'data-ws-id="{ws_id}"' in body
|
||||
# Template placeholder is fully substituted.
|
||||
assert "{{WS_ID}}" not in body
|
||||
# Sanity: the shared static imports are wired.
|
||||
assert "/shared/base.css" in body
|
||||
assert "/static/coordinator/coordinator.js" in body
|
||||
|
||||
|
||||
def test_non_hex_ws_id_returns_400(client):
|
||||
"""Only hex chars are allowed to avoid HTML injection."""
|
||||
resp = client.get("/coordinator/not-hex-chars-here")
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_ws_id_too_long_returns_400(client):
|
||||
resp = client.get("/coordinator/" + "a" * 65)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_uppercase_hex_rejected(client):
|
||||
# Our ws_ids are lowercase hex; reject mixed/upper to avoid surprises.
|
||||
resp = client.get("/coordinator/" + "A" * 32)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
"""Smoke guard for two layers of the coord chat frontend: the
|
||||
children-tree inline approve/deny block (the original Chunk 3
|
||||
landing) and the PR #447 tool-batch construct that replaced the
|
||||
pinned approval dock for the coord-self surface. Both layers'
|
||||
helper symbols must remain reachable in the served JS so a
|
||||
refactor that accidentally renames or removes them surfaces here
|
||||
instead of in production where the affected gates silently stop
|
||||
rendering. Asserts string presence only — no DOM parsing —
|
||||
since coord.js has no JS test framework today (per the plan's
|
||||
testing notes)."""
|
||||
from pathlib import Path
|
||||
|
||||
coord_js = Path(__file__).resolve().parent.parent / (
|
||||
"turnstone/console/static/coordinator/coordinator.js"
|
||||
)
|
||||
body = coord_js.read_text(encoding="utf-8")
|
||||
# Approval-block rendering helpers
|
||||
assert "function renderApprovalBlock" in body
|
||||
assert "function _maxSeverityItem" in body
|
||||
assert "function _renderSubItem" in body
|
||||
# The submit + 409 race-handling path
|
||||
assert "function submitChildApproval" in body or "submitChildApproval(" in body
|
||||
# The shared approve POST helper (parameterized for child ws_ids)
|
||||
assert "function approveWorkstream" in body or "approveWorkstream(" in body
|
||||
# The urgent live-bulk fetch option that fires on activity_state
|
||||
# transitions in/out of "approval"
|
||||
assert "{ urgent: true }" in body or "urgent: true" in body
|
||||
# Server-side payload field — drift here means the JS reads stale keys
|
||||
assert "pending_approval_detail" in body
|
||||
# Reconnect parity (chunk 4): the SSE re-open handler must drop
|
||||
# non-permanent entries from the live-badge cache so a stale
|
||||
# pending_approval_detail (left from before the disconnect)
|
||||
# can't render zombie approve/deny buttons on a row whose
|
||||
# approval was resolved during the gap. The implementation
|
||||
# iterates the cache and deletes only !permanent entries —
|
||||
# asserting the literal Map iteration form keeps a refactor
|
||||
# back to liveBadgeCache.clear() (which would re-pay 403s on
|
||||
# every reconnect for denied ids) from sneaking in.
|
||||
assert "liveBadgeCache.delete" in body
|
||||
# Edge-case matrix sentinel labels — POLICY-BLOCKED renders when
|
||||
# an item has error set + needs_approval=False (server-side
|
||||
# tool policy already blocked the call); "(judge unavailable)"
|
||||
# renders when no verdict (judge or heuristic) and no
|
||||
# judge_pending. Refactors that drop either branch silently
|
||||
# regress to a buttoned approve UI on the wrong state.
|
||||
assert "POLICY-BLOCKED" in body
|
||||
assert "judge unavailable" in body
|
||||
# Critical-risk handling — bug-1 was that risk_level='critical'
|
||||
# rendered as low because RISK_SEVERITY only mapped 'crit'.
|
||||
# Both aliases must remain in the table so a 'critical' verdict
|
||||
# ranks at 3 and renders with the .risk.crit pill.
|
||||
assert "critical: 3" in body
|
||||
# Child approves must round-trip through the routing proxy at
|
||||
# /v1/api/route/workstreams/{ws_id}/approve — the bare
|
||||
# /v1/api/workstreams/.../approve path only works for the
|
||||
# coord-self ws_id (the coord lives on the console process).
|
||||
# Children live on cluster nodes and 404 without the prefix.
|
||||
assert "/v1/api/route/workstreams/" in body
|
||||
# Late-judge polling — the LLM judge runs async on the child
|
||||
# node and never pushes a signal that reaches the coord, so
|
||||
# the row's pending_approval_detail with judge_pending=true
|
||||
# would freeze on heuristic verdicts forever without this
|
||||
# poll loop. The poller is GLOBAL (not per-row) so off-screen
|
||||
# rows still refresh — a per-row poller's scheduleLiveFetch
|
||||
# call short-circuits on non-visible rows, leaving them stuck.
|
||||
assert "_maybeStartJudgePoll" in body
|
||||
assert "_judgePollTick" in body
|
||||
# Reload parity for the coord-self approval gate: init() must
|
||||
# consume the authoritative GET /workstreams snapshot's
|
||||
# pending_approval_detail so a freshly opened tab can render
|
||||
# Approve/Deny before SSE replay arrives.
|
||||
assert "wsSnapshot.pending_approval_detail" in body
|
||||
assert "appendToolBatch(pendingDetail.items" in body
|
||||
# Tool-batch construct (PR #447) — the inline replacement for the
|
||||
# pinned approval-dock pattern. These helpers carry the
|
||||
# state-machine that pairs each tool call with its result and
|
||||
# embeds the approval flow. Refactors that rename or drop them
|
||||
# silently regress the entire coord-self approval surface — the
|
||||
# most novel and risky behavior in the PR.
|
||||
assert "function appendToolBatch" in body
|
||||
assert "function _morphBatchResolved" in body
|
||||
assert "function _resolveBatchAction" in body
|
||||
assert "function _refreshBatchTier" in body
|
||||
assert "function _refreshRowStatus" in body
|
||||
# State modifiers driven by the upgrade-in-place path
|
||||
# (--running orphan promoted to --pending or --auto when SSE
|
||||
# arrives with the authoritative shape). Both class names must
|
||||
# remain reachable from JS — dropping either breaks the reload
|
||||
# state machine that PR #447's review pass surfaced.
|
||||
assert "coord-tool-batch--running" in body
|
||||
assert "coord-tool-batch--pending" in body
|
||||
# History replay's outcome classifier — denied / errored tool
|
||||
# turns must render with the correct batch state on reload, not
|
||||
# the contradictory "✓ approved" pill that pre-fix showed for
|
||||
# any prior denial. bug-1 / bug-3 from the second /review pass.
|
||||
assert "Denied by user" in body
|
||||
assert "callOutcomes" in body
|
||||
|
||||
|
||||
def test_coordinator_js_handle_child_state_reads_sse_pending_approval_detail():
|
||||
"""Lock the Shape A behavior change: child_ws_state SSE events now
|
||||
carry ``pending_approval_detail`` directly so the browser mutates
|
||||
``liveBadgeCache`` without firing an urgent live-bulk fetch on
|
||||
every activity_state transition into/out of approval. A refactor
|
||||
that re-introduces the urgent-fetch path on routine transitions
|
||||
(or drops the SSE-source merge guard in flushLiveFetches) would
|
||||
re-open the load-storm pattern this PR is fixing.
|
||||
|
||||
Structural assertions (regex against multi-line source) — symbol-
|
||||
presence alone wouldn't catch a guard that keeps the names but
|
||||
inverts the comparison or drops the ``prev.live`` check. This
|
||||
codebase has no JS test framework, so locking the guard's shape
|
||||
here is the next-best thing to a behavioral test."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
coord_js = Path(__file__).resolve().parent.parent / (
|
||||
"turnstone/console/static/coordinator/coordinator.js"
|
||||
)
|
||||
body = coord_js.read_text(encoding="utf-8")
|
||||
|
||||
# handleChildState now reads the SSE-supplied detail.
|
||||
assert "ev.pending_approval_detail" in body
|
||||
# The pre-fix urgent-fetch on activity_state transitions is
|
||||
# gone (the 409 retry path keeps its own ``{ urgent: true }``
|
||||
# for stale-call_id refresh — that's a different scenario).
|
||||
assert "enteredApproval" not in body
|
||||
assert "leftApproval" not in body
|
||||
|
||||
# SSE-authoritative window constant is defined and used.
|
||||
assert re.search(r"\bconst\s+SSE_AUTHORITATIVE_MS\s*=\s*\d+", body), (
|
||||
"SSE_AUTHORITATIVE_MS constant must be defined as a numeric literal"
|
||||
)
|
||||
|
||||
# handleChildState writes sseUpdatedAt = Date.now() into the cache
|
||||
# entry it sets. This is the SSE-source tag; without it, the
|
||||
# merge guard in flushLiveFetches has nothing to gate on.
|
||||
assert re.search(
|
||||
r"sseUpdatedAt:\s*Date\.now\(\)",
|
||||
body,
|
||||
), "handleChildState must write sseUpdatedAt: Date.now() onto liveBadgeCache entries"
|
||||
|
||||
# flushLiveFetches' merge guard structure: SSE-set pending_approval
|
||||
# / _detail wins over a stale bulk-poll snapshot when (live) AND
|
||||
# (prev exists) AND (prev.sseUpdatedAt set) AND (within window)
|
||||
# AND (prev.live exists). Inverting the comparison or dropping
|
||||
# any of these guards reopens the clobber bug.
|
||||
merge_guard = re.search(
|
||||
r"if\s*\(\s*live\s*&&\s*prev\s*&&\s*prev\.sseUpdatedAt\s*&&\s*"
|
||||
r"now\s*-\s*prev\.sseUpdatedAt\s*<\s*SSE_AUTHORITATIVE_MS\s*&&\s*"
|
||||
r"prev\.live\s*\)",
|
||||
body,
|
||||
)
|
||||
assert merge_guard is not None, (
|
||||
"flushLiveFetches merge guard must be the conjunction "
|
||||
"(live && prev && prev.sseUpdatedAt && now - prev.sseUpdatedAt < "
|
||||
"SSE_AUTHORITATIVE_MS && prev.live). An inverted comparison or "
|
||||
"missing prev.live check would let a stale bulk-poll clobber a "
|
||||
"fresh SSE-set approval."
|
||||
)
|
||||
|
||||
# The merge body must preserve BOTH pending_approval and
|
||||
# pending_approval_detail from prev — preserving only one would
|
||||
# render a row with a phantom badge but no buttons (or vice versa).
|
||||
merge_body = re.search(
|
||||
r"mergedLive\s*=\s*Object\.assign\(\s*\{\}\s*,\s*live\s*,\s*\{"
|
||||
r"[^}]*pending_approval:\s*prev\.live\.pending_approval[^}]*"
|
||||
r"pending_approval_detail:\s*prev\.live\.pending_approval_detail",
|
||||
body,
|
||||
)
|
||||
assert merge_body is not None, (
|
||||
"Merge body must preserve both pending_approval AND "
|
||||
"pending_approval_detail from prev.live — preserving only one "
|
||||
"creates a half-rendered approval row."
|
||||
)
|
||||
|
||||
# flushLiveFetches must forward sseUpdatedAt onto the new cache
|
||||
# entry so the SSE-source tag survives the bulk-poll write back —
|
||||
# without this, every bulk-poll resets the window and the next
|
||||
# late-arriving poll silently clobbers.
|
||||
assert re.search(
|
||||
r"sseUpdatedAt:\s*prev\s*\?\s*prev\.sseUpdatedAt",
|
||||
body,
|
||||
), (
|
||||
"flushLiveFetches must forward prev.sseUpdatedAt onto the new "
|
||||
"cache entry (preserving the SSE-source window across bulk-poll "
|
||||
"cycles) — without this, the second bulk-poll after an SSE "
|
||||
"transition silently clobbers."
|
||||
)
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Tests for console _proxy_auth_headers preserving the coordinator src claim.
|
||||
|
||||
Verifies C8 of the coordinator plan: when a console handler processes an
|
||||
inbound request authenticated with a coordinator-minted JWT (``src ==
|
||||
"coordinator"``), the upstream JWT the console mints for the proxied
|
||||
request preserves that source plus the ``coord_ws_id`` custom claim.
|
||||
For non-coordinator inbound tokens the re-mint still uses
|
||||
``"console-proxy"`` as before — the existing behaviour is unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import jwt as pyjwt
|
||||
|
||||
from turnstone.console.server import _proxy_auth_headers
|
||||
from turnstone.core.auth import JWT_AUD_SERVER, AuthResult
|
||||
|
||||
_SECRET = "x" * 64
|
||||
|
||||
|
||||
def _build_request(auth_result: AuthResult | None):
|
||||
"""Minimal Request-alike for _proxy_auth_headers."""
|
||||
state = SimpleNamespace(auth_result=auth_result)
|
||||
app_state = SimpleNamespace(jwt_secret=_SECRET, proxy_token_mgr=None)
|
||||
app = MagicMock()
|
||||
app.state = app_state
|
||||
req = MagicMock()
|
||||
req.state = state
|
||||
req.app = app
|
||||
return req
|
||||
|
||||
|
||||
def _decode(headers: dict[str, str]) -> dict:
|
||||
token = headers["Authorization"].removeprefix("Bearer ")
|
||||
return pyjwt.decode(token, _SECRET, algorithms=["HS256"], audience=JWT_AUD_SERVER)
|
||||
|
||||
|
||||
def test_console_proxy_uses_console_proxy_source_by_default():
|
||||
"""Non-coordinator inbound tokens still mint src='console-proxy'."""
|
||||
auth = AuthResult(
|
||||
user_id="user-1",
|
||||
scopes=frozenset({"write"}),
|
||||
token_source="jwt",
|
||||
permissions=frozenset(),
|
||||
)
|
||||
headers = _proxy_auth_headers(_build_request(auth))
|
||||
payload = _decode(headers)
|
||||
assert payload["src"] == "console-proxy"
|
||||
assert "coord_ws_id" not in payload
|
||||
|
||||
|
||||
def test_coordinator_source_is_preserved_on_remint():
|
||||
"""Inbound src='coordinator' → outbound src='coordinator'."""
|
||||
auth = AuthResult(
|
||||
user_id="user-1",
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="coordinator",
|
||||
permissions=frozenset({"admin.coordinator"}),
|
||||
extra_claims={"coord_ws_id": "coord-42"},
|
||||
)
|
||||
headers = _proxy_auth_headers(_build_request(auth))
|
||||
payload = _decode(headers)
|
||||
assert payload["src"] == "coordinator"
|
||||
assert payload["coord_ws_id"] == "coord-42"
|
||||
|
||||
|
||||
def test_coord_ws_id_absent_when_not_in_inbound_claims():
|
||||
"""Defensive: if the inbound token is src=coordinator but missing the
|
||||
coord_ws_id claim (shouldn't happen in practice), the re-mint skips
|
||||
the custom claim rather than panicking."""
|
||||
auth = AuthResult(
|
||||
user_id="user-1",
|
||||
scopes=frozenset({"write"}),
|
||||
token_source="coordinator",
|
||||
permissions=frozenset(),
|
||||
)
|
||||
headers = _proxy_auth_headers(_build_request(auth))
|
||||
payload = _decode(headers)
|
||||
assert payload["src"] == "coordinator"
|
||||
assert "coord_ws_id" not in payload
|
||||
|
||||
|
||||
def test_empty_auth_falls_back_to_service_token_or_empty():
|
||||
"""Without auth_result.user_id, falls through to ServiceTokenManager."""
|
||||
auth = AuthResult(
|
||||
user_id="",
|
||||
scopes=frozenset(),
|
||||
token_source="config",
|
||||
permissions=frozenset(),
|
||||
)
|
||||
# No proxy_token_mgr configured → empty headers.
|
||||
headers = _proxy_auth_headers(_build_request(auth))
|
||||
assert headers == {}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,15 @@
|
||||
"""Tests for turnstone.core.hash_ring."""
|
||||
|
||||
from turnstone.core.hash_ring import bucket_of
|
||||
|
||||
|
||||
class TestBucketOf:
|
||||
def test_known_vectors(self):
|
||||
assert bucket_of("a3f1" + "0" * 28) == 0xA3F1
|
||||
assert bucket_of("0000" + "a" * 28) == 0
|
||||
assert bucket_of("ffff" + "b" * 28) == 65535
|
||||
|
||||
def test_hex_prefix(self):
|
||||
# Only the first 4 hex chars matter — the rest is ignored.
|
||||
assert bucket_of("abcd0000") == bucket_of("abcdffff")
|
||||
assert bucket_of("abcd0000") == 0xABCD
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Tests for the hash ring routing storage methods."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class TestHashRingBuckets:
|
||||
def test_list_empty(self, storage):
|
||||
assert storage.list_ring_buckets() == []
|
||||
|
||||
def test_seed_and_list(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-b"), (2, "node-a")])
|
||||
rows = storage.list_ring_buckets()
|
||||
assert len(rows) == 3
|
||||
assert rows[0] == {"bucket": 0, "node_id": "node-a"}
|
||||
assert rows[1] == {"bucket": 1, "node_id": "node-b"}
|
||||
assert rows[2] == {"bucket": 2, "node_id": "node-a"}
|
||||
|
||||
def test_seed_idempotent(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-b")])
|
||||
# Re-seed with conflicting assignment: should keep original
|
||||
storage.seed_ring_buckets([(0, "node-x"), (2, "node-c")])
|
||||
rows = storage.list_ring_buckets()
|
||||
by_bucket = {r["bucket"]: r["node_id"] for r in rows}
|
||||
assert by_bucket[0] == "node-a" # original preserved
|
||||
assert by_bucket[1] == "node-b"
|
||||
assert by_bucket[2] == "node-c" # new bucket added
|
||||
|
||||
def test_assign_buckets(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-a"), (2, "node-b")])
|
||||
storage.assign_buckets([0, 1], "node-c")
|
||||
rows = storage.list_ring_buckets()
|
||||
by_bucket = {r["bucket"]: r["node_id"] for r in rows}
|
||||
assert by_bucket[0] == "node-c"
|
||||
assert by_bucket[1] == "node-c"
|
||||
assert by_bucket[2] == "node-b"
|
||||
|
||||
def test_assign_returns_count(self, storage):
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-a")])
|
||||
count = storage.assign_buckets([0, 1], "node-b")
|
||||
assert count == 2
|
||||
# Empty list returns 0
|
||||
assert storage.assign_buckets([], "node-x") == 0
|
||||
|
||||
def test_assign_large_list_exceeds_chunk_size(self, storage):
|
||||
"""Regression: lists larger than chunk_size must not hit param limits."""
|
||||
n = 1200 # exceeds SQLite chunk_size (500) and exercises multi-chunk path
|
||||
storage.seed_ring_buckets([(i, "node-a") for i in range(n)])
|
||||
count = storage.assign_buckets(list(range(n)), "node-b")
|
||||
assert count == n
|
||||
rows = storage.list_ring_buckets()
|
||||
assert all(r["node_id"] == "node-b" for r in rows)
|
||||
|
||||
def test_assign_deduplicates_input(self, storage):
|
||||
"""Duplicates in the input list should not inflate rowcount."""
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-a")])
|
||||
count = storage.assign_buckets([0, 1, 0, 1, 0], "node-b")
|
||||
assert count == 2
|
||||
|
||||
|
||||
class TestBucketStats:
|
||||
def test_increment_creates_row(self, storage):
|
||||
storage.increment_bucket_count(42)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert len(stats) == 1
|
||||
assert stats[0]["bucket"] == 42
|
||||
assert stats[0]["ws_count"] == 1
|
||||
assert stats[0]["active_count"] == 0
|
||||
|
||||
def test_increment_active(self, storage):
|
||||
storage.increment_bucket_count(10, active=True)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 1
|
||||
assert stats[0]["active_count"] == 1
|
||||
# Increment again without active
|
||||
storage.increment_bucket_count(10)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 2
|
||||
assert stats[0]["active_count"] == 1
|
||||
|
||||
def test_decrement(self, storage):
|
||||
storage.increment_bucket_count(5, active=True)
|
||||
storage.increment_bucket_count(5, active=True)
|
||||
storage.decrement_bucket_count(5, active=True)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 1
|
||||
assert stats[0]["active_count"] == 1
|
||||
|
||||
def test_decrement_clamps_at_zero(self, storage):
|
||||
storage.increment_bucket_count(7)
|
||||
storage.decrement_bucket_count(7)
|
||||
storage.decrement_bucket_count(7) # already at 0
|
||||
stats = storage.list_bucket_stats()
|
||||
# ws_count is 0, so should not appear (filter ws_count > 0)
|
||||
assert len(stats) == 0
|
||||
|
||||
def test_adjust_active_only(self, storage):
|
||||
storage.increment_bucket_count(20, active=True)
|
||||
storage.increment_bucket_count(20, active=True)
|
||||
# Decrease active without changing ws_count
|
||||
storage.adjust_bucket_active(20, -1)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["ws_count"] == 2
|
||||
assert stats[0]["active_count"] == 1
|
||||
# Clamp at zero
|
||||
storage.adjust_bucket_active(20, -5)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert stats[0]["active_count"] == 0
|
||||
|
||||
def test_list_sparse(self, storage):
|
||||
storage.increment_bucket_count(100)
|
||||
storage.increment_bucket_count(200)
|
||||
storage.increment_bucket_count(300)
|
||||
# Decrement 200 to zero
|
||||
storage.decrement_bucket_count(200)
|
||||
stats = storage.list_bucket_stats()
|
||||
buckets = [s["bucket"] for s in stats]
|
||||
assert 100 in buckets
|
||||
assert 200 not in buckets
|
||||
assert 300 in buckets
|
||||
|
||||
def test_set_bucket_stat_creates(self, storage):
|
||||
"""set_bucket_stat upserts a new row."""
|
||||
storage.set_bucket_stat(42, 5, 2)
|
||||
stats = storage.list_bucket_stats()
|
||||
row = next(s for s in stats if s["bucket"] == 42)
|
||||
assert row["ws_count"] == 5
|
||||
assert row["active_count"] == 2
|
||||
|
||||
def test_set_bucket_stat_overwrites(self, storage):
|
||||
"""set_bucket_stat overwrites existing values."""
|
||||
storage.set_bucket_stat(42, 10, 3)
|
||||
storage.set_bucket_stat(42, 2, 0)
|
||||
stats = storage.list_bucket_stats()
|
||||
row = next(s for s in stats if s["bucket"] == 42)
|
||||
assert row["ws_count"] == 2
|
||||
assert row["active_count"] == 0
|
||||
|
||||
def test_set_bucket_stat_zero_removes_from_sparse(self, storage):
|
||||
"""Setting ws_count=0 means list_bucket_stats excludes it (sparse)."""
|
||||
storage.set_bucket_stat(42, 5, 1)
|
||||
storage.set_bucket_stat(42, 0, 0)
|
||||
stats = storage.list_bucket_stats()
|
||||
assert not any(s["bucket"] == 42 for s in stats)
|
||||
|
||||
|
||||
class TestWorkstreamOverrides:
|
||||
def test_set_and_list(self, storage):
|
||||
storage.set_workstream_override("ws-001", "node-a", reason="affinity")
|
||||
overrides = storage.list_workstream_overrides()
|
||||
assert len(overrides) == 1
|
||||
assert overrides[0]["ws_id"] == "ws-001"
|
||||
assert overrides[0]["node_id"] == "node-a"
|
||||
assert overrides[0]["reason"] == "affinity"
|
||||
|
||||
def test_upsert(self, storage):
|
||||
storage.set_workstream_override("ws-002", "node-a")
|
||||
storage.set_workstream_override("ws-002", "node-b", reason="migration")
|
||||
overrides = storage.list_workstream_overrides()
|
||||
assert len(overrides) == 1
|
||||
assert overrides[0]["node_id"] == "node-b"
|
||||
assert overrides[0]["reason"] == "migration"
|
||||
|
||||
def test_delete(self, storage):
|
||||
storage.set_workstream_override("ws-003", "node-a")
|
||||
result = storage.delete_workstream_override("ws-003")
|
||||
assert result is True
|
||||
assert storage.list_workstream_overrides() == []
|
||||
|
||||
def test_delete_nonexistent(self, storage):
|
||||
result = storage.delete_workstream_override("ws-nope")
|
||||
assert result is False
|
||||
|
||||
def test_list_empty(self, storage):
|
||||
assert storage.list_workstream_overrides() == []
|
||||
@@ -1,248 +0,0 @@
|
||||
"""Tests for InteractiveAdapter.
|
||||
|
||||
Focus: the ``emit_closed`` transport contract (sole path for
|
||||
``ws_closed`` onto the process-wide queue) and ``cleanup_ui``
|
||||
behavior (unblock pending events, broadcast ``ws_closed`` to per-UI
|
||||
listeners, cancel + close session). The SessionManager-level tests
|
||||
in ``test_session_manager.py`` cover the adapter-agnostic lifecycle.
|
||||
|
||||
The other three :class:`SessionEventEmitter` methods
|
||||
(``emit_created`` / ``emit_state`` / ``emit_rehydrated``) are
|
||||
documented no-op stubs — ``ws_created`` is fired by the create HTTP
|
||||
handler after attachment validation, and ``ws_state`` is fired by
|
||||
``WebUI._broadcast_state`` with the full payload. No-op assertions
|
||||
on those methods would be tautological given the class docstring,
|
||||
so they're not retested here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from turnstone.core.adapters.interactive_adapter import InteractiveAdapter
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind
|
||||
|
||||
|
||||
class _StubUI:
|
||||
"""Stub matching the subset of WebUI the adapter touches."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._approval_event = threading.Event()
|
||||
self._approval_result: tuple[bool, str | None] = (True, "initial")
|
||||
self._plan_event = threading.Event()
|
||||
self._plan_result: str = "accept"
|
||||
self._fg_event = threading.Event()
|
||||
self._listeners_lock = threading.Lock()
|
||||
self._listeners: list[queue.Queue[dict[str, Any]]] = []
|
||||
|
||||
|
||||
class _StubSession:
|
||||
def __init__(self) -> None:
|
||||
self.cancelled = False
|
||||
self.closed = False
|
||||
self.model = "gpt-5"
|
||||
self.model_alias = "default"
|
||||
|
||||
def cancel(self) -> None:
|
||||
self.cancelled = True
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _make_adapter(
|
||||
*,
|
||||
ui_factory: Any = None,
|
||||
session_factory: Any = None,
|
||||
) -> tuple[InteractiveAdapter, queue.Queue[dict[str, Any]]]:
|
||||
gq: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=100)
|
||||
adapter = InteractiveAdapter(
|
||||
global_queue=gq,
|
||||
ui_factory=ui_factory or (lambda ws: _StubUI()),
|
||||
session_factory=session_factory or (lambda *a, **kw: _StubSession()),
|
||||
)
|
||||
return adapter, gq
|
||||
|
||||
|
||||
def _make_ws(**overrides: Any) -> Workstream:
|
||||
ws = Workstream(id="ws-1", name="hello")
|
||||
ws.kind = WorkstreamKind.INTERACTIVE
|
||||
ws.user_id = "u1"
|
||||
ws.ui = _StubUI()
|
||||
ws.session = _StubSession()
|
||||
for k, v in overrides.items():
|
||||
setattr(ws, k, v)
|
||||
return ws
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Transport — emit_closed (the only emit_* with real behavior on interactive;
|
||||
# emit_created / emit_state / emit_rehydrated are documented no-op stubs)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_emit_closed_defaults_to_closed_reason() -> None:
|
||||
adapter, gq = _make_adapter()
|
||||
adapter.emit_closed("ws-1", name="my-ws")
|
||||
event = gq.get_nowait()
|
||||
assert event == {
|
||||
"type": "ws_closed",
|
||||
"ws_id": "ws-1",
|
||||
"reason": "closed",
|
||||
"name": "my-ws",
|
||||
}
|
||||
|
||||
|
||||
def test_emit_closed_propagates_evicted_reason_and_name() -> None:
|
||||
adapter, gq = _make_adapter()
|
||||
adapter.emit_closed("ws-1", reason="evicted", name="my-ws")
|
||||
event = gq.get_nowait()
|
||||
assert event["reason"] == "evicted"
|
||||
assert event["name"] == "my-ws"
|
||||
|
||||
|
||||
def test_emit_closed_default_name_is_empty_string() -> None:
|
||||
adapter, gq = _make_adapter()
|
||||
adapter.emit_closed("ws-1")
|
||||
assert gq.get_nowait()["name"] == ""
|
||||
|
||||
|
||||
def test_emit_swallows_queue_full_without_raising() -> None:
|
||||
gq: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1)
|
||||
gq.put({"type": "filler"})
|
||||
adapter = InteractiveAdapter(
|
||||
global_queue=gq,
|
||||
ui_factory=lambda ws: _StubUI(),
|
||||
session_factory=lambda *a, **kw: _StubSession(),
|
||||
)
|
||||
adapter.emit_closed("ws-1") # must not raise even though queue is full
|
||||
assert gq.qsize() == 1 # nothing added on a full queue
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cleanup_ui
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cleanup_ui_unblocks_pending_approval_plan_fg_events() -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
# Simulate pending events
|
||||
ws.ui._approval_event.clear() # type: ignore[attr-defined]
|
||||
ws.ui._plan_event.clear() # type: ignore[attr-defined]
|
||||
ws.ui._fg_event.clear() # type: ignore[attr-defined]
|
||||
|
||||
adapter.cleanup_ui(ws)
|
||||
|
||||
assert ws.ui._approval_event.is_set() # type: ignore[attr-defined]
|
||||
assert ws.ui._plan_event.is_set() # type: ignore[attr-defined]
|
||||
assert ws.ui._fg_event.is_set() # type: ignore[attr-defined]
|
||||
# Approval result flipped to "deny" so the waiter sees a sensible value.
|
||||
assert ws.ui._approval_result == (False, None) # type: ignore[attr-defined]
|
||||
assert ws.ui._plan_result == "reject" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_cleanup_ui_broadcasts_ws_closed_to_listener_queues() -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
lq1: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=10)
|
||||
lq2: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=10)
|
||||
ws.ui._listeners.extend([lq1, lq2]) # type: ignore[attr-defined]
|
||||
|
||||
adapter.cleanup_ui(ws)
|
||||
|
||||
assert lq1.get_nowait() == {"type": "ws_closed"}
|
||||
assert lq2.get_nowait() == {"type": "ws_closed"}
|
||||
# Listeners cleared so subsequent events don't fan out to dead generators.
|
||||
assert ws.ui._listeners == [] # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_cleanup_ui_broadcast_evicts_stale_head_when_listener_queue_full() -> None:
|
||||
"""Per the old _cleanup_ui fallback: when a listener queue is full,
|
||||
drop the oldest event and put ws_closed. Ensures an unresponsive
|
||||
browser tab doesn't block close."""
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
lq: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1)
|
||||
lq.put_nowait({"type": "stale"})
|
||||
ws.ui._listeners.append(lq) # type: ignore[attr-defined]
|
||||
|
||||
adapter.cleanup_ui(ws)
|
||||
|
||||
assert lq.get_nowait() == {"type": "ws_closed"}
|
||||
assert lq.empty()
|
||||
|
||||
|
||||
def test_cleanup_ui_cancels_and_closes_session() -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.cleanup_ui(ws)
|
||||
assert ws.session.cancelled is True # type: ignore[attr-defined]
|
||||
assert ws.session.closed is True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_cleanup_ui_tolerates_missing_session_and_ui() -> None:
|
||||
"""A placeholder workstream whose session build failed may arrive
|
||||
at cleanup_ui with session=None or ui=None. Must not crash."""
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
ws.session = None
|
||||
ws.ui = None
|
||||
adapter.cleanup_ui(ws) # no crash
|
||||
|
||||
|
||||
def test_cleanup_ui_tolerates_stub_ui_without_events() -> None:
|
||||
"""A stub UI missing _approval_event / etc. (test scaffolding
|
||||
code) must not crash cleanup_ui — the hasattr guards matter."""
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
ws.ui = MagicMock(spec=[]) # empty spec — attribute accesses miss
|
||||
adapter.cleanup_ui(ws) # no crash
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction passthrough
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_ui_delegates_to_ui_factory() -> None:
|
||||
captured_ws: list[Workstream] = []
|
||||
|
||||
def _ui_factory(ws: Workstream) -> Any:
|
||||
captured_ws.append(ws)
|
||||
return _StubUI()
|
||||
|
||||
adapter, _ = _make_adapter(ui_factory=_ui_factory)
|
||||
ws = _make_ws()
|
||||
result = adapter.build_ui(ws)
|
||||
assert captured_ws == [ws]
|
||||
assert isinstance(result, _StubUI)
|
||||
|
||||
|
||||
def test_build_session_forwards_all_kwargs_to_session_factory() -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _sf(ui: Any, model: str | None, ws_id: str, **kwargs: Any) -> Any:
|
||||
captured["ui"] = ui
|
||||
captured["model"] = model
|
||||
captured["ws_id"] = ws_id
|
||||
captured.update(kwargs)
|
||||
return _StubSession()
|
||||
|
||||
adapter, _ = _make_adapter(session_factory=_sf)
|
||||
ws = _make_ws()
|
||||
adapter.build_session(
|
||||
ws, skill="coder", model="gpt-5", client_type="web", judge_model="gpt-4.1"
|
||||
)
|
||||
assert captured["ui"] is ws.ui
|
||||
assert captured["model"] == "gpt-5"
|
||||
assert captured["ws_id"] == ws.id
|
||||
assert captured["skill"] == "coder"
|
||||
assert captured["client_type"] == "web"
|
||||
assert captured["kind"] == WorkstreamKind.INTERACTIVE
|
||||
assert captured["parent_ws_id"] is None
|
||||
# Kind-specific passthrough — interactive session_factory accepts judge_model.
|
||||
assert captured["judge_model"] == "gpt-4.1"
|
||||
@@ -703,113 +703,3 @@ class TestHeuristicNewLowRules:
|
||||
def test_web_search(self):
|
||||
v = evaluate_heuristic("web_search", {"query": "python"}, "web_search")
|
||||
assert v.risk_level == "low"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Alias resolution — regression guard for the "did not return a verdict"
|
||||
# silent no-op surfaced during coordinator harness testing.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestModelAliasResolution:
|
||||
"""When ``judge.model`` points at a registry alias whose underlying
|
||||
provider differs from the session's, the judge MUST resolve through
|
||||
the registry — not fall back to the session provider with the
|
||||
underlying model id. Pre-resolving the alias to the model id in the
|
||||
session_factory stranded the alias and made every coordinator tool
|
||||
verdict come back ``llm_fallback / "did not return a verdict"``.
|
||||
"""
|
||||
|
||||
def _make_alias_registry(
|
||||
self,
|
||||
alias: str,
|
||||
alias_provider: MagicMock,
|
||||
alias_client: MagicMock,
|
||||
underlying_model: str,
|
||||
) -> MagicMock:
|
||||
registry = MagicMock()
|
||||
cfg = MagicMock()
|
||||
cfg.context_window = 50_000
|
||||
registry.has_alias.side_effect = lambda a: a == alias
|
||||
registry.resolve.return_value = (alias_client, underlying_model, cfg)
|
||||
registry.get_provider.return_value = alias_provider
|
||||
return registry
|
||||
|
||||
def test_alias_uses_registry_provider_not_session_provider(self):
|
||||
"""Judge with model=alias should resolve via registry — provider, client,
|
||||
and concrete model name all come from the alias."""
|
||||
# Session provider/client — would be used if resolution falls back.
|
||||
session_provider = _make_mock_provider(
|
||||
response_content=_good_verdict_json(intent_summary="from-session"),
|
||||
)
|
||||
session_provider.provider_name = "anthropic"
|
||||
session_client = MagicMock()
|
||||
session_client.base_url = "https://session.example/v1"
|
||||
session_client.api_key = "session-key"
|
||||
|
||||
# Alias provider/client — what the judge SHOULD use.
|
||||
alias_provider = _make_mock_provider(
|
||||
response_content=_good_verdict_json(intent_summary="from-alias"),
|
||||
)
|
||||
alias_provider.provider_name = "openai"
|
||||
alias_client = MagicMock()
|
||||
alias_client.base_url = "https://alias.example/v1"
|
||||
alias_client.api_key = "alias-key"
|
||||
|
||||
registry = self._make_alias_registry(
|
||||
"judge-mini", alias_provider, alias_client, "gpt-5-mini-resolved"
|
||||
)
|
||||
|
||||
config = JudgeConfig(enabled=True, model="judge-mini")
|
||||
judge = IntentJudge(
|
||||
config=config,
|
||||
session_provider=session_provider,
|
||||
session_client=session_client,
|
||||
session_model="session-default-model",
|
||||
context_window=100_000,
|
||||
model_registry=registry,
|
||||
)
|
||||
|
||||
assert judge._provider is alias_provider
|
||||
assert judge._model == "gpt-5-mini-resolved"
|
||||
# Client factory args reflect the alias's client, not the session's.
|
||||
assert judge._client_factory_args["base_url"] == "https://alias.example/v1"
|
||||
assert judge._client_factory_args["api_key"] == "alias-key"
|
||||
assert judge._client_factory_args["provider_name"] == "openai"
|
||||
|
||||
def test_coordinator_tool_call_returns_llm_verdict_not_fallback(self):
|
||||
"""Happy-path regression for coordinator tool calls: with a properly
|
||||
resolved provider, the verdict tier must be ``llm`` — the
|
||||
``llm_fallback`` failure mode flagged in the harness was uniform
|
||||
across every coordinator tool, so guard the happy path explicitly.
|
||||
"""
|
||||
provider = _make_mock_provider(
|
||||
response_content=_good_verdict_json(
|
||||
intent_summary="Spawn a child workstream",
|
||||
risk_level="medium",
|
||||
recommendation="approve",
|
||||
),
|
||||
)
|
||||
judge = _make_judge(provider)
|
||||
|
||||
callback_results: list[IntentVerdict] = []
|
||||
coord_item = _make_item(
|
||||
func_name="spawn_workstream",
|
||||
func_args={"initial_message": "do the thing", "skill": "engineer"},
|
||||
approval_label="spawn_workstream",
|
||||
)
|
||||
judge.evaluate(
|
||||
[coord_item],
|
||||
[{"role": "user", "content": "delegate the audit"}],
|
||||
callback_results.append,
|
||||
)
|
||||
# Wait for daemon thread.
|
||||
for _ in range(20):
|
||||
if callback_results:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
assert callback_results, "judge never delivered a verdict"
|
||||
assert callback_results[0].tier == "llm"
|
||||
assert callback_results[0].tier != "llm_fallback"
|
||||
assert "did not return a verdict" not in callback_results[0].reasoning
|
||||
|
||||
@@ -114,59 +114,6 @@ class TestIntentVerdictCRUD:
|
||||
assert ok is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bulk insert
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIntentVerdictBulkInsert:
|
||||
"""Coverage for ``create_intent_verdicts_bulk`` — backs the
|
||||
``approve_tools`` per-turn heuristic-verdict persistence path so a
|
||||
fan-out turn pays one commit instead of N.
|
||||
"""
|
||||
|
||||
def test_bulk_insert_creates_all_rows(self, db):
|
||||
db.create_intent_verdicts_bulk(
|
||||
[
|
||||
_make_verdict_kwargs(verdict_id="b1", call_id="c1"),
|
||||
_make_verdict_kwargs(verdict_id="b2", call_id="c2"),
|
||||
_make_verdict_kwargs(verdict_id="b3", call_id="c3"),
|
||||
]
|
||||
)
|
||||
for vid in ("b1", "b2", "b3"):
|
||||
v = db.get_intent_verdict(vid)
|
||||
assert v is not None
|
||||
assert v["verdict_id"] == vid
|
||||
|
||||
def test_bulk_insert_empty_list_is_noop(self, db):
|
||||
# Must not raise and must not commit a phantom row.
|
||||
db.create_intent_verdicts_bulk([])
|
||||
assert db.list_intent_verdicts() == []
|
||||
|
||||
def test_bulk_insert_preserves_distinct_field_values(self, db):
|
||||
db.create_intent_verdicts_bulk(
|
||||
[
|
||||
_make_verdict_kwargs(
|
||||
verdict_id="b1",
|
||||
risk_level="low",
|
||||
tier="heuristic",
|
||||
confidence=0.4,
|
||||
),
|
||||
_make_verdict_kwargs(
|
||||
verdict_id="b2",
|
||||
risk_level="high",
|
||||
tier="llm",
|
||||
confidence=0.95,
|
||||
),
|
||||
]
|
||||
)
|
||||
v1 = db.get_intent_verdict("b1")
|
||||
v2 = db.get_intent_verdict("b2")
|
||||
assert v1 is not None and v2 is not None
|
||||
assert v1["risk_level"] == "low" and v1["tier"] == "heuristic"
|
||||
assert v2["risk_level"] == "high" and v2["tier"] == "llm"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# List queries
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+13
-15
@@ -158,7 +158,7 @@ class TestExecLoadSkill:
|
||||
"name": "code-review",
|
||||
"description": "Reviews code for quality",
|
||||
"content": "# Code Review\nReview all code.",
|
||||
"risk_level": "safe",
|
||||
"scan_status": "safe",
|
||||
"category": "engineering",
|
||||
}
|
||||
]
|
||||
@@ -185,7 +185,7 @@ class TestExecLoadSkill:
|
||||
assert session._set_skill_called == []
|
||||
|
||||
def test_load_calls_ui_on_tool_result(self) -> None:
|
||||
skills = [{"name": "test", "content": "content", "description": "", "risk_level": ""}]
|
||||
skills = [{"name": "test", "content": "content", "description": "", "scan_status": ""}]
|
||||
session, _, fake_get = _make_session(skills)
|
||||
|
||||
with patch("turnstone.core.session.get_skill_by_name", side_effect=fake_get):
|
||||
@@ -200,7 +200,7 @@ class TestExecLoadSkill:
|
||||
"name": "code-review",
|
||||
"description": "Reviews code",
|
||||
"category": "eng",
|
||||
"risk_level": "safe",
|
||||
"scan_status": "safe",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
},
|
||||
@@ -208,7 +208,7 @@ class TestExecLoadSkill:
|
||||
"name": "docs-writer",
|
||||
"description": "Writes docs",
|
||||
"category": "general",
|
||||
"risk_level": "low",
|
||||
"scan_status": "low",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
},
|
||||
@@ -232,7 +232,7 @@ class TestExecLoadSkill:
|
||||
"name": f"skill-{i}",
|
||||
"description": f"Desc {i}",
|
||||
"category": "general",
|
||||
"risk_level": "",
|
||||
"scan_status": "",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
}
|
||||
@@ -262,13 +262,13 @@ class TestExecLoadSkill:
|
||||
|
||||
assert "no skills found" in result.lower()
|
||||
|
||||
def test_search_includes_risk_level(self) -> None:
|
||||
def test_search_includes_scan_status(self) -> None:
|
||||
skills = [
|
||||
{
|
||||
"name": "risky",
|
||||
"description": "Risky skill",
|
||||
"category": "ops",
|
||||
"risk_level": "high",
|
||||
"scan_status": "high",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
},
|
||||
@@ -301,7 +301,7 @@ class TestExecLoadSkill:
|
||||
"name": "disabled-skill",
|
||||
"content": "x",
|
||||
"description": "",
|
||||
"risk_level": "",
|
||||
"scan_status": "",
|
||||
"enabled": False,
|
||||
}
|
||||
]
|
||||
@@ -315,7 +315,7 @@ class TestExecLoadSkill:
|
||||
assert session._set_skill_called == []
|
||||
|
||||
def test_load_already_active_skill(self) -> None:
|
||||
skills = [{"name": "active", "content": "x", "description": "", "risk_level": "safe"}]
|
||||
skills = [{"name": "active", "content": "x", "description": "", "scan_status": "safe"}]
|
||||
session, _, fake_get = _make_session(skills)
|
||||
session._skill_name = "active"
|
||||
|
||||
@@ -332,7 +332,7 @@ class TestExecLoadSkill:
|
||||
"name": "enabled-skill",
|
||||
"description": "Good",
|
||||
"category": "gen",
|
||||
"risk_level": "",
|
||||
"scan_status": "",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
"enabled": True,
|
||||
@@ -341,7 +341,7 @@ class TestExecLoadSkill:
|
||||
"name": "disabled-skill",
|
||||
"description": "Bad",
|
||||
"category": "gen",
|
||||
"risk_level": "",
|
||||
"scan_status": "",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
"enabled": False,
|
||||
@@ -365,7 +365,7 @@ class TestExecLoadSkill:
|
||||
"name": "code-review",
|
||||
"description": "Reviews code for quality",
|
||||
"category": "eng",
|
||||
"risk_level": "",
|
||||
"scan_status": "",
|
||||
"tags": "[]",
|
||||
"activation": "named",
|
||||
},
|
||||
@@ -420,8 +420,7 @@ class TestSkillCatalogDisclosure:
|
||||
session.system_messages = []
|
||||
session._agent_system_messages = []
|
||||
session.reasoning_effort = "medium"
|
||||
session._pending_tool_advisories = []
|
||||
session._pending_user_advisories = []
|
||||
session._pending_nudge = []
|
||||
session._tool_search = None
|
||||
session._mcp_client = None
|
||||
session._notify_on_complete = "{}"
|
||||
@@ -431,7 +430,6 @@ class TestSkillCatalogDisclosure:
|
||||
session._tools = []
|
||||
session._client_type = ClientType.CLI
|
||||
session._username = ""
|
||||
session._kind = "interactive"
|
||||
|
||||
# Memory stubs
|
||||
session._memory_config = MagicMock()
|
||||
|
||||
@@ -17,7 +17,7 @@ from turnstone.core.mcp_client import (
|
||||
_mcp_to_openai,
|
||||
load_mcp_config,
|
||||
)
|
||||
from turnstone.core.tools import INTERACTIVE_TOOLS, TOOLS, merge_mcp_tools
|
||||
from turnstone.core.tools import TOOLS, merge_mcp_tools
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -349,15 +349,14 @@ class TestSessionIntegration:
|
||||
|
||||
def test_session_without_mcp(self, tmp_db):
|
||||
session = self._make_session(mcp_client=None)
|
||||
# Interactive session surface — coordinator tools excluded.
|
||||
assert session._tools is INTERACTIVE_TOOLS
|
||||
assert session._tools is TOOLS
|
||||
assert session._mcp_client is None
|
||||
|
||||
def test_session_with_mcp(self, tmp_db):
|
||||
mock_mcp = MagicMock()
|
||||
mock_mcp.get_tools.return_value = [_fake_openai_tool()]
|
||||
session = self._make_session(mcp_client=mock_mcp)
|
||||
assert len(session._tools) == len(INTERACTIVE_TOOLS) + 1
|
||||
assert len(session._tools) == len(TOOLS) + 1
|
||||
assert session._tools[-1]["function"]["name"] == "mcp__test__search"
|
||||
|
||||
def test_task_tools_include_mcp(self, tmp_db):
|
||||
|
||||
@@ -8,7 +8,6 @@ from turnstone.core.metacognition import (
|
||||
NUDGE_RESUME,
|
||||
NUDGE_START,
|
||||
NUDGE_TOOL_ERROR,
|
||||
RepeatDetector,
|
||||
detect_completion,
|
||||
detect_correction,
|
||||
format_nudge,
|
||||
@@ -309,70 +308,3 @@ class TestRepeatNudge:
|
||||
"""Repeat nudge should fire even with zero memories."""
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("repeat", state, message_count=5, memory_count=0) is True
|
||||
|
||||
|
||||
class TestRepeatDetector:
|
||||
"""Repeat-detection streak machine — fires only when the same signature
|
||||
is recorded ``threshold`` times *consecutively* (default 3). Recording
|
||||
any different signature resets the streak, so an interrupted repeat
|
||||
isn't flagged as a stuck loop."""
|
||||
|
||||
def test_below_threshold_does_not_fire(self):
|
||||
det = RepeatDetector()
|
||||
assert det.record("a") is False
|
||||
assert det.record("a") is False # second call still under threshold
|
||||
|
||||
def test_at_threshold_fires(self):
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_continues_to_fire_past_threshold(self):
|
||||
# Caller is responsible for clearing after a fire — until they do,
|
||||
# subsequent identical calls keep returning True.
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("a") is True
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_clear_resets_count(self):
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
det.clear()
|
||||
assert det.record("a") is False # back to 1 after clear
|
||||
|
||||
def test_intervening_sig_resets_streak(self):
|
||||
# The streak is consecutive: recording any other sig mid-streak
|
||||
# discards the in-progress count. An alternating pattern like
|
||||
# [A, A, B, A, A] is two short streaks of 2, not a streak of 4.
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("b") is False # b at count 1; a's streak is gone
|
||||
assert det.record("a") is False # a starts fresh at 1
|
||||
assert det.record("a") is False # a at 2
|
||||
assert det.record("a") is True # a hits 3 — fresh streak completes
|
||||
|
||||
def test_errored_signature_counts_toward_repeat(self):
|
||||
# Regression: when metacog was split out of the system message,
|
||||
# the error-output skip got reintroduced and stuck-loop detection
|
||||
# silently broke for tools that kept failing. Detector itself is
|
||||
# signature-only — error vs. success is the caller's policy.
|
||||
det = RepeatDetector()
|
||||
# Caller records an errored call's sig the same as a successful one;
|
||||
# the streak is what matters.
|
||||
for _ in range(3):
|
||||
last = det.record("bash:ls /nonexistent")
|
||||
assert last is True
|
||||
|
||||
def test_custom_threshold(self):
|
||||
det = RepeatDetector(threshold=2)
|
||||
assert det.record("a") is False
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_threshold_one_fires_immediately(self):
|
||||
det = RepeatDetector(threshold=1)
|
||||
assert det.record("a") is True
|
||||
|
||||
@@ -770,76 +770,16 @@ class TestRegistryReload:
|
||||
assert reg.has_alias("b")
|
||||
assert reg.default == "b"
|
||||
|
||||
def test_reload_keeps_clients_when_connection_target_unchanged(self) -> None:
|
||||
"""Selective teardown: a model edit that leaves base_url / api_key /
|
||||
provider intact (e.g. admin tweaks the underlying ``model`` name or
|
||||
``temperature``) keeps the cached HTTP client warm — no need to
|
||||
re-establish TLS+pool when the endpoint is the same."""
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m1", provider="openai")}
|
||||
def test_reload_clears_clients(self) -> None:
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m")}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
# Force client creation
|
||||
reg.get_client("a")
|
||||
client_before = reg._clients["a"]
|
||||
provider_before = reg.get_provider("a")
|
||||
|
||||
# Same endpoint (base_url, api_key, provider), only ``model`` changed.
|
||||
new_models = {"a": ModelConfig("a", "http://x/v1", "key", "m2", provider="openai")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
assert "a" in reg._clients
|
||||
assert reg._clients["a"] is client_before
|
||||
assert "a" in reg._providers
|
||||
assert reg._providers["a"] is provider_before
|
||||
|
||||
def test_reload_drops_client_when_base_url_changes(self) -> None:
|
||||
"""A ``base_url`` change drops the cached client (different
|
||||
endpoint = new connection) but keeps the cached provider —
|
||||
``LLMProvider`` is keyed only on the provider string, which
|
||||
didn't change."""
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m", provider="openai")}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
reg.get_client("a")
|
||||
provider_before = reg.get_provider("a")
|
||||
|
||||
new_models = {"a": ModelConfig("a", "http://y/v1", "key", "m", provider="openai")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
# Reload with same models — clients should be cleared
|
||||
reg.reload(dict(models), "a")
|
||||
assert "a" not in reg._clients
|
||||
assert "a" in reg._providers
|
||||
assert reg._providers["a"] is provider_before
|
||||
|
||||
def test_reload_drops_provider_when_provider_string_changes(self) -> None:
|
||||
"""A provider-type swap (e.g. openai → anthropic) drops both the
|
||||
client AND the provider so the next resolve picks up the right
|
||||
``LLMProvider`` implementation against the new SDK."""
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m", provider="openai")}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
reg.get_client("a")
|
||||
reg.get_provider("a")
|
||||
|
||||
new_models = {"a": ModelConfig("a", "http://x/v1", "key", "m", provider="anthropic")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
assert "a" not in reg._clients
|
||||
assert "a" not in reg._providers
|
||||
|
||||
def test_reload_drops_clients_for_removed_aliases(self) -> None:
|
||||
"""Aliases removed from the registry must release their cached
|
||||
clients — otherwise a deleted endpoint's connection pool would
|
||||
outlive the alias indefinitely."""
|
||||
models = {
|
||||
"a": ModelConfig("a", "http://x/v1", "key", "m"),
|
||||
"b": ModelConfig("b", "http://y/v1", "key", "m"),
|
||||
}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
reg.get_client("a")
|
||||
reg.get_client("b")
|
||||
|
||||
# Drop "b" entirely.
|
||||
new_models = {"a": ModelConfig("a", "http://x/v1", "key", "m")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
assert "a" in reg._clients # unchanged endpoint, kept warm
|
||||
assert "b" not in reg._clients
|
||||
|
||||
def test_reload_validates_default(self) -> None:
|
||||
models_a = {"a": ModelConfig("a", "x", "x", "m")}
|
||||
@@ -1265,26 +1205,11 @@ class TestSessionAgentModel:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_manager(session_factory: Any) -> Any:
|
||||
"""Construct a SessionManager with an interactive adapter that
|
||||
forwards to the supplied session_factory. Storage is mocked — the
|
||||
only thing the model-alias tests exercise is the factory passthrough."""
|
||||
import queue
|
||||
|
||||
from turnstone.core.adapters.interactive_adapter import InteractiveAdapter
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
|
||||
adapter = InteractiveAdapter(
|
||||
global_queue=queue.Queue(maxsize=100),
|
||||
ui_factory=lambda ws: MagicMock(),
|
||||
session_factory=session_factory,
|
||||
)
|
||||
return SessionManager(adapter, storage=MagicMock(), max_active=10, event_emitter=adapter)
|
||||
|
||||
|
||||
class TestWorkstreamModelParam:
|
||||
def test_create_with_model(self) -> None:
|
||||
"""SessionManager.create passes model_alias to session_factory."""
|
||||
"""WorkstreamManager.create passes model_alias to session_factory."""
|
||||
from turnstone.core.workstream import WorkstreamManager
|
||||
|
||||
captured_alias = None
|
||||
|
||||
def factory(
|
||||
@@ -1296,8 +1221,8 @@ class TestWorkstreamModelParam:
|
||||
mock_session.ws_id = "test123"
|
||||
return mock_session
|
||||
|
||||
mgr = _make_manager(factory)
|
||||
mgr.create(user_id="", name="test", model="openai")
|
||||
mgr = WorkstreamManager(factory)
|
||||
mgr.create(name="test", model="openai")
|
||||
assert captured_alias == "openai"
|
||||
|
||||
def test_create_without_model(self) -> None:
|
||||
@@ -1312,8 +1237,10 @@ class TestWorkstreamModelParam:
|
||||
mock_session.ws_id = "test123"
|
||||
return mock_session
|
||||
|
||||
mgr = _make_manager(factory)
|
||||
mgr.create(user_id="", name="test")
|
||||
from turnstone.core.workstream import WorkstreamManager
|
||||
|
||||
mgr = WorkstreamManager(factory)
|
||||
mgr.create(name="test")
|
||||
assert captured_alias is None
|
||||
|
||||
|
||||
|
||||
@@ -5,20 +5,8 @@ from __future__ import annotations
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core import node_info
|
||||
from turnstone.core.node_info import (
|
||||
_collect_interfaces,
|
||||
_detect_aws_metadata,
|
||||
_detect_azure_metadata,
|
||||
_detect_cloud_metadata,
|
||||
_detect_cloud_provider_from_dmi,
|
||||
_detect_cpu_model,
|
||||
_detect_gcp_metadata,
|
||||
_detect_gpus,
|
||||
_detect_memory_gb,
|
||||
_imds_field,
|
||||
_is_loopback_or_link_local,
|
||||
collect_node_info,
|
||||
)
|
||||
@@ -147,669 +135,3 @@ class TestIsLoopbackOrLinkLocal:
|
||||
assert _is_loopback_or_link_local("10.0.0.5") is False
|
||||
assert _is_loopback_or_link_local("192.168.1.1") is False
|
||||
assert _is_loopback_or_link_local("2001:db8::1") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernel-interface helpers — capability detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _seed_drm_layout(tmp_path, cards):
|
||||
"""Build a fake ``/sys/class/drm`` layout under ``tmp_path``.
|
||||
|
||||
``cards`` is a list of ``(name, vendor_id, device_id)`` tuples.
|
||||
Use ``vendor_id=None`` to skip writing the vendor file (simulates
|
||||
a permission/missing-attr failure that the detector must skip
|
||||
cleanly). Returns the DRM root path.
|
||||
"""
|
||||
drm = tmp_path / "drm"
|
||||
drm.mkdir()
|
||||
for name, vendor_id, device_id in cards:
|
||||
device_dir = drm / name / "device"
|
||||
device_dir.mkdir(parents=True)
|
||||
if vendor_id is not None:
|
||||
(device_dir / "vendor").write_text(vendor_id + "\n")
|
||||
if device_id is not None:
|
||||
(device_dir / "device").write_text(device_id + "\n")
|
||||
return str(drm)
|
||||
|
||||
|
||||
class TestDetectGPUs:
|
||||
"""Sysfs-DRM enumeration — vendor-agnostic, no userspace binary."""
|
||||
|
||||
def test_returns_empty_when_drm_dir_missing(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", "/nonexistent/path/that/should/not/exist")
|
||||
assert _detect_gpus() == []
|
||||
|
||||
def test_returns_empty_when_no_card_dirs(self, tmp_path, monkeypatch):
|
||||
# Empty /sys/class/drm — no GPUs registered.
|
||||
drm = tmp_path / "drm"
|
||||
drm.mkdir()
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", str(drm))
|
||||
assert _detect_gpus() == []
|
||||
|
||||
def test_detects_nvidia_gpu(self, tmp_path, monkeypatch):
|
||||
drm_dir = _seed_drm_layout(tmp_path, [("card0", "0x10de", "0x2330")])
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
gpus = _detect_gpus()
|
||||
assert len(gpus) == 1
|
||||
assert gpus[0] == {
|
||||
"index": "0",
|
||||
"vendor": "nvidia",
|
||||
"pci_vendor": "0x10de",
|
||||
"pci_device": "0x2330",
|
||||
}
|
||||
|
||||
def test_detects_amd_gpu(self, tmp_path, monkeypatch):
|
||||
drm_dir = _seed_drm_layout(tmp_path, [("card0", "0x1002", "0x74a1")])
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
gpus = _detect_gpus()
|
||||
assert len(gpus) == 1
|
||||
assert gpus[0]["vendor"] == "amd"
|
||||
|
||||
def test_detects_intel_gpu(self, tmp_path, monkeypatch):
|
||||
drm_dir = _seed_drm_layout(tmp_path, [("card0", "0x8086", "0x56a0")])
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
gpus = _detect_gpus()
|
||||
assert gpus[0]["vendor"] == "intel"
|
||||
|
||||
def test_unknown_vendor_id_is_filtered_out(self, tmp_path, monkeypatch):
|
||||
"""A DRM ``cardN`` whose PCI vendor isn't in the GPU
|
||||
allow-list (Hyper-V synthetic 0x1414, AWS Nitro VGA, QEMU
|
||||
virtio-gpu, etc.) MUST NOT count as a GPU. Counting them
|
||||
mis-labels CPU-only VMs as GPU nodes — observed on a CI
|
||||
runner."""
|
||||
drm_dir = _seed_drm_layout(tmp_path, [("card0", "0xdead", "0xbeef")])
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
assert _detect_gpus() == []
|
||||
|
||||
def test_hyper_v_synthetic_adapter_is_filtered_out(self, tmp_path, monkeypatch):
|
||||
"""Specific regression: Hyper-V's synthetic display adapter
|
||||
(vendor 0x1414, device 0x06) registers a ``/sys/class/drm/
|
||||
card0`` entry on Linux but is NOT a compute GPU. A CI
|
||||
runner reproduced this and came back with ``gpu_count=1``
|
||||
before the vendor allow-list filter."""
|
||||
drm_dir = _seed_drm_layout(tmp_path, [("card0", "0x1414", "0x06")])
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
assert _detect_gpus() == []
|
||||
|
||||
def test_mixed_known_and_unknown_keeps_only_known(self, tmp_path, monkeypatch):
|
||||
"""A node with a real GPU (NVIDIA) AND a synthetic display
|
||||
adapter (Hyper-V) only counts the real GPU."""
|
||||
drm_dir = _seed_drm_layout(
|
||||
tmp_path,
|
||||
[
|
||||
("card0", "0x1414", "0x06"), # Hyper-V synthetic
|
||||
("card1", "0x10de", "0x2330"), # NVIDIA H100
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
gpus = _detect_gpus()
|
||||
assert len(gpus) == 1
|
||||
assert gpus[0]["vendor"] == "nvidia"
|
||||
assert gpus[0]["index"] == "1"
|
||||
|
||||
def test_skips_render_nodes(self, tmp_path, monkeypatch):
|
||||
"""``renderD*`` nodes are per-card render-only interfaces that
|
||||
share the same physical device as a ``cardN`` entry; counting
|
||||
them would double the GPU count. The card-name regex
|
||||
excludes them."""
|
||||
drm = tmp_path / "drm"
|
||||
drm.mkdir()
|
||||
for name in ("card0", "renderD128"):
|
||||
device = drm / name / "device"
|
||||
device.mkdir(parents=True)
|
||||
(device / "vendor").write_text("0x10de")
|
||||
(device / "device").write_text("0x2330")
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", str(drm))
|
||||
gpus = _detect_gpus()
|
||||
assert len(gpus) == 1 # only card0, not renderD128
|
||||
|
||||
def test_multi_gpu_node(self, tmp_path, monkeypatch):
|
||||
drm_dir = _seed_drm_layout(
|
||||
tmp_path,
|
||||
[
|
||||
("card0", "0x10de", "0x2330"),
|
||||
("card1", "0x10de", "0x2330"),
|
||||
("card2", "0x10de", "0x2330"),
|
||||
("card3", "0x10de", "0x2330"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
gpus = _detect_gpus()
|
||||
assert len(gpus) == 4
|
||||
assert [g["index"] for g in gpus] == ["0", "1", "2", "3"]
|
||||
|
||||
def test_card_with_missing_vendor_is_skipped(self, tmp_path, monkeypatch):
|
||||
"""A card whose vendor file can't be read (permissions /
|
||||
partial sysfs) is silently skipped — the rest of the
|
||||
enumeration must still complete."""
|
||||
drm = tmp_path / "drm"
|
||||
drm.mkdir()
|
||||
# card0 has no vendor file; card1 is well-formed.
|
||||
(drm / "card0" / "device").mkdir(parents=True)
|
||||
good = drm / "card1" / "device"
|
||||
good.mkdir(parents=True)
|
||||
(good / "vendor").write_text("0x10de")
|
||||
(good / "device").write_text("0x2330")
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", str(drm))
|
||||
gpus = _detect_gpus()
|
||||
assert len(gpus) == 1
|
||||
assert gpus[0]["index"] == "1"
|
||||
|
||||
|
||||
class TestDetectMemoryGB:
|
||||
def test_parses_meminfo(self, tmp_path, monkeypatch):
|
||||
meminfo = tmp_path / "meminfo"
|
||||
# 32 GiB = 32 * 1024 * 1024 KiB = 33554432 KiB
|
||||
meminfo.write_text(
|
||||
"MemTotal: 33554432 kB\n"
|
||||
"MemFree: 5000000 kB\n"
|
||||
"MemAvailable: 28000000 kB\n"
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_MEMINFO_PATH", str(meminfo))
|
||||
assert _detect_memory_gb() == 32
|
||||
|
||||
def test_rounds_down(self, tmp_path, monkeypatch):
|
||||
"""31.5 GiB worth of KiB rounds down to 31 — operators that
|
||||
write ``filters={"memory_gb": 32}`` shouldn't match a node
|
||||
that's actually 31.5."""
|
||||
meminfo = tmp_path / "meminfo"
|
||||
# 31.5 GiB = 31.5 * 1024 * 1024 = 33030144 KiB
|
||||
meminfo.write_text(f"MemTotal: {31 * 1024 * 1024 + 512 * 1024} kB\n")
|
||||
monkeypatch.setattr(node_info, "_MEMINFO_PATH", str(meminfo))
|
||||
assert _detect_memory_gb() == 31
|
||||
|
||||
def test_returns_none_when_meminfo_missing(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_MEMINFO_PATH", "/nonexistent/meminfo")
|
||||
assert _detect_memory_gb() is None
|
||||
|
||||
def test_returns_none_when_no_memtotal_line(self, tmp_path, monkeypatch):
|
||||
meminfo = tmp_path / "meminfo"
|
||||
meminfo.write_text("MemFree: 5000000 kB\n") # no MemTotal
|
||||
monkeypatch.setattr(node_info, "_MEMINFO_PATH", str(meminfo))
|
||||
assert _detect_memory_gb() is None
|
||||
|
||||
|
||||
class TestDetectCPUModel:
|
||||
def test_parses_intel_brand(self, tmp_path, monkeypatch):
|
||||
cpuinfo = tmp_path / "cpuinfo"
|
||||
cpuinfo.write_text(
|
||||
"processor\t: 0\n"
|
||||
"model name\t: Intel(R) Xeon(R) Platinum 8488C\n"
|
||||
"cpu MHz\t\t: 2400.000\n"
|
||||
"processor\t: 1\n"
|
||||
"model name\t: Intel(R) Xeon(R) Platinum 8488C\n"
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_CPUINFO_PATH", str(cpuinfo))
|
||||
assert _detect_cpu_model() == "Intel(R) Xeon(R) Platinum 8488C"
|
||||
|
||||
def test_parses_amd_brand(self, tmp_path, monkeypatch):
|
||||
cpuinfo = tmp_path / "cpuinfo"
|
||||
cpuinfo.write_text("model name\t: AMD EPYC 9654 96-Core Processor\n")
|
||||
monkeypatch.setattr(node_info, "_CPUINFO_PATH", str(cpuinfo))
|
||||
assert _detect_cpu_model() == "AMD EPYC 9654 96-Core Processor"
|
||||
|
||||
def test_returns_none_on_arm_with_no_model_name(self, tmp_path, monkeypatch):
|
||||
"""ARM cpuinfo uses ``Hardware`` / ``Processor`` instead of
|
||||
``model name``; we return None and operators set ``cpu_model``
|
||||
in [metadata] config to taste."""
|
||||
cpuinfo = tmp_path / "cpuinfo"
|
||||
cpuinfo.write_text("Hardware\t: Apple M1\nProcessor\t: ARMv8\n")
|
||||
monkeypatch.setattr(node_info, "_CPUINFO_PATH", str(cpuinfo))
|
||||
assert _detect_cpu_model() is None
|
||||
|
||||
def test_returns_none_when_cpuinfo_missing(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_CPUINFO_PATH", "/nonexistent/cpuinfo")
|
||||
assert _detect_cpu_model() is None
|
||||
|
||||
|
||||
def _seed_dmi_layout(tmp_path, fields):
|
||||
"""Build a fake /sys/class/dmi/id with given key→value text files."""
|
||||
dmi = tmp_path / "dmi"
|
||||
dmi.mkdir()
|
||||
for key, value in fields.items():
|
||||
(dmi / key).write_text(value + "\n")
|
||||
return str(dmi)
|
||||
|
||||
|
||||
class TestDetectCloudProviderFromDMI:
|
||||
"""DMI-based cloud-provider detection — pure kernel interface."""
|
||||
|
||||
def test_aws_via_sys_vendor(self, tmp_path, monkeypatch):
|
||||
dmi = _seed_dmi_layout(tmp_path, {"sys_vendor": "Amazon EC2"})
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", dmi)
|
||||
assert _detect_cloud_provider_from_dmi() == "aws"
|
||||
|
||||
def test_aws_via_bios_vendor(self, tmp_path, monkeypatch):
|
||||
"""Older Nitro instances set bios_vendor instead of sys_vendor."""
|
||||
dmi = _seed_dmi_layout(
|
||||
tmp_path,
|
||||
{"sys_vendor": "Xen", "bios_vendor": "Amazon EC2"},
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", dmi)
|
||||
assert _detect_cloud_provider_from_dmi() == "aws"
|
||||
|
||||
def test_gcp_via_sys_vendor(self, tmp_path, monkeypatch):
|
||||
dmi = _seed_dmi_layout(
|
||||
tmp_path,
|
||||
{"sys_vendor": "Google", "product_name": "Google Compute Engine"},
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", dmi)
|
||||
assert _detect_cloud_provider_from_dmi() == "gcp"
|
||||
|
||||
def test_azure_via_chassis_asset_tag(self, tmp_path, monkeypatch):
|
||||
"""The chassis_asset_tag prefix distinguishes Azure VMs from
|
||||
plain Microsoft Hyper-V on baremetal — same sys_vendor, but
|
||||
only Azure VMs carry the well-known asset tag."""
|
||||
dmi = _seed_dmi_layout(
|
||||
tmp_path,
|
||||
{
|
||||
"sys_vendor": "Microsoft Corporation",
|
||||
"chassis_asset_tag": "7783-7084-3265-9085-8269-3286-77",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", dmi)
|
||||
assert _detect_cloud_provider_from_dmi() == "azure"
|
||||
|
||||
def test_microsoft_without_azure_tag_is_unknown(self, tmp_path, monkeypatch):
|
||||
"""Plain Hyper-V on baremetal — Microsoft sys_vendor but no
|
||||
Azure asset tag. Must not auto-detect as azure."""
|
||||
dmi = _seed_dmi_layout(
|
||||
tmp_path,
|
||||
{
|
||||
"sys_vendor": "Microsoft Corporation",
|
||||
"chassis_asset_tag": "Default string",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", dmi)
|
||||
assert _detect_cloud_provider_from_dmi() == "unknown"
|
||||
|
||||
def test_baremetal_is_unknown(self, tmp_path, monkeypatch):
|
||||
dmi = _seed_dmi_layout(tmp_path, {"sys_vendor": "Dell Inc.", "bios_vendor": "Dell Inc."})
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", dmi)
|
||||
assert _detect_cloud_provider_from_dmi() == "unknown"
|
||||
|
||||
def test_missing_dmi_dir_is_unknown(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_DMI_DIR", "/nonexistent/dmi")
|
||||
assert _detect_cloud_provider_from_dmi() == "unknown"
|
||||
|
||||
|
||||
class TestIMDSDetectors:
|
||||
"""Vendor-specific IMDS parsers — exercise the body-shape parsing
|
||||
without making real network calls."""
|
||||
|
||||
def test_aws_imds_v2_token_failure(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: None)
|
||||
assert _detect_aws_metadata() == {}
|
||||
|
||||
def test_aws_imds_parses_identity_doc(self, monkeypatch):
|
||||
responses = iter(
|
||||
[
|
||||
"TOKEN-ABCD", # PUT /api/token
|
||||
json.dumps(
|
||||
{
|
||||
"region": "us-east-1",
|
||||
"availabilityZone": "us-east-1a",
|
||||
"instanceType": "p5.48xlarge",
|
||||
"instanceId": "i-0123456789abcdef0",
|
||||
}
|
||||
), # GET /dynamic/instance-identity/document
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: next(responses))
|
||||
result = _detect_aws_metadata()
|
||||
assert result == {
|
||||
"cloud_region": "us-east-1",
|
||||
"cloud_zone": "us-east-1a",
|
||||
"cloud_instance_type": "p5.48xlarge",
|
||||
"cloud_instance_id": "i-0123456789abcdef0",
|
||||
}
|
||||
|
||||
def test_aws_malformed_identity_doc_returns_empty(self, monkeypatch):
|
||||
responses = iter(["TOKEN-ABCD", "not-json"])
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: next(responses))
|
||||
assert _detect_aws_metadata() == {}
|
||||
|
||||
def test_gcp_zone_parsing(self, monkeypatch):
|
||||
# GCP returns paths like "projects/12345/zones/us-east1-a";
|
||||
# we surface the tail and derive region by chopping the
|
||||
# trailing "-a" letter.
|
||||
responses = {
|
||||
"zone": "projects/12345/zones/us-east1-a",
|
||||
"machine-type": "projects/12345/machineTypes/n1-standard-4",
|
||||
"id": "9876543210",
|
||||
}
|
||||
|
||||
def fake(url, headers=None, **_kw):
|
||||
for key, body in responses.items():
|
||||
if url.endswith("/" + key):
|
||||
return body
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(node_info, "_imds_get", fake)
|
||||
result = _detect_gcp_metadata()
|
||||
assert result["cloud_zone"] == "us-east1-a"
|
||||
assert result["cloud_region"] == "us-east1"
|
||||
assert result["cloud_instance_type"] == "n1-standard-4"
|
||||
assert result["cloud_instance_id"] == "9876543210"
|
||||
|
||||
def test_gcp_no_zone_returns_empty(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: None)
|
||||
assert _detect_gcp_metadata() == {}
|
||||
|
||||
def test_azure_compute_block_parsing(self, monkeypatch):
|
||||
body = json.dumps(
|
||||
{
|
||||
"compute": {
|
||||
"location": "eastus",
|
||||
"zone": "1",
|
||||
"vmSize": "Standard_NC24ads_A100_v4",
|
||||
"vmId": "abcd1234-...",
|
||||
}
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: body)
|
||||
result = _detect_azure_metadata()
|
||||
assert result == {
|
||||
"cloud_region": "eastus",
|
||||
"cloud_zone": "1",
|
||||
"cloud_instance_type": "Standard_NC24ads_A100_v4",
|
||||
"cloud_instance_id": "abcd1234-...",
|
||||
}
|
||||
|
||||
def test_azure_missing_compute_block_returns_empty(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: json.dumps({}))
|
||||
assert _detect_azure_metadata() == {}
|
||||
|
||||
|
||||
class TestDetectCloudMetadata:
|
||||
"""End-to-end cloud metadata detection: DMI gate + IMDS probe."""
|
||||
|
||||
def test_baremetal_skips_imds(self, monkeypatch):
|
||||
"""No DMI cloud signal → no IMDS probe → empty result, no
|
||||
startup latency cost. This is the property we wanted from
|
||||
the kernel-interface refactor."""
|
||||
called = {"imds": 0}
|
||||
|
||||
def _spy(*args, **kwargs):
|
||||
called["imds"] += 1
|
||||
return "should-never-be-called"
|
||||
|
||||
monkeypatch.setattr(node_info, "_detect_cloud_provider_from_dmi", lambda: "unknown")
|
||||
monkeypatch.setattr(node_info, "_imds_get", _spy)
|
||||
assert _detect_cloud_metadata() == {}
|
||||
assert called["imds"] == 0
|
||||
|
||||
def test_aws_detection_path(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_detect_cloud_provider_from_dmi", lambda: "aws")
|
||||
monkeypatch.setattr(
|
||||
node_info,
|
||||
"_detect_aws_metadata",
|
||||
lambda: {"cloud_region": "us-west-2", "cloud_instance_type": "p4d.24xlarge"},
|
||||
)
|
||||
result = _detect_cloud_metadata()
|
||||
assert result["cloud_provider"] == "aws"
|
||||
assert result["cloud_region"] == "us-west-2"
|
||||
assert result["cloud_instance_type"] == "p4d.24xlarge"
|
||||
|
||||
def test_imds_probe_failure_still_surfaces_provider(self, monkeypatch):
|
||||
"""If DMI says we're on AWS but IMDS times out, we still
|
||||
surface ``cloud_provider=aws`` from DMI alone. Operators
|
||||
can route on provider even when region/instance-type
|
||||
couldn't be probed."""
|
||||
monkeypatch.setattr(node_info, "_detect_cloud_provider_from_dmi", lambda: "aws")
|
||||
monkeypatch.setattr(node_info, "_detect_aws_metadata", lambda: {})
|
||||
result = _detect_cloud_metadata()
|
||||
assert result == {"cloud_provider": "aws"}
|
||||
|
||||
def test_opt_out_skips_imds_but_keeps_provider(self, monkeypatch):
|
||||
"""``TURNSTONE_AUTO_CLOUD_METADATA=0`` skips the network probe
|
||||
entirely. ``cloud_provider`` from DMI still populates because
|
||||
it's a kernel interface, not a network call."""
|
||||
monkeypatch.setenv("TURNSTONE_AUTO_CLOUD_METADATA", "0")
|
||||
monkeypatch.setattr(node_info, "_detect_cloud_provider_from_dmi", lambda: "gcp")
|
||||
|
||||
def _imds_should_not_run(*a, **kw):
|
||||
pytest.fail("IMDS probe must not run when TURNSTONE_AUTO_CLOUD_METADATA=0")
|
||||
|
||||
monkeypatch.setattr(node_info, "_imds_get", _imds_should_not_run)
|
||||
result = _detect_cloud_metadata()
|
||||
assert result == {"cloud_provider": "gcp"}
|
||||
|
||||
def test_imds_exception_does_not_propagate(self, monkeypatch):
|
||||
"""A buggy IMDS parser (raises unexpectedly) must not crash
|
||||
the collector — the ``except Exception`` wrapper inside
|
||||
``_detect_cloud_metadata`` swallows and logs."""
|
||||
monkeypatch.setattr(node_info, "_detect_cloud_provider_from_dmi", lambda: "azure")
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("simulated parser bug")
|
||||
|
||||
monkeypatch.setattr(node_info, "_detect_azure_metadata", _boom)
|
||||
result = _detect_cloud_metadata()
|
||||
# cloud_provider survives; region/zone are missing.
|
||||
assert result == {"cloud_provider": "azure"}
|
||||
|
||||
|
||||
class TestCollectNodeInfoCapabilityIntegration:
|
||||
"""End-to-end checks on the public ``collect_node_info`` entry
|
||||
point — confirms the new kernel-interface helpers wire up
|
||||
correctly and that one helper failing doesn't suppress the others."""
|
||||
|
||||
def test_gpu_keys_appear_when_gpus_detected(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
node_info,
|
||||
"_detect_gpus",
|
||||
lambda: [
|
||||
{"index": "0", "vendor": "nvidia", "pci_vendor": "0x10de", "pci_device": "0x2330"},
|
||||
],
|
||||
)
|
||||
info = collect_node_info()
|
||||
assert info["gpu_count"] == 1
|
||||
assert info["has_gpu"] is True
|
||||
assert info["gpu_vendors"] == ["nvidia"]
|
||||
assert info["gpu_has_nvidia"] is True
|
||||
assert info["gpus"][0]["pci_device"] == "0x2330"
|
||||
# Singular ``gpu_vendor`` is intentionally NOT exposed —
|
||||
# multi-vendor nodes would only be filterable under one
|
||||
# vendor, hiding them from the other; per-vendor booleans
|
||||
# avoid the false-negative.
|
||||
assert "gpu_vendor" not in info
|
||||
|
||||
def test_gpu_keys_absent_when_no_gpus(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_detect_gpus", lambda: [])
|
||||
info = collect_node_info()
|
||||
for k in ("gpu_count", "gpu_vendors", "gpus", "has_gpu"):
|
||||
assert k not in info
|
||||
# No spurious ``gpu_has_*`` keys when there are no GPUs.
|
||||
assert not any(k.startswith("gpu_has_") for k in info)
|
||||
|
||||
def test_multi_vendor_node_filterable_under_each_vendor(self, monkeypatch):
|
||||
"""A mixed AMD+NVIDIA node MUST be filterable under both
|
||||
vendors. Pre-fix the singular ``gpu_vendor`` flat key was
|
||||
set to ``vendors[0]`` (alphabetical first = ``amd``) and
|
||||
``filters={"gpu_vendor": "nvidia"}`` would mismatch the
|
||||
NVIDIA card on the bus. Per-vendor booleans avoid the
|
||||
false-negative entirely."""
|
||||
monkeypatch.setattr(
|
||||
node_info,
|
||||
"_detect_gpus",
|
||||
lambda: [
|
||||
{"index": "0", "vendor": "amd", "pci_vendor": "0x1002", "pci_device": "0x74a1"},
|
||||
{"index": "1", "vendor": "nvidia", "pci_vendor": "0x10de", "pci_device": "0x2330"},
|
||||
],
|
||||
)
|
||||
info = collect_node_info()
|
||||
# Both per-vendor flags True — filter under EITHER vendor matches.
|
||||
assert info["gpu_has_amd"] is True
|
||||
assert info["gpu_has_nvidia"] is True
|
||||
# Sorted unique vendors carry the full list for tooling that
|
||||
# wants the set.
|
||||
assert info["gpu_vendors"] == ["amd", "nvidia"]
|
||||
assert info["gpu_count"] == 2
|
||||
assert info["has_gpu"] is True
|
||||
|
||||
def test_memory_key_appears(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_detect_memory_gb", lambda: 256)
|
||||
info = collect_node_info()
|
||||
assert info["memory_gb"] == 256
|
||||
|
||||
def test_memory_zero_omitted(self, monkeypatch):
|
||||
"""A reading of 0 GiB is degenerate — likely a parse error
|
||||
rather than a real zero-RAM machine. Skip the key rather
|
||||
than advertise a false value."""
|
||||
monkeypatch.setattr(node_info, "_detect_memory_gb", lambda: 0)
|
||||
info = collect_node_info()
|
||||
assert "memory_gb" not in info
|
||||
|
||||
def test_cpu_model_key_appears(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_detect_cpu_model", lambda: "AMD EPYC 9654")
|
||||
info = collect_node_info()
|
||||
assert info["cpu_model"] == "AMD EPYC 9654"
|
||||
|
||||
def test_cloud_keys_merged(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
node_info,
|
||||
"_detect_cloud_metadata",
|
||||
lambda: {
|
||||
"cloud_provider": "aws",
|
||||
"cloud_region": "us-east-1",
|
||||
"cloud_instance_type": "p5.48xlarge",
|
||||
},
|
||||
)
|
||||
info = collect_node_info()
|
||||
assert info["cloud_provider"] == "aws"
|
||||
assert info["cloud_region"] == "us-east-1"
|
||||
assert info["cloud_instance_type"] == "p5.48xlarge"
|
||||
|
||||
def test_one_capability_failure_does_not_block_others(self, monkeypatch):
|
||||
"""If GPU detection raises, memory + cpu + cloud detection
|
||||
must still run. Mirrors the existing per-field-failsafe
|
||||
contract on the basic fields."""
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("simulated DRM failure")
|
||||
|
||||
monkeypatch.setattr(node_info, "_detect_gpus", _boom)
|
||||
monkeypatch.setattr(node_info, "_detect_memory_gb", lambda: 64)
|
||||
monkeypatch.setattr(node_info, "_detect_cpu_model", lambda: "AMD EPYC 9654")
|
||||
info = collect_node_info()
|
||||
assert "gpu_count" not in info
|
||||
assert info["memory_gb"] == 64
|
||||
assert info["cpu_model"] == "AMD EPYC 9654"
|
||||
|
||||
def test_synthetic_display_adapter_does_not_register_as_gpu(self, tmp_path, monkeypatch):
|
||||
"""End-to-end: a Hyper-V synthetic display adapter on the
|
||||
host's /sys/class/drm doesn't reach ``collect_node_info``'s
|
||||
GPU surface at all. The vendor allow-list filter in
|
||||
``_detect_gpus`` drops it before it gets to ``has_gpu`` /
|
||||
``gpu_count`` / ``gpu_has_*``. Pre-fix this would mis-label
|
||||
a CPU-only Hyper-V VM as a GPU node."""
|
||||
drm_dir = _seed_drm_layout(tmp_path, [("card0", "0x1414", "0x06")])
|
||||
monkeypatch.setattr(node_info, "_DRM_DIR", drm_dir)
|
||||
info = collect_node_info()
|
||||
for k in ("gpu_count", "has_gpu", "gpus", "gpu_vendors"):
|
||||
assert k not in info
|
||||
assert not any(k.startswith("gpu_has_") for k in info)
|
||||
|
||||
|
||||
class TestIMDSFieldSanitiser:
|
||||
"""``_imds_field`` strips control chars + length-caps each
|
||||
persisted value. Defense-in-depth against an attacker-controlled
|
||||
IMDS responder injecting prompt-payload bytes into coord LLM
|
||||
context via ``list_nodes``."""
|
||||
|
||||
def test_passes_clean_string_through(self):
|
||||
assert _imds_field("us-east-1") == "us-east-1"
|
||||
|
||||
def test_strips_control_characters(self):
|
||||
# Newline + NUL would otherwise survive into list_nodes
|
||||
# output and could break parsing or inject content into
|
||||
# downstream renderers.
|
||||
out = _imds_field("us-east-1\n\x00 injected")
|
||||
assert "\n" not in (out or "")
|
||||
assert "\x00" not in (out or "")
|
||||
assert out == "us-east-1 injected"
|
||||
|
||||
def test_caps_length(self):
|
||||
from turnstone.core.node_info import _IMDS_MAX_FIELD_CHARS
|
||||
|
||||
out = _imds_field("X" * (_IMDS_MAX_FIELD_CHARS * 4))
|
||||
assert out is not None
|
||||
assert len(out) == _IMDS_MAX_FIELD_CHARS
|
||||
|
||||
def test_returns_none_for_non_string(self):
|
||||
assert _imds_field(None) is None
|
||||
assert _imds_field(42) is None
|
||||
assert _imds_field(["us-east-1"]) is None
|
||||
|
||||
def test_returns_none_for_empty_or_whitespace(self):
|
||||
assert _imds_field("") is None
|
||||
assert _imds_field(" ") is None
|
||||
|
||||
|
||||
class TestIMDSResponseHardening:
|
||||
"""Regression guards on the AWS / Azure non-dict-JSON paths and
|
||||
the GCP hostname → IP-literal switch."""
|
||||
|
||||
def test_aws_handles_non_dict_json_without_raising(self, monkeypatch):
|
||||
"""If a hostile/misbehaving IMDS returns a JSON list rather
|
||||
than the documented identity-document object, the previous
|
||||
shape would AttributeError on ``doc.get(src)``. The
|
||||
``isinstance(doc, dict)`` guard makes this a clean miss."""
|
||||
responses = iter(["TOKEN-ABCD", "[1, 2, 3]"])
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: next(responses))
|
||||
# Must not raise.
|
||||
assert _detect_aws_metadata() == {}
|
||||
|
||||
def test_aws_handles_scalar_json_without_raising(self, monkeypatch):
|
||||
responses = iter(["TOKEN-ABCD", "42"])
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: next(responses))
|
||||
assert _detect_aws_metadata() == {}
|
||||
|
||||
def test_azure_handles_non_dict_json_without_raising(self, monkeypatch):
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: '["not-an-object"]')
|
||||
# Must not raise.
|
||||
assert _detect_azure_metadata() == {}
|
||||
|
||||
def test_gcp_uses_link_local_ip_literal(self, monkeypatch):
|
||||
"""The GCP probe must target ``169.254.169.254`` directly so
|
||||
a host with attacker-controlled DNS can't redirect the probe
|
||||
via ``metadata.google.internal``. Pin the URL prefix."""
|
||||
called_urls: list[str] = []
|
||||
|
||||
def _spy(url, *args, **kwargs):
|
||||
called_urls.append(url)
|
||||
return None # all probes fail; that's fine — we're inspecting URLs
|
||||
|
||||
monkeypatch.setattr(node_info, "_imds_get", _spy)
|
||||
_detect_gcp_metadata()
|
||||
assert called_urls, "GCP detector must issue at least one IMDS call"
|
||||
for url in called_urls:
|
||||
assert url.startswith("http://169.254.169.254/"), (
|
||||
f"GCP probe leaked through DNS-resolvable hostname: {url}"
|
||||
)
|
||||
|
||||
def test_imds_field_sanitises_aws_response(self, monkeypatch):
|
||||
"""End-to-end: a hostile IMDS response body with a control
|
||||
character lands sanitised in the AWS detector's output."""
|
||||
responses = iter(
|
||||
[
|
||||
"TOKEN-ABCD",
|
||||
json.dumps(
|
||||
{
|
||||
"region": "us-east-1\nrm -rf", # control char injection
|
||||
"instanceType": "p5.48xlarge",
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(node_info, "_imds_get", lambda *a, **kw: next(responses))
|
||||
result = _detect_aws_metadata()
|
||||
assert "\n" not in result["cloud_region"]
|
||||
# Sanitiser preserves the leading meaningful prefix, drops
|
||||
# the control character. Trailing content survives stripped
|
||||
# of control chars.
|
||||
assert "us-east-1" in result["cloud_region"]
|
||||
assert "rm -rf" in result["cloud_region"] # text still there, just newline-free
|
||||
|
||||
+5
-84
@@ -26,36 +26,22 @@ class TestServerSpec:
|
||||
paths = set(spec["paths"].keys())
|
||||
expected = {
|
||||
"/v1/api/workstreams",
|
||||
"/v1/api/workstreams/{ws_id}",
|
||||
"/v1/api/workstreams/{ws_id}/history",
|
||||
"/v1/api/workstreams/{ws_id}/send",
|
||||
"/v1/api/workstreams/{ws_id}/approve",
|
||||
"/v1/api/workstreams/{ws_id}/cancel",
|
||||
"/v1/api/workstreams/{ws_id}/close",
|
||||
"/v1/api/workstreams/{ws_id}/events",
|
||||
"/v1/api/dashboard",
|
||||
"/v1/api/workstreams/saved",
|
||||
"/v1/api/send",
|
||||
"/v1/api/approve",
|
||||
"/v1/api/plan",
|
||||
"/v1/api/command",
|
||||
"/v1/api/events",
|
||||
"/v1/api/events/global",
|
||||
"/v1/api/workstreams/new",
|
||||
"/v1/api/workstreams/close",
|
||||
"/v1/api/auth/login",
|
||||
"/v1/api/auth/logout",
|
||||
"/health",
|
||||
}
|
||||
assert expected.issubset(paths), f"Missing: {expected - paths}"
|
||||
|
||||
def test_workstream_history_has_limit_query_param(self):
|
||||
"""Mirror of the coord-side history limit param test — server now
|
||||
exposes the same endpoint via the lifted factory."""
|
||||
from turnstone.api.server_spec import build_server_spec
|
||||
|
||||
spec = build_server_spec()
|
||||
op = spec["paths"]["/v1/api/workstreams/{ws_id}/history"]["get"]
|
||||
param_names = [p["name"] for p in op.get("parameters", [])]
|
||||
assert "ws_id" in param_names
|
||||
assert "limit" in param_names
|
||||
|
||||
def test_schemas_not_empty(self):
|
||||
from turnstone.api.server_spec import build_server_spec
|
||||
|
||||
@@ -73,7 +59,7 @@ class TestServerSpec:
|
||||
from turnstone.api.server_spec import build_server_spec
|
||||
|
||||
spec = build_server_spec()
|
||||
send = spec["paths"]["/v1/api/workstreams/{ws_id}/send"]["post"]
|
||||
send = spec["paths"]["/v1/api/send"]["post"]
|
||||
assert "requestBody" in send
|
||||
assert "application/json" in send["requestBody"]["content"]
|
||||
|
||||
@@ -125,68 +111,3 @@ class TestConsoleSpec:
|
||||
param_names = [p["name"] for p in nodes["parameters"]]
|
||||
assert "sort" in param_names
|
||||
assert "limit" in param_names
|
||||
|
||||
def test_has_coordinator_endpoints(self):
|
||||
"""Phase 1-3 coordinator routes must appear in the OpenAPI catalog —
|
||||
the spec was missing every coordinator endpoint except ``/open``,
|
||||
so SDK consumers and operators couldn't discover the surface
|
||||
from /docs. Pin the full set so a future regression that drops
|
||||
one fails loudly."""
|
||||
from turnstone.api.console_spec import build_console_spec
|
||||
|
||||
spec = build_console_spec()
|
||||
paths = set(spec["paths"].keys())
|
||||
expected = {
|
||||
"/v1/api/workstreams/new",
|
||||
"/v1/api/workstreams",
|
||||
"/v1/api/workstreams/{ws_id}",
|
||||
"/v1/api/workstreams/{ws_id}/open",
|
||||
"/v1/api/workstreams/{ws_id}/send",
|
||||
"/v1/api/workstreams/{ws_id}/approve",
|
||||
"/v1/api/workstreams/{ws_id}/cancel",
|
||||
"/v1/api/workstreams/{ws_id}/close",
|
||||
"/v1/api/workstreams/{ws_id}/events",
|
||||
"/v1/api/workstreams/{ws_id}/history",
|
||||
"/v1/api/workstreams/{ws_id}/children",
|
||||
"/v1/api/workstreams/{ws_id}/tasks",
|
||||
"/v1/api/cluster/ws/{ws_id}/detail",
|
||||
}
|
||||
assert expected.issubset(paths), f"Missing: {expected - paths}"
|
||||
|
||||
def test_coordinator_create_has_request_body_and_200(self):
|
||||
"""Coordinator create returns 200 and accepts a body.
|
||||
|
||||
Pre-1.5.0 this returned 201 (REST-strict for create); the lifted
|
||||
``make_create_handler`` factory converges on 200 across both
|
||||
kinds for response-shape parity with every other shared verb.
|
||||
"""
|
||||
from turnstone.api.console_spec import build_console_spec
|
||||
|
||||
spec = build_console_spec()
|
||||
op = spec["paths"]["/v1/api/workstreams/new"]["post"]
|
||||
assert "requestBody" in op
|
||||
assert "application/json" in op["requestBody"]["content"]
|
||||
assert "200" in op["responses"]
|
||||
|
||||
def test_coordinator_history_has_limit_query_param(self):
|
||||
from turnstone.api.console_spec import build_console_spec
|
||||
|
||||
spec = build_console_spec()
|
||||
op = spec["paths"]["/v1/api/workstreams/{ws_id}/history"]["get"]
|
||||
param_names = [p["name"] for p in op.get("parameters", [])]
|
||||
assert "ws_id" in param_names # auto-added from path
|
||||
assert "limit" in param_names
|
||||
|
||||
def test_coordinator_endpoints_share_tag(self):
|
||||
"""All coordinator endpoints (including the cluster-inspect one)
|
||||
live under the same OpenAPI tag so /docs groups them together."""
|
||||
from turnstone.api.console_spec import build_console_spec
|
||||
|
||||
spec = build_console_spec()
|
||||
coord_paths = [p for p in spec["paths"] if "/coordinator" in p]
|
||||
coord_paths.append("/v1/api/cluster/ws/{ws_id}/detail")
|
||||
for path in coord_paths:
|
||||
for op in spec["paths"][path].values():
|
||||
assert "Coordinator" in op.get("tags", []), (
|
||||
f"{path} missing Coordinator tag (tags={op.get('tags')})"
|
||||
)
|
||||
|
||||
@@ -1,502 +0,0 @@
|
||||
"""Tests for the phase-6 polish endpoints (#q-1).
|
||||
|
||||
Covers:
|
||||
|
||||
- GET /v1/api/cluster/ws/live — bulk live-block fetch (admin.cluster.inspect).
|
||||
- GET /v1/api/workstreams/{ws_id}/metrics — per-coordinator health snapshot.
|
||||
|
||||
Both endpoints ride on the same test harness as
|
||||
``test_coordinator_endpoints.py`` — a minimal Starlette app with an
|
||||
auth-injecting middleware, TestClient + MockTransport for the
|
||||
upstream node fetches.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from tests._coord_test_helpers import (
|
||||
_AuthMiddleware,
|
||||
_build_mgr,
|
||||
_fake_registry,
|
||||
_FakeConfigStore,
|
||||
)
|
||||
from turnstone.console.server import (
|
||||
cluster_ws_live_bulk,
|
||||
coordinator_metrics,
|
||||
)
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "phase6.db"))
|
||||
|
||||
|
||||
def _make_client(storage, *, coord_mgr=None) -> TestClient:
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route("/v1/api/cluster/ws/live", cluster_ws_live_bulk, methods=["GET"]),
|
||||
Route(
|
||||
"/v1/api/workstreams/{ws_id}/metrics",
|
||||
coordinator_metrics,
|
||||
methods=["GET"],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_AuthMiddleware)],
|
||||
)
|
||||
app.state.coord_mgr = coord_mgr
|
||||
app.state.coord_adapter = coord_mgr._adapter if coord_mgr is not None else None
|
||||
app.state.config_store = _FakeConfigStore({"coordinator.model_alias": "gpt-4"})
|
||||
app.state.coord_registry = _fake_registry() if coord_mgr is not None else None
|
||||
app.state.coord_registry_error = "" if coord_mgr else "registry missing"
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "x" * 64
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _seed_workstream(
|
||||
storage: SQLiteBackend,
|
||||
*,
|
||||
ws_id: str,
|
||||
node_id: str,
|
||||
user_id: str = "user-1",
|
||||
kind: str = "interactive",
|
||||
state: str = "idle",
|
||||
parent_ws_id: str | None = None,
|
||||
created: str | None = None,
|
||||
) -> None:
|
||||
storage.register_workstream(
|
||||
ws_id,
|
||||
node_id=node_id,
|
||||
user_id=user_id,
|
||||
name=f"ws-{ws_id[:4]}",
|
||||
state=state,
|
||||
kind=kind,
|
||||
parent_ws_id=parent_ws_id,
|
||||
)
|
||||
if created is not None:
|
||||
# Override the created timestamp directly — register_workstream
|
||||
# stamps "now", so we need a second write to test the
|
||||
# spawns_last_hour boundary.
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._sqlite import workstreams
|
||||
|
||||
with storage._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(created=created)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /v1/api/cluster/ws/live — bulk live-block fetch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_ADMIN_HEADERS = {"X-Test-User": "user-1", "X-Test-Perms": "admin.cluster.inspect"}
|
||||
_OWNER_HEADERS = _ADMIN_HEADERS # same caller; permission grants inspect
|
||||
|
||||
|
||||
def test_bulk_live_requires_permission(storage):
|
||||
client = _make_client(storage, coord_mgr=_build_mgr(storage))
|
||||
resp = client.get(
|
||||
"/v1/api/cluster/ws/live?ids=" + "a" * 32,
|
||||
headers={"X-Test-User": "u", "X-Test-Perms": "read"},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
def test_bulk_live_empty_ids_returns_empty_body(storage):
|
||||
client = _make_client(storage, coord_mgr=_build_mgr(storage))
|
||||
resp = client.get("/v1/api/cluster/ws/live?ids=", headers=_ADMIN_HEADERS)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body == {"results": {}, "denied": [], "truncated": False}
|
||||
|
||||
|
||||
def test_bulk_live_strips_invalid_ids(storage):
|
||||
"""IDs failing the hex-regex are silently dropped; duplicates
|
||||
collapse."""
|
||||
client = _make_client(storage, coord_mgr=_build_mgr(storage))
|
||||
# NOT-HEX is invalid; the valid id is 32 chars hex but unknown to
|
||||
# storage → shows up as denied.
|
||||
resp = client.get(
|
||||
"/v1/api/cluster/ws/live?ids=NOT-HEX,NOT-HEX,," + ("a" * 32) + "," + ("a" * 32),
|
||||
headers=_ADMIN_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
# Invalid / empty / duplicate ids trimmed; only the one valid-but-
|
||||
# missing id is reported as denied.
|
||||
assert body["denied"] == ["a" * 32]
|
||||
assert body["results"] == {}
|
||||
|
||||
|
||||
def test_bulk_live_caps_ids_at_50(storage):
|
||||
"""Ids past the server-side cap truncate with truncated=true."""
|
||||
client = _make_client(storage, coord_mgr=_build_mgr(storage))
|
||||
# 60 fake ids → cap=50 keeps the first 50 (dedup preserves order).
|
||||
ids = ",".join(f"{i:064x}" for i in range(60))
|
||||
resp = client.get(
|
||||
"/v1/api/cluster/ws/live?ids=" + ids,
|
||||
headers=_ADMIN_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["truncated"] is True
|
||||
# All 50 kept ids resolve to 'denied' (no storage rows) — their
|
||||
# inclusion in the response proves the cap took the head 50.
|
||||
assert len(body["denied"]) == 50
|
||||
|
||||
|
||||
def test_bulk_live_admin_bypass_returns_live(storage):
|
||||
"""An admin user (holds admin.users or admin.roles, not just
|
||||
admin.cluster.inspect) bypasses tenancy and sees non-owned rows'
|
||||
live blocks. Coordinator live-block synthesis is in-process, so
|
||||
results is populated without any upstream node fetch."""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="other-user")
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
"/v1/api/cluster/ws/live?ids=" + ws.id,
|
||||
headers={
|
||||
"X-Test-User": "user-1",
|
||||
# admin.users grants the _is_admin bypass in addition to
|
||||
# admin.cluster.inspect for the endpoint itself.
|
||||
"X-Test-Perms": "admin.cluster.inspect,admin.users",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert ws.id in body["results"]
|
||||
assert body["denied"] == []
|
||||
|
||||
|
||||
def test_bulk_live_cluster_wide_visibility(storage):
|
||||
"""Trusted-team visibility: any ``admin.cluster.inspect`` caller
|
||||
sees every row in ``results``. ``denied`` is reserved for ids
|
||||
that don't correspond to a persisted workstream (no existence
|
||||
oracle for unknown ids)."""
|
||||
ws_id = "b" * 32
|
||||
_seed_workstream(storage, ws_id=ws_id, node_id="node-a", user_id="stranger")
|
||||
client = _make_client(storage, coord_mgr=_build_mgr(storage))
|
||||
resp = client.get(
|
||||
f"/v1/api/cluster/ws/live?ids={ws_id}",
|
||||
headers={"X-Test-User": "user-1", "X-Test-Perms": "admin.cluster.inspect"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert ws_id in body["results"]
|
||||
assert body["denied"] == []
|
||||
|
||||
|
||||
def test_bulk_live_unknown_ids_route_to_denied(storage):
|
||||
"""Unknown ids (not in storage) land in ``denied`` so the endpoint
|
||||
can't be used as an existence oracle."""
|
||||
ws_id = "c" * 32 # not seeded
|
||||
client = _make_client(storage, coord_mgr=_build_mgr(storage))
|
||||
resp = client.get(
|
||||
f"/v1/api/cluster/ws/live?ids={ws_id}",
|
||||
headers={"X-Test-User": "user-1", "X-Test-Perms": "admin.cluster.inspect"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["denied"] == [ws_id]
|
||||
assert body["results"] == {}
|
||||
|
||||
|
||||
def test_bulk_live_coordinator_row_uses_manager_snapshot(storage):
|
||||
"""A coordinator ws_id routes through _fetch_live_block's
|
||||
coordinator branch — live is populated from the in-process manager
|
||||
even though the pseudo-node has no /dashboard endpoint."""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/cluster/ws/live?ids={ws.id}",
|
||||
headers=_OWNER_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert ws.id in body["results"]
|
||||
live = body["results"][ws.id]
|
||||
assert live is not None
|
||||
assert "pending_approval" in live
|
||||
# New field always present on the wire — None when no approval
|
||||
# is pending so the JS can `key in row` without surprise.
|
||||
assert "pending_approval_detail" in live
|
||||
assert live["pending_approval_detail"] is None
|
||||
|
||||
|
||||
def test_bulk_live_coordinator_row_includes_pending_approval_detail(storage):
|
||||
"""When _pending_approval is set on a coord UI, the live block
|
||||
surfaces the merged items + judge_verdict payload through the
|
||||
coord-pseudo-node path. End-to-end equivalent of the dashboard
|
||||
test in test_server_authz, but for the console live-bulk
|
||||
endpoint that the coord tree UI actually consumes."""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
ws.ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{
|
||||
"call_id": "c-99",
|
||||
"header": "spawn_workstream",
|
||||
"preview": "{...}",
|
||||
"func_name": "spawn_workstream",
|
||||
"approval_label": "spawn_workstream",
|
||||
"needs_approval": True,
|
||||
}
|
||||
],
|
||||
"judge_pending": False,
|
||||
}
|
||||
ws.ui._llm_verdicts["c-99"] = {
|
||||
"recommendation": "approve",
|
||||
"risk_level": "low",
|
||||
"tier": "llm",
|
||||
}
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/cluster/ws/live?ids={ws.id}",
|
||||
headers=_OWNER_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
live = resp.json()["results"][ws.id]
|
||||
assert live["pending_approval"] is True # boolean derived flag
|
||||
detail = live["pending_approval_detail"]
|
||||
assert detail is not None
|
||||
assert detail["call_id"] == "c-99"
|
||||
assert detail["items"][0]["func_name"] == "spawn_workstream"
|
||||
assert detail["items"][0]["judge_verdict"]["recommendation"] == "approve"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /v1/api/workstreams/{ws_id}/metrics — per-coordinator health snapshot
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_METRICS_HEADERS = {"X-Test-User": "user-1", "X-Test-Perms": "admin.coordinator"}
|
||||
|
||||
|
||||
def test_metrics_requires_permission(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers={"X-Test-User": "user-1", "X-Test-Perms": "read"},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
def test_metrics_invalid_ws_id_400(storage):
|
||||
mgr = _build_mgr(storage)
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
"/v1/api/workstreams/NOT-HEX/metrics",
|
||||
headers=_METRICS_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_metrics_any_admin_coordinator_caller_can_read(storage):
|
||||
"""Trusted-team visibility: metrics are readable by any caller
|
||||
with ``admin.coordinator`` regardless of the coordinator owner."""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="stranger")
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers=_METRICS_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["ws_id"] == ws.id
|
||||
|
||||
|
||||
def test_metrics_empty_coordinator_defaults(storage):
|
||||
"""A freshly created coordinator with no spawns / no verdicts
|
||||
returns zero / empty defaults."""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers=_METRICS_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["ws_id"] == ws.id
|
||||
assert body["spawns_total"] == 0
|
||||
assert body["spawns_last_hour"] == 0
|
||||
assert body["child_state_counts"] == {}
|
||||
assert body["judge_fallback_rate"] == 0.0
|
||||
assert body["wait_completions"] == 0
|
||||
assert body["wait_timeouts"] == 0
|
||||
assert body["wait_avg_elapsed"] == 0.0
|
||||
|
||||
|
||||
def test_metrics_spawns_and_state_counts(storage):
|
||||
"""spawns_total counts ALL children (including closed); state
|
||||
histogram groups by current state. All children share the
|
||||
coordinator's owner so the non-admin tenant filter on the
|
||||
aggregate queries counts them all (see next test for the
|
||||
cross-tenant filter behaviour)."""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="aa" * 16,
|
||||
node_id="node-a",
|
||||
user_id="user-1",
|
||||
parent_ws_id=ws.id,
|
||||
state="idle",
|
||||
)
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="bb" * 16,
|
||||
node_id="node-a",
|
||||
user_id="user-1",
|
||||
parent_ws_id=ws.id,
|
||||
state="running",
|
||||
)
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="cc" * 16,
|
||||
node_id="node-a",
|
||||
user_id="user-1",
|
||||
parent_ws_id=ws.id,
|
||||
state="closed",
|
||||
)
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers=_METRICS_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["spawns_total"] == 3
|
||||
assert body["child_state_counts"] == {"idle": 1, "running": 1, "closed": 1}
|
||||
|
||||
|
||||
def test_metrics_cluster_wide_aggregates(storage):
|
||||
"""Trusted-team model: aggregates are cluster-wide across every
|
||||
caller with ``admin.coordinator``. Every child under the
|
||||
coordinator counts, regardless of the ``user_id`` on the row.
|
||||
"""
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="alice")
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="aa" * 16,
|
||||
node_id="node-a",
|
||||
user_id="alice",
|
||||
parent_ws_id=ws.id,
|
||||
state="idle",
|
||||
)
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="bb" * 16,
|
||||
node_id="node-a",
|
||||
user_id="bob",
|
||||
parent_ws_id=ws.id,
|
||||
state="running",
|
||||
)
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
|
||||
# Every admin.coordinator caller sees both children.
|
||||
for caller in ("alice", "bob", "admin-1"):
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers={"X-Test-User": caller, "X-Test-Perms": "admin.coordinator"},
|
||||
)
|
||||
assert resp.status_code == 200, caller
|
||||
body = resp.json()
|
||||
assert body["spawns_total"] == 2, caller
|
||||
assert body["child_state_counts"] == {"idle": 1, "running": 1}, caller
|
||||
|
||||
|
||||
def test_metrics_judge_fallback_rate_substring_match(storage):
|
||||
"""judge_fallback_rate is computed from any verdict whose ``tier``
|
||||
field contains 'fallback' (case-insensitive). Supports tiers like
|
||||
'llm_fallback', 'LLM_FALLBACK', 'fallback_deterministic'."""
|
||||
import uuid
|
||||
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
# Three verdicts, two marked fallback (one LLM_FALLBACK, one
|
||||
# llm_fallback → both match case-insensitive substring).
|
||||
for tier in ("llm_primary", "LLM_FALLBACK", "llm_fallback"):
|
||||
storage.create_intent_verdict(
|
||||
verdict_id=uuid.uuid4().hex,
|
||||
ws_id=ws.id,
|
||||
call_id="c-" + tier,
|
||||
func_name="f",
|
||||
func_args="{}",
|
||||
intent_summary="",
|
||||
risk_level="low",
|
||||
confidence=0.9,
|
||||
recommendation="allow",
|
||||
reasoning="",
|
||||
evidence="",
|
||||
tier=tier,
|
||||
judge_model="j",
|
||||
latency_ms=1,
|
||||
)
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers=_METRICS_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
# 2 / 3 verdicts matched → 0.667 (rounded to 3 places).
|
||||
assert body["judge_fallback_rate"] == pytest.approx(0.667, abs=1e-3)
|
||||
assert body["intent_verdicts_sample"] == 3
|
||||
|
||||
|
||||
def test_metrics_spawns_last_hour_boundary(storage):
|
||||
"""Only children whose created timestamp is within the last 3600s
|
||||
count toward spawns_last_hour; older children count toward
|
||||
spawns_total but not the hour bucket."""
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
mgr = _build_mgr(storage)
|
||||
ws = mgr.create(user_id="user-1")
|
||||
# One child created "now" (within the window); one created 2
|
||||
# hours ago (outside the window).
|
||||
recent_iso = datetime.fromtimestamp(time.time(), tz=UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
old_iso = (datetime.fromtimestamp(time.time(), tz=UTC) - timedelta(hours=2)).strftime(
|
||||
"%Y-%m-%dT%H:%M:%S"
|
||||
)
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="aa" * 16,
|
||||
node_id="node-a",
|
||||
parent_ws_id=ws.id,
|
||||
state="idle",
|
||||
created=recent_iso,
|
||||
)
|
||||
_seed_workstream(
|
||||
storage,
|
||||
ws_id="bb" * 16,
|
||||
node_id="node-a",
|
||||
parent_ws_id=ws.id,
|
||||
state="closed",
|
||||
created=old_iso,
|
||||
)
|
||||
client = _make_client(storage, coord_mgr=mgr)
|
||||
resp = client.get(
|
||||
f"/v1/api/workstreams/{ws.id}/metrics",
|
||||
headers=_METRICS_HEADERS,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["spawns_total"] == 2
|
||||
assert body["spawns_last_hour"] == 1
|
||||
@@ -440,58 +440,40 @@ class TestResumeDeletedTemplate:
|
||||
|
||||
class TestSkillFactoryPassthrough:
|
||||
def test_skill_passed_through_workstream_create(self, tmp_db):
|
||||
"""SessionManager.create(skill=...) propagates to session factory."""
|
||||
import queue
|
||||
|
||||
from turnstone.core.adapters.interactive_adapter import InteractiveAdapter
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
"""WorkstreamManager.create(skill=...) propagates to session factory."""
|
||||
from turnstone.core.storage import get_storage
|
||||
from turnstone.core.workstream import WorkstreamManager
|
||||
|
||||
db = get_storage()
|
||||
_create_template(db, "t1", "factory-tpl", "FACTORY_CONTENT", is_default=False)
|
||||
|
||||
captured_skill = None
|
||||
|
||||
def factory(ui, model_alias=None, ws_id=None, *, skill=None, **_kwargs):
|
||||
def factory(ui, model_alias=None, ws_id=None, *, skill=None):
|
||||
nonlocal captured_skill
|
||||
captured_skill = skill
|
||||
return _make_session(skill=captured_skill)
|
||||
|
||||
gq: queue.Queue[dict] = queue.Queue(maxsize=1000)
|
||||
adapter = InteractiveAdapter(
|
||||
global_queue=gq,
|
||||
ui_factory=lambda ws: NullUI(),
|
||||
session_factory=factory,
|
||||
)
|
||||
mgr = SessionManager(adapter, storage=MagicMock(), max_active=10, event_emitter=adapter)
|
||||
ws = mgr.create(user_id="", name="test", skill="factory-tpl")
|
||||
mgr = WorkstreamManager(factory)
|
||||
ws = mgr.create(name="test", skill="factory-tpl")
|
||||
assert captured_skill == "factory-tpl"
|
||||
assert ws.session is not None
|
||||
assert ws.session._skill_name == "factory-tpl"
|
||||
assert "FACTORY_CONTENT" in _sys_content(ws.session)
|
||||
|
||||
def test_skill_none_uses_defaults(self, tmp_db):
|
||||
"""SessionManager.create() without skill passes None."""
|
||||
import queue
|
||||
|
||||
from turnstone.core.adapters.interactive_adapter import InteractiveAdapter
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
|
||||
"""WorkstreamManager.create() without skill passes None."""
|
||||
captured_skill = "sentinel"
|
||||
|
||||
def factory(ui, model_alias=None, ws_id=None, *, skill=None, **_kwargs):
|
||||
def factory(ui, model_alias=None, ws_id=None, *, skill=None):
|
||||
nonlocal captured_skill
|
||||
captured_skill = skill
|
||||
return _make_session(skill=skill)
|
||||
|
||||
gq: queue.Queue[dict] = queue.Queue(maxsize=1000)
|
||||
adapter = InteractiveAdapter(
|
||||
global_queue=gq,
|
||||
ui_factory=lambda ws: NullUI(),
|
||||
session_factory=factory,
|
||||
)
|
||||
mgr = SessionManager(adapter, storage=MagicMock(), max_active=10, event_emitter=adapter)
|
||||
mgr.create(user_id="", name="test")
|
||||
from turnstone.core.workstream import WorkstreamManager
|
||||
|
||||
mgr = WorkstreamManager(factory)
|
||||
mgr.create(name="test")
|
||||
assert captured_skill is None
|
||||
|
||||
|
||||
|
||||
@@ -350,93 +350,6 @@ def test_tools_excluded_when_no_tools() -> None:
|
||||
assert "TOOL PATTERNS" not in result
|
||||
|
||||
|
||||
def test_coordinator_kind_selects_coord_tools() -> None:
|
||||
"""kind='coordinator' swaps in tools_coordinator.md with the right patterns."""
|
||||
coord_tools = frozenset(
|
||||
{
|
||||
"spawn_workstream",
|
||||
"send_to_workstream",
|
||||
"inspect_workstream",
|
||||
"close_workstream",
|
||||
"list_workstreams",
|
||||
"list_nodes",
|
||||
"list_skills",
|
||||
"tasks",
|
||||
}
|
||||
)
|
||||
result = compose_system_message(
|
||||
ClientType.WEB,
|
||||
_VALID_CTX,
|
||||
coord_tools,
|
||||
kind="coordinator",
|
||||
)
|
||||
# Coordinator tool patterns are present.
|
||||
assert "spawn_workstream" in result
|
||||
assert "inspect_workstream" in result
|
||||
assert "tasks" in result
|
||||
# IC tool patterns are NOT present — the model must not be instructed
|
||||
# to call tools it doesn't have.
|
||||
for phantom in (
|
||||
"read_file",
|
||||
"edit_file",
|
||||
"write_file",
|
||||
"bash",
|
||||
"plan_agent",
|
||||
"web_fetch",
|
||||
"web_search",
|
||||
):
|
||||
assert phantom not in result, (
|
||||
f"coordinator prompt must not advertise phantom tool {phantom!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_coordinator_kind_uses_orchestrator_persona() -> None:
|
||||
"""kind='coordinator' swaps in base_coordinator.md."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
frozenset({"spawn_workstream"}),
|
||||
kind="coordinator",
|
||||
)
|
||||
# IC-framing phrases from base.md should NOT appear.
|
||||
for ic_phrase in ("read before you edit", "commits you make"):
|
||||
assert ic_phrase not in result, f"coordinator persona leaked IC framing: {ic_phrase!r}"
|
||||
# Orchestrator-framing phrases from base_coordinator.md should appear.
|
||||
assert "orchestrate" in result
|
||||
assert "delegate" in result
|
||||
|
||||
|
||||
def test_coordinator_kind_skips_env_block() -> None:
|
||||
"""Coordinators don't render rich output, so the ENV block is omitted.
|
||||
|
||||
Regression-locks the orchestration-vs-rendering split: a coordinator
|
||||
composing a system message with any client_type must not pick up the
|
||||
user-facing formatting principles (Mermaid / KaTeX / chat platform
|
||||
quirks). client_type still validates — only the loaded content is
|
||||
skipped.
|
||||
"""
|
||||
for ct in (ClientType.WEB, ClientType.CLI, ClientType.CHAT):
|
||||
result = compose_system_message(
|
||||
ct,
|
||||
_VALID_CTX,
|
||||
frozenset({"spawn_workstream"}),
|
||||
kind="coordinator",
|
||||
)
|
||||
for env_phrase in ("Output Environment", "Available rendering", "Formatting principles"):
|
||||
assert env_phrase not in result, f"coordinator on {ct} leaked ENV phrase {env_phrase!r}"
|
||||
|
||||
|
||||
def test_interactive_kind_default_still_loads_ic_tools() -> None:
|
||||
"""Default kind='interactive' still loads tools.md (no regression)."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
)
|
||||
assert "read_file" in result
|
||||
assert "bash" in result
|
||||
|
||||
|
||||
def test_tools_included_when_tools_available() -> None:
|
||||
"""TOOLS module is included when available_tools is non-empty."""
|
||||
result = compose_system_message(
|
||||
@@ -445,24 +358,3 @@ def test_tools_included_when_tools_available() -> None:
|
||||
_ALL_TOOLS,
|
||||
)
|
||||
assert "TOOL PATTERNS" in result
|
||||
|
||||
|
||||
def test_session_kind_in_context_interactive() -> None:
|
||||
"""Default interactive kind appears next to the user line."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
)
|
||||
assert "Session kind:** interactive" in result
|
||||
|
||||
|
||||
def test_session_kind_in_context_coordinator() -> None:
|
||||
"""Coordinator kind appears in the context block."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
frozenset({"spawn_workstream"}),
|
||||
kind="coordinator",
|
||||
)
|
||||
assert "Session kind:** coordinator" in result
|
||||
|
||||
+1
-55
@@ -224,25 +224,6 @@ class TestOpenAIProvider:
|
||||
sanitize_messages([original])
|
||||
assert original["content"] is None
|
||||
|
||||
def test_sanitize_messages_strips_underscore_sibling_keys(self) -> None:
|
||||
"""Internal sibling metadata (``_reminders``, ``_reminders_delivered``,
|
||||
``_attachments_meta``, ``_provider_content``) must be stripped
|
||||
before the wire — the OpenAI-compat APIs reject unknown fields."""
|
||||
msgs = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi",
|
||||
"_reminders": [{"type": "correction", "text": "watch"}],
|
||||
"_reminders_delivered": True,
|
||||
"_attachments_meta": [{"kind": "image"}],
|
||||
}
|
||||
]
|
||||
result = sanitize_messages(msgs)
|
||||
assert result == [{"role": "user", "content": "hi"}]
|
||||
assert "_reminders" not in result[0]
|
||||
assert "_reminders_delivered" not in result[0]
|
||||
assert "_attachments_meta" not in result[0]
|
||||
|
||||
# -- sanitize_messages: orphan detection -----------------------------------
|
||||
|
||||
def test_sanitize_orphaned_tool_call_synthesized(self) -> None:
|
||||
@@ -1795,31 +1776,6 @@ class TestOpenAIParameterGating:
|
||||
assert "temperature" not in kwargs
|
||||
assert kwargs["reasoning_effort"] == "medium" # fell back from unsupported "low"
|
||||
|
||||
def test_gpt55_1m_context_and_effort(self) -> None:
|
||||
"""GPT-5.5: 1M context, temperature when effort=none, xhigh supported."""
|
||||
caps = self.provider.get_capabilities("gpt-5.5")
|
||||
assert caps.context_window == 1050000
|
||||
assert caps.supports_tool_search is True
|
||||
assert caps.supports_vision is True
|
||||
kwargs: dict[str, Any] = {}
|
||||
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none")
|
||||
assert kwargs["temperature"] == 0.7
|
||||
assert "reasoning_effort" not in kwargs
|
||||
kwargs2: dict[str, Any] = {}
|
||||
apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh")
|
||||
assert "temperature" not in kwargs2
|
||||
assert kwargs2["reasoning_effort"] == "xhigh"
|
||||
|
||||
def test_gpt55_pro_no_temperature_always_reasoning(self) -> None:
|
||||
"""GPT-5.5 pro: no temperature, medium/high/xhigh only."""
|
||||
caps = self.provider.get_capabilities("gpt-5.5-pro")
|
||||
assert caps.context_window == 1050000
|
||||
assert caps.supports_tool_search is True
|
||||
kwargs: dict[str, Any] = {}
|
||||
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low")
|
||||
assert "temperature" not in kwargs
|
||||
assert kwargs["reasoning_effort"] == "medium" # fell back from unsupported "low"
|
||||
|
||||
|
||||
class TestAnthropicOrphanedToolUse:
|
||||
"""Verify _convert_messages synthesizes tool_results for orphaned tool_use."""
|
||||
@@ -3445,17 +3401,7 @@ class TestOpenAIPromptCaching:
|
||||
|
||||
def test_cache_retention_set_for_gpt5(self) -> None:
|
||||
"""GPT-5.x models get prompt_cache_retention=24h."""
|
||||
for model in (
|
||||
"gpt-5",
|
||||
"gpt-5.1",
|
||||
"gpt-5.2",
|
||||
"gpt-5.4",
|
||||
"gpt-5.4-pro",
|
||||
"gpt-5.5",
|
||||
"gpt-5.5-pro",
|
||||
"gpt-5-mini",
|
||||
"gpt-5-pro",
|
||||
):
|
||||
for model in ("gpt-5", "gpt-5.1", "gpt-5.2", "gpt-5.4", "gpt-5-mini", "gpt-5-pro"):
|
||||
kwargs: dict[str, Any] = {}
|
||||
apply_cache_retention(kwargs, model)
|
||||
assert kwargs.get("prompt_cache_retention") == "24h", f"Failed for {model}"
|
||||
|
||||
+11
-11
@@ -61,15 +61,15 @@ class TestRateLimiter:
|
||||
def test_disabled_allows_everything(self):
|
||||
limiter = RateLimiter(enabled=False, rate=1.0, burst=1)
|
||||
for _ in range(100):
|
||||
allowed, retry = limiter.check("1.2.3.4", "/api/workstreams/abc/send")
|
||||
allowed, retry = limiter.check("1.2.3.4", "/api/send")
|
||||
assert allowed is True
|
||||
assert retry == 0.0
|
||||
|
||||
def test_exempt_paths_bypass(self):
|
||||
limiter = RateLimiter(enabled=True, rate=1.0, burst=1)
|
||||
# Exhaust the bucket on a normal path
|
||||
limiter.check("1.2.3.4", "/api/workstreams/abc/send")
|
||||
limiter.check("1.2.3.4", "/api/workstreams/abc/send")
|
||||
limiter.check("1.2.3.4", "/api/send")
|
||||
limiter.check("1.2.3.4", "/api/send")
|
||||
|
||||
# Exempt paths should still pass
|
||||
allowed, retry = limiter.check("1.2.3.4", "/health")
|
||||
@@ -84,18 +84,18 @@ class TestRateLimiter:
|
||||
limiter = RateLimiter(enabled=True, rate=1.0, burst=1)
|
||||
|
||||
# Exhaust IP A
|
||||
allowed_a, _ = limiter.check("10.0.0.1", "/api/workstreams/abc/send")
|
||||
allowed_a, _ = limiter.check("10.0.0.1", "/api/send")
|
||||
assert allowed_a is True
|
||||
allowed_a, _ = limiter.check("10.0.0.1", "/api/workstreams/abc/send")
|
||||
allowed_a, _ = limiter.check("10.0.0.1", "/api/send")
|
||||
assert allowed_a is False
|
||||
|
||||
# IP B should still have its own bucket
|
||||
allowed_b, _ = limiter.check("10.0.0.2", "/api/workstreams/abc/send")
|
||||
allowed_b, _ = limiter.check("10.0.0.2", "/api/send")
|
||||
assert allowed_b is True
|
||||
|
||||
def test_burst_then_reject(self):
|
||||
limiter = RateLimiter(enabled=True, rate=10.0, burst=3)
|
||||
results = [limiter.check("1.2.3.4", "/api/workstreams/abc/send")[0] for _ in range(5)]
|
||||
results = [limiter.check("1.2.3.4", "/api/send")[0] for _ in range(5)]
|
||||
assert results == [True, True, True, False, False]
|
||||
|
||||
def test_cleanup_removes_stale(self):
|
||||
@@ -104,8 +104,8 @@ class TestRateLimiter:
|
||||
limiter = RateLimiter(enabled=True, rate=10.0, burst=5)
|
||||
|
||||
# Create buckets for two IPs
|
||||
limiter.check("10.0.0.1", "/api/workstreams/abc/send")
|
||||
limiter.check("10.0.0.2", "/api/workstreams/abc/send")
|
||||
limiter.check("10.0.0.1", "/api/send")
|
||||
limiter.check("10.0.0.2", "/api/send")
|
||||
|
||||
# Advance time past max_age for both
|
||||
mock_time.return_value = 5000.0
|
||||
@@ -120,11 +120,11 @@ class TestRateLimiter:
|
||||
mock_time.return_value = 1000.0
|
||||
limiter = RateLimiter(enabled=True, rate=10.0, burst=5)
|
||||
|
||||
limiter.check("10.0.0.1", "/api/workstreams/abc/send")
|
||||
limiter.check("10.0.0.1", "/api/send")
|
||||
|
||||
# Only 60s later — well within max_age
|
||||
mock_time.return_value = 1060.0
|
||||
limiter.check("10.0.0.2", "/api/workstreams/abc/send")
|
||||
limiter.check("10.0.0.2", "/api/send")
|
||||
|
||||
mock_time.return_value = 1060.0
|
||||
removed = limiter.cleanup(max_age=3600.0)
|
||||
|
||||
@@ -0,0 +1,561 @@
|
||||
"""Tests for turnstone.console.rebalancer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.console.rebalancer import Rebalancer
|
||||
from turnstone.core.hash_ring import RING_SIZE
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
"""Fresh SQLite backend for each test."""
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
def _register_nodes(storage: SQLiteBackend, count: int, *, weight: int = 1) -> None:
|
||||
"""Register *count* server nodes in the services table."""
|
||||
for i in range(count):
|
||||
meta = json.dumps({"weight": weight, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", f"node-{i}", f"http://node-{i}:8080", metadata=meta)
|
||||
|
||||
|
||||
def _register_weighted_nodes(storage: SQLiteBackend, weights: dict[str, int]) -> None:
|
||||
"""Register nodes with specific weights."""
|
||||
for node_id, w in weights.items():
|
||||
meta = json.dumps({"weight": w, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", node_id, f"http://{node_id}:8080", metadata=meta)
|
||||
|
||||
|
||||
def _get_version(storage: SQLiteBackend) -> int:
|
||||
"""Read the rebalancer_version from system_settings."""
|
||||
raw = storage.get_system_setting("rebalancer_version", node_id="")
|
||||
if raw is None:
|
||||
return 0
|
||||
try:
|
||||
return int(json.loads(raw.get("value", "0")))
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
class TestFirstRunSeed:
|
||||
def test_first_run_seeds_ring(self, storage):
|
||||
"""Empty assignment table + 2 nodes -> seed all 65536 rows."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
result = rb.rebalance_once()
|
||||
|
||||
assert result.seeded is True
|
||||
assert result.noop is False
|
||||
assert result.nodes == 2
|
||||
|
||||
buckets = storage.list_ring_buckets()
|
||||
assert len(buckets) == RING_SIZE
|
||||
|
||||
# All buckets should be assigned to one of the two nodes
|
||||
node_ids = {b["node_id"] for b in buckets}
|
||||
assert node_ids == {"node-0", "node-1"}
|
||||
|
||||
|
||||
class TestSeedPopulatesRouter:
|
||||
def test_seed_populates_router_directly(self, storage):
|
||||
"""On first seed, the router cache is populated without a DB read-back."""
|
||||
from turnstone.console.router import ConsoleRouter
|
||||
|
||||
_register_nodes(storage, 2)
|
||||
router = ConsoleRouter(storage)
|
||||
assert not router.is_ready()
|
||||
|
||||
rb = Rebalancer(storage=storage, router=router)
|
||||
result = rb.rebalance_once()
|
||||
|
||||
assert result.seeded is True
|
||||
assert router.is_ready()
|
||||
assert router.node_count() == 2
|
||||
|
||||
# Routing should work for any valid ws_id
|
||||
ws_id = "0000" + "a" * 28
|
||||
ref = router.route(ws_id)
|
||||
assert ref.node_id in {"node-0", "node-1"}
|
||||
|
||||
|
||||
class TestIdempotent:
|
||||
def test_second_run_is_noop(self, storage):
|
||||
"""Running rebalance twice with same membership produces noop on second pass."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
|
||||
r1 = rb.rebalance_once()
|
||||
assert r1.seeded is True
|
||||
|
||||
r2 = rb.rebalance_once()
|
||||
assert r2.noop is True
|
||||
assert r2.moves == 0
|
||||
|
||||
|
||||
class TestNewNodeRebalances:
|
||||
def test_adding_node_moves_buckets(self, storage):
|
||||
"""Seed with 2 nodes, add 3rd -> some buckets move to the new node."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Verify only 2 nodes initially
|
||||
buckets_before = storage.list_ring_buckets()
|
||||
nodes_before = {b["node_id"] for b in buckets_before}
|
||||
assert nodes_before == {"node-0", "node-1"}
|
||||
|
||||
# Add a third node
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is False
|
||||
assert result.moves > 0
|
||||
assert result.nodes == 3
|
||||
|
||||
# Verify all three nodes have buckets
|
||||
buckets_after = storage.list_ring_buckets()
|
||||
nodes_after = {b["node_id"] for b in buckets_after}
|
||||
assert "node-2" in nodes_after
|
||||
|
||||
|
||||
class TestDeadNodeReassigned:
|
||||
def test_dead_node_buckets_move_to_survivors(self, storage):
|
||||
"""Seed with 3 nodes, deregister one -> its buckets move to survivors."""
|
||||
_register_nodes(storage, 3)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Verify node-2 has some buckets
|
||||
buckets = storage.list_ring_buckets()
|
||||
node2_count = sum(1 for b in buckets if b["node_id"] == "node-2")
|
||||
assert node2_count > 0
|
||||
|
||||
# Deregister node-2
|
||||
storage.deregister_service("server", "node-2")
|
||||
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is False
|
||||
assert result.moves > 0
|
||||
|
||||
# Verify no buckets assigned to dead node
|
||||
buckets_after = storage.list_ring_buckets()
|
||||
nodes_after = {b["node_id"] for b in buckets_after}
|
||||
assert "node-2" not in nodes_after
|
||||
|
||||
|
||||
class TestSingleNodeNoop:
|
||||
def test_single_node_already_assigned_is_noop(self, storage):
|
||||
"""1 node with all buckets assigned -> noop."""
|
||||
_register_nodes(storage, 1)
|
||||
rb = Rebalancer(storage=storage)
|
||||
|
||||
# Seed with single node
|
||||
rb.rebalance_once()
|
||||
|
||||
# Second run should be noop
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is True
|
||||
|
||||
|
||||
class TestWeightedDistribution:
|
||||
def test_weight_2_gets_more_buckets(self, storage):
|
||||
"""Node with weight=2 gets roughly 2x the buckets of weight=1."""
|
||||
_register_weighted_nodes(storage, {"heavy": 2, "light": 1})
|
||||
rb = Rebalancer(storage=storage, vnodes_per_unit=150)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
buckets = storage.list_ring_buckets()
|
||||
heavy_count = sum(1 for b in buckets if b["node_id"] == "heavy")
|
||||
light_count = sum(1 for b in buckets if b["node_id"] == "light")
|
||||
|
||||
# heavy should have roughly 2/3 of total, light roughly 1/3
|
||||
# Allow 10% tolerance
|
||||
expected_heavy = RING_SIZE * 2 // 3
|
||||
assert abs(heavy_count - expected_heavy) < RING_SIZE * 0.10
|
||||
assert heavy_count > light_count
|
||||
|
||||
|
||||
class TestVersionIncremented:
|
||||
def test_version_bumps_on_seed(self, storage):
|
||||
"""Verify rebalancer_version increments after seed."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
|
||||
v0 = _get_version(storage)
|
||||
assert v0 == 0
|
||||
|
||||
rb.rebalance_once()
|
||||
v1 = _get_version(storage)
|
||||
assert v1 == 1
|
||||
|
||||
def test_version_bumps_on_rebalance(self, storage):
|
||||
"""Version bumps on actual moves, not on noops."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed: version -> 1
|
||||
|
||||
# Noop: version stays at 1
|
||||
rb.rebalance_once()
|
||||
assert _get_version(storage) == 1
|
||||
|
||||
# Add node: version -> 2
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
rb.rebalance_once()
|
||||
assert _get_version(storage) == 2
|
||||
|
||||
|
||||
class TestReconcileStats:
|
||||
def test_bucket_stats_corrected(self, storage):
|
||||
"""Create workstreams in DB, verify bucket_stats are reconciled."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Create some workstreams — ws_id starts with hex bucket
|
||||
# Bucket 0x0000 = 0, bucket 0x0001 = 1
|
||||
storage.register_workstream("0000" + "a" * 28, state="idle")
|
||||
storage.register_workstream("0000" + "b" * 28, state="running")
|
||||
storage.register_workstream("0001" + "c" * 28, state="idle")
|
||||
|
||||
# Set bogus stats that will be corrected
|
||||
storage.increment_bucket_count(0) # says 1, should be 2
|
||||
storage.increment_bucket_count(5) # says 1, should be 0
|
||||
|
||||
rb._reconcile_bucket_stats()
|
||||
|
||||
stats = storage.list_bucket_stats()
|
||||
stats_map = {s["bucket"]: s for s in stats}
|
||||
|
||||
# Bucket 0 should have 2 ws, 1 active (running)
|
||||
assert stats_map[0]["ws_count"] == 2
|
||||
assert stats_map[0]["active_count"] == 1
|
||||
|
||||
# Bucket 1 should have 1 ws, 0 active
|
||||
assert stats_map[1]["ws_count"] == 1
|
||||
assert stats_map[1]["active_count"] == 0
|
||||
|
||||
# Bucket 5 should have been removed (ws_count=0)
|
||||
assert 5 not in stats_map
|
||||
|
||||
|
||||
class TestTransferPriorityEmptyFirst:
|
||||
def test_empty_buckets_moved_before_occupied(self, storage):
|
||||
"""Verify the sort key puts empty buckets before occupied ones.
|
||||
|
||||
Rather than asserting specific bucket assignments (which depend on
|
||||
hash ring placement), we verify the sorting invariant directly by
|
||||
checking that moves with zero occupancy come before occupied ones
|
||||
in the internal ordering.
|
||||
"""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.01)
|
||||
rb.rebalance_once() # seed
|
||||
|
||||
# Create workstreams in a few buckets owned by node-0
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_buckets = [b["bucket"] for b in buckets if b["node_id"] == "node-0"]
|
||||
|
||||
occupied = set()
|
||||
for b in node0_buckets[:3]:
|
||||
ws_id = f"{b:04x}" + "d" * 28
|
||||
storage.register_workstream(ws_id, state="running")
|
||||
storage.increment_bucket_count(b, active=True)
|
||||
occupied.add(b)
|
||||
|
||||
# Reconcile stats so the rebalancer sees them
|
||||
rb._reconcile_bucket_stats()
|
||||
|
||||
# Read stats to verify ordering assumptions
|
||||
stats = storage.list_bucket_stats()
|
||||
stats_map = {s["bucket"]: (s["ws_count"], s["active_count"]) for s in stats}
|
||||
|
||||
# The sort key is (active_count, ws_count) — occupied buckets
|
||||
# must sort AFTER empty buckets
|
||||
for b in occupied:
|
||||
assert stats_map[b][0] > 0 # ws_count > 0
|
||||
assert stats_map[b][1] > 0 # active_count > 0
|
||||
|
||||
# Empty buckets have (0, 0) which sorts before (1, 1)
|
||||
assert (0, 0) < (1, 1)
|
||||
|
||||
|
||||
class TestLeaderElection:
|
||||
def test_two_rebalancers_one_runs(self, storage):
|
||||
"""Two rebalancers compete — only one acquires the lock."""
|
||||
_register_nodes(storage, 2)
|
||||
rb1 = Rebalancer(storage=storage)
|
||||
rb2 = Rebalancer(storage=storage)
|
||||
|
||||
# rb1 acquires the lock
|
||||
assert rb1._try_acquire_lock() is True
|
||||
|
||||
# rb2 cannot acquire (lock is fresh)
|
||||
assert rb2._try_acquire_lock() is False
|
||||
|
||||
# rb1 releases
|
||||
rb1._release_lock()
|
||||
|
||||
# Now rb2 can acquire
|
||||
assert rb2._try_acquire_lock() is True
|
||||
rb2._release_lock()
|
||||
|
||||
|
||||
class TestZeroNodes:
|
||||
def test_no_nodes_returns_noop(self, storage):
|
||||
"""Zero live nodes -> noop result."""
|
||||
rb = Rebalancer(storage=storage)
|
||||
result = rb.rebalance_once()
|
||||
assert result.noop is True
|
||||
assert result.nodes == 0
|
||||
|
||||
|
||||
class TestStartStop:
|
||||
def test_start_stop_lifecycle(self, storage):
|
||||
"""Verify start/stop lifecycle doesn't hang or crash."""
|
||||
_register_nodes(storage, 1)
|
||||
rb = Rebalancer(storage=storage, interval=1)
|
||||
rb.start()
|
||||
assert rb._thread is not None
|
||||
assert rb._thread.is_alive()
|
||||
rb.stop()
|
||||
assert not rb._thread.is_alive()
|
||||
|
||||
def test_trigger_wakes_thread(self, storage):
|
||||
"""Verify trigger() causes an immediate pass."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, interval=3600) # long interval
|
||||
rb.start()
|
||||
try:
|
||||
rb.trigger()
|
||||
# Give it a moment to process
|
||||
rb._stop_event.wait(timeout=2)
|
||||
finally:
|
||||
rb.stop()
|
||||
# After trigger, the ring should be seeded
|
||||
assert len(storage.list_ring_buckets()) == RING_SIZE
|
||||
|
||||
|
||||
class TestGetStatus:
|
||||
def test_status_before_any_run(self, storage):
|
||||
"""Status returns version=0 and no last_result before any run."""
|
||||
rb = Rebalancer(storage=storage)
|
||||
status = rb.get_status()
|
||||
assert status["version"] == 0
|
||||
assert status["last_result"] is None
|
||||
|
||||
def test_status_after_seed(self, storage):
|
||||
"""Status reflects the seed run when result is stored."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage)
|
||||
result = rb.rebalance_once()
|
||||
# The loop normally sets _last_result; simulate that here
|
||||
rb._last_result = result
|
||||
status = rb.get_status()
|
||||
assert status["version"] == 1
|
||||
assert status["last_result"] is not None
|
||||
assert status["last_result"]["seeded"] is True
|
||||
|
||||
|
||||
class TestEagerMigration:
|
||||
def test_eager_migrate_posts_to_source_nodes(self, storage):
|
||||
"""When eager_migrate=True, rebalancer POSTs /_internal/migrate for idle workstreams."""
|
||||
import httpx
|
||||
|
||||
# Seed ring with 2 nodes
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, eager_migrate=True)
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
# Create a workstream on node-0's bucket range
|
||||
# Find a bucket assigned to node-0
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_bucket = None
|
||||
for b in buckets:
|
||||
if b["node_id"] == "node-0":
|
||||
node0_bucket = b["bucket"]
|
||||
break
|
||||
assert node0_bucket is not None
|
||||
|
||||
ws_id = f"{node0_bucket:04x}" + "a" * 28
|
||||
storage.register_workstream(ws_id, node_id="node-0", name="test")
|
||||
storage.increment_bucket_count(node0_bucket)
|
||||
|
||||
# Add a 3rd node — this will trigger rebalance
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
# Track migrate calls
|
||||
migrate_calls: list[tuple[str, str]] = [] # (url, ws_id)
|
||||
|
||||
class FakeTransport(httpx.BaseTransport):
|
||||
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
migrate_calls.append((str(request.url), body.get("ws_id", "")))
|
||||
return httpx.Response(200, json={"status": "ok", "ws_id": body["ws_id"]})
|
||||
|
||||
# Monkey-patch httpx.Client to use our fake transport
|
||||
original_init = httpx.Client.__init__
|
||||
|
||||
def patched_init(self_client, **kwargs):
|
||||
kwargs["transport"] = FakeTransport()
|
||||
original_init(self_client, **kwargs)
|
||||
|
||||
import unittest.mock
|
||||
|
||||
with unittest.mock.patch.object(httpx.Client, "__init__", patched_init):
|
||||
result = rb.rebalance_once(trigger="test")
|
||||
|
||||
# If the bucket moved to a different node, the workstream should be migrated
|
||||
new_buckets = storage.list_ring_buckets()
|
||||
new_owner = None
|
||||
for b in new_buckets:
|
||||
if b["bucket"] == node0_bucket:
|
||||
new_owner = b["node_id"]
|
||||
break
|
||||
|
||||
if new_owner != "node-0":
|
||||
# Bucket moved — migration should have happened
|
||||
assert result.migrations > 0
|
||||
assert any(ws_id in call[1] for call in migrate_calls)
|
||||
else:
|
||||
# Bucket stayed — no migration needed for this ws
|
||||
assert result.migrations >= 0 # other workstreams might have been migrated
|
||||
|
||||
def test_eager_migrate_skips_active_workstreams(self, storage):
|
||||
"""Active workstreams are not eagerly migrated (would disrupt in-flight work)."""
|
||||
import httpx
|
||||
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, eager_migrate=True)
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
# Find a bucket on node-0
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_bucket = None
|
||||
for b in buckets:
|
||||
if b["node_id"] == "node-0":
|
||||
node0_bucket = b["bucket"]
|
||||
break
|
||||
assert node0_bucket is not None
|
||||
|
||||
# Create an ACTIVE workstream (state="running")
|
||||
ws_id = f"{node0_bucket:04x}" + "b" * 28
|
||||
storage.register_workstream(ws_id, node_id="node-0", name="active-ws")
|
||||
storage.update_workstream_state(ws_id, "running")
|
||||
storage.increment_bucket_count(node0_bucket, active=True)
|
||||
|
||||
# Add 3rd node to trigger rebalance
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
migrate_calls: list[str] = []
|
||||
|
||||
class FakeTransport(httpx.BaseTransport):
|
||||
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
migrate_calls.append(body.get("ws_id", ""))
|
||||
return httpx.Response(200, json={"status": "ok"})
|
||||
|
||||
original_init = httpx.Client.__init__
|
||||
|
||||
def patched_init(self_client, **kwargs):
|
||||
kwargs["transport"] = FakeTransport()
|
||||
original_init(self_client, **kwargs)
|
||||
|
||||
import unittest.mock
|
||||
|
||||
with unittest.mock.patch.object(httpx.Client, "__init__", patched_init):
|
||||
rb.rebalance_once(trigger="test")
|
||||
|
||||
# The active workstream should NOT have been migrated
|
||||
assert ws_id not in migrate_calls
|
||||
|
||||
def test_eager_migrate_disabled_by_default(self, storage):
|
||||
"""When eager_migrate=False (default), no migrate calls happen."""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage) # eager_migrate defaults to False
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
# Create workstream and trigger rebalance
|
||||
buckets = storage.list_ring_buckets()
|
||||
node0_bucket = next(b["bucket"] for b in buckets if b["node_id"] == "node-0")
|
||||
ws_id = f"{node0_bucket:04x}" + "c" * 28
|
||||
storage.register_workstream(ws_id, node_id="node-0", name="test")
|
||||
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
|
||||
result = rb.rebalance_once(trigger="test")
|
||||
assert result.migrations == 0 # no eager migration when disabled
|
||||
|
||||
|
||||
class TestMinimalTransfer:
|
||||
def test_new_node_only_receives_never_shuffles(self, storage):
|
||||
"""Adding a 3rd node moves buckets TO it, never between existing nodes.
|
||||
|
||||
This is the key property of the minimal-transfer algorithm: nodes A
|
||||
and B should not exchange buckets with each other — only donate to C.
|
||||
"""
|
||||
_register_nodes(storage, 2)
|
||||
rb = Rebalancer(storage=storage, threshold=0.05)
|
||||
rb.rebalance_once() # seeds: node-0 gets 32768, node-1 gets 32768
|
||||
|
||||
# Record which node owns each bucket before adding node-2
|
||||
before = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Add a third node
|
||||
meta = json.dumps({"weight": 1, "started": "2026-01-01T00:00:00Z"})
|
||||
storage.register_service("server", "node-2", "http://node-2:8080", metadata=meta)
|
||||
result = rb.rebalance_once()
|
||||
|
||||
after = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Verify: every bucket that moved went TO node-2
|
||||
for bucket in range(RING_SIZE):
|
||||
old = before[bucket]
|
||||
new = after[bucket]
|
||||
if old != new:
|
||||
assert new == "node-2", (
|
||||
f"bucket {bucket} moved {old} -> {new}, expected all moves to target node-2"
|
||||
)
|
||||
|
||||
# Verify: node-2 got roughly 1/3 of all buckets
|
||||
node2_count = sum(1 for nid in after.values() if nid == "node-2")
|
||||
assert 19000 < node2_count < 24000, f"node-2 got {node2_count} buckets"
|
||||
assert result.moves > 0
|
||||
|
||||
def test_remove_node_distributes_proportionally(self, storage):
|
||||
"""Removing a node distributes its buckets to remaining nodes
|
||||
proportionally — doesn't shuffle between survivors."""
|
||||
_register_nodes(storage, 3)
|
||||
rb = Rebalancer(storage=storage, threshold=0.05)
|
||||
rb.rebalance_once() # seeds
|
||||
|
||||
before = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Remove node-2
|
||||
storage.deregister_service("server", "node-2")
|
||||
result = rb.rebalance_once()
|
||||
|
||||
after = {r["bucket"]: r["node_id"] for r in storage.list_ring_buckets()}
|
||||
|
||||
# Every moved bucket should have been owned by node-2 (the dead node)
|
||||
for bucket in range(RING_SIZE):
|
||||
old = before[bucket]
|
||||
new = after[bucket]
|
||||
if old != new:
|
||||
assert old == "node-2", (
|
||||
f"bucket {bucket} moved {old} -> {new}, but only node-2's buckets should move"
|
||||
)
|
||||
|
||||
# node-2 should have zero buckets now
|
||||
node2_count = sum(1 for nid in after.values() if nid == "node-2")
|
||||
assert node2_count == 0
|
||||
assert result.moves > 0
|
||||
@@ -1,626 +0,0 @@
|
||||
"""Smoke tests for ``turnstone/shared_static/renderer.js``.
|
||||
|
||||
The renderer is browser-only JS with no test framework on the project
|
||||
side. These tests drive it through ``node`` against a minimal browser-
|
||||
shim harness so a regression on the markdown / KaTeX wiring surfaces
|
||||
in CI rather than at runtime in the operator's browser.
|
||||
|
||||
Each test invokes ``node -e`` with a small wrapper that loads
|
||||
``utils.js`` + ``renderer.js`` via ``vm.runInThisContext``, stubs
|
||||
``document`` / ``katex`` enough for the renderer to run, then prints
|
||||
the rendered HTML for a sample input. The assertions check the
|
||||
resulting markup contains the expected ``<span class="katex">…</span>``
|
||||
placeholder and not the raw delimiter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
_UTILS_JS = _REPO_ROOT / "turnstone/shared_static/utils.js"
|
||||
_RENDERER_JS = _REPO_ROOT / "turnstone/shared_static/renderer.js"
|
||||
|
||||
|
||||
def _has_node() -> bool:
|
||||
return shutil.which("node") is not None
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(not _has_node(), reason="node not available")
|
||||
|
||||
|
||||
_HARNESS_TEMPLATE = """
|
||||
const vm = require('vm');
|
||||
const fs = require('fs');
|
||||
global.document = {
|
||||
createElement: () => {
|
||||
let t = '';
|
||||
return {
|
||||
get textContent() { return t; },
|
||||
set textContent(v) { t = v; },
|
||||
get innerHTML() {
|
||||
return t.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
|
||||
},
|
||||
};
|
||||
},
|
||||
addEventListener: () => {},
|
||||
};
|
||||
global.katex = {
|
||||
renderToString: (tex, opts) =>
|
||||
'<span class="katex">[KATEX:' +
|
||||
tex.replace(/\\n/g, '\\\\n') +
|
||||
(opts.displayMode ? ':display' : ':inline') +
|
||||
']</span>',
|
||||
};
|
||||
global.window = global;
|
||||
vm.runInThisContext(fs.readFileSync(%(utils)s, 'utf8'));
|
||||
vm.runInThisContext(fs.readFileSync(%(renderer)s, 'utf8'));
|
||||
const input = %(input)s;
|
||||
process.stdout.write(renderMarkdown(input));
|
||||
"""
|
||||
|
||||
|
||||
def _render(markdown: str) -> str:
|
||||
"""Render ``markdown`` through renderer.js + return the HTML."""
|
||||
harness = _HARNESS_TEMPLATE % {
|
||||
"utils": json.dumps(str(_UTILS_JS)),
|
||||
"renderer": json.dumps(str(_RENDERER_JS)),
|
||||
"input": json.dumps(markdown),
|
||||
}
|
||||
result = subprocess.run(
|
||||
["node", "-e", harness],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KaTeX delimiter handling — both TeX and LaTeX styles
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tex_inline_math_renders() -> None:
|
||||
out = _render("The formula $E = mc^2$ is famous.")
|
||||
assert '<span class="katex">' in out
|
||||
assert "[KATEX:E = mc^2:inline]" in out
|
||||
assert "$E = mc^2$" not in out # raw delimiters consumed
|
||||
|
||||
|
||||
def test_tex_display_math_renders() -> None:
|
||||
out = _render("$$\nE = mc^2\n$$")
|
||||
assert '<span class="katex">' in out
|
||||
assert ":display]" in out
|
||||
|
||||
|
||||
def test_latex_inline_math_renders() -> None:
|
||||
r"""LaTeX-style \(...\) inline math. GPT-5 / o-series / Claude
|
||||
with reasoning effort emit this style by default; without
|
||||
explicit support the model output passed through as raw \(x\)
|
||||
text in coord + interactive UIs."""
|
||||
out = _render(r"The formula \(E = mc^2\) is famous.")
|
||||
assert '<span class="katex">' in out
|
||||
assert "[KATEX:E = mc^2:inline]" in out
|
||||
assert r"\(E = mc^2\)" not in out
|
||||
|
||||
|
||||
def test_latex_display_math_renders() -> None:
|
||||
r"""LaTeX-style \[...\] display math."""
|
||||
out = _render("Intro\n\n\\[\nE = mc^2\n\\]\n\nMore")
|
||||
assert '<span class="katex">' in out
|
||||
assert ":display]" in out
|
||||
assert "\\[" not in out
|
||||
assert "\\]" not in out
|
||||
|
||||
|
||||
def test_latex_math_in_list_item_renders() -> None:
|
||||
"""Nested-in-markdown-block — the original bug report. The list
|
||||
item is processed via line-by-line + inlineMarkdown; the math
|
||||
placeholder must survive that path."""
|
||||
out = _render(r"- Item with \(E = mc^2\) math")
|
||||
assert "<li>" in out
|
||||
assert '<span class="katex">' in out
|
||||
assert "[KATEX:E = mc^2:inline]" in out
|
||||
|
||||
|
||||
def test_latex_math_in_blockquote_renders() -> None:
|
||||
out = _render(r"> Note: \(x^2\) is squared.")
|
||||
assert "<blockquote>" in out
|
||||
assert '<span class="katex">' in out
|
||||
|
||||
|
||||
def test_latex_math_in_bold_renders() -> None:
|
||||
out = _render(r"Then **\(x^2\)** end.")
|
||||
assert "<strong>" in out
|
||||
assert '<span class="katex">' in out
|
||||
|
||||
|
||||
def test_mixed_tex_and_latex_styles() -> None:
|
||||
out = _render(r"Here $x$ then \(y\) end.")
|
||||
assert out.count('<span class="katex">') == 2
|
||||
assert "[KATEX:x:inline]" in out
|
||||
assert "[KATEX:y:inline]" in out
|
||||
|
||||
|
||||
def test_latex_math_inside_inline_code_preserved() -> None:
|
||||
r"""\(...\) inside inline code must NOT render as math —
|
||||
code is escaped + left literal."""
|
||||
out = _render(r"Code: `\(x\)` raw.")
|
||||
assert r"<code>\(x\)</code>" in out
|
||||
assert '<span class="katex">' not in out
|
||||
|
||||
|
||||
def test_latex_math_inside_fenced_code_preserved() -> None:
|
||||
r"""\(...\) inside a fenced block must stay literal."""
|
||||
out = _render("```\nA \\(x\\) sample\n```")
|
||||
assert "<pre><code>" in out
|
||||
assert '<span class="katex">' not in out
|
||||
|
||||
|
||||
def test_solo_escaped_bracket_does_not_render_as_math() -> None:
|
||||
r"""A lone \[ with no matching \] is not math — it's a markdown
|
||||
bracket escape. Don't hijack it."""
|
||||
out = _render(r"No math: \[ alone.")
|
||||
assert '<span class="katex">' not in out
|
||||
|
||||
|
||||
def test_markdown_link_unaffected_by_math_protection() -> None:
|
||||
r"""Math regex uses \[ / \] (escaped brackets), not bare [...].
|
||||
Markdown links must still render."""
|
||||
out = _render("See [docs](https://example.com).")
|
||||
assert '<a href="https://example.com"' in out
|
||||
assert ">docs</a>" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge cases — Copilot review on PR #425
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_display_math_inside_inline_code_stays_literal() -> None:
|
||||
r"""``$$...$$`` inside backticks must NOT trigger display-math
|
||||
extraction — otherwise the math sentinel ends up wrapped inside
|
||||
the <code> placeholder and leaks into rendered HTML as a raw
|
||||
null-byte sentinel string.
|
||||
|
||||
Pre-#425 ordering ran display-math before inline code, which
|
||||
caused this leak. The reordering makes inline code seal first.
|
||||
"""
|
||||
out = _render(r"Use `$$x$$` for display math.")
|
||||
assert "<code>$$x$$</code>" in out
|
||||
assert '<span class="katex">' not in out
|
||||
assert "\x00" not in out # no leaked sentinel
|
||||
|
||||
|
||||
def test_latex_display_math_inside_inline_code_stays_literal() -> None:
|
||||
r"""Same as above, but for the LaTeX-style \[...\] delimiter."""
|
||||
out = _render(r"Use `\[x\]` for display math.")
|
||||
assert r"<code>\[x\]</code>" in out
|
||||
assert '<span class="katex">' not in out
|
||||
assert "\x00" not in out
|
||||
|
||||
|
||||
def test_inline_latex_math_does_not_span_paragraphs() -> None:
|
||||
r"""An unterminated \(...\) on one line must not eat the
|
||||
following paragraph until it finds a closing \) — that would
|
||||
consume large chunks of text under streaming markdown where
|
||||
the closer hasn't arrived yet. Mirrors the $...$ behavior."""
|
||||
src = "Open \\(unterminated\n\nNext paragraph with \\(x\\) here."
|
||||
out = _render(src)
|
||||
# The bare \( on line 1 should NOT match; the well-formed \(x\)
|
||||
# on the second paragraph should render normally.
|
||||
assert out.count('<span class="katex">') == 1
|
||||
assert "[KATEX:x:inline]" in out
|
||||
# The "unterminated" stays as raw text.
|
||||
assert "unterminated" in out
|
||||
|
||||
|
||||
def test_inline_tex_math_does_not_span_newlines() -> None:
|
||||
"""Existing $...$ behavior — regression guard."""
|
||||
src = "Open $unterminated\n\nNext paragraph $x$ here."
|
||||
out = _render(src)
|
||||
assert out.count('<span class="katex">') == 1
|
||||
assert "[KATEX:x:inline]" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mermaid progressive rendering — source-keyed SVG cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_MERMAID_HARNESS_TEMPLATE = """
|
||||
const vm = require('vm');
|
||||
const fs = require('fs');
|
||||
|
||||
// Minimal DOM fake — enough surface for postRenderMermaid + the
|
||||
// mermaid render path. Each created element tracks its attributes,
|
||||
// classList, children, and parent so replaceWith works.
|
||||
function makeEl(tag) {
|
||||
const el = {
|
||||
tagName: tag.toUpperCase(),
|
||||
_attrs: {},
|
||||
_classes: new Set(),
|
||||
children: [],
|
||||
parent: null,
|
||||
_innerHTML: '',
|
||||
_textContent: '',
|
||||
setAttribute(k, v) { this._attrs[k] = v; },
|
||||
getAttribute(k) { return this._attrs[k] !== undefined ? this._attrs[k] : null; },
|
||||
get classList() {
|
||||
const self = this;
|
||||
return {
|
||||
add(...c) { c.forEach(x => self._classes.add(x)); },
|
||||
remove(...c) { c.forEach(x => self._classes.delete(x)); },
|
||||
contains(c) { return self._classes.has(c); },
|
||||
};
|
||||
},
|
||||
get className() { return Array.from(this._classes).join(' '); },
|
||||
set className(v) {
|
||||
this._classes = new Set(String(v).split(/\\s+/).filter(Boolean));
|
||||
},
|
||||
get textContent() {
|
||||
return this._textContent || this.children.map(c => c.textContent || '').join('');
|
||||
},
|
||||
set textContent(v) { this._textContent = v; this.children = []; },
|
||||
get innerHTML() { return this._innerHTML; },
|
||||
set innerHTML(v) { this._innerHTML = v; this.children = []; },
|
||||
get isConnected() {
|
||||
// In real DOM this checks attachment to the document; for the
|
||||
// test harness we approximate via the parent chain. After
|
||||
// replaceWith, the displaced element's parent is nulled so
|
||||
// its isConnected goes false — which is exactly the
|
||||
// detached-during-streaming case the production guard
|
||||
// protects against.
|
||||
return !!this.parent;
|
||||
},
|
||||
appendChild(c) {
|
||||
c.parent = this;
|
||||
this.children.push(c);
|
||||
return c;
|
||||
},
|
||||
closest(selector) {
|
||||
const t = selector.toUpperCase();
|
||||
let cur = this;
|
||||
while (cur) {
|
||||
if (cur.tagName === t) return cur;
|
||||
cur = cur.parent;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
replaceWith(other) {
|
||||
if (!this.parent) return;
|
||||
const idx = this.parent.children.indexOf(this);
|
||||
if (idx === -1) return;
|
||||
this.parent.children[idx] = other;
|
||||
other.parent = this.parent;
|
||||
this.parent = null;
|
||||
},
|
||||
querySelectorAll(selector) {
|
||||
// Only supports the literal "pre code.language-mermaid"
|
||||
// selector that postRenderMermaid uses.
|
||||
const out = [];
|
||||
function walk(node) {
|
||||
for (const c of (node.children || [])) {
|
||||
if (
|
||||
c.tagName === 'CODE' &&
|
||||
c.parent && c.parent.tagName === 'PRE' &&
|
||||
c._classes.has('language-mermaid')
|
||||
) {
|
||||
out.push(c);
|
||||
}
|
||||
walk(c);
|
||||
}
|
||||
}
|
||||
walk(this);
|
||||
return out;
|
||||
},
|
||||
};
|
||||
return el;
|
||||
}
|
||||
|
||||
global.document = {
|
||||
createElement: makeEl,
|
||||
addEventListener: () => {},
|
||||
getElementById: () => null,
|
||||
head: { appendChild: () => {} },
|
||||
documentElement: {},
|
||||
};
|
||||
global.window = global;
|
||||
global.getComputedStyle = () => ({ getPropertyValue: () => '' });
|
||||
|
||||
let renderCallCount = 0;
|
||||
let renderShouldFail = false;
|
||||
global.mermaid = {
|
||||
initialize: () => {},
|
||||
render: (id, source) => {
|
||||
renderCallCount++;
|
||||
if (renderShouldFail) {
|
||||
return Promise.reject(new Error('bad diagram: ' + source));
|
||||
}
|
||||
return Promise.resolve({
|
||||
svg: '<svg data-source="' + source + '">rendered</svg>',
|
||||
bindFunctions: null,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
vm.runInThisContext(fs.readFileSync(%(utils)s, 'utf8'));
|
||||
vm.runInThisContext(fs.readFileSync(%(renderer)s, 'utf8'));
|
||||
|
||||
// Mermaid is normally lazy-loaded via _loadMermaid which fetches a
|
||||
// script tag. Force-mark it ready so postRenderMermaid invokes the
|
||||
// render path synchronously without trying to inject a script.
|
||||
_mermaidState = 'ready';
|
||||
|
||||
%(scenario)s
|
||||
"""
|
||||
|
||||
|
||||
def _run_mermaid_scenario(scenario_js: str) -> dict[str, Any]:
|
||||
"""Run a JS snippet against the mermaid-aware harness, return JSON output."""
|
||||
harness = _MERMAID_HARNESS_TEMPLATE % {
|
||||
"utils": json.dumps(str(_UTILS_JS)),
|
||||
"renderer": json.dumps(str(_RENDERER_JS)),
|
||||
"scenario": scenario_js,
|
||||
}
|
||||
result = subprocess.run(
|
||||
["node", "-e", harness],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
parsed: dict[str, Any] = json.loads(result.stdout)
|
||||
return parsed
|
||||
|
||||
|
||||
def _build_mermaid_container_js(sources: list[str]) -> str:
|
||||
"""JS expression that builds a container with ``<pre><code language-mermaid>`` blocks."""
|
||||
src_array = "[" + ", ".join(json.dumps(s) for s in sources) + "]"
|
||||
return f"""
|
||||
function buildContainer(sources) {{
|
||||
const container = document.createElement('div');
|
||||
for (const src of sources) {{
|
||||
const pre = document.createElement('pre');
|
||||
const code = document.createElement('code');
|
||||
code.classList.add('language-mermaid');
|
||||
code.textContent = src;
|
||||
pre.appendChild(code);
|
||||
container.appendChild(pre);
|
||||
}}
|
||||
return container;
|
||||
}}
|
||||
const sources = {src_array};
|
||||
const container = buildContainer(sources);
|
||||
"""
|
||||
|
||||
|
||||
# Drain microtasks + global mermaid render chain. Wraps the async
|
||||
# work in a setTimeout(0) hop so all queued microtasks (including
|
||||
# the per-source pending list draining via _mermaidRenderChain)
|
||||
# flush before the assertion script reads cache state.
|
||||
_MERMAID_DRAIN_JS = """
|
||||
function drainAndReport(report) {
|
||||
// Two setTimeout hops give the global chain time to resolve
|
||||
// mermaid.render's promise + the .then handlers that populate
|
||||
// the cache and call _applyMermaidSvg.
|
||||
setTimeout(() => setTimeout(() => {
|
||||
process.stdout.write(JSON.stringify(report()));
|
||||
}, 0), 0);
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def test_mermaid_cache_hit_skips_render_call() -> None:
|
||||
"""Identical source on a second postRenderMermaid call must serve
|
||||
from the cache — mermaid.render runs exactly once across both
|
||||
invocations. This is the core invariant that lets streamingRender
|
||||
fire postRenderMermaid on every rAF tick without thrashing."""
|
||||
scenario = (
|
||||
_build_mermaid_container_js(["graph TD\n A --> B"])
|
||||
+ _MERMAID_DRAIN_JS
|
||||
+ """
|
||||
postRenderMermaid(container);
|
||||
setTimeout(() => setTimeout(() => {
|
||||
// Second invocation — fresh container, same source. Should NOT
|
||||
// call mermaid.render again because the cache holds the SVG.
|
||||
const container2 = buildContainer(sources);
|
||||
postRenderMermaid(container2);
|
||||
setTimeout(() => {
|
||||
process.stdout.write(JSON.stringify({
|
||||
renderCalls: renderCallCount,
|
||||
cacheSize: _mermaidSvgCache.size,
|
||||
firstClass: container.children[0].className,
|
||||
secondClass: container2.children[0].className,
|
||||
}));
|
||||
}, 0);
|
||||
}, 0), 0);
|
||||
"""
|
||||
)
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
assert out["renderCalls"] == 1, "second postRenderMermaid call invoked render — cache miss"
|
||||
assert out["cacheSize"] == 1
|
||||
# Both containers end up with the rendered class — second from cache.
|
||||
assert "mermaid-rendered" in out["firstClass"]
|
||||
assert "mermaid-rendered" in out["secondClass"]
|
||||
|
||||
|
||||
def test_mermaid_distinct_sources_render_independently() -> None:
|
||||
"""Two distinct sources each trigger mermaid.render once and are
|
||||
cached separately. Verifies the cache key is the source string,
|
||||
not e.g. a positional index."""
|
||||
scenario = (
|
||||
_build_mermaid_container_js(["graph TD\n A --> B", "sequenceDiagram\n A->>B: hi"])
|
||||
+ """
|
||||
postRenderMermaid(container);
|
||||
// Drain twice — across-source serialization means the second
|
||||
// render starts only after the first lands.
|
||||
setTimeout(() => setTimeout(() => setTimeout(() => {
|
||||
process.stdout.write(JSON.stringify({
|
||||
renderCalls: renderCallCount,
|
||||
cacheSize: _mermaidSvgCache.size,
|
||||
}));
|
||||
}, 0), 0), 0);
|
||||
"""
|
||||
)
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
assert out["renderCalls"] == 2
|
||||
assert out["cacheSize"] == 2
|
||||
|
||||
|
||||
def test_mermaid_error_cached_to_avoid_thrash() -> None:
|
||||
"""A mermaid render failure caches the error message keyed by
|
||||
source, so subsequent postRenderMermaid calls on the same source
|
||||
don't re-invoke mermaid.render only to re-fail."""
|
||||
scenario = (
|
||||
_build_mermaid_container_js(["bogus diagram"])
|
||||
+ """
|
||||
renderShouldFail = true;
|
||||
postRenderMermaid(container);
|
||||
setTimeout(() => setTimeout(() => {
|
||||
// Re-run with same source — should hit error cache.
|
||||
const container2 = buildContainer(sources);
|
||||
postRenderMermaid(container2);
|
||||
setTimeout(() => {
|
||||
process.stdout.write(JSON.stringify({
|
||||
renderCalls: renderCallCount,
|
||||
errorCacheSize: _mermaidErrorCache.size,
|
||||
svgCacheSize: _mermaidSvgCache.size,
|
||||
secondClass: container2.children[0].className,
|
||||
}));
|
||||
}, 0);
|
||||
}, 0), 0);
|
||||
"""
|
||||
)
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
assert out["renderCalls"] == 1, "errored source re-invoked mermaid.render — error cache miss"
|
||||
assert out["errorCacheSize"] == 1
|
||||
assert out["svgCacheSize"] == 0
|
||||
# Second container shows the error class without re-rendering.
|
||||
assert "mermaid-error" in out["secondClass"]
|
||||
|
||||
|
||||
def test_mermaid_cache_evicts_oldest_at_cap() -> None:
|
||||
"""FIFO eviction at _MERMAID_CACHE_MAX prevents unbounded growth
|
||||
on long sessions emitting many distinct diagrams."""
|
||||
scenario = """
|
||||
const cap = _MERMAID_CACHE_MAX;
|
||||
for (let i = 0; i < cap + 5; i++) {
|
||||
_cacheMermaidEntry(_mermaidSvgCache, 'src-' + i, {svg: 'svg-' + i, bindFunctions: null});
|
||||
}
|
||||
process.stdout.write(JSON.stringify({
|
||||
size: _mermaidSvgCache.size,
|
||||
hasOldest: _mermaidSvgCache.has('src-0'),
|
||||
hasNewest: _mermaidSvgCache.has('src-' + (cap + 4)),
|
||||
}));
|
||||
"""
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
assert out["size"] == 64
|
||||
assert out["hasOldest"] is False
|
||||
assert out["hasNewest"] is True
|
||||
|
||||
|
||||
def test_mermaid_overwrite_does_not_evict() -> None:
|
||||
"""Overwriting an existing key is an in-place update, not a new
|
||||
insertion — should not evict the oldest entry. Pre-fix, an
|
||||
update at cap would unnecessarily drop an unrelated cached SVG."""
|
||||
scenario = """
|
||||
const cap = _MERMAID_CACHE_MAX;
|
||||
// Fill exactly to cap.
|
||||
for (let i = 0; i < cap; i++) {
|
||||
_cacheMermaidEntry(_mermaidSvgCache, 'src-' + i, {svg: 'svg-' + i, bindFunctions: null});
|
||||
}
|
||||
// Overwrite an existing entry — must not evict src-0.
|
||||
_cacheMermaidEntry(_mermaidSvgCache, 'src-5', {svg: 'svg-updated', bindFunctions: null});
|
||||
process.stdout.write(JSON.stringify({
|
||||
size: _mermaidSvgCache.size,
|
||||
hasOldest: _mermaidSvgCache.has('src-0'),
|
||||
updated: _mermaidSvgCache.get('src-5').svg,
|
||||
}));
|
||||
"""
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
assert out["size"] == 64
|
||||
assert out["hasOldest"] is True, "overwrite evicted oldest unnecessarily"
|
||||
assert out["updated"] == "svg-updated"
|
||||
|
||||
|
||||
def test_mermaid_cache_cleared_on_init() -> None:
|
||||
"""_initMermaid must clear both caches so a theme change via
|
||||
reRenderAllMermaid doesn't serve stale SVG keyed by source-only
|
||||
— the rendered output depends on themeVariables which change
|
||||
on init."""
|
||||
scenario = """
|
||||
_cacheMermaidEntry(_mermaidSvgCache, 'src-1', {svg: 'old', bindFunctions: null});
|
||||
_cacheMermaidEntry(_mermaidErrorCache, 'src-bad', 'old error');
|
||||
_initMermaid();
|
||||
process.stdout.write(JSON.stringify({
|
||||
svgSize: _mermaidSvgCache.size,
|
||||
errorSize: _mermaidErrorCache.size,
|
||||
}));
|
||||
"""
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
assert out["svgSize"] == 0
|
||||
assert out["errorSize"] == 0
|
||||
|
||||
|
||||
def test_mermaid_cache_hit_reapplies_bind_functions() -> None:
|
||||
"""bindFunctions returned by mermaid.render attach link/click
|
||||
handlers to the rendered SVG. Cache hits must re-invoke this
|
||||
on the new container instance — pre-fix, only the first render
|
||||
got bindings; subsequent cache hits via innerHTML left the SVG
|
||||
inert."""
|
||||
scenario = (
|
||||
_build_mermaid_container_js(["graph TD\n A --> B"])
|
||||
+ """
|
||||
let bindCallCount = 0;
|
||||
const origRender = mermaid.render;
|
||||
mermaid.render = (id, source) => {
|
||||
return Promise.resolve({
|
||||
svg: '<svg>render</svg>',
|
||||
bindFunctions: () => { bindCallCount++; },
|
||||
});
|
||||
};
|
||||
postRenderMermaid(container);
|
||||
setTimeout(() => setTimeout(() => {
|
||||
// Second invocation — cache hit, should still call
|
||||
// bindFunctions on the new container.
|
||||
const container2 = buildContainer(sources);
|
||||
postRenderMermaid(container2);
|
||||
setTimeout(() => {
|
||||
process.stdout.write(JSON.stringify({
|
||||
bindCallCount: bindCallCount,
|
||||
}));
|
||||
}, 0);
|
||||
}, 0), 0);
|
||||
"""
|
||||
)
|
||||
out = _run_mermaid_scenario(scenario)
|
||||
# First render binds; cache hit on second container also binds.
|
||||
assert out["bindCallCount"] == 2, (
|
||||
"bindFunctions was not re-applied on cache hit — interactive "
|
||||
"diagram features (links, callbacks) would silently break"
|
||||
)
|
||||
|
||||
|
||||
def test_streaming_render_invokes_mermaid_post_render() -> None:
|
||||
"""_streamingRenderApply must call postRenderMermaid so closed
|
||||
mermaid fences appear progressively during streaming, not only
|
||||
at stream_end via streamingRenderFinalize."""
|
||||
body = _RENDERER_JS.read_text(encoding="utf-8")
|
||||
# Bound the search to a window after the function declaration —
|
||||
# avoids the brittleness of stopping at the first inner-block
|
||||
# closing brace.
|
||||
start = body.index("function _streamingRenderApply")
|
||||
mermaid_call = body.find("postRenderMermaid(el)", start, start + 4000)
|
||||
assert mermaid_call != -1, (
|
||||
"_streamingRenderApply must call postRenderMermaid for "
|
||||
"progressive diagram rendering during streaming"
|
||||
)
|
||||
@@ -1,138 +0,0 @@
|
||||
"""Tests for turnstone.core.rendezvous (HRW routing primitive)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.rendezvous import NoAvailableNodeError, NodeRef, fnv1a_32, select, select_all
|
||||
|
||||
|
||||
class TestFnv1aVectors:
|
||||
"""Pin the FNV-1a-32 implementation against the documented test
|
||||
vectors so cross-language readers (Go, TS) stay in sync."""
|
||||
|
||||
def test_empty_input(self) -> None:
|
||||
assert fnv1a_32(b"") == 0x811C9DC5 # basis
|
||||
|
||||
def test_foobar(self) -> None:
|
||||
assert fnv1a_32(b"foobar") == 0xBF9CF968
|
||||
|
||||
def test_single_byte(self) -> None:
|
||||
# Hand-computed: (basis ^ 0x61) * prime, masked to 32 bits.
|
||||
expected = ((0x811C9DC5 ^ 0x61) * 0x01000193) & 0xFFFFFFFF
|
||||
assert fnv1a_32(b"a") == expected
|
||||
|
||||
|
||||
class TestSelect:
|
||||
def test_empty_node_list_raises(self) -> None:
|
||||
with pytest.raises(NoAvailableNodeError):
|
||||
select("any-key", [])
|
||||
|
||||
def test_single_node_always_wins(self) -> None:
|
||||
only = NodeRef("solo", "http://solo")
|
||||
for key in ("a", "b", "00ff" + "0" * 28):
|
||||
assert select(key, [only]) is only
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(5)]
|
||||
key = "deadbeef" * 4
|
||||
first = select(key, nodes)
|
||||
for _ in range(20):
|
||||
assert select(key, nodes) is first
|
||||
|
||||
def test_independent_of_node_list_order(self) -> None:
|
||||
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(5)]
|
||||
key = "feedface" * 4
|
||||
forward = select(key, nodes)
|
||||
backward = select(key, list(reversed(nodes)))
|
||||
assert forward.node_id == backward.node_id
|
||||
|
||||
def test_distribution_roughly_uniform(self) -> None:
|
||||
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(4)]
|
||||
counts = {n.node_id: 0 for n in nodes}
|
||||
# Use sequential keys — 32 hex chars is what the router actually
|
||||
# passes in. Sequential isn't a problem because FNV-1a smears.
|
||||
for i in range(4000):
|
||||
key = f"{i:08x}" + "0" * 24
|
||||
counts[select(key, nodes).node_id] += 1
|
||||
# Each node should win ~25% (1000); allow ±15% drift.
|
||||
for c in counts.values():
|
||||
assert 850 < c < 1150, counts
|
||||
|
||||
|
||||
class TestMinimalMoves:
|
||||
def test_join_only_moves_to_new_node(self) -> None:
|
||||
old = [NodeRef(f"n{i}", f"http://n{i}") for i in range(3)]
|
||||
new = [*old, NodeRef("n3", "http://n3")]
|
||||
moved_correctly = 0
|
||||
moved_incorrectly = 0
|
||||
for i in range(2000):
|
||||
key = f"{i:08x}" + "0" * 24
|
||||
before = select(key, old).node_id
|
||||
after = select(key, new).node_id
|
||||
if before == after:
|
||||
continue
|
||||
if after == "n3":
|
||||
moved_correctly += 1
|
||||
else:
|
||||
moved_incorrectly += 1
|
||||
# Strict invariant: a join must never move a key between two
|
||||
# surviving nodes.
|
||||
assert moved_incorrectly == 0
|
||||
# Sanity: some keys did move.
|
||||
assert moved_correctly > 0
|
||||
|
||||
def test_leave_does_not_disturb_surviving_nodes(self) -> None:
|
||||
old = [NodeRef(f"n{i}", f"http://n{i}") for i in range(4)]
|
||||
new = old[:-1] # n3 leaves
|
||||
for i in range(2000):
|
||||
key = f"{i:08x}" + "0" * 24
|
||||
before = select(key, old).node_id
|
||||
after = select(key, new).node_id
|
||||
if before == "n3":
|
||||
# Must rehome to a survivor.
|
||||
assert after in {"n0", "n1", "n2"}
|
||||
else:
|
||||
# Must not move.
|
||||
assert after == before
|
||||
|
||||
|
||||
class TestWeights:
|
||||
def test_higher_weight_wins_more_often(self) -> None:
|
||||
nodes = [
|
||||
NodeRef("light", "http://l", weight=1),
|
||||
NodeRef("heavy", "http://h", weight=4),
|
||||
]
|
||||
on_heavy = 0
|
||||
for i in range(5000):
|
||||
key = f"{i:08x}" + "0" * 24
|
||||
if select(key, nodes).node_id == "heavy":
|
||||
on_heavy += 1
|
||||
# Heavy gets clearly more than half; tolerance for the simple
|
||||
# hash×weight formulation is wide.
|
||||
assert on_heavy / 5000 > 0.65
|
||||
|
||||
def test_zero_weight_clamped_to_one(self) -> None:
|
||||
# A weight-0 node still participates as if weight 1 — defended
|
||||
# at both NodeRef construction and _score(). Use the public
|
||||
# surface to sanity check.
|
||||
nodes = [
|
||||
NodeRef("a", "http://a", weight=0),
|
||||
NodeRef("b", "http://b", weight=0),
|
||||
]
|
||||
# Just confirms it doesn't divide-by-zero or score to 0.
|
||||
winner = select("any-key", nodes)
|
||||
assert winner.node_id in {"a", "b"}
|
||||
|
||||
|
||||
class TestSelectAll:
|
||||
def test_returns_all_nodes_in_score_order(self) -> None:
|
||||
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(4)]
|
||||
ranked = select_all("some-key", nodes)
|
||||
assert len(ranked) == 4
|
||||
assert {n.node_id for n in ranked} == {"n0", "n1", "n2", "n3"}
|
||||
# Top of the ranked list matches the single-select winner.
|
||||
assert ranked[0] is select("some-key", nodes)
|
||||
|
||||
def test_empty_list_returns_empty(self) -> None:
|
||||
assert select_all("any-key", []) == []
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user