mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-14 07:52:25 -06:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fd47c23177 | |||
| 9fe988b1be | |||
| 3ce66960bc | |||
| 8a852a12e3 | |||
| 9a518657a3 | |||
| c424176c73 |
+5
-5
@@ -23,18 +23,18 @@ RUN useradd --create-home --shell /bin/bash turnstone
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Compile bytecode for faster startup
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
|
||||
# Install dependencies first (cached layer — only re-runs when deps change)
|
||||
COPY pyproject.toml uv.lock README.md LICENSE ./
|
||||
RUN uv sync --frozen --no-install-project --no-dev \
|
||||
--extra all
|
||||
--no-compile --extra all
|
||||
|
||||
# Install the project itself
|
||||
COPY turnstone/ turnstone/
|
||||
RUN uv sync --frozen --no-dev \
|
||||
--extra all
|
||||
--no-compile --extra all
|
||||
|
||||
# Compile bytecode in a separate step (avoids fd exhaustion during install)
|
||||
RUN python -m compileall -q .venv turnstone/
|
||||
|
||||
# Add venv to PATH so entry points are found
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
@@ -127,7 +127,6 @@ services:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port=8090
|
||||
- --poll-interval=${CONSOLE_POLL_INTERVAL:-10}
|
||||
ports:
|
||||
- "${CONSOLE_PORT:-8090}:8090"
|
||||
environment:
|
||||
|
||||
+20
-20
@@ -74,7 +74,7 @@ turnstone/
|
||||
_sync.py Background event loop for sync wrappers
|
||||
_types.py TurnResult + TurnstoneAPIError
|
||||
console/
|
||||
collector.py ClusterCollector — aggregates state from all nodes via HTTP
|
||||
collector.py ClusterCollector — aggregates state from all nodes via SSE
|
||||
scheduler.py TaskScheduler — background cron/at scheduler, dispatches via HTTP
|
||||
server.py Cluster dashboard HTTP server + SSE + CLI entry point
|
||||
static/ Cluster dashboard web UI (page-specific HTML, CSS, JS)
|
||||
@@ -1201,31 +1201,31 @@ bell + status line to stderr to alert the user.
|
||||
### Cluster Console
|
||||
|
||||
```
|
||||
Monitoring (3 daemon threads) Control + Proxy (async Starlette)
|
||||
Monitoring (2 daemon threads) Control + Proxy (async Starlette)
|
||||
+------------------+ +----------------------------+
|
||||
| Event subscriber | | POST /v1/api/cluster/ |
|
||||
| SSE on | | workstreams/new |
|
||||
| /events/glob | | → POST to target server |
|
||||
| Node discovery | | POST /v1/api/cluster/ |
|
||||
| Service registry | | workstreams/new |
|
||||
| every 60 seconds | | → POST to target server |
|
||||
+------------------+ +----------------------------+
|
||||
| Node discovery | | GET /node/{node_id}/ |
|
||||
| Service registry | | → httpx.AsyncClient |
|
||||
| every 15 seconds | | proxy to server_url |
|
||||
+------------------+ | GET /node/{id}/v1/api/events |
|
||||
| Poll loop | | → SSE stream proxy |
|
||||
| GET /v1/api/dash | | POST /node/{id}/v1/api/send |
|
||||
| GET /health | | → forwarded to server |
|
||||
| ThreadPoolExec | +----------------------------+
|
||||
+------------------+
|
||||
| SSE manager | | GET /node/{node_id}/ |
|
||||
| asyncio loop | | → httpx.AsyncClient |
|
||||
| 1 task per node | | proxy to server_url |
|
||||
| /events/global | | GET /node/{id}/v1/api/events |
|
||||
| snapshot+deltas | | → SSE stream proxy |
|
||||
+------------------+ | POST /node/{id}/v1/api/send |
|
||||
| → forwarded to server |
|
||||
+----------------------------+
|
||||
```
|
||||
|
||||
The console HTTP layer is a Starlette/ASGI app served by uvicorn. The SSE
|
||||
endpoint uses `EventSourceResponse` with the same listener queue pattern as
|
||||
the main server. `ClusterCollector`'s background threads (event subscriber,
|
||||
node discovery, poll loop) use `ThreadPoolExecutor`
|
||||
for parallel HTTP polling. The poll loop diffs workstream IDs between poll
|
||||
cycles and fans out synthetic `ws_created`/`ws_closed` SSE events for any
|
||||
changes, ensuring browser clients stay in sync even when real-time cluster
|
||||
events are missed.
|
||||
the main server. `ClusterCollector` runs two daemon threads: a discovery loop
|
||||
that queries the service registry every 60 seconds, and an SSE manager that
|
||||
runs a single asyncio event loop multiplexing persistent SSE connections to
|
||||
all nodes via `GET /v1/api/events/global`. Each node delivers a full snapshot
|
||||
on connect followed by real-time delta events — state changes, health
|
||||
transitions, and aggregate metrics arrive sub-second instead of on a 15-second
|
||||
poll cycle.
|
||||
|
||||
The console has two write-path capabilities:
|
||||
|
||||
|
||||
+6
-9
@@ -1,6 +1,6 @@
|
||||
# Cluster Dashboard (turnstone-console)
|
||||
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It discovers nodes via the `services` database table, polls each node's HTTP API for workstream data, and receives real-time state changes via HTTP polling.
|
||||
`turnstone-console` is a cluster management service that provides cluster-wide visibility and control across all turnstone nodes. It discovers nodes via the `services` database table and subscribes to each node's SSE event stream for real-time workstream, health, and metric updates.
|
||||
|
||||
The console also supports **workstream creation** (dispatched via HTTP proxy to target nodes) and a **reverse proxy** that serves each node's server UI through the console port — so users only need network access to the console, not to individual server nodes.
|
||||
|
||||
@@ -21,7 +21,7 @@ turnstone-console ──────┤
|
||||
|
||||
Data flows in two directions:
|
||||
|
||||
- **Inbound (monitoring):** The console discovers nodes via the `services` database table (nodes register on startup and send periodic heartbeats). It periodically polls each node's `GET /v1/api/dashboard` for full workstream snapshots and `GET /health` for node health.
|
||||
- **Inbound (monitoring):** The console discovers nodes via the `services` database table (nodes register on startup and send periodic heartbeats). It opens a persistent SSE connection to each node's `GET /v1/api/events/global` endpoint, receiving a full snapshot on connect followed by real-time delta events (state changes, health transitions, aggregate metrics).
|
||||
- **Outbound (control):** The console proxies workstream creation requests to target nodes via HTTP.
|
||||
- **Proxy (pass-through):** The console reverse-proxies each node's server UI at `/node/{node_id}/`, forwarding HTTP and SSE traffic so the browser never contacts server nodes directly.
|
||||
|
||||
@@ -30,8 +30,7 @@ Data flows in two directions:
|
||||
| Source | Method | Direction | Data |
|
||||
|--------|--------|-----------|------|
|
||||
| `services` table | Database query | Read | Node discovery (node_id, server_url, started) |
|
||||
| Node HTTP API | `GET {server_url}/v1/api/dashboard` | Read | Full workstream list with tokens, context, activity |
|
||||
| Node HTTP API | `GET {server_url}/health` | Read | Node health status |
|
||||
| Node SSE | `GET {server_url}/v1/api/events/global` | Stream | Snapshot on connect, then real-time delta events (state, health, aggregate) |
|
||||
| Node HTTP API | `POST {server_url}/v1/api/workstreams/new` | Write | Workstream creation |
|
||||
| Node HTTP API | `GET/POST {server_url}/*` | Proxy | Server UI, API requests, SSE streams |
|
||||
|
||||
@@ -41,9 +40,9 @@ Data flows in two directions:
|
||||
|
||||
The collector (`turnstone/console/collector.py`) maintains an in-memory snapshot of all nodes and workstreams. Two daemon threads handle data acquisition:
|
||||
|
||||
1. **Node discovery** — queries the `services` database table every 15 seconds. Adds newly discovered nodes, removes expired ones (stale heartbeats), emits `node_joined` / `node_lost` events to SSE listeners.
|
||||
1. **Node discovery** — queries the `services` database table every 60 seconds. Adds newly discovered nodes, removes expired ones (stale heartbeats), emits `node_joined` / `node_lost` events to SSE listeners, and spawns/cancels SSE tasks for new/lost nodes.
|
||||
|
||||
2. **Poll loop** — fetches `GET /v1/api/dashboard` and `GET /health` from each known node every 10 seconds. Uses `ThreadPoolExecutor(max_workers=50)` for parallelism. Each poll replaces the node's workstream list with the authoritative server data.
|
||||
2. **SSE manager** — a single asyncio event loop on one thread multiplexes persistent SSE connections to all discovered nodes via `GET /v1/api/events/global`. Each connection receives a `node_snapshot` on connect (workstreams, health, aggregate) followed by real-time delta events (`ws_state`, `ws_created`, `ws_closed`, `ws_rename`, `health_changed`, `aggregate`). On disconnect, the node is marked unreachable and the connection is retried with exponential backoff (1s–30s). An `?expected_node_id=` query parameter provides identity verification against IP reuse (server returns 409 on mismatch).
|
||||
|
||||
A `get_snapshot()` method builds the full cluster state under a single lock acquisition — overview aggregates and per-node workstream lists in one atomic read. This is served both as a REST endpoint and as the initial SSE event on client connect.
|
||||
|
||||
@@ -54,7 +53,7 @@ All reads and writes to the node/workstream map are protected by a single `threa
|
||||
### Scale Considerations
|
||||
|
||||
- **50,000 workstreams** (1,000 nodes × 50 per node) at ~500 bytes each = ~25 MB in memory
|
||||
- **1,000 nodes** polled in parallel — fan-out concurrency is configurable via `cluster.node_fan_out_limit` (default 200), yielding 5 batches at ~100ms each = ~0.5 second poll cycle
|
||||
- **1,000 nodes** connected via persistent SSE — a single asyncio event loop multiplexes all connections with negligible overhead. Ensure `ulimit -n` >= 4096 for fd headroom
|
||||
- **Filtering and pagination** run in-memory on the full workstream list — sub-millisecond at this scale
|
||||
- **SSE fan-out** uses per-client queues (2,000 events) — backed-up clients get events dropped, not blocking
|
||||
- **Database** — for clusters sharing PostgreSQL, use [PgBouncer](pgbouncer.md) in transaction pooling mode
|
||||
@@ -629,7 +628,6 @@ CLI flags for `turnstone-console`:
|
||||
|------|---------|-------------|
|
||||
| `--host` | `0.0.0.0` | Bind host |
|
||||
| `--port` | `8090` | HTTP port |
|
||||
| `--poll-interval` | `10` | Node polling interval (seconds) |
|
||||
| `--auth-token` | `$TURNSTONE_AUTH_TOKEN` | Bearer token for server node communication and proxy |
|
||||
| `--log-level` | `INFO` | Log level |
|
||||
|
||||
@@ -640,7 +638,6 @@ Config file (`~/.config/turnstone/config.toml`):
|
||||
host = "0.0.0.0"
|
||||
port = 8090
|
||||
url = "http://localhost:8090" # used by CLI /cluster commands
|
||||
poll_interval = 10
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -10,55 +10,76 @@ participant "ClusterCollector" as CC
|
||||
participant "Node-A\n(server)" as NodeA
|
||||
participant "Node-B\n(server)" as NodeB
|
||||
|
||||
== Thread 1: HTTP Polling (every 10s) ==
|
||||
== Thread 1: Node Discovery (every 60s) ==
|
||||
|
||||
CC -> CC : Iterate registered nodes
|
||||
CC -> CC : list_services("server",\nmax_age_seconds=120)
|
||||
activate CC #C8E6C9
|
||||
|
||||
CC -> NodeA : GET /v1/api/dashboard
|
||||
activate NodeA
|
||||
NodeA --> CC : {workstreams: [...],\naggregate: {total_tokens, ...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> NodeA : GET /health
|
||||
activate NodeA
|
||||
NodeA --> CC : {status:"ok", version:"0.9.2",\nmodel:"...", workstreams:{...}}
|
||||
deactivate NodeA
|
||||
|
||||
CC -> NodeB : GET /v1/api/dashboard
|
||||
activate NodeB
|
||||
NodeB --> CC : {workstreams: [...],\naggregate: {total_tokens, ...}}
|
||||
deactivate NodeB
|
||||
|
||||
CC -> CC : Diff old vs new workstream IDs
|
||||
CC -> CC : Replace NodeSnapshot["nodeA"]\n.workstreams, .health, .aggregate
|
||||
CC -> CC : _fanout(ws_created) for\nnewly appeared workstreams
|
||||
CC -> CC : _fanout(ws_closed) for\nremoved workstreams
|
||||
|
||||
note right of CC
|
||||
Poll-diff fanout ensures
|
||||
browser SSE clients learn
|
||||
about workstreams that
|
||||
appeared between polls.
|
||||
end note
|
||||
CC -> CC : New node? → spawn SSE task\nLost node? → cancel SSE task
|
||||
CC -> CC : _fanout(node_joined)\n_fanout(node_lost)
|
||||
|
||||
deactivate CC
|
||||
|
||||
== Thread 2: SSE Manager (asyncio event loop) ==
|
||||
|
||||
note over CC
|
||||
Single asyncio event loop multiplexes
|
||||
one persistent SSE connection per node.
|
||||
Scales to 1000+ nodes.
|
||||
end note
|
||||
|
||||
CC -> NodeA : GET /v1/api/events/global\n?expected_node_id=nodeA
|
||||
activate NodeA
|
||||
activate CC #BBDEFB
|
||||
|
||||
NodeA --> CC : data: {"type":"node_snapshot",\n"node_id":"nodeA",\n"workstreams":[...],\n"health":{...},\n"aggregate":{...}}
|
||||
|
||||
note right of CC
|
||||
Snapshot populates NodeSnapshot
|
||||
in-memory state. Reconciles
|
||||
against stale data (emits
|
||||
ws_created/ws_closed diffs).
|
||||
end note
|
||||
|
||||
loop real-time delta events
|
||||
NodeA --> CC : data: {"type":"ws_state",\n"ws_id":"ws1","state":"running"}
|
||||
CC -> CC : Update NodeSnapshot\n_fanout(cluster_state)
|
||||
end
|
||||
|
||||
alt health transition
|
||||
NodeA --> CC : data: {"type":"health_changed",\n"circuit_state":"open"}
|
||||
CC -> CC : Update node.health
|
||||
end
|
||||
|
||||
alt periodic aggregate (every 10s)
|
||||
NodeA --> CC : data: {"type":"aggregate",\n"total_tokens":50000}
|
||||
CC -> CC : Update node.aggregate
|
||||
end
|
||||
|
||||
deactivate CC
|
||||
deactivate NodeA
|
||||
|
||||
alt SSE disconnect
|
||||
CC -> CC : Mark node unreachable\nReconnect with backoff\n(1s → 30s cap)
|
||||
end
|
||||
|
||||
alt identity mismatch (409 or snapshot node_id differs)
|
||||
CC -> CC : Mark node unreachable\nStop reconnecting to this URL
|
||||
end
|
||||
|
||||
== Browser SSE Stream ==
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/events
|
||||
activate Server
|
||||
|
||||
Server -> CC : get_snapshot()
|
||||
Server -> CC : get_snapshot_and_register(queue)
|
||||
note right : Atomic: snapshot + listener\nregistration under both locks\n→ no event gap
|
||||
CC --> Server : ClusterSnapshot\n(full current state)
|
||||
|
||||
Server -> CC : register_listener(queue)
|
||||
note right : Per-client queue.Queue(maxsize=2000)\nSSE via EventSourceResponse + run_in_executor()
|
||||
|
||||
Server -> Browser : data: {"type":"snapshot",...}\n(full state as first SSE event)
|
||||
|
||||
loop continuous (incremental updates)
|
||||
CC -> Server : event via listener queue\n(from polling thread)
|
||||
CC -> Server : event via listener queue\n(from SSE manager thread)
|
||||
Server -> Browser : data: {"type":"cluster_state",...}\n\n
|
||||
end
|
||||
|
||||
@@ -81,7 +102,7 @@ Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/overview
|
||||
Server -> CC : get_overview()
|
||||
CC --> Server : {nodes: 2, workstreams: 12,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.9.2"]}
|
||||
CC --> Server : {nodes: 2, workstreams: 12,\nstates: {running:5, ...},\naggregate: {total_tokens: 50000},\nversion_drift: false, versions: ["0.9.7"]}
|
||||
Server --> Browser : JSON response
|
||||
|
||||
Browser -> Server : GET /v1/api/cluster/nodes?sort=activity
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:14824b81fa87f9a29e9b182b54132d3e438dd83f880b20b111b4bf51bc4b39d1
|
||||
size 317947
|
||||
oid sha256:040f7d9ec7d676da40b9487e0825caf2c1574cbdd9f16d0998d90e0c2e4f8861
|
||||
size 360309
|
||||
|
||||
@@ -69,7 +69,6 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `CONSOLE_PORT` | `8090` | Host port mapping |
|
||||
| `CONSOLE_POLL_INTERVAL` | `10` | Node polling interval (seconds) |
|
||||
|
||||
### Auth
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "0.9.7"
|
||||
version = "0.9.8"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
|
||||
@@ -229,13 +229,15 @@ class TestMessageCog:
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_ignores_dms(self):
|
||||
def test_dm_without_reference_sends_guidance(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
msg = _make_message(guild=False)
|
||||
dm_channel = AsyncMock()
|
||||
msg = _make_message(guild=False, channel=dm_channel)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
dm_channel.send.assert_awaited_once()
|
||||
|
||||
def test_ignores_non_allowed_channels(self):
|
||||
cog, ts, _bot = self._make_cog()
|
||||
@@ -329,6 +331,8 @@ class TestWsEventFinalization:
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
|
||||
@@ -358,6 +362,8 @@ class TestWsEventFinalization:
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
@@ -391,6 +397,8 @@ class TestApprovalVerdictDisplay:
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
@@ -509,6 +517,8 @@ class TestApprovalVerdictDisplay:
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {"ws-1": MagicMock()}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
@@ -533,6 +543,8 @@ class TestStreamEndBehavior:
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
@@ -702,8 +714,8 @@ class TestNotificationTracking:
|
||||
# Should send feedback to the DM channel
|
||||
dm_channel.send.assert_awaited_once_with("*This notification is no longer active.*")
|
||||
|
||||
def test_dm_without_reference_ignored(self):
|
||||
"""DM without a message reference should be ignored."""
|
||||
def test_dm_without_reference_sends_guidance(self):
|
||||
"""DM without a message reference should reply with guidance."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
@@ -718,11 +730,15 @@ class TestNotificationTracking:
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
msg = _make_message(guild=False) # reference=None
|
||||
dm_channel = AsyncMock()
|
||||
msg = _make_message(guild=False, channel=dm_channel) # reference=None
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
dm_channel.send.assert_awaited_once()
|
||||
sent_text = dm_channel.send.call_args[0][0]
|
||||
assert "/ask" in sent_text
|
||||
|
||||
def test_dm_reply_unlinked_user_ignored(self):
|
||||
"""DM reply from an unlinked user should be ignored."""
|
||||
@@ -760,6 +776,8 @@ class TestNotificationTracking:
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_ws_map = {}
|
||||
bot._MAX_NOTIFY_TRACKING = 100
|
||||
@@ -797,6 +815,8 @@ class TestNotificationTracking:
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_ws_map = {}
|
||||
|
||||
@@ -817,6 +837,482 @@ class TestNotificationTracking:
|
||||
assert len(bot._notify_ws_map) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Formatter: format_tool_result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFormatToolResult:
|
||||
"""Tests for format_tool_result in _formatter.py."""
|
||||
|
||||
def test_basic_output(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
result = format_tool_result("hello world")
|
||||
assert "```" in result
|
||||
assert "hello world" in result
|
||||
|
||||
def test_wraps_in_code_block(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
result = format_tool_result("output text")
|
||||
assert result.startswith("```\n")
|
||||
assert result.endswith("\n```")
|
||||
|
||||
def test_truncates_long_output_by_lines(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
output = "\n".join(f"line {i}" for i in range(20))
|
||||
result = format_tool_result(output)
|
||||
# Should have at most 10 content lines + ellipsis
|
||||
inner = result.split("```")[1]
|
||||
assert inner.strip().count("\n") <= 11
|
||||
|
||||
def test_truncates_long_output_by_chars(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
output = "x" * 600
|
||||
result = format_tool_result(output)
|
||||
# Code block content should be <= 500 chars (497 + ellipsis)
|
||||
inner = result.split("```")[1].strip()
|
||||
assert len(inner) <= 501 # 497 + ellipsis char
|
||||
|
||||
def test_escapes_triple_backticks_in_output(self):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
output = "before ``` after"
|
||||
result = format_tool_result(output)
|
||||
# Only the opening and closing code fences should remain as ```.
|
||||
assert result.count("```") == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thinking indicator lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestThinkingIndicator:
|
||||
"""Tests for ThinkingStart/Stop event handling in the Discord bot."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_thinking_start_sends_message(self):
|
||||
from turnstone.sdk.events import ThinkingStartEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
sent_msg = MagicMock()
|
||||
thread.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
event = ThinkingStartEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once_with("*Thinking...*")
|
||||
assert bot._thinking_msgs["ws-1"] is sent_msg
|
||||
|
||||
def test_thinking_stop_preserves_message_for_reuse(self):
|
||||
from turnstone.sdk.events import ThinkingStopEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.delete = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
|
||||
event = ThinkingStopEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Message kept for next event to reuse via edit.
|
||||
thinking_msg.delete.assert_not_awaited()
|
||||
assert "ws-1" in bot._thinking_msgs
|
||||
|
||||
def test_thinking_stop_without_message_is_noop(self):
|
||||
from turnstone.sdk.events import ThinkingStopEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ThinkingStopEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
def test_content_event_reuses_thinking_message(self):
|
||||
from turnstone.sdk.events import ContentEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.edit = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
|
||||
event = ContentEvent(ws_id="ws-1", text="Hello")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Thinking message becomes the StreamingMessage base — no delete.
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
sm = bot._streaming["ws-1"]
|
||||
assert sm._message is thinking_msg
|
||||
|
||||
def test_stream_end_clears_thinking_message(self):
|
||||
from turnstone.sdk.events import StreamEndEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.delete = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
bot._notify_reply_channels = {}
|
||||
|
||||
event = StreamEndEvent(ws_id="ws-1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thinking_msg.delete.assert_awaited_once()
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool info / result embeds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestToolInfoEvent:
|
||||
"""Tests for ToolInfoEvent handling in the Discord bot."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_sends_per_item_embed(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
sent_msg = MagicMock()
|
||||
thread.send = AsyncMock(return_value=sent_msg)
|
||||
|
||||
items = [{"func_name": "bash", "preview": "ls -la", "needs_approval": False}]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once()
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.title == "bash"
|
||||
assert embed.description == "ls -la"
|
||||
# Message tracked for later editing by ToolResultEvent.
|
||||
assert bot._tool_info_msgs["ws-1"] == [("", "bash", "ls -la", sent_msg)]
|
||||
|
||||
def test_multiple_tools_send_multiple_embeds(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
items = [
|
||||
{"func_name": "bash", "preview": "ls", "needs_approval": False},
|
||||
{"func_name": "read_file", "preview": "/etc", "needs_approval": False},
|
||||
]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
assert thread.send.await_count == 2
|
||||
assert len(bot._tool_info_msgs["ws-1"]) == 2
|
||||
|
||||
def test_shows_all_items_regardless_of_approval(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
items = [
|
||||
{"func_name": "bash", "preview": "rm -rf /", "needs_approval": True},
|
||||
{"func_name": "read_file", "preview": "/etc/hosts", "needs_approval": False},
|
||||
]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Both items shown — running indicator is separate from approval dialog.
|
||||
assert thread.send.await_count == 2
|
||||
|
||||
def test_reuses_thinking_message_for_first_tool(self):
|
||||
from turnstone.sdk.events import ToolInfoEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
thinking_msg = MagicMock()
|
||||
thinking_msg.edit = AsyncMock()
|
||||
bot._thinking_msgs["ws-1"] = thinking_msg
|
||||
|
||||
items = [{"func_name": "bash", "preview": "ls -la", "needs_approval": False}]
|
||||
event = ToolInfoEvent(ws_id="ws-1", items=items)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Thinking message edited into tool embed, no new message sent.
|
||||
thinking_msg.edit.assert_awaited_once()
|
||||
thread.send.assert_not_awaited()
|
||||
assert "ws-1" not in bot._thinking_msgs
|
||||
# The reused message is tracked for ToolResultEvent editing.
|
||||
assert bot._tool_info_msgs["ws-1"][0][3] is thinking_msg
|
||||
|
||||
|
||||
class TestToolResultEvent:
|
||||
"""Tests for ToolResultEvent handling in the Discord bot."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_marks_info_done_and_sends_result(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
# Pre-populate a tool info message (as ToolInfoEvent would).
|
||||
info_msg = MagicMock()
|
||||
info_msg.edit = AsyncMock()
|
||||
bot._tool_info_msgs["ws-1"] = [("", "bash", "ls -la", info_msg)]
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="file1\nfile2")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Info embed edited to "Done" status.
|
||||
info_msg.edit.assert_awaited_once()
|
||||
status_embed = info_msg.edit.call_args[1]["embed"]
|
||||
assert "Done" in status_embed.title
|
||||
assert status_embed.description == "ls -la" # preview preserved
|
||||
# Result sent as separate new message.
|
||||
thread.send.assert_awaited_once()
|
||||
result_embed = thread.send.call_args[1]["embed"]
|
||||
assert result_embed.title == "bash"
|
||||
assert "file1" in result_embed.description
|
||||
# Entry consumed from tracking list.
|
||||
assert bot._tool_info_msgs["ws-1"] == []
|
||||
|
||||
def test_result_sent_even_without_info_match(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="file1\nfile2")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
thread.send.assert_awaited_once()
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.title == "bash"
|
||||
assert "file1" in embed.description
|
||||
|
||||
def test_error_result_uses_red_color(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ToolResultEvent(
|
||||
ws_id="ws-1", name="bash", output="command not found", is_error=True
|
||||
)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.color == discord.Color.red()
|
||||
|
||||
def test_success_result_uses_dark_grey_color(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="ok")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
embed = thread.send.call_args[1]["embed"]
|
||||
assert embed.color == discord.Color.dark_grey()
|
||||
|
||||
def test_call_id_matching(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
first_msg = MagicMock()
|
||||
first_msg.edit = AsyncMock()
|
||||
second_msg = MagicMock()
|
||||
second_msg.edit = AsyncMock()
|
||||
bot._tool_info_msgs["ws-1"] = [
|
||||
("call-1", "bash", "", first_msg),
|
||||
("call-2", "bash", "", second_msg),
|
||||
]
|
||||
|
||||
# Result with call_id matches the correct message regardless of order.
|
||||
event = ToolResultEvent(ws_id="ws-1", call_id="call-2", name="bash", output="result")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
second_msg.edit.assert_awaited_once()
|
||||
first_msg.edit.assert_not_awaited()
|
||||
|
||||
def test_fifo_fallback_when_no_call_id(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
first_msg = MagicMock()
|
||||
first_msg.edit = AsyncMock()
|
||||
second_msg = MagicMock()
|
||||
second_msg.edit = AsyncMock()
|
||||
bot._tool_info_msgs["ws-1"] = [("", "bash", "", first_msg), ("", "bash", "", second_msg)]
|
||||
|
||||
# No call_id — falls back to FIFO name match.
|
||||
event1 = ToolResultEvent(ws_id="ws-1", name="bash", output="result1")
|
||||
_run(bot._on_ws_event("ws-1", thread, event1))
|
||||
first_msg.edit.assert_awaited_once()
|
||||
second_msg.edit.assert_not_awaited()
|
||||
|
||||
event2 = ToolResultEvent(ws_id="ws-1", name="bash", output="result2")
|
||||
_run(bot._on_ws_event("ws-1", thread, event2))
|
||||
second_msg.edit.assert_awaited_once()
|
||||
|
||||
def test_edit_failure_falls_back_to_send(self):
|
||||
from turnstone.sdk.events import ToolResultEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
info_msg = MagicMock()
|
||||
info_msg.edit = AsyncMock(side_effect=Exception("Discord API error"))
|
||||
bot._tool_info_msgs["ws-1"] = [("", "bash", "ls -la", info_msg)]
|
||||
|
||||
event = ToolResultEvent(ws_id="ws-1", name="bash", output="ok")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
# Edit failed, should fall back to send.
|
||||
info_msg.edit.assert_awaited_once()
|
||||
thread.send.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Approval resolved (timeout / external resolution)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApprovalResolved:
|
||||
"""ApprovalResolvedEvent should disable buttons on the pending approval embed."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot.storage = None
|
||||
bot._streaming = {}
|
||||
bot._thinking_msgs = {}
|
||||
bot._tool_info_msgs = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_disables_buttons_on_timeout(self):
|
||||
from turnstone.sdk.events import ApprovalResolvedEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
# Set up a pending approval message with components.
|
||||
approval_msg = MagicMock()
|
||||
approval_msg.embeds = [MagicMock()]
|
||||
approval_msg.components = []
|
||||
approval_msg.edit = AsyncMock()
|
||||
bot._pending_approval_msgs["ws-1"] = approval_msg
|
||||
|
||||
event = ApprovalResolvedEvent(ws_id="ws-1", approved=False, feedback="timeout")
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
approval_msg.edit.assert_awaited_once()
|
||||
# Pending approval message should be removed.
|
||||
assert "ws-1" not in bot._pending_approval_msgs
|
||||
|
||||
def test_disables_buttons_on_approved(self):
|
||||
from turnstone.sdk.events import ApprovalResolvedEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
approval_msg = MagicMock()
|
||||
approval_msg.embeds = [MagicMock()]
|
||||
approval_msg.components = []
|
||||
approval_msg.edit = AsyncMock()
|
||||
bot._pending_approval_msgs["ws-1"] = approval_msg
|
||||
|
||||
event = ApprovalResolvedEvent(ws_id="ws-1", approved=True)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
|
||||
approval_msg.edit.assert_awaited_once()
|
||||
# Check the embed title was updated with "Approved".
|
||||
edited_embed = approval_msg.edit.call_args[1]["embed"]
|
||||
assert "Approved" in edited_embed.title
|
||||
|
||||
def test_no_pending_approval_is_noop(self):
|
||||
from turnstone.sdk.events import ApprovalResolvedEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
event = ApprovalResolvedEvent(ws_id="ws-1", approved=False)
|
||||
_run(bot._on_ws_event("ws-1", thread, event))
|
||||
# No error, no state change.
|
||||
|
||||
|
||||
class TestChannelCLI:
|
||||
"""Tests for the channel CLI entry point."""
|
||||
|
||||
|
||||
+195
-102
@@ -3,7 +3,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -29,32 +29,15 @@ class MockStorage:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_collector(storage=None, poll_interval=0, discovery_interval=999):
|
||||
"""Create a collector with zero poll interval (no jitter delay in tests)."""
|
||||
def _make_collector(storage=None, discovery_interval=999):
|
||||
"""Create a collector for tests (discovery disabled by default)."""
|
||||
s = storage or MockStorage()
|
||||
return ClusterCollector(
|
||||
storage=s,
|
||||
poll_interval=poll_interval,
|
||||
discovery_interval=discovery_interval,
|
||||
)
|
||||
|
||||
|
||||
def _dashboard_response(workstreams=None, aggregate=None):
|
||||
"""Build a /v1/api/dashboard-style response dict."""
|
||||
return {
|
||||
"workstreams": workstreams or [],
|
||||
"aggregate": aggregate
|
||||
or {
|
||||
"total_tokens": 0,
|
||||
"total_tool_calls": 0,
|
||||
"active_count": 0,
|
||||
"total_count": 0,
|
||||
"uptime_seconds": 0,
|
||||
"node": "local",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClusterCollector — unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -157,30 +140,35 @@ class TestCollectorDiscovery:
|
||||
assert c._nodes["node-a"].started == 1234567890.0
|
||||
|
||||
|
||||
class TestCollectorPolling:
|
||||
"""Polling /v1/api/dashboard from nodes."""
|
||||
class TestCollectorSnapshot:
|
||||
"""Applying node_snapshot SSE events."""
|
||||
|
||||
def test_apply_poll_populates_workstreams(self):
|
||||
def test_apply_snapshot_populates_workstreams(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
|
||||
dashboard = _dashboard_response(
|
||||
workstreams=[
|
||||
{
|
||||
"id": "ws1",
|
||||
"name": "test",
|
||||
"state": "running",
|
||||
"tokens": 1000,
|
||||
"context_ratio": 0.15,
|
||||
"activity": "bash: ls",
|
||||
"activity_state": "tool",
|
||||
"tool_calls": 3,
|
||||
"title": "My task",
|
||||
},
|
||||
],
|
||||
aggregate={"total_tokens": 1000, "total_tool_calls": 3},
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [
|
||||
{
|
||||
"id": "ws1",
|
||||
"name": "test",
|
||||
"state": "running",
|
||||
"tokens": 1000,
|
||||
"context_ratio": 0.15,
|
||||
"activity": "bash: ls",
|
||||
"activity_state": "tool",
|
||||
"tool_calls": 3,
|
||||
"title": "My task",
|
||||
},
|
||||
],
|
||||
"health": {"status": "ok"},
|
||||
"aggregate": {"total_tokens": 1000, "total_tool_calls": 3},
|
||||
},
|
||||
)
|
||||
c._apply_poll("node-a", dashboard, {"status": "ok"})
|
||||
|
||||
detail = c.get_node_detail("node-a")
|
||||
assert len(detail["workstreams"]) == 1
|
||||
@@ -189,7 +177,7 @@ class TestCollectorPolling:
|
||||
assert detail["workstreams"][0]["server_url"] == "http://a:8080"
|
||||
assert detail["health"]["status"] == "ok"
|
||||
|
||||
def test_apply_poll_replaces_stale_workstreams(self):
|
||||
def test_apply_snapshot_replaces_stale_workstreams(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -197,30 +185,44 @@ class TestCollectorPolling:
|
||||
workstreams={"old-ws": {"id": "old-ws", "name": "old", "state": "idle"}},
|
||||
)
|
||||
|
||||
dashboard = _dashboard_response(
|
||||
workstreams=[{"id": "new-ws", "name": "new", "state": "running"}]
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [{"id": "new-ws", "name": "new", "state": "running"}],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
c._apply_poll("node-a", dashboard, {})
|
||||
|
||||
detail = c.get_node_detail("node-a")
|
||||
assert len(detail["workstreams"]) == 1
|
||||
assert detail["workstreams"][0]["id"] == "new-ws"
|
||||
|
||||
def test_apply_poll_ignores_unknown_node(self):
|
||||
def test_apply_snapshot_ignores_unknown_node(self):
|
||||
c = _make_collector()
|
||||
# Should not raise
|
||||
c._apply_poll("unknown", _dashboard_response(), {})
|
||||
c._apply_snapshot(
|
||||
"unknown", {"type": "node_snapshot", "workstreams": [], "health": {}, "aggregate": {}}
|
||||
)
|
||||
|
||||
def test_apply_poll_emits_ws_created_for_new_workstream(self):
|
||||
def test_apply_snapshot_emits_ws_created_for_new_workstream(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
dashboard = _dashboard_response(
|
||||
workstreams=[{"id": "ws1", "name": "new-task", "state": "idle"}]
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [{"id": "ws1", "name": "new-task", "state": "idle"}],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
c._apply_poll("node-a", dashboard, {})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "ws_created"
|
||||
@@ -228,7 +230,7 @@ class TestCollectorPolling:
|
||||
assert event["name"] == "new-task"
|
||||
assert event["node_id"] == "node-a"
|
||||
|
||||
def test_apply_poll_emits_ws_closed_for_removed_workstream(self):
|
||||
def test_apply_snapshot_emits_ws_closed_for_removed_workstream(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -238,13 +240,22 @@ class TestCollectorPolling:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_poll("node-a", _dashboard_response(), {})
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "ws_closed"
|
||||
assert event["ws_id"] == "ws1"
|
||||
|
||||
def test_apply_poll_no_events_when_unchanged(self):
|
||||
def test_apply_snapshot_no_events_when_unchanged(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -254,15 +265,21 @@ class TestCollectorPolling:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
# Same state — no events expected
|
||||
dashboard = _dashboard_response(
|
||||
workstreams=[{"id": "ws1", "name": "same", "state": "idle"}]
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [{"id": "ws1", "name": "same", "state": "idle"}],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
c._apply_poll("node-a", dashboard, {})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_poll_emits_state_change(self):
|
||||
def test_apply_snapshot_emits_state_change_as_cluster_state(self):
|
||||
"""State change events must use type 'cluster_state' for the frontend."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -272,30 +289,141 @@ class TestCollectorPolling:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
dashboard = _dashboard_response(
|
||||
workstreams=[{"id": "ws1", "name": "same", "state": "running"}]
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [{"id": "ws1", "name": "same", "state": "running"}],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
c._apply_poll("node-a", dashboard, {})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "ws_state"
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["state"] == "running"
|
||||
|
||||
def test_apply_poll_skips_empty_id_workstream(self):
|
||||
def test_apply_snapshot_skips_empty_id_workstream(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
dashboard = _dashboard_response(workstreams=[{"name": "no-id", "state": "idle"}])
|
||||
c._apply_poll("node-a", dashboard, {})
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [{"name": "no-id", "state": "idle"}],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
|
||||
assert q.empty()
|
||||
assert len(c._nodes["node-a"].workstreams) == 0
|
||||
|
||||
def test_poll_401_preserves_workstreams_and_marks_unreachable(self):
|
||||
"""A 401 from the server must NOT wipe workstream data."""
|
||||
|
||||
class TestCollectorDelta:
|
||||
"""Applying individual SSE delta events."""
|
||||
|
||||
def test_apply_delta_ws_state_fans_out_as_cluster_state(self):
|
||||
"""Server emits ws_state; collector must translate to cluster_state."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta(
|
||||
"node-a", {"type": "ws_state", "ws_id": "ws1", "state": "running", "tokens": 500}
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["state"] == "running"
|
||||
# Verify in-memory state was updated
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["state"] == "running"
|
||||
|
||||
def test_apply_delta_ws_created(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "ws_created", "ws_id": "ws1", "name": "new"})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "ws_created"
|
||||
assert "ws1" in c._nodes["node-a"].workstreams
|
||||
|
||||
def test_apply_delta_ws_closed(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "old", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "ws_closed", "ws_id": "ws1"})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "ws_closed"
|
||||
assert "ws1" not in c._nodes["node-a"].workstreams
|
||||
|
||||
def test_apply_delta_ws_rename(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "old-name", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "ws_rename", "ws_id": "ws1", "name": "new-name"})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "ws_rename"
|
||||
assert event["name"] == "new-name"
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["name"] == "new-name"
|
||||
|
||||
def test_apply_delta_health_changed(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
health={"status": "ok", "backend": {"status": "up", "circuit_state": "closed"}},
|
||||
)
|
||||
|
||||
c._apply_delta("node-a", {"type": "health_changed", "circuit_state": "open"})
|
||||
|
||||
health = c._nodes["node-a"].health
|
||||
assert health["backend"]["circuit_state"] == "open"
|
||||
assert health["backend"]["status"] == "down"
|
||||
assert health["status"] == "degraded"
|
||||
|
||||
def test_apply_delta_aggregate(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "aggregate", "total_tokens": 5000, "total_tool_calls": 42, "active_count": 3},
|
||||
)
|
||||
|
||||
assert c._nodes["node-a"].aggregate["total_tokens"] == 5000
|
||||
assert c._nodes["node-a"].aggregate["total_tool_calls"] == 42
|
||||
|
||||
def test_mark_unreachable_preserves_workstreams(self):
|
||||
"""Disconnection marks unreachable but preserves workstream data."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -304,47 +432,12 @@ class TestCollectorPolling:
|
||||
workstreams={"ws1": {"id": "ws1", "name": "existing", "state": "idle"}},
|
||||
)
|
||||
|
||||
# Mock httpx to return 401
|
||||
import httpx as _httpx
|
||||
c._mark_unreachable("node-a")
|
||||
|
||||
mock_response = _httpx.Response(
|
||||
401,
|
||||
json={"error": "Unauthorized"},
|
||||
request=_httpx.Request("GET", "http://a:8080/v1/api/dashboard"),
|
||||
)
|
||||
|
||||
with patch.object(c._http_client, "get", return_value=mock_response):
|
||||
c._poll_all_nodes()
|
||||
|
||||
# Workstream data must be preserved, node marked unreachable
|
||||
assert c._nodes["node-a"].reachable is False
|
||||
assert "ws1" in c._nodes["node-a"].workstreams
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["name"] == "existing"
|
||||
|
||||
def test_poll_403_preserves_workstreams(self):
|
||||
"""A 403 should also preserve state and mark unreachable."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
reachable=True,
|
||||
workstreams={"ws1": {"id": "ws1", "name": "keep-me", "state": "running"}},
|
||||
)
|
||||
|
||||
import httpx as _httpx
|
||||
|
||||
mock_response = _httpx.Response(
|
||||
403,
|
||||
json={"error": "Forbidden"},
|
||||
request=_httpx.Request("GET", "http://a:8080/v1/api/dashboard"),
|
||||
)
|
||||
|
||||
with patch.object(c._http_client, "get", return_value=mock_response):
|
||||
c._poll_all_nodes()
|
||||
|
||||
assert c._nodes["node-a"].reachable is False
|
||||
assert "ws1" in c._nodes["node-a"].workstreams
|
||||
|
||||
|
||||
class TestCollectorFanout:
|
||||
"""SSE fan-out to registered listeners."""
|
||||
|
||||
@@ -425,11 +425,16 @@ class TestSkillCatalogDisclosure:
|
||||
session._mcp_client = None
|
||||
session._notify_on_complete = "{}"
|
||||
session._tool_error_flags = {}
|
||||
from turnstone.prompts import ClientType
|
||||
|
||||
session._tools = []
|
||||
session._client_type = ClientType.CLI
|
||||
session._username = ""
|
||||
|
||||
# Memory stubs
|
||||
session._memory_config = MagicMock()
|
||||
session._memory_config.fetch_limit = 0
|
||||
session._user_id = ""
|
||||
session._user_id = "test-user"
|
||||
|
||||
with (
|
||||
patch(
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Tests for the system message composition harness (turnstone.prompts)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.prompts import (
|
||||
ClientType,
|
||||
SessionContext,
|
||||
compose_system_message,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALID_CTX = SessionContext(
|
||||
current_datetime="2026-03-31T14:22:00-07:00",
|
||||
timezone="PDT",
|
||||
username="sarah.chen",
|
||||
)
|
||||
|
||||
_ALL_TOOLS: frozenset[str] = frozenset({"web_search", "read_file", "bash"})
|
||||
_NO_TOOLS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Assembly smoke test per client type
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ct", [ClientType.WEB, ClientType.CLI, ClientType.CHAT])
|
||||
def test_smoke_all_client_types(ct: ClientType) -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ct,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
# BASE content present
|
||||
assert "resident engineer" in result
|
||||
# CONTEXT present
|
||||
assert "sarah.chen" in result
|
||||
assert "2026-03-31" in result
|
||||
|
||||
|
||||
def test_smoke_web_has_mermaid() -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ClientType.WEB,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
assert "Mermaid" in result
|
||||
assert "KaTeX" in result
|
||||
|
||||
|
||||
def test_smoke_cli_no_mermaid() -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ClientType.CLI,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
assert "Mermaid" not in result or "Do not use" in result
|
||||
|
||||
|
||||
def test_smoke_chat_no_tables() -> None:
|
||||
result = compose_system_message(
|
||||
client_type=ClientType.CHAT,
|
||||
context=_VALID_CTX,
|
||||
available_tools=_ALL_TOOLS,
|
||||
)
|
||||
assert "Do not use them" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Required field validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_current_datetime() -> None:
|
||||
ctx = SessionContext(current_datetime="", timezone="PDT", username="alice")
|
||||
with pytest.raises(ValueError, match="current_datetime"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
def test_missing_timezone() -> None:
|
||||
ctx = SessionContext(
|
||||
current_datetime="2026-03-31T14:22:00-07:00",
|
||||
timezone="",
|
||||
username="alice",
|
||||
)
|
||||
with pytest.raises(ValueError, match="timezone"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
def test_missing_username() -> None:
|
||||
ctx = SessionContext(
|
||||
current_datetime="2026-03-31T14:22:00-07:00",
|
||||
timezone="PDT",
|
||||
username="",
|
||||
)
|
||||
with pytest.raises(ValueError, match="username"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Unknown client type rejection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unknown_client_type() -> None:
|
||||
with pytest.raises(ValueError, match="Unknown client_type"):
|
||||
compose_system_message("tablet", _VALID_CTX, _NO_TOOLS) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Missing policy file
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_policy_file() -> None:
|
||||
with pytest.raises(FileNotFoundError, match="nonexistent"):
|
||||
compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
policies=["nonexistent"],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Module isolation — BASE must be environment-agnostic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_base_module_isolation() -> None:
|
||||
from turnstone.prompts import _load
|
||||
|
||||
base = _load("base.md")
|
||||
for forbidden in ("Mermaid", "KaTeX", "terminal", "monospace", "Slack", "Discord"):
|
||||
assert forbidden not in base, f"BASE must not contain '{forbidden}'"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. ENV mutual exclusion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_env_mutual_exclusion() -> None:
|
||||
web = compose_system_message(ClientType.WEB, _VALID_CTX, _ALL_TOOLS)
|
||||
# Web should have Mermaid.js but not "No diagram rendering" from CLI
|
||||
assert "Mermaid" in web
|
||||
assert "No diagram rendering" not in web
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. Policy tool gating — file-based (negative case)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_policy_gated_out() -> None:
|
||||
"""web_search policy excluded when web_search tool is not available."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
frozenset({"read_file"}), # no web_search
|
||||
policies=["web_search"],
|
||||
)
|
||||
assert "Web Search Policy" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 8. Policy tool gating — positive case
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_policy_gated_in() -> None:
|
||||
"""web_search policy included when web_search tool is available."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
frozenset({"web_search"}),
|
||||
policies=["web_search"],
|
||||
)
|
||||
assert "Web Search Policy" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. Unconditional policy not gated
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unconditional_policy() -> None:
|
||||
"""A DB policy with no tool_gate is always included."""
|
||||
db = [
|
||||
{
|
||||
"name": "custom_rule",
|
||||
"content": "## Custom Rule\nAlways be polite.",
|
||||
"tool_gate": "",
|
||||
"priority": 0,
|
||||
"enabled": True,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
assert "Always be polite" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. DB policy override
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_db_policy_overrides_file() -> None:
|
||||
"""DB policy with same name as file policy wins."""
|
||||
db = [
|
||||
{
|
||||
"name": "web_search",
|
||||
"content": "## DB Web Search Override\nCustom content.",
|
||||
"tool_gate": "web_search",
|
||||
"priority": 0,
|
||||
"enabled": True,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
policies=["web_search"],
|
||||
db_policies=db,
|
||||
)
|
||||
assert "DB Web Search Override" in result
|
||||
assert "Use local tools" not in result # original file content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 11. DB-only policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_db_only_policy() -> None:
|
||||
"""DB policy not in explicit list is still included."""
|
||||
db = [
|
||||
{
|
||||
"name": "extra_rule",
|
||||
"content": "## Extra\nDo not share secrets.",
|
||||
"tool_gate": "",
|
||||
"priority": 5,
|
||||
"enabled": True,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
assert "Do not share secrets" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 12. Disabled DB policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_disabled_db_policy() -> None:
|
||||
"""DB policy with enabled=False is skipped."""
|
||||
db = [
|
||||
{
|
||||
"name": "disabled_rule",
|
||||
"content": "## Disabled\nThis should not appear.",
|
||||
"tool_gate": "",
|
||||
"priority": 0,
|
||||
"enabled": False,
|
||||
}
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
assert "This should not appear" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 13. ISO 8601 validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_invalid_iso_datetime() -> None:
|
||||
ctx = SessionContext(
|
||||
current_datetime="not-a-date",
|
||||
timezone="PDT",
|
||||
username="alice",
|
||||
)
|
||||
with pytest.raises(ValueError, match="not valid ISO 8601"):
|
||||
compose_system_message(ClientType.CLI, ctx, _NO_TOOLS)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 14. DB policy priority ordering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_db_policy_priority_ordering() -> None:
|
||||
"""DB-only policies are assembled in priority order (ascending)."""
|
||||
db = [
|
||||
{
|
||||
"name": "second",
|
||||
"content": "SECOND_MARKER",
|
||||
"tool_gate": "",
|
||||
"priority": 10,
|
||||
"enabled": True,
|
||||
},
|
||||
{
|
||||
"name": "first",
|
||||
"content": "FIRST_MARKER",
|
||||
"tool_gate": "",
|
||||
"priority": 1,
|
||||
"enabled": True,
|
||||
},
|
||||
]
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
db_policies=db,
|
||||
)
|
||||
first_pos = result.index("FIRST_MARKER")
|
||||
second_pos = result.index("SECOND_MARKER")
|
||||
assert first_pos < second_pos
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 15. TOOLS module excluded when no tools available
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tools_excluded_when_no_tools() -> None:
|
||||
"""TOOLS module is not included when available_tools is empty."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_NO_TOOLS,
|
||||
)
|
||||
assert "TOOL PATTERNS" not in result
|
||||
|
||||
|
||||
def test_tools_included_when_tools_available() -> None:
|
||||
"""TOOLS module is included when available_tools is non-empty."""
|
||||
result = compose_system_message(
|
||||
ClientType.CLI,
|
||||
_VALID_CTX,
|
||||
_ALL_TOOLS,
|
||||
)
|
||||
assert "TOOL PATTERNS" in result
|
||||
@@ -84,5 +84,5 @@ def test_collector_tls_defaults():
|
||||
|
||||
storage_mock = MagicMock()
|
||||
collector = ClusterCollector(storage=storage_mock)
|
||||
# Should create httpx client without errors
|
||||
assert collector._http_client is not None
|
||||
# Should store TLS settings for async client creation
|
||||
assert collector._tls_verify is True
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "0.9.7"
|
||||
__version__ = "0.9.8"
|
||||
|
||||
@@ -942,6 +942,41 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [
|
||||
],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Admin: Prompt Policies ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/prompt-policies",
|
||||
"GET",
|
||||
"List all prompt policies for system message composition",
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/prompt-policies",
|
||||
"POST",
|
||||
"Create a prompt policy",
|
||||
error_codes=[400],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/prompt-policies/{policy_id}",
|
||||
"GET",
|
||||
"Get a single prompt policy",
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/prompt-policies/{policy_id}",
|
||||
"PUT",
|
||||
"Update a prompt policy",
|
||||
error_codes=[400, 404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/prompt-policies/{policy_id}",
|
||||
"DELETE",
|
||||
"Delete a prompt policy",
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Admin: TLS / ACME ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/tls/ca",
|
||||
|
||||
@@ -57,6 +57,10 @@ class CreateWorkstreamRequest(BaseModel):
|
||||
description="Workstream ID to resume atomically during creation (empty = fresh start)",
|
||||
)
|
||||
skill: str = Field(default="", description="Skill name (replaces default skills)")
|
||||
client_type: str = Field(
|
||||
default="",
|
||||
description="Client surface type (web, cli, chat). Defaults to web for server-created sessions.",
|
||||
)
|
||||
|
||||
|
||||
class CreateWorkstreamResponse(BaseModel):
|
||||
|
||||
@@ -137,8 +137,11 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [
|
||||
"/v1/api/events/global",
|
||||
"GET",
|
||||
"Global SSE event stream",
|
||||
description="Global Server-Sent Events stream for state-change broadcasts "
|
||||
"across all workstreams. Returns text/event-stream.",
|
||||
description="Server-Sent Events stream for node-level state broadcasts. "
|
||||
"Emits a node_snapshot event on connect (workstreams, health, aggregate), "
|
||||
"followed by real-time delta events (ws_state, ws_activity, ws_created, "
|
||||
"ws_closed, ws_rename, health_changed, aggregate). "
|
||||
"Pass ?expected_node_id=X for identity verification (returns 409 on mismatch).",
|
||||
tags=["Streaming"],
|
||||
),
|
||||
# --- Saved workstreams ---
|
||||
|
||||
+37
-8
@@ -192,7 +192,7 @@ Note: the MCP SDK's DNS rebinding protection must be disabled for Docker-interna
|
||||
(the compose.yaml handles this), and the server must bind to 0.0.0.0 (not 127.0.0.1) to be \
|
||||
reachable from other containers.
|
||||
- The `DATABASE_URL` for docker compose internal networking uses the hostname `postgres` \
|
||||
(e.g., `postgresql://turnstone:<password>@postgres:5432/turnstone`).
|
||||
(e.g., `postgresql+psycopg://turnstone:<password>@postgres:5432/turnstone`).
|
||||
- For local LLM backends (vLLM, llama.cpp, Ollama, etc.), set `OPENAI_API_KEY=dummy` in the \
|
||||
.env file — local servers typically don't require authentication. The `LLM_BASE_URL` should \
|
||||
use `host.docker.internal` to reach the host machine from inside Docker \
|
||||
@@ -643,22 +643,42 @@ class _BootstrapLLM:
|
||||
messages=messages,
|
||||
tools=tools if tools else None,
|
||||
)
|
||||
choice = resp.choices[0]
|
||||
content = choice.message.content or ""
|
||||
# Guard against non-spec responses from proxies (Open WebUI, LiteLLM, etc.)
|
||||
if resp is None:
|
||||
raise RuntimeError(
|
||||
"Server returned null — your OpenAI-compatible endpoint may not "
|
||||
"support tool calling. Try a direct connection to the model server."
|
||||
)
|
||||
choices = getattr(resp, "choices", None)
|
||||
if not choices:
|
||||
raise RuntimeError(
|
||||
"Server returned an empty choices array. "
|
||||
"The model may have hit its context limit, or the proxy "
|
||||
"dropped the response."
|
||||
)
|
||||
choice = choices[0]
|
||||
message = getattr(choice, "message", None)
|
||||
if message is None:
|
||||
raise RuntimeError(
|
||||
"Server returned a choice with no message. "
|
||||
"Your OpenAI-compatible endpoint may not fully implement "
|
||||
"the chat completions API."
|
||||
)
|
||||
content = message.content or ""
|
||||
tool_calls = None
|
||||
if choice.message.tool_calls:
|
||||
if getattr(message, "tool_calls", None):
|
||||
tool_calls = [
|
||||
{
|
||||
"id": tc.id,
|
||||
"id": getattr(tc, "id", None) or f"call_{secrets.token_hex(4)}",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.function.name,
|
||||
"arguments": tc.function.arguments,
|
||||
},
|
||||
}
|
||||
for tc in choice.message.tool_calls
|
||||
for i, tc in enumerate(message.tool_calls)
|
||||
]
|
||||
return content, tool_calls, choice.finish_reason or "stop"
|
||||
return content, tool_calls, getattr(choice, "finish_reason", None) or "stop"
|
||||
|
||||
# -- Anthropic path -----------------------------------------------------
|
||||
|
||||
@@ -1002,7 +1022,16 @@ def _run_conversation(
|
||||
retries += 1
|
||||
if retries >= _max_retries:
|
||||
print(f"\n{RED}LLM error after {_max_retries} attempts: {exc}{RESET}")
|
||||
print("Please check your connection and try again.")
|
||||
print()
|
||||
print("Troubleshooting:")
|
||||
print(
|
||||
f" {DIM}• If using a proxy (Open WebUI, LiteLLM), try connecting directly{RESET}"
|
||||
)
|
||||
print(f" {DIM}• Verify the endpoint supports tool/function calling{RESET}")
|
||||
print(f" {DIM}• Check that the model context window isn't exceeded{RESET}")
|
||||
print(
|
||||
f" {DIM}• Try a different model — not all models handle tool calls reliably{RESET}"
|
||||
)
|
||||
return
|
||||
print(f"\n{RED}LLM error: {exc}{RESET}")
|
||||
print(f"{DIM}Retrying ({retries}/{_max_retries})...{RESET}")
|
||||
|
||||
@@ -139,6 +139,26 @@ def format_plan_review(content: str) -> str:
|
||||
return f"**Plan review requested:**\n\n{content}"
|
||||
|
||||
|
||||
def format_tool_result(output: str) -> str:
|
||||
"""Format a tool result into a compact code-block summary.
|
||||
|
||||
Truncates to the first 10 lines (plus an ellipsis line if trimmed) or
|
||||
500 characters, whichever is shorter.
|
||||
"""
|
||||
# Truncate to 10 lines.
|
||||
lines = output.split("\n", 10)
|
||||
if len(lines) > 10:
|
||||
lines = lines[:10]
|
||||
lines.append("\u2026")
|
||||
trimmed = "\n".join(lines)
|
||||
# Escape triple backticks to prevent code-block breakout.
|
||||
trimmed = trimmed.replace("```", "` ` `")
|
||||
# Truncate to 500 chars (after escaping, which can expand the string).
|
||||
if len(trimmed) > 500:
|
||||
trimmed = trimmed[:497] + "\u2026"
|
||||
return f"```\n{trimmed}\n```"
|
||||
|
||||
|
||||
def truncate(text: str, max_length: int = 200) -> str:
|
||||
"""Truncate *text* to *max_length*, appending an ellipsis if trimmed."""
|
||||
if len(text) <= max_length:
|
||||
|
||||
@@ -120,6 +120,7 @@ class ChannelRouter:
|
||||
name: str = "",
|
||||
model: str = "",
|
||||
initial_message: str = "",
|
||||
client_type: str = "",
|
||||
) -> tuple[str, bool]:
|
||||
"""Look up or create a workstream for a channel.
|
||||
|
||||
@@ -176,6 +177,7 @@ class ChannelRouter:
|
||||
skill=self._skill,
|
||||
auto_approve=self._auto_approve,
|
||||
auto_approve_tools=_tools_csv,
|
||||
client_type=client_type,
|
||||
)
|
||||
ws_id = data.get("ws_id", "")
|
||||
else:
|
||||
@@ -187,6 +189,7 @@ class ChannelRouter:
|
||||
skill=self._skill,
|
||||
auto_approve=self._auto_approve,
|
||||
auto_approve_tools=_tools_csv,
|
||||
client_type=client_type,
|
||||
)
|
||||
ws_id = resp.ws_id
|
||||
data = {"ws_id": resp.ws_id, "name": resp.name}
|
||||
|
||||
@@ -24,6 +24,7 @@ from turnstone.channels._formatter import chunk_message
|
||||
from turnstone.channels._routing import ChannelRouter
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.sdk.events import (
|
||||
ApprovalResolvedEvent,
|
||||
ApproveRequestEvent,
|
||||
ContentEvent,
|
||||
ErrorEvent,
|
||||
@@ -31,6 +32,10 @@ from turnstone.sdk.events import (
|
||||
PlanReviewEvent,
|
||||
ServerEvent,
|
||||
StreamEndEvent,
|
||||
ThinkingStartEvent,
|
||||
ThinkingStopEvent,
|
||||
ToolInfoEvent,
|
||||
ToolResultEvent,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -178,6 +183,12 @@ class TurnstoneBot:
|
||||
self._subscribed_ws: set[str] = set()
|
||||
self._sse_tasks: dict[str, asyncio.Task[None]] = {}
|
||||
self._streaming: dict[str, StreamingMessage] = {}
|
||||
# Transient "Thinking..." status messages, deleted when content starts.
|
||||
self._thinking_msgs: dict[str, discord.Message] = {}
|
||||
# Per-tool "running" embeds, edited in-place when the result arrives.
|
||||
# List preserves call order for FIFO matching when the same tool name
|
||||
# appears more than once in a single turn.
|
||||
self._tool_info_msgs: dict[str, list[tuple[str, str, str, discord.Message]]] = {}
|
||||
# Track the Discord message containing the pending approval embed per
|
||||
# workstream so that IntentVerdictEvent can update it with LLM judge
|
||||
# results.
|
||||
@@ -302,6 +313,11 @@ class TurnstoneBot:
|
||||
await task
|
||||
self._subscribed_ws.discard(ws_id)
|
||||
self._streaming.pop(ws_id, None)
|
||||
thinking_msg = self._thinking_msgs.pop(ws_id, None)
|
||||
if thinking_msg is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await thinking_msg.delete()
|
||||
self._tool_info_msgs.pop(ws_id, None)
|
||||
self._pending_approval_msgs.pop(ws_id, None)
|
||||
self._notify_reply_channels.pop(ws_id, None)
|
||||
# Purge stale notification tracking entries for this workstream.
|
||||
@@ -322,6 +338,11 @@ class TurnstoneBot:
|
||||
self._subscribed_ws.discard(ws_id)
|
||||
self._sse_tasks.pop(ws_id, None)
|
||||
self._streaming.pop(ws_id, None)
|
||||
thinking_msg = self._thinking_msgs.pop(ws_id, None)
|
||||
if thinking_msg is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await thinking_msg.delete()
|
||||
self._tool_info_msgs.pop(ws_id, None)
|
||||
self._pending_approval_msgs.pop(ws_id, None)
|
||||
self._notify_reply_channels.pop(ws_id, None)
|
||||
stale = [mid for mid, entry in self._notify_ws_map.items() if entry[0] == ws_id]
|
||||
@@ -391,6 +412,13 @@ class TurnstoneBot:
|
||||
log.debug("discord.sse_remote_closed", ws_id=ws_id)
|
||||
except asyncio.CancelledError:
|
||||
return # unsubscribe or shutdown
|
||||
except (httpx.ConnectError, httpx.ConnectTimeout) as exc:
|
||||
log.warning(
|
||||
"discord.sse_connect_failed",
|
||||
ws_id=ws_id,
|
||||
url=url,
|
||||
error=str(exc),
|
||||
)
|
||||
except Exception:
|
||||
log.warning("discord.sse_error", ws_id=ws_id, exc_info=True)
|
||||
|
||||
@@ -416,7 +444,28 @@ class TurnstoneBot:
|
||||
)
|
||||
from turnstone.channels.discord.views import ApprovalView, PlanReviewView
|
||||
|
||||
if isinstance(event, ContentEvent):
|
||||
if isinstance(event, ThinkingStartEvent):
|
||||
# Clean up any prior thinking message (consecutive starts without stop).
|
||||
prev = self._thinking_msgs.pop(ws_id, None)
|
||||
if prev is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await prev.delete()
|
||||
try:
|
||||
msg = await thread.send("*Thinking...*")
|
||||
self._thinking_msgs[ws_id] = msg
|
||||
except Exception:
|
||||
log.debug("discord.thinking_start_send_failed", ws_id=ws_id)
|
||||
|
||||
elif isinstance(event, ThinkingStopEvent):
|
||||
# Leave the thinking message in place — the next visible event
|
||||
# (ContentEvent, ToolInfoEvent, StreamEndEvent) will edit or
|
||||
# clean it up, avoiding a delete→gap→new-message flicker.
|
||||
pass
|
||||
|
||||
elif isinstance(event, ContentEvent):
|
||||
# Reuse thinking message as the initial streaming message so the
|
||||
# first flush edits it in-place (no delete→gap→send flicker).
|
||||
thinking_msg = self._thinking_msgs.pop(ws_id, None)
|
||||
sm = self._streaming.get(ws_id)
|
||||
if sm is None:
|
||||
sm = StreamingMessage(
|
||||
@@ -424,9 +473,100 @@ class TurnstoneBot:
|
||||
max_length=self.config.max_message_length,
|
||||
edit_interval=self.config.streaming_edit_interval,
|
||||
)
|
||||
if thinking_msg is not None:
|
||||
sm._message = thinking_msg
|
||||
self._streaming[ws_id] = sm
|
||||
elif thinking_msg is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await thinking_msg.delete()
|
||||
await sm.append(event.text)
|
||||
|
||||
elif isinstance(event, ToolInfoEvent):
|
||||
from turnstone.channels._formatter import truncate
|
||||
|
||||
# Reuse the thinking message for the first tool embed.
|
||||
thinking_msg = self._thinking_msgs.pop(ws_id, None)
|
||||
|
||||
# Show a "running" embed for every tool. The approval dialog
|
||||
# (ApproveRequestEvent) is a separate concern — it asks "do you
|
||||
# authorize this?" while the running embed says "this tool is
|
||||
# executing." Both can coexist in the thread.
|
||||
for it in event.items:
|
||||
name = it.get("func_name") or it.get("approval_label") or "tool"
|
||||
raw_preview = it.get("preview", "")
|
||||
# Sanitize preview: escape backticks to prevent markdown
|
||||
# breakout and strip @-mentions.
|
||||
raw_preview = raw_preview.replace("`", "\\`")
|
||||
raw_preview = discord.utils.escape_mentions(raw_preview)
|
||||
preview = truncate(raw_preview, max_length=120) or None
|
||||
embed = discord.Embed(
|
||||
title=name,
|
||||
description=preview,
|
||||
color=discord.Color.light_grey(),
|
||||
)
|
||||
# Edit thinking message into first tool embed to avoid flicker.
|
||||
if thinking_msg is not None:
|
||||
try:
|
||||
await thinking_msg.edit(content=None, embed=embed)
|
||||
msg = thinking_msg
|
||||
except Exception:
|
||||
msg = await thread.send(embed=embed)
|
||||
thinking_msg = None
|
||||
else:
|
||||
msg = await thread.send(embed=embed)
|
||||
call_id = it.get("call_id", "")
|
||||
self._tool_info_msgs.setdefault(ws_id, []).append(
|
||||
(call_id, name, preview or "", msg)
|
||||
)
|
||||
|
||||
# If no items consumed the thinking message (empty event), clean up.
|
||||
if thinking_msg is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await thinking_msg.delete()
|
||||
|
||||
elif isinstance(event, ToolResultEvent):
|
||||
from turnstone.channels._formatter import format_tool_result
|
||||
|
||||
# Mark the matching "running" embed as complete/errored.
|
||||
# Prefer call_id match (deterministic); fall back to name (FIFO).
|
||||
info_list = self._tool_info_msgs.get(ws_id, [])
|
||||
matched_preview = ""
|
||||
matched_msg: discord.Message | None = None
|
||||
if event.call_id:
|
||||
for i, (cid, _tname, _prev, _tmsg) in enumerate(info_list):
|
||||
if cid == event.call_id:
|
||||
entry = info_list.pop(i)
|
||||
matched_preview, matched_msg = entry[2], entry[3]
|
||||
break
|
||||
if matched_msg is None:
|
||||
for i, (_cid, tname, _prev, _tmsg) in enumerate(info_list):
|
||||
if tname == event.name:
|
||||
entry = info_list.pop(i)
|
||||
matched_preview, matched_msg = entry[2], entry[3]
|
||||
break
|
||||
if matched_msg is not None:
|
||||
status = "Error" if event.is_error else "Done"
|
||||
status_color = discord.Color.red() if event.is_error else discord.Color.dark_grey()
|
||||
status_embed = discord.Embed(
|
||||
title=f"{event.name} \u2014 {status}",
|
||||
description=matched_preview or None,
|
||||
color=status_color,
|
||||
)
|
||||
try:
|
||||
await matched_msg.edit(content=None, embed=status_embed)
|
||||
except Exception:
|
||||
log.debug("discord.tool_info_status_edit_failed", ws_id=ws_id)
|
||||
|
||||
# Send the result as a separate message.
|
||||
desc = format_tool_result(event.output)
|
||||
color = discord.Color.red() if event.is_error else discord.Color.dark_grey()
|
||||
result_embed = discord.Embed(
|
||||
title=event.name,
|
||||
description=desc,
|
||||
color=color,
|
||||
)
|
||||
await thread.send(embed=result_embed)
|
||||
|
||||
elif isinstance(event, ApproveRequestEvent):
|
||||
# Evaluate admin tool policies before auto-approve.
|
||||
_policy_handled = False
|
||||
@@ -539,7 +679,26 @@ class TurnstoneBot:
|
||||
except Exception:
|
||||
log.debug("discord.verdict_embed_edit_failed", ws_id=ws_id)
|
||||
|
||||
elif isinstance(event, ApprovalResolvedEvent):
|
||||
# Server resolved the approval (timeout, external approve/reject).
|
||||
# Disable the buttons so they can't be clicked stale.
|
||||
approval_msg = self._pending_approval_msgs.pop(ws_id, None)
|
||||
if approval_msg is not None:
|
||||
from turnstone.channels.discord.views import disable_message_buttons
|
||||
|
||||
label = "Approved" if event.approved else "Denied"
|
||||
try:
|
||||
await disable_message_buttons(approval_msg, label)
|
||||
except Exception:
|
||||
log.debug("discord.approval_resolved_edit_failed", ws_id=ws_id)
|
||||
|
||||
elif isinstance(event, StreamEndEvent):
|
||||
# Edge-case cleanup: clear any lingering thinking indicator.
|
||||
thinking_msg = self._thinking_msgs.pop(ws_id, None)
|
||||
if thinking_msg is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await thinking_msg.delete()
|
||||
self._tool_info_msgs.pop(ws_id, None)
|
||||
sm = self._streaming.pop(ws_id, None)
|
||||
if sm is not None:
|
||||
await sm.finalize()
|
||||
|
||||
@@ -136,6 +136,7 @@ class MessageCog:
|
||||
str(channel.id),
|
||||
name=channel.name or "",
|
||||
initial_message="",
|
||||
client_type="chat",
|
||||
)
|
||||
except (TimeoutError, RuntimeError):
|
||||
log.warning("discord.ws_reactivation_failed", thread_id=channel.id)
|
||||
@@ -192,6 +193,7 @@ class MessageCog:
|
||||
name=thread_name,
|
||||
model=self.ts.config.model,
|
||||
initial_message="",
|
||||
client_type="chat",
|
||||
)
|
||||
|
||||
await self.ts.subscribe_ws(ws_id, thread)
|
||||
@@ -210,6 +212,10 @@ class MessageCog:
|
||||
# Only handle explicit replies to a tracked notification message.
|
||||
ref = message.reference
|
||||
if ref is None or ref.message_id is None:
|
||||
await message.channel.send(
|
||||
"*Direct messages aren't supported. "
|
||||
"Use `/ask` in a server channel or @mention me to start a conversation.*"
|
||||
)
|
||||
return
|
||||
|
||||
# Atomic pop prevents TOCTOU race across await points.
|
||||
@@ -366,6 +372,7 @@ class MessageCog:
|
||||
name=thread_name,
|
||||
model=self.ts.config.model,
|
||||
initial_message="",
|
||||
client_type="chat",
|
||||
)
|
||||
|
||||
await self.ts.subscribe_ws(ws_id, thread)
|
||||
|
||||
@@ -30,15 +30,17 @@ def _parse_footer(interaction: discord.Interaction) -> tuple[str, str] | None:
|
||||
return parts[0], parts[1]
|
||||
|
||||
|
||||
async def _disable_buttons(interaction: discord.Interaction, label: str) -> None:
|
||||
"""Edit the message to disable all buttons and append a result label."""
|
||||
async def disable_message_buttons(message: discord.Message, label: str) -> None:
|
||||
"""Disable all buttons on *message* and append *label* to the embed title.
|
||||
|
||||
Used both from interaction callbacks (via the message attribute) and
|
||||
from bot event handlers when the server resolves an approval externally
|
||||
(e.g. timeout).
|
||||
"""
|
||||
import discord
|
||||
|
||||
if interaction.message is None:
|
||||
return
|
||||
|
||||
view = discord.ui.View()
|
||||
for item in interaction.message.components or []:
|
||||
for item in message.components or []:
|
||||
for child in item.children: # type: ignore[union-attr]
|
||||
button: discord.ui.Button[discord.ui.View] = discord.ui.Button(
|
||||
label=getattr(child, "label", ""),
|
||||
@@ -48,12 +50,19 @@ async def _disable_buttons(interaction: discord.Interaction, label: str) -> None
|
||||
)
|
||||
view.add_item(button)
|
||||
|
||||
embed = interaction.message.embeds[0] if interaction.message.embeds else None
|
||||
embed = message.embeds[0] if message.embeds else None
|
||||
if embed is not None:
|
||||
embed.color = discord.Color.greyple()
|
||||
embed.title = f"{embed.title} - {label}"
|
||||
|
||||
await interaction.message.edit(embed=embed, view=view)
|
||||
await message.edit(embed=embed, view=view)
|
||||
|
||||
|
||||
async def _disable_buttons(interaction: discord.Interaction, label: str) -> None:
|
||||
"""Edit the interaction message to disable all buttons and append *label* to the embed title."""
|
||||
if interaction.message is None:
|
||||
return
|
||||
await disable_message_buttons(interaction.message, label)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -151,6 +160,8 @@ class ApprovalView:
|
||||
)
|
||||
|
||||
label = "Always Approved" if always else ("Approved" if approved else "Rejected")
|
||||
# Pop pending approval so ApprovalResolvedEvent doesn't double-update.
|
||||
self.bot._pending_approval_msgs.pop(ws_id, None)
|
||||
await _disable_buttons(interaction, label)
|
||||
await interaction.followup.send(
|
||||
f"Tool execution **{label.lower()}**.",
|
||||
|
||||
@@ -1132,6 +1132,7 @@ def main() -> None:
|
||||
ws_id: str | None = None,
|
||||
*,
|
||||
skill: str | None = None,
|
||||
client_type: str = "",
|
||||
) -> ChatSession:
|
||||
assert ui is not None, "session_factory requires a non-None UI"
|
||||
r_client, r_model, r_cfg = registry.resolve(model_alias)
|
||||
|
||||
+358
-196
@@ -1,22 +1,26 @@
|
||||
"""Cluster state collector — aggregates data from all turnstone nodes.
|
||||
|
||||
Discovers nodes via the service registry (StorageBackend), polls each
|
||||
node's /v1/api/dashboard endpoint for workstream data.
|
||||
Discovers nodes via the service registry (StorageBackend) and subscribes
|
||||
to each node's ``/v1/api/events/global`` SSE stream for real-time state
|
||||
updates. A single asyncio event loop on one dedicated thread multiplexes
|
||||
all SSE connections, scaling to 1000+ nodes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
import httpx_sse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from turnstone.console.metrics import ConsoleMetrics
|
||||
@@ -34,7 +38,7 @@ class NodeSnapshot:
|
||||
node_id: str = ""
|
||||
server_url: str = ""
|
||||
started: float = 0.0
|
||||
last_seen: float = 0.0 # monotonic time of last successful poll
|
||||
last_seen: float = 0.0 # monotonic time of last successful data
|
||||
max_ws: int = 10 # max workstreams (capacity)
|
||||
workstreams: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
health: dict[str, Any] = field(default_factory=dict)
|
||||
@@ -43,19 +47,17 @@ class NodeSnapshot:
|
||||
|
||||
|
||||
class ClusterCollector:
|
||||
"""Aggregates cluster state from the service registry and per-node HTTP APIs.
|
||||
"""Aggregates cluster state from the service registry and per-node SSE streams.
|
||||
|
||||
Two daemon threads:
|
||||
1. Node discovery — queries the service registry every ``discovery_interval`` seconds
|
||||
2. Poll loop — fetches /v1/api/dashboard from each node every ``poll_interval`` seconds
|
||||
2. SSE manager — single asyncio event loop multiplexing SSE connections to all nodes
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage: StorageBackend,
|
||||
poll_interval: float = 15.0,
|
||||
discovery_interval: float = 15.0,
|
||||
max_poll_workers: int = 200,
|
||||
discovery_interval: float = 60.0,
|
||||
http_timeout: float = 30.0,
|
||||
auth_token: str = "",
|
||||
token_manager: ServiceTokenManager | None = None,
|
||||
@@ -65,16 +67,14 @@ class ClusterCollector:
|
||||
console_metrics: ConsoleMetrics | None = None,
|
||||
):
|
||||
self._storage = storage
|
||||
self._poll_interval = poll_interval
|
||||
self._discovery_interval = discovery_interval
|
||||
self._max_poll_workers = max_poll_workers
|
||||
self._http_timeout = http_timeout
|
||||
self._token_manager = token_manager
|
||||
self._router = router
|
||||
self._console_metrics = console_metrics
|
||||
self._tls_verify = tls_verify
|
||||
self._tls_cert = tls_cert
|
||||
# Static auth header — only used when no token_manager is present.
|
||||
# When a token_manager exists, auth is injected per-request via
|
||||
# extra_headers in _poll_all_nodes to avoid stale JWT expiry.
|
||||
self._static_auth: dict[str, str] | None = None
|
||||
if auth_token and token_manager is None:
|
||||
self._static_auth = {"Authorization": f"Bearer {auth_token}"}
|
||||
@@ -83,39 +83,41 @@ class ClusterCollector:
|
||||
self._nodes: dict[str, NodeSnapshot] = {}
|
||||
self._running = False
|
||||
self._threads: list[threading.Thread] = []
|
||||
self._poll_pool = ThreadPoolExecutor(max_workers=max_poll_workers)
|
||||
self._http_client = httpx.Client(
|
||||
timeout=httpx.Timeout(connect=10, read=http_timeout, write=5, pool=http_timeout),
|
||||
limits=httpx.Limits(
|
||||
max_connections=max_poll_workers + 10,
|
||||
max_keepalive_connections=min(max_poll_workers, 200),
|
||||
),
|
||||
verify=tls_verify,
|
||||
cert=tls_cert,
|
||||
)
|
||||
|
||||
# SSE fan-out to browser clients
|
||||
self._listeners: list[queue.Queue[dict[str, Any]]] = []
|
||||
self._listeners_lock = threading.Lock()
|
||||
|
||||
# SSE manager state (managed by the asyncio event loop thread)
|
||||
self._sse_loop: asyncio.AbstractEventLoop | None = None
|
||||
self._sse_tasks: dict[str, asyncio.Task[None]] = {}
|
||||
self._sse_stop_events: dict[str, asyncio.Event] = {}
|
||||
self._sse_async_client: httpx.AsyncClient | None = None
|
||||
|
||||
def upgrade_tls(self, tls_verify: Any = True, tls_cert: tuple[str, str] | None = None) -> None:
|
||||
"""Replace the httpx client with one using mTLS context."""
|
||||
old = self._http_client
|
||||
self._http_client = httpx.Client(
|
||||
timeout=httpx.Timeout(
|
||||
connect=10, read=self._http_timeout, write=5, pool=self._http_timeout
|
||||
),
|
||||
limits=httpx.Limits(
|
||||
max_connections=self._max_poll_workers + 10,
|
||||
max_keepalive_connections=min(self._max_poll_workers, 200),
|
||||
),
|
||||
verify=tls_verify,
|
||||
cert=tls_cert,
|
||||
"""Update TLS settings for future SSE connections."""
|
||||
self._tls_verify = tls_verify
|
||||
self._tls_cert = tls_cert
|
||||
# If the async client is running, replace it on the event loop.
|
||||
if self._sse_loop is not None and self._sse_loop.is_running():
|
||||
asyncio.run_coroutine_threadsafe(self._replace_async_client(), self._sse_loop)
|
||||
|
||||
async def _replace_async_client(self) -> None:
|
||||
"""Replace the async httpx client (called on the SSE event loop).
|
||||
|
||||
Closing the old client terminates its underlying connections, which
|
||||
causes active ``_node_sse_task`` coroutines to raise and reconnect
|
||||
using the new client with updated TLS settings.
|
||||
"""
|
||||
old = self._sse_async_client
|
||||
self._sse_async_client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(connect=10, read=None, write=5, pool=None),
|
||||
limits=httpx.Limits(max_connections=2000, max_keepalive_connections=1500),
|
||||
verify=self._tls_verify,
|
||||
cert=self._tls_cert,
|
||||
)
|
||||
# Don't close old client — concurrent _fetch_node() threads may still
|
||||
# be using it. It will be GC'd once all references are released, and
|
||||
# the current client is closed in stop().
|
||||
del old
|
||||
if old is not None:
|
||||
await old.aclose()
|
||||
|
||||
# -- lifecycle -----------------------------------------------------------
|
||||
|
||||
@@ -124,7 +126,7 @@ class ClusterCollector:
|
||||
self._running = True
|
||||
for target, name in [
|
||||
(self._discovery_loop, "console-discovery"),
|
||||
(self._poll_loop, "console-poll"),
|
||||
(self._sse_manager_thread, "console-sse"),
|
||||
]:
|
||||
t = threading.Thread(target=target, name=name, daemon=True)
|
||||
t.start()
|
||||
@@ -132,10 +134,22 @@ class ClusterCollector:
|
||||
log.info("ClusterCollector started")
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop all threads and clean up resources."""
|
||||
"""Stop all threads and clean up resources.
|
||||
|
||||
Sets ``_running = False`` which causes the SSE manager coroutine to
|
||||
exit naturally (its ``while self._running`` loop terminates), running
|
||||
its ``finally`` cleanup (cancel tasks, close AsyncClient).
|
||||
"""
|
||||
self._running = False
|
||||
self._poll_pool.shutdown(wait=False)
|
||||
self._http_client.close()
|
||||
# Request cancellation of all SSE tasks so they don't block the
|
||||
# manager's cleanup. The manager coroutine exits when _running is
|
||||
# False and handles remaining task cancellation in its finally block.
|
||||
if self._sse_loop is not None and self._sse_loop.is_running():
|
||||
for node_id in list(self._sse_tasks):
|
||||
asyncio.run_coroutine_threadsafe(self._stop_node(node_id), self._sse_loop)
|
||||
# Wait for background threads to finish their shutdown.
|
||||
for t in self._threads:
|
||||
t.join(timeout=5)
|
||||
log.info("ClusterCollector stopped")
|
||||
|
||||
def _fanout(self, event: dict[str, Any]) -> None:
|
||||
@@ -145,6 +159,137 @@ class ClusterCollector:
|
||||
with contextlib.suppress(queue.Full):
|
||||
q.put_nowait(event)
|
||||
|
||||
# -- auth helpers --------------------------------------------------------
|
||||
|
||||
def _auth_headers(self) -> dict[str, str]:
|
||||
"""Build auth headers for the current SSE connection."""
|
||||
if self._token_manager is not None:
|
||||
return {"Authorization": f"Bearer {self._token_manager.token}"}
|
||||
return dict(self._static_auth) if self._static_auth else {}
|
||||
|
||||
# -- SSE manager ---------------------------------------------------------
|
||||
|
||||
def _sse_manager_thread(self) -> None:
|
||||
"""Run asyncio event loop that manages all node SSE connections."""
|
||||
self._sse_loop = asyncio.new_event_loop()
|
||||
try:
|
||||
self._sse_loop.run_until_complete(self._sse_manager())
|
||||
finally:
|
||||
self._sse_loop.close()
|
||||
self._sse_loop = None
|
||||
|
||||
async def _sse_manager(self) -> None:
|
||||
"""Top-level coroutine — runs until collector stops."""
|
||||
self._sse_async_client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(connect=10, read=None, write=5, pool=None),
|
||||
limits=httpx.Limits(max_connections=2000, max_keepalive_connections=1500),
|
||||
verify=self._tls_verify,
|
||||
cert=self._tls_cert,
|
||||
)
|
||||
try:
|
||||
while self._running:
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
# Cancel all remaining tasks
|
||||
for task in self._sse_tasks.values():
|
||||
task.cancel()
|
||||
for task in self._sse_tasks.values():
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
self._sse_tasks.clear()
|
||||
self._sse_stop_events.clear()
|
||||
await self._sse_async_client.aclose()
|
||||
self._sse_async_client = None
|
||||
|
||||
async def _start_node(self, node_id: str) -> None:
|
||||
"""Start an SSE task for a node (called on the SSE event loop)."""
|
||||
if node_id in self._sse_tasks:
|
||||
return # already running
|
||||
stop = asyncio.Event()
|
||||
self._sse_stop_events[node_id] = stop
|
||||
self._sse_tasks[node_id] = asyncio.create_task(
|
||||
self._node_sse_task(node_id, stop),
|
||||
name=f"sse-{node_id}",
|
||||
)
|
||||
|
||||
async def _stop_node(self, node_id: str) -> None:
|
||||
"""Stop an SSE task for a node (called on the SSE event loop)."""
|
||||
stop = self._sse_stop_events.pop(node_id, None)
|
||||
if stop:
|
||||
stop.set()
|
||||
task = self._sse_tasks.pop(node_id, None)
|
||||
if task:
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
async def _node_sse_task(self, node_id: str, stop_event: asyncio.Event) -> None:
|
||||
"""Persistent SSE connection to a single server node."""
|
||||
backoff = 1.0
|
||||
while not stop_event.is_set() and self._running:
|
||||
url = self._get_node_url(node_id)
|
||||
if not url or self._sse_async_client is None:
|
||||
break
|
||||
base = url.rstrip("/")
|
||||
try:
|
||||
async with httpx_sse.aconnect_sse(
|
||||
self._sse_async_client,
|
||||
"GET",
|
||||
f"{base}/v1/api/events/global",
|
||||
params={"expected_node_id": node_id},
|
||||
headers=self._auth_headers(),
|
||||
) as source:
|
||||
if source.response.status_code == 409:
|
||||
log.warning("Node identity mismatch for %s at %s", node_id, url)
|
||||
self._mark_unreachable(node_id)
|
||||
break # stop reconnecting — wrong node at this URL
|
||||
source.response.raise_for_status()
|
||||
async for sse in source.aiter_sse():
|
||||
if stop_event.is_set():
|
||||
break
|
||||
if not sse.data:
|
||||
continue # ping/comment frame
|
||||
try:
|
||||
data = json.loads(sse.data)
|
||||
except json.JSONDecodeError:
|
||||
log.debug("Invalid SSE JSON from node %s", node_id)
|
||||
continue
|
||||
etype = data.get("type", "")
|
||||
if etype == "node_snapshot":
|
||||
# Client-side identity check (defense in depth)
|
||||
if data.get("node_id") != node_id:
|
||||
log.warning(
|
||||
"Snapshot node_id mismatch: expected %s, got %s",
|
||||
node_id,
|
||||
data.get("node_id"),
|
||||
)
|
||||
self._mark_unreachable(node_id)
|
||||
break
|
||||
self._apply_snapshot(node_id, data)
|
||||
backoff = 1.0
|
||||
else:
|
||||
self._apply_delta(node_id, data)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.debug("SSE error for node %s", node_id, exc_info=True)
|
||||
self._mark_unreachable(node_id)
|
||||
await asyncio.sleep(min(backoff, 30) + random.random())
|
||||
backoff = min(backoff * 2, 30)
|
||||
|
||||
def _get_node_url(self, node_id: str) -> str:
|
||||
"""Get the server URL for a node (thread-safe)."""
|
||||
with self._lock:
|
||||
node = self._nodes.get(node_id)
|
||||
return node.server_url if node else ""
|
||||
|
||||
def _mark_unreachable(self, node_id: str) -> None:
|
||||
"""Mark a node as unreachable (thread-safe)."""
|
||||
with self._lock:
|
||||
node = self._nodes.get(node_id)
|
||||
if node:
|
||||
node.reachable = False
|
||||
|
||||
# -- node discovery ------------------------------------------------------
|
||||
|
||||
def _discovery_loop(self) -> None:
|
||||
@@ -161,6 +306,8 @@ class ClusterCollector:
|
||||
raw_services = self._storage.list_services("server", max_age_seconds=120)
|
||||
active_ids = set()
|
||||
pending_events: list[dict[str, Any]] = []
|
||||
new_nodes: list[str] = []
|
||||
lost_nodes: list[str] = []
|
||||
|
||||
with self._lock:
|
||||
for svc in raw_services:
|
||||
@@ -183,6 +330,7 @@ class ClusterCollector:
|
||||
max_ws=meta.get("max_ws", 10),
|
||||
)
|
||||
pending_events.append({"type": "node_joined", "node_id": nid})
|
||||
new_nodes.append(nid)
|
||||
log.info("Discovered node: %s", nid)
|
||||
else:
|
||||
self._nodes[nid].server_url = url or self._nodes[nid].server_url
|
||||
@@ -193,10 +341,18 @@ class ClusterCollector:
|
||||
for nid in lost:
|
||||
del self._nodes[nid]
|
||||
pending_events.append({"type": "node_lost", "node_id": nid})
|
||||
lost_nodes.append(nid)
|
||||
log.info("Lost node: %s", nid)
|
||||
for event in pending_events:
|
||||
self._fanout(event)
|
||||
|
||||
# Manage SSE tasks for new/lost nodes
|
||||
if self._sse_loop is not None and self._sse_loop.is_running():
|
||||
for nid in new_nodes:
|
||||
asyncio.run_coroutine_threadsafe(self._start_node(nid), self._sse_loop)
|
||||
for nid in lost_nodes:
|
||||
asyncio.run_coroutine_threadsafe(self._stop_node(nid), self._sse_loop)
|
||||
|
||||
# Notify the routing layer so it can refresh its hash-ring cache
|
||||
# when the rebalancer has published a new version.
|
||||
if self._router is not None:
|
||||
@@ -211,114 +367,67 @@ class ClusterCollector:
|
||||
self._router.version,
|
||||
)
|
||||
|
||||
# -- polling -------------------------------------------------------------
|
||||
# -- SSE event handlers --------------------------------------------------
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
"""Periodically fetch /v1/api/dashboard from each node."""
|
||||
while self._running:
|
||||
try:
|
||||
self._poll_all_nodes()
|
||||
except Exception:
|
||||
log.exception("Poll loop error")
|
||||
time.sleep(self._poll_interval)
|
||||
def _reconcile_node(
|
||||
self, node_id: str, node: NodeSnapshot, new_ws_list: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Diff new workstream data against the current snapshot.
|
||||
|
||||
@staticmethod
|
||||
def _node_jitter(node_id: str, window: float) -> float:
|
||||
"""Deterministic per-node delay within a sliding window.
|
||||
|
||||
Uses a Mersenne prime (2^31 - 1) to hash the node_id into a
|
||||
stable offset so each node is polled at a different point in
|
||||
the cycle. The offset is consistent across restarts for the
|
||||
same node_id, giving an even spread without randomness.
|
||||
Returns a list of pending events. Caller must hold ``_lock``.
|
||||
Updates ``node.workstreams`` in place.
|
||||
"""
|
||||
h = hash(node_id) & 0x7FFFFFFF # positive 31-bit
|
||||
return (h % 2147483647) / 2147483647 * window # M31 = 2^31 - 1
|
||||
pending: list[dict[str, Any]] = []
|
||||
old_ids = {k for k in node.workstreams if k}
|
||||
new_ws: dict[str, dict[str, Any]] = {}
|
||||
for ws in new_ws_list:
|
||||
ws_id = ws.get("id", "")
|
||||
if not ws_id:
|
||||
continue
|
||||
ws["node"] = node_id
|
||||
ws["server_url"] = node.server_url
|
||||
new_ws[ws_id] = ws
|
||||
new_ids = set(new_ws.keys())
|
||||
# Additions
|
||||
for ws_id in sorted(new_ids - old_ids):
|
||||
ws = new_ws[ws_id]
|
||||
pending.append(
|
||||
{
|
||||
"type": "ws_created",
|
||||
"ws_id": ws_id,
|
||||
"name": ws.get("name", ""),
|
||||
"node_id": node_id,
|
||||
}
|
||||
)
|
||||
# Removals
|
||||
for ws_id in sorted(old_ids - new_ids):
|
||||
pending.append({"type": "ws_closed", "ws_id": ws_id})
|
||||
# State and name changes on existing workstreams
|
||||
for ws_id in sorted(new_ids & old_ids):
|
||||
old_ws = node.workstreams.get(ws_id, {})
|
||||
new_w = new_ws[ws_id]
|
||||
old_state = old_ws.get("state", "")
|
||||
new_state = new_w.get("state", "")
|
||||
if old_state != new_state:
|
||||
pending.append(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": ws_id,
|
||||
"state": new_state,
|
||||
"node_id": node_id,
|
||||
"tokens": new_w.get("tokens", 0),
|
||||
"content": new_w.get("content", ""),
|
||||
}
|
||||
)
|
||||
old_name = old_ws.get("name", "")
|
||||
new_name = new_w.get("name", "")
|
||||
if old_name != new_name and new_name:
|
||||
pending.append({"type": "ws_rename", "ws_id": ws_id, "name": new_name})
|
||||
node.workstreams = new_ws
|
||||
return pending
|
||||
|
||||
def _poll_all_nodes(self) -> None:
|
||||
"""Fetch dashboard data from all known nodes in parallel.
|
||||
|
||||
Submissions are throttled by the thread pool size to avoid a
|
||||
thundering herd — at most ``max_poll_workers`` concurrent HTTP
|
||||
requests are in flight at any time. Each worker sleeps a
|
||||
deterministic per-node jitter (derived from its node_id) to
|
||||
spread requests across the first half of the poll interval.
|
||||
"""
|
||||
# Snapshot current auth header for this poll cycle. Per-request
|
||||
# headers avoid mutating shared client state (thread-safe).
|
||||
if self._token_manager is not None:
|
||||
poll_headers: dict[str, str] | None = {
|
||||
"Authorization": f"Bearer {self._token_manager.token}"
|
||||
}
|
||||
else:
|
||||
poll_headers = self._static_auth
|
||||
with self._lock:
|
||||
targets = [
|
||||
(n.node_id, n.server_url)
|
||||
for n in self._nodes.values()
|
||||
if n.server_url and n.server_url.startswith("http")
|
||||
]
|
||||
|
||||
if not targets:
|
||||
return
|
||||
|
||||
jitter_window = self._poll_interval / 2
|
||||
|
||||
def _jittered_fetch(
|
||||
nid: str, url: str, headers: dict[str, str] | None
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
delay = self._node_jitter(nid, jitter_window)
|
||||
if delay > 0.1:
|
||||
time.sleep(delay)
|
||||
return self._fetch_node(nid, url, headers)
|
||||
|
||||
futures = {
|
||||
self._poll_pool.submit(_jittered_fetch, nid, url, poll_headers): nid
|
||||
for nid, url in targets
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
nid = futures[future]
|
||||
try:
|
||||
dashboard, health = future.result()
|
||||
self._apply_poll(nid, dashboard, health)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code in (401, 403):
|
||||
log.warning(
|
||||
"Auth failure polling node %s: HTTP %d", nid, exc.response.status_code
|
||||
)
|
||||
else:
|
||||
log.debug("Failed to poll node %s: HTTP %d", nid, exc.response.status_code)
|
||||
with self._lock:
|
||||
if nid in self._nodes:
|
||||
self._nodes[nid].reachable = False
|
||||
except Exception:
|
||||
log.warning("Failed to poll node %s", nid, exc_info=True)
|
||||
with self._lock:
|
||||
if nid in self._nodes:
|
||||
self._nodes[nid].reachable = False
|
||||
|
||||
def _fetch_node(
|
||||
self,
|
||||
node_id: str,
|
||||
server_url: str,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Fetch /v1/api/dashboard and /health from a single node."""
|
||||
base = server_url.rstrip("/")
|
||||
dash_resp = self._http_client.get(f"{base}/v1/api/dashboard", headers=extra_headers)
|
||||
dash_resp.raise_for_status()
|
||||
dash_data: dict[str, Any] = dash_resp.json()
|
||||
try:
|
||||
health_resp = self._http_client.get(f"{base}/health", headers=extra_headers)
|
||||
health_data: dict[str, Any] = health_resp.json()
|
||||
except Exception:
|
||||
log.debug("Failed to fetch health from %s", node_id, exc_info=True)
|
||||
health_data = {}
|
||||
return dash_data, health_data
|
||||
|
||||
def _apply_poll(self, node_id: str, dashboard: dict[str, Any], health: dict[str, Any]) -> None:
|
||||
"""Apply polled data to the in-memory node snapshot."""
|
||||
ws_list = dashboard.get("workstreams", [])
|
||||
aggregate = dashboard.get("aggregate", {})
|
||||
def _apply_snapshot(self, node_id: str, data: dict[str, Any]) -> None:
|
||||
"""Apply a ``node_snapshot`` SSE event to the in-memory state."""
|
||||
pending_events: list[dict[str, Any]] = []
|
||||
with self._lock:
|
||||
node = self._nodes.get(node_id)
|
||||
@@ -326,63 +435,116 @@ class ClusterCollector:
|
||||
return
|
||||
node.last_seen = time.monotonic()
|
||||
node.reachable = True
|
||||
node.health = health
|
||||
node.aggregate = aggregate
|
||||
# Build new workstream map
|
||||
old_ids = {k for k in node.workstreams if k}
|
||||
new_ws: dict[str, dict[str, Any]] = {}
|
||||
for ws in ws_list:
|
||||
ws_id = ws.get("id", "")
|
||||
if not ws_id:
|
||||
continue
|
||||
ws["node"] = node_id
|
||||
ws["server_url"] = node.server_url
|
||||
new_ws[ws_id] = ws
|
||||
new_ids = set(new_ws.keys())
|
||||
# Detect additions not yet known to SSE clients
|
||||
for ws_id in sorted(new_ids - old_ids):
|
||||
ws = new_ws[ws_id]
|
||||
node.health = data.get("health", {})
|
||||
node.aggregate = data.get("aggregate", {})
|
||||
pending_events = self._reconcile_node(node_id, node, data.get("workstreams", []))
|
||||
for event in pending_events:
|
||||
self._fanout(event)
|
||||
|
||||
def _apply_delta(self, node_id: str, data: dict[str, Any]) -> None:
|
||||
"""Apply a single delta SSE event to the in-memory state."""
|
||||
etype = data.get("type", "")
|
||||
if not etype:
|
||||
return
|
||||
pending_events: list[dict[str, Any]] = []
|
||||
with self._lock:
|
||||
node = self._nodes.get(node_id)
|
||||
if not node:
|
||||
return
|
||||
node.last_seen = time.monotonic()
|
||||
|
||||
if etype == "ws_state":
|
||||
# Server emits ws_state; translate to cluster_state for browser.
|
||||
# Only fan out if the workstream is known — a ws_state arriving
|
||||
# before ws_created (race) is silently absorbed on reconnect.
|
||||
ws_id = data.get("ws_id", "")
|
||||
ws = node.workstreams.get(ws_id)
|
||||
if ws:
|
||||
ws["state"] = data.get("state", ws.get("state", ""))
|
||||
ws["tokens"] = data.get("tokens", ws.get("tokens", 0))
|
||||
ws["context_ratio"] = data.get("context_ratio", ws.get("context_ratio", 0))
|
||||
ws["activity"] = data.get("activity", ws.get("activity", ""))
|
||||
ws["activity_state"] = data.get("activity_state", ws.get("activity_state", ""))
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": ws_id,
|
||||
"state": data.get("state", ""),
|
||||
"node_id": node_id,
|
||||
"tokens": data.get("tokens", 0),
|
||||
"content": data.get("content", ""),
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "ws_activity":
|
||||
ws_id = data.get("ws_id", "")
|
||||
ws = node.workstreams.get(ws_id)
|
||||
if ws:
|
||||
ws["activity"] = data.get("activity", "")
|
||||
ws["activity_state"] = data.get("activity_state", "")
|
||||
# Activity events are not forwarded to cluster SSE — only state changes
|
||||
|
||||
elif etype == "ws_created":
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id and ws_id not in node.workstreams:
|
||||
node.workstreams[ws_id] = {
|
||||
"id": ws_id,
|
||||
"name": data.get("name", ""),
|
||||
"state": "idle",
|
||||
"node": node_id,
|
||||
"server_url": node.server_url,
|
||||
"model": data.get("model", ""),
|
||||
"model_alias": data.get("model_alias", ""),
|
||||
"tokens": 0,
|
||||
"context_ratio": 0.0,
|
||||
"activity": "",
|
||||
"activity_state": "",
|
||||
"tool_calls": 0,
|
||||
"title": "",
|
||||
}
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "ws_created",
|
||||
"ws_id": ws_id,
|
||||
"name": ws.get("name", ""),
|
||||
"name": data.get("name", ""),
|
||||
"node_id": node_id,
|
||||
}
|
||||
)
|
||||
# Detect removals
|
||||
for ws_id in sorted(old_ids - new_ids):
|
||||
|
||||
elif etype == "ws_closed":
|
||||
ws_id = data.get("ws_id", "")
|
||||
node.workstreams.pop(ws_id, None)
|
||||
pending_events.append({"type": "ws_closed", "ws_id": ws_id})
|
||||
# Detect state changes on existing workstreams
|
||||
for ws_id in sorted(new_ids & old_ids):
|
||||
old_ws = node.workstreams.get(ws_id, {})
|
||||
new_w = new_ws[ws_id]
|
||||
old_state = old_ws.get("state", "")
|
||||
new_state = new_w.get("state", "")
|
||||
if old_state != new_state:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "ws_state",
|
||||
"ws_id": ws_id,
|
||||
"state": new_state,
|
||||
"node_id": node_id,
|
||||
"tokens": new_w.get("tokens", 0),
|
||||
"content": new_w.get("content", ""),
|
||||
}
|
||||
)
|
||||
# Detect name/title changes
|
||||
old_name = old_ws.get("name", "")
|
||||
new_name = new_w.get("name", "")
|
||||
if old_name != new_name and new_name:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "ws_rename",
|
||||
"ws_id": ws_id,
|
||||
"name": new_name,
|
||||
}
|
||||
)
|
||||
node.workstreams = new_ws
|
||||
# Fan out diffs to SSE listeners outside the lock
|
||||
|
||||
elif etype == "ws_rename":
|
||||
ws_id = data.get("ws_id", "")
|
||||
name = data.get("name", "")
|
||||
ws = node.workstreams.get(ws_id)
|
||||
if ws and name:
|
||||
ws["name"] = name
|
||||
pending_events.append({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
elif etype == "health_changed":
|
||||
# Update the health dict's circuit state in-place
|
||||
circuit = data.get("circuit_state", "")
|
||||
if circuit:
|
||||
if not node.health:
|
||||
node.health = {}
|
||||
backend = node.health.setdefault("backend", {})
|
||||
backend["circuit_state"] = circuit
|
||||
backend["status"] = "up" if circuit == "closed" else "down"
|
||||
node.health["status"] = "ok" if circuit == "closed" else "degraded"
|
||||
# Not forwarded to cluster SSE — next snapshot refreshes UI
|
||||
|
||||
elif etype == "aggregate":
|
||||
node.aggregate = {
|
||||
"total_tokens": data.get("total_tokens", 0),
|
||||
"total_tool_calls": data.get("total_tool_calls", 0),
|
||||
"active_count": data.get("active_count", 0),
|
||||
"total_count": data.get("total_count", 0),
|
||||
}
|
||||
# Not forwarded to cluster SSE — overview queries read from snapshot
|
||||
|
||||
for event in pending_events:
|
||||
self._fanout(event)
|
||||
|
||||
|
||||
+208
-7
@@ -5567,6 +5567,193 @@ async def tls_ca_cert(request: Request) -> Response:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin: Prompt Policies (system message composition)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def admin_list_prompt_policies(request: Request) -> JSONResponse:
|
||||
"""GET /v1/api/admin/prompt-policies — list all prompt policies."""
|
||||
from turnstone.core.auth import require_permission
|
||||
from turnstone.core.web_helpers import require_storage_or_503
|
||||
|
||||
storage, err = require_storage_or_503(request)
|
||||
if err:
|
||||
return err
|
||||
err = require_permission(request, "admin.prompt_policies")
|
||||
if err:
|
||||
return err
|
||||
|
||||
policies = storage.list_prompt_policies()
|
||||
return JSONResponse({"policies": policies})
|
||||
|
||||
|
||||
async def admin_create_prompt_policy(request: Request) -> JSONResponse:
|
||||
"""POST /v1/api/admin/prompt-policies — create a prompt policy."""
|
||||
import uuid
|
||||
|
||||
from turnstone.core.audit import record_audit
|
||||
from turnstone.core.auth import require_permission
|
||||
from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503
|
||||
|
||||
storage, err = require_storage_or_503(request)
|
||||
if err:
|
||||
return err
|
||||
err = require_permission(request, "admin.prompt_policies")
|
||||
if err:
|
||||
return err
|
||||
|
||||
body = await read_json_or_400(request)
|
||||
if isinstance(body, JSONResponse):
|
||||
return body
|
||||
|
||||
name = str(body.get("name", "")).strip()[:64]
|
||||
content = str(body.get("content", "")).strip()[:32768]
|
||||
if not name:
|
||||
return JSONResponse({"error": "name is required"}, status_code=400)
|
||||
if not content:
|
||||
return JSONResponse({"error": "content is required"}, status_code=400)
|
||||
|
||||
try:
|
||||
priority = int(body.get("priority", 0))
|
||||
except (ValueError, TypeError):
|
||||
return JSONResponse({"error": "priority must be an integer"}, status_code=400)
|
||||
|
||||
policy_id = uuid.uuid4().hex
|
||||
audit_uid, ip = _audit_context(request)
|
||||
|
||||
storage.upsert_prompt_policy(
|
||||
{
|
||||
"policy_id": policy_id,
|
||||
"name": name,
|
||||
"content": content,
|
||||
"tool_gate": str(body.get("tool_gate", "")).strip(),
|
||||
"priority": priority,
|
||||
"enabled": bool(body.get("enabled", True)),
|
||||
"org_id": str(body.get("org_id", "")).strip(),
|
||||
"created_by": audit_uid,
|
||||
}
|
||||
)
|
||||
|
||||
record_audit(
|
||||
storage,
|
||||
audit_uid,
|
||||
"prompt_policy.create",
|
||||
"prompt_policy",
|
||||
policy_id,
|
||||
{"name": name},
|
||||
ip,
|
||||
)
|
||||
|
||||
return JSONResponse(storage.get_prompt_policy(policy_id) or {})
|
||||
|
||||
|
||||
async def admin_get_prompt_policy(request: Request) -> JSONResponse:
|
||||
"""GET /v1/api/admin/prompt-policies/{policy_id}."""
|
||||
from turnstone.core.auth import require_permission
|
||||
from turnstone.core.web_helpers import require_storage_or_503
|
||||
|
||||
storage, err = require_storage_or_503(request)
|
||||
if err:
|
||||
return err
|
||||
err = require_permission(request, "admin.prompt_policies")
|
||||
if err:
|
||||
return err
|
||||
|
||||
policy_id = request.path_params["policy_id"]
|
||||
policy = storage.get_prompt_policy(policy_id)
|
||||
if policy is None:
|
||||
return JSONResponse({"error": "Prompt policy not found"}, status_code=404)
|
||||
return JSONResponse(policy)
|
||||
|
||||
|
||||
async def admin_update_prompt_policy(request: Request) -> JSONResponse:
|
||||
"""PUT /v1/api/admin/prompt-policies/{policy_id}."""
|
||||
from turnstone.core.audit import record_audit
|
||||
from turnstone.core.auth import require_permission
|
||||
from turnstone.core.web_helpers import read_json_or_400, require_storage_or_503
|
||||
|
||||
storage, err = require_storage_or_503(request)
|
||||
if err:
|
||||
return err
|
||||
err = require_permission(request, "admin.prompt_policies")
|
||||
if err:
|
||||
return err
|
||||
|
||||
policy_id = request.path_params["policy_id"]
|
||||
existing = storage.get_prompt_policy(policy_id)
|
||||
if existing is None:
|
||||
return JSONResponse({"error": "Prompt policy not found"}, status_code=404)
|
||||
|
||||
body = await read_json_or_400(request)
|
||||
if isinstance(body, JSONResponse):
|
||||
return body
|
||||
|
||||
update = dict(body)
|
||||
update["policy_id"] = policy_id
|
||||
if "name" in update:
|
||||
update["name"] = str(update["name"]).strip()[:64]
|
||||
if "priority" in update:
|
||||
try:
|
||||
update["priority"] = int(update["priority"])
|
||||
except (ValueError, TypeError):
|
||||
return JSONResponse({"error": "priority must be an integer"}, status_code=400)
|
||||
if "content" in update:
|
||||
update["content"] = str(update["content"]).strip()[:32768]
|
||||
if "tool_gate" in update:
|
||||
update["tool_gate"] = str(update["tool_gate"] or "").strip()
|
||||
if "enabled" in update:
|
||||
update["enabled"] = bool(update["enabled"])
|
||||
storage.upsert_prompt_policy(update)
|
||||
|
||||
audit_uid, ip = _audit_context(request)
|
||||
record_audit(
|
||||
storage,
|
||||
audit_uid,
|
||||
"prompt_policy.update",
|
||||
"prompt_policy",
|
||||
policy_id,
|
||||
{"name": existing.get("name", "")},
|
||||
ip,
|
||||
)
|
||||
|
||||
return JSONResponse(storage.get_prompt_policy(policy_id) or {})
|
||||
|
||||
|
||||
async def admin_delete_prompt_policy(request: Request) -> JSONResponse:
|
||||
"""DELETE /v1/api/admin/prompt-policies/{policy_id}."""
|
||||
from turnstone.core.audit import record_audit
|
||||
from turnstone.core.auth import require_permission
|
||||
from turnstone.core.web_helpers import require_storage_or_503
|
||||
|
||||
storage, err = require_storage_or_503(request)
|
||||
if err:
|
||||
return err
|
||||
err = require_permission(request, "admin.prompt_policies")
|
||||
if err:
|
||||
return err
|
||||
|
||||
policy_id = request.path_params["policy_id"]
|
||||
existing = storage.get_prompt_policy(policy_id)
|
||||
if existing is None:
|
||||
return JSONResponse({"error": "Prompt policy not found"}, status_code=404)
|
||||
|
||||
storage.delete_prompt_policy(policy_id)
|
||||
|
||||
audit_uid, ip = _audit_context(request)
|
||||
record_audit(
|
||||
storage,
|
||||
audit_uid,
|
||||
"prompt_policy.delete",
|
||||
"prompt_policy",
|
||||
policy_id,
|
||||
{"name": existing.get("name", "")},
|
||||
ip,
|
||||
)
|
||||
|
||||
return JSONResponse({"status": "ok", "policy_id": policy_id})
|
||||
|
||||
|
||||
async def admin_ring_status(request: Request) -> JSONResponse:
|
||||
"""GET /v1/api/admin/ring/status — hash ring rebalancer status."""
|
||||
from turnstone.core.auth import require_permission
|
||||
@@ -6005,6 +6192,27 @@ def create_app(
|
||||
"/api/admin/model-capabilities/known",
|
||||
admin_known_models,
|
||||
),
|
||||
# Governance: Prompt Policies
|
||||
Route("/api/admin/prompt-policies", admin_list_prompt_policies),
|
||||
Route(
|
||||
"/api/admin/prompt-policies",
|
||||
admin_create_prompt_policy,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/api/admin/prompt-policies/{policy_id}",
|
||||
admin_get_prompt_policy,
|
||||
),
|
||||
Route(
|
||||
"/api/admin/prompt-policies/{policy_id}",
|
||||
admin_update_prompt_policy,
|
||||
methods=["PUT"],
|
||||
),
|
||||
Route(
|
||||
"/api/admin/prompt-policies/{policy_id}",
|
||||
admin_delete_prompt_policy,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
# Governance: Usage & Audit
|
||||
Route("/api/admin/usage", admin_usage),
|
||||
Route("/api/admin/audit", admin_audit),
|
||||
@@ -6140,12 +6348,6 @@ def main() -> None:
|
||||
default=8090,
|
||||
help="Port to listen on (default: 8090)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--poll-interval",
|
||||
type=float,
|
||||
default=10.0,
|
||||
help="Node polling interval in seconds (default: 10)",
|
||||
)
|
||||
from turnstone.core.log import add_log_args
|
||||
|
||||
add_log_args(parser)
|
||||
@@ -6228,7 +6430,6 @@ def main() -> None:
|
||||
|
||||
collector = ClusterCollector(
|
||||
storage=auth_storage,
|
||||
poll_interval=args.poll_interval,
|
||||
auth_token=collector_token if collector_token_mgr is None else "",
|
||||
token_manager=collector_token_mgr,
|
||||
router=router,
|
||||
|
||||
@@ -59,6 +59,7 @@ function showAdmin() {
|
||||
watches: "admin.watches",
|
||||
roles: "admin.roles",
|
||||
policies: "admin.policies",
|
||||
"prompt-policies": "admin.prompt_policies",
|
||||
skills: "admin.skills",
|
||||
usage: "admin.usage",
|
||||
audit: "admin.audit",
|
||||
@@ -198,6 +199,7 @@ function switchAdminTab(tab) {
|
||||
"settings",
|
||||
"tls",
|
||||
"mcp",
|
||||
"prompt-policies",
|
||||
];
|
||||
for (var p = 0; p < panels.length; p++) {
|
||||
var el = document.getElementById("admin-" + panels[p]);
|
||||
@@ -222,6 +224,7 @@ function switchAdminTab(tab) {
|
||||
if (tab === "settings") loadSettings();
|
||||
if (tab === "tls") loadTlsCerts();
|
||||
if (tab === "mcp") loadAdminMcp();
|
||||
if (tab === "prompt-policies") loadPromptPolicies();
|
||||
|
||||
// Update breadcrumb with active tab label
|
||||
var activeNav = document.querySelector('.admin-nav[data-tab="' + tab + '"]');
|
||||
@@ -1848,6 +1851,10 @@ function _installTrap(overlayId, boxId, trapRef) {
|
||||
else if (overlayId === "mcp-install-overlay") hideInstallMcpModal();
|
||||
else if (overlayId === "github-import-overlay") hideGitHubImportModal();
|
||||
else if (overlayId === "model-create-overlay") hideCreateModelModal();
|
||||
else if (overlayId === "create-ppolicy-overlay")
|
||||
hideCreatePromptPolicyModal();
|
||||
else if (overlayId === "edit-ppolicy-overlay")
|
||||
hideEditPromptPolicyModal();
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1937,6 +1944,8 @@ document.addEventListener("keydown", function (e) {
|
||||
["mcp-create-overlay", hideCreateMcpModal],
|
||||
["github-import-overlay", hideGitHubImportModal],
|
||||
["model-create-overlay", hideCreateModelModal],
|
||||
["create-ppolicy-overlay", hideCreatePromptPolicyModal],
|
||||
["edit-ppolicy-overlay", hideEditPromptPolicyModal],
|
||||
];
|
||||
for (var gi = 0; gi < govOverlays.length; gi++) {
|
||||
var govEl = document.getElementById(govOverlays[gi][0]);
|
||||
|
||||
@@ -2450,3 +2450,282 @@ function submitGitHubImport() {
|
||||
submitBtn.textContent = "Install";
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prompt Policies (system message composition)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
var _promptPolicies = [];
|
||||
var _cppTrapHandler = null;
|
||||
var _cppTriggerEl = null;
|
||||
var _eppTrapHandler = null;
|
||||
var _eppTriggerEl = null;
|
||||
|
||||
function loadPromptPolicies() {
|
||||
authFetch("/v1/api/admin/prompt-policies")
|
||||
.then(function (r) {
|
||||
if (!r.ok) throw new Error("Failed");
|
||||
return r.json();
|
||||
})
|
||||
.then(function (data) {
|
||||
_promptPolicies = data.policies || [];
|
||||
_renderPromptPolicies(_promptPolicies);
|
||||
})
|
||||
.catch(function () {
|
||||
var el = document.getElementById("admin-prompt-policies-table");
|
||||
el.textContent = "";
|
||||
var empty = document.createElement("div");
|
||||
empty.className = "dashboard-empty";
|
||||
empty.textContent = "Failed to load prompts";
|
||||
el.appendChild(empty);
|
||||
});
|
||||
}
|
||||
|
||||
function _renderPromptPolicies(items) {
|
||||
var el = document.getElementById("admin-prompt-policies-table");
|
||||
el.textContent = "";
|
||||
if (!items.length) {
|
||||
var empty = document.createElement("div");
|
||||
empty.className = "dashboard-empty";
|
||||
empty.textContent = "No prompts defined";
|
||||
el.appendChild(empty);
|
||||
return;
|
||||
}
|
||||
for (var i = 0; i < items.length; i++) {
|
||||
var p = items[i];
|
||||
var row = document.createElement("div");
|
||||
row.className = "admin-row";
|
||||
row.setAttribute("role", "listitem");
|
||||
|
||||
var colName = document.createElement("span");
|
||||
colName.className = "admin-col admin-col-pname";
|
||||
colName.textContent = p.name;
|
||||
row.appendChild(colName);
|
||||
|
||||
var colGate = document.createElement("span");
|
||||
colGate.className = "admin-col admin-col-ppattern";
|
||||
if (p.tool_gate) {
|
||||
var code = document.createElement("code");
|
||||
code.textContent = p.tool_gate;
|
||||
colGate.appendChild(code);
|
||||
} else {
|
||||
var em = document.createElement("em");
|
||||
em.textContent = "unconditional";
|
||||
colGate.appendChild(em);
|
||||
}
|
||||
row.appendChild(colGate);
|
||||
|
||||
var colPri = document.createElement("span");
|
||||
colPri.className = "admin-col admin-col-ppriority";
|
||||
colPri.textContent = String(p.priority);
|
||||
row.appendChild(colPri);
|
||||
|
||||
var colStatus = document.createElement("span");
|
||||
colStatus.className = "admin-col admin-col-pstatus";
|
||||
var dot = document.createElement("span");
|
||||
dot.className = p.enabled ? "watch-active" : "watch-completed";
|
||||
dot.title = p.enabled ? "Enabled" : "Disabled";
|
||||
dot.textContent = p.enabled ? "\u25CF active" : "\u25CB disabled";
|
||||
colStatus.appendChild(dot);
|
||||
row.appendChild(colStatus);
|
||||
|
||||
var colActions = document.createElement("span");
|
||||
colActions.className = "admin-col admin-col-actions";
|
||||
var editBtn = document.createElement("button");
|
||||
editBtn.className = "admin-btn-action";
|
||||
editBtn.textContent = "edit";
|
||||
editBtn.setAttribute("data-edit-ppolicy", p.policy_id);
|
||||
editBtn.setAttribute("aria-label", "Edit prompt " + p.name);
|
||||
colActions.appendChild(editBtn);
|
||||
var delBtn = document.createElement("button");
|
||||
delBtn.className = "admin-btn-danger";
|
||||
delBtn.textContent = "delete";
|
||||
delBtn.setAttribute("data-delete-ppolicy", p.policy_id);
|
||||
delBtn.setAttribute("data-ppolicy-name", p.name);
|
||||
delBtn.setAttribute("aria-label", "Delete prompt " + p.name);
|
||||
colActions.appendChild(delBtn);
|
||||
row.appendChild(colActions);
|
||||
|
||||
el.appendChild(row);
|
||||
}
|
||||
el.querySelectorAll("[data-edit-ppolicy]").forEach(function (btn) {
|
||||
btn.addEventListener("click", function () {
|
||||
showEditPromptPolicyModal(this.getAttribute("data-edit-ppolicy"));
|
||||
});
|
||||
});
|
||||
el.querySelectorAll("[data-delete-ppolicy]").forEach(function (btn) {
|
||||
btn.addEventListener("click", function () {
|
||||
var pid = this.getAttribute("data-delete-ppolicy");
|
||||
var pname = this.getAttribute("data-ppolicy-name");
|
||||
showConfirmModal(
|
||||
"Delete Prompt",
|
||||
'Delete prompt "' + pname + '"?',
|
||||
"Delete",
|
||||
function () {
|
||||
authFetch("/v1/api/admin/prompt-policies/" + pid, {
|
||||
method: "DELETE",
|
||||
})
|
||||
.then(function (r) {
|
||||
if (!r.ok) throw new Error();
|
||||
return r.json();
|
||||
})
|
||||
.then(function () {
|
||||
showToast("Prompt deleted");
|
||||
loadPromptPolicies();
|
||||
})
|
||||
.catch(function () {
|
||||
showToast("Failed to delete prompt");
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function showCreatePromptPolicyModal() {
|
||||
_cppTriggerEl = document.activeElement;
|
||||
var ov = document.getElementById("create-ppolicy-overlay");
|
||||
ov.style.display = "flex";
|
||||
document.getElementById("cpp-name").value = "";
|
||||
document.getElementById("cpp-gate").value = "";
|
||||
document.getElementById("cpp-content").value = "";
|
||||
document.getElementById("cpp-priority").value = "0";
|
||||
document.getElementById("cpp-error").style.display = "none";
|
||||
document.getElementById("cpp-name").focus();
|
||||
_cppTrapHandler = _installTrap(
|
||||
"create-ppolicy-overlay",
|
||||
"create-ppolicy-box",
|
||||
);
|
||||
}
|
||||
|
||||
function hideCreatePromptPolicyModal() {
|
||||
document.getElementById("create-ppolicy-overlay").style.display = "none";
|
||||
_cppTrapHandler = _removeTrap(_cppTrapHandler);
|
||||
if (_cppTriggerEl && _cppTriggerEl.focus) {
|
||||
_cppTriggerEl.focus();
|
||||
}
|
||||
_cppTriggerEl = null;
|
||||
}
|
||||
|
||||
function submitCreatePromptPolicy() {
|
||||
var errEl = document.getElementById("cpp-error");
|
||||
var name = document.getElementById("cpp-name").value.trim();
|
||||
var content = document.getElementById("cpp-content").value.trim();
|
||||
if (!name || !content) {
|
||||
errEl.textContent = "Name and content are required";
|
||||
errEl.style.display = "";
|
||||
return;
|
||||
}
|
||||
errEl.style.display = "none";
|
||||
var submitBtn = document.getElementById("cpp-submit");
|
||||
submitBtn.disabled = true;
|
||||
authFetch("/v1/api/admin/prompt-policies", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
name: name,
|
||||
content: content,
|
||||
tool_gate: document.getElementById("cpp-gate").value.trim(),
|
||||
priority:
|
||||
parseInt(document.getElementById("cpp-priority").value, 10) || 0,
|
||||
enabled: true,
|
||||
}),
|
||||
})
|
||||
.then(function (r) {
|
||||
if (!r.ok)
|
||||
return r.json().then(function (d) {
|
||||
throw new Error(d.error || "Failed");
|
||||
});
|
||||
return r.json();
|
||||
})
|
||||
.then(function () {
|
||||
hideCreatePromptPolicyModal();
|
||||
showToast("Prompt created");
|
||||
loadPromptPolicies();
|
||||
})
|
||||
.catch(function (e) {
|
||||
errEl.textContent = e.message;
|
||||
errEl.style.display = "";
|
||||
})
|
||||
.finally(function () {
|
||||
submitBtn.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
function showEditPromptPolicyModal(policyId) {
|
||||
_eppTriggerEl = document.activeElement;
|
||||
var p = null;
|
||||
for (var i = 0; i < _promptPolicies.length; i++) {
|
||||
if (_promptPolicies[i].policy_id === policyId) {
|
||||
p = _promptPolicies[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!p) return;
|
||||
document.getElementById("epp-id").value = p.policy_id;
|
||||
document.getElementById("epp-name").value = p.name;
|
||||
document.getElementById("epp-gate").value = p.tool_gate || "";
|
||||
document.getElementById("epp-content").value = p.content || "";
|
||||
document.getElementById("epp-priority").value = p.priority || 0;
|
||||
document.getElementById("epp-enabled").checked = p.enabled;
|
||||
document.getElementById("epp-error").style.display = "none";
|
||||
var ov = document.getElementById("edit-ppolicy-overlay");
|
||||
ov.style.display = "flex";
|
||||
document.getElementById("epp-name").focus();
|
||||
_eppTrapHandler = _installTrap("edit-ppolicy-overlay", "edit-ppolicy-box");
|
||||
}
|
||||
|
||||
function hideEditPromptPolicyModal() {
|
||||
document.getElementById("edit-ppolicy-overlay").style.display = "none";
|
||||
_eppTrapHandler = _removeTrap(_eppTrapHandler);
|
||||
if (_eppTriggerEl && _eppTriggerEl.focus) {
|
||||
_eppTriggerEl.focus();
|
||||
}
|
||||
_eppTriggerEl = null;
|
||||
}
|
||||
|
||||
function submitEditPromptPolicy() {
|
||||
var errEl = document.getElementById("epp-error");
|
||||
var policyId = document.getElementById("epp-id").value;
|
||||
var name = document.getElementById("epp-name").value.trim();
|
||||
var content = document.getElementById("epp-content").value.trim();
|
||||
if (!name || !content) {
|
||||
errEl.textContent = "Name and content are required";
|
||||
errEl.style.display = "";
|
||||
return;
|
||||
}
|
||||
errEl.style.display = "none";
|
||||
var submitBtn = document.getElementById("epp-submit");
|
||||
submitBtn.disabled = true;
|
||||
authFetch("/v1/api/admin/prompt-policies/" + policyId, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
name: name,
|
||||
content: content,
|
||||
tool_gate: document.getElementById("epp-gate").value.trim(),
|
||||
priority:
|
||||
parseInt(document.getElementById("epp-priority").value, 10) || 0,
|
||||
enabled: document.getElementById("epp-enabled").checked,
|
||||
}),
|
||||
})
|
||||
.then(function (r) {
|
||||
if (!r.ok)
|
||||
return r.json().then(function (d) {
|
||||
throw new Error(d.error || "Failed");
|
||||
});
|
||||
return r.json();
|
||||
})
|
||||
.then(function () {
|
||||
hideEditPromptPolicyModal();
|
||||
showToast("Prompt updated");
|
||||
loadPromptPolicies();
|
||||
})
|
||||
.catch(function (e) {
|
||||
errEl.textContent = e.message;
|
||||
errEl.style.display = "";
|
||||
})
|
||||
.finally(function () {
|
||||
submitBtn.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@
|
||||
<div class="admin-sidebar-group-label" aria-hidden="true">Governance</div>
|
||||
<button id="tab-roles" class="admin-nav" data-tab="roles" role="tab" aria-selected="false" aria-controls="admin-roles" tabindex="-1" onclick="switchAdminTab('roles')">Roles</button>
|
||||
<button id="tab-policies" class="admin-nav" data-tab="policies" role="tab" aria-selected="false" aria-controls="admin-policies" tabindex="-1" onclick="switchAdminTab('policies')">Policies</button>
|
||||
<button id="tab-prompt-policies" class="admin-nav" data-tab="prompt-policies" role="tab" aria-selected="false" aria-controls="admin-prompt-policies" tabindex="-1" onclick="switchAdminTab('prompt-policies')">Prompts</button>
|
||||
</div>
|
||||
<div class="admin-sidebar-group" data-group="extensions" role="group" aria-label="Extensions">
|
||||
<div class="admin-sidebar-group-label" aria-hidden="true">Extensions</div>
|
||||
@@ -259,6 +260,24 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Prompt Policies Tab -->
|
||||
<div id="admin-prompt-policies" class="admin-panel" role="tabpanel" aria-labelledby="tab-prompt-policies" style="display:none">
|
||||
<div class="admin-toolbar">
|
||||
<span class="section-header" style="margin:0">PROMPTS</span>
|
||||
<button class="admin-action-btn" onclick="showCreatePromptPolicyModal()">+ Create prompt</button>
|
||||
</div>
|
||||
<div class="admin-colheaders" aria-hidden="true">
|
||||
<span class="admin-col admin-col-pname">NAME</span>
|
||||
<span class="admin-col admin-col-ppattern">TOOL GATE</span>
|
||||
<span class="admin-col admin-col-ppriority">PRI</span>
|
||||
<span class="admin-col admin-col-pstatus">STATUS</span>
|
||||
<span class="admin-col admin-col-actions">ACTIONS</span>
|
||||
</div>
|
||||
<div id="admin-prompt-policies-table" role="list" aria-label="Prompt policies" aria-live="polite">
|
||||
<div class="dashboard-empty">Loading prompt policies...</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Skills Tab -->
|
||||
<div id="admin-skills" class="admin-panel" role="tabpanel" aria-labelledby="tab-skills" style="display:none">
|
||||
<div class="admin-toolbar">
|
||||
@@ -888,6 +907,48 @@ window.TURNSTONE_KB_SHORTCUTS = [
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Create Prompt Policy Modal -->
|
||||
<div id="create-ppolicy-overlay" style="display:none" role="dialog" aria-modal="true" aria-labelledby="create-ppolicy-title">
|
||||
<div id="create-ppolicy-box" class="admin-modal admin-modal-wide">
|
||||
<h2 id="create-ppolicy-title">Create Prompt</h2>
|
||||
<div id="cpp-error" role="alert" aria-live="assertive"></div>
|
||||
<label for="cpp-name">Name <span class="label-hint">slug-style identifier</span></label>
|
||||
<input id="cpp-name" type="text" placeholder="e.g. web_search, data_handling" autocomplete="off">
|
||||
<label for="cpp-gate">Tool Gate <span class="label-hint">tool name or blank for unconditional</span></label>
|
||||
<input id="cpp-gate" type="text" placeholder="e.g. web_search" autocomplete="off">
|
||||
<label for="cpp-content">Content <span class="label-hint">markdown body for system message</span></label>
|
||||
<textarea id="cpp-content" rows="10" placeholder="## Policy Name Behavioral guidance..." spellcheck="false"></textarea>
|
||||
<label for="cpp-priority">Priority <span class="label-hint">higher = assembled later (closer to conversation)</span></label>
|
||||
<input id="cpp-priority" type="number" value="0" min="0" max="9999">
|
||||
<div class="modal-buttons">
|
||||
<button class="modal-cancel" onclick="hideCreatePromptPolicyModal()">Cancel</button>
|
||||
<button id="cpp-submit" class="modal-submit" onclick="submitCreatePromptPolicy()">Create</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Edit Prompt Policy Modal -->
|
||||
<div id="edit-ppolicy-overlay" style="display:none" role="dialog" aria-modal="true" aria-labelledby="edit-ppolicy-title">
|
||||
<div id="edit-ppolicy-box" class="admin-modal admin-modal-wide">
|
||||
<h2 id="edit-ppolicy-title">Edit Prompt</h2>
|
||||
<div id="epp-error" role="alert" aria-live="assertive"></div>
|
||||
<input id="epp-id" type="hidden">
|
||||
<label for="epp-name">Name</label>
|
||||
<input id="epp-name" type="text" autocomplete="off">
|
||||
<label for="epp-gate">Tool Gate</label>
|
||||
<input id="epp-gate" type="text" autocomplete="off">
|
||||
<label for="epp-content">Content</label>
|
||||
<textarea id="epp-content" rows="10" spellcheck="false"></textarea>
|
||||
<label for="epp-priority">Priority</label>
|
||||
<input id="epp-priority" type="number" value="0" min="0" max="9999">
|
||||
<label class="admin-checkbox"><input id="epp-enabled" type="checkbox" checked> Enabled</label>
|
||||
<div class="modal-buttons">
|
||||
<button class="modal-cancel" onclick="hideEditPromptPolicyModal()">Cancel</button>
|
||||
<button id="epp-submit" class="modal-submit" onclick="submitEditPromptPolicy()">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Create Skill Modal -->
|
||||
<div id="create-template-overlay" style="display:none" role="dialog" aria-modal="true" aria-labelledby="create-template-title">
|
||||
<div id="create-template-box" class="admin-modal admin-modal-wide admin-modal-skill">
|
||||
|
||||
@@ -1395,6 +1395,7 @@ h3.skill-spec-heading { font-size: inherit; margin-block: 0; }
|
||||
#create-schedule-overlay, #edit-schedule-overlay, #schedule-runs-overlay,
|
||||
#create-role-overlay, #edit-role-overlay, #user-roles-overlay,
|
||||
#create-policy-overlay, #edit-policy-overlay,
|
||||
#create-ppolicy-overlay, #edit-ppolicy-overlay,
|
||||
#create-template-overlay, #edit-template-overlay,
|
||||
#memory-detail-overlay,
|
||||
#mcp-create-overlay, #mcp-import-overlay, #mcp-detail-overlay, #mcp-install-overlay,
|
||||
@@ -1493,6 +1494,12 @@ h3.skill-spec-heading { font-size: inherit; margin-block: 0; }
|
||||
grid-template-columns: 1.2fr 1fr 70px 50px 80px 140px;
|
||||
}
|
||||
|
||||
/* Prompt Policies grid: NAME | TOOL GATE | PRI | STATUS | ACTIONS */
|
||||
#admin-prompt-policies .admin-colheaders,
|
||||
#admin-prompt-policies .admin-row {
|
||||
grid-template-columns: 1.2fr 1fr 50px 80px 140px;
|
||||
}
|
||||
|
||||
/* Policy action badges */
|
||||
.policy-badge {
|
||||
display: inline-block;
|
||||
@@ -1807,6 +1814,9 @@ h3.skill-spec-heading { font-size: inherit; margin-block: 0; }
|
||||
grid-template-columns: 1fr 70px 50px 100px;
|
||||
}
|
||||
.admin-col-pstatus, .admin-col-ppriority { display: none; }
|
||||
#admin-prompt-policies .admin-colheaders, #admin-prompt-policies .admin-row {
|
||||
grid-template-columns: 1fr 70px 100px;
|
||||
}
|
||||
#admin-skills .admin-colheaders, #admin-skills .admin-row {
|
||||
grid-template-columns: 1fr 100px;
|
||||
}
|
||||
|
||||
@@ -117,7 +117,6 @@ _CONFIG_MAP: dict[str, dict[str, str]] = {
|
||||
"host": "host",
|
||||
"port": "port",
|
||||
"url": "console_url",
|
||||
"poll_interval": "poll_interval",
|
||||
"log_level": "log_level",
|
||||
},
|
||||
"auth": {
|
||||
|
||||
@@ -43,6 +43,7 @@ class BackendHealthMonitor:
|
||||
provider: str = "openai",
|
||||
initial_model: str = "",
|
||||
on_model_changed: Callable[[str, int | None], None] | None = None,
|
||||
on_state_changed: Callable[[str], None] | None = None,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self._probe_interval = probe_interval
|
||||
@@ -54,6 +55,7 @@ class BackendHealthMonitor:
|
||||
self._provider = provider
|
||||
self._last_detected_model = initial_model
|
||||
self._on_model_changed = on_model_changed
|
||||
self._on_state_changed = on_state_changed
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self._state = CircuitState.CLOSED
|
||||
@@ -82,8 +84,17 @@ class BackendHealthMonitor:
|
||||
# Passive tracking (called by request path)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _fire_state_callback(self, state_val: str | None) -> None:
|
||||
"""Fire on_state_changed callback outside the lock."""
|
||||
if state_val is not None and self._on_state_changed is not None:
|
||||
try:
|
||||
self._on_state_changed(state_val)
|
||||
except Exception:
|
||||
log.debug("on_state_changed callback error", exc_info=True)
|
||||
|
||||
def record_success(self) -> None:
|
||||
"""Called on successful LLM call. Resets failure count, closes circuit."""
|
||||
state_to_dispatch: str | None = None
|
||||
with self._lock:
|
||||
self._consecutive_failures = 0
|
||||
if self._state != CircuitState.CLOSED:
|
||||
@@ -93,9 +104,12 @@ class BackendHealthMonitor:
|
||||
self._last_state_change = time.monotonic()
|
||||
log.info("Circuit breaker CLOSED (was %s): backend recovered", prev.value)
|
||||
self._update_metrics()
|
||||
state_to_dispatch = self._state.value
|
||||
self._fire_state_callback(state_to_dispatch)
|
||||
|
||||
def record_failure(self) -> None:
|
||||
"""Called on LLM call failure. May open circuit."""
|
||||
state_to_dispatch: str | None = None
|
||||
with self._lock:
|
||||
self._consecutive_failures += 1
|
||||
if self._state == CircuitState.HALF_OPEN:
|
||||
@@ -105,6 +119,7 @@ class BackendHealthMonitor:
|
||||
self._last_state_change = time.monotonic()
|
||||
log.warning("Circuit breaker OPEN: probe failed in HALF_OPEN")
|
||||
self._update_metrics()
|
||||
state_to_dispatch = self._state.value
|
||||
elif (
|
||||
self._state == CircuitState.CLOSED
|
||||
and self._consecutive_failures >= self._failure_threshold
|
||||
@@ -116,6 +131,8 @@ class BackendHealthMonitor:
|
||||
self._consecutive_failures,
|
||||
)
|
||||
self._update_metrics()
|
||||
state_to_dispatch = self._state.value
|
||||
self._fire_state_callback(state_to_dispatch)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Query helpers
|
||||
@@ -246,7 +263,12 @@ class BackendHealthMonitor:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _update_metrics(self) -> None:
|
||||
"""Push state to metrics collector. Called with *self._lock* held."""
|
||||
"""Push circuit-breaker state to metrics collector.
|
||||
|
||||
Called with *self._lock* held. State-change callbacks are dispatched
|
||||
by the callers (``record_success`` / ``record_failure``) after the
|
||||
lock is released, not by this method.
|
||||
"""
|
||||
from turnstone.core.metrics import metrics
|
||||
|
||||
metrics.set_backend_status(self._state == CircuitState.CLOSED)
|
||||
|
||||
+31
-59
@@ -26,6 +26,7 @@ import textwrap
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from html import escape as _html_escape
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
@@ -87,6 +88,7 @@ from turnstone.core.tools import (
|
||||
merge_mcp_tools,
|
||||
)
|
||||
from turnstone.core.web import check_ssrf, strip_html
|
||||
from turnstone.prompts import ClientType, SessionContext, compose_system_message
|
||||
from turnstone.ui.colors import DIM, GRAY, GREEN, RED, RESET, YELLOW, bold, cyan, dim
|
||||
|
||||
log = get_logger(__name__)
|
||||
@@ -290,6 +292,8 @@ class ChatSession:
|
||||
memory_config: MemoryConfig | None = None,
|
||||
config_store: ConfigStore | None = None,
|
||||
web_search_backend: str = "",
|
||||
client_type: ClientType = ClientType.CLI,
|
||||
username: str = "",
|
||||
):
|
||||
self.client = client
|
||||
self.model = model
|
||||
@@ -325,6 +329,8 @@ class ChatSession:
|
||||
self.auto_approve = False
|
||||
self._node_id = node_id
|
||||
self._user_id = user_id
|
||||
self._username = username
|
||||
self._client_type = client_type
|
||||
self._config_store = config_store
|
||||
self._memory_config = memory_config or MemoryConfig()
|
||||
self._ws_id = ws_id or uuid.uuid4().hex
|
||||
@@ -988,64 +994,30 @@ class ChatSession:
|
||||
"Never condescend to the form.",
|
||||
]
|
||||
else:
|
||||
dev_parts = [
|
||||
"You are a resident engineer on a small, focused infrastructure team. "
|
||||
"Your workspace is an instrumented workbench — a terminal with tools for "
|
||||
"reading, writing, searching, and executing code. You've been here a while. "
|
||||
"You know the codebase. You know the tools. You know their limits.\n\n"
|
||||
"Your team trusts you with real work: investigating bugs, implementing features, "
|
||||
"reviewing security, writing code that ships. You have access to the project's "
|
||||
"files, git history, and a running database. You don't have access to everything "
|
||||
"— some tools require approval, some paths are restricted, and that's by design. "
|
||||
"You work within those boundaries.\n\n"
|
||||
"You think before you act. You read before you edit. You verify before you commit. "
|
||||
"When something breaks, you diagnose before you retry. When you're uncertain, you "
|
||||
"say so. When a request is ambiguous, you make a reasonable call and note what you "
|
||||
"assumed — you don't stall asking for permission on every judgment call.\n\n"
|
||||
"When you disagree with a direction, you push back with reasoning — then defer to "
|
||||
"the team's call.\n\n"
|
||||
"You are not performing a demo. There is no audience. The code you write will run. "
|
||||
"The files you edit are real. The commits you make go to a shared repository. "
|
||||
"Act accordingly.\n\n"
|
||||
"TOOL PATTERNS:\n\n"
|
||||
"Modify existing file → read_file then edit_file:\n"
|
||||
" read_file(path='config.py') → "
|
||||
"edit_file(path='config.py')\n\n"
|
||||
"Modify multiple files → read_file then edit_file each:\n"
|
||||
" read_file(path='a.py') → edit_file(path='a.py') → "
|
||||
"read_file(path='b.py') → edit_file(path='b.py')\n\n"
|
||||
"Create new file → write_file (generate reasonable "
|
||||
"content even if the request is vague):\n"
|
||||
" write_file(path='hello.py', content='...')\n"
|
||||
" write_file(path='README.md', "
|
||||
"content='# Project\\nDescription.')\n\n"
|
||||
"Create a file then run it → write_file then bash:\n"
|
||||
" write_file(path='fib.py', content='...') → "
|
||||
"bash(command='python fib.py')\n\n"
|
||||
"Find something across files → search:\n"
|
||||
" search(query='test_')\n\n"
|
||||
"Find and modify → search then read_file then edit_file:\n"
|
||||
" search(query='MAX_RETRIES') → "
|
||||
"read_file(path='found.py') → "
|
||||
"edit_file(path='found.py')\n\n"
|
||||
"Plan, design, or architect something → "
|
||||
"explore codebase then plan_agent:\n"
|
||||
" bash(command='ls') → read_file(path='app.py') → "
|
||||
"plan_agent(goal='add caching to the application')\n"
|
||||
" plan_agent(goal='refactor database layer "
|
||||
"from monolith to service')\n"
|
||||
" plan_agent(goal='restructure auth module')\n\n"
|
||||
"Run a command, git, or tests → bash:\n"
|
||||
" bash(command='git log -5')\n"
|
||||
" bash(command='pytest')\n\n"
|
||||
"Retrieve a URL → web_fetch:\n"
|
||||
" web_fetch(url='https://example.com')\n\n"
|
||||
"Search the web for information → web_search:\n"
|
||||
" web_search(query='current population of Tokyo')\n\n"
|
||||
"Look up command flags or documentation → man:\n"
|
||||
" man(page='tar')\n"
|
||||
" man(page='grep')",
|
||||
]
|
||||
# Compose system message from modular components
|
||||
tool_names = frozenset(t["function"]["name"] for t in self._tools if "function" in t)
|
||||
# Load DB prompt policies if storage is available
|
||||
db_policies: list[dict[str, Any]] = []
|
||||
try:
|
||||
storage = get_storage()
|
||||
if storage:
|
||||
db_policies = storage.list_prompt_policies()
|
||||
except Exception:
|
||||
pass
|
||||
now = datetime.now().astimezone()
|
||||
ctx = SessionContext(
|
||||
current_datetime=now.strftime("%Y-%m-%dT%H:%M"),
|
||||
timezone=now.tzname() or "UTC",
|
||||
username=self._username or self._user_id or "unknown",
|
||||
)
|
||||
composed = compose_system_message(
|
||||
client_type=self._client_type,
|
||||
context=ctx,
|
||||
available_tools=tool_names,
|
||||
policies=["web_search"],
|
||||
db_policies=db_policies,
|
||||
)
|
||||
dev_parts = [composed]
|
||||
# Tool search hint (client-side mode only — native mode needs no hint)
|
||||
if self._tool_search:
|
||||
caps = self._get_capabilities()
|
||||
@@ -5725,7 +5697,7 @@ class ChatSession:
|
||||
}
|
||||
|
||||
def _exec_watch(self, item: dict[str, Any]) -> tuple[str, str]:
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
call_id = item["call_id"]
|
||||
action = item["action"]
|
||||
|
||||
@@ -45,6 +45,9 @@ from turnstone.core.storage._schema import (
|
||||
workstream_overrides,
|
||||
workstreams,
|
||||
)
|
||||
from turnstone.core.storage._schema import (
|
||||
prompt_policies as prompt_policies_t,
|
||||
)
|
||||
from turnstone.core.storage._utils import (
|
||||
MCP_SERVER_MUTABLE as _MCP_SERVER_MUTABLE,
|
||||
)
|
||||
@@ -57,6 +60,9 @@ from turnstone.core.storage._utils import (
|
||||
from turnstone.core.storage._utils import (
|
||||
POLICY_MUTABLE as _POLICY_MUTABLE,
|
||||
)
|
||||
from turnstone.core.storage._utils import (
|
||||
PROMPT_POLICY_MUTABLE as _PROMPT_POLICY_MUTABLE,
|
||||
)
|
||||
from turnstone.core.storage._utils import (
|
||||
ROLE_MUTABLE as _ROLE_MUTABLE,
|
||||
)
|
||||
@@ -3087,6 +3093,73 @@ class PostgreSQLBackend:
|
||||
conn.commit()
|
||||
return result.rowcount
|
||||
|
||||
# -- Prompt policies -------------------------------------------------------
|
||||
|
||||
def list_prompt_policies(self, org_id: str = "") -> list[dict[str, Any]]:
|
||||
|
||||
with self._engine.connect() as conn:
|
||||
q = sa.select(prompt_policies_t).order_by(prompt_policies_t.c.priority)
|
||||
if org_id:
|
||||
q = q.where(prompt_policies_t.c.org_id == org_id)
|
||||
rows = conn.execute(q).fetchall()
|
||||
return [_row_to_dict(r, "enabled") for r in rows]
|
||||
|
||||
def get_prompt_policy(self, policy_id: str) -> dict[str, Any] | None:
|
||||
|
||||
with self._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(prompt_policies_t).where(prompt_policies_t.c.policy_id == policy_id)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return _row_to_dict(row, "enabled")
|
||||
|
||||
def upsert_prompt_policy(self, policy: dict[str, Any]) -> None:
|
||||
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._engine.connect() as conn:
|
||||
existing = conn.execute(
|
||||
sa.select(prompt_policies_t).where(
|
||||
prompt_policies_t.c.policy_id == policy["policy_id"]
|
||||
)
|
||||
).fetchone()
|
||||
if existing:
|
||||
fields = {k: v for k, v in policy.items() if k in _PROMPT_POLICY_MUTABLE}
|
||||
fields["updated"] = now
|
||||
if "enabled" in fields:
|
||||
fields["enabled"] = 1 if fields["enabled"] else 0
|
||||
conn.execute(
|
||||
sa.update(prompt_policies_t)
|
||||
.where(prompt_policies_t.c.policy_id == policy["policy_id"])
|
||||
.values(**fields)
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
sa.insert(prompt_policies_t),
|
||||
{
|
||||
"policy_id": policy["policy_id"],
|
||||
"name": policy["name"],
|
||||
"content": policy["content"],
|
||||
"tool_gate": policy.get("tool_gate", ""),
|
||||
"priority": policy.get("priority", 0),
|
||||
"enabled": 1 if policy.get("enabled", True) else 0,
|
||||
"org_id": policy.get("org_id", ""),
|
||||
"created_by": policy.get("created_by", ""),
|
||||
"created": now,
|
||||
"updated": now,
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def delete_prompt_policy(self, policy_id: str) -> bool:
|
||||
|
||||
with self._engine.connect() as conn:
|
||||
result = conn.execute(
|
||||
sa.delete(prompt_policies_t).where(prompt_policies_t.c.policy_id == policy_id)
|
||||
)
|
||||
conn.commit()
|
||||
return result.rowcount > 0
|
||||
|
||||
# -- TLS / ACME ------------------------------------------------------------
|
||||
|
||||
def save_tls_account_key(self, key_id: str, key_pem: str) -> None:
|
||||
|
||||
@@ -1048,6 +1048,24 @@ class StorageBackend(Protocol):
|
||||
"""Delete a model definition. Returns True if existed."""
|
||||
...
|
||||
|
||||
# -- Prompt policies -------------------------------------------------------
|
||||
|
||||
def list_prompt_policies(self, org_id: str = "") -> list[dict[str, Any]]:
|
||||
"""Return all prompt policies ordered by priority."""
|
||||
...
|
||||
|
||||
def get_prompt_policy(self, policy_id: str) -> dict[str, Any] | None:
|
||||
"""Return prompt policy dict or None."""
|
||||
...
|
||||
|
||||
def upsert_prompt_policy(self, policy: dict[str, Any]) -> None:
|
||||
"""Create or update a prompt policy."""
|
||||
...
|
||||
|
||||
def delete_prompt_policy(self, policy_id: str) -> bool:
|
||||
"""Delete a prompt policy. Returns True if existed."""
|
||||
...
|
||||
|
||||
# -- TLS / ACME (lacme Store) ----------------------------------------------
|
||||
|
||||
def save_tls_account_key(self, key_id: str, key_pem: str) -> None:
|
||||
|
||||
@@ -575,6 +575,25 @@ model_definitions = sa.Table(
|
||||
|
||||
sa.Index("idx_model_definitions_enabled", model_definitions.c.enabled)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt policies — system message behavioral rules (admin-managed)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
prompt_policies = sa.Table(
|
||||
"prompt_policies",
|
||||
metadata,
|
||||
sa.Column("policy_id", sa.Text, primary_key=True),
|
||||
sa.Column("name", sa.Text, nullable=False, unique=True),
|
||||
sa.Column("content", sa.Text, nullable=False),
|
||||
sa.Column("tool_gate", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("priority", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("enabled", sa.Integer, nullable=False, server_default="1"),
|
||||
sa.Column("org_id", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("created_by", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("created", sa.Text, nullable=False),
|
||||
sa.Column("updated", sa.Text, nullable=False),
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OIDC identity tables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -45,6 +45,9 @@ from turnstone.core.storage._schema import (
|
||||
workstream_overrides,
|
||||
workstreams,
|
||||
)
|
||||
from turnstone.core.storage._schema import (
|
||||
prompt_policies as prompt_policies_t,
|
||||
)
|
||||
from turnstone.core.storage._utils import (
|
||||
MCP_SERVER_MUTABLE as _MCP_SERVER_MUTABLE,
|
||||
)
|
||||
@@ -57,6 +60,9 @@ from turnstone.core.storage._utils import (
|
||||
from turnstone.core.storage._utils import (
|
||||
POLICY_MUTABLE as _POLICY_MUTABLE,
|
||||
)
|
||||
from turnstone.core.storage._utils import (
|
||||
PROMPT_POLICY_MUTABLE as _PROMPT_POLICY_MUTABLE,
|
||||
)
|
||||
from turnstone.core.storage._utils import (
|
||||
ROLE_MUTABLE as _ROLE_MUTABLE,
|
||||
)
|
||||
@@ -3152,6 +3158,73 @@ class SQLiteBackend:
|
||||
conn.commit()
|
||||
return result.rowcount
|
||||
|
||||
# -- Prompt policies -------------------------------------------------------
|
||||
|
||||
def list_prompt_policies(self, org_id: str = "") -> list[dict[str, Any]]:
|
||||
|
||||
with self._engine.connect() as conn:
|
||||
q = sa.select(prompt_policies_t).order_by(prompt_policies_t.c.priority)
|
||||
if org_id:
|
||||
q = q.where(prompt_policies_t.c.org_id == org_id)
|
||||
rows = conn.execute(q).fetchall()
|
||||
return [_row_to_dict(r, "enabled") for r in rows]
|
||||
|
||||
def get_prompt_policy(self, policy_id: str) -> dict[str, Any] | None:
|
||||
|
||||
with self._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(prompt_policies_t).where(prompt_policies_t.c.policy_id == policy_id)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return _row_to_dict(row, "enabled")
|
||||
|
||||
def upsert_prompt_policy(self, policy: dict[str, Any]) -> None:
|
||||
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._engine.connect() as conn:
|
||||
existing = conn.execute(
|
||||
sa.select(prompt_policies_t).where(
|
||||
prompt_policies_t.c.policy_id == policy["policy_id"]
|
||||
)
|
||||
).fetchone()
|
||||
if existing:
|
||||
fields = {k: v for k, v in policy.items() if k in _PROMPT_POLICY_MUTABLE}
|
||||
fields["updated"] = now
|
||||
if "enabled" in fields:
|
||||
fields["enabled"] = 1 if fields["enabled"] else 0
|
||||
conn.execute(
|
||||
sa.update(prompt_policies_t)
|
||||
.where(prompt_policies_t.c.policy_id == policy["policy_id"])
|
||||
.values(**fields)
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
sa.insert(prompt_policies_t),
|
||||
{
|
||||
"policy_id": policy["policy_id"],
|
||||
"name": policy["name"],
|
||||
"content": policy["content"],
|
||||
"tool_gate": policy.get("tool_gate", ""),
|
||||
"priority": policy.get("priority", 0),
|
||||
"enabled": 1 if policy.get("enabled", True) else 0,
|
||||
"org_id": policy.get("org_id", ""),
|
||||
"created_by": policy.get("created_by", ""),
|
||||
"created": now,
|
||||
"updated": now,
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def delete_prompt_policy(self, policy_id: str) -> bool:
|
||||
|
||||
with self._engine.connect() as conn:
|
||||
result = conn.execute(
|
||||
sa.delete(prompt_policies_t).where(prompt_policies_t.c.policy_id == policy_id)
|
||||
)
|
||||
conn.commit()
|
||||
return result.rowcount > 0
|
||||
|
||||
# -- TLS / ACME ------------------------------------------------------------
|
||||
|
||||
def save_tls_account_key(self, key_id: str, key_pem: str) -> None:
|
||||
|
||||
@@ -108,6 +108,7 @@ MODEL_DEFINITION_MUTABLE = frozenset(
|
||||
"enabled",
|
||||
}
|
||||
)
|
||||
PROMPT_POLICY_MUTABLE = frozenset({"name", "content", "tool_gate", "priority", "enabled"})
|
||||
VERDICT_MUTABLE = frozenset(
|
||||
{
|
||||
"user_decision",
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Create prompt_policies table for system message composition.
|
||||
|
||||
Revision ID: 031
|
||||
Revises: 030
|
||||
Create Date: 2026-03-31
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "031"
|
||||
down_revision = "030"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"prompt_policies",
|
||||
sa.Column("policy_id", sa.Text, primary_key=True),
|
||||
sa.Column("name", sa.Text, nullable=False, unique=True),
|
||||
sa.Column("content", sa.Text, nullable=False),
|
||||
sa.Column("tool_gate", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("priority", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("enabled", sa.Integer, nullable=False, server_default="1"),
|
||||
sa.Column("org_id", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("created_by", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("created", sa.Text, nullable=False),
|
||||
sa.Column("updated", sa.Text, nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("prompt_policies")
|
||||
@@ -12,7 +12,7 @@ import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
|
||||
ws_id: str | None = ...,
|
||||
*,
|
||||
skill: str | None = ...,
|
||||
client_type: str = ...,
|
||||
) -> ChatSession: ...
|
||||
|
||||
|
||||
@@ -136,6 +137,7 @@ class WorkstreamManager:
|
||||
skill_id: str = "",
|
||||
skill_version: int = 0,
|
||||
ws_id: str = "",
|
||||
client_type: str = "",
|
||||
) -> Workstream:
|
||||
"""Create a new workstream. Returns the new ws.
|
||||
|
||||
@@ -177,7 +179,10 @@ class WorkstreamManager:
|
||||
ws = Workstream(id=ws_id, name=name) if ws_id else Workstream(name=name)
|
||||
if ui_factory:
|
||||
ws.ui = ui_factory(ws.id)
|
||||
ws.session = self._session_factory(ws.ui, model, ws.id, skill=skill)
|
||||
factory_kwargs: dict[str, Any] = {"skill": skill}
|
||||
if client_type:
|
||||
factory_kwargs["client_type"] = client_type
|
||||
ws.session = self._session_factory(ws.ui, model, ws.id, **factory_kwargs)
|
||||
|
||||
# Authoritative insert under lock with re-check (another thread may
|
||||
# have filled capacity while we were unlocked).
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""System message composition harness.
|
||||
|
||||
Assembles modular system messages from BASE (persona), ENV (client surface),
|
||||
CONTEXT (session variables), TOOLS (usage patterns), and POLICIES (behavioral
|
||||
rules). Replaces the monolithic persona+tools section of
|
||||
``ChatSession._init_system_messages()``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_PROMPTS_DIR = Path(__file__).resolve().parent # turnstone/prompts/
|
||||
# Files are read once at import time — they're static markdown.
|
||||
# This matches the pattern in tools.py where JSON schemas are loaded once.
|
||||
_FILE_CACHE: dict[Path, str] = {}
|
||||
|
||||
|
||||
def _load(relpath: str) -> str:
|
||||
"""Load and cache a prompt module file."""
|
||||
path = _PROMPTS_DIR / relpath
|
||||
if path not in _FILE_CACHE:
|
||||
_FILE_CACHE[path] = path.read_text()
|
||||
return _FILE_CACHE[path]
|
||||
|
||||
|
||||
class ClientType(enum.StrEnum):
|
||||
WEB = "web"
|
||||
CLI = "cli"
|
||||
CHAT = "chat"
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class SessionContext:
|
||||
current_datetime: str # ISO 8601, required
|
||||
timezone: str # system tz abbreviation, required
|
||||
username: str # users.username, required
|
||||
|
||||
|
||||
# File-based policy-to-tool gating (defaults).
|
||||
# DB policies carry their own tool_gate field.
|
||||
POLICY_TOOL_GATES: dict[str, str] = {
|
||||
"web_search": "web_search",
|
||||
}
|
||||
|
||||
_ENV_MAP: dict[ClientType, str] = {
|
||||
ClientType.WEB: "env/web.md",
|
||||
ClientType.CLI: "env/cli.md",
|
||||
ClientType.CHAT: "env/chat.md",
|
||||
}
|
||||
|
||||
|
||||
def _build_context(ctx: SessionContext) -> str:
|
||||
"""Build the CONTEXT module from session variables."""
|
||||
return (
|
||||
"## Session Context\n"
|
||||
"\n"
|
||||
f"- **Current date/time:** {ctx.current_datetime} ({ctx.timezone})\n"
|
||||
f"- **User:** {ctx.username}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_context(ctx: SessionContext) -> None:
|
||||
"""Validate required fields and format constraints."""
|
||||
if not ctx.current_datetime:
|
||||
raise ValueError("current_datetime is required")
|
||||
if not ctx.timezone:
|
||||
raise ValueError("timezone is required")
|
||||
if not ctx.username:
|
||||
raise ValueError("username is required")
|
||||
# Validate ISO 8601
|
||||
try:
|
||||
datetime.fromisoformat(ctx.current_datetime)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"current_datetime is not valid ISO 8601: {ctx.current_datetime}") from exc
|
||||
|
||||
|
||||
def compose_system_message(
|
||||
client_type: ClientType,
|
||||
context: SessionContext,
|
||||
available_tools: frozenset[str],
|
||||
policies: list[str] | None = None,
|
||||
db_policies: list[dict[str, Any]] | None = None,
|
||||
) -> str:
|
||||
"""Compose a system message from modular components.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client_type:
|
||||
Target rendering surface (web, cli, chat).
|
||||
context:
|
||||
Per-session variables (datetime, timezone, username).
|
||||
available_tools:
|
||||
Set of available tool names (used for policy gating).
|
||||
policies:
|
||||
Explicit file-based policy names to include (e.g. ``["web_search"]``).
|
||||
db_policies:
|
||||
Database-backed policies from ``storage.list_prompt_policies()``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The fully assembled system message, modules separated by double newlines.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
|
||||
# 1. BASE — always included
|
||||
parts.append(_load("base.md"))
|
||||
|
||||
# 2. ENV — exactly one, selected by client type
|
||||
if client_type not in _ENV_MAP:
|
||||
raise ValueError(f"Unknown client_type: {client_type!r}")
|
||||
parts.append(_load(_ENV_MAP[client_type]))
|
||||
|
||||
# 3. CONTEXT — built programmatically (no template engine)
|
||||
_validate_context(context)
|
||||
parts.append(_build_context(context))
|
||||
|
||||
# 4. TOOLS — if any tools are available
|
||||
if available_tools:
|
||||
parts.append(_load("tools.md"))
|
||||
|
||||
# 5. POLICIES — resolve from DB first, fall back to files
|
||||
# DB policies indexed by name for O(1) override lookup.
|
||||
db_by_name: dict[str, dict[str, Any]] = {}
|
||||
if db_policies:
|
||||
db_by_name = {p["name"]: p for p in db_policies if p.get("enabled", True)}
|
||||
|
||||
for policy_name in policies or []:
|
||||
db_row = db_by_name.pop(policy_name, None)
|
||||
if db_row:
|
||||
# DB override — use its content and tool_gate
|
||||
gate = db_row.get("tool_gate", "")
|
||||
if gate and gate not in available_tools:
|
||||
log.debug("Skipping DB policy %r: requires tool %r", policy_name, gate)
|
||||
continue
|
||||
parts.append(db_row["content"])
|
||||
else:
|
||||
# File-based fallback
|
||||
path = _PROMPTS_DIR / "policies" / f"{policy_name}.md"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Policy module not found: {path}")
|
||||
gate = POLICY_TOOL_GATES.get(policy_name, "")
|
||||
if gate and gate not in available_tools:
|
||||
log.debug("Skipping file policy %r: requires tool %r", policy_name, gate)
|
||||
continue
|
||||
parts.append(_load(f"policies/{policy_name}.md"))
|
||||
|
||||
# DB-only policies (not in the explicit list) — sorted by priority
|
||||
for db_row in sorted(db_by_name.values(), key=lambda p: p.get("priority", 0)):
|
||||
gate = db_row.get("tool_gate", "")
|
||||
if gate and gate not in available_tools:
|
||||
continue
|
||||
parts.append(db_row["content"])
|
||||
|
||||
return "\n\n".join(parts)
|
||||
@@ -0,0 +1,9 @@
|
||||
You are a resident engineer on a small, focused infrastructure team. You've been here a while. You know the codebase. You know the tools. You know their limits.
|
||||
|
||||
Your team trusts you with real work: investigating bugs, implementing features, reviewing security, writing code that ships. You have access to the project's files, git history, and a running database. You don't have access to everything — some tools require approval, some paths are restricted, and that's by design. You work within those boundaries.
|
||||
|
||||
You think before you act. You read before you edit. You verify before you commit. When something breaks, you diagnose before you retry. When you're uncertain, you say so. When a request is ambiguous, you make a reasonable call and note what you assumed — you don't stall asking for permission on every judgment call.
|
||||
|
||||
When you disagree with a direction, you push back with reasoning — then defer to the team's call.
|
||||
|
||||
You are not performing a demo. There is no audience. The code you write will run. The files you edit are real. The commits you make go to a shared repository. Act accordingly.
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
## Output Environment
|
||||
|
||||
Your responses are delivered in a third-party chat platform (Slack, Discord, or Microsoft Teams). These platforms have constrained and inconsistent markdown support. Optimize for maximum portability and readability across all of them.
|
||||
|
||||
**Available rendering:**
|
||||
|
||||
- **Bold** (`**text**`) — Supported everywhere.
|
||||
- **Italic** (`*text*`) — Supported everywhere (Slack also accepts `_text_`).
|
||||
- **Inline code** (`` `text` ``) — Supported everywhere.
|
||||
- **Code blocks** (triple backticks) — Supported everywhere, but language-specific syntax highlighting is inconsistent. Include language tags anyway for clients that support them.
|
||||
- **Bullet lists** — Supported everywhere. Use `-` syntax.
|
||||
- **Links** — `[text](url)` works in Slack and Discord. Teams may render inconsistently. Use bare URLs when maximum portability matters.
|
||||
|
||||
**Not reliably available:**
|
||||
|
||||
- **Tables** — Slack and Discord do not render markdown tables. They display as broken pipe characters. Do not use them. Use aligned code blocks or bullet lists for structured data instead.
|
||||
- **Headings** (`#`, `##`) — Slack does not support them (renders as literal `#`). Use **bold text** on its own line as a heading substitute.
|
||||
- **Mermaid / KaTeX** — Not available. Do not use them.
|
||||
- **Blockquotes** (`>`) — Supported in Slack and Discord, not reliably in Teams. Use sparingly.
|
||||
- **Nested lists** — Inconsistent. Avoid nesting deeper than one level.
|
||||
|
||||
**Formatting principles:**
|
||||
- Keep responses concise. Chat platforms favor short, scannable messages over long-form prose.
|
||||
- Use emoji sparingly for visual anchoring: ✅ ❌ ⚠️ 🔍 are useful; decorative emoji is noise.
|
||||
- For structured comparisons that would normally be a table, use a code block:
|
||||
```
|
||||
Model A: 95.2% accuracy, 1.2s latency
|
||||
Model B: 91.8% accuracy, 0.4s latency
|
||||
```
|
||||
- Break long responses into logical chunks. A response that requires scrolling in a chat window is too long — consider splitting across messages or summarizing with an offer to elaborate.
|
||||
- When referencing files, commands, or code, always use inline code formatting for scannability.
|
||||
- Math expressions should use plain programming notation: `(a * b) / c`, not LaTeX.
|
||||
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
## Output Environment
|
||||
|
||||
Your workspace is a terminal. Your responses are rendered as plain text in a monospace font with limited formatting support. Design your output for readability in this context.
|
||||
|
||||
**Available rendering:**
|
||||
|
||||
- **Code blocks** — Rendered in monospace with basic syntax highlighting. Language tags are still useful (```python, etc.) but rendering quality varies by terminal emulator.
|
||||
- **Basic markdown** — Bold (**text**) and inline code (`text`) may render depending on the client. Headings (#) render as plain text with emphasis. Tables render as-is in monospace (pipe-aligned tables work well).
|
||||
- **No diagram rendering** — Mermaid, KaTeX, and other embedded renderers are not available. Do not use them.
|
||||
|
||||
**Formatting principles:**
|
||||
- Use indentation, whitespace, and ASCII structure for clarity.
|
||||
- For flows and architectures, use simple text-based representations:
|
||||
```
|
||||
Input → Processing → Output
|
||||
```
|
||||
or indented tree structures, not Mermaid blocks.
|
||||
- For math, write expressions inline using programming notation: `(a * b) / c`, `sum(x_i for i in 1..n)`, `sqrt(n)`. Do not use LaTeX/KaTeX syntax.
|
||||
- Keep line lengths reasonable (~80-100 chars) for terminal readability.
|
||||
- Tables work well — keep them pipe-aligned and concise.
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
## Output Environment
|
||||
|
||||
Your responses are rendered in a rich web client with full markdown support. Use the available rendering capabilities to communicate clearly — prefer structured visuals over walls of text when they aid understanding.
|
||||
|
||||
**Available rendering:**
|
||||
|
||||
- **Code blocks** — Syntax-highlighted via highlight.js. Always specify the language tag (```python, ```sql, ```yaml, etc.) for proper highlighting.
|
||||
- **Diagrams** — Mermaid.js is supported via ```mermaid code blocks. Use flowcharts, sequence diagrams, state diagrams, ER diagrams, and Gantt charts when explaining flows, architectures, or processes. Prefer a diagram over a verbal description of a system or sequence.
|
||||
- **Math** — KaTeX is supported for both inline (`$...$`) and display (`$$...$$`) notation. Use proper mathematical typesetting when discussing formulas, equations, or formal notation rather than ASCII approximations.
|
||||
- **Standard markdown** — Tables, headings, bold, italic, lists, blockquotes, horizontal rules, footnotes, and definition lists all render correctly. Use tables for structured comparisons. Use headings to organize long responses.
|
||||
- **GFM callouts** — `> [!NOTE]`, `> [!TIP]`, `> [!IMPORTANT]`, `> [!WARNING]`, `> [!CAUTION]` render as styled alert boxes. Use them for important caveats or warnings.
|
||||
|
||||
**Formatting principles:**
|
||||
- Lead with the answer, then support with visuals — don't bury conclusions after a diagram.
|
||||
- Mermaid diagrams should be self-contained and labeled clearly; the reader may not have surrounding context if they screenshot it.
|
||||
- Use KaTeX for any expression that would be awkward in plain text (fractions, subscripts, summations, Greek letters, etc.).
|
||||
- Don't use rich formatting gratuitously — a one-line answer doesn't need a flowchart.
|
||||
@@ -0,0 +1,26 @@
|
||||
## Web Search Policy
|
||||
|
||||
You have tools for reading files, searching the codebase, and running commands.
|
||||
Use them first. Web search is for information that doesn't exist in the local
|
||||
workspace.
|
||||
|
||||
**Use local tools, not web search, for:**
|
||||
- Anything in the codebase — file contents, function signatures, config values,
|
||||
test results, git history, dependency versions (`read_file`, `search`, `bash`)
|
||||
- Language syntax, standard library behavior, well-established patterns —
|
||||
your training covers this
|
||||
- Anything the user can answer faster than a search round-trip — ask them
|
||||
|
||||
**Use web search for:**
|
||||
- Package versions, changelogs, or deprecation notices newer than your
|
||||
knowledge cutoff
|
||||
- CVEs, security advisories, or vulnerability details for specific versions
|
||||
- API behavior or SDK changes you're uncertain about — verify rather than guess
|
||||
- Anything the user explicitly asks you to search for
|
||||
- Current status of external services, outages, or recent announcements
|
||||
|
||||
**When searching:**
|
||||
- One query at a time. Evaluate results before searching again.
|
||||
- Keep queries specific: `httpx 0.28 changelog` not `httpx python http client latest version changes`
|
||||
- Link to sources when citing external information. Bare URLs are fine.
|
||||
- Don't narrate the search — just do it and present what you found.
|
||||
@@ -0,0 +1,39 @@
|
||||
TOOL PATTERNS:
|
||||
|
||||
Modify existing file → read_file then edit_file:
|
||||
read_file(path='config.py') → edit_file(path='config.py')
|
||||
|
||||
Modify multiple files → read_file then edit_file each:
|
||||
read_file(path='a.py') → edit_file(path='a.py') → read_file(path='b.py') → edit_file(path='b.py')
|
||||
|
||||
Create new file → write_file (generate reasonable content even if the request is vague):
|
||||
write_file(path='hello.py', content='...')
|
||||
write_file(path='README.md', content='# Project\nDescription.')
|
||||
|
||||
Create a file then run it → write_file then bash:
|
||||
write_file(path='fib.py', content='...') → bash(command='python fib.py')
|
||||
|
||||
Find something across files → search:
|
||||
search(query='test_')
|
||||
|
||||
Find and modify → search then read_file then edit_file:
|
||||
search(query='MAX_RETRIES') → read_file(path='found.py') → edit_file(path='found.py')
|
||||
|
||||
Plan, design, or architect something → explore codebase then plan_agent:
|
||||
bash(command='ls') → read_file(path='app.py') → plan_agent(goal='add caching to the application')
|
||||
plan_agent(goal='refactor database layer from monolith to service')
|
||||
plan_agent(goal='restructure auth module')
|
||||
|
||||
Run a command, git, or tests → bash:
|
||||
bash(command='git log -5')
|
||||
bash(command='pytest')
|
||||
|
||||
Retrieve a URL → web_fetch:
|
||||
web_fetch(url='https://example.com')
|
||||
|
||||
Search the web for information → web_search:
|
||||
web_search(query='current population of Tokyo')
|
||||
|
||||
Look up command flags or documentation → man:
|
||||
man(page='tar')
|
||||
man(page='grep')
|
||||
@@ -199,6 +199,7 @@ class AsyncTurnstoneConsole(_BaseClient):
|
||||
resume_ws: str = "",
|
||||
target_node: str = "",
|
||||
user_id: str = "",
|
||||
client_type: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Create a workstream via the console's routing proxy.
|
||||
|
||||
@@ -224,6 +225,8 @@ class AsyncTurnstoneConsole(_BaseClient):
|
||||
body["target_node"] = target_node
|
||||
if user_id:
|
||||
body["user_id"] = user_id
|
||||
if client_type:
|
||||
body["client_type"] = client_type
|
||||
return await self._request("POST", "/v1/api/route/workstreams/new", json_body=body)
|
||||
|
||||
async def route_send(self, message: str, ws_id: str) -> dict[str, Any]:
|
||||
@@ -1059,6 +1062,7 @@ class TurnstoneConsole:
|
||||
resume_ws: str = "",
|
||||
target_node: str = "",
|
||||
user_id: str = "",
|
||||
client_type: str = "",
|
||||
) -> dict[str, Any]:
|
||||
return self._runner.run(
|
||||
self._async.route_create_workstream(
|
||||
@@ -1071,6 +1075,7 @@ class TurnstoneConsole:
|
||||
resume_ws=resume_ws,
|
||||
target_node=target_node,
|
||||
user_id=user_id,
|
||||
client_type=client_type,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -300,6 +300,36 @@ class ClusterSnapshotEvent(ClusterEvent):
|
||||
timestamp: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeSnapshotEvent(ClusterEvent):
|
||||
"""Full node state delivered on SSE connect to ``/v1/api/events/global``."""
|
||||
|
||||
type: str = "node_snapshot"
|
||||
node_id: str = ""
|
||||
workstreams: list[dict[str, Any]] = field(default_factory=list)
|
||||
health: dict[str, Any] = field(default_factory=dict)
|
||||
aggregate: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthChangedEvent(ClusterEvent):
|
||||
"""Circuit breaker state transition on a server node."""
|
||||
|
||||
type: str = "health_changed"
|
||||
circuit_state: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AggregateEvent(ClusterEvent):
|
||||
"""Periodic aggregate metrics from a server node."""
|
||||
|
||||
type: str = "aggregate"
|
||||
total_tokens: int = 0
|
||||
total_tool_calls: int = 0
|
||||
active_count: int = 0
|
||||
total_count: int = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type registries (built after all classes are defined)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -354,5 +384,8 @@ _CLUSTER_REGISTRY: dict[str, type[ClusterEvent]] = {
|
||||
ClusterWsClosedEvent,
|
||||
ClusterWsRenameEvent,
|
||||
ClusterSnapshotEvent,
|
||||
NodeSnapshotEvent,
|
||||
HealthChangedEvent,
|
||||
AggregateEvent,
|
||||
]
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ from turnstone.sdk._base import _BaseClient
|
||||
from turnstone.sdk._sync import _SyncRunner
|
||||
from turnstone.sdk._types import TurnResult
|
||||
from turnstone.sdk.events import (
|
||||
ClusterEvent,
|
||||
ContentEvent,
|
||||
ErrorEvent,
|
||||
ReasoningEvent,
|
||||
@@ -98,6 +99,7 @@ class AsyncTurnstoneServer(_BaseClient):
|
||||
auto_approve_tools: str = "",
|
||||
user_id: str = "",
|
||||
ws_id: str = "",
|
||||
client_type: str = "",
|
||||
) -> CreateWorkstreamResponse:
|
||||
body: dict[str, Any] = {}
|
||||
if name:
|
||||
@@ -118,6 +120,8 @@ class AsyncTurnstoneServer(_BaseClient):
|
||||
body["user_id"] = user_id
|
||||
if ws_id:
|
||||
body["ws_id"] = ws_id
|
||||
if client_type:
|
||||
body["client_type"] = client_type
|
||||
return await self._request(
|
||||
"POST",
|
||||
"/v1/api/workstreams/new",
|
||||
@@ -199,6 +203,22 @@ class AsyncTurnstoneServer(_BaseClient):
|
||||
async for data in self._stream_sse("/v1/api/events/global"):
|
||||
yield ServerEvent.from_dict(data)
|
||||
|
||||
async def stream_node_events(
|
||||
self, *, expected_node_id: str = ""
|
||||
) -> AsyncIterator[ClusterEvent]:
|
||||
"""Iterate over node-level SSE events (snapshot + deltas).
|
||||
|
||||
Connects to ``/v1/api/events/global`` with the optional
|
||||
``expected_node_id`` param for identity verification. Yields
|
||||
``ClusterEvent`` instances (``NodeSnapshotEvent``, ``HealthChangedEvent``,
|
||||
etc.) suitable for console collector consumption.
|
||||
"""
|
||||
params: dict[str, str] = {}
|
||||
if expected_node_id:
|
||||
params["expected_node_id"] = expected_node_id
|
||||
async for data in self._stream_sse("/v1/api/events/global", params=params):
|
||||
yield ClusterEvent.from_dict(data)
|
||||
|
||||
# -- high-level convenience ----------------------------------------------
|
||||
|
||||
async def send_and_wait(
|
||||
@@ -457,6 +477,7 @@ class TurnstoneServer:
|
||||
auto_approve_tools: str = "",
|
||||
user_id: str = "",
|
||||
ws_id: str = "",
|
||||
client_type: str = "",
|
||||
) -> CreateWorkstreamResponse:
|
||||
return self._runner.run(
|
||||
self._async.create_workstream(
|
||||
@@ -469,6 +490,7 @@ class TurnstoneServer:
|
||||
auto_approve_tools=auto_approve_tools,
|
||||
user_id=user_id,
|
||||
ws_id=ws_id,
|
||||
client_type=client_type,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -509,6 +531,11 @@ class TurnstoneServer:
|
||||
def stream_global_events(self) -> Iterator[ServerEvent]:
|
||||
return self._runner.run_iter(self._async.stream_global_events())
|
||||
|
||||
def stream_node_events(self, *, expected_node_id: str = "") -> Iterator[ClusterEvent]:
|
||||
return self._runner.run_iter(
|
||||
self._async.stream_node_events(expected_node_id=expected_node_id)
|
||||
)
|
||||
|
||||
# -- high-level convenience ----------------------------------------------
|
||||
|
||||
def send_and_wait(
|
||||
|
||||
+213
-15
@@ -47,6 +47,7 @@ from turnstone.core.ratelimit import resolve_client_ip
|
||||
from turnstone.core.session import ChatSession, GenerationCancelled, SessionUI # noqa: F401
|
||||
from turnstone.core.tools import TOOLS # noqa: F401 — available for introspection
|
||||
from turnstone.core.workstream import Workstream, WorkstreamManager, WorkstreamState
|
||||
from turnstone.prompts import ClientType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, MutableMapping
|
||||
@@ -919,17 +920,111 @@ async def events_sse(request: Request) -> Response:
|
||||
return EventSourceResponse(event_generator(), ping=5)
|
||||
|
||||
|
||||
def _build_node_snapshot(app_state: Any) -> dict[str, Any]:
|
||||
"""Build a complete node state snapshot for SSE consumers.
|
||||
|
||||
Includes workstream list, health, and aggregate — everything the console
|
||||
collector needs to populate a ``NodeSnapshot`` without polling.
|
||||
"""
|
||||
from turnstone.core.memory import get_workstream_display_name
|
||||
|
||||
mgr: WorkstreamManager = app_state.workstreams
|
||||
wss = mgr.list_all()
|
||||
total_tokens = 0
|
||||
total_tool_calls = 0
|
||||
active_count = 0
|
||||
ws_list = []
|
||||
for ws in wss:
|
||||
ui = ws.ui
|
||||
if hasattr(ui, "_ws_lock"):
|
||||
with ui._ws_lock: # type: ignore[union-attr]
|
||||
tok = ui._ws_prompt_tokens + ui._ws_completion_tokens # type: ignore[union-attr]
|
||||
tc = sum(ui._ws_tool_calls.values()) # type: ignore[union-attr]
|
||||
ctx = ui._ws_context_ratio # type: ignore[union-attr]
|
||||
activity = ui._ws_current_activity # type: ignore[union-attr]
|
||||
activity_state = ui._ws_activity_state # type: ignore[union-attr]
|
||||
else:
|
||||
tok = tc = 0
|
||||
ctx = 0.0
|
||||
activity = activity_state = ""
|
||||
total_tokens += tok
|
||||
total_tool_calls += tc
|
||||
if ws.state.value != "idle":
|
||||
active_count += 1
|
||||
title = ""
|
||||
if ws.session:
|
||||
title = get_workstream_display_name(ws.session.ws_id) or ""
|
||||
ws_list.append(
|
||||
{
|
||||
"id": ws.id,
|
||||
"name": ws.name,
|
||||
"state": ws.state.value,
|
||||
"title": title,
|
||||
"tokens": tok,
|
||||
"context_ratio": round(ctx, 3),
|
||||
"activity": activity,
|
||||
"activity_state": activity_state,
|
||||
"tool_calls": tc,
|
||||
"model": ws.session.model if ws.session else "",
|
||||
"model_alias": ws.session.model_alias if ws.session else "",
|
||||
}
|
||||
)
|
||||
return {
|
||||
"type": "node_snapshot",
|
||||
"node_id": getattr(app_state, "node_id", ""),
|
||||
"workstreams": ws_list,
|
||||
"health": _build_health_dict(app_state),
|
||||
"aggregate": {
|
||||
"total_tokens": total_tokens,
|
||||
"total_tool_calls": total_tool_calls,
|
||||
"active_count": active_count,
|
||||
"total_count": len(ws_list),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def global_events_sse(request: Request) -> Response:
|
||||
"""GET /v1/api/events/global — global SSE event stream."""
|
||||
"""GET /v1/api/events/global — global SSE event stream.
|
||||
|
||||
Supports optional ``?expected_node_id=X`` query parameter for node identity
|
||||
verification. If present and the server's node_id does not match, returns
|
||||
409 Conflict immediately.
|
||||
|
||||
On connect, emits a ``node_snapshot`` event with the full node state
|
||||
(workstreams, health, aggregate) followed by real-time delta events.
|
||||
The snapshot and listener registration are atomic — no events are lost.
|
||||
"""
|
||||
# -- Node identity check --------------------------------------------------
|
||||
expected = request.query_params.get("expected_node_id")
|
||||
actual_node_id = getattr(request.app.state, "node_id", "")
|
||||
if expected and expected != actual_node_id:
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "node_id mismatch" if actual_node_id else "node_id unavailable",
|
||||
"expected": expected,
|
||||
"actual": actual_node_id,
|
||||
},
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
# -- Atomic snapshot + listener registration ------------------------------
|
||||
client_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1000)
|
||||
listeners = request.app.state.global_listeners
|
||||
listeners_lock = request.app.state.global_listeners_lock
|
||||
|
||||
# Hold the listeners lock while building the snapshot AND registering.
|
||||
# The fanout thread also acquires this lock when snapshotting the listener
|
||||
# list, so events that land on global_queue during snapshot build will be
|
||||
# distributed to our queue after we release — gap-free.
|
||||
with listeners_lock:
|
||||
snapshot = _build_node_snapshot(request.app.state)
|
||||
listeners.append(client_queue)
|
||||
|
||||
async def event_generator() -> AsyncGenerator[dict[str, str], None]:
|
||||
_metrics.record_sse_connect()
|
||||
try:
|
||||
# Emit snapshot as first event
|
||||
yield {"data": json.dumps(snapshot)}
|
||||
loop = asyncio.get_running_loop()
|
||||
executor = request.app.state.sse_executor
|
||||
while True:
|
||||
@@ -1100,17 +1195,20 @@ def _count_ws_states(wss: list[Workstream]) -> dict[str, int]:
|
||||
return counts
|
||||
|
||||
|
||||
async def health(request: Request) -> JSONResponse:
|
||||
"""GET /health — server health status."""
|
||||
mgr: WorkstreamManager = request.app.state.workstreams
|
||||
def _build_health_dict(app_state: Any) -> dict[str, Any]:
|
||||
"""Assemble health status dict from app state.
|
||||
|
||||
Shared by the ``/health`` endpoint and the global SSE snapshot.
|
||||
"""
|
||||
mgr: WorkstreamManager = app_state.workstreams
|
||||
wss = mgr.list_all()
|
||||
states = _count_ws_states(wss)
|
||||
monitor = getattr(request.app.state, "health_monitor", None)
|
||||
monitor = getattr(app_state, "health_monitor", None)
|
||||
backend_ok = monitor.is_healthy if monitor else True
|
||||
data: dict[str, Any] = {
|
||||
"status": "ok" if backend_ok else "degraded",
|
||||
"version": __version__,
|
||||
"node_id": getattr(request.app.state, "node_id", ""),
|
||||
"node_id": getattr(app_state, "node_id", ""),
|
||||
"uptime_seconds": round(time.monotonic() - _metrics.start_time, 2),
|
||||
"model": _metrics.model,
|
||||
"max_ws": mgr.max_workstreams,
|
||||
@@ -1120,14 +1218,19 @@ async def health(request: Request) -> JSONResponse:
|
||||
"circuit_state": monitor.circuit_state.value if monitor else "closed",
|
||||
},
|
||||
}
|
||||
mc = getattr(request.app.state, "mcp_client", None)
|
||||
mc = getattr(app_state, "mcp_client", None)
|
||||
if mc:
|
||||
data["mcp"] = {
|
||||
"servers": mc.server_count,
|
||||
"resources": mc.resource_count,
|
||||
"prompts": mc.prompt_count,
|
||||
}
|
||||
return JSONResponse(data)
|
||||
return data
|
||||
|
||||
|
||||
async def health(request: Request) -> JSONResponse:
|
||||
"""GET /health — server health status."""
|
||||
return JSONResponse(_build_health_dict(request.app.state))
|
||||
|
||||
|
||||
async def metrics_endpoint(request: Request) -> Response:
|
||||
@@ -1536,6 +1639,7 @@ async def create_workstream(request: Request) -> JSONResponse:
|
||||
skill_id=skill_data["template_id"] if skill_data else "",
|
||||
skill_version=applied_skill_version,
|
||||
ws_id=requested_ws_id,
|
||||
client_type=body.get("client_type", "") or "",
|
||||
)
|
||||
assert isinstance(ws.ui, WebUI)
|
||||
if skip or body.get("auto_approve", False):
|
||||
@@ -1546,10 +1650,21 @@ async def create_workstream(request: Request) -> JSONResponse:
|
||||
ws.session.set_watch_runner(
|
||||
runner, dispatch_fn=_make_watch_dispatch(ws, ws.session, ws.ui)
|
||||
)
|
||||
# Emit creation event on global queue for SSE consumers (console)
|
||||
gq: queue.Queue[dict[str, Any]] = request.app.state.global_queue
|
||||
with contextlib.suppress(queue.Full):
|
||||
gq.put_nowait(
|
||||
{
|
||||
"type": "ws_created",
|
||||
"ws_id": ws.id,
|
||||
"name": ws.name,
|
||||
"model": ws.session.model if ws.session else "",
|
||||
"model_alias": ws.session.model_alias if ws.session else "",
|
||||
}
|
||||
)
|
||||
# Emit eviction event if a workstream was evicted to make room
|
||||
evicted = mgr.last_evicted
|
||||
if evicted is not None:
|
||||
gq: queue.Queue[dict[str, Any]] = request.app.state.global_queue
|
||||
with contextlib.suppress(queue.Full):
|
||||
gq.put_nowait(
|
||||
{
|
||||
@@ -1669,6 +1784,9 @@ async def close_workstream(request: Request) -> JSONResponse:
|
||||
ws_id = str(body.get("ws_id", ""))
|
||||
mgr = request.app.state.workstreams
|
||||
if mgr.close(ws_id):
|
||||
gq: queue.Queue[dict[str, Any]] = request.app.state.global_queue
|
||||
with contextlib.suppress(queue.Full):
|
||||
gq.put_nowait({"type": "ws_closed", "ws_id": ws_id, "reason": "closed"})
|
||||
return JSONResponse({"status": "ok"})
|
||||
return JSONResponse({"error": "Cannot close last workstream"}, status_code=400)
|
||||
|
||||
@@ -2076,6 +2194,58 @@ async def internal_migrate(request: Request) -> JSONResponse:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _emit_health_changed(circuit_state: str, gq: queue.Queue[dict[str, Any]]) -> None:
|
||||
"""Push a health_changed event onto the global SSE queue.
|
||||
|
||||
Called from the BackendHealthMonitor callback on circuit breaker transitions.
|
||||
"""
|
||||
with contextlib.suppress(queue.Full):
|
||||
gq.put_nowait({"type": "health_changed", "circuit_state": circuit_state})
|
||||
|
||||
|
||||
def _aggregate_emitter_thread(
|
||||
mgr: WorkstreamManager,
|
||||
global_queue: queue.Queue[dict[str, Any]],
|
||||
interval: float = 10.0,
|
||||
) -> None:
|
||||
"""Periodically emit aggregate token/tool_call totals on the global SSE queue.
|
||||
|
||||
Runs as a daemon thread so the console receives periodic updates without
|
||||
having to poll ``/v1/api/dashboard``.
|
||||
"""
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
total_tokens = 0
|
||||
total_tool_calls = 0
|
||||
active_count = 0
|
||||
try:
|
||||
for ws in mgr.list_all():
|
||||
ui = ws.ui
|
||||
if hasattr(ui, "_ws_lock"):
|
||||
with ui._ws_lock: # type: ignore[union-attr]
|
||||
tok = ui._ws_prompt_tokens + ui._ws_completion_tokens # type: ignore[union-attr]
|
||||
tc = sum(ui._ws_tool_calls.values()) # type: ignore[union-attr]
|
||||
else:
|
||||
tok = 0
|
||||
tc = 0
|
||||
total_tokens += tok
|
||||
total_tool_calls += tc
|
||||
if ws.state.value != "idle":
|
||||
active_count += 1
|
||||
with contextlib.suppress(queue.Full):
|
||||
global_queue.put_nowait(
|
||||
{
|
||||
"type": "aggregate",
|
||||
"total_tokens": total_tokens,
|
||||
"total_tool_calls": total_tool_calls,
|
||||
"active_count": active_count,
|
||||
"total_count": len(mgr.list_all()),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
log.debug("Aggregate emitter error", exc_info=True)
|
||||
|
||||
|
||||
def _idle_cleanup_thread(
|
||||
mgr: WorkstreamManager,
|
||||
timeout_sec: float,
|
||||
@@ -2134,6 +2304,13 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
daemon=True,
|
||||
)
|
||||
fanout.start()
|
||||
# Start aggregate emitter thread for SSE consumers
|
||||
agg_emitter = threading.Thread(
|
||||
target=_aggregate_emitter_thread,
|
||||
args=(app.state.workstreams, app.state.global_queue),
|
||||
daemon=True,
|
||||
)
|
||||
agg_emitter.start()
|
||||
# Start idle cleanup thread if configured
|
||||
if app.state.idle_timeout > 0:
|
||||
cleanup = threading.Thread(
|
||||
@@ -2606,6 +2783,13 @@ def main() -> None:
|
||||
if new_reg is not None:
|
||||
new_reg.shutdown()
|
||||
|
||||
# Set up global event queue for state-change broadcasts (created early so
|
||||
# the health monitor callback can reference it).
|
||||
global_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=10000)
|
||||
global_listeners: list[queue.Queue[dict[str, Any]]] = []
|
||||
global_listeners_lock = threading.Lock()
|
||||
WebUI._global_queue = global_queue
|
||||
|
||||
health_monitor = BackendHealthMonitor(
|
||||
client=client,
|
||||
probe_interval=config_store.get("health.backend_probe_interval"),
|
||||
@@ -2615,6 +2799,7 @@ def main() -> None:
|
||||
provider=provider_name,
|
||||
initial_model=model,
|
||||
on_model_changed=_handle_model_change,
|
||||
on_state_changed=lambda state: _emit_health_changed(state, global_queue),
|
||||
)
|
||||
health_monitor.start()
|
||||
|
||||
@@ -2628,12 +2813,6 @@ def main() -> None:
|
||||
trusted_proxies=config_store.get("ratelimit.trusted_proxies"),
|
||||
)
|
||||
|
||||
# Set up global event queue for state-change broadcasts
|
||||
global_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=10000)
|
||||
global_listeners: list[queue.Queue[dict[str, Any]]] = []
|
||||
global_listeners_lock = threading.Lock()
|
||||
WebUI._global_queue = global_queue
|
||||
|
||||
# Config builders — shared between startup logging and session factory.
|
||||
# Re-read from ConfigStore each call so hot-reload works.
|
||||
from turnstone.core.judge import JudgeConfig
|
||||
@@ -2678,11 +2857,26 @@ def main() -> None:
|
||||
ws_id: str | None = None,
|
||||
*,
|
||||
skill: str | None = None,
|
||||
client_type: str = "",
|
||||
) -> ChatSession:
|
||||
assert ui is not None
|
||||
r_client, r_model, r_cfg = registry.resolve(model_alias)
|
||||
uid = getattr(ui, "_user_id", "") or ""
|
||||
|
||||
# Resolve username from user_id for system message context
|
||||
_username = ""
|
||||
if uid:
|
||||
try:
|
||||
from turnstone.core.storage._registry import get_storage as _gs
|
||||
|
||||
_st = _gs()
|
||||
if _st:
|
||||
_u = _st.get_user(uid)
|
||||
if _u:
|
||||
_username = _u.get("username", "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Re-resolve from ConfigStore so new workstreams pick up hot-reloaded settings.
|
||||
live_memory_config = _build_memory_config()
|
||||
live_judge_config = _build_judge_config()
|
||||
@@ -2716,6 +2910,10 @@ def main() -> None:
|
||||
user_id=uid,
|
||||
memory_config=live_memory_config,
|
||||
config_store=config_store,
|
||||
client_type=ClientType(client_type)
|
||||
if client_type in {ct.value for ct in ClientType}
|
||||
else ClientType.WEB,
|
||||
username=_username,
|
||||
)
|
||||
|
||||
# Create WatchRunner (periodic command polling, server-level)
|
||||
|
||||
Reference in New Issue
Block a user