mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 07:22:24 -06:00
Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bd9f780b21 | |||
| 5d14b5f675 | |||
| 4693fa95f1 | |||
| c3423d6606 | |||
| 53f1222c22 | |||
| 802d87a57f | |||
| 8349d9994d | |||
| ac1fd67137 | |||
| 1b40ae79f9 | |||
| b078ddccf0 | |||
| 4b6c93a0e9 | |||
| 9d283e951f | |||
| 4e407e7d4f | |||
| 7ab24e500b | |||
| 5bcbcb73b9 | |||
| af6749421a | |||
| 4d6cb77075 | |||
| 5c225ef39b | |||
| f5a843f44a | |||
| cf44841624 | |||
| 7ffab6a272 | |||
| ba3bc9d989 | |||
| dbe023b4dd | |||
| 3d3a8b7367 | |||
| 99eff73a97 | |||
| 99fcd30299 | |||
| 423c2e80b7 | |||
| a0eb77360d | |||
| 3dd0e196fe | |||
| 19c3db5329 | |||
| 0bea72019e | |||
| b9ff52d582 | |||
| 961f999c93 | |||
| 8bdb916064 | |||
| 25fe4e728a | |||
| 0d1a32ff65 |
+1
-1
@@ -8,7 +8,7 @@ FROM python:3.14-slim
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.8 /uv /usr/local/bin/uv
|
||||
|
||||
# Remove the slim image's man page exclusion so man-db has actual content
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/docker
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "1.5.0"
|
||||
version = "1.5.5"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
|
||||
@@ -35,11 +35,10 @@ def _seed_children(
|
||||
The production path populates the registry via the cluster-event
|
||||
fan-out thread observing ``ws_created`` events. These tests just
|
||||
need a known-children set for the endpoint handlers to iterate —
|
||||
inject directly under ``_children_lock`` rather than spinning up
|
||||
the collector + fan-out plumbing.
|
||||
inject directly via the registry's bulk-merge surface rather than
|
||||
spinning up the collector + fan-out plumbing.
|
||||
"""
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked(coord_ws_id, child_ws_ids)
|
||||
adapter._registry.merge_children(coord_ws_id, child_ws_ids)
|
||||
|
||||
|
||||
class _AuthMiddleware(BaseHTTPMiddleware):
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Console-side coord_registry auto-refresh on model-definition CRUD + reload.
|
||||
|
||||
The console builds ``app.state.coord_registry`` once at lifespan startup
|
||||
and the coordinator session factory closes over that exact instance.
|
||||
Without these refresh hooks, an admin who edits a model definition
|
||||
through the UI sees the DB change immediately but coordinator sessions
|
||||
keep calling the prior model name — the on-disk truth diverges from the
|
||||
in-process registry until the console is restarted.
|
||||
|
||||
These tests cover both the helper (``_refresh_coord_registry``)
|
||||
and the four wired endpoints (create / update / delete / explicit reload)
|
||||
to lock in:
|
||||
|
||||
- in-place mutation: ``coord_registry`` object identity is preserved
|
||||
across refreshes (factory closure must not be invalidated);
|
||||
- failure isolation: a load or reload failure leaves the existing
|
||||
registry intact rather than tearing down a working coordinator;
|
||||
- no-op safety: the helper short-circuits when ``coord_registry`` is
|
||||
``None`` so a coord-less console (no model rows at boot) doesn't
|
||||
500 on routine model-definition CRUD.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from tests._coord_test_helpers import _AuthMiddleware
|
||||
from turnstone.console.server import (
|
||||
_refresh_coord_registry,
|
||||
admin_create_model_definition,
|
||||
admin_delete_model_definition,
|
||||
admin_model_reload,
|
||||
admin_update_model_definition,
|
||||
)
|
||||
from turnstone.core.model_registry import ModelConfig, ModelRegistry
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path: Any) -> SQLiteBackend:
|
||||
return SQLiteBackend(str(tmp_path / "models.db"))
|
||||
|
||||
|
||||
def _seed_model_def(
|
||||
storage: SQLiteBackend,
|
||||
*,
|
||||
definition_id: str,
|
||||
alias: str,
|
||||
model: str,
|
||||
base_url: str = "http://localhost:8000/v1",
|
||||
enabled: bool = True,
|
||||
) -> None:
|
||||
"""Insert a model definition row directly via the storage API."""
|
||||
storage.create_model_definition(
|
||||
definition_id=definition_id,
|
||||
alias=alias,
|
||||
model=model,
|
||||
provider="openai-compatible",
|
||||
base_url=base_url,
|
||||
api_key="sk-test",
|
||||
context_window=8192,
|
||||
capabilities="{}",
|
||||
enabled=enabled,
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
|
||||
def _make_config(alias: str, model: str) -> ModelConfig:
|
||||
return ModelConfig(
|
||||
alias=alias,
|
||||
base_url="http://localhost:8000/v1",
|
||||
api_key="sk-test",
|
||||
model=model,
|
||||
context_window=8192,
|
||||
provider="openai-compatible",
|
||||
source="db",
|
||||
)
|
||||
|
||||
|
||||
def _make_registry(
|
||||
*,
|
||||
alias: str = "local",
|
||||
model: str = "old-model",
|
||||
extras: dict[str, str] | None = None,
|
||||
) -> ModelRegistry:
|
||||
"""Build a real ModelRegistry seeded with ``alias`` (the default) plus
|
||||
any ``extras`` (alias → model). ``ModelRegistry.__init__`` rejects an
|
||||
empty model dict so tests that exercise the helper need at least one
|
||||
entry; pass ``extras`` for multi-alias scenarios (e.g. delete-by-alias).
|
||||
"""
|
||||
configs = {alias: _make_config(alias, model)}
|
||||
for extra_alias, extra_model in (extras or {}).items():
|
||||
configs[extra_alias] = _make_config(extra_alias, extra_model)
|
||||
return ModelRegistry(configs, default=alias)
|
||||
|
||||
|
||||
class _AppState:
|
||||
"""Shim mirroring Starlette's ``app.state`` for direct helper tests."""
|
||||
|
||||
coord_registry: ModelRegistry | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper-level tests — ``_refresh_coord_registry`` semantics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_helper_rebuilds_registry_from_db(storage: SQLiteBackend) -> None:
|
||||
"""Helper pulls the latest DB rows into the existing registry."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="new-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="old-model")
|
||||
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "new-model"
|
||||
|
||||
|
||||
def test_helper_preserves_object_identity(storage: SQLiteBackend) -> None:
|
||||
"""The factory closes over the registry object — refresh must mutate
|
||||
in place rather than swap the attribute."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="new-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry()
|
||||
before = id(state.coord_registry)
|
||||
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert id(state.coord_registry) == before
|
||||
|
||||
|
||||
def test_helper_noop_when_coord_registry_none(storage: SQLiteBackend) -> None:
|
||||
"""Console boot with no model rows leaves coord_registry = None.
|
||||
The helper must not 500 in that state — CRUD that lands the FIRST
|
||||
row would otherwise fail before the operator can recover."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m")
|
||||
state = _AppState()
|
||||
state.coord_registry = None
|
||||
|
||||
_refresh_coord_registry(state, storage) # must not raise
|
||||
|
||||
assert state.coord_registry is None
|
||||
|
||||
|
||||
def test_helper_preserves_registry_when_load_fails(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An unexpected error from ``load_model_registry`` (e.g. config.toml
|
||||
parse failure, programming bug) must not tear down a working
|
||||
registry — log + leave the existing instance intact."""
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="old-model")
|
||||
|
||||
def _boom(**_kw: Any) -> ModelRegistry:
|
||||
raise RuntimeError("simulated loader failure")
|
||||
|
||||
monkeypatch.setattr("turnstone.core.model_registry.load_model_registry", _boom)
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "old-model"
|
||||
|
||||
|
||||
def test_helper_preserves_registry_when_strict_load_fails(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""``load_model_registry`` normally swallows storage read errors and
|
||||
would return a config.toml-only registry on a transient DB outage —
|
||||
applying that via ``reload()`` would silently drop every DB-sourced
|
||||
alias. The helper passes ``strict=True`` so the loader re-raises
|
||||
instead, the helper's outer except catches it, and the existing
|
||||
registry survives intact."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="db-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="db-model")
|
||||
|
||||
def _broken(**_kw: Any) -> Any:
|
||||
raise RuntimeError("simulated transient DB outage")
|
||||
|
||||
monkeypatch.setattr(storage, "list_model_definitions", _broken)
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
# Existing registry untouched — strict=True surfaced the storage
|
||||
# error to the helper before the loader's silent fallback could
|
||||
# produce a truncated registry for reload().
|
||||
assert state.coord_registry.get_config("local").model == "db-model"
|
||||
|
||||
|
||||
def test_helper_preserves_registry_when_no_enabled_rows(storage: SQLiteBackend) -> None:
|
||||
"""All rows disabled/deleted: ModelRegistry.__init__ rejects an empty
|
||||
model dict (raises ValueError). Helper must catch and preserve the
|
||||
existing registry so coord stays usable while admin restores rows."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m", enabled=False)
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="cached-model")
|
||||
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "cached-model"
|
||||
|
||||
|
||||
def test_helper_preserves_registry_on_reload_validation_error(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A reload that raises mid-mutation (e.g. validation guard) must
|
||||
leave the existing registry instance functional."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="new-model")
|
||||
state = _AppState()
|
||||
state.coord_registry = _make_registry(alias="local", model="old-model")
|
||||
|
||||
def _broken_reload(*_a: Any, **_kw: Any) -> None:
|
||||
raise ValueError("simulated reload validation failure")
|
||||
|
||||
monkeypatch.setattr(state.coord_registry, "reload", _broken_reload)
|
||||
_refresh_coord_registry(state, storage)
|
||||
|
||||
# Existing registry still reachable; the broken reload was a no-op
|
||||
# at the public-facing level.
|
||||
assert state.coord_registry is not None
|
||||
assert state.coord_registry.get_config("local").model == "old-model"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoint-level integration tests — verify wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_client(storage: SQLiteBackend, registry: ModelRegistry | None) -> TestClient:
|
||||
"""Build a TestClient wired to the four model-definition endpoints.
|
||||
|
||||
Uses the shared header-driven ``_AuthMiddleware`` from
|
||||
``tests/_coord_test_helpers``; default headers below grant
|
||||
``admin.models`` permission so the endpoint gate passes.
|
||||
"""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions",
|
||||
admin_create_model_definition,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions/reload",
|
||||
admin_model_reload,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions/{definition_id}",
|
||||
admin_update_model_definition,
|
||||
methods=["PUT"],
|
||||
),
|
||||
Route(
|
||||
"/v1/api/admin/model-definitions/{definition_id}",
|
||||
admin_delete_model_definition,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_AuthMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
app.state.coord_registry = registry
|
||||
# Reload endpoint also touches these — stub them so the test focuses
|
||||
# on the registry-refresh behaviour without dragging in a full
|
||||
# collector / proxy_client wiring.
|
||||
app.state.collector = MagicMock()
|
||||
app.state.collector.get_all_nodes.return_value = []
|
||||
app.state.proxy_client = MagicMock()
|
||||
app.state.config_store = MagicMock()
|
||||
client = TestClient(app)
|
||||
client.headers.update({"X-Test-User": "admin", "X-Test-Perms": "admin.models"})
|
||||
return client
|
||||
|
||||
|
||||
def test_create_endpoint_refreshes_registry(storage: SQLiteBackend) -> None:
|
||||
"""POST /api/admin/model-definitions bumps the in-process registry
|
||||
so newly-spawned coord sessions see the new alias immediately."""
|
||||
# Pre-existing alias (registry needs at least one row)
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m")
|
||||
registry = _make_registry(alias="local", model="m")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
resp = client.post(
|
||||
"/v1/api/admin/model-definitions",
|
||||
json={
|
||||
"alias": "fast",
|
||||
"model": "fast-model",
|
||||
"provider": "openai-compatible",
|
||||
"base_url": "http://localhost:9000/v1",
|
||||
"api_key": "sk-x",
|
||||
"context_window": 4096,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert registry.has_alias("fast")
|
||||
assert registry.get_config("fast").model == "fast-model"
|
||||
|
||||
|
||||
def test_update_endpoint_refreshes_registry(storage: SQLiteBackend) -> None:
|
||||
"""PUT swaps the underlying model name behind a stable alias — the
|
||||
user's reported regression."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="old-model")
|
||||
registry = _make_registry(alias="local", model="old-model")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
resp = client.put(
|
||||
"/v1/api/admin/model-definitions/m1",
|
||||
json={"model": "new-model"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert registry.get_config("local").model == "new-model"
|
||||
|
||||
|
||||
def test_update_endpoint_skips_refresh_on_empty_body(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An empty PUT body must skip the registry refresh — the
|
||||
``if updates:`` gate exists because ``load_model_registry`` is
|
||||
non-trivial and a no-op refresh on every PUT would burn cycles
|
||||
rebuilding state that hasn't changed. Spy on the helper to lock
|
||||
the gate down: a regression that drops the conditional would
|
||||
register a call here and trip the assertion.
|
||||
"""
|
||||
from turnstone.console import server as server_module
|
||||
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="locked-in")
|
||||
registry = _make_registry(alias="local", model="locked-in")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
calls: list[tuple[Any, Any]] = []
|
||||
|
||||
def _spy(app_state: Any, storage: Any) -> None:
|
||||
calls.append((app_state, storage))
|
||||
|
||||
monkeypatch.setattr(server_module, "_refresh_coord_registry", _spy)
|
||||
|
||||
resp = client.put("/v1/api/admin/model-definitions/m1", json={})
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert calls == [] # gate held: empty body did not trigger a refresh
|
||||
|
||||
|
||||
def test_delete_endpoint_refreshes_registry(storage: SQLiteBackend) -> None:
|
||||
"""DELETE drops the alias from the in-process registry too — a
|
||||
coord session that tried to resolve the deleted alias would
|
||||
otherwise hit a stale cached client."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="m")
|
||||
_seed_model_def(storage, definition_id="m2", alias="extra", model="x")
|
||||
registry = _make_registry(alias="local", model="m", extras={"extra": "x"})
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
resp = client.delete("/v1/api/admin/model-definitions/m2")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert not registry.has_alias("extra")
|
||||
assert registry.has_alias("local") # default alias unaffected
|
||||
|
||||
|
||||
def test_reload_endpoint_refreshes_registry(
|
||||
storage: SQLiteBackend, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The explicit reload button must refresh the console's own
|
||||
registry — until this PR it only fanned out to nodes."""
|
||||
_seed_model_def(storage, definition_id="m1", alias="local", model="initial")
|
||||
registry = _make_registry(alias="local", model="initial")
|
||||
client = _make_client(storage, registry)
|
||||
|
||||
# Bypass the CRUD endpoints to mimic an out-of-band DB change (e.g.
|
||||
# an operator psql session) and verify the explicit reload path
|
||||
# still pulls the change in.
|
||||
storage.update_model_definition("m1", model="reloaded-model")
|
||||
|
||||
# Stub the async cluster fan-out helpers — they require a fully-wired
|
||||
# collector / proxy_client which is orthogonal to the helper under test.
|
||||
async def _noop_publish(_request: Any) -> None:
|
||||
return None
|
||||
|
||||
async def _noop_notify(_request: Any) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr("turnstone.console.server._publish_config_change", _noop_publish)
|
||||
monkeypatch.setattr("turnstone.console.server._notify_nodes_model_reload", _noop_notify)
|
||||
|
||||
resp = client.post("/v1/api/admin/model-definitions/reload")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert registry.get_config("local").model == "reloaded-model"
|
||||
@@ -92,3 +92,71 @@ def test_tool_error_does_not_overwrite_approval_badge() -> None:
|
||||
"badge instead so the approval verdict stays visible alongside "
|
||||
"the error."
|
||||
)
|
||||
|
||||
|
||||
def test_replay_history_renders_content_before_tool_block() -> None:
|
||||
"""In ``replayHistory``'s ``role === "assistant"`` branch, the
|
||||
``msg.content`` render must precede the ``msg.tool_calls`` render.
|
||||
|
||||
Two reasons, both load-bearing:
|
||||
|
||||
1. **Structural** — the next loop iteration's ``role === "tool"``
|
||||
message anchors to ``lastToolBlock``. The tool-block branch sets
|
||||
that anchor; the content branch clears it. If content runs after
|
||||
the tool block, the clear silently drops the upcoming tool
|
||||
result. Pre-fix, every interactive tool result was missing from
|
||||
saved-workstream replays whenever the assistant turn carried
|
||||
both narration and tool calls (very common output shape).
|
||||
|
||||
2. **Visual** — the live SSE path renders content first
|
||||
(``stream_text`` streams before ``tool_info`` /
|
||||
``approve_request``), so replay should match.
|
||||
|
||||
The test pins the order via the offsets of the ``msg.content`` and
|
||||
``msg.tool_calls`` branch headers inside the function body."""
|
||||
body = _APP_JS.read_text(encoding="utf-8")
|
||||
start = body.index("Pane.prototype.replayHistory = function")
|
||||
end = body.index("Pane.prototype._attachRetryToLastAssistant", start)
|
||||
fn = body[start:end]
|
||||
# Locate the assistant branch and bound the search to its body —
|
||||
# the function also handles user / tool roles which would otherwise
|
||||
# confuse the offset comparison.
|
||||
asst_start = fn.index('msg.role === "assistant"')
|
||||
asst_end = fn.index('msg.role === "tool"', asst_start)
|
||||
asst = fn[asst_start:asst_end]
|
||||
content_idx = asst.index("if (msg.content)")
|
||||
tool_calls_idx = asst.index("if (msg.tool_calls && msg.tool_calls.length)")
|
||||
assert content_idx < tool_calls_idx, (
|
||||
"replayHistory must render msg.content BEFORE msg.tool_calls "
|
||||
"inside the assistant branch — otherwise the lastToolBlock "
|
||||
"anchor is clobbered before the next iteration's tool result "
|
||||
"can attach to it (and the visual order also drifts from the "
|
||||
"live SSE flow)."
|
||||
)
|
||||
|
||||
|
||||
def test_replay_history_renders_persisted_verdict_badge() -> None:
|
||||
"""Saved-workstream replays must paint the persisted intent verdict
|
||||
next to each tool div, using the same ``renderVerdictBadge`` helper
|
||||
the live ``showInlineToolBlock`` path uses. Pre-fix the audit trail
|
||||
was complete in storage (``intent_verdicts`` table) but never
|
||||
surfaced on replay — operators reviewing a saved workstream
|
||||
couldn't see what the heuristic / LLM judge thought of any tool
|
||||
call. This test pins the call site so a refactor that drops the
|
||||
decoration regresses the audit surface."""
|
||||
body = _APP_JS.read_text(encoding="utf-8")
|
||||
start = body.index("Pane.prototype.replayHistory = function")
|
||||
end = body.index("Pane.prototype._attachRetryToLastAssistant", start)
|
||||
fn = body[start:end]
|
||||
# Match a `renderVerdictBadge(<something>.verdict, ...)` call inside
|
||||
# the replay loop. Loose on whitespace + identifier so a future
|
||||
# rename of the iteration variable doesn't trip CI.
|
||||
badge_call_re = re.compile(
|
||||
r"renderVerdictBadge\(\s*\w+\.verdict\b",
|
||||
)
|
||||
assert badge_call_re.search(fn), (
|
||||
"replayHistory must call renderVerdictBadge(tc.verdict, ...) "
|
||||
"when a persisted verdict is attached to a tool_call entry — "
|
||||
"otherwise the audit-trail data persisted to intent_verdicts "
|
||||
"doesn't surface on saved-workstream replays."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Unit tests for :mod:`turnstone.core.child_source`.
|
||||
|
||||
Covers both strategies in isolation against fakes — no live collector,
|
||||
no live SessionManager. Adapter-level integration coverage continues to
|
||||
live in ``test_coordinator_adapter.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.core.child_source import ClusterChildSource, SameNodeChildSource
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import queue
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SameNodeChildSource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeManager:
|
||||
"""Minimal SessionManager stand-in implementing the subscribe API."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.subscribers: list[Any] = []
|
||||
|
||||
def subscribe_to_state(self, callback: Any) -> None:
|
||||
self.subscribers.append(callback)
|
||||
|
||||
def unsubscribe_from_state(self, callback: Any) -> None:
|
||||
with contextlib.suppress(ValueError):
|
||||
self.subscribers.remove(callback)
|
||||
|
||||
def fire(self, ws_id: str, state: WorkstreamState) -> None:
|
||||
for cb in self.subscribers:
|
||||
cb(ws_id, state)
|
||||
|
||||
|
||||
class TestSameNodeChildSource:
|
||||
def test_start_subscribes_to_manager(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
assert len(mgr.subscribers) == 1
|
||||
|
||||
def test_state_change_for_known_child_pushes_to_sink(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
registry.add_child("p1", "c1")
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
|
||||
mgr.fire("c1", WorkstreamState.RUNNING)
|
||||
|
||||
assert len(sink_calls) == 1
|
||||
ev = sink_calls[0]
|
||||
assert ev["type"] == "cluster_state"
|
||||
assert ev["ws_id"] == "c1"
|
||||
assert ev["state"] == "running"
|
||||
|
||||
def test_state_change_for_unknown_workstream_is_dropped(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
|
||||
# No registry entry — pre-filter drops the event without
|
||||
# invoking the sink.
|
||||
mgr.fire("ws-unknown", WorkstreamState.IDLE)
|
||||
assert sink_calls == []
|
||||
|
||||
def test_shutdown_unsubscribes(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(mgr.subscribers) == 1
|
||||
src.shutdown()
|
||||
assert mgr.subscribers == []
|
||||
|
||||
def test_start_is_idempotent(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
src.start(sink=lambda ev: None)
|
||||
src.start(sink=lambda ev: None)
|
||||
# Second start is a no-op; only one subscription.
|
||||
assert len(mgr.subscribers) == 1
|
||||
|
||||
def test_sink_exception_does_not_propagate(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
registry.add_child("p1", "c1")
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
|
||||
def bad_sink(ev: dict[str, Any]) -> None:
|
||||
raise RuntimeError("sink boom")
|
||||
|
||||
src.start(sink=bad_sink)
|
||||
# Should not raise — the strategy catches sink failures and logs.
|
||||
mgr.fire("c1", WorkstreamState.RUNNING)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClusterChildSource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeCollector:
|
||||
"""Minimal ClusterCollector stand-in providing the listener API."""
|
||||
|
||||
def __init__(self, snapshot: dict[str, Any] | None = None) -> None:
|
||||
self._snapshot = snapshot or {"nodes": []}
|
||||
self.queues: list[queue.Queue[dict[str, Any]]] = []
|
||||
self.unregistered: list[queue.Queue[dict[str, Any]]] = []
|
||||
|
||||
def get_snapshot_and_register(self, q: queue.Queue[dict[str, Any]]) -> dict[str, Any]:
|
||||
self.queues.append(q)
|
||||
return self._snapshot
|
||||
|
||||
def unregister_listener(self, q: queue.Queue[dict[str, Any]]) -> None:
|
||||
self.unregistered.append(q)
|
||||
|
||||
def emit(self, event: dict[str, Any]) -> None:
|
||||
"""Push an event to all registered listener queues."""
|
||||
for q in self.queues:
|
||||
q.put(event)
|
||||
|
||||
|
||||
class TestClusterChildSource:
|
||||
def test_start_subscribes_to_collector(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(coll.queues) == 1
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_start_primes_registry_from_snapshot(self) -> None:
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
{
|
||||
"workstreams": [
|
||||
{"id": "c1", "parent_ws_id": "p1"},
|
||||
{"id": "c2", "parent_ws_id": "p1"},
|
||||
# Unknown parent — dropped
|
||||
{"id": "x", "parent_ws_id": "p-unknown"},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
coll = _FakeCollector(snapshot)
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=lambda: ["p1"],
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
assert registry.parent_for("x") is None
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_event_dispatched_to_sink(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
src.start(sink=sink_calls.append)
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c1", "state": "running"})
|
||||
# Daemon thread loop has 1.0s queue timeout; poll briefly.
|
||||
for _ in range(20):
|
||||
if sink_calls:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(sink_calls) == 1
|
||||
assert sink_calls[0]["ws_id"] == "c1"
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_shutdown_unregisters_and_joins_thread(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
src.start(sink=lambda ev: None)
|
||||
src.shutdown()
|
||||
assert coll.unregistered == coll.queues
|
||||
# Second shutdown is a no-op (idempotent).
|
||||
src.shutdown()
|
||||
|
||||
def test_start_is_idempotent(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(coll.queues) == 1
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_sink_exception_does_not_kill_thread(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
survived_calls: list[dict[str, Any]] = []
|
||||
call_count = [0]
|
||||
|
||||
def flaky_sink(ev: dict[str, Any]) -> None:
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
raise RuntimeError("first one boom")
|
||||
survived_calls.append(ev)
|
||||
|
||||
try:
|
||||
src.start(sink=flaky_sink)
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c1", "state": "x"})
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c2", "state": "y"})
|
||||
for _ in range(40):
|
||||
if survived_calls:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(survived_calls) == 1
|
||||
assert survived_calls[0]["ws_id"] == "c2"
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
|
||||
# Multi-subscriber observer tests for ``SessionManager.subscribe_to_state``
|
||||
# / ``unsubscribe_from_state`` live in ``test_session_manager.py`` where
|
||||
# the proper FakeAdapter / FakeStorage construction helpers already exist.
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Unit tests for :class:`turnstone.core.children_registry.ChildrenRegistry`.
|
||||
|
||||
The registry was lifted from ``CoordinatorAdapter`` in Stage 3 Step 1.
|
||||
Adapter-level coverage for the integrated behavior already lives in
|
||||
``test_coordinator_adapter.py``; this file pins the data structure
|
||||
invariants in isolation so the registry can be reused by future
|
||||
``ChildSource`` strategies (Step 2) without re-deriving the behavior
|
||||
from the adapter test surface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
|
||||
|
||||
class _Sentinel:
|
||||
"""Lightweight UI stand-in; identity-comparable, no behavior."""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry() -> ChildrenRegistry:
|
||||
return ChildrenRegistry()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# install / uninstall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInstallUninstall:
|
||||
def test_install_seeds_empty_child_set_and_presence(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.children_of("p1") == []
|
||||
assert registry.ui_for("p1") is ui
|
||||
assert registry.parents() == ["p1"]
|
||||
|
||||
def test_install_is_idempotent_repoints_ui_keeps_children(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui_a = _Sentinel()
|
||||
ui_b = _Sentinel()
|
||||
registry.install("p1", ui_a)
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
registry.install("p1", ui_b)
|
||||
assert registry.ui_for("p1") is ui_b
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
|
||||
def test_uninstall_clears_forward_reverse_and_presence(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
registry.uninstall("p1")
|
||||
assert registry.children_of("p1") == []
|
||||
assert registry.ui_for("p1") is None
|
||||
assert registry.parents() == []
|
||||
assert registry.parent_for("c1") is None
|
||||
assert registry.parent_for("c2") is None
|
||||
|
||||
def test_uninstall_unknown_parent_is_noop(self, registry: ChildrenRegistry) -> None:
|
||||
registry.uninstall("never-installed") # must not raise
|
||||
|
||||
def test_uninstall_does_not_clobber_other_parents(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
registry.install("p2", _Sentinel())
|
||||
registry.merge_children("p1", ["c1"])
|
||||
registry.merge_children("p2", ["c2"])
|
||||
registry.uninstall("p1")
|
||||
assert registry.parent_for("c1") is None
|
||||
assert registry.parent_for("c2") == "p2"
|
||||
assert registry.parents() == ["p2"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# add_child — atomic check-and-route
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAddChild:
|
||||
def test_add_child_returns_ui_on_success(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.add_child("p1", "c1") is ui
|
||||
assert registry.parent_for("c1") == "p1"
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
def test_add_child_returns_none_when_parent_not_installed(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
assert registry.add_child("absent", "c1") is None
|
||||
assert registry.parent_for("c1") is None
|
||||
|
||||
def test_add_child_returns_none_on_duplicate(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.add_child("p1", "c1") is ui
|
||||
# second add for same child returns None — caller must not
|
||||
# double-dispatch.
|
||||
assert registry.add_child("p1", "c1") is None
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_children — bulk seeding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergeChildren:
|
||||
def test_merge_seeds_forward_and_reverse(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["c1", "c2", "c3"])
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2", "c3"}
|
||||
for cid in ("c1", "c2", "c3"):
|
||||
assert registry.parent_for(cid) == "p1"
|
||||
|
||||
def test_merge_is_idempotent(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["c1"])
|
||||
registry.merge_children("p1", ["c1"])
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
def test_merge_skips_empty_or_falsy_ids(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["", "c1", "", "c2"])
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
|
||||
def test_merge_does_not_require_install(self, registry: ChildrenRegistry) -> None:
|
||||
# Snapshot-priming may run before the parent's install fires —
|
||||
# the merge still seeds the forward set so the install picks
|
||||
# the children up. (Storage-seeded rebuild relies on this.)
|
||||
registry.merge_children("p1", ["c1"])
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
# ui_for is still None because install hasn't run
|
||||
assert registry.ui_for("p1") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lookups — return copies, not live refs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLookups:
|
||||
def test_children_of_returns_copy(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
snap = registry.children_of("p1")
|
||||
snap.append("c3-injected")
|
||||
assert "c3-injected" not in registry.children_of("p1")
|
||||
|
||||
def test_children_of_unknown_parent_returns_empty(self, registry: ChildrenRegistry) -> None:
|
||||
assert registry.children_of("absent") == []
|
||||
|
||||
def test_parent_for_unknown_child_returns_none(self, registry: ChildrenRegistry) -> None:
|
||||
assert registry.parent_for("absent") is None
|
||||
|
||||
def test_parents_returns_copy(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
snap = registry.parents()
|
||||
snap.append("p2-injected")
|
||||
assert "p2-injected" not in registry.parents()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concurrency — concurrent add_child must not exceed the unique-set
|
||||
# invariant or leave a half-installed reverse-index entry.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConcurrency:
|
||||
def test_concurrent_add_child_returns_ui_exactly_once_per_unique(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
results: list[object] = []
|
||||
results_lock = threading.Lock()
|
||||
|
||||
def attempt_add(child_id: str) -> None:
|
||||
r = registry.add_child("p1", child_id)
|
||||
with results_lock:
|
||||
results.append(r)
|
||||
|
||||
threads = [threading.Thread(target=attempt_add, args=("c1",)) for _ in range(20)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Exactly one thread sees the UI; the remaining 19 see None
|
||||
# (duplicate). The forward + reverse indexes carry exactly one
|
||||
# entry for c1.
|
||||
successes = [r for r in results if r is ui]
|
||||
nones = [r for r in results if r is None]
|
||||
assert len(successes) == 1
|
||||
assert len(nones) == 19
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
assert registry.parent_for("c1") == "p1"
|
||||
|
||||
def test_concurrent_install_and_add_child_no_resurrect(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
# add_child racing with uninstall: either lands first (registry
|
||||
# populated) or the parent is gone (returns None). Must NOT
|
||||
# leave a forward-set entry without presence — that would be
|
||||
# the "resurrected after close" leak the locked dispatch path
|
||||
# was guarding against.
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
|
||||
outcomes: list[object] = []
|
||||
|
||||
def adder() -> None:
|
||||
outcomes.append(registry.add_child("p1", "c1"))
|
||||
|
||||
def uninstaller() -> None:
|
||||
registry.uninstall("p1")
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=adder),
|
||||
threading.Thread(target=uninstaller),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# If add_child landed first: c1 is in the forward set, then
|
||||
# uninstall clears everything. End state: nothing.
|
||||
# If uninstall landed first: add_child sees no presence,
|
||||
# returns None, no entry added. End state: nothing.
|
||||
# Either way, the leak invariant holds: child set is empty or
|
||||
# parent is gone, never "child set populated but no presence".
|
||||
children = registry.children_of("p1")
|
||||
ui_present = registry.ui_for("p1") is not None
|
||||
if children:
|
||||
assert ui_present, "registry leaked: children set without presence"
|
||||
@@ -314,6 +314,46 @@ class TestCollectorSnapshot:
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["state"] == "running"
|
||||
|
||||
def test_apply_snapshot_state_change_does_not_carry_pending_approval_detail(self):
|
||||
"""Stage 3 cleanup — the snapshot-resync cluster_state event no
|
||||
longer piggybacks ``pending_approval_detail`` (the field is
|
||||
gone from cluster_state entirely). On reconnect the browser's
|
||||
bulk fetch — triggered by the ``activity_state="approval"``
|
||||
transition in the reducer — pulls the items directly from
|
||||
``ui.serialize_pending_approval_detail()`` via the dashboard
|
||||
endpoint."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "same", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
"type": "node_snapshot",
|
||||
"node_id": "node-a",
|
||||
"workstreams": [
|
||||
{
|
||||
"id": "ws1",
|
||||
"name": "same",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
}
|
||||
],
|
||||
"health": {},
|
||||
"aggregate": {},
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in event
|
||||
|
||||
def test_apply_snapshot_skips_empty_id_workstream(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
@@ -359,6 +399,36 @@ class TestCollectorDelta:
|
||||
# Verify in-memory state was updated
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["state"] == "running"
|
||||
|
||||
def test_apply_delta_ws_state_does_not_carry_pending_approval_detail(self):
|
||||
"""Stage 3 cleanup — ``cluster_state`` no longer carries the
|
||||
``pending_approval_detail`` piggyback. Approval items now arrive
|
||||
via bulk fetch on activity_state transition; verdicts via the
|
||||
explicit ``intent_verdict`` event class. Symmetric event flow,
|
||||
no piggyback to dedupe against."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
"type": "ws_state",
|
||||
"ws_id": "ws1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in event
|
||||
|
||||
def test_apply_delta_ws_created(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
@@ -404,6 +474,139 @@ class TestCollectorDelta:
|
||||
assert event["name"] == "new-name"
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["name"] == "new-name"
|
||||
|
||||
def test_apply_delta_intent_verdict_forwards_verbatim(self):
|
||||
"""Stage 3 Step 5 — node-emitted intent_verdict events flow
|
||||
through _apply_delta to cluster fan-out so coord adapters can
|
||||
re-emit as child_ws_intent_verdict on the parent's SSE."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.9,
|
||||
"recommendation": "approve",
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "intent_verdict", "ws_id": "ws1", "verdict": verdict},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "intent_verdict"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["verdict"] == verdict
|
||||
|
||||
def test_apply_delta_intent_verdict_drops_when_ws_id_missing(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "intent_verdict", "verdict": {}})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_delta_approval_resolved_forwards_verbatim(self):
|
||||
"""Stage 3 Step 5 — paired with intent_verdict; clears the
|
||||
coord tree's pending-approval pill in lockstep with the
|
||||
actual decision rather than waiting for the state-change
|
||||
piggyback."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "ws1",
|
||||
"approved": True,
|
||||
"feedback": "lgtm",
|
||||
"always": False,
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "approval_resolved"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["approved"] is True
|
||||
assert event["feedback"] == "lgtm"
|
||||
assert event["always"] is False
|
||||
|
||||
def test_apply_delta_approve_request_forwards_detail(self):
|
||||
"""Push path for the initial approval items — eliminates the
|
||||
bulk-fetch race that left the coord row stuck on a loading
|
||||
placeholder when the bulk fetch landed in the gap between
|
||||
_emit_state(ATTENTION) and approve_tools setting _pending_approval."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "approve_request", "ws_id": "ws1", "detail": detail},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "approve_request"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["detail"] == detail
|
||||
|
||||
def test_apply_delta_approve_request_drops_when_ws_id_missing(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "approve_request", "detail": {}})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_delta_approval_resolved_coerces_missing_fields(self):
|
||||
"""Defensive: ``approved`` / ``always`` / ``feedback`` may be
|
||||
omitted by older nodes mid-rolling-upgrade; collector coerces
|
||||
to safe defaults."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "approval_resolved", "ws_id": "ws1"})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["approved"] is False
|
||||
assert event["feedback"] == ""
|
||||
assert event["always"] is False
|
||||
|
||||
def test_apply_delta_health_changed(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Tests for the console's coordinator idle-cleanup thread helper.
|
||||
|
||||
The helper itself is a tiny loop wrapping ``mgr.close_idle``; the heavy
|
||||
lifting is in ``SessionManager.close_idle`` (covered in
|
||||
``test_session_manager.py``) and ``bulk_close_stale_orphans`` (covered
|
||||
in ``test_storage_sqlite.py``). These tests verify the glue:
|
||||
|
||||
- the helper runs an initial sweep BEFORE its first sleep (cold-start
|
||||
cleanup without blocking the lifespan),
|
||||
- the helper swallows exceptions so a transient DB blip can't kill the
|
||||
daemon thread,
|
||||
- the helper exits cleanly when ``stop_event`` is set.
|
||||
|
||||
The ``stop_event`` parameter is exclusively for tests — production
|
||||
callers pass ``None`` and the daemon runs for process lifetime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
from turnstone.console.server import _coord_idle_cleanup_thread
|
||||
|
||||
|
||||
class _StubMgr:
|
||||
def __init__(
|
||||
self, *, stop_event: threading.Event, expected_calls: int, raise_after: int = -1
|
||||
) -> None:
|
||||
self.calls: list[float] = []
|
||||
self.sleep_calls_at_each_close: list[int] = []
|
||||
self._stop_event = stop_event
|
||||
self._expected = expected_calls
|
||||
self._raise_after = raise_after
|
||||
self._sleep_count = 0
|
||||
|
||||
def close_idle(self, timeout_sec: float) -> list[str]:
|
||||
# Snapshot how many sleeps preceded this close — lets the
|
||||
# "initial sweep" test verify the first close_idle ran with
|
||||
# zero preceding sleeps.
|
||||
self.sleep_calls_at_each_close.append(self._sleep_count)
|
||||
self.calls.append(timeout_sec)
|
||||
try:
|
||||
if 0 <= self._raise_after < len(self.calls):
|
||||
raise RuntimeError("simulated DB blip")
|
||||
finally:
|
||||
# Set stop after the helper has been exercised enough,
|
||||
# regardless of whether this call raised.
|
||||
if len(self.calls) >= self._expected:
|
||||
self._stop_event.set()
|
||||
return []
|
||||
|
||||
def record_sleep(self, _seconds: float) -> None:
|
||||
self._sleep_count += 1
|
||||
|
||||
|
||||
def _run_until_done(mgr: _StubMgr, stop_event: threading.Event, timeout_sec: float) -> None:
|
||||
with patch("turnstone.console.server.time.sleep", mgr.record_sleep):
|
||||
thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(mgr, timeout_sec, stop_event),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
thread.join(timeout=2.0)
|
||||
assert not thread.is_alive(), "helper failed to exit on stop_event"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_runs_initial_sweep_before_sleep() -> None:
|
||||
"""The first close_idle call must happen BEFORE the first time.sleep —
|
||||
otherwise cold-start orphans wait one ``check_every`` interval (~30 min
|
||||
on default 2h timeout) for the first reap. Crucial because the
|
||||
lifespan no longer does a synchronous initial sweep."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert mgr.sleep_calls_at_each_close == [0], "first close_idle should run before any sleep"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_calls_close_idle_each_tick() -> None:
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=3)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert len(mgr.calls) == 3
|
||||
assert all(t == 120.0 for t in mgr.calls)
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_survives_close_idle_exceptions() -> None:
|
||||
"""A transient DB error must not kill the daemon thread — the next
|
||||
tick should still fire close_idle. Without the try/except, a single
|
||||
blip would silently leak orphans forever."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=4, raise_after=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
# All four calls must have fired despite calls 2-4 raising.
|
||||
assert len(mgr.calls) == 4
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_exits_cleanly_on_stop_event() -> None:
|
||||
"""The stop_event mechanism is the test contract; verify the thread
|
||||
actually exits when the event is set, without needing exceptions or
|
||||
daemon-process termination."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=2)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert stop_event.is_set()
|
||||
@@ -464,3 +464,148 @@ def test_coord_budget_override_survives_wildcard_allow_policy() -> None:
|
||||
assert approved is True
|
||||
types = [e.get("type") for e in captured_events]
|
||||
assert "approve_request" in types, "Wildcard allow must not strip the budget-override prompt"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cluster-bus broadcast hooks — _broadcast_intent_verdict / _approval_resolved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBroadcastIntentVerdict:
|
||||
"""``ConsoleCoordinatorUI._broadcast_intent_verdict`` overrides the
|
||||
no-op base hook to push the verdict onto the cluster bus via
|
||||
``ClusterCollector.emit_console_ws_intent_verdict``. The far more
|
||||
common path is the per-node ``WebUI`` override (covered in
|
||||
test_webui_content.py); this lights up the rare coord-self path
|
||||
(a coord that runs its own LLM judge).
|
||||
"""
|
||||
|
||||
def test_calls_collector_emit_with_ws_id_and_verdict(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "high",
|
||||
"confidence": 0.91,
|
||||
}
|
||||
ui._broadcast_intent_verdict(verdict)
|
||||
collector.emit_console_ws_intent_verdict.assert_called_once_with(
|
||||
"coord-a",
|
||||
verdict,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_intent_verdict.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise — collector failures are observational only.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
class TestBroadcastApprovalResolved:
|
||||
"""``ConsoleCoordinatorUI._broadcast_approval_resolved`` overrides
|
||||
the base hook to push the resolution onto the cluster bus via
|
||||
``ClusterCollector.emit_console_ws_approval_resolved``."""
|
||||
|
||||
def test_calls_collector_with_decision_fields(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
ui._broadcast_approval_resolved(True, "lgtm", always=True)
|
||||
collector.emit_console_ws_approval_resolved.assert_called_once_with(
|
||||
"coord-a",
|
||||
approved=True,
|
||||
feedback="lgtm",
|
||||
always=True,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_normalises_none_feedback_to_empty_string(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
ui._broadcast_approval_resolved(False, None)
|
||||
collector.emit_console_ws_approval_resolved.assert_called_once_with(
|
||||
"coord-a",
|
||||
approved=False,
|
||||
feedback="",
|
||||
always=False,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, None)
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_approval_resolved.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, "ok")
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
class TestBroadcastApproveRequest:
|
||||
"""Coord-side override for the approve_request push. Same rationale
|
||||
as the WebUI override — the coord-self path is rare today, but
|
||||
parity keeps the override symmetric with the rest of the broadcast
|
||||
family."""
|
||||
|
||||
def test_calls_collector_emit_with_ws_id_and_detail(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
ui._broadcast_approve_request(detail)
|
||||
collector.emit_console_ws_approve_request.assert_called_once_with(
|
||||
"coord-a",
|
||||
detail,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_approve_request.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
@@ -378,13 +378,21 @@ class TestCoordinatorAdapterWorkerDispatch:
|
||||
|
||||
|
||||
class TestCoordinatorAdapterChildrenRegistry:
|
||||
def test_emit_created_seeds_empty_children_set(self) -> None:
|
||||
"""Adapter-level integration with :class:`ChildrenRegistry`.
|
||||
|
||||
Pure-registry invariants (forward/reverse consistency, idempotent
|
||||
merge, locking) live in ``test_children_registry.py``. These
|
||||
tests cover the adapter's wiring: that ``emit_*`` paths drive the
|
||||
registry correctly and that the snapshot-priming bridge between
|
||||
a collector snapshot and the registry preserves merge semantics.
|
||||
"""
|
||||
|
||||
def test_emit_created_installs_parent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.emit_created(ws)
|
||||
assert ws.id in adapter._children
|
||||
assert adapter._children[ws.id] == set()
|
||||
assert adapter._active_coords[ws.id] is ws.ui
|
||||
assert adapter._registry.children_of(ws.id) == []
|
||||
assert adapter._registry.ui_for(ws.id) is ws.ui
|
||||
|
||||
def test_emit_rehydrated_calls_rebuild(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
@@ -398,42 +406,36 @@ class TestCoordinatorAdapterChildrenRegistry:
|
||||
adapter.emit_rehydrated(ws)
|
||||
assert calls == [ws.id]
|
||||
|
||||
def test_emit_closed_clears_forward_and_reverse_indexes(self) -> None:
|
||||
def test_emit_closed_uninstalls_parent_and_clears_children(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._merge_child_ids_locked("coord-b", ["child-b1"])
|
||||
adapter._active_coords["coord-a"] = object()
|
||||
adapter._active_coords["coord-b"] = object()
|
||||
adapter._registry.install("coord-a", object())
|
||||
adapter._registry.install("coord-b", object())
|
||||
adapter._registry.merge_children("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._registry.merge_children("coord-b", ["child-b1"])
|
||||
|
||||
adapter.emit_closed("coord-a")
|
||||
|
||||
assert "coord-a" not in adapter._children
|
||||
assert "coord-a" not in adapter._active_coords
|
||||
assert "child-a1" not in adapter._child_to_coord
|
||||
assert "child-a2" not in adapter._child_to_coord
|
||||
assert adapter._registry.ui_for("coord-a") is None
|
||||
assert adapter._registry.children_of("coord-a") == []
|
||||
assert adapter._registry.parent_for("child-a1") is None
|
||||
assert adapter._registry.parent_for("child-a2") is None
|
||||
# coord-b untouched
|
||||
assert adapter._child_to_coord["child-b1"] == "coord-b"
|
||||
assert "coord-b" in adapter._children
|
||||
|
||||
def test_merge_child_ids_locked_is_idempotent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
assert adapter._children["coord-a"] == {"child-1"}
|
||||
assert adapter._child_to_coord == {"child-1": "coord-a"}
|
||||
assert adapter._registry.parent_for("child-b1") == "coord-b"
|
||||
assert adapter._registry.ui_for("coord-b") is not None
|
||||
|
||||
def test_prime_children_from_snapshot_merges_without_overwriting(self) -> None:
|
||||
# Snapshot priming now lives on ClusterChildSource (production
|
||||
# path). The adapter no longer carries its own duplicate copy.
|
||||
from turnstone.core.child_source import ClusterChildSource
|
||||
|
||||
adapter, _ = _make_adapter()
|
||||
# Seed one in-memory coord + one existing child
|
||||
coord_ws = _make_ws()
|
||||
coord_ws.id = "coord-a"
|
||||
mgr = MagicMock()
|
||||
mgr.list_all.return_value = [coord_ws]
|
||||
adapter.attach(mgr)
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
|
||||
source = ClusterChildSource(
|
||||
collector=MagicMock(),
|
||||
registry=adapter._registry,
|
||||
parents_provider=lambda: ["coord-a"],
|
||||
)
|
||||
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
@@ -448,10 +450,13 @@ class TestCoordinatorAdapterChildrenRegistry:
|
||||
},
|
||||
],
|
||||
}
|
||||
adapter._prime_children_from_snapshot(snapshot)
|
||||
assert adapter._children["coord-a"] == {"child-a1", "child-a2"}
|
||||
assert adapter._child_to_coord["child-a2"] == "coord-a"
|
||||
assert "child-x" not in adapter._child_to_coord
|
||||
source._prime_from_snapshot(snapshot)
|
||||
assert set(adapter._registry.children_of("coord-a")) == {
|
||||
"child-a1",
|
||||
"child-a2",
|
||||
}
|
||||
assert adapter._registry.parent_for("child-a2") == "coord-a"
|
||||
assert adapter._registry.parent_for("child-x") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -478,9 +483,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
coord_ws.id = coord_id
|
||||
recorder = _UIRecorder()
|
||||
coord_ws.ui = recorder # type: ignore[assignment]
|
||||
with adapter._children_lock:
|
||||
adapter._children.setdefault(coord_id, set())
|
||||
adapter._active_coords[coord_id] = recorder
|
||||
adapter._registry.install(coord_id, recorder)
|
||||
adapter.attach(_StubManager(coord_ws)) # type: ignore[arg-type]
|
||||
return adapter, recorder, coord_ws
|
||||
|
||||
@@ -510,12 +513,11 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
# Reverse index updated for subsequent cluster_state events.
|
||||
assert adapter._child_to_coord["child-a1"] == "coord-a"
|
||||
assert adapter._registry.parent_for("child-a1") == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_routes_via_reverse_index(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -533,8 +535,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
|
||||
def test_dispatch_ws_closed_routes_to_parent_coord(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{"type": "ws_closed", "ws_id": "child-a1", "reason": "evicted"}
|
||||
)
|
||||
@@ -548,8 +549,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
"""perf-6: _enqueue_on_ui mutates the payload dict in place with
|
||||
the coord's ws_id so the browser can discriminate child events."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -558,3 +558,161 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
}
|
||||
)
|
||||
assert recorder.enqueued[0]["ws_id"] == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_does_not_carry_pending_approval_detail(
|
||||
self,
|
||||
) -> None:
|
||||
"""Stage 3 cleanup — the ``pending_approval_detail`` piggyback
|
||||
on ``cluster_state`` is gone. Approval items now arrive via
|
||||
bulk fetch (triggered by ``activity_state="approval"`` in the
|
||||
browser); verdicts via ``child_ws_intent_verdict``; resolution
|
||||
via ``child_ws_approval_resolved``. The state event carries
|
||||
only state + activity_state — no detail field."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_state"
|
||||
assert payload["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in payload
|
||||
|
||||
def test_dispatch_intent_verdict_emits_child_ws_intent_verdict(self) -> None:
|
||||
"""Stage 3 Step 6 — explicit verdict events are re-emitted as
|
||||
child_ws_intent_verdict on the parent's SSE so the tree UI
|
||||
renders the risk pill without polling."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.92,
|
||||
"recommendation": "approve",
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_intent_verdict"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["node_id"] == "node-1"
|
||||
assert payload["verdict"] == verdict
|
||||
|
||||
def test_dispatch_intent_verdict_unknown_child_drops(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": "ws-orphan",
|
||||
"verdict": {"call_id": "c1"},
|
||||
}
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_approval_resolved_emits_child_ws_approval_resolved(
|
||||
self,
|
||||
) -> None:
|
||||
"""Stage 3 Step 6 — paired with intent_verdict; clears the
|
||||
pending-approval pill on the parent's tree UI in lockstep
|
||||
with the actual decision."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"approved": True,
|
||||
"feedback": "lgtm",
|
||||
"always": False,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_approval_resolved"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["approved"] is True
|
||||
assert payload["feedback"] == "lgtm"
|
||||
assert payload["always"] is False
|
||||
|
||||
def test_dispatch_approval_resolved_coerces_missing_fields(self) -> None:
|
||||
"""Older nodes mid-rolling-upgrade may omit approved / always /
|
||||
feedback; dispatch coerces to safe defaults."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event({"type": "approval_resolved", "ws_id": "child-a1"})
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["approved"] is False
|
||||
assert payload["feedback"] == ""
|
||||
assert payload["always"] is False
|
||||
|
||||
def test_dispatch_approval_resolved_unknown_child_drops(self) -> None:
|
||||
"""Symmetric to the intent_verdict drop test — events for
|
||||
ws_ids the registry doesn't know about silently drop instead
|
||||
of fanning out to a parent that has no business seeing them."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "ws-orphan",
|
||||
"approved": True,
|
||||
},
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_approve_request_emits_child_ws_approve_request(
|
||||
self,
|
||||
) -> None:
|
||||
"""Push path for the initial approval items — eliminates the
|
||||
bulk-fetch race that left the coord row stuck on a loading
|
||||
placeholder when the bulk fetch landed in the gap between
|
||||
_emit_state(ATTENTION) and approve_tools setting _pending_approval."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"detail": detail,
|
||||
},
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_approve_request"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["node_id"] == "node-1"
|
||||
assert payload["detail"] == detail
|
||||
|
||||
def test_dispatch_approve_request_unknown_child_drops(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": "ws-orphan",
|
||||
"detail": {"items": []},
|
||||
},
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
+134
-16
@@ -79,9 +79,10 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
assert "function submitChildApproval" in body or "submitChildApproval(" in body
|
||||
# The shared approve POST helper (parameterized for child ws_ids)
|
||||
assert "function approveWorkstream" in body or "approveWorkstream(" in body
|
||||
# The urgent live-bulk fetch option that fires on activity_state
|
||||
# transitions in/out of "approval"
|
||||
assert "{ urgent: true }" in body or "urgent: true" in body
|
||||
# The 409 stale-call_id retry path uses invalidateLiveBadge +
|
||||
# scheduleLiveFetch (Stage 3 cleanup removed the urgent flag —
|
||||
# cache invalidation makes the TTL gate fall through naturally).
|
||||
assert "invalidateLiveBadge(targetWsId)" in body
|
||||
# Server-side payload field — drift here means the JS reads stale keys
|
||||
assert "pending_approval_detail" in body
|
||||
# Reconnect parity (chunk 4): the SSE re-open handler must drop
|
||||
@@ -90,10 +91,10 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# can't render zombie approve/deny buttons on a row whose
|
||||
# approval was resolved during the gap. The implementation
|
||||
# iterates the cache and deletes only !permanent entries —
|
||||
# asserting the literal Map iteration form keeps a refactor
|
||||
# back to liveBadgeCache.clear() (which would re-pay 403s on
|
||||
# every reconnect for denied ids) from sneaking in.
|
||||
assert "liveBadgeCache.delete" in body
|
||||
# asserting the literal helper call keeps a refactor back to
|
||||
# _liveBadgeCacheClear() (which would re-pay 403s on every
|
||||
# reconnect for denied ids) from sneaking in.
|
||||
assert "_liveBadgeCacheDelete" in body
|
||||
# Edge-case matrix sentinel labels — POLICY-BLOCKED renders when
|
||||
# an item has error set + needs_approval=False (server-side
|
||||
# tool policy already blocked the call); "(judge unavailable)"
|
||||
@@ -113,15 +114,19 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# coord-self ws_id (the coord lives on the console process).
|
||||
# Children live on cluster nodes and 404 without the prefix.
|
||||
assert "/v1/api/route/workstreams/" in body
|
||||
# Late-judge polling — the LLM judge runs async on the child
|
||||
# node and never pushes a signal that reaches the coord, so
|
||||
# the row's pending_approval_detail with judge_pending=true
|
||||
# would freeze on heuristic verdicts forever without this
|
||||
# poll loop. The poller is GLOBAL (not per-row) so off-screen
|
||||
# rows still refresh — a per-row poller's scheduleLiveFetch
|
||||
# call short-circuits on non-visible rows, leaving them stuck.
|
||||
assert "_maybeStartJudgePoll" in body
|
||||
assert "_judgePollTick" in body
|
||||
# Late-arriving LLM judge verdicts — Stage 3 Step 5 promoted
|
||||
# ``intent_verdict`` and ``approval_resolved`` to first-class
|
||||
# cluster-bus event types, so the coord adapter dispatches them
|
||||
# as ``child_ws_intent_verdict`` / ``child_ws_approval_resolved``
|
||||
# on the parent's SSE stream. The browser handlers write
|
||||
# directly to liveBadgeCache (bypassing scheduleLiveFetch's
|
||||
# visibility gate cleanly) so off-screen rows pick up verdicts
|
||||
# without polling. Replaced the old ``_judgePollTick`` 90-second
|
||||
# global poll loop and its visibility-gate-bypass workaround.
|
||||
assert "handleChildIntentVerdict" in body
|
||||
assert "handleChildApprovalResolved" in body
|
||||
assert "child_ws_intent_verdict" in body
|
||||
assert "child_ws_approval_resolved" in body
|
||||
# Reload parity for the coord-self approval gate: init() must
|
||||
# consume the authoritative GET /workstreams snapshot's
|
||||
# pending_approval_detail so a freshly opened tab can render
|
||||
@@ -152,3 +157,116 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# any prior denial. bug-1 / bug-3 from the second /review pass.
|
||||
assert "Denied by user" in body
|
||||
assert "callOutcomes" in body
|
||||
|
||||
|
||||
def test_coordinator_js_handle_child_state_no_longer_reads_sse_pending_approval_detail():
|
||||
"""Stage 3 cleanup — ``pending_approval_detail`` is no longer
|
||||
piggybacked on child_ws_state events. Approval items now arrive
|
||||
via bulk fetch on the activity_state="approval" transition;
|
||||
verdicts via the explicit ``child_ws_intent_verdict`` event class;
|
||||
resolution via ``child_ws_approval_resolved``. A refactor that
|
||||
re-introduces the piggyback would silently re-open the
|
||||
duplicate-path race the dedicated event classes were added to
|
||||
eliminate.
|
||||
|
||||
Structural assertions (regex against multi-line source) — symbol-
|
||||
presence alone wouldn't catch a guard that keeps the names but
|
||||
inverts the comparison or drops the ``prev.live`` check. This
|
||||
codebase has no JS test framework, so locking the guard's shape
|
||||
here is the next-best thing to a behavioral test."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
coord_js = Path(__file__).resolve().parent.parent / (
|
||||
"turnstone/console/static/coordinator/coordinator.js"
|
||||
)
|
||||
body = coord_js.read_text(encoding="utf-8")
|
||||
|
||||
# The piggyback read is gone from handleChildState. (The string
|
||||
# may still appear elsewhere — e.g. handleChildIntentVerdict
|
||||
# reading from cache, or comments — but never as ``ev.pending_approval_detail``.)
|
||||
assert "ev.pending_approval_detail" not in body
|
||||
# The pre-fix urgent-fetch on activity_state transitions is gone.
|
||||
assert "enteredApproval" not in body
|
||||
assert "leftApproval" not in body
|
||||
# ``pendingApproval`` flag derivation must check BOTH state and
|
||||
# activity_state. The worker thread can fire the state transition
|
||||
# to "attention" before approve_tools updates activity_state, so
|
||||
# checking only activity_state misses children that legitimately
|
||||
# need approval. Pin the disjunction so the regression doesn't
|
||||
# silently re-introduce.
|
||||
assert re.search(
|
||||
r'existing\.state\s*===\s*"attention"\s*\|\|\s*'
|
||||
r'existing\.activity_state\s*===\s*"approval"',
|
||||
body,
|
||||
), (
|
||||
"handleChildState must derive pendingApproval from "
|
||||
"(state==='attention' || activity_state==='approval')"
|
||||
)
|
||||
|
||||
# SSE-authoritative window constant is defined and used.
|
||||
assert re.search(r"\bconst\s+SSE_AUTHORITATIVE_MS\s*=\s*\d+", body), (
|
||||
"SSE_AUTHORITATIVE_MS constant must be defined as a numeric literal"
|
||||
)
|
||||
|
||||
# SSE writers tag entries with sseUpdatedAt: Date.now() so the
|
||||
# merge guard in flushLiveFetches preserves them against stale
|
||||
# bulk-fetch responses. handleChildState only stamps when it
|
||||
# AUTHORITATIVELY clears the detail (off-approval transition);
|
||||
# writers that stamp unconditionally are intent_verdict (verdict
|
||||
# stamp), approval_resolved (clear), and the optimistic-clear
|
||||
# path in submitChildApproval. Pinning the literal Date.now()
|
||||
# call keeps a refactor that drops the SSE-source tag entirely
|
||||
# from sneaking in.
|
||||
assert re.search(
|
||||
r"sseUpdatedAt:\s*Date\.now\(\)",
|
||||
body,
|
||||
), "Critical SSE writers must stamp sseUpdatedAt: Date.now()"
|
||||
|
||||
# flushLiveFetches' merge guard structure: SSE-set pending_approval
|
||||
# / _detail wins over a stale bulk-poll snapshot when (live) AND
|
||||
# (prev exists) AND (prev.sseUpdatedAt set) AND (within window)
|
||||
# AND (prev.live exists). Inverting the comparison or dropping
|
||||
# any of these guards reopens the clobber bug.
|
||||
merge_guard = re.search(
|
||||
r"if\s*\(\s*live\s*&&\s*prev\s*&&\s*prev\.sseUpdatedAt\s*&&\s*"
|
||||
r"now\s*-\s*prev\.sseUpdatedAt\s*<\s*SSE_AUTHORITATIVE_MS\s*&&\s*"
|
||||
r"prev\.live\s*\)",
|
||||
body,
|
||||
)
|
||||
assert merge_guard is not None, (
|
||||
"flushLiveFetches merge guard must be the conjunction "
|
||||
"(live && prev && prev.sseUpdatedAt && now - prev.sseUpdatedAt < "
|
||||
"SSE_AUTHORITATIVE_MS && prev.live). An inverted comparison or "
|
||||
"missing prev.live check would let a stale bulk-poll clobber a "
|
||||
"fresh SSE-set approval."
|
||||
)
|
||||
|
||||
# The merge body must preserve BOTH pending_approval and
|
||||
# pending_approval_detail from prev — preserving only one would
|
||||
# render a row with a phantom badge but no buttons (or vice versa).
|
||||
merge_body = re.search(
|
||||
r"mergedLive\s*=\s*Object\.assign\(\s*\{\}\s*,\s*live\s*,\s*\{"
|
||||
r"[^}]*pending_approval:\s*prev\.live\.pending_approval[^}]*"
|
||||
r"pending_approval_detail:\s*prev\.live\.pending_approval_detail",
|
||||
body,
|
||||
)
|
||||
assert merge_body is not None, (
|
||||
"Merge body must preserve both pending_approval AND "
|
||||
"pending_approval_detail from prev.live — preserving only one "
|
||||
"creates a half-rendered approval row."
|
||||
)
|
||||
|
||||
# flushLiveFetches must forward sseUpdatedAt onto the new cache
|
||||
# entry so the SSE-source tag survives the bulk-poll write back —
|
||||
# without this, every bulk-poll resets the window and the next
|
||||
# late-arriving poll silently clobbers.
|
||||
assert re.search(
|
||||
r"sseUpdatedAt:\s*prev\s*\?\s*prev\.sseUpdatedAt",
|
||||
body,
|
||||
), (
|
||||
"flushLiveFetches must forward prev.sseUpdatedAt onto the new "
|
||||
"cache entry (preserving the SSE-source window across bulk-poll "
|
||||
"cycles) — without this, the second bulk-poll after an SSE "
|
||||
"transition silently clobbers."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Unit tests for ``turnstone.core.history_decoration``.
|
||||
|
||||
The decoration helpers are shared between two surfaces — interactive's
|
||||
SSE replay (``_build_history``) and the lifted ``/history`` REST
|
||||
endpoint (``make_history_handler``, used by both interactive and
|
||||
coord). Pinning the wire shape here lets a future schema/projection
|
||||
change land in one file rather than spread across the two surfaces.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from turnstone.core.history_decoration import (
|
||||
build_output_assessment_payload,
|
||||
build_verdict_payload,
|
||||
decorate_history_messages,
|
||||
decorate_tool_call,
|
||||
)
|
||||
|
||||
|
||||
class TestBuildVerdictPayload:
|
||||
"""The wire-shape projection that's the single source of truth for
|
||||
what intent_verdict fields ship to the client."""
|
||||
|
||||
def test_skips_unflagged_baseline(self) -> None:
|
||||
"""``risk_level`` "none" is the unflagged-tool baseline; the
|
||||
client filters those anyway, so projecting None at the wire
|
||||
layer keeps the payload tight on long workstreams."""
|
||||
row = {"risk_level": "none", "recommendation": "approve", "tier": "heuristic"}
|
||||
assert build_verdict_payload(row) is None
|
||||
|
||||
def test_drops_call_id_and_func_name(self) -> None:
|
||||
"""The client already has these on ``tc.id`` / ``tc.name``;
|
||||
re-shipping them per-tool_call would balloon long replays."""
|
||||
row = {
|
||||
"call_id": "call_abc",
|
||||
"func_name": "bash",
|
||||
"risk_level": "medium",
|
||||
"recommendation": "review",
|
||||
"confidence": 0.8,
|
||||
"intent_summary": "summary",
|
||||
"tier": "heuristic",
|
||||
}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "call_id" not in out
|
||||
assert "func_name" not in out
|
||||
# Sanity — the kept fields are the ones renderVerdictBadge reads.
|
||||
assert out["risk_level"] == "medium"
|
||||
assert out["recommendation"] == "review"
|
||||
assert out["confidence"] == 0.8
|
||||
assert out["intent_summary"] == "summary"
|
||||
assert out["tier"] == "heuristic"
|
||||
|
||||
def test_includes_reasoning_for_either_tier_when_present(self) -> None:
|
||||
"""Heuristic verdicts in this project emit structured
|
||||
rationales (one per matched pattern) — e.g.
|
||||
``policy.py`` writes a reasoning string per heuristic hit.
|
||||
Ship the field for either tier when it has content; only
|
||||
omit when the row didn't write one."""
|
||||
for tier in ("heuristic", "llm"):
|
||||
row = {
|
||||
"risk_level": "high",
|
||||
"tier": tier,
|
||||
"reasoning": "The command exfiltrates ~/.ssh/id_rsa over an external connection.",
|
||||
}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "id_rsa" in out["reasoning"]
|
||||
|
||||
def test_omits_reasoning_when_empty(self) -> None:
|
||||
"""An absent / empty reasoning string shouldn't ship as
|
||||
``reasoning: ""`` — the rationale ``<details>`` block on the
|
||||
client renders an empty disclosure when the field is present
|
||||
but empty."""
|
||||
row = {"risk_level": "high", "tier": "heuristic", "reasoning": ""}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "reasoning" not in out
|
||||
|
||||
def test_includes_judge_model_when_present(self) -> None:
|
||||
"""``judge_model`` rides through so the batch tier badge can
|
||||
render ``⚖ llm:claude-haiku-4`` on history-only replays
|
||||
rather than the bare ``⚖ llm`` label."""
|
||||
row = {"risk_level": "high", "tier": "llm", "judge_model": "claude-haiku-4"}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert out["judge_model"] == "claude-haiku-4"
|
||||
|
||||
def test_omits_judge_model_when_empty(self) -> None:
|
||||
row = {"risk_level": "medium", "tier": "heuristic", "judge_model": ""}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "judge_model" not in out
|
||||
|
||||
|
||||
class TestBuildOutputAssessmentPayload:
|
||||
"""Output-guard wire shape — flags decoded from JSON string at
|
||||
this layer so the client never has to parse twice."""
|
||||
|
||||
def test_skips_unflagged_baseline(self) -> None:
|
||||
row = {"risk_level": "none", "flags": "[]"}
|
||||
assert build_output_assessment_payload(row) is None
|
||||
|
||||
def test_decodes_flags_from_json(self) -> None:
|
||||
row = {"risk_level": "high", "flags": '["api_key","email"]', "redacted": 1}
|
||||
out = build_output_assessment_payload(row)
|
||||
assert out is not None
|
||||
assert out["flags"] == ["api_key", "email"]
|
||||
assert out["redacted"] is True
|
||||
assert out["risk_level"] == "high"
|
||||
|
||||
def test_handles_malformed_flags_json(self) -> None:
|
||||
"""Bad JSON in ``flags`` must not block the rest of the
|
||||
assessment from rendering — degrade to empty list."""
|
||||
row = {"risk_level": "medium", "flags": "not-json", "redacted": 0}
|
||||
out = build_output_assessment_payload(row)
|
||||
assert out is not None
|
||||
assert out["flags"] == []
|
||||
assert out["redacted"] is False
|
||||
|
||||
|
||||
class TestDecorateToolCall:
|
||||
"""In-place mutation of either OpenAI-format or flattened tool_call
|
||||
entries — both shapes carry ``id`` at the top level."""
|
||||
|
||||
def test_attaches_verdict_when_present(self) -> None:
|
||||
tc: dict[str, object] = {"id": "call_1", "function": {"name": "bash", "arguments": "{}"}}
|
||||
verdicts = {
|
||||
"call_1": {
|
||||
"risk_level": "medium",
|
||||
"recommendation": "review",
|
||||
"confidence": 0.7,
|
||||
"intent_summary": "summary",
|
||||
"tier": "heuristic",
|
||||
}
|
||||
}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" in tc
|
||||
assert tc["verdict"]["risk_level"] == "medium" # type: ignore[index]
|
||||
|
||||
def test_skips_when_no_call_id_match(self) -> None:
|
||||
tc: dict[str, object] = {"id": "call_other", "name": "bash"}
|
||||
verdicts = {
|
||||
"call_1": {"risk_level": "medium", "tier": "heuristic"},
|
||||
}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
def test_skips_unflagged_verdict(self) -> None:
|
||||
"""``build_verdict_payload`` returns None for unflagged rows;
|
||||
decorate_tool_call must not stamp ``verdict`` in that case."""
|
||||
tc: dict[str, object] = {"id": "call_1", "name": "bash"}
|
||||
verdicts = {"call_1": {"risk_level": "none", "tier": "heuristic"}}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
def test_handles_empty_id(self) -> None:
|
||||
"""A tool_call with no id can't be paired against the lookup
|
||||
table — must not raise (or stamp the wrong row's verdict)."""
|
||||
tc: dict[str, object] = {"id": "", "name": "bash"}
|
||||
verdicts = {"call_1": {"risk_level": "high", "tier": "heuristic"}}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
|
||||
class TestDecorateHistoryMessages:
|
||||
"""End-to-end mutation of a /history-shaped message list — covers
|
||||
the full transform applied by ``make_history_handler``."""
|
||||
|
||||
def test_decorates_tool_calls_and_marks_truncated(self) -> None:
|
||||
verdicts = {
|
||||
"call_a": {
|
||||
"risk_level": "high",
|
||||
"recommendation": "deny",
|
||||
"confidence": 0.95,
|
||||
"intent_summary": "exfil",
|
||||
"tier": "llm",
|
||||
"reasoning": "ssh key access",
|
||||
}
|
||||
}
|
||||
assessments = {
|
||||
"call_a": {"risk_level": "high", "flags": '["secret"]', "redacted": 1},
|
||||
}
|
||||
# Tool result content of exactly TOOL_RESULT_STORAGE_CAP chars
|
||||
# hits the storage cap (longer is impossible — storage clamps
|
||||
# at the cap). Reference the constant rather than a literal so
|
||||
# this test stays correct if the cap moves again.
|
||||
from turnstone.core.history_decoration import TOOL_RESULT_STORAGE_CAP
|
||||
|
||||
truncated_content = "x" * TOOL_RESULT_STORAGE_CAP
|
||||
messages: list[dict[str, object]] = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "running",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_a",
|
||||
"function": {"name": "bash", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_a", "content": truncated_content},
|
||||
{"role": "tool", "tool_call_id": "call_b", "content": "short"},
|
||||
]
|
||||
decorate_history_messages(messages, verdicts, assessments)
|
||||
# Assistant tool_calls got both decorations.
|
||||
tc = messages[1]["tool_calls"][0] # type: ignore[index]
|
||||
assert tc["verdict"]["risk_level"] == "high"
|
||||
assert tc["verdict"]["tier"] == "llm"
|
||||
assert "reasoning" in tc["verdict"]
|
||||
assert tc["output_assessment"]["flags"] == ["secret"]
|
||||
assert tc["output_assessment"]["redacted"] is True
|
||||
# Truncated tool message got the flag; the short one did not.
|
||||
assert messages[2].get("truncated") is True
|
||||
assert "truncated" not in messages[3]
|
||||
|
||||
def test_no_op_on_empty_indexes(self) -> None:
|
||||
"""When neither table has rows for the workstream, the wire
|
||||
shape passes through unchanged — replay must degrade
|
||||
gracefully when verdict storage is empty / unavailable."""
|
||||
messages: list[dict[str, object]] = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "call_a", "function": {"name": "bash", "arguments": "{}"}}],
|
||||
},
|
||||
]
|
||||
decorate_history_messages(messages, {}, {})
|
||||
tc = messages[0]["tool_calls"][0] # type: ignore[index]
|
||||
assert "verdict" not in tc
|
||||
assert "output_assessment" not in tc
|
||||
@@ -8,6 +8,7 @@ from turnstone.core.metacognition import (
|
||||
NUDGE_RESUME,
|
||||
NUDGE_START,
|
||||
NUDGE_TOOL_ERROR,
|
||||
RepeatDetector,
|
||||
detect_completion,
|
||||
detect_correction,
|
||||
format_nudge,
|
||||
@@ -308,3 +309,70 @@ class TestRepeatNudge:
|
||||
"""Repeat nudge should fire even with zero memories."""
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("repeat", state, message_count=5, memory_count=0) is True
|
||||
|
||||
|
||||
class TestRepeatDetector:
|
||||
"""Repeat-detection streak machine — fires only when the same signature
|
||||
is recorded ``threshold`` times *consecutively* (default 3). Recording
|
||||
any different signature resets the streak, so an interrupted repeat
|
||||
isn't flagged as a stuck loop."""
|
||||
|
||||
def test_below_threshold_does_not_fire(self):
|
||||
det = RepeatDetector()
|
||||
assert det.record("a") is False
|
||||
assert det.record("a") is False # second call still under threshold
|
||||
|
||||
def test_at_threshold_fires(self):
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_continues_to_fire_past_threshold(self):
|
||||
# Caller is responsible for clearing after a fire — until they do,
|
||||
# subsequent identical calls keep returning True.
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("a") is True
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_clear_resets_count(self):
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
det.clear()
|
||||
assert det.record("a") is False # back to 1 after clear
|
||||
|
||||
def test_intervening_sig_resets_streak(self):
|
||||
# The streak is consecutive: recording any other sig mid-streak
|
||||
# discards the in-progress count. An alternating pattern like
|
||||
# [A, A, B, A, A] is two short streaks of 2, not a streak of 4.
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("b") is False # b at count 1; a's streak is gone
|
||||
assert det.record("a") is False # a starts fresh at 1
|
||||
assert det.record("a") is False # a at 2
|
||||
assert det.record("a") is True # a hits 3 — fresh streak completes
|
||||
|
||||
def test_errored_signature_counts_toward_repeat(self):
|
||||
# Regression: when metacog was split out of the system message,
|
||||
# the error-output skip got reintroduced and stuck-loop detection
|
||||
# silently broke for tools that kept failing. Detector itself is
|
||||
# signature-only — error vs. success is the caller's policy.
|
||||
det = RepeatDetector()
|
||||
# Caller records an errored call's sig the same as a successful one;
|
||||
# the streak is what matters.
|
||||
for _ in range(3):
|
||||
last = det.record("bash:ls /nonexistent")
|
||||
assert last is True
|
||||
|
||||
def test_custom_threshold(self):
|
||||
det = RepeatDetector(threshold=2)
|
||||
assert det.record("a") is False
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_threshold_one_fires_immediately(self):
|
||||
det = RepeatDetector(threshold=1)
|
||||
assert det.record("a") is True
|
||||
|
||||
@@ -770,16 +770,76 @@ class TestRegistryReload:
|
||||
assert reg.has_alias("b")
|
||||
assert reg.default == "b"
|
||||
|
||||
def test_reload_clears_clients(self) -> None:
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m")}
|
||||
def test_reload_keeps_clients_when_connection_target_unchanged(self) -> None:
|
||||
"""Selective teardown: a model edit that leaves base_url / api_key /
|
||||
provider intact (e.g. admin tweaks the underlying ``model`` name or
|
||||
``temperature``) keeps the cached HTTP client warm — no need to
|
||||
re-establish TLS+pool when the endpoint is the same."""
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m1", provider="openai")}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
# Force client creation
|
||||
reg.get_client("a")
|
||||
assert "a" in reg._clients
|
||||
client_before = reg._clients["a"]
|
||||
provider_before = reg.get_provider("a")
|
||||
|
||||
# Same endpoint (base_url, api_key, provider), only ``model`` changed.
|
||||
new_models = {"a": ModelConfig("a", "http://x/v1", "key", "m2", provider="openai")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
assert "a" in reg._clients
|
||||
assert reg._clients["a"] is client_before
|
||||
assert "a" in reg._providers
|
||||
assert reg._providers["a"] is provider_before
|
||||
|
||||
def test_reload_drops_client_when_base_url_changes(self) -> None:
|
||||
"""A ``base_url`` change drops the cached client (different
|
||||
endpoint = new connection) but keeps the cached provider —
|
||||
``LLMProvider`` is keyed only on the provider string, which
|
||||
didn't change."""
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m", provider="openai")}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
reg.get_client("a")
|
||||
provider_before = reg.get_provider("a")
|
||||
|
||||
new_models = {"a": ModelConfig("a", "http://y/v1", "key", "m", provider="openai")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
# Reload with same models — clients should be cleared
|
||||
reg.reload(dict(models), "a")
|
||||
assert "a" not in reg._clients
|
||||
assert "a" in reg._providers
|
||||
assert reg._providers["a"] is provider_before
|
||||
|
||||
def test_reload_drops_provider_when_provider_string_changes(self) -> None:
|
||||
"""A provider-type swap (e.g. openai → anthropic) drops both the
|
||||
client AND the provider so the next resolve picks up the right
|
||||
``LLMProvider`` implementation against the new SDK."""
|
||||
models = {"a": ModelConfig("a", "http://x/v1", "key", "m", provider="openai")}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
reg.get_client("a")
|
||||
reg.get_provider("a")
|
||||
|
||||
new_models = {"a": ModelConfig("a", "http://x/v1", "key", "m", provider="anthropic")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
assert "a" not in reg._clients
|
||||
assert "a" not in reg._providers
|
||||
|
||||
def test_reload_drops_clients_for_removed_aliases(self) -> None:
|
||||
"""Aliases removed from the registry must release their cached
|
||||
clients — otherwise a deleted endpoint's connection pool would
|
||||
outlive the alias indefinitely."""
|
||||
models = {
|
||||
"a": ModelConfig("a", "http://x/v1", "key", "m"),
|
||||
"b": ModelConfig("b", "http://y/v1", "key", "m"),
|
||||
}
|
||||
reg = ModelRegistry(models=models, default="a")
|
||||
reg.get_client("a")
|
||||
reg.get_client("b")
|
||||
|
||||
# Drop "b" entirely.
|
||||
new_models = {"a": ModelConfig("a", "http://x/v1", "key", "m")}
|
||||
reg.reload(new_models, "a")
|
||||
|
||||
assert "a" in reg._clients # unchanged endpoint, kept warm
|
||||
assert "b" not in reg._clients
|
||||
|
||||
def test_reload_validates_default(self) -> None:
|
||||
models_a = {"a": ModelConfig("a", "x", "x", "m")}
|
||||
|
||||
@@ -224,6 +224,25 @@ class TestOpenAIProvider:
|
||||
sanitize_messages([original])
|
||||
assert original["content"] is None
|
||||
|
||||
def test_sanitize_messages_strips_underscore_sibling_keys(self) -> None:
|
||||
"""Internal sibling metadata (``_reminders``, ``_reminders_delivered``,
|
||||
``_attachments_meta``, ``_provider_content``) must be stripped
|
||||
before the wire — the OpenAI-compat APIs reject unknown fields."""
|
||||
msgs = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi",
|
||||
"_reminders": [{"type": "correction", "text": "watch"}],
|
||||
"_reminders_delivered": True,
|
||||
"_attachments_meta": [{"kind": "image"}],
|
||||
}
|
||||
]
|
||||
result = sanitize_messages(msgs)
|
||||
assert result == [{"role": "user", "content": "hi"}]
|
||||
assert "_reminders" not in result[0]
|
||||
assert "_reminders_delivered" not in result[0]
|
||||
assert "_attachments_meta" not in result[0]
|
||||
|
||||
# -- sanitize_messages: orphan detection -----------------------------------
|
||||
|
||||
def test_sanitize_orphaned_tool_call_synthesized(self) -> None:
|
||||
|
||||
+837
-114
File diff suppressed because it is too large
Load Diff
@@ -18,13 +18,22 @@ from __future__ import annotations
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.session_manager import SessionKindAdapter, SessionManager
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
from turnstone.core.workstream import (
|
||||
BULK_CLOSE_STATE_VALUES,
|
||||
Workstream,
|
||||
WorkstreamKind,
|
||||
WorkstreamState,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test fixtures
|
||||
@@ -156,6 +165,8 @@ class _Row:
|
||||
kind: str
|
||||
state: str = "idle"
|
||||
parent_ws_id: str | None = None
|
||||
updated: str = ""
|
||||
node_id: str | None = None
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
@@ -164,8 +175,19 @@ class FakeStorage:
|
||||
def __init__(self) -> None:
|
||||
self.rows: dict[str, _Row] = {}
|
||||
self.state_updates: list[tuple[str, str]] = []
|
||||
self.touch_calls: list[str] = []
|
||||
self.register_raises = False
|
||||
self.lock = threading.Lock()
|
||||
# Live-services lookup target for close_idle pass 2. Map
|
||||
# service_type → list of live service_ids. Tests that exercise
|
||||
# liveness scoping populate this directly; default empty means
|
||||
# "no peers alive" (every row unprotected by liveness).
|
||||
self.live_services: dict[str, list[str]] = {}
|
||||
self.list_services_raises = False
|
||||
|
||||
@staticmethod
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
def register_workstream(
|
||||
self,
|
||||
@@ -178,6 +200,8 @@ class FakeStorage:
|
||||
parent_ws_id: str | None = None,
|
||||
skill_id: str = "",
|
||||
skill_version: int = 0,
|
||||
state: str = "idle",
|
||||
updated: str | None = None,
|
||||
) -> None:
|
||||
if self.register_raises:
|
||||
raise RuntimeError("register forced failure")
|
||||
@@ -188,14 +212,66 @@ class FakeStorage:
|
||||
user_id=user_id or "",
|
||||
name=name,
|
||||
kind=kind_str,
|
||||
state=state,
|
||||
parent_ws_id=parent_ws_id,
|
||||
updated=updated if updated is not None else self._now_iso(),
|
||||
node_id=node_id,
|
||||
)
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
with self.lock:
|
||||
self.touch_calls.append(ws_id)
|
||||
if ws_id in self.rows:
|
||||
self.rows[ws_id].updated = self._now_iso()
|
||||
|
||||
def update_workstream_state(self, ws_id: str, state: str) -> None:
|
||||
with self.lock:
|
||||
self.state_updates.append((ws_id, state))
|
||||
if ws_id in self.rows:
|
||||
self.rows[ws_id].state = state
|
||||
self.rows[ws_id].updated = self._now_iso()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
kind_str = kind.value if isinstance(kind, WorkstreamKind) else str(kind)
|
||||
excluded = set(exclude_ws_ids)
|
||||
live_set = set(live_node_ids) if live_node_ids else set()
|
||||
now = self._now_iso()
|
||||
closed: list[str] = []
|
||||
with self.lock:
|
||||
for ws_id, row in self.rows.items():
|
||||
if (
|
||||
row.kind == kind_str
|
||||
and row.state in BULK_CLOSE_STATE_VALUES
|
||||
and row.updated < cutoff
|
||||
and ws_id not in excluded
|
||||
):
|
||||
# Liveness gate: when live_node_ids was provided AND
|
||||
# non-empty, protect rows whose owner is in the live
|
||||
# set. NULL node_id is always eligible. When
|
||||
# live_node_ids is None or empty, no protection
|
||||
# (mirror of the real backends).
|
||||
if live_node_ids and row.node_id is not None and row.node_id in live_set:
|
||||
continue
|
||||
row.state = "closed"
|
||||
row.updated = now
|
||||
self.state_updates.append((ws_id, "closed"))
|
||||
closed.append(ws_id)
|
||||
return closed
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
if self.list_services_raises:
|
||||
raise RuntimeError("list_services forced failure")
|
||||
with self.lock:
|
||||
return [
|
||||
{"service_id": sid, "service_type": service_type}
|
||||
for sid in self.live_services.get(service_type, [])
|
||||
]
|
||||
|
||||
def get_workstream(self, ws_id: str) -> dict[str, Any] | None:
|
||||
with self.lock:
|
||||
@@ -228,6 +304,7 @@ def _make_manager(
|
||||
max_active: int = 5,
|
||||
storage: FakeStorage | None = None,
|
||||
event_emitter: Any = _EMITTER_DEFAULT,
|
||||
node_id: str | None = None,
|
||||
) -> tuple[SessionManager, FakeAdapter, FakeStorage]:
|
||||
"""Build a SessionManager wired to a FakeAdapter for both Protocols.
|
||||
|
||||
@@ -246,6 +323,7 @@ def _make_manager(
|
||||
storage=storage,
|
||||
max_active=max_active,
|
||||
event_emitter=emitter,
|
||||
node_id=node_id,
|
||||
)
|
||||
return mgr, adapter, storage
|
||||
|
||||
@@ -579,6 +657,24 @@ def test_open_resurrects_closed_state() -> None:
|
||||
assert ws_id in [e.ws_id for e in adapter.events_of("rehydrated")]
|
||||
|
||||
|
||||
def test_open_touches_workstream_on_rehydrate() -> None:
|
||||
"""Rehydrating a workstream must bump its ``updated`` so a concurrent
|
||||
close_idle pass-2 in this same process can't clobber the freshly-loaded
|
||||
row to ``closed`` because its DB ``updated`` is older than the cutoff.
|
||||
The touch is best-effort (try/except in open()) but must fire on the
|
||||
happy path."""
|
||||
mgr, _, storage = _make_manager()
|
||||
ws = mgr.create(user_id="u1")
|
||||
ws_id = ws.id
|
||||
mgr.close(ws_id)
|
||||
storage.touch_calls.clear() # only care about touches from rehydrate
|
||||
|
||||
reopened = mgr.open(ws_id)
|
||||
|
||||
assert reopened is not None
|
||||
assert ws_id in storage.touch_calls
|
||||
|
||||
|
||||
def test_open_ignores_owner_mismatch() -> None:
|
||||
# Turnstone is a trusted-team tool; row-level ownership is
|
||||
# metadata, not an access boundary. ``open`` no longer cares
|
||||
@@ -827,6 +923,197 @@ def test_close_idle_on_empty_manager_returns_empty_list() -> None:
|
||||
assert mgr.close_idle(max_age_seconds=1.0) == []
|
||||
|
||||
|
||||
def test_close_idle_runs_db_orphan_pass() -> None:
|
||||
"""DB rows of this kind that aren't loaded into the manager get
|
||||
bulk-closed when their ``updated`` is older than the cutoff. Catches
|
||||
the orphan-after-process-restart case the original close_idle missed."""
|
||||
mgr, _, storage = _make_manager()
|
||||
# Orphan rows live in storage but were never loaded via mgr.create.
|
||||
storage.register_workstream(
|
||||
"orphan-1",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"orphan-2",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
state="thinking",
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert set(closed) == {"orphan-1", "orphan-2"}
|
||||
assert ("orphan-1", "closed") in storage.state_updates
|
||||
assert ("orphan-2", "closed") in storage.state_updates
|
||||
assert storage.rows["orphan-1"].state == "closed"
|
||||
assert storage.rows["orphan-2"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_excludes_loaded_workstreams_from_db_pass() -> None:
|
||||
"""A workstream loaded into memory must NOT be reaped by the DB
|
||||
orphan pass even when its storage ``updated`` is stale — the
|
||||
in-memory pass owns those. Verifies the exclude_ws_ids plumbing."""
|
||||
mgr, _, storage = _make_manager()
|
||||
ws = mgr.create(user_id="u1")
|
||||
# Force the storage row's ``updated`` to look stale. In practice
|
||||
# ``set_state`` would bump it, but we're simulating a long-running
|
||||
# active workstream whose updated drifted older than the cutoff.
|
||||
storage.rows[ws.id].updated = "2020-01-01T00:00:00"
|
||||
|
||||
# Huge timeout so the in-memory IDLE pass skips it (stays loaded).
|
||||
closed = mgr.close_idle(max_age_seconds=10_000.0)
|
||||
|
||||
assert ws.id not in closed
|
||||
assert mgr.get(ws.id) is not None
|
||||
assert storage.rows[ws.id].state == "idle"
|
||||
|
||||
|
||||
def test_close_idle_filters_db_orphans_by_kind() -> None:
|
||||
"""An interactive manager's close_idle must not touch coordinator
|
||||
rows in storage and vice versa. Without this filter, both managers
|
||||
would race to close each other's rows."""
|
||||
mgr, _, storage = _make_manager() # interactive by default
|
||||
storage.register_workstream(
|
||||
"coord-orphan",
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"interactive-orphan",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert "interactive-orphan" in closed
|
||||
assert "coord-orphan" not in closed
|
||||
assert storage.rows["coord-orphan"].state == "idle"
|
||||
assert storage.rows["interactive-orphan"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_protects_rows_owned_by_live_services() -> None:
|
||||
"""Multi-node correctness: rows whose ``node_id`` matches a service
|
||||
with a recent heartbeat must NOT be reaped, even when *this* manager
|
||||
is on a different node — the alive peer may legitimately have them
|
||||
loaded. Liveness is the rendezvous router's primitive (post-PR-#384);
|
||||
using it here keeps reap scoping aligned with routing.
|
||||
|
||||
Default ``_make_manager`` uses an INTERACTIVE adapter, which derives
|
||||
``service_type='server'`` — so live_services seeded under "server"
|
||||
are what the manager queries."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.live_services["server"] = ["node-b"] # only node-b is alive
|
||||
storage.register_workstream(
|
||||
"ours-from-dead-node",
|
||||
node_id="node-a", # dead pod (not in live_services)
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"theirs-still-alive",
|
||||
node_id="node-b",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["ours-from-dead-node"]
|
||||
assert storage.rows["ours-from-dead-node"].state == "closed"
|
||||
assert storage.rows["theirs-still-alive"].state == "idle"
|
||||
|
||||
|
||||
def test_close_idle_protects_live_services_for_coordinator_kind() -> None:
|
||||
"""Coord-side parity: a coordinator manager derives
|
||||
``service_type='console'``, so live_services seeded under "console"
|
||||
are what gets queried. Mirrors the interactive test to ensure both
|
||||
halves of the production wiring are exercised."""
|
||||
coord_adapter = FakeAdapter(kind=WorkstreamKind.COORDINATOR)
|
||||
mgr, _, storage = _make_manager(coord_adapter)
|
||||
storage.live_services["console"] = ["console"] # console is alive
|
||||
storage.register_workstream(
|
||||
"alive-console-coord",
|
||||
node_id="console",
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"dead-console-coord",
|
||||
node_id="dead-console-instance", # not in live set
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["dead-console-coord"]
|
||||
assert storage.rows["alive-console-coord"].state == "idle"
|
||||
assert storage.rows["dead-console-coord"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_reaps_rows_with_null_node_id() -> None:
|
||||
"""A row with no ``node_id`` has no owner identity — age alone gates
|
||||
the reap. Defends against a NULL silently propagating through ``NOT
|
||||
IN (live)`` and protecting orphans forever."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.live_services["server"] = ["node-a"]
|
||||
storage.register_workstream(
|
||||
"no-owner",
|
||||
node_id=None,
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["no-owner"]
|
||||
|
||||
|
||||
def test_close_idle_reaps_all_orphans_when_no_peers_alive() -> None:
|
||||
"""When ``list_services`` returns an empty list (no heartbeating
|
||||
peers), every stale orphan is unprotected and gets reaped. This is
|
||||
the cold-start / single-process / dead-cluster-recovery case."""
|
||||
mgr, _, storage = _make_manager()
|
||||
# storage.live_services["server"] left empty — no peers heartbeating
|
||||
storage.register_workstream(
|
||||
"any-node-1",
|
||||
node_id="node-a",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"any-node-2",
|
||||
node_id="node-b",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert set(closed) == {"any-node-1", "any-node-2"}
|
||||
|
||||
|
||||
def test_close_idle_skips_pass_2_when_list_services_fails() -> None:
|
||||
"""Conservative fallback: if list_services fails we can't enumerate
|
||||
live owners safely, so pass 2 must skip rather than reap blind. Pass
|
||||
1 (in-memory IDLE) still runs."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.list_services_raises = True
|
||||
storage.register_workstream(
|
||||
"would-be-orphan",
|
||||
node_id="node-a",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == []
|
||||
assert storage.rows["would-be-orphan"].state == "idle"
|
||||
|
||||
|
||||
def test_list_all_returns_creation_order() -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
a = mgr.create(user_id="u1")
|
||||
@@ -1084,3 +1371,127 @@ class TestSessionManagerWithStateWriter:
|
||||
assert "running" not in ws_writes, (
|
||||
f"set_state after close enqueued through buffer: {ws_writes}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-subscriber observer — subscribe_to_state / unsubscribe_from_state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStateSubscribers:
|
||||
"""Multi-subscriber observer for ``set_state``.
|
||||
|
||||
Used by the CLI's background-attention notifier and by
|
||||
``SameNodeChildSource``. Subscribe / unsubscribe must be safe under
|
||||
concurrent dispatch, and dispatch must not skip / repeat callbacks
|
||||
when subscribers register or unregister mid-iteration.
|
||||
"""
|
||||
|
||||
def test_subscribe_fires_on_set_state(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
events: list[tuple[str, str]] = []
|
||||
|
||||
def cb(ws_id: str, state: WorkstreamState) -> None:
|
||||
events.append((ws_id, state.value))
|
||||
|
||||
mgr.subscribe_to_state(cb)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert events == [(ws.id, "running")]
|
||||
|
||||
def test_unsubscribe_stops_firing(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
events: list[str] = []
|
||||
|
||||
def cb(_ws_id: str, state: WorkstreamState) -> None:
|
||||
events.append(state.value)
|
||||
|
||||
mgr.subscribe_to_state(cb)
|
||||
mgr.unsubscribe_from_state(cb)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert events == []
|
||||
|
||||
def test_unsubscribe_unknown_is_noop(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
# Doesn't raise.
|
||||
mgr.unsubscribe_from_state(lambda *_: None)
|
||||
|
||||
def test_multiple_subscribers_fire_in_registration_order(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
order: list[int] = []
|
||||
|
||||
def make(i: int) -> Callable[[str, WorkstreamState], None]:
|
||||
def cb(_ws_id: str, _state: WorkstreamState) -> None:
|
||||
order.append(i)
|
||||
|
||||
return cb
|
||||
|
||||
mgr.subscribe_to_state(make(1))
|
||||
mgr.subscribe_to_state(make(2))
|
||||
mgr.subscribe_to_state(make(3))
|
||||
mgr.set_state(ws.id, WorkstreamState.IDLE)
|
||||
assert order == [1, 2, 3]
|
||||
|
||||
def test_subscriber_exception_does_not_block_others(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
survived: list[str] = []
|
||||
|
||||
def boom(*_: Any) -> None:
|
||||
raise RuntimeError("subscriber crash")
|
||||
|
||||
def good(_ws_id: str, state: WorkstreamState) -> None:
|
||||
survived.append(state.value)
|
||||
|
||||
mgr.subscribe_to_state(boom)
|
||||
mgr.subscribe_to_state(good)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert survived == ["running"]
|
||||
|
||||
@pytest.mark.parametrize("n_threads", [10, 50])
|
||||
def test_concurrent_subscribe(self, n_threads: int) -> None:
|
||||
"""Subscribe from many threads; all callbacks land in the list.
|
||||
|
||||
Validates the lock around mutation — without it the underlying
|
||||
list.append could lose entries under contention.
|
||||
"""
|
||||
mgr, _, _ = _make_manager()
|
||||
callbacks = [lambda *_, i=i: None for i in range(n_threads)]
|
||||
threads = [threading.Thread(target=mgr.subscribe_to_state, args=(cb,)) for cb in callbacks]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
# Snapshot under the lock to read the count safely.
|
||||
with mgr._state_subscribers_lock:
|
||||
assert len(mgr._state_subscribers) == n_threads
|
||||
|
||||
def test_subscribe_during_dispatch_does_not_corrupt_iteration(self) -> None:
|
||||
"""A subscriber that calls subscribe_to_state during its own
|
||||
callback must not affect the in-flight dispatch (snapshot
|
||||
isolation). This is the bug-1 invariant: mutation during
|
||||
iteration can't shift the iterator's index because dispatch
|
||||
iterates a snapshot, not the live list.
|
||||
"""
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
fired: list[str] = []
|
||||
|
||||
def late(_ws_id: str, state: WorkstreamState) -> None:
|
||||
fired.append("late:" + state.value)
|
||||
|
||||
def first(_ws_id: str, state: WorkstreamState) -> None:
|
||||
fired.append("first:" + state.value)
|
||||
mgr.subscribe_to_state(late) # mid-dispatch addition
|
||||
|
||||
mgr.subscribe_to_state(first)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
# ``late`` was added during dispatch but the snapshot was
|
||||
# already frozen — so it doesn't fire on this round.
|
||||
assert fired == ["first:running"]
|
||||
# Next round it does fire, in registration order after first.
|
||||
fired.clear()
|
||||
mgr.set_state(ws.id, WorkstreamState.IDLE)
|
||||
assert fired == ["first:idle", "late:idle"]
|
||||
|
||||
@@ -4,6 +4,10 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import workstreams
|
||||
|
||||
# -- Workstream registration ---------------------------------------------------
|
||||
|
||||
|
||||
@@ -664,6 +668,277 @@ class TestBatchPrimitives:
|
||||
assert result == {"never-seen": 0}
|
||||
|
||||
|
||||
# -- bulk_close_stale_orphans --------------------------------------------------
|
||||
|
||||
|
||||
def _force_updated(backend: Any, ws_id: str, updated: str) -> None:
|
||||
"""Stamp a workstream row's ``updated`` column directly.
|
||||
|
||||
The public surface only sets ``updated`` to ``now``, which makes it
|
||||
impossible to fabricate a stale row through register/update calls.
|
||||
Reaches into ``backend._engine`` — same access pattern conftest uses
|
||||
for cross-backend cleanup.
|
||||
"""
|
||||
with backend._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=updated)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
class TestBulkCloseStaleOrphans:
|
||||
def test_closes_stale_non_terminal_rows_of_kind(self, backend):
|
||||
backend.register_workstream("stale-idle", kind="interactive")
|
||||
backend.register_workstream("stale-thinking", kind="interactive")
|
||||
backend.update_workstream_state("stale-thinking", "thinking")
|
||||
backend.register_workstream("fresh-idle", kind="interactive")
|
||||
_force_updated(backend, "stale-idle", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "stale-thinking", "2020-01-01T00:00:00")
|
||||
# fresh-idle stays at registration time (effectively now)
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"stale-idle", "stale-thinking"}
|
||||
rows = backend.get_workstreams_batch(["stale-idle", "stale-thinking", "fresh-idle"])
|
||||
assert rows["stale-idle"]["state"] == "closed"
|
||||
assert rows["stale-thinking"]["state"] == "closed"
|
||||
assert rows["fresh-idle"]["state"] == "idle"
|
||||
|
||||
def test_skips_already_closed(self, backend):
|
||||
backend.register_workstream("already-closed", kind="interactive")
|
||||
backend.update_workstream_state("already-closed", "closed")
|
||||
_force_updated(backend, "already-closed", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == []
|
||||
|
||||
def test_filters_by_kind(self, backend):
|
||||
backend.register_workstream("interactive-stale", kind="interactive")
|
||||
backend.register_workstream("coord-stale", kind="coordinator")
|
||||
_force_updated(backend, "interactive-stale", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "coord-stale", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == ["interactive-stale"]
|
||||
rows = backend.get_workstreams_batch(["interactive-stale", "coord-stale"])
|
||||
assert rows["interactive-stale"]["state"] == "closed"
|
||||
assert rows["coord-stale"]["state"] == "idle"
|
||||
|
||||
def test_excludes_loaded_ws_ids(self, backend):
|
||||
backend.register_workstream("ws-keep", kind="interactive")
|
||||
backend.register_workstream("ws-close", kind="interactive")
|
||||
_force_updated(backend, "ws-keep", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "ws-close", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=["ws-keep"]
|
||||
)
|
||||
|
||||
assert closed == ["ws-close"]
|
||||
rows = backend.get_workstreams_batch(["ws-keep", "ws-close"])
|
||||
assert rows["ws-keep"]["state"] == "idle"
|
||||
assert rows["ws-close"]["state"] == "closed"
|
||||
|
||||
def test_empty_exclude_list_does_not_break_sql(self, backend):
|
||||
backend.register_workstream("orphan", kind="interactive")
|
||||
_force_updated(backend, "orphan", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == ["orphan"]
|
||||
|
||||
def test_no_orphans_returns_empty(self, backend):
|
||||
backend.register_workstream("fresh", kind="interactive")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == []
|
||||
|
||||
def test_closes_all_non_terminal_states(self, backend):
|
||||
for ws_id, state in [
|
||||
("o-idle", "idle"),
|
||||
("o-thinking", "thinking"),
|
||||
("o-attention", "attention"),
|
||||
("o-running", "running"),
|
||||
]:
|
||||
backend.register_workstream(ws_id, kind="interactive")
|
||||
if state != "idle":
|
||||
backend.update_workstream_state(ws_id, state)
|
||||
_force_updated(backend, ws_id, "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"o-idle", "o-thinking", "o-attention", "o-running"}
|
||||
|
||||
def test_bumps_updated_on_close(self, backend):
|
||||
stale_updated = "2020-01-01T00:00:00"
|
||||
backend.register_workstream("orphan", kind="interactive")
|
||||
_force_updated(backend, "orphan", stale_updated)
|
||||
|
||||
backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
# ``updated`` must change away from the forced stale value. Asserting
|
||||
# inequality from the seed (rather than ``> "2024-01-01..."``) keeps
|
||||
# the test independent of wall-clock date.
|
||||
with backend._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(workstreams.c.updated).where(workstreams.c.ws_id == "orphan")
|
||||
).one()
|
||||
assert row[0] != stale_updated
|
||||
|
||||
def test_protects_rows_owned_by_live_services(self, backend):
|
||||
"""Liveness scoping (post-#384 rendezvous-routing world): rows
|
||||
whose ``node_id`` matches a heartbeating service must NOT be
|
||||
reaped, because that owner may legitimately have them loaded on
|
||||
another worker. Rows whose ``node_id`` matches a dead service
|
||||
ARE eligible — that's how dead-pod orphans get reclaimed in
|
||||
containerized deployments with dynamic hostnames."""
|
||||
backend.register_workstream("dead-node", node_id="dead-pod-x4k2", kind="interactive")
|
||||
backend.register_workstream("alive-node", node_id="alive-pod-y9p3", kind="interactive")
|
||||
_force_updated(backend, "dead-node", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "alive-node", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=["alive-pod-y9p3"],
|
||||
)
|
||||
|
||||
assert closed == ["dead-node"]
|
||||
rows = backend.get_workstreams_batch(["dead-node", "alive-node"])
|
||||
assert rows["dead-node"]["state"] == "closed"
|
||||
assert rows["alive-node"]["state"] == "idle"
|
||||
|
||||
def test_null_node_id_always_eligible(self, backend):
|
||||
"""A row with NULL ``node_id`` has no owner identity — age alone
|
||||
gates the reap. Belt-and-suspenders against ``NULL NOT IN (...)``
|
||||
evaluating to NULL (not TRUE) and silently protecting orphans
|
||||
forever."""
|
||||
backend.register_workstream("no-owner", node_id=None, kind="interactive")
|
||||
_force_updated(backend, "no-owner", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=["some-other-node"],
|
||||
)
|
||||
|
||||
assert closed == ["no-owner"]
|
||||
|
||||
def test_live_node_ids_none_skips_filter(self, backend):
|
||||
"""``live_node_ids=None`` is the single-process / operator-backfill
|
||||
mode — all rows of *kind* are eligible regardless of node_id."""
|
||||
backend.register_workstream("node-a", node_id="node-a", kind="interactive")
|
||||
backend.register_workstream("node-b", node_id="node-b", kind="interactive")
|
||||
_force_updated(backend, "node-a", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "node-b", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"node-a", "node-b"}
|
||||
|
||||
def test_empty_live_node_ids_treats_all_as_dead(self, backend):
|
||||
"""Empty list ``live_node_ids=[]`` means "no nodes alive" — every
|
||||
row's owner is unprotected. Useful for operator scripts that
|
||||
want to reap regardless of liveness."""
|
||||
backend.register_workstream("any", node_id="node-a", kind="interactive")
|
||||
_force_updated(backend, "any", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=[],
|
||||
)
|
||||
|
||||
assert closed == ["any"]
|
||||
|
||||
def test_combines_live_node_ids_and_exclude_ws_ids(self, backend):
|
||||
"""Both filters stack as AND clauses on the UPDATE. Covers the
|
||||
full 2x2 matrix to catch a future edit that replaces an AND with
|
||||
an OR or drops one of the filters: only the (orphan + dead-node)
|
||||
cell should be reaped."""
|
||||
# All four registered with the same stale ``updated``.
|
||||
for ws_id, node in [
|
||||
("loaded-alive", "alive-node"),
|
||||
("loaded-dead", "dead-node"),
|
||||
("orphan-alive", "alive-node"),
|
||||
("orphan-dead", "dead-node"),
|
||||
]:
|
||||
backend.register_workstream(ws_id, node_id=node, kind="interactive")
|
||||
_force_updated(backend, ws_id, "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=["loaded-alive", "loaded-dead"],
|
||||
live_node_ids=["alive-node"],
|
||||
)
|
||||
|
||||
# Only orphan-dead is unprotected by both filters.
|
||||
assert closed == ["orphan-dead"]
|
||||
rows = backend.get_workstreams_batch(
|
||||
["loaded-alive", "loaded-dead", "orphan-alive", "orphan-dead"]
|
||||
)
|
||||
assert rows["loaded-alive"]["state"] == "idle"
|
||||
assert rows["loaded-dead"]["state"] == "idle"
|
||||
assert rows["orphan-alive"]["state"] == "idle"
|
||||
assert rows["orphan-dead"]["state"] == "closed"
|
||||
|
||||
|
||||
# -- touch_workstream ----------------------------------------------------------
|
||||
|
||||
|
||||
class TestTouchWorkstream:
|
||||
def test_bumps_updated_only(self, backend):
|
||||
"""Used by ``open()`` on rehydrate to defend against the orphan
|
||||
reaper clobbering a freshly-loaded row. Must not change ``state``
|
||||
(the open() path explicitly avoids state writes to dodge a race
|
||||
with concurrent close())."""
|
||||
stale_updated = "2020-01-01T00:00:00"
|
||||
backend.register_workstream("ws-touch", kind="interactive")
|
||||
backend.update_workstream_state("ws-touch", "closed") # simulate prior close
|
||||
_force_updated(backend, "ws-touch", stale_updated)
|
||||
|
||||
backend.touch_workstream("ws-touch")
|
||||
|
||||
with backend._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(workstreams.c.state, workstreams.c.updated).where(
|
||||
workstreams.c.ws_id == "ws-touch"
|
||||
)
|
||||
).one()
|
||||
assert row[0] == "closed", "state must not be modified by touch"
|
||||
# Compare against the forced stale value rather than a fixed calendar
|
||||
# date so the test is independent of wall-clock time.
|
||||
assert row[1] != stale_updated, "updated must be bumped"
|
||||
|
||||
def test_unknown_id_is_noop(self, backend):
|
||||
"""Touch on a missing id must not raise — open()'s exception
|
||||
handler is best-effort."""
|
||||
backend.touch_workstream("nonexistent") # must not raise
|
||||
|
||||
|
||||
# -- Lifecycle -----------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -155,3 +155,179 @@ class TestContentAccumulation:
|
||||
assert len(idle_events) == 1
|
||||
# Content should be capped, not contain everything
|
||||
assert len(idle_events[0]["content"]) <= _MAX_TURN_CONTENT_CHARS + 1024
|
||||
|
||||
|
||||
class TestPendingApprovalDetailNotPiggybacked:
|
||||
"""Stage 3 cleanup — ``pending_approval_detail`` is no longer
|
||||
piggybacked on ``ws_state`` events. Approval items now arrive via
|
||||
bulk fetch when the coord tree's reducer sees the
|
||||
``activity_state="approval"`` transition; verdicts via the explicit
|
||||
``intent_verdict`` event class; resolution via
|
||||
``approval_resolved``. These tests lock the no-piggyback contract
|
||||
down so a future regression doesn't silently re-introduce the
|
||||
duplicated path."""
|
||||
|
||||
def test_state_broadcast_omits_field_when_no_approval_pending(self):
|
||||
ui = _make_ui()
|
||||
assert ui._pending_approval is None
|
||||
ui._broadcast_state("running")
|
||||
|
||||
events = _drain_global()
|
||||
running_events = [e for e in events if e.get("state") == "running"]
|
||||
assert len(running_events) == 1
|
||||
assert "pending_approval_detail" not in running_events[0]
|
||||
|
||||
def test_state_broadcast_omits_field_even_when_approval_pending(self):
|
||||
"""The piggyback is gone: even when ``_pending_approval`` is set,
|
||||
the state broadcast must NOT carry ``pending_approval_detail``.
|
||||
The browser triggers a bulk fetch off the
|
||||
``activity_state="approval"`` transition to get the items."""
|
||||
ui = _make_ui()
|
||||
ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{
|
||||
"call_id": "c1",
|
||||
"header": "tool x",
|
||||
"func_args": "{}",
|
||||
"intent_summary": "do x",
|
||||
"needs_approval": True,
|
||||
}
|
||||
],
|
||||
"judge_pending": False,
|
||||
}
|
||||
ui._broadcast_state("attention")
|
||||
|
||||
events = _drain_global()
|
||||
attn = [e for e in events if e.get("state") == "attention"]
|
||||
assert len(attn) == 1
|
||||
assert "pending_approval_detail" not in attn[0]
|
||||
|
||||
def test_field_stays_absent_after_approval_resolves(self):
|
||||
ui = _make_ui()
|
||||
ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
ui._broadcast_state("attention")
|
||||
_drain_global()
|
||||
|
||||
ui._pending_approval = None
|
||||
ui._broadcast_state("running")
|
||||
events = _drain_global()
|
||||
running = [e for e in events if e.get("state") == "running"]
|
||||
assert len(running) == 1
|
||||
assert "pending_approval_detail" not in running[0]
|
||||
|
||||
|
||||
class TestBroadcastIntentVerdict:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_intent_verdict``.
|
||||
|
||||
The collector-side test (``test_apply_delta_intent_verdict_*`` in
|
||||
test_console.py) covers consumption; this pins the event shape the
|
||||
producer puts on the global queue. A field rename or missed key
|
||||
here would slip past the consumer test because the consumer reads
|
||||
via ``data.get(...)``.
|
||||
"""
|
||||
|
||||
def test_pushes_intent_verdict_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.92,
|
||||
"recommendation": "approve",
|
||||
"reasoning": "tool reads only",
|
||||
}
|
||||
ui._broadcast_intent_verdict(verdict)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "intent_verdict"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["verdict"] == verdict
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
def test_queue_full_swallowed(self):
|
||||
# Force a tiny queue then fill it so the next put_nowait
|
||||
# raises queue.Full — the broadcast must absorb it without
|
||||
# propagating (matches _broadcast_state's queue.Full handling).
|
||||
WebUI._global_queue = queue.Queue(maxsize=1)
|
||||
WebUI._global_queue.put_nowait({"sentinel": True})
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
|
||||
class TestBroadcastApprovalResolved:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_approval_resolved``."""
|
||||
|
||||
def test_pushes_approval_resolved_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(True, "lgtm", always=False)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "approval_resolved"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["approved"] is True
|
||||
assert ev["feedback"] == "lgtm"
|
||||
assert ev["always"] is False
|
||||
|
||||
def test_normalises_none_feedback_to_empty_string(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(False, None)
|
||||
|
||||
events = _drain_global()
|
||||
assert events[0]["feedback"] == ""
|
||||
assert events[0]["approved"] is False
|
||||
assert events[0]["always"] is False
|
||||
|
||||
def test_always_kwarg_propagates(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(True, "ok", always=True)
|
||||
events = _drain_global()
|
||||
assert events[0]["always"] is True
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, None)
|
||||
|
||||
|
||||
class TestBroadcastApproveRequest:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_approve_request`` —
|
||||
push path for the initial approval items so a coord parent's tree
|
||||
UI can render the inline approve/deny block immediately without
|
||||
waiting for a bulk-fetch round-trip."""
|
||||
|
||||
def test_pushes_approve_request_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
ui._broadcast_approve_request(detail)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "approve_request"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["detail"] == detail
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
|
||||
@@ -899,6 +899,138 @@ class TestHistoryInteractive:
|
||||
assert client.get(base, params={"limit": 999}).status_code == 200
|
||||
|
||||
|
||||
class TestBuildHistoryReminderPropagation:
|
||||
"""``_build_history`` must surface the ``_reminders`` side-channel on
|
||||
each entry so a tab reconnecting via ``/history`` renders the same
|
||||
metacognitive nudge bubble the originating tab saw via the live
|
||||
``user_reminder`` SSE event.
|
||||
"""
|
||||
|
||||
def _session_with_messages(self, messages: list[dict]) -> MagicMock:
|
||||
session = MagicMock()
|
||||
session.messages = messages
|
||||
return session
|
||||
|
||||
def test_reminders_sidechannel_surfaces_on_entry(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "ah no",
|
||||
"_reminders": [{"type": "correction", "text": "watch out"}],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == "ah no"
|
||||
assert history[0]["reminders"] == [{"type": "correction", "text": "watch out"}]
|
||||
|
||||
def test_no_reminders_key_when_sidechannel_absent(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages([{"role": "user", "content": "just a message"}])
|
||||
history = _build_history(session)
|
||||
assert "reminders" not in history[0]
|
||||
|
||||
def test_no_reminders_key_when_sidechannel_empty(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages([{"role": "user", "content": "hi", "_reminders": []}])
|
||||
history = _build_history(session)
|
||||
assert "reminders" not in history[0]
|
||||
|
||||
def test_multiple_reminders_preserved_in_order(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "x",
|
||||
"_reminders": [
|
||||
{"type": "denial", "text": "FIRST"},
|
||||
{"type": "correction", "text": "SECOND"},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["reminders"] == [
|
||||
{"type": "denial", "text": "FIRST"},
|
||||
{"type": "correction", "text": "SECOND"},
|
||||
]
|
||||
|
||||
def test_reminders_coexist_with_attachments(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "data:..."}},
|
||||
],
|
||||
"_reminders": [{"type": "correction", "text": "watch"}],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == "look"
|
||||
assert history[0]["attachments"] == [{"kind": "image", "filename": "", "mime_type": ""}]
|
||||
assert history[0]["reminders"] == [{"type": "correction", "text": "watch"}]
|
||||
|
||||
def test_malformed_reminders_filtered_out(self):
|
||||
"""Defensive: a non-dict element in the list (corruption / bug)
|
||||
is dropped rather than crashing the history serialisation."""
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "x",
|
||||
"_reminders": [
|
||||
{"type": "correction", "text": "ok"},
|
||||
"not-a-dict",
|
||||
{"type": "denial"}, # missing text
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
# Non-dicts dropped; missing-text fills with empty string.
|
||||
assert history[0]["reminders"] == [
|
||||
{"type": "correction", "text": "ok"},
|
||||
{"type": "denial", "text": ""},
|
||||
]
|
||||
|
||||
def test_clean_message_passes_through_unchanged(self):
|
||||
"""No reminders, plain content — _build_history is a no-op for the
|
||||
reminder field and ``content`` rides through verbatim."""
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[{"role": "user", "content": "just a normal message"}]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == "just a normal message"
|
||||
assert "reminders" not in history[0]
|
||||
|
||||
def test_assistant_content_with_literal_reminder_tag_unchanged(self):
|
||||
"""Assistant output may legitimately reference the tag (e.g. when
|
||||
the model is explaining the reminder system itself). No
|
||||
transformation should ever apply to assistant content."""
|
||||
from turnstone.server import _build_history
|
||||
|
||||
content = "Here is a <system-reminder> tag in assistant output."
|
||||
session = self._session_with_messages([{"role": "assistant", "content": content}])
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == content
|
||||
|
||||
|
||||
class TestDetailInteractive:
|
||||
"""Interactive parity for the lifted ``GET /v1/api/workstreams/{ws_id}``.
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "1.5.0"
|
||||
__version__ = "1.5.5"
|
||||
|
||||
+24
-1
@@ -312,6 +312,29 @@ class TerminalUI(SessionUI):
|
||||
sys.stdout.write(f"{RED}{message}{RESET}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
def _print_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
"""Render a metacognitive reminder list as ``[metacognition · type] text``
|
||||
lines in the terminal — the CLI's equivalent of the web UI's
|
||||
yellow themed bubble. Used by both ``on_user_reminder`` and
|
||||
``on_tool_reminder``; the rendering is identical because
|
||||
terminal output is anchor-by-flow rather than DOM-by-anchor.
|
||||
"""
|
||||
for r in reminders:
|
||||
nt = str(r.get("type", "") or "")
|
||||
text = str(r.get("text", "") or "")
|
||||
label = "metacognition" + (f" · {nt}" if nt else "")
|
||||
sys.stdout.write(f"{YELLOW}[{label}]{RESET} {text}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
self._print_reminder(reminders)
|
||||
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None:
|
||||
# tool_call_id ignored — the CLI anchors by output sequence
|
||||
# (the line lands directly after the tool result that
|
||||
# triggered the batch's reminder).
|
||||
self._print_reminder(reminders)
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
pass # base TerminalUI ignores state changes
|
||||
|
||||
@@ -1219,7 +1242,7 @@ def main() -> None:
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
manager._on_state_change = _bg_attention_notify
|
||||
manager.subscribe_to_state(_bg_attention_notify)
|
||||
|
||||
# Print banner
|
||||
print(f"\n{bold('Chat')} with {cyan(model)}")
|
||||
|
||||
@@ -634,6 +634,60 @@ class ClusterCollector:
|
||||
ws["name"] = name
|
||||
pending_events.append({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
elif etype == "intent_verdict":
|
||||
# Pure pass-through for LLM judge verdicts. The node's
|
||||
# WebUI._broadcast_intent_verdict pushes onto the
|
||||
# global queue (Stage 3 Step 5); the cluster fan-out
|
||||
# forwards to subscribers (e.g. CoordinatorAdapter,
|
||||
# which dispatches to the parent's coord SSE as
|
||||
# ``child_ws_intent_verdict``).
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"verdict": data.get("verdict") or {},
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "approval_resolved":
|
||||
# Pure pass-through for approve/deny resolutions. Pairs
|
||||
# with ``intent_verdict`` above so the parent
|
||||
# coordinator's tree UI can clear the pending-approval
|
||||
# pill the moment the user decides, rather than waiting
|
||||
# for the subsequent state-change piggyback.
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"approved": bool(data.get("approved", False)),
|
||||
"feedback": data.get("feedback", "") or "",
|
||||
"always": bool(data.get("always", False)),
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "approve_request":
|
||||
# Push path for the initial approval items. Eliminates
|
||||
# the bulk-fetch race that otherwise leaves the coord
|
||||
# tree stuck on a loading placeholder when the bulk
|
||||
# fetch lands in the gap between the state transition
|
||||
# to ATTENTION and ``_pending_approval`` being set.
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"detail": data.get("detail") or {},
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "health_changed":
|
||||
# Update the health dict's backend status in-place
|
||||
bstatus = data.get("backend_status", "")
|
||||
@@ -1203,3 +1257,70 @@ class ClusterCollector:
|
||||
return
|
||||
entry["name"] = name
|
||||
self._fanout({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
def emit_console_ws_intent_verdict(self, ws_id: str, verdict: dict[str, Any]) -> None:
|
||||
"""Fan an LLM intent-judge verdict for a console-pseudo-node ws.
|
||||
|
||||
Gives coord-spawned approval flows a first-class cluster-bus
|
||||
event so the parent's tree UI can render the risk pill +
|
||||
verdict result without polling.
|
||||
``CoordinatorAdapter._dispatch_child_event`` re-emits these as
|
||||
``child_ws_intent_verdict`` for the parent coordinator's SSE
|
||||
stream. The dispatch path filters by registry membership;
|
||||
downstream subscribers tolerate verdicts for ws_ids they don't
|
||||
own (silently drop), so we skip the membership pre-check that
|
||||
would otherwise add a lock acquisition per emit on a path
|
||||
that fires once per tool-call during heuristic+LLM judging.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
|
||||
def emit_console_ws_approval_resolved(
|
||||
self,
|
||||
ws_id: str,
|
||||
*,
|
||||
approved: bool,
|
||||
feedback: str = "",
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision for a console-pseudo-node ws.
|
||||
|
||||
Paired with :meth:`emit_console_ws_intent_verdict` so the
|
||||
coord tree UI clears the pending-approval pill in lockstep
|
||||
with the actual decision. Same lock-skip rationale as the
|
||||
intent-verdict emit above.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"approved": approved,
|
||||
"feedback": feedback,
|
||||
"always": always,
|
||||
}
|
||||
)
|
||||
|
||||
def emit_console_ws_approve_request(self, ws_id: str, detail: dict[str, Any]) -> None:
|
||||
"""Fan an ``approve_request`` payload for a console-pseudo-node ws.
|
||||
|
||||
Push path for the initial approval items so a coord parent's
|
||||
tree UI can render the inline approve/deny block immediately
|
||||
without a bulk-fetch round-trip. ``CoordinatorAdapter._dispatch_child_event``
|
||||
re-emits as ``child_ws_approve_request`` for the parent
|
||||
coordinator's SSE stream.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -15,17 +15,17 @@ for the storage-seeded children rebuild.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.core import session_worker
|
||||
from turnstone.core.adapters._ui_cleanup import cleanup_session_ui
|
||||
from turnstone.core.child_source import ClusterChildSource
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
@@ -56,38 +56,25 @@ class CoordinatorAdapter:
|
||||
# the manager to the adapter's ``__init__`` — break the cycle
|
||||
# with a setter called from console startup.
|
||||
self._manager: SessionManager | None = None
|
||||
# Per-coordinator known-child ws_id set. Populated lazily on
|
||||
# create/open from storage and updated live as the cluster fan-out
|
||||
# thread sees ws_created events with matching parent_ws_id.
|
||||
# Closed / deleted children stay in the registry so the tree UI
|
||||
# can keep rendering them grayed out; the authoritative render
|
||||
# path reads storage for state. Bounded by eventual coordinator
|
||||
# close/eviction.
|
||||
self._children: dict[str, set[str]] = {}
|
||||
# Reverse index for O(1) child → coord lookup on every cluster
|
||||
# event. Without this, every cluster event incurs a linear scan
|
||||
# over every coordinator's child set while holding the fan-out
|
||||
# lock — a hot-path tax that scales with both active coordinators
|
||||
# and their retained-history depth.
|
||||
self._child_to_coord: dict[str, str] = {}
|
||||
self._children_lock = threading.Lock()
|
||||
# Cluster-event fan-out: subscribes to the ClusterCollector's
|
||||
# listener channel, filters by known child ws_ids, and re-emits
|
||||
# child_ws_* events on the matching coordinator's UI. Configured
|
||||
# lazily via ``start_child_event_fanout(collector)`` from the
|
||||
# console lifespan once both the manager and collector exist.
|
||||
self._collector_queue: queue.Queue[dict[str, Any]] | None = None
|
||||
self._fanout_thread: threading.Thread | None = None
|
||||
self._fanout_stop = threading.Event()
|
||||
# Coord-ws-id → UI map for the fan-out dispatch path. Read
|
||||
# and written under ``self._children_lock`` alongside the
|
||||
# forward/reverse child maps. (Previously the value was
|
||||
# ``(user_id, ui)`` with a copy-on-write dict swap so the
|
||||
# dispatch could read it lock-free — but _dispatch_child_event
|
||||
# already re-validates the parent under _children_lock anyway,
|
||||
# so the lock-free snapshot was premature. The user_id half is
|
||||
# also dead after a46dab1 dropped row-level ownership gates.)
|
||||
self._active_coords: dict[str, Any] = {}
|
||||
# Children registry — universal parent → children + reverse
|
||||
# lookup primitive. Lifted from inline data on this adapter to
|
||||
# ``turnstone.core.children_registry.ChildrenRegistry`` (Stage 3
|
||||
# Step 1) so the same primitive can serve interactive
|
||||
# workstreams when they gain spawn capability. The dispatch
|
||||
# path (`_dispatch_child_event`) calls into the registry
|
||||
# atomically; everything else delegates through the legacy
|
||||
# method names which still exist as thin shims for the
|
||||
# cluster-routing + cleanup callers.
|
||||
self._registry = ChildrenRegistry()
|
||||
# Cross-node child events arrive via ``ClusterChildSource``
|
||||
# (Stage 3 Step 2): a strategy that subscribes to the
|
||||
# collector's listener channel and runs a daemon thread that
|
||||
# drains the queue, pushing each event to the sink. The sink
|
||||
# is :meth:`_dispatch_child_event` so the existing translation
|
||||
# logic (cluster_state → child_ws_state, etc.) stays in one
|
||||
# place. Constructed lazily by ``start_child_event_fanout`` so
|
||||
# the collector reference is available.
|
||||
self._child_source: ClusterChildSource | None = None
|
||||
|
||||
def attach(self, manager: SessionManager) -> None:
|
||||
"""Late-bind the owning :class:`SessionManager`.
|
||||
@@ -109,7 +96,7 @@ class CoordinatorAdapter:
|
||||
# needs the empty forward/presence entries so
|
||||
# ``_dispatch_child_event`` recognises this coordinator when
|
||||
# its first child is spawned.
|
||||
self._install_coord_registry(ws)
|
||||
self._registry.install(ws.id, ws.ui)
|
||||
self._fanout_console_ws_created(ws)
|
||||
|
||||
def emit_rehydrated(self, ws: Workstream) -> None:
|
||||
@@ -117,21 +104,10 @@ class CoordinatorAdapter:
|
||||
# it from storage after the registry seed so a ``ws_created``
|
||||
# for an already-spawned child that fires mid-rebuild merges
|
||||
# cleanly.
|
||||
self._install_coord_registry(ws)
|
||||
self._registry.install(ws.id, ws.ui)
|
||||
self._rebuild_children_registry(ws.id)
|
||||
self._fanout_console_ws_created(ws)
|
||||
|
||||
def _install_coord_registry(self, ws: Workstream) -> None:
|
||||
"""Seed the children registry + presence map for ``ws``.
|
||||
|
||||
Shared by ``emit_created`` and ``emit_rehydrated`` — the
|
||||
difference between the two is purely whether we then rebuild
|
||||
from storage.
|
||||
"""
|
||||
with self._children_lock:
|
||||
self._children.setdefault(ws.id, set())
|
||||
self._active_coords[ws.id] = ws.ui
|
||||
|
||||
def _fanout_console_ws_created(self, ws: Workstream) -> None:
|
||||
try:
|
||||
self._collector.emit_console_ws_created(
|
||||
@@ -197,14 +173,11 @@ class CoordinatorAdapter:
|
||||
# toast only fire for real-node (interactive) ws_closed events.
|
||||
del reason, name
|
||||
# Drop the coordinator's children-registry entries AND its
|
||||
# presence slot. Mirrors the eviction/close paths from the old
|
||||
# CoordinatorManager (which did the same under _children_lock
|
||||
# + _lock respectively). A plain _children.pop without clearing
|
||||
# the reverse index would leak every evicted coordinator's
|
||||
# child→parent pointers forever.
|
||||
with self._children_lock:
|
||||
self._pop_coord_registry_locked(ws_id)
|
||||
self._active_coords.pop(ws_id, None)
|
||||
# presence slot. A plain pop without clearing the reverse
|
||||
# index would leak every evicted coordinator's child→parent
|
||||
# pointers forever — :meth:`ChildrenRegistry.uninstall`
|
||||
# handles forward set + reverse entries + presence atomically.
|
||||
self._registry.uninstall(ws_id)
|
||||
try:
|
||||
self._collector.emit_console_ws_closed(ws_id)
|
||||
except Exception:
|
||||
@@ -356,25 +329,6 @@ class CoordinatorAdapter:
|
||||
# Children registry
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_child_ids_locked(self, coord_ws_id: str, child_ids: Iterable[str]) -> None:
|
||||
"""Merge ``child_ids`` into ``coord_ws_id``'s forward + reverse maps.
|
||||
|
||||
Caller MUST hold ``self._children_lock``. Idempotent — re-adding
|
||||
an existing child is a no-op (the reverse-index pointer is
|
||||
already correct). Empty / falsy entries in ``child_ids`` are
|
||||
skipped.
|
||||
|
||||
Sole write-path for bulk registry updates so
|
||||
``_rebuild_children_registry`` (storage-seeded) and
|
||||
``_prime_children_from_snapshot`` (collector-seeded) agree on
|
||||
ordering and reverse-index invariants.
|
||||
"""
|
||||
existing = self._children.setdefault(coord_ws_id, set())
|
||||
for cid in child_ids:
|
||||
if cid and cid not in existing:
|
||||
existing.add(cid)
|
||||
self._child_to_coord[cid] = coord_ws_id
|
||||
|
||||
def _rebuild_children_registry(self, coord_ws_id: str) -> None:
|
||||
"""Populate ``self._children[coord_ws_id]`` from storage.
|
||||
|
||||
@@ -445,174 +399,88 @@ class CoordinatorAdapter:
|
||||
if not child_id:
|
||||
continue
|
||||
child_ids.append(child_id)
|
||||
with self._children_lock:
|
||||
self._merge_child_ids_locked(coord_ws_id, child_ids)
|
||||
|
||||
def _coord_for_child(self, child_ws_id: str) -> str | None:
|
||||
"""Reverse-lookup: which coordinator owns this child ws_id?
|
||||
|
||||
O(1) via the ``_child_to_coord`` reverse index. Cluster events
|
||||
fire on every token tick across the cluster; a linear scan here
|
||||
turned into a hot-path tax as the retained-history set grew.
|
||||
"""
|
||||
with self._children_lock:
|
||||
return self._child_to_coord.get(child_ws_id)
|
||||
self._registry.merge_children(coord_ws_id, child_ids)
|
||||
|
||||
def children_snapshot(self, coord_ws_id: str) -> list[str]:
|
||||
"""Return a snapshot of the coordinator's direct child ws_ids.
|
||||
|
||||
Used by ``stop_cascade`` to iterate children without holding the
|
||||
registry lock during the per-child HTTP dispatch. A mutation
|
||||
racing with the snapshot (child spawned mid-cascade) either
|
||||
lands before the snapshot and gets cancelled, or lands after
|
||||
and is out of scope for this batch — both outcomes are safe.
|
||||
Returns an empty list for unknown coordinators.
|
||||
Used by ``stop_cascade`` to iterate children without holding
|
||||
the registry lock during the per-child HTTP dispatch. A
|
||||
mutation racing with the snapshot (child spawned mid-cascade)
|
||||
either lands before (cancelled) or after (out of scope for
|
||||
this batch) — both safe. Returns an empty list for unknown
|
||||
coordinators.
|
||||
"""
|
||||
with self._children_lock:
|
||||
child_set = self._children.get(coord_ws_id)
|
||||
return list(child_set) if child_set else []
|
||||
|
||||
def _pop_coord_registry_locked(self, coord_ws_id: str) -> None:
|
||||
"""Remove a coordinator's forward set + reverse-index entries.
|
||||
|
||||
Caller MUST hold ``self._children_lock``. Used by close /
|
||||
eviction paths so stale coordinators don't leak registry
|
||||
entries. No-op if the coordinator is unknown.
|
||||
"""
|
||||
child_set = self._children.pop(coord_ws_id, None)
|
||||
if child_set is None:
|
||||
return
|
||||
for cid in child_set:
|
||||
# Defensive: only clear the reverse entry if it still points
|
||||
# at THIS coordinator. If a child has since been reassigned
|
||||
# (unusual but possible on schema changes), we don't want to
|
||||
# orphan the new owner's entry.
|
||||
if self._child_to_coord.get(cid) == coord_ws_id:
|
||||
self._child_to_coord.pop(cid, None)
|
||||
return self._registry.children_of(coord_ws_id)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cluster-event fan-out thread
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def start_child_event_fanout(self, collector: ClusterCollector) -> None:
|
||||
"""Subscribe to cluster events and start the filter + re-emit thread.
|
||||
"""Subscribe to cluster events via :class:`ClusterChildSource`.
|
||||
|
||||
Idempotent — calling twice is a no-op (already-started fan-out
|
||||
thread stays). Called once from the console lifespan after both
|
||||
the collector and the session manager are constructed.
|
||||
Idempotent — already-started ChildSource stays. Called once
|
||||
from the console lifespan after both the collector and the
|
||||
session manager are constructed.
|
||||
"""
|
||||
if self._fanout_thread is not None and self._fanout_thread.is_alive():
|
||||
if self._child_source is not None:
|
||||
return
|
||||
self._collector = collector
|
||||
self._collector_queue = queue.Queue(maxsize=1000)
|
||||
# Ensure the "console" pseudo-node exists in the snapshot map so
|
||||
# emit_console_ws_* calls from create / close / open land on a
|
||||
# real node entry the snapshot will surface.
|
||||
# Coord-specific transport setup: the "console" pseudo-node
|
||||
# must exist in the snapshot map BEFORE any
|
||||
# ``emit_console_ws_*`` calls (and before the snapshot is
|
||||
# taken inside ``ChildSource.start``) so those emits land on
|
||||
# a real node entry the snapshot surfaces.
|
||||
collector.ensure_console_pseudo_node()
|
||||
# Register with the collector — use the existing listener channel
|
||||
# the browser SSE fan-out uses; the collector treats our queue as
|
||||
# just another subscriber.
|
||||
snapshot = collector.get_snapshot_and_register(self._collector_queue)
|
||||
# Prime the child registry from the snapshot so a coordinator
|
||||
# that opens right after a console restart sees already-live
|
||||
# children without waiting for the next ``ws_state`` tick to
|
||||
# discover them via the fan-out path.
|
||||
self._prime_children_from_snapshot(snapshot)
|
||||
# Seed the pseudo-node with any coordinators already loaded in
|
||||
# memory when the collector binds. Prevents a race where early
|
||||
# creates happened before the collector was wired up and their
|
||||
# rows never showed on the snapshot.
|
||||
mgr = self._manager
|
||||
if mgr is not None:
|
||||
for ws in mgr.list_all():
|
||||
try:
|
||||
collector.emit_console_ws_created(
|
||||
ws.id,
|
||||
name=ws.name,
|
||||
user_id=ws.user_id or "",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=ws.state.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_adapter.collector_seed_failed ws=%s",
|
||||
ws.id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
self._fanout_stop.clear()
|
||||
t = threading.Thread(
|
||||
target=self._fanout_loop,
|
||||
name="coord-adapter-child-fanout",
|
||||
daemon=True,
|
||||
)
|
||||
self._fanout_thread = t
|
||||
t.start()
|
||||
|
||||
def _prime_children_from_snapshot(self, snapshot: dict[str, Any]) -> None:
|
||||
"""Populate ``_children`` + ``_child_to_coord`` from a collector snapshot.
|
||||
|
||||
The snapshot's per-node workstreams carry ``parent_ws_id``. For
|
||||
every workstream whose parent is an in-memory coordinator,
|
||||
record the child so the fan-out filter sees it immediately.
|
||||
"""
|
||||
nodes = snapshot.get("nodes", []) if isinstance(snapshot, dict) else []
|
||||
if not nodes:
|
||||
return
|
||||
mgr = self._manager
|
||||
if mgr is None:
|
||||
raise RuntimeError(
|
||||
"CoordinatorAdapter: manager not attached — call attach(mgr) after construction"
|
||||
)
|
||||
by_parent: dict[str, list[str]] = {}
|
||||
known = {ws.id for ws in mgr.list_all()}
|
||||
for node in nodes:
|
||||
for entry in node.get("workstreams", []) or []:
|
||||
parent = entry.get("parent_ws_id") or ""
|
||||
child_id = entry.get("id") or ""
|
||||
if not parent or not child_id or parent not in known:
|
||||
continue
|
||||
by_parent.setdefault(parent, []).append(child_id)
|
||||
if not by_parent:
|
||||
return
|
||||
with self._children_lock:
|
||||
for parent, kids in by_parent.items():
|
||||
self._merge_child_ids_locked(parent, kids)
|
||||
# Build + start the strategy. The sink is
|
||||
# :meth:`_dispatch_child_event` so cluster events flow through
|
||||
# the same translation path that synthesises ``child_ws_*``
|
||||
# payloads for the parent's UI.
|
||||
source = ClusterChildSource(
|
||||
collector=collector,
|
||||
registry=self._registry,
|
||||
parents_provider=lambda: [ws.id for ws in mgr.list_all()],
|
||||
)
|
||||
source.start(sink=self._dispatch_child_event)
|
||||
self._child_source = source
|
||||
# Seed the pseudo-node with any coordinators already loaded in
|
||||
# memory when the collector binds. Prevents a race where early
|
||||
# creates happened before the collector was wired up and their
|
||||
# rows never showed on the snapshot. (Coord-specific — interactive
|
||||
# has no analogous pseudo-node.)
|
||||
for ws in mgr.list_all():
|
||||
try:
|
||||
collector.emit_console_ws_created(
|
||||
ws.id,
|
||||
name=ws.name,
|
||||
user_id=ws.user_id or "",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=ws.state.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_adapter.collector_seed_failed ws=%s",
|
||||
ws.id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the fan-out thread and unregister from the collector.
|
||||
"""Stop the ChildSource and unregister from the collector.
|
||||
|
||||
Safe to call multiple times; idempotent. Invoked from the
|
||||
console lifespan teardown so SSE listener queues don't leak.
|
||||
"""
|
||||
self._fanout_stop.set()
|
||||
t = self._fanout_thread
|
||||
q = self._collector_queue
|
||||
coll = self._collector
|
||||
self._fanout_thread = None
|
||||
self._collector_queue = None
|
||||
if coll is not None and q is not None:
|
||||
try:
|
||||
coll.unregister_listener(q)
|
||||
except Exception:
|
||||
log.debug("coord_adapter.unregister_listener_failed", exc_info=True)
|
||||
if t is not None:
|
||||
t.join(timeout=2.0)
|
||||
|
||||
def _fanout_loop(self) -> None:
|
||||
"""Drain collector events, filter by known children, dispatch."""
|
||||
q = self._collector_queue
|
||||
if q is None:
|
||||
return
|
||||
while not self._fanout_stop.is_set():
|
||||
try:
|
||||
event = q.get(timeout=1.0)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
self._dispatch_child_event(event)
|
||||
except Exception:
|
||||
log.debug("coord_adapter.fanout.dispatch_failed", exc_info=True)
|
||||
src = self._child_source
|
||||
self._child_source = None
|
||||
if src is not None:
|
||||
src.shutdown()
|
||||
|
||||
def _dispatch_child_event(self, event: dict[str, Any]) -> None:
|
||||
"""Match a cluster event to a coordinator and re-emit on its UI.
|
||||
@@ -622,10 +490,12 @@ class CoordinatorAdapter:
|
||||
- ``ws_created`` with ``parent_ws_id`` matching an in-memory
|
||||
coordinator → add to registry + re-emit as
|
||||
``child_ws_created``.
|
||||
- ``cluster_state`` / ``ws_closed`` / ``ws_rename`` whose
|
||||
``ws_id`` is in any coordinator's known-children registry →
|
||||
re-emit as ``child_ws_state`` / ``child_ws_closed`` /
|
||||
``child_ws_rename``.
|
||||
- ``cluster_state`` / ``ws_closed`` / ``ws_rename`` /
|
||||
``intent_verdict`` / ``approval_resolved`` whose ``ws_id``
|
||||
is in any coordinator's known-children registry → re-emit as
|
||||
``child_ws_state`` / ``child_ws_closed`` /
|
||||
``child_ws_rename`` / ``child_ws_intent_verdict`` /
|
||||
``child_ws_approval_resolved``.
|
||||
|
||||
Events for ws_ids we don't own silently drop — the filter lives
|
||||
on the server so each coordinator's SSE stream stays small.
|
||||
@@ -639,23 +509,17 @@ class CoordinatorAdapter:
|
||||
parent = event.get("parent_ws_id") or ""
|
||||
if not parent:
|
||||
return
|
||||
# Presence check + registry mutation under the same lock:
|
||||
# a concurrent close()/eviction can pop the entry between
|
||||
# the check and the mutation, after which a bare setdefault
|
||||
# would resurrect the entry — leaking the registry key and
|
||||
# enqueuing onto the closed coordinator's UI. Trusted-team
|
||||
# posture (#400 / a46dab1) means no per-event tenant gate
|
||||
# here; scope-level auth at the SSE endpoint is the only
|
||||
# boundary.
|
||||
with self._children_lock:
|
||||
coord_ui = self._active_coords.get(parent)
|
||||
if coord_ui is None:
|
||||
return
|
||||
existing = self._children.setdefault(parent, set())
|
||||
if ws_id in existing:
|
||||
return
|
||||
existing.add(ws_id)
|
||||
self._child_to_coord[ws_id] = parent
|
||||
# Atomic check-and-route under the registry's lock: a
|
||||
# concurrent close()/eviction can pop the parent's entry
|
||||
# between the presence check and the mutation, so the two
|
||||
# must happen together. ``add_child`` returns the parent's
|
||||
# UI on success or None on (a) parent not installed, or
|
||||
# (b) duplicate child. Trusted-team posture (#400 / a46dab1)
|
||||
# means no per-event tenant gate here; scope-level auth at
|
||||
# the SSE endpoint is the only boundary.
|
||||
coord_ui = self._registry.add_child(parent, ws_id)
|
||||
if coord_ui is None:
|
||||
return
|
||||
payload = {
|
||||
"type": "child_ws_created",
|
||||
"ws_id": ws_id,
|
||||
@@ -668,8 +532,15 @@ class CoordinatorAdapter:
|
||||
_enqueue_on_ui(coord_ui, parent, payload)
|
||||
return
|
||||
|
||||
if etype in ("cluster_state", "ws_closed", "ws_rename"):
|
||||
coord_id = self._coord_for_child(ws_id)
|
||||
if etype in (
|
||||
"cluster_state",
|
||||
"ws_closed",
|
||||
"ws_rename",
|
||||
"intent_verdict",
|
||||
"approval_resolved",
|
||||
"approve_request",
|
||||
):
|
||||
coord_id = self._registry.parent_for(ws_id)
|
||||
if coord_id is None:
|
||||
return
|
||||
mgr = self._manager
|
||||
@@ -684,11 +555,12 @@ class CoordinatorAdapter:
|
||||
"state": event.get("state", ""),
|
||||
"tokens": event.get("tokens", 0),
|
||||
"node_id": event.get("node_id", ""),
|
||||
# activity_state lets the JS detect approval-state
|
||||
# transitions and fire urgent live-bulk fetches so
|
||||
# inline approve/deny buttons render in lockstep
|
||||
# with the child entering attention (instead of
|
||||
# waiting up to 5s for the next TTL window).
|
||||
# ``activity_state`` lets the JS detect approval
|
||||
# transitions and trigger a bulk fetch for the
|
||||
# initial detail. The detail itself no longer
|
||||
# piggybacks here (Stage 3 cleanup) — verdicts
|
||||
# arrive via the explicit ``intent_verdict`` event
|
||||
# class and resolution via ``approval_resolved``.
|
||||
"activity_state": event.get("activity_state", ""),
|
||||
}
|
||||
elif etype == "ws_closed":
|
||||
@@ -698,13 +570,52 @@ class CoordinatorAdapter:
|
||||
"parent_ws_id": coord_id,
|
||||
"reason": event.get("reason", ""),
|
||||
}
|
||||
else: # ws_rename
|
||||
elif etype == "ws_rename":
|
||||
child_event = {
|
||||
"type": "child_ws_rename",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"name": event.get("name", ""),
|
||||
}
|
||||
elif etype == "intent_verdict":
|
||||
# Per-coord re-emit of an explicit verdict event so
|
||||
# the tree UI can render the risk pill + verdict
|
||||
# result without polling. The corresponding
|
||||
# ``cluster_state`` event no longer carries the
|
||||
# detail (the piggyback was removed end-to-end);
|
||||
# the bulk fetch on initial approval entry plus this
|
||||
# explicit event class are the only carriers.
|
||||
child_event = {
|
||||
"type": "child_ws_intent_verdict",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"verdict": event.get("verdict") or {},
|
||||
}
|
||||
elif etype == "approval_resolved":
|
||||
child_event = {
|
||||
"type": "child_ws_approval_resolved",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"approved": bool(event.get("approved", False)),
|
||||
"feedback": event.get("feedback", "") or "",
|
||||
"always": bool(event.get("always", False)),
|
||||
}
|
||||
else: # approve_request
|
||||
# Push path for the initial approval items —
|
||||
# eliminates the bulk-fetch race that previously left
|
||||
# the coord row stuck on a loading placeholder when
|
||||
# the bulk fetch landed in the gap between the state
|
||||
# transition to ATTENTION and ``_pending_approval``
|
||||
# being set inside ``approve_tools``.
|
||||
child_event = {
|
||||
"type": "child_ws_approve_request",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"detail": event.get("detail") or {},
|
||||
}
|
||||
_enqueue_on_ui(owning_ws.ui, coord_id, child_event)
|
||||
|
||||
|
||||
|
||||
@@ -81,13 +81,13 @@ WAIT_MAX_TIMEOUT: float = 600.0
|
||||
WAIT_POLL_INTERVAL: float = 0.5
|
||||
|
||||
# Per-ws cap on the inline ``message`` field bundled into wait_for_workstream
|
||||
# results. Sized so a fan-out of 32 children at the cap is ~192 KiB of
|
||||
# results. Sized so a fan-out of 32 children at the cap is ~320 KiB of
|
||||
# tool output — large but not catastrophic on commercial models, and
|
||||
# typical waits run with a handful of children. Truncation is from the
|
||||
# END (the lead is usually more informative than the tail) and sets a
|
||||
# ``truncated=True`` flag so the model can opt into a follow-up read if
|
||||
# the trailing bytes matter.
|
||||
WAIT_MESSAGE_MAX_BYTES: int = 6 * 1024
|
||||
WAIT_MESSAGE_MAX_BYTES: int = 10 * 1024
|
||||
|
||||
# How many tail messages ``wait_for_workstream`` reads when extracting a
|
||||
# child's last assistant turn. The conversation tail almost always
|
||||
|
||||
@@ -181,6 +181,81 @@ class ConsoleCoordinatorUI(SessionUIBase):
|
||||
with self._ws_lock:
|
||||
self._last_broadcast_activity = current
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
"""Fan an LLM intent-judge verdict to the cluster collector.
|
||||
|
||||
Stage 3 Step 5 — overrides the no-op base hook so a coord
|
||||
workstream that produces its own verdict (rare — coord agents
|
||||
don't typically run the LLM judge) gets a first-class
|
||||
cluster-bus event for the parent's tree UI. The far more
|
||||
common path is a CHILD workstream firing its verdict on a
|
||||
real node; that path goes through ``WebUI._broadcast_intent_verdict``
|
||||
→ global SSE → collector ``_apply_delta``.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_intent_verdict(self.ws_id, verdict)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.intent_verdict_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool,
|
||||
feedback: str | None = None,
|
||||
*,
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision to the cluster collector.
|
||||
|
||||
Stage 3 Step 5 — paired with :meth:`_broadcast_intent_verdict`.
|
||||
Same rationale: coord-direct approvals are rare; the typical
|
||||
path is a child workstream resolving on its node, with that
|
||||
node's WebUI broadcasting through the global queue.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_approval_resolved(
|
||||
self.ws_id,
|
||||
approved=approved,
|
||||
feedback=feedback or "",
|
||||
always=always,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.approval_resolved_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None:
|
||||
"""Fan an ``approve_request`` payload to the cluster collector.
|
||||
|
||||
Push path for the initial approval items. Same rationale as
|
||||
the other two broadcast hooks: coord-self approvals are rare
|
||||
(the LLM judge isn't wired on the console coord today), but
|
||||
the override exists for symmetry and lights up the same path
|
||||
a future coord-self gate would use.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_approve_request(self.ws_id, detail)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.approve_request_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
# Flow state transitions through the unified SessionManager so
|
||||
# the storage write + adapter emit_state fan-out stay in lockstep
|
||||
|
||||
+213
-10
@@ -23,6 +23,7 @@ import queue
|
||||
import re
|
||||
import secrets
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -89,6 +90,7 @@ if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
|
||||
from turnstone.core.session import ChatSession
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
from turnstone.core.storage._protocol import StorageBackend
|
||||
|
||||
log = logging.getLogger("turnstone.console.server")
|
||||
@@ -844,8 +846,20 @@ async def _fetch_live_block(
|
||||
live = {k: entry.get(k) for k in _CLUSTER_WS_LIVE_KEYS if k in entry}
|
||||
# Derived field — kept in lockstep with
|
||||
# _coordinator_live_snapshot so both origins produce the
|
||||
# same keys.
|
||||
live["pending_approval"] = live.get("activity_state") == "approval"
|
||||
# same keys. ``state="attention"`` is the canonical signal;
|
||||
# ``activity_state="approval"`` is set inside approve_tools
|
||||
# AFTER the state transition fires, so a bulk fetch that
|
||||
# races with that window can see state=attention and
|
||||
# activity_state="" simultaneously. A non-null
|
||||
# ``pending_approval_detail`` is also a definitive signal
|
||||
# (the serializer only emits non-None when ``_pending_approval``
|
||||
# is set on the UI). Any of the three flips this true; the
|
||||
# frontend reducer mirrors the same disjunction.
|
||||
live["pending_approval"] = (
|
||||
live.get("activity_state") == "approval"
|
||||
or entry.get("state") == "attention"
|
||||
or live.get("pending_approval_detail") is not None
|
||||
)
|
||||
return live
|
||||
return None
|
||||
|
||||
@@ -2453,7 +2467,7 @@ def _require_admin_coordinator(
|
||||
)
|
||||
|
||||
|
||||
def _resolve_coordinator_or_404(
|
||||
async def _resolve_coordinator_or_404(
|
||||
request: Request,
|
||||
coord_mgr: Any,
|
||||
storage: Any,
|
||||
@@ -2484,7 +2498,11 @@ def _resolve_coordinator_or_404(
|
||||
if storage is None:
|
||||
return None, miss
|
||||
try:
|
||||
row = storage.get_workstream(ws_id)
|
||||
# Cold-cache path (every console restart, eviction, console
|
||||
# proxy hop) — offload the sync DB call so the coord
|
||||
# children/tasks handlers don't block the event loop on the
|
||||
# same DB the rest of the handler is unblocking.
|
||||
row = await asyncio.to_thread(storage.get_workstream, ws_id)
|
||||
except Exception:
|
||||
log.debug("resolve_coordinator.storage_failed ws=%s", ws_id[:8], exc_info=True)
|
||||
return None, miss
|
||||
@@ -2910,7 +2928,7 @@ async def coordinator_children(request: Request) -> JSONResponse:
|
||||
if not _VALID_WS_ID_RE.match(ws_id):
|
||||
return JSONResponse({"error": "invalid ws_id"}, status_code=400)
|
||||
user_id = _auth_user_id(request)
|
||||
_ws, err404 = _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
_ws, err404 = await _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
if err404 is not None:
|
||||
return err404
|
||||
|
||||
@@ -2918,7 +2936,8 @@ async def coordinator_children(request: Request) -> JSONResponse:
|
||||
# the full child subtree. ``user_id`` stays on each row as
|
||||
# metadata, not a filter.
|
||||
try:
|
||||
raw = storage.list_workstreams(
|
||||
raw = await asyncio.to_thread(
|
||||
storage.list_workstreams,
|
||||
limit=_CHILDREN_PAGE_LIMIT + 1,
|
||||
parent_ws_id=ws_id,
|
||||
kind=None,
|
||||
@@ -3033,7 +3052,7 @@ async def coordinator_metrics(request: Request) -> JSONResponse:
|
||||
if not _VALID_WS_ID_RE.match(ws_id):
|
||||
return JSONResponse({"error": "invalid ws_id"}, status_code=400)
|
||||
user_id = _auth_user_id(request)
|
||||
_ws, err404 = _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
_ws, err404 = await _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
if err404 is not None:
|
||||
return err404
|
||||
|
||||
@@ -3233,7 +3252,7 @@ async def _resolve_coord_session(
|
||||
if not _VALID_WS_ID_RE.match(ws_id):
|
||||
return JSONResponse({"error": "invalid ws_id"}, status_code=400)
|
||||
user_id = _auth_user_id(request)
|
||||
ws, err404 = _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
ws, err404 = await _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
if err404 is not None:
|
||||
return err404
|
||||
if ws is None or ws.session is None:
|
||||
@@ -3523,11 +3542,11 @@ async def coordinator_tasks(request: Request) -> JSONResponse:
|
||||
if not _VALID_WS_ID_RE.match(ws_id):
|
||||
return JSONResponse({"error": "invalid ws_id"}, status_code=400)
|
||||
user_id = _auth_user_id(request)
|
||||
_ws, err404 = _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
_ws, err404 = await _resolve_coordinator_or_404(request, coord_mgr, storage, ws_id, user_id)
|
||||
if err404 is not None:
|
||||
return err404
|
||||
|
||||
envelope, _corrupt = load_task_envelope(storage, ws_id)
|
||||
envelope, _corrupt = await asyncio.to_thread(load_task_envelope, storage, ws_id)
|
||||
return JSONResponse(envelope)
|
||||
|
||||
|
||||
@@ -3682,6 +3701,50 @@ async def _verify_collector_service_scope(app: Starlette, client: httpx.AsyncCli
|
||||
)
|
||||
|
||||
|
||||
def _coord_idle_cleanup_thread(
|
||||
mgr: SessionManager,
|
||||
timeout_sec: float,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> None:
|
||||
"""Periodically reap idle + DB-orphan coordinator workstreams.
|
||||
|
||||
Mirrors the regular server's ``_idle_cleanup_thread`` (turnstone/server.py)
|
||||
but skips the rate-limiter / global-queue arms — the console doesn't have
|
||||
those. ``mgr.close_idle`` does the work: closes loaded IDLE rows AND
|
||||
bulk-closes DB rows of this kind whose ``updated`` is past the cutoff
|
||||
and which aren't currently loaded. The latter pass catches coords left
|
||||
behind by prior console process incarnations.
|
||||
|
||||
Runs an initial sweep BEFORE the first sleep so cold-start orphans are
|
||||
reaped immediately rather than waiting one ``check_every`` interval (~30
|
||||
min on default 2h timeout). This intentionally diverges from the regular
|
||||
server pattern, which has no initial sweep — the regular server runs
|
||||
inside a normal request-handling lifecycle, the console-side coord pool
|
||||
is a small fixed-size cache where orphans dominate the row count after
|
||||
a cold boot.
|
||||
|
||||
``stop_event`` is for tests — when set, the thread exits cleanly after
|
||||
the next loop check. Production callers pass ``None`` (the daemon is
|
||||
process-lifetime).
|
||||
"""
|
||||
check_every = min(300.0, timeout_sec / 4)
|
||||
# Initial sweep — runs once before entering the sleep loop.
|
||||
try:
|
||||
mgr.close_idle(timeout_sec)
|
||||
except Exception:
|
||||
log.debug("console.coord_idle_cleanup_initial_failed", exc_info=True)
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
return
|
||||
time.sleep(check_every)
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
return
|
||||
try:
|
||||
mgr.close_idle(timeout_sec)
|
||||
except Exception:
|
||||
log.debug("console.coord_idle_cleanup_failed", exc_info=True)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
# Create async HTTP clients for proxy routes. Auth headers are NOT baked
|
||||
@@ -3750,6 +3813,18 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
audit_exec = ThreadPoolExecutor(max_workers=4, thread_name_prefix="coord-audit")
|
||||
app.state.audit_executor = audit_exec
|
||||
_set_audit_executor(audit_exec)
|
||||
# Dedicated executor for coord SSE queue polling, mirroring the
|
||||
# interactive-side ``sse_executor`` in ``turnstone/server.py``.
|
||||
# Each coord ``events`` SSE listener parks a thread on
|
||||
# ``client_queue.get(timeout=5)`` for the connection lifetime.
|
||||
# Without this pool, those parks land on Python's default
|
||||
# ThreadPoolExecutor (~min(32, cpu_count+4)) and compete with
|
||||
# every other ``asyncio.to_thread`` caller — a few coord tabs
|
||||
# against a multi-child workstream are enough to stall new
|
||||
# request handlers.
|
||||
app.state.coord_sse_executor = ThreadPoolExecutor(
|
||||
max_workers=200, thread_name_prefix="coord-sse"
|
||||
)
|
||||
# Populate the router's services cache if a router is configured
|
||||
_router: ConsoleRouter | None = getattr(app.state, "router", None)
|
||||
if _router is not None:
|
||||
@@ -3975,6 +4050,27 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
coord_adapter.start_child_event_fanout(app.state.collector)
|
||||
except Exception:
|
||||
log.warning("console.coordinator_child_fanout_init_failed", exc_info=True)
|
||||
# Idle cleanup: closes loaded-but-stale coords AND DB orphans
|
||||
# left behind by prior console processes. The thread runs an
|
||||
# initial sweep on entry (no synchronous lifespan call needed —
|
||||
# see ``_coord_idle_cleanup_thread``) so cold-start cleanup
|
||||
# doesn't block startup. Reuses the regular-server
|
||||
# ``server.workstream_idle_timeout`` setting — the same cadence
|
||||
# makes sense for both kinds and avoids a redundant config knob.
|
||||
try:
|
||||
idle_minutes = int(config_store.get("server.workstream_idle_timeout"))
|
||||
except Exception:
|
||||
idle_minutes = 0
|
||||
if idle_minutes > 0:
|
||||
timeout_sec = float(idle_minutes * 60)
|
||||
cleanup_thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(coord_mgr, timeout_sec),
|
||||
name="coord-idle-cleanup",
|
||||
daemon=True,
|
||||
)
|
||||
cleanup_thread.start()
|
||||
app.state.coord_idle_cleanup_thread = cleanup_thread
|
||||
log.info(
|
||||
"console.coordinator_mgr_ready max_active=%s",
|
||||
config_store.get("coordinator.max_active"),
|
||||
@@ -4029,6 +4125,15 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
if audit_exec_shutdown is not None:
|
||||
_set_audit_executor(None)
|
||||
audit_exec_shutdown.shutdown(wait=True)
|
||||
# Drain the coord SSE pool AFTER ``coord_adapter.shutdown()`` above:
|
||||
# adapter shutdown deregisters listeners so no new events handlers
|
||||
# arrive at this pool; in-flight ``client_queue.get`` futures
|
||||
# already running are bounded by their 5s timeout and finish
|
||||
# naturally. ``cancel_futures=True`` discards any queued-but-not-
|
||||
# started futures so we don't block lifespan teardown on them.
|
||||
coord_sse_exec_shutdown = getattr(app.state, "coord_sse_executor", None)
|
||||
if coord_sse_exec_shutdown is not None:
|
||||
coord_sse_exec_shutdown.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -7996,6 +8101,82 @@ async def _collect_model_status(
|
||||
return {nid: models for nid, models in results if models is not None}
|
||||
|
||||
|
||||
def _refresh_coord_registry(app_state: Any, storage: Any) -> None:
|
||||
"""Rebuild ``app_state.coord_registry`` in place from DB model definitions.
|
||||
|
||||
The console-side coordinator session factory closes over the
|
||||
``coord_registry`` instance built at lifespan startup
|
||||
(see this module's lifespan setup and ``console/session_factory.py``).
|
||||
Replacing the attribute would orphan the closure — new sessions would
|
||||
still resolve through the stale object. Mutating in place via
|
||||
``ModelRegistry.reload()`` preserves identity, so:
|
||||
|
||||
- new coordinator sessions see the new state at create-time;
|
||||
- active coordinator sessions auto-pick up the swap at next ``send()``
|
||||
via ``ChatSession._refresh_model_from_registry`` (the per-send
|
||||
check compares ``cfg.model`` against ``self.model`` and re-resolves
|
||||
on mismatch).
|
||||
|
||||
Errors are logged + swallowed. The DB write that triggered this
|
||||
refresh has already succeeded, and the explicit reload button
|
||||
remains the user-facing recovery path. Validation failures
|
||||
(e.g. admin deleted the alias that ``registry.default`` points at)
|
||||
leave the existing registry intact rather than tearing down a
|
||||
working coordinator.
|
||||
"""
|
||||
from turnstone.core.model_registry import load_model_registry
|
||||
|
||||
existing = getattr(app_state, "coord_registry", None)
|
||||
if existing is None:
|
||||
# Lifespan didn't build a coord_registry (no DB model rows at boot)
|
||||
# — the entire coord subsystem stayed uninitialized, so a console
|
||||
# restart is required after the operator adds the first row.
|
||||
return
|
||||
try:
|
||||
# ``strict=True`` so a transient DB read error surfaces here.
|
||||
# Without it, the loader degrades to a config.toml-only registry
|
||||
# and ``existing.reload()`` would silently drop every DB-sourced
|
||||
# alias.
|
||||
new_registry = load_model_registry(storage=storage, strict=True)
|
||||
except ValueError as exc:
|
||||
# ModelRegistry.__init__ raises ValueError for several distinct
|
||||
# config issues — empty models, default/fallback/agent/plan/task
|
||||
# alias not present in the loaded set. Log the actual reason so
|
||||
# operators can tell "no enabled rows" from "default alias typo
|
||||
# in config.toml". Existing registry stays in place either way.
|
||||
log.warning("console.coord_registry_refresh_skipped reason=%s", exc)
|
||||
return
|
||||
except Exception:
|
||||
log.warning("console.coord_registry_refresh_load_failed", exc_info=True)
|
||||
return
|
||||
try:
|
||||
existing.reload(
|
||||
new_registry.models,
|
||||
new_registry.default,
|
||||
new_registry.fallback,
|
||||
new_registry.agent_model,
|
||||
plan_model=new_registry.plan_model,
|
||||
task_model=new_registry.task_model,
|
||||
plan_effort=new_registry.plan_effort,
|
||||
task_effort=new_registry.task_effort,
|
||||
)
|
||||
except Exception:
|
||||
log.warning("console.coord_registry_refresh_reload_failed", exc_info=True)
|
||||
finally:
|
||||
# Defensive — load_model_registry doesn't eagerly create clients
|
||||
# (ModelRegistry.__init__ leaves _clients/_providers empty; they
|
||||
# populate lazily on first resolve), so shutdown() iterates empty
|
||||
# dicts in practice. Kept against the day the loader grows
|
||||
# eager-init or a future caller pre-warms the throwaway, and
|
||||
# wrapped because shutdown() in finally would otherwise escape
|
||||
# after a successful in-place reload — surfacing as 500 with the
|
||||
# registry actually mutated and the audit row recording success.
|
||||
try:
|
||||
new_registry.shutdown()
|
||||
except Exception:
|
||||
log.warning("console.coord_registry_refresh_shutdown_failed", exc_info=True)
|
||||
|
||||
|
||||
async def _notify_nodes_model_reload(request: Request) -> dict[str, Any]:
|
||||
"""Tell all nodes to re-read model definitions from DB and rebuild registry."""
|
||||
collector: ClusterCollector = request.app.state.collector
|
||||
@@ -8232,6 +8413,8 @@ async def admin_create_model_definition(request: Request) -> JSONResponse:
|
||||
ip,
|
||||
)
|
||||
|
||||
await asyncio.to_thread(_refresh_coord_registry, request.app.state, storage)
|
||||
|
||||
created = storage.get_model_definition(definition_id)
|
||||
if created is None:
|
||||
return JSONResponse(
|
||||
@@ -8389,6 +8572,9 @@ async def admin_update_model_definition(request: Request) -> JSONResponse:
|
||||
ip,
|
||||
)
|
||||
|
||||
if updates:
|
||||
await asyncio.to_thread(_refresh_coord_registry, request.app.state, storage)
|
||||
|
||||
model_def = storage.get_model_definition(definition_id)
|
||||
return JSONResponse(_mask_model_secrets(model_def or {}))
|
||||
|
||||
@@ -8424,6 +8610,8 @@ async def admin_delete_model_definition(request: Request) -> JSONResponse:
|
||||
ip,
|
||||
)
|
||||
|
||||
await asyncio.to_thread(_refresh_coord_registry, request.app.state, storage)
|
||||
|
||||
return JSONResponse({"status": "ok", "definition_id": definition_id})
|
||||
|
||||
|
||||
@@ -8443,6 +8631,13 @@ async def admin_model_reload(request: Request) -> JSONResponse:
|
||||
# before they rebuild their model registries.
|
||||
await _publish_config_change(request)
|
||||
|
||||
# Refresh the console's own coord_registry first. The node fan-out
|
||||
# below carries DB→nodes propagation; the console hosts coordinator
|
||||
# sessions itself and must mutate its in-process registry too —
|
||||
# otherwise the coord LLM keeps calling the prior model name even
|
||||
# after a successful reload.
|
||||
await asyncio.to_thread(_refresh_coord_registry, request.app.state, storage)
|
||||
|
||||
results = await _notify_nodes_model_reload(request)
|
||||
return JSONResponse({"status": "ok", "results": results})
|
||||
|
||||
@@ -10196,6 +10391,14 @@ def create_app(
|
||||
# tombstones are non-resurrectable.
|
||||
saved_state_filter="closed",
|
||||
saved_loaded_lookup=_coord_saved_loaded_lookup,
|
||||
# Isolate coord SSE polling on its own 200-thread pool so a
|
||||
# handful of coord tabs (each parking a thread on
|
||||
# ``client_queue.get``) can't starve the default executor and
|
||||
# stall every other ``asyncio.to_thread`` caller (storage,
|
||||
# router, audit). Mirrors the interactive endpoint's
|
||||
# ``sse_executor_lookup`` wiring on ``interactive_endpoint_config``
|
||||
# in ``turnstone/server.py``.
|
||||
sse_executor_lookup=lambda request: request.app.state.coord_sse_executor,
|
||||
)
|
||||
coord_workstream_routes: list[Any] = []
|
||||
register_session_routes(
|
||||
|
||||
+160
-10
@@ -1851,6 +1851,16 @@ var _savedCoordsRetry = false;
|
||||
|
||||
function loadSavedCoordinators() {
|
||||
if (!_hasCoordPermission()) return;
|
||||
// Freeze the list while the user is multi-selecting — re-rendering
|
||||
// mid-mode would shuffle the visible page out from under them. The
|
||||
// delete-mode wrapper drains the retry flag on cancel/onClose.
|
||||
if (
|
||||
typeof _coordDeleteController !== "undefined" &&
|
||||
_coordDeleteController.inMode()
|
||||
) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
}
|
||||
if (_savedCoordsInFlight) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
@@ -1861,6 +1871,16 @@ function loadSavedCoordinators() {
|
||||
return r.ok ? r.json() : { workstreams: [] };
|
||||
})
|
||||
.then(function (data) {
|
||||
// Belt-and-braces: if the user entered delete mode while this
|
||||
// fetch was already in flight, defer the render — re-rendering
|
||||
// mid-selection would shuffle visible cards and reshape selections.
|
||||
if (
|
||||
typeof _coordDeleteController !== "undefined" &&
|
||||
_coordDeleteController.inMode()
|
||||
) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
}
|
||||
renderSavedCoordinators(data.workstreams || []);
|
||||
})
|
||||
.catch(function () {
|
||||
@@ -1878,7 +1898,52 @@ function loadSavedCoordinators() {
|
||||
});
|
||||
}
|
||||
|
||||
// Saved Coordinators: paginated card list + multi-select delete.
|
||||
// The shared controller (createSavedCardsController in /shared/cards.js)
|
||||
// owns mode state, checkbox decoration, the toolbar, and the modal.
|
||||
// Pagination caps Select-All fan-out at COORD_PAGE_SIZE — the controller
|
||||
// only ever sees the visible page, so a confirm-all batch is bounded to
|
||||
// COORD_PAGE_SIZE parallel POSTs against the routing proxy.
|
||||
var COORD_PAGE_SIZE = 24;
|
||||
var _coordPage = 0;
|
||||
var _coordSavedItems = [];
|
||||
var _coordDeleteController = createSavedCardsController({
|
||||
idPrefix: "coord-delete",
|
||||
buttonId: "coord-delete-btn",
|
||||
noun: "coordinator",
|
||||
activateLabel: function (s) {
|
||||
return "Resume coordinator: " + (s.alias || s.title || s.name || s.ws_id);
|
||||
},
|
||||
// Coordinators live on whichever node owns the ws_id, so we can't fire
|
||||
// a path-keyed delete the way ui/static does. The router proxy reads
|
||||
// ws_id from the body, resolves the owning node via the consistent-
|
||||
// hash ring, and forwards to that node's POST workstreams/{ws_id}/delete.
|
||||
buildDeleteRequest: function (wsId) {
|
||||
return {
|
||||
url: "/v1/api/route/workstreams/delete",
|
||||
options: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ ws_id: wsId }),
|
||||
},
|
||||
};
|
||||
},
|
||||
render: function () {
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
},
|
||||
onClose: function () {
|
||||
// Drain queued retries before the explicit reload — without this,
|
||||
// _savedCoordsRetry is still true from SSE events that arrived
|
||||
// during the freeze, so loadSavedCoordinators's .finally() would
|
||||
// re-fire a second fetch immediately after the first resolves.
|
||||
// Same idiom as cancelCoordDeleteMode below.
|
||||
_savedCoordsRetry = false;
|
||||
loadSavedCoordinators();
|
||||
},
|
||||
});
|
||||
|
||||
function renderSavedCoordinators(items) {
|
||||
_coordSavedItems = items;
|
||||
var section = document.getElementById("saved-coordinators");
|
||||
var cards = document.getElementById("saved-coord-cards");
|
||||
var countEl = document.getElementById("saved-coord-count");
|
||||
@@ -1887,24 +1952,31 @@ function renderSavedCoordinators(items) {
|
||||
section.style.display = "none";
|
||||
cards.replaceChildren();
|
||||
if (countEl) countEl.textContent = "";
|
||||
_coordPage = 0;
|
||||
if (_coordDeleteController.inMode()) _coordDeleteController.cancel();
|
||||
_renderCoordPagination();
|
||||
return;
|
||||
}
|
||||
// Clamp the page index after deletes (or upstream churn) shrink the list.
|
||||
var pages = Math.max(1, Math.ceil(items.length / COORD_PAGE_SIZE));
|
||||
if (_coordPage > pages - 1) _coordPage = pages - 1;
|
||||
if (_coordPage < 0) _coordPage = 0;
|
||||
var visible = items.slice(
|
||||
_coordPage * COORD_PAGE_SIZE,
|
||||
(_coordPage + 1) * COORD_PAGE_SIZE,
|
||||
);
|
||||
_coordDeleteController.setItems(visible);
|
||||
|
||||
section.style.display = "";
|
||||
if (countEl) countEl.textContent = "(" + items.length + ")";
|
||||
cards.replaceChildren();
|
||||
items.forEach(function (sess) {
|
||||
visible.forEach(function (sess) {
|
||||
var card = renderSessionCard(sess, {
|
||||
ariaLabel: function (s) {
|
||||
return (
|
||||
"Resume coordinator: " + (s.alias || s.title || s.name || s.ws_id)
|
||||
);
|
||||
},
|
||||
ariaLabel: _coordDeleteController.ariaLabel,
|
||||
onActivate: function (s, cardEl) {
|
||||
if (_coordDeleteController.blockActivate()) return;
|
||||
// POST /open BEFORE navigating so capacity issues surface as a
|
||||
// toast instead of a broken-looking detail page. The /open
|
||||
// endpoint calls the same lazy-rehydrate path the GET would,
|
||||
// but we get the status code synchronously so the user learns
|
||||
// "all slots in use" instead of staring at a 404.
|
||||
// toast instead of a broken-looking detail page.
|
||||
cardEl.classList.add("is-busy");
|
||||
authFetch(
|
||||
"/v1/api/workstreams/" + encodeURIComponent(s.ws_id) + "/open",
|
||||
@@ -1936,8 +2008,86 @@ function renderSavedCoordinators(items) {
|
||||
});
|
||||
},
|
||||
});
|
||||
_coordDeleteController.decorateCard(card, sess);
|
||||
cards.appendChild(card);
|
||||
});
|
||||
if (_coordDeleteController.inMode()) _coordDeleteController.refreshBar();
|
||||
_renderCoordPagination();
|
||||
}
|
||||
|
||||
function _renderCoordPagination() {
|
||||
var pag = document.getElementById("coord-pagination");
|
||||
if (!pag) return;
|
||||
var total = _coordSavedItems.length;
|
||||
var pages = Math.max(1, Math.ceil(total / COORD_PAGE_SIZE));
|
||||
// Single-page lists and delete-mode hide the controls — page changes
|
||||
// would invalidate the user's checkbox selections, so we lock them out.
|
||||
if (pages <= 1 || _coordDeleteController.inMode()) {
|
||||
pag.style.display = "none";
|
||||
return;
|
||||
}
|
||||
pag.style.display = "";
|
||||
var label = document.getElementById("coord-page-label");
|
||||
if (label) {
|
||||
/* Visible text uses the terse "X / Y" form to match the
|
||||
filtered-pagination control elsewhere in the console; the long
|
||||
form sits on the parent's aria-label so screen readers still get
|
||||
a full sentence. */
|
||||
label.textContent = _coordPage + 1 + " / " + pages;
|
||||
pag.setAttribute(
|
||||
"aria-label",
|
||||
"Saved coordinators pagination — page " +
|
||||
(_coordPage + 1) +
|
||||
" of " +
|
||||
pages,
|
||||
);
|
||||
}
|
||||
var prev = document.getElementById("coord-page-prev");
|
||||
if (prev) prev.disabled = _coordPage <= 0;
|
||||
var next = document.getElementById("coord-page-next");
|
||||
if (next) next.disabled = _coordPage >= pages - 1;
|
||||
}
|
||||
|
||||
function coordPagePrev() {
|
||||
if (_coordPage > 0) {
|
||||
_coordPage--;
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
}
|
||||
}
|
||||
|
||||
function coordPageNext() {
|
||||
var pages = Math.max(1, Math.ceil(_coordSavedItems.length / COORD_PAGE_SIZE));
|
||||
if (_coordPage < pages - 1) {
|
||||
_coordPage++;
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
}
|
||||
}
|
||||
|
||||
// HTML inline-onclick wrappers — keep the global names the markup binds
|
||||
// to and forward to the shared controller.
|
||||
function startCoordDeleteMode() {
|
||||
_coordDeleteController.start();
|
||||
}
|
||||
function cancelCoordDeleteMode() {
|
||||
_coordDeleteController.cancel();
|
||||
// The freeze gate (see loadSavedCoordinators) may have queued retries
|
||||
// while we were multi-selecting; drain them now that we're idle again.
|
||||
if (_savedCoordsRetry) {
|
||||
_savedCoordsRetry = false;
|
||||
loadSavedCoordinators();
|
||||
}
|
||||
}
|
||||
function toggleCoordSelectAll() {
|
||||
_coordDeleteController.toggleAll();
|
||||
}
|
||||
function confirmCoordDeleteSelection() {
|
||||
_coordDeleteController.confirmSelection();
|
||||
}
|
||||
function cancelCoordDelete() {
|
||||
_coordDeleteController.closeModal();
|
||||
}
|
||||
function confirmCoordDelete() {
|
||||
_coordDeleteController.confirm();
|
||||
}
|
||||
|
||||
// --- Init ---
|
||||
|
||||
@@ -396,6 +396,87 @@
|
||||
background: color-mix(in srgb, var(--err) 12%, var(--panel-2));
|
||||
}
|
||||
|
||||
/* Output-guard finding rendered under its specific .coord-tool-row.
|
||||
Stays anchored to the call that tripped the guard rather than
|
||||
floating into the chat log as a generic "[output guard]" line —
|
||||
matches interactive's `.output-warning` placement convention.
|
||||
Severity drives the hue (matches .verdict-badge.verdict-* palette
|
||||
so an operator scanning a workstream reads risk consistently
|
||||
across both surfaces). */
|
||||
.coord-tool-row-warning {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
margin-top: 4px;
|
||||
padding: 2px 8px;
|
||||
font-size: 11px;
|
||||
font-family: var(--font-mono);
|
||||
border: 1px solid var(--hair);
|
||||
border-radius: 3px;
|
||||
border-left-width: 3px;
|
||||
background: var(--panel-2);
|
||||
color: var(--ink-2);
|
||||
max-width: max-content;
|
||||
}
|
||||
.coord-tool-row-warning--low {
|
||||
color: color-mix(in srgb, var(--ok) 70%, var(--ink-2));
|
||||
border-left-color: var(--ok);
|
||||
background: color-mix(in srgb, var(--ok) 12%, var(--panel-2));
|
||||
}
|
||||
.coord-tool-row-warning--medium {
|
||||
color: color-mix(in srgb, var(--warn) 70%, var(--ink-2));
|
||||
border-left-color: var(--warn);
|
||||
background: var(--warn-tint);
|
||||
}
|
||||
.coord-tool-row-warning--high,
|
||||
.coord-tool-row-warning--critical {
|
||||
color: color-mix(in srgb, var(--err) 70%, var(--ink-2));
|
||||
border-left-color: var(--err);
|
||||
background: color-mix(in srgb, var(--err) 12%, var(--panel-2));
|
||||
}
|
||||
.coord-tool-row-warning-redacted {
|
||||
color: var(--ink-3);
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
/* Storage-truncation indicator — same convention as the interactive
|
||||
UI's `.tool-output-truncated` pill (transparent bg, dim border,
|
||||
small font) so the operator reads the affordance the same way on
|
||||
both surfaces. Sibling node next to .coord-tool-row-result rather
|
||||
than text-in-content so a future "best-effort JSON repair" pass
|
||||
on the result body doesn't have to strip a marker string. */
|
||||
.coord-tool-truncated {
|
||||
display: inline-block;
|
||||
margin-top: 4px;
|
||||
margin-left: 6px;
|
||||
padding: 1px 6px;
|
||||
font-size: 10px;
|
||||
font-family: var(--font-mono);
|
||||
color: var(--ink-3);
|
||||
background: transparent;
|
||||
border: 1px solid var(--ink-3);
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
/* memory/recall calls are background metadata — the audit trail is
|
||||
useful but they crowd the tree on workstreams with heavy memory
|
||||
usage. Dim the row by default; full opacity on hover so they
|
||||
stay inspectable. Mirrors the interactive UI's metacog dim rule
|
||||
(style.css `.ts-approval-tool[data-func-name="memory"]`). The
|
||||
row stamps `data-tool-name` from item.func_name in coordinator.js
|
||||
so this selector has something to match. */
|
||||
.coord-tool-row[data-tool-name="memory"],
|
||||
.coord-tool-row[data-tool-name="recall"] {
|
||||
opacity: 0.55;
|
||||
transition: opacity 120ms ease-out;
|
||||
}
|
||||
.coord-tool-row[data-tool-name="memory"]:hover,
|
||||
.coord-tool-row[data-tool-name="memory"]:focus-within,
|
||||
.coord-tool-row[data-tool-name="recall"]:hover,
|
||||
.coord-tool-row[data-tool-name="recall"]:focus-within {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Tool result block — paired under its row, mono pre-block. Capped
|
||||
at 240px with internal scroll so a long tool output doesn't push
|
||||
the rest of the chat off-screen. The interactive UI uses a
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -182,6 +182,36 @@
|
||||
font-size: 11px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
/* Loading-state placeholder rendered while the bulk fetch is
|
||||
in-flight. Same outer container as the real block so the row
|
||||
height is stable when the real content swaps in. */
|
||||
.ch-row .approval-block-loading {
|
||||
color: var(--ink-3);
|
||||
}
|
||||
.ch-row .approval-loading-spin {
|
||||
display: inline-block;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
border: 1.5px solid var(--accent);
|
||||
border-top-color: transparent;
|
||||
margin-right: 5px;
|
||||
animation: ts-spin 0.9s linear infinite;
|
||||
vertical-align: middle;
|
||||
}
|
||||
/* Inline status note shown when the operator clicks Approve/Deny
|
||||
but the call_id is stale (already resolved on another channel).
|
||||
Quieter than a toast — the row is about to be replaced wholesale
|
||||
by the refresh, so this only needs to bridge ~350ms. */
|
||||
.ch-row .approval-stale-note {
|
||||
font-size: 11px;
|
||||
color: var(--ink-3);
|
||||
font-style: italic;
|
||||
margin-top: 4px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ch-row .approval-loading-spin { animation: none; }
|
||||
}
|
||||
/* Auto-approved pill — same 22px indent as .approval-block / .meta
|
||||
so a child row showing both stacks cleanly. Lower visual weight
|
||||
than the live approve/deny block (this is informational, not
|
||||
|
||||
@@ -165,6 +165,14 @@
|
||||
<h2 class="home-section-title">
|
||||
<span>Saved Coordinators</span>
|
||||
<span id="saved-coord-count" class="home-section-count"></span>
|
||||
<button
|
||||
id="coord-delete-btn"
|
||||
class="ws-delete-btn home-section-action"
|
||||
onclick="startCoordDeleteMode()"
|
||||
title="Delete coordinators"
|
||||
>
|
||||
<span aria-hidden="true">🗑</span> Delete
|
||||
</button>
|
||||
</h2>
|
||||
<div
|
||||
id="saved-coord-cards"
|
||||
@@ -172,6 +180,61 @@
|
||||
role="list"
|
||||
aria-live="polite"
|
||||
></div>
|
||||
<div
|
||||
id="coord-pagination"
|
||||
class="pagination"
|
||||
style="display: none"
|
||||
role="navigation"
|
||||
aria-label="Saved coordinators pagination"
|
||||
>
|
||||
<button
|
||||
id="coord-page-prev"
|
||||
type="button"
|
||||
onclick="coordPagePrev()"
|
||||
>
|
||||
◄ Prev
|
||||
</button>
|
||||
<span id="coord-page-label" aria-live="polite" aria-atomic="true">
|
||||
</span>
|
||||
<button
|
||||
id="coord-page-next"
|
||||
type="button"
|
||||
onclick="coordPageNext()"
|
||||
>
|
||||
Next ►
|
||||
</button>
|
||||
</div>
|
||||
<div id="coord-delete-bar" class="ws-delete-bar">
|
||||
<span
|
||||
class="ws-delete-count-label"
|
||||
id="coord-delete-bar-count"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
aria-atomic="true"
|
||||
>0 selected</span
|
||||
>
|
||||
<button
|
||||
class="ws-delete-cancel-btn"
|
||||
onclick="cancelCoordDeleteMode()"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
class="ws-delete-selectall-btn"
|
||||
id="coord-delete-bar-select-all"
|
||||
onclick="toggleCoordSelectAll()"
|
||||
>
|
||||
Select All
|
||||
</button>
|
||||
<button
|
||||
class="ws-delete-bar-btn"
|
||||
id="coord-delete-bar-delete"
|
||||
onclick="confirmCoordDeleteSelection()"
|
||||
disabled
|
||||
>
|
||||
Delete Selected
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Cluster details — node list, always visible. The list is
|
||||
@@ -3907,6 +3970,40 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Delete coordinators confirmation modal (batch) -->
|
||||
<div
|
||||
id="coord-delete-overlay"
|
||||
class="ws-delete-modal-overlay"
|
||||
style="display: none"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="coord-delete-title"
|
||||
>
|
||||
<div id="coord-delete-box" class="ws-delete-modal-box">
|
||||
<h3 id="coord-delete-title">Delete Coordinators</h3>
|
||||
<div id="coord-delete-error" role="alert" aria-live="assertive"></div>
|
||||
<p id="coord-delete-count"></p>
|
||||
<div id="coord-delete-list" class="ws-delete-modal-list"></div>
|
||||
<div id="coord-delete-buttons" class="ws-delete-modal-buttons">
|
||||
<button
|
||||
id="coord-delete-cancel-btn"
|
||||
type="button"
|
||||
onclick="cancelCoordDelete()"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
id="coord-delete-confirm-btn"
|
||||
class="ws-delete-confirm"
|
||||
type="button"
|
||||
onclick="confirmCoordDelete()"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="/static/admin.js"></script>
|
||||
<script src="/static/governance.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
|
||||
@@ -133,6 +133,17 @@
|
||||
font-size: 11px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
/* Right-align action buttons (e.g. Saved Coordinators "Delete") inside
|
||||
.home-section-title without breaking the count's natural left position. */
|
||||
.home-section-title .home-section-action {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
/* Saved Coordinators reuses the existing .pagination control (see the
|
||||
"Pagination" block below) — the visible page is capped at
|
||||
COORD_PAGE_SIZE so Select-All fan-out is bounded. Pagination is
|
||||
hidden in delete mode and when there's only one page (see
|
||||
_renderCoordPagination). */
|
||||
|
||||
.home-coord-list {
|
||||
border: 1px solid var(--border);
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Strategy interface for delivering child workstream lifecycle events.
|
||||
|
||||
Two implementations bind to the unified :class:`ChildrenRegistry`:
|
||||
|
||||
- :class:`SameNodeChildSource` — subscribes to a :class:`SessionManager`'s
|
||||
state-change callbacks. In-process, no transport. For interactive
|
||||
workstreams that spawn children locally (no cluster routing).
|
||||
- :class:`ClusterChildSource` — subscribes to a :class:`ClusterCollector`'s
|
||||
listener channel and runs a daemon thread that drains the queue and
|
||||
pushes events to the sink. For coordinator workstreams whose
|
||||
children are routed across the cluster by hash bucket.
|
||||
|
||||
The strategy doesn't translate events into UI-shaped payloads; that's
|
||||
the sink's job. This split keeps the strategy generic across kinds and
|
||||
lets the consumer (e.g. ``CoordinatorAdapter._dispatch_child_event``)
|
||||
own the per-kind translation.
|
||||
|
||||
Sink signature: ``Callable[[dict[str, Any]], None]``. The sink is
|
||||
responsible for filtering by registry membership; strategies push raw
|
||||
events without registry-side filtering so the sink can decide whether
|
||||
to act based on its own state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class ChildSource(Protocol):
|
||||
"""Subscription strategy for child workstream lifecycle events."""
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
"""Begin delivering events to ``sink``. Idempotent."""
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the strategy. Idempotent; safe to call multiple times."""
|
||||
|
||||
|
||||
class _CollectorProtocol(Protocol):
|
||||
"""Subset of :class:`ClusterCollector` that :class:`ClusterChildSource` consumes.
|
||||
|
||||
Defined here (not imported) to keep ``turnstone/core/`` free of
|
||||
``turnstone/console/`` imports AND to let test fakes satisfy the
|
||||
type signature without subclassing the real collector.
|
||||
"""
|
||||
|
||||
def get_snapshot_and_register(self, q: queue.Queue[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Register ``q`` and return the current snapshot."""
|
||||
|
||||
def unregister_listener(self, q: queue.Queue[dict[str, Any]]) -> None:
|
||||
"""Drop ``q`` from the listener set."""
|
||||
|
||||
|
||||
class _ManagerProtocol(Protocol):
|
||||
"""Subset of :class:`SessionManager` that :class:`SameNodeChildSource` consumes.
|
||||
|
||||
Lets test fakes participate in the strategy's typed surface
|
||||
without forcing a full SessionManager construction.
|
||||
"""
|
||||
|
||||
def subscribe_to_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Register ``callback`` for state-change events."""
|
||||
|
||||
def unsubscribe_from_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Remove a previously-registered ``callback``."""
|
||||
|
||||
|
||||
class SameNodeChildSource:
|
||||
"""In-process child events via :class:`SessionManager` state observer.
|
||||
|
||||
Subscribes to the manager's state-change callbacks (registered via
|
||||
:meth:`SessionManager.subscribe_to_state`). For each transition on
|
||||
a workstream that's a known child (per the
|
||||
:class:`ChildrenRegistry` reverse index), synthesises a
|
||||
cluster-state-shaped event and pushes it to the sink.
|
||||
|
||||
Used by interactive workstreams when they gain spawn capability —
|
||||
children live on the same node as the parent, so no cluster routing
|
||||
is needed and event fan-out is in-process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
manager: _ManagerProtocol,
|
||||
registry: ChildrenRegistry,
|
||||
) -> None:
|
||||
self._manager = manager
|
||||
self._registry = registry
|
||||
self._sink: Callable[[dict[str, Any]], None] | None = None
|
||||
self._callback: Callable[[str, WorkstreamState], None] | None = None
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
if self._callback is not None:
|
||||
return # idempotent — already started
|
||||
self._sink = sink
|
||||
|
||||
def _on_state(ws_id: str, state: WorkstreamState) -> None:
|
||||
sink_fn = self._sink
|
||||
if sink_fn is None:
|
||||
return
|
||||
# Cheap pre-filter: skip dispatch for transitions on
|
||||
# workstreams that aren't children of any in-memory parent.
|
||||
# ``has_children`` is a lock-free dict-truthiness read; the
|
||||
# ``parent_for`` call below would otherwise acquire the
|
||||
# registry lock on every state change even when no
|
||||
# children exist (the steady state for an interactive
|
||||
# manager). The cluster strategy can't pre-filter because
|
||||
# the collector queue carries all events; here we have the
|
||||
# information to skip the synthesis entirely.
|
||||
if not self._registry.has_children():
|
||||
return
|
||||
if self._registry.parent_for(ws_id) is None:
|
||||
return
|
||||
# ``pending_approval_detail`` deliberately omitted — the
|
||||
# field was removed from cluster_state end-to-end in the
|
||||
# Stage 3 cleanup pass. Approval items arrive via bulk
|
||||
# fetch; verdicts via the explicit intent_verdict event;
|
||||
# resolution via approval_resolved.
|
||||
event = {
|
||||
"type": "cluster_state",
|
||||
"ws_id": ws_id,
|
||||
"state": state.value,
|
||||
"node_id": "",
|
||||
"tokens": 0,
|
||||
"activity_state": "",
|
||||
}
|
||||
try:
|
||||
sink_fn(event)
|
||||
except Exception:
|
||||
log.debug("same_node_child_source.sink_failed", exc_info=True)
|
||||
|
||||
self._callback = _on_state
|
||||
self._manager.subscribe_to_state(_on_state)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
cb = self._callback
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
self._manager.unsubscribe_from_state(cb)
|
||||
except Exception:
|
||||
log.debug("same_node_child_source.unsubscribe_failed", exc_info=True)
|
||||
self._callback = None
|
||||
self._sink = None
|
||||
|
||||
|
||||
class ClusterChildSource:
|
||||
"""Cross-node child events via :class:`ClusterCollector` subscription.
|
||||
|
||||
Refactor of the existing fan-out machinery from
|
||||
``CoordinatorAdapter`` (was ``_collector_queue`` +
|
||||
``_fanout_thread`` + ``_fanout_loop``). Subscribes as a listener on
|
||||
the collector's broadcast channel and runs a daemon thread that
|
||||
drains the queue, pushing each event to the sink.
|
||||
|
||||
On :meth:`start`, also primes the registry from the collector's
|
||||
snapshot so a parent that re-installs after a console restart sees
|
||||
its already-live children without waiting for the next state tick.
|
||||
The ``parents_provider`` callback returns the set of in-memory
|
||||
parent ws_ids for snapshot filtering — only children whose parent
|
||||
is currently installed get merged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
collector: _CollectorProtocol,
|
||||
registry: ChildrenRegistry,
|
||||
*,
|
||||
parents_provider: Callable[[], Iterable[str]],
|
||||
) -> None:
|
||||
self._collector = collector
|
||||
self._registry = registry
|
||||
self._parents_provider = parents_provider
|
||||
self._sink: Callable[[dict[str, Any]], None] | None = None
|
||||
self._queue: queue.Queue[dict[str, Any]] | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return # idempotent — already started
|
||||
self._sink = sink
|
||||
self._queue = queue.Queue(maxsize=1000)
|
||||
snapshot = self._collector.get_snapshot_and_register(self._queue)
|
||||
self._prime_from_snapshot(snapshot)
|
||||
self._stop.clear()
|
||||
t = threading.Thread(
|
||||
target=self._loop,
|
||||
name="cluster-child-source",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread = t
|
||||
t.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._stop.set()
|
||||
t = self._thread
|
||||
q = self._queue
|
||||
coll = self._collector
|
||||
self._thread = None
|
||||
self._queue = None
|
||||
if coll is not None and q is not None:
|
||||
try:
|
||||
coll.unregister_listener(q)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"cluster_child_source.unregister_listener_failed",
|
||||
exc_info=True,
|
||||
)
|
||||
if t is not None:
|
||||
t.join(timeout=2.0)
|
||||
self._sink = None
|
||||
|
||||
def _loop(self) -> None:
|
||||
q = self._queue
|
||||
if q is None:
|
||||
return
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
event = q.get(timeout=1.0)
|
||||
except queue.Empty:
|
||||
continue
|
||||
sink = self._sink
|
||||
if sink is None:
|
||||
continue
|
||||
try:
|
||||
sink(event)
|
||||
except Exception:
|
||||
log.debug("cluster_child_source.dispatch_failed", exc_info=True)
|
||||
|
||||
def _prime_from_snapshot(self, snapshot: dict[str, Any]) -> None:
|
||||
"""Populate the registry from a collector snapshot.
|
||||
|
||||
For every workstream in the snapshot whose ``parent_ws_id``
|
||||
names a currently-installed parent (per ``parents_provider``),
|
||||
merge it into the registry. Caller-installed parents that
|
||||
appear in the snapshot are seeded; unknown parents are skipped
|
||||
— they'll be picked up by the live fan-out path once their
|
||||
``ws_created`` event arrives.
|
||||
"""
|
||||
nodes = snapshot.get("nodes", []) if isinstance(snapshot, dict) else []
|
||||
if not nodes:
|
||||
return
|
||||
known_parents = set(self._parents_provider())
|
||||
if not known_parents:
|
||||
return
|
||||
by_parent: dict[str, list[str]] = {}
|
||||
for node in nodes:
|
||||
for entry in node.get("workstreams", []) or []:
|
||||
parent = entry.get("parent_ws_id") or ""
|
||||
child_id = entry.get("id") or ""
|
||||
if not parent or not child_id or parent not in known_parents:
|
||||
continue
|
||||
by_parent.setdefault(parent, []).append(child_id)
|
||||
for parent, kids in by_parent.items():
|
||||
self._registry.merge_children(parent, kids)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Universal parent → children registry for SessionManager.
|
||||
|
||||
Pure data + lookups; no IO, no transport. Lifted from
|
||||
:class:`turnstone.console.coordinator_adapter.CoordinatorAdapter` where
|
||||
it lived bound to the coordinator kind. The lift is what lets the
|
||||
``ChildSource`` strategies (Step 2) plug into a single shared primitive
|
||||
regardless of whether children are local (interactive) or cluster-routed
|
||||
(coordinator).
|
||||
|
||||
Storage rebuild and snapshot priming happen in the caller — typically
|
||||
the ``ChildSource`` implementation that owns the relevant transport.
|
||||
The registry exposes :meth:`merge_children` for bulk seeding so callers
|
||||
that compute child id lists from any source can feed them in without
|
||||
the registry needing to know about storage shapes or collector
|
||||
snapshots.
|
||||
|
||||
Threading: every public method is internally locked. Helpers suffixed
|
||||
``_locked`` require the caller to already hold :attr:`_lock`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class ChildrenRegistry:
|
||||
"""Tracks parent → children + reverse lookup for in-memory parents.
|
||||
|
||||
The forward index (``_children``) is parent_ws_id → set of child ws_ids.
|
||||
The reverse index (``_child_to_parent``) is child_ws_id → parent_ws_id.
|
||||
The presence map (``_active``) is parent_ws_id → UI ref, used by the
|
||||
dispatch path to atomically check-and-route in one lock acquisition.
|
||||
|
||||
Closed / deleted children stay in the registry until their owning
|
||||
parent is uninstalled — the tree UI keeps rendering them grayed out;
|
||||
state authority lives in storage, not here.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._children: dict[str, set[str]] = {}
|
||||
self._child_to_parent: dict[str, str] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._active: dict[str, Any] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle — install / uninstall a parent
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def install(self, parent_ws_id: str, ui: Any) -> None:
|
||||
"""Seed the forward set + presence map for a new parent.
|
||||
|
||||
Idempotent — re-installing re-points the UI but leaves the
|
||||
existing child set intact. Mirrors the original
|
||||
``_install_coord_registry`` semantics so a coordinator that
|
||||
rehydrates after a crash doesn't lose its known-children.
|
||||
"""
|
||||
with self._lock:
|
||||
self._children.setdefault(parent_ws_id, set())
|
||||
self._active[parent_ws_id] = ui
|
||||
|
||||
def uninstall(self, parent_ws_id: str) -> None:
|
||||
"""Drop a parent: forward set, reverse-index entries, presence.
|
||||
|
||||
No-op if the parent is unknown. Used by close / eviction paths.
|
||||
"""
|
||||
with self._lock:
|
||||
self._uninstall_locked(parent_ws_id)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Mutation — register children under a parent
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_child(self, parent_ws_id: str, child_ws_id: str) -> Any | None:
|
||||
"""Register a child under a parent. Returns parent's UI or None.
|
||||
|
||||
Returns the parent's UI on success (so the dispatch path can
|
||||
atomically check-and-route in one lock acquisition). Returns
|
||||
``None`` if the parent isn't installed (concurrent close /
|
||||
eviction) or if the child is already registered (duplicate
|
||||
ws_created from the cluster fan-out).
|
||||
"""
|
||||
with self._lock:
|
||||
ui = self._active.get(parent_ws_id)
|
||||
if ui is None:
|
||||
return None
|
||||
existing = self._children.setdefault(parent_ws_id, set())
|
||||
if child_ws_id in existing:
|
||||
return None
|
||||
existing.add(child_ws_id)
|
||||
self._child_to_parent[child_ws_id] = parent_ws_id
|
||||
return ui
|
||||
|
||||
def merge_children(self, parent_ws_id: str, child_ws_ids: Iterable[str]) -> None:
|
||||
"""Bulk-merge child_ids under a parent. Idempotent.
|
||||
|
||||
Sole bulk write-path — used by both storage-seeded rebuilds and
|
||||
snapshot-seeded priming so reverse-index ordering invariants
|
||||
hold regardless of which seed source races first.
|
||||
"""
|
||||
with self._lock:
|
||||
self._merge_locked(parent_ws_id, child_ws_ids)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lookups
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def parent_for(self, child_ws_id: str) -> str | None:
|
||||
"""Reverse lookup: which parent owns this child? O(1)."""
|
||||
with self._lock:
|
||||
return self._child_to_parent.get(child_ws_id)
|
||||
|
||||
def has_children(self) -> bool:
|
||||
"""Lock-free fast path: any child registered under any parent?
|
||||
|
||||
Reads ``bool(self._child_to_parent)`` without taking the lock.
|
||||
Dict-truthiness is a single GIL-atomic read, so callers on
|
||||
the hot state-broadcast path can short-circuit without paying
|
||||
the lock acquisition when the registry is empty (the steady
|
||||
state for an interactive manager today). The answer is best-
|
||||
effort — if a child is added concurrently with the read the
|
||||
caller may falsely return ``False``, but the next state event
|
||||
will pick up the change correctly.
|
||||
"""
|
||||
return bool(self._child_to_parent)
|
||||
|
||||
def children_of(self, parent_ws_id: str) -> list[str]:
|
||||
"""Snapshot copy of the parent's child ws_ids.
|
||||
|
||||
Returned list is a copy so callers can iterate without holding
|
||||
the registry lock during per-child work. A mutation racing with
|
||||
the snapshot either lands before (included) or after (excluded)
|
||||
— both outcomes are safe for cascade-style dispatch.
|
||||
"""
|
||||
with self._lock:
|
||||
child_set = self._children.get(parent_ws_id)
|
||||
return list(child_set) if child_set else []
|
||||
|
||||
def ui_for(self, parent_ws_id: str) -> Any | None:
|
||||
"""Look up the UI registered for a parent."""
|
||||
with self._lock:
|
||||
return self._active.get(parent_ws_id)
|
||||
|
||||
def parents(self) -> list[str]:
|
||||
"""Snapshot copy of installed parent ws_ids."""
|
||||
with self._lock:
|
||||
return list(self._active)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Locked helpers — caller must hold ``self._lock``
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_locked(self, parent_ws_id: str, child_ws_ids: Iterable[str]) -> None:
|
||||
"""Idempotent merge under caller's lock. Empty/falsy ids skipped."""
|
||||
existing = self._children.setdefault(parent_ws_id, set())
|
||||
for cid in child_ws_ids:
|
||||
if cid and cid not in existing:
|
||||
existing.add(cid)
|
||||
self._child_to_parent[cid] = parent_ws_id
|
||||
|
||||
def _uninstall_locked(self, parent_ws_id: str) -> None:
|
||||
"""Pop forward set + presence + own reverse-index entries.
|
||||
|
||||
Defensive: only clears reverse entries that still point at
|
||||
``parent_ws_id``. Schema-shaped reassignments (rare but
|
||||
possible) shouldn't orphan the new owner's entry.
|
||||
"""
|
||||
child_set = self._children.pop(parent_ws_id, None)
|
||||
self._active.pop(parent_ws_id, None)
|
||||
if child_set is None:
|
||||
return
|
||||
for cid in child_set:
|
||||
if self._child_to_parent.get(cid) == parent_ws_id:
|
||||
self._child_to_parent.pop(cid, None)
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Shared history-replay decoration helpers.
|
||||
|
||||
Both surfaces that build a history wire payload — interactive's SSE
|
||||
``_build_history`` and the lifted ``make_history_handler`` REST
|
||||
endpoint — need the same audit-trail data attached to each
|
||||
``tool_calls`` entry: the persisted intent verdict (``intent_verdicts``
|
||||
table) and the output-guard assessment (``output_assessments`` table).
|
||||
|
||||
Centralising the lookup + decoration here keeps the two surfaces from
|
||||
drifting on which fields ship to the client and how they're shaped.
|
||||
The shared helpers also let us project only the fields the UI actually
|
||||
renders, dropping redundant ones (``call_id``/``func_name`` already
|
||||
carried on ``tc.id``/``tc.name``) so the wire payload stays tight.
|
||||
|
||||
All functions are pure I/O or pure transforms — safe to call from
|
||||
either an async caller (via ``asyncio.to_thread``) or a sync hook.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# Tool results are clamped at this length per row at storage time
|
||||
# (see ``session.py``'s ``store_text = raw_output[:TOOL_RESULT_STORAGE_CAP]``).
|
||||
# Keeping the constant here lets the truncation flag detection in
|
||||
# ``decorate_history_messages`` stay in sync without a magic number
|
||||
# duplicated across server.py / session.py.
|
||||
#
|
||||
# Raised from 2000 → 10000 because a 2000-char clip routinely cut
|
||||
# the body of a single grep / file read mid-line, leaving the
|
||||
# historical record useless for retrospective debugging. FTS5
|
||||
# index + row size grow proportionally; the per-tool upper bound is
|
||||
# still bounded upstream by ``_truncate_output``'s context-budget
|
||||
# clamp (so a single huge result can't blow past the live context
|
||||
# window).
|
||||
TOOL_RESULT_STORAGE_CAP = 10000
|
||||
|
||||
|
||||
def load_verdict_indexes(
|
||||
ws_id: str,
|
||||
) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]:
|
||||
"""Bulk-load intent verdicts and output assessments for a workstream.
|
||||
|
||||
Returns ``(verdicts_by_call_id, assessments_by_call_id)``. Both
|
||||
tables are indexed by ws_id so the queries are O(rows-for-ws); the
|
||||
DESC ordering plus first-seen-wins dedupe leaves the newest
|
||||
verdict per call_id (LLM upgrade beats heuristic when both exist).
|
||||
|
||||
Pure storage I/O — safe to run in ``asyncio.to_thread`` from an
|
||||
async caller. Returns empty dicts when storage is unavailable or
|
||||
the lookup raises (best-effort: replay must never block on
|
||||
audit-trail decoration).
|
||||
"""
|
||||
verdicts_by_call_id: dict[str, dict[str, Any]] = {}
|
||||
assessments_by_call_id: dict[str, dict[str, Any]] = {}
|
||||
if not ws_id:
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
try:
|
||||
from turnstone.core.storage._registry import get_storage
|
||||
|
||||
storage = get_storage()
|
||||
if storage is None:
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
for v in storage.list_intent_verdicts(ws_id=ws_id, limit=10000):
|
||||
cid = v.get("call_id") or ""
|
||||
if cid and cid not in verdicts_by_call_id:
|
||||
verdicts_by_call_id[cid] = v
|
||||
for a in storage.list_output_assessments(ws_id=ws_id, limit=10000):
|
||||
cid = a.get("call_id") or ""
|
||||
if cid and cid not in assessments_by_call_id:
|
||||
assessments_by_call_id[cid] = a
|
||||
except Exception:
|
||||
# Missing storage / migration drift / driver error must not
|
||||
# block replay — degrade to an unannotated history.
|
||||
log.debug(
|
||||
"verdict/assessment lookup failed; replay continues unannotated",
|
||||
exc_info=True,
|
||||
)
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
|
||||
|
||||
def build_verdict_payload(vrow: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Project a stored ``intent_verdicts`` row into the wire shape.
|
||||
|
||||
Returns ``None`` when the verdict is the unflagged baseline
|
||||
(``risk_level == "none"``) — the client's ``renderVerdictBadge``
|
||||
helper would suppress those anyway, so skipping at the wire layer
|
||||
keeps the payload tight on long workstreams.
|
||||
|
||||
Drops ``call_id`` and ``func_name`` from the wire payload — they're
|
||||
already carried on the parent ``tc.id`` / ``tc.name`` fields.
|
||||
Ships ``reasoning`` for either tier when the row has non-empty
|
||||
prose (heuristic rules in this project DO write meaningful
|
||||
rationales — e.g. ``policy.py`` emits structured reasoning per
|
||||
matched pattern). ``judge_model`` rides through so the batch tier
|
||||
badge can render ``⚖ llm:claude-haiku-4`` on history-only batches
|
||||
rather than the bare ``⚖ llm`` label.
|
||||
"""
|
||||
if (vrow.get("risk_level") or "none") == "none":
|
||||
return None
|
||||
payload: dict[str, Any] = {
|
||||
"risk_level": vrow.get("risk_level", "medium"),
|
||||
"recommendation": vrow.get("recommendation", "review"),
|
||||
"confidence": vrow.get("confidence", 0.0),
|
||||
"intent_summary": vrow.get("intent_summary", ""),
|
||||
"tier": vrow.get("tier", "heuristic"),
|
||||
}
|
||||
if vrow.get("reasoning"):
|
||||
payload["reasoning"] = vrow.get("reasoning", "")
|
||||
judge_model = vrow.get("judge_model") or ""
|
||||
if judge_model:
|
||||
payload["judge_model"] = judge_model
|
||||
return payload
|
||||
|
||||
|
||||
def build_output_assessment_payload(arow: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Project a stored ``output_assessments`` row into the wire shape.
|
||||
|
||||
Returns ``None`` when the assessment is the unflagged baseline
|
||||
(``risk_level == "none"``) — same skip-on-clean pattern as
|
||||
:func:`build_verdict_payload`.
|
||||
|
||||
Decodes ``flags`` from its JSON string form here so the client
|
||||
never has to parse twice. Falls back to an empty list on bad JSON
|
||||
rather than raising — the rest of the assessment is still useful.
|
||||
"""
|
||||
if (arow.get("risk_level") or "none") == "none":
|
||||
return None
|
||||
flags_raw = arow.get("flags") or "[]"
|
||||
try:
|
||||
flags = json.loads(flags_raw) if isinstance(flags_raw, str) else flags_raw
|
||||
except (ValueError, TypeError):
|
||||
flags = []
|
||||
return {
|
||||
"risk_level": arow.get("risk_level", "none"),
|
||||
"flags": flags if isinstance(flags, list) else [],
|
||||
"redacted": bool(arow.get("redacted", 0)),
|
||||
}
|
||||
|
||||
|
||||
def decorate_tool_call(
|
||||
tc: dict[str, Any],
|
||||
verdicts_by_call_id: dict[str, dict[str, Any]],
|
||||
assessments_by_call_id: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Mutate ``tc`` in place, attaching ``verdict`` / ``output_assessment``.
|
||||
|
||||
Works on either tool_call shape:
|
||||
- OpenAI format (``{id, function: {name, arguments}}``) — used by
|
||||
``/history`` REST.
|
||||
- Flattened format (``{id, name, arguments}``) — used by SSE replay.
|
||||
|
||||
Both carry ``id`` at the top level, which is the only field this
|
||||
helper reads. No-ops cleanly when the call_id has no matching
|
||||
row (unflagged tools stay clean).
|
||||
"""
|
||||
call_id = tc.get("id", "") or ""
|
||||
if not call_id:
|
||||
return
|
||||
vrow = verdicts_by_call_id.get(call_id)
|
||||
if vrow is not None:
|
||||
verdict = build_verdict_payload(vrow)
|
||||
if verdict is not None:
|
||||
tc["verdict"] = verdict
|
||||
arow = assessments_by_call_id.get(call_id)
|
||||
if arow is not None:
|
||||
assessment = build_output_assessment_payload(arow)
|
||||
if assessment is not None:
|
||||
tc["output_assessment"] = assessment
|
||||
|
||||
|
||||
def decorate_history_messages(
|
||||
messages: list[dict[str, Any]],
|
||||
verdicts_by_call_id: dict[str, dict[str, Any]],
|
||||
assessments_by_call_id: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Mutate a list of OpenAI-format messages, decorating tool_calls.
|
||||
|
||||
Used by the ``/history`` REST endpoint after ``load_messages``
|
||||
returns. For each assistant message with ``tool_calls``, runs
|
||||
:func:`decorate_tool_call` on every entry. For each tool message
|
||||
whose content hits the storage cap, sets ``truncated: True`` so
|
||||
the client can render the "… truncated in storage" pill.
|
||||
|
||||
Pure transform — no I/O. Async callers should pre-load the
|
||||
indexes via :func:`load_verdict_indexes` (in ``to_thread``) and
|
||||
pass them in.
|
||||
"""
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
if role == "assistant":
|
||||
tcs = msg.get("tool_calls")
|
||||
if isinstance(tcs, list):
|
||||
for tc in tcs:
|
||||
if isinstance(tc, dict):
|
||||
decorate_tool_call(tc, verdicts_by_call_id, assessments_by_call_id)
|
||||
elif role == "tool":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str) and len(content) >= TOOL_RESULT_STORAGE_CAP:
|
||||
msg["truncated"] = True
|
||||
@@ -432,7 +432,7 @@ LAST_ERROR_CONFIG_KEY = "last_error"
|
||||
# such error per workstream would bloat workstream_config and the model
|
||||
# prompt the coord LLM ingests on inspect. 1024 chars matches the
|
||||
# practical "useful for triage" length while staying well under the
|
||||
# WAIT_MESSAGE_MAX_BYTES (6 KiB) cap so the truncate happens here at
|
||||
# WAIT_MESSAGE_MAX_BYTES (10 KiB) cap so the truncate happens here at
|
||||
# write time, not later at the wait surface.
|
||||
LAST_ERROR_MAX_LEN = 1024
|
||||
|
||||
|
||||
@@ -5,7 +5,50 @@ from __future__ import annotations
|
||||
import re
|
||||
import time
|
||||
|
||||
_COOLDOWN_SECS = 300 # 5 minutes between nudges of the same type
|
||||
# Default cooldown (s) between nudges of the same type. Production
|
||||
# paths pass ``cooldown_secs`` explicitly from
|
||||
# ``MemoryConfig.nudge_cooldown`` (config-store ``memory.nudge_cooldown``,
|
||||
# default 300); this constant is the fallback for tests and unit-style
|
||||
# callers without a ``MemoryConfig`` and is kept aligned with that
|
||||
# canonical default so both paths behave the same.
|
||||
_COOLDOWN_SECS = 300
|
||||
|
||||
# Repeat-detection threshold — number of *consecutive* identical tool
|
||||
# calls (same name + same arguments) before a repeat warning fires.
|
||||
# Two-in-a-row is too noisy because legitimate retries on transient
|
||||
# failures look identical; three-in-a-row is the cheapest signal that
|
||||
# the model is stuck on the same call.
|
||||
_REPEAT_THRESHOLD = 3
|
||||
|
||||
|
||||
class RepeatDetector:
|
||||
"""Detect a streak of identical tool-call signatures.
|
||||
|
||||
``record(sig)`` returns ``True`` once *sig* has been recorded
|
||||
``threshold`` times in a row (default 3). Recording a different
|
||||
signature resets the streak — interleaved tool calls aren't a
|
||||
stuck loop, only repeated identical ones are. After a fire, the
|
||||
caller is expected to call ``clear()`` to start a fresh streak.
|
||||
"""
|
||||
|
||||
def __init__(self, threshold: int = _REPEAT_THRESHOLD) -> None:
|
||||
self._threshold = threshold
|
||||
self._sig: str | None = None
|
||||
self._count = 0
|
||||
|
||||
def record(self, sig: str) -> bool:
|
||||
"""Record *sig*; return ``True`` when the streak hits the threshold."""
|
||||
if sig == self._sig:
|
||||
self._count += 1
|
||||
else:
|
||||
self._sig = sig
|
||||
self._count = 1
|
||||
return self._count >= self._threshold
|
||||
|
||||
def clear(self) -> None:
|
||||
self._sig = None
|
||||
self._count = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nudge messages (brief, model-facing hints)
|
||||
|
||||
@@ -230,6 +230,7 @@ class ModelRegistry:
|
||||
if task_model and task_model not in models:
|
||||
raise ValueError(f"Task model '{task_model}' not found in registry")
|
||||
with self._client_lock:
|
||||
old_models = self._models
|
||||
self._models = dict(models)
|
||||
self.default = default
|
||||
self.fallback = list(fallback) if fallback else []
|
||||
@@ -238,11 +239,32 @@ class ModelRegistry:
|
||||
self.task_model = task_model
|
||||
self.plan_effort = plan_effort
|
||||
self.task_effort = task_effort
|
||||
for client in self._clients.values():
|
||||
if hasattr(client, "close"):
|
||||
client.close()
|
||||
self._clients.clear()
|
||||
self._providers.clear()
|
||||
# Selective teardown — close + drop only clients whose
|
||||
# connection target actually changed (alias removed, or
|
||||
# base_url / api_key / provider differs). Keeps connection
|
||||
# pools warm for the common admin-edit case where only
|
||||
# ``model`` / ``temperature`` / ``context_window`` changed.
|
||||
for alias, client in list(self._clients.items()):
|
||||
old_cfg = old_models.get(alias)
|
||||
new_cfg = self._models.get(alias)
|
||||
if (
|
||||
new_cfg is None
|
||||
or old_cfg is None
|
||||
or old_cfg.base_url != new_cfg.base_url
|
||||
or old_cfg.api_key != new_cfg.api_key
|
||||
or old_cfg.provider != new_cfg.provider
|
||||
):
|
||||
if hasattr(client, "close"):
|
||||
client.close()
|
||||
del self._clients[alias]
|
||||
# Providers are keyed on alias but only depend on
|
||||
# ``cfg.provider`` — drop only when the provider string
|
||||
# changed or the alias was removed.
|
||||
for alias in list(self._providers.keys()):
|
||||
old_cfg = old_models.get(alias)
|
||||
new_cfg = self._models.get(alias)
|
||||
if new_cfg is None or old_cfg is None or old_cfg.provider != new_cfg.provider:
|
||||
del self._providers[alias]
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Close all cached client connections."""
|
||||
@@ -300,6 +322,7 @@ def load_model_registry(
|
||||
context_window: int = 32768,
|
||||
provider: str = "openai",
|
||||
storage: Any | None = None,
|
||||
strict: bool = False,
|
||||
) -> ModelRegistry:
|
||||
"""Build a ModelRegistry from CLI args, ``config.toml``, and database.
|
||||
|
||||
@@ -317,6 +340,15 @@ def load_model_registry(
|
||||
``[model].plan_effort``, ``[model].task_effort`` control routing.
|
||||
``plan_model``/``task_model`` override ``agent_model`` per sub-agent
|
||||
role; both fall back to it when unset.
|
||||
|
||||
``strict``: when True, a storage read failure during the DB-rows step
|
||||
re-raises instead of degrading to a config.toml-only registry.
|
||||
Callers that hot-reload an existing registry need this so a transient
|
||||
DB outage doesn't silently drop every DB-sourced alias when the
|
||||
truncated result is applied via ``ModelRegistry.reload``. Callers
|
||||
that build a fresh registry from scratch (CLI, lifespan startup) want
|
||||
the default behaviour — boot succeeds with a config-only fallback
|
||||
rather than crashing on a flaky DB.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
@@ -370,6 +402,8 @@ def load_model_registry(
|
||||
server_compat=row_server_compat,
|
||||
)
|
||||
except Exception:
|
||||
if strict:
|
||||
raise
|
||||
log.warning("Failed to load model definitions from storage", exc_info=True)
|
||||
|
||||
# 2. Build configs from [models.*] sections (overrides DB for same alias)
|
||||
|
||||
+412
-206
@@ -44,6 +44,7 @@ from turnstone.core.attachments import (
|
||||
)
|
||||
from turnstone.core.config import get_tavily_key
|
||||
from turnstone.core.edit import find_occurrences, pick_nearest
|
||||
from turnstone.core.history_decoration import TOOL_RESULT_STORAGE_CAP
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.memory import (
|
||||
count_structured_memories,
|
||||
@@ -81,6 +82,7 @@ from turnstone.core.memory_relevance import (
|
||||
score_memories,
|
||||
)
|
||||
from turnstone.core.metacognition import (
|
||||
RepeatDetector,
|
||||
detect_completion,
|
||||
detect_correction,
|
||||
format_nudge,
|
||||
@@ -90,6 +92,7 @@ from turnstone.core.providers import create_provider
|
||||
from turnstone.core.safety import is_command_blocked, sanitize_command
|
||||
from turnstone.core.sandbox import execute_math_sandboxed
|
||||
from turnstone.core.storage._registry import get_storage
|
||||
from turnstone.core.tool_advisory import escape_wrapper_tags, render_system_reminder
|
||||
from turnstone.core.tool_search import ToolSearchManager
|
||||
from turnstone.core.tools import (
|
||||
AGENT_AUTO_TOOLS,
|
||||
@@ -275,6 +278,8 @@ class SessionUI(Protocol):
|
||||
def on_plan_review(self, content: str) -> str: ...
|
||||
def on_info(self, message: str) -> None: ...
|
||||
def on_error(self, message: str) -> None: ...
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None: ...
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None: ...
|
||||
def on_state_change(self, state: str) -> None: ...
|
||||
def on_rename(self, name: str) -> None: ...
|
||||
def on_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
@@ -474,8 +479,12 @@ class ChatSession:
|
||||
collections.OrderedDict()
|
||||
)
|
||||
self._queued_lock = threading.Lock()
|
||||
# Repeat detection: track recent tool call signatures
|
||||
self._recent_tool_sigs: set[str] = set()
|
||||
# Repeat detection: streak counter over tool-call signatures.
|
||||
# Fires when a (name, args) signature has been seen N times in
|
||||
# a row; recording any different signature resets the streak.
|
||||
# Also cleared after a write tool succeeds (state changed) or
|
||||
# after a warning fires (clean slate, re-fire on the next streak).
|
||||
self._repeat_detector = RepeatDetector()
|
||||
# Tool error tracking: call_id → is_error for message persistence
|
||||
self._tool_error_flags: dict[str, bool] = {}
|
||||
# Cooperative cancellation: set from outside to stop generation
|
||||
@@ -1293,7 +1302,7 @@ class ChatSession:
|
||||
self._ws_id = ws_id
|
||||
self.messages = messages
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
self._last_usage = None
|
||||
self._calibrated_msg_count = 0
|
||||
self._title_generated = True # don't re-title resumed workstreams
|
||||
@@ -1638,6 +1647,124 @@ class ChatSession:
|
||||
"""System messages + conversation history."""
|
||||
return self.system_messages + self.messages
|
||||
|
||||
def _apply_reminders_for_provider(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return a transient copy of *messages* with ``_reminders`` rendered
|
||||
inline for the model.
|
||||
|
||||
Metacognitive nudges live on the message dict's ``_reminders``
|
||||
side-channel regardless of role — user messages carry
|
||||
user-channel nudges (correction / denial / resume / start /
|
||||
completion), tool messages carry tool-channel nudges
|
||||
(tool_error / repeat). Both ride the same side-channel so
|
||||
``self.messages`` and every downstream consumer (UI replay,
|
||||
compaction, title gen, channel adapters, DB) see clean
|
||||
``content``; only the wire-bound copy here carries the
|
||||
rendered reminder.
|
||||
|
||||
For each message that has ``_reminders`` AND has not yet been
|
||||
flagged delivered, build a shallow copy and splice the reminders
|
||||
as ``<system-reminder>`` blocks onto the trailing edge of the
|
||||
copy's ``content`` — string content gets a tail block, list
|
||||
content gets the block on the trailing text part (or a new text
|
||||
part if there isn't one). Messages without ``_reminders`` (or
|
||||
already delivered) pass through unchanged (same object
|
||||
reference) so the common case is allocation-free.
|
||||
|
||||
**Reminder lifecycle.** After a successful provider stream
|
||||
``_mark_reminders_delivered`` flips ``_reminders_delivered`` to
|
||||
``True`` on every message (user or tool) that carried reminders
|
||||
into that call, so subsequent provider calls skip them — the
|
||||
model sees each reminder once, the turn it advised. The
|
||||
``_reminders`` key itself stays on the message dict for the
|
||||
lifetime of the in-memory session so ``/history`` (reconnecting
|
||||
tabs, multi-tab live mirrors) still renders the same nudge
|
||||
bubbles the originating tab saw; only the wire-side replay is
|
||||
suppressed. Compaction is the natural full drain (it replaces
|
||||
``self.messages`` wholesale).
|
||||
|
||||
``sanitize_messages`` later drops both leading-underscore sibling
|
||||
keys (``_reminders`` and ``_reminders_delivered``) on the way to
|
||||
the wire, so the provider sees only ``content`` with the
|
||||
reminder spliced in.
|
||||
|
||||
**Read-only contract on the returned list.** The pass-through
|
||||
path returns the original ``msg`` by reference; callers must
|
||||
not mutate the returned dicts in place (today's only callers —
|
||||
``sanitize_messages`` + provider conversion — construct new
|
||||
dicts, so the contract holds). Mutations on the spliced copy
|
||||
are safe; mutations on a pass-through reference would bleed
|
||||
back into ``self.messages``.
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for msg in messages:
|
||||
raw_reminders = msg.get("_reminders")
|
||||
if not raw_reminders or msg.get("_reminders_delivered"):
|
||||
out.append(msg)
|
||||
continue
|
||||
# Defensive filter — only dict entries are valid; a string /
|
||||
# None / other shape from corruption or partial state must
|
||||
# not abort the whole send via an AttributeError on .get().
|
||||
# Mirrors the same filter ``_build_history`` applies on the
|
||||
# wire-out side.
|
||||
reminders = [r for r in raw_reminders if isinstance(r, dict)]
|
||||
if not reminders:
|
||||
out.append(msg)
|
||||
continue
|
||||
block = "\n\n" + "\n\n".join(
|
||||
render_system_reminder(r.get("text", "")) for r in reminders
|
||||
)
|
||||
copy = dict(msg)
|
||||
content = copy.get("content")
|
||||
if isinstance(content, str):
|
||||
copy["content"] = escape_wrapper_tags(content) + block
|
||||
elif isinstance(content, list):
|
||||
# Shallow-copy the parts list and any text parts we'll
|
||||
# mutate so the original list/dicts in self.messages stay
|
||||
# untouched.
|
||||
new_parts = [
|
||||
dict(p) if isinstance(p, dict) and p.get("type") == "text" else p
|
||||
for p in content
|
||||
]
|
||||
text_parts = [
|
||||
p for p in new_parts if isinstance(p, dict) and p.get("type") == "text"
|
||||
]
|
||||
for part in text_parts:
|
||||
part["text"] = escape_wrapper_tags(part.get("text", ""))
|
||||
if text_parts:
|
||||
text_parts[-1]["text"] = text_parts[-1]["text"] + block
|
||||
else:
|
||||
new_parts.append({"type": "text", "text": block})
|
||||
copy["content"] = new_parts
|
||||
else:
|
||||
# Unexpected shape (None, etc.) — attach as a text-only
|
||||
# content rather than dropping the reminder silently.
|
||||
copy["content"] = block.lstrip()
|
||||
out.append(copy)
|
||||
return out
|
||||
|
||||
def _mark_reminders_delivered(self) -> None:
|
||||
"""Flag every message's ``_reminders`` as delivered.
|
||||
|
||||
Role-agnostic — both user-channel reminders (set by
|
||||
``_attach_pending_user_reminders`` on user messages) and
|
||||
tool-channel reminders (set by the per-result loop on tool
|
||||
messages) ride the same ``_reminders`` side-channel and the
|
||||
same delivered flag. Called after a successful provider
|
||||
stream; subsequent calls to ``_apply_reminders_for_provider``
|
||||
skip messages with this flag, so the model sees each reminder
|
||||
exactly once (the turn it advised). The flag is a sibling key
|
||||
like ``_reminders`` itself; ``sanitize_messages`` strips both
|
||||
before the wire and ``_build_history`` ignores the delivered
|
||||
flag entirely so UI replay parity is preserved across
|
||||
reconnects.
|
||||
"""
|
||||
for msg in self.messages:
|
||||
if msg.get("_reminders") and not msg.get("_reminders_delivered"):
|
||||
msg["_reminders_delivered"] = True
|
||||
|
||||
def _emit_state(self, state: str) -> None:
|
||||
"""Notify UI of a workstream state transition.
|
||||
|
||||
@@ -2113,12 +2240,13 @@ class ChatSession:
|
||||
# Metacognitive user-channel drain: any nudges queued via
|
||||
# _queue_user_advisory (correction/start/completion from this
|
||||
# turn, denial from the previous tool batch, resume from
|
||||
# rehydrate) splice in as <system-reminder> blocks at the
|
||||
# trailing edge of the user content. The DB row stores
|
||||
# ``user_input`` only (line below) so these blocks stay
|
||||
# ephemeral — they advise the next assistant turn and do not
|
||||
# persist across reloads.
|
||||
self._splice_pending_user_advisories(user_msg)
|
||||
# rehydrate) attach to the user message dict's ``_reminders``
|
||||
# side-channel — content stays clean. The wire-side splice
|
||||
# happens later in _apply_reminders_for_provider against a
|
||||
# transient copy. The DB row stores ``user_input`` only (line
|
||||
# below) so reminders stay in-memory only and don't persist
|
||||
# across reloads.
|
||||
self._attach_pending_user_reminders(user_msg)
|
||||
self.messages.append(user_msg)
|
||||
self._msg_tokens.append(max(1, int(self._msg_char_count(user_msg) / self._chars_per_token)))
|
||||
# DB row stores the raw text only; attachments are joined back in
|
||||
@@ -2198,7 +2326,7 @@ class ChatSession:
|
||||
try:
|
||||
while True:
|
||||
self._check_cancelled(my_generation)
|
||||
msgs = self._full_messages()
|
||||
msgs = self._apply_reminders_for_provider(self._full_messages())
|
||||
|
||||
if self.debug:
|
||||
self._debug_print_request(msgs)
|
||||
@@ -2235,7 +2363,7 @@ class ChatSession:
|
||||
self.ui.on_thinking_stop()
|
||||
try:
|
||||
self._compact_messages(auto=True)
|
||||
msgs = self._full_messages()
|
||||
msgs = self._apply_reminders_for_provider(self._full_messages())
|
||||
self.ui.on_thinking_start()
|
||||
stream = self._create_stream_with_retry(msgs)
|
||||
except Exception:
|
||||
@@ -2257,7 +2385,19 @@ class ChatSession:
|
||||
if self._generation != my_generation:
|
||||
return
|
||||
|
||||
self._update_token_table(assistant_msg)
|
||||
# Reuse the wire-bound ``msgs`` we already built for the
|
||||
# stream call instead of re-applying the reminder splice
|
||||
# (perf-2). After mark-delivered runs below, a fresh
|
||||
# _apply_reminders_for_provider would skip the
|
||||
# just-rendered reminders and undercount; passing the
|
||||
# already-rendered list keeps calibration char count
|
||||
# aligned with what the provider actually counted.
|
||||
self._update_token_table(assistant_msg, msgs=msgs)
|
||||
# Reminders that rode this stream have now reached the
|
||||
# model; flag delivered so the next provider call skips
|
||||
# them (one-shot semantics for the wire; UI replay still
|
||||
# surfaces ``_reminders`` for reconnect parity).
|
||||
self._mark_reminders_delivered()
|
||||
self._print_status_line() # Report usage for EVERY API call
|
||||
self.messages.append(assistant_msg)
|
||||
self._msg_tokens.append(
|
||||
@@ -2275,16 +2415,7 @@ class ChatSession:
|
||||
if assistant_msg.get("_provider_content"):
|
||||
provider_data = json.dumps(assistant_msg["_provider_content"])
|
||||
|
||||
# Build tool_calls JSON (excluding memory tools)
|
||||
tool_calls_json: str | None = None
|
||||
if tc:
|
||||
filtered_tc = [
|
||||
call
|
||||
for call in tc
|
||||
if call.get("function", {}).get("name", "") not in ("memory", "recall")
|
||||
]
|
||||
if filtered_tc:
|
||||
tool_calls_json = json.dumps(filtered_tc)
|
||||
tool_calls_json: str | None = json.dumps(tc) if tc else None
|
||||
|
||||
# Save assistant message atomically (content + tool_calls in one row)
|
||||
if content or provider_data is not None or tool_calls_json:
|
||||
@@ -2332,92 +2463,10 @@ class ChatSession:
|
||||
if self._generation != my_generation:
|
||||
return
|
||||
|
||||
# Repeat detection: warn when a tool is called with identical args.
|
||||
# Skip error outputs — retrying a failed tool is valid.
|
||||
# Skip JSON outputs (MCP structured results) — appending
|
||||
# text would corrupt the payload.
|
||||
_tc_by_id = {c["id"]: c for c in tool_calls}
|
||||
_repeat_detected = False
|
||||
_error_prefixes = (
|
||||
"Error",
|
||||
"JSON parse error",
|
||||
"Unknown tool",
|
||||
"Command timed out",
|
||||
"Blocked:",
|
||||
"Denied",
|
||||
)
|
||||
|
||||
# Clear dedup sigs when a write tool executed successfully —
|
||||
# the state has changed so re-running a read tool is valid.
|
||||
_write_tools = frozenset({"write_file", "edit_file", "bash"})
|
||||
if any(
|
||||
tc["function"]["name"] in _write_tools
|
||||
and not any(
|
||||
cid == tc["id"] and isinstance(out, str) and out.startswith(_error_prefixes)
|
||||
for cid, out in results
|
||||
)
|
||||
for tc in tool_calls
|
||||
):
|
||||
self._recent_tool_sigs.clear()
|
||||
for i, (tc_id, output) in enumerate(results):
|
||||
tc = _tc_by_id.get(tc_id)
|
||||
if tc and isinstance(output, str) and not output.startswith(_error_prefixes):
|
||||
raw = tc["function"]["name"] + ":" + tc["function"]["arguments"]
|
||||
sig = hashlib.sha256(raw.encode()).hexdigest()
|
||||
is_json = output.lstrip().startswith(("{", "["))
|
||||
if sig in self._recent_tool_sigs:
|
||||
_repeat_detected = True
|
||||
if not is_json:
|
||||
output += (
|
||||
"\n\n⚠ Warning: this is an identical repeat of a "
|
||||
"previous tool call. The result is the same. "
|
||||
"Try a different approach."
|
||||
)
|
||||
results[i] = (tc_id, output)
|
||||
self.ui.on_info(
|
||||
f"{GRAY}[repeat: {tc['function']['name']}() "
|
||||
f"called with same arguments]{RESET}"
|
||||
)
|
||||
self._recent_tool_sigs.add(sig)
|
||||
if _repeat_detected:
|
||||
# Reset so the model gets a clean slate after the warning.
|
||||
# If it repeats again, a new warning fires.
|
||||
self._recent_tool_sigs.clear()
|
||||
if self._mem_cfg.nudges and should_nudge(
|
||||
"repeat",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
):
|
||||
self._queue_tool_advisory("repeat", format_nudge("repeat"))
|
||||
|
||||
# Tool-error nudge — checked here (pre-iteration) so the
|
||||
# MetacognitiveAdvisory rides the same _collect_advisories
|
||||
# drain pass that handles guard findings and user
|
||||
# interjections. Cooldown gating in should_nudge keeps
|
||||
# this to one nudge per batch even with many failing
|
||||
# tools.
|
||||
if (
|
||||
self._mem_cfg.nudges
|
||||
and any(
|
||||
isinstance(out, str)
|
||||
and (
|
||||
out.startswith("Error")
|
||||
or " error: " in out[:50]
|
||||
or out.startswith("Command timed out")
|
||||
or out.startswith("Unknown tool:")
|
||||
)
|
||||
for _, out in results
|
||||
)
|
||||
and should_nudge(
|
||||
"tool_error",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
memory_count=self._visible_memory_count(),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
)
|
||||
):
|
||||
self._queue_tool_advisory("tool_error", format_nudge("tool_error"))
|
||||
# Repeat-detection + tool-error nudge. Mutates *results*
|
||||
# in place to inject inline warning text on identical
|
||||
# repeats; queues advisories for the next drain pass.
|
||||
self._apply_post_execute_advisories(tool_calls, results)
|
||||
|
||||
# Map tool_call_id → tool name for logging
|
||||
from turnstone.core.tool_advisory import wrap_tool_result
|
||||
@@ -2456,19 +2505,30 @@ class ChatSession:
|
||||
# Capture raw output for DB storage before advisory wrapping
|
||||
raw_output = output
|
||||
|
||||
# Advisory injection: wrap tool output with advisories
|
||||
# (output guard findings, queued user messages, etc.)
|
||||
advisories = self._collect_advisories(
|
||||
# Advisory injection: persistent advisories (output
|
||||
# guard findings, queued user interjections) wrap
|
||||
# into the tool-result envelope and stay in
|
||||
# self.messages. Metacognitive tool-channel
|
||||
# reminders (tool_error / repeat) ride a side-channel
|
||||
# — never inside content — so the model sees the
|
||||
# splice only at the wire boundary via
|
||||
# _apply_reminders_for_provider, while UI/replay
|
||||
# surfaces them as a themed bubble below the tool
|
||||
# result.
|
||||
persistent_advisories, metacog_reminders = self._collect_advisories(
|
||||
assessment, _tc_names.get(tc_id, ""), _ri == _last_idx
|
||||
)
|
||||
if isinstance(output, str):
|
||||
output = wrap_tool_result(output, advisories)
|
||||
elif isinstance(output, list) and advisories:
|
||||
output = wrap_tool_result(output, persistent_advisories)
|
||||
elif isinstance(output, list) and persistent_advisories:
|
||||
# Structured/image output — append advisories as a
|
||||
# text part so they aren't silently dropped.
|
||||
output = [
|
||||
*output,
|
||||
{"type": "text", "text": wrap_tool_result("", advisories)},
|
||||
{
|
||||
"type": "text",
|
||||
"text": wrap_tool_result("", persistent_advisories),
|
||||
},
|
||||
]
|
||||
|
||||
tool_msg: dict[str, Any] = {
|
||||
@@ -2478,6 +2538,15 @@ class ChatSession:
|
||||
}
|
||||
if self._tool_error_flags.pop(tc_id, False):
|
||||
tool_msg["is_error"] = True
|
||||
if metacog_reminders:
|
||||
tool_msg["_reminders"] = metacog_reminders
|
||||
try:
|
||||
self.ui.on_tool_reminder(metacog_reminders, tc_id)
|
||||
except Exception:
|
||||
log.warning(
|
||||
"ui.on_tool_reminder failed; reminder still attached",
|
||||
exc_info=True,
|
||||
)
|
||||
self.messages.append(tool_msg)
|
||||
|
||||
# Token estimation — image content uses a fixed heuristic
|
||||
@@ -2494,27 +2563,27 @@ class ChatSession:
|
||||
tok_est = max(1, int(len(output) / self._chars_per_token))
|
||||
self._msg_tokens.append(tok_est)
|
||||
|
||||
# Log tool result (skip memory tools to avoid noise).
|
||||
# Use raw_output (pre-advisory-wrap) so DB stores clean
|
||||
# tool output without ephemeral advisory XML.
|
||||
# Log tool result. Use raw_output (pre-advisory-wrap)
|
||||
# so the DB stores clean tool output without ephemeral
|
||||
# advisory XML. memory/recall persist alongside every
|
||||
# other tool: replays show the full audit trail, and
|
||||
# output already passes through _truncate_output above
|
||||
# so size is bounded by the same budget every other
|
||||
# tool uses.
|
||||
_tname = _tc_names.get(tc_id, "")
|
||||
if _tname not in (
|
||||
"memory",
|
||||
"recall",
|
||||
):
|
||||
if isinstance(raw_output, list):
|
||||
store_text = " ".join(
|
||||
p.get("text", "") for p in raw_output if p.get("type") == "text"
|
||||
)[:2000]
|
||||
else:
|
||||
store_text = raw_output[:2000]
|
||||
save_message(
|
||||
self._ws_id,
|
||||
"tool",
|
||||
store_text,
|
||||
_tname,
|
||||
tool_call_id=tc_id,
|
||||
)
|
||||
if isinstance(raw_output, list):
|
||||
store_text = " ".join(
|
||||
p.get("text", "") for p in raw_output if p.get("type") == "text"
|
||||
)[:TOOL_RESULT_STORAGE_CAP]
|
||||
else:
|
||||
store_text = raw_output[:TOOL_RESULT_STORAGE_CAP]
|
||||
save_message(
|
||||
self._ws_id,
|
||||
"tool",
|
||||
store_text,
|
||||
_tname,
|
||||
tool_call_id=tc_id,
|
||||
)
|
||||
# Inject user feedback from approval prompt (e.g. "y, use full path")
|
||||
if user_feedback:
|
||||
self.messages.append({"role": "user", "content": user_feedback})
|
||||
@@ -2583,10 +2652,7 @@ class ChatSession:
|
||||
# Drain any queued user messages so they appear in the
|
||||
# conversation and are visible on the next send().
|
||||
self._flush_queued_messages()
|
||||
# Tool-channel nudges queued earlier in this generation
|
||||
# (tool_error, repeat) belong to the abandoned batch — drop
|
||||
# them so they don't bleed into the next send()'s tool loop.
|
||||
self._pending_tool_advisories.clear()
|
||||
self._drain_pending_advisories()
|
||||
# No need to clear _cancel_event — it's replaced per-generation
|
||||
# in send(), so this generation's event is simply discarded.
|
||||
self.ui.on_info("[Generation cancelled]")
|
||||
@@ -2596,15 +2662,29 @@ class ChatSession:
|
||||
except KeyboardInterrupt as exc:
|
||||
self._synthesize_cancelled_results("Interrupted by user.")
|
||||
self._flush_queued_messages()
|
||||
self._pending_tool_advisories.clear()
|
||||
self._drain_pending_advisories()
|
||||
self._record_fatal_error(exc)
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._flush_queued_messages()
|
||||
self._pending_tool_advisories.clear()
|
||||
self._drain_pending_advisories()
|
||||
self._record_fatal_error(exc)
|
||||
raise
|
||||
|
||||
def _drain_pending_advisories(self) -> None:
|
||||
"""Drop both advisory channels' pending buffers.
|
||||
|
||||
Both channels are scoped to the current generation: tool-channel
|
||||
nudges (``tool_error``, ``repeat``) queued earlier in this batch
|
||||
and user-channel nudges (``correction``, ``denial``, …) queued
|
||||
during ``_check_metacognitive_nudge`` but not yet drained. When
|
||||
a generation is abandoned (cancel, KeyboardInterrupt, unexpected
|
||||
exception) both must drop so they don't bleed into the next
|
||||
send's tool loop or next user turn.
|
||||
"""
|
||||
self._pending_tool_advisories.clear()
|
||||
self._pending_user_advisories.clear()
|
||||
|
||||
def _synthesize_cancelled_results(self, reason: str) -> None:
|
||||
"""Synthesize tool_result messages for orphaned tool_calls after cancel.
|
||||
|
||||
@@ -3137,8 +3217,24 @@ class ChatSession:
|
||||
text_chars, images, doc_chars = self._msg_text_chars(msg)
|
||||
return text_chars + doc_chars + int(images * self._IMAGE_TOKENS * self._chars_per_token)
|
||||
|
||||
def _update_token_table(self, assistant_msg: dict[str, Any]) -> None:
|
||||
"""Update per-message token estimates using API usage data."""
|
||||
def _update_token_table(
|
||||
self,
|
||||
assistant_msg: dict[str, Any],
|
||||
*,
|
||||
msgs: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
"""Update per-message token estimates using API usage data.
|
||||
|
||||
*msgs* (optional) is the wire-bound message list already built
|
||||
for the stream call — passing it avoids a redundant
|
||||
``_apply_reminders_for_provider`` walk and, more importantly,
|
||||
ensures the char count matches the bytes the provider counted
|
||||
even after ``_mark_reminders_delivered`` has flipped the flag
|
||||
on the reminders that rode the stream. When *msgs* is None the
|
||||
caller didn't pre-build (rare path) — fall back to applying
|
||||
the splice on the fly, but be aware the result will be
|
||||
reminder-free if delivered flags are already set.
|
||||
"""
|
||||
if not self._last_usage:
|
||||
return
|
||||
|
||||
@@ -3148,8 +3244,16 @@ class ChatSession:
|
||||
# Calibrate chars_per_token ratio from actual usage.
|
||||
# Images get a fixed token budget (subtracted). Documents
|
||||
# tokenize non-linearly depending on provider — excluded from
|
||||
# calibration so they don't skew the text ratio.
|
||||
all_msgs = self._full_messages() # system + self.messages (before append)
|
||||
# calibration so they don't skew the text ratio. ``all_msgs``
|
||||
# must reflect what the provider actually counted in
|
||||
# ``prompt_tokens``: when called from the loop with the
|
||||
# pre-built ``msgs``, that's exact; without it, fall back to
|
||||
# applying the splice fresh (post-mark-delivered the result
|
||||
# may undercount, but no caller currently takes this path
|
||||
# after a successful stream).
|
||||
all_msgs = (
|
||||
msgs if msgs is not None else self._apply_reminders_for_provider(self._full_messages())
|
||||
) # system + self.messages (before append)
|
||||
active_tools = self._get_active_tools() or []
|
||||
tool_def_chars = sum(len(json.dumps(t)) for t in active_tools)
|
||||
text_chars = 0
|
||||
@@ -3391,7 +3495,7 @@ class ChatSession:
|
||||
self.messages = [summary_user, summary_asst]
|
||||
# File contents are gone after compaction — force re-read before edit_file
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
|
||||
# Rebuild token table
|
||||
su_tok = max(1, int(self._msg_char_count(summary_user) / self._chars_per_token))
|
||||
@@ -3780,12 +3884,26 @@ class ChatSession:
|
||||
assessment: OutputAssessment | None,
|
||||
func_name: str,
|
||||
is_last_in_batch: bool,
|
||||
) -> list[ToolAdvisory]:
|
||||
) -> tuple[list[ToolAdvisory], list[dict[str, str]]]:
|
||||
"""Gather advisories to attach to a tool result message.
|
||||
|
||||
Returns an empty list when no advisories apply (common case).
|
||||
Guard advisories attach per-result; user messages drain on the
|
||||
last result in the batch only.
|
||||
Returns ``(persistent, metacog_reminders)``:
|
||||
|
||||
- ``persistent`` — guard findings + user interjections that ride
|
||||
inside the tool-result envelope via ``wrap_tool_result``.
|
||||
These are conversation history and must persist in
|
||||
``self.messages``.
|
||||
- ``metacog_reminders`` — list of ``{"type", "text"}`` dicts for
|
||||
``tool_error`` / ``repeat`` nudges that the caller attaches to
|
||||
the tool message dict's ``_reminders`` side-channel. Like
|
||||
user-channel reminders, they are spliced into ``content`` only
|
||||
at the wire boundary by ``_apply_reminders_for_provider`` and
|
||||
surfaced separately on the UI as a themed bubble below the
|
||||
tool result.
|
||||
|
||||
Both lists are empty when no advisories apply (common case).
|
||||
Guard advisories attach per-result; user messages and
|
||||
metacognitive nudges drain on the last result in the batch only.
|
||||
"""
|
||||
from turnstone.core.tool_advisory import GuardAdvisory, UserInterjection
|
||||
|
||||
@@ -3801,26 +3919,25 @@ class ChatSession:
|
||||
if is_last_in_batch:
|
||||
self._pending_tool_advisories.clear()
|
||||
self._flush_queued_messages()
|
||||
return []
|
||||
return [], []
|
||||
|
||||
advisories: list[ToolAdvisory] = []
|
||||
persistent: list[ToolAdvisory] = []
|
||||
metacog_reminders: list[dict[str, str]] = []
|
||||
|
||||
# Output guard advisory
|
||||
# Output guard advisory — persists with the tool result.
|
||||
if assessment is not None:
|
||||
advisories.append(GuardAdvisory(assessment=assessment, func_name=func_name))
|
||||
persistent.append(GuardAdvisory(assessment=assessment, func_name=func_name))
|
||||
|
||||
# Metacognitive tool-channel drain — fires once per batch on the
|
||||
# last result. Queued by _queue_tool_advisory from the
|
||||
# tool_error and repeat detection paths just before this loop.
|
||||
# last result. Queued by _queue_tool_advisory from the
|
||||
# tool_error / repeat detection paths just before this loop.
|
||||
# Lands on the tool message dict's ``_reminders`` side-channel
|
||||
# (caller's responsibility) so it stays out of persisted content
|
||||
# and rides the wire only via the transient-copy splice.
|
||||
if is_last_in_batch and self._pending_tool_advisories:
|
||||
from turnstone.core.tool_advisory import MetacognitiveAdvisory
|
||||
|
||||
drained = list(self._pending_tool_advisories)
|
||||
self._pending_tool_advisories.clear()
|
||||
advisories.extend(
|
||||
MetacognitiveAdvisory(nudge_type=nt, message=text) for nt, text in drained
|
||||
)
|
||||
self._emit_nudge_ping(nt for nt, _ in drained)
|
||||
metacog_reminders.extend({"type": nt, "text": text} for nt, text in drained)
|
||||
|
||||
# Drain queued user messages on the last result in the batch.
|
||||
# Attachment-bearing items fall back to a full multipart user
|
||||
@@ -3834,7 +3951,7 @@ class ChatSession:
|
||||
if att_ids:
|
||||
attachment_items.append((queue_msg_id, msg, priority, att_ids))
|
||||
else:
|
||||
advisories.append(UserInterjection(message=msg, priority=priority))
|
||||
persistent.append(UserInterjection(message=msg, priority=priority))
|
||||
if attachment_items:
|
||||
from turnstone.core.tool_advisory import PRIORITY_IMPORTANT
|
||||
|
||||
@@ -3845,7 +3962,7 @@ class ChatSession:
|
||||
)
|
||||
self._append_user_turn(text, resolved, send_id=queue_msg_id)
|
||||
|
||||
return advisories
|
||||
return persistent, metacog_reminders
|
||||
|
||||
# -- Two-phase tool execution -----------------------------------------------
|
||||
#
|
||||
@@ -3974,7 +4091,14 @@ class ChatSession:
|
||||
)
|
||||
return item["call_id"], item["error"]
|
||||
if item.get("denied"):
|
||||
return item["call_id"], item.get("denial_msg", "Denied by user")
|
||||
msg = item.get("denial_msg", "Denied by user")
|
||||
self._report_tool_result(
|
||||
item["call_id"],
|
||||
item.get("func_name", "unknown"),
|
||||
msg,
|
||||
is_error=True,
|
||||
)
|
||||
return item["call_id"], msg
|
||||
try:
|
||||
result: tuple[str, str | list[dict[str, Any]]] = item["execute"](item)
|
||||
return result
|
||||
@@ -5377,59 +5501,52 @@ class ChatSession:
|
||||
def _queue_user_advisory(self, nudge_type: str, text: str) -> None:
|
||||
"""Queue a metacognitive nudge for the next user turn.
|
||||
|
||||
Drains in ``_append_user_turn`` as a ``<system-reminder>`` block
|
||||
appended to the user message body. Used for nudges that respond
|
||||
to user behaviour: ``correction``, ``denial``, ``resume``,
|
||||
Drains in ``_append_user_turn`` onto the user message dict's
|
||||
``_reminders`` side-channel. Used for nudges that respond to
|
||||
user behaviour: ``correction``, ``denial``, ``resume``,
|
||||
``start``, ``completion``.
|
||||
"""
|
||||
self._pending_user_advisories.append((nudge_type, text))
|
||||
|
||||
def _splice_pending_user_advisories(self, user_msg: dict[str, Any]) -> None:
|
||||
"""Drain ``_pending_user_advisories`` into *user_msg*'s content.
|
||||
def _attach_pending_user_reminders(self, user_msg: dict[str, Any]) -> None:
|
||||
"""Drain ``_pending_user_advisories`` onto *user_msg*'s ``_reminders``
|
||||
sibling key (a side-channel, never inside ``content``).
|
||||
|
||||
Mutates *user_msg* in place — the caller appends it after.
|
||||
Renders each queued nudge as a ``<system-reminder>`` block
|
||||
(same envelope as ``wrap_tool_result``) and attaches them to
|
||||
the trailing edge of the user content. Every text segment in
|
||||
the user content is passed through ``escape_wrapper_tags``
|
||||
first so a user typing literal ``<system-reminder>`` cannot
|
||||
fabricate an envelope the model would treat as a
|
||||
Turnstone-issued reminder. For attachment-bearing turns the
|
||||
blocks land on the trailing text part so they stay glued to
|
||||
the same multipart turn.
|
||||
Mutates *user_msg* in place — the caller appends it after. The
|
||||
rendered ``<system-reminder>`` envelope is built later in
|
||||
``_apply_reminders_for_provider`` against a transient copy, so
|
||||
the model still sees the reminder spliced into ``content`` at
|
||||
the wire boundary while ``self.messages`` and every downstream
|
||||
consumer (UI replay, compaction, title gen, channel adapters,
|
||||
DB) see clean user text.
|
||||
|
||||
``_reminders`` rides the leading-underscore convention used by
|
||||
other internal sibling metadata (``_attachments_meta``,
|
||||
``_provider_content``); ``sanitize_messages`` strips it before
|
||||
the wire on its own pass.
|
||||
|
||||
Also fires the live ``on_user_reminder`` UI hook so any open
|
||||
SSE consumers (other browser tabs, CLI mirrors, eventual
|
||||
channel adapters) can render the reminder bubble in lockstep
|
||||
with the originating tab's optimistic render. Hook failures
|
||||
are logged and swallowed: the side-channel write is the
|
||||
load-bearing op, and a UI implementation throwing here must
|
||||
not abort the user's send (which would otherwise drop both the
|
||||
user message and the queued nudges).
|
||||
"""
|
||||
if not self._pending_user_advisories:
|
||||
return
|
||||
from turnstone.core.tool_advisory import escape_wrapper_tags, render_system_reminder
|
||||
|
||||
items = list(self._pending_user_advisories)
|
||||
self._pending_user_advisories.clear()
|
||||
|
||||
block = "\n\n" + "\n\n".join(render_system_reminder(text) for _, text in items)
|
||||
content = user_msg["content"]
|
||||
if isinstance(content, str):
|
||||
user_msg["content"] = escape_wrapper_tags(content) + block
|
||||
else:
|
||||
text_parts = [p for p in content if isinstance(p, dict) and p.get("type") == "text"]
|
||||
for part in text_parts:
|
||||
part["text"] = escape_wrapper_tags(part.get("text", ""))
|
||||
if text_parts:
|
||||
text_parts[-1]["text"] = text_parts[-1]["text"] + block
|
||||
else:
|
||||
content.append({"type": "text", "text": block})
|
||||
reminders = [{"type": nudge_type, "text": text} for nudge_type, text in items]
|
||||
user_msg["_reminders"] = reminders
|
||||
|
||||
self._emit_nudge_ping(nudge_type for nudge_type, _ in items)
|
||||
|
||||
def _emit_nudge_ping(self, types: Iterable[str]) -> None:
|
||||
"""Surface the ``[metacognition: nudge injected — …]`` UI line.
|
||||
|
||||
Centralised so both drain sites (tool channel via
|
||||
``_collect_advisories``, user channel via
|
||||
``_splice_pending_user_advisories``) emit the same wording.
|
||||
"""
|
||||
joined = ", ".join(types)
|
||||
if joined:
|
||||
self.ui.on_info(f"{GRAY}[metacognition: nudge injected — {joined}]{RESET}")
|
||||
try:
|
||||
self.ui.on_user_reminder(reminders)
|
||||
except Exception:
|
||||
log.warning("ui.on_user_reminder failed; reminder still attached", exc_info=True)
|
||||
|
||||
def _queue_tool_advisory(self, nudge_type: str, text: str) -> None:
|
||||
"""Queue a metacognitive nudge for the next tool-result batch.
|
||||
@@ -5441,6 +5558,95 @@ class ChatSession:
|
||||
"""
|
||||
self._pending_tool_advisories.append((nudge_type, text))
|
||||
|
||||
def _apply_post_execute_advisories(
|
||||
self,
|
||||
tool_calls: list[dict[str, Any]],
|
||||
results: list[tuple[str, str | list[dict[str, Any]]]],
|
||||
) -> None:
|
||||
"""Run repeat detection + tool-error nudge over a freshly-executed batch.
|
||||
|
||||
Mutates *results* in place when an identical-repeat warning is
|
||||
appended to a tool's text output. Updates ``self._repeat_detector``,
|
||||
``self._pending_tool_advisories``, and ``self._metacog_state``
|
||||
(cooldown timestamp via ``should_nudge``). The operator-visible
|
||||
signal is the themed ``tool_reminder`` bubble below the tool
|
||||
block — emitted by the per-result loop downstream when the
|
||||
drained metacog reminders attach to the tool message dict's
|
||||
``_reminders`` side-channel.
|
||||
|
||||
Repeat detection's job is to nudge a flaky local model out of a
|
||||
loop where it keeps making the same tool call ("``bash(cmd='echo
|
||||
test')`` × 3" being the canonical example). It fires on the
|
||||
consecutive-streak signal alone, with no regard for the tool's
|
||||
success / failure / output content — same (name, args) for N
|
||||
turns in a row is by definition stuck. ``RepeatDetector.record``
|
||||
already resets the streak on any different signature, so an
|
||||
intervening tool call (read, write, anything different) breaks
|
||||
the streak naturally without an explicit clear here.
|
||||
|
||||
``_tool_error_flags`` is the authoritative is_error signal —
|
||||
consumed below for the tool-error nudge gate; the per-result
|
||||
loop in ``_run_loop`` ``.pop``s it after this returns.
|
||||
"""
|
||||
# Repeat detection: warn when a tool is called with identical
|
||||
# args N times in a row. Independent of success/failure — the
|
||||
# stuck-loop pattern is sig-driven, not state-driven. JSON
|
||||
# outputs (MCP structured results) are tracked but exempt from
|
||||
# the inline warning text (appending text would corrupt the
|
||||
# payload).
|
||||
_tc_by_id = {c["id"]: c for c in tool_calls}
|
||||
_repeat_detected = False
|
||||
|
||||
for i, (tc_id, output) in enumerate(results):
|
||||
tc = _tc_by_id.get(tc_id)
|
||||
if tc and isinstance(output, str):
|
||||
raw = tc["function"]["name"] + ":" + tc["function"]["arguments"]
|
||||
sig = hashlib.sha256(raw.encode()).hexdigest()
|
||||
is_json = output.lstrip().startswith(("{", "["))
|
||||
if self._repeat_detector.record(sig):
|
||||
_repeat_detected = True
|
||||
if not is_json:
|
||||
output += (
|
||||
"\n\n⚠ Warning: this is an identical repeat of a "
|
||||
"previous tool call. The result is the same. "
|
||||
"Try a different approach."
|
||||
)
|
||||
results[i] = (tc_id, output)
|
||||
# The themed ``tool_reminder`` bubble below the tool
|
||||
# block carries the operator-visible signal; the
|
||||
# tool-name context comes from the visible tool
|
||||
# block immediately above the bubble, so a separate
|
||||
# diagnostic info line would just duplicate it.
|
||||
|
||||
if _repeat_detected:
|
||||
# Reset so the model gets a clean slate after the warning.
|
||||
# If it repeats again, a new warning fires.
|
||||
self._repeat_detector.clear()
|
||||
if self._mem_cfg.nudges and should_nudge(
|
||||
"repeat",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
):
|
||||
self._queue_tool_advisory("repeat", format_nudge("repeat"))
|
||||
|
||||
# Tool-error nudge — queued so the MetacognitiveAdvisory rides
|
||||
# the same _collect_advisories drain pass as guard findings and
|
||||
# user interjections. Cooldown gating in should_nudge keeps
|
||||
# this to one nudge per batch even with many failing tools.
|
||||
if (
|
||||
self._mem_cfg.nudges
|
||||
and any(self._tool_error_flags.get(tc_id) for tc_id, _ in results)
|
||||
and should_nudge(
|
||||
"tool_error",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
memory_count=self._visible_memory_count(),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
)
|
||||
):
|
||||
self._queue_tool_advisory("tool_error", format_nudge("tool_error"))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Coordinator tools — reachable only when ``kind == "coordinator"``.
|
||||
# All six dispatch through ``self._coord_client`` which is None when
|
||||
@@ -9203,7 +9409,7 @@ class ChatSession:
|
||||
elif cmd == "/clear":
|
||||
self.messages.clear()
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
self._last_usage = None
|
||||
self._calibrated_msg_count = 0
|
||||
self._msg_tokens = []
|
||||
@@ -9214,7 +9420,7 @@ class ChatSession:
|
||||
|
||||
self.messages.clear()
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
self._last_usage = None
|
||||
self._calibrated_msg_count = 0
|
||||
self._msg_tokens = []
|
||||
|
||||
@@ -13,6 +13,7 @@ import contextlib
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
@@ -28,6 +29,22 @@ if TYPE_CHECKING:
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# Maps each workstream kind to the ``services.service_type`` its hosting
|
||||
# process registers under. Used by ``SessionManager.close_idle`` pass 2
|
||||
# to enumerate live peer processes for orphan-reaper liveness scoping.
|
||||
# Server processes register as ``("server", node_id, ...)`` (see
|
||||
# ``turnstone/server.py``); the console process as ``("console",
|
||||
# "console", ...)`` (see ``turnstone/console/server.py``). Deriving from
|
||||
# kind here removes a duplicated-config footgun: any caller that builds
|
||||
# a ``SessionManager`` automatically gets the correct service_type for
|
||||
# its kind, with no risk of miswiring INTERACTIVE→"console" or vice
|
||||
# versa.
|
||||
_KIND_SERVICE_TYPE: dict[WorkstreamKind, str] = {
|
||||
WorkstreamKind.INTERACTIVE: "server",
|
||||
WorkstreamKind.COORDINATOR: "console",
|
||||
}
|
||||
|
||||
|
||||
class SessionKindAdapter(Protocol):
|
||||
"""Per-kind construction + cleanup policies the shared ``SessionManager`` delegates to.
|
||||
|
||||
@@ -197,13 +214,21 @@ class SessionManager:
|
||||
# manager never reads them.
|
||||
self._active_id: str | None = None
|
||||
self._eviction_count: int = 0
|
||||
# Optional state-change observer. The CLI sets this to a
|
||||
# callback that prints a background-attention notification
|
||||
# when a non-focused workstream transitions to ATTENTION.
|
||||
# Web/coord paths use the event_emitter's emit_state for their
|
||||
# own fan-out; this is a second, manager-level hook for callers
|
||||
# that don't consume SSE.
|
||||
self._on_state_change: Callable[[str, WorkstreamState], None] | None = None
|
||||
# State-change subscribers. Multi-subscriber to support the
|
||||
# CLI's background-attention notification AND the in-process
|
||||
# ``SameNodeChildSource`` strategy that delivers child
|
||||
# workstream state changes to a parent's UI without going
|
||||
# through the cluster bus. Each callback fires under
|
||||
# exception-suppression so one failing subscriber doesn't
|
||||
# block the others. Subscribers register via
|
||||
# :meth:`subscribe_to_state`. ``_state_subscribers_lock``
|
||||
# guards mutation + snapshot — set_state copies the list
|
||||
# under the lock then iterates the snapshot unlocked so a
|
||||
# slow subscriber doesn't block subscribe/unsubscribe (and
|
||||
# so concurrent subscribe/unsubscribe during a state event
|
||||
# can't shift the iterator's index — caught by /review bug-1).
|
||||
self._state_subscribers: list[Callable[[str, WorkstreamState], None]] = []
|
||||
self._state_subscribers_lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Properties
|
||||
@@ -217,6 +242,16 @@ class SessionManager:
|
||||
def kind(self) -> WorkstreamKind:
|
||||
return self._adapter.kind
|
||||
|
||||
@property
|
||||
def _service_type(self) -> str | None:
|
||||
"""``services.service_type`` this manager's hosting process registers
|
||||
under, derived from its ``kind``. Used by ``close_idle`` pass 2 to
|
||||
enumerate live peer processes. Returns ``None`` for kinds that have
|
||||
no production service mapping (only the two existing kinds map
|
||||
today; ``None`` would be a marker for a future kind without a
|
||||
clustered hosting model)."""
|
||||
return _KIND_SERVICE_TYPE.get(self.kind)
|
||||
|
||||
@property
|
||||
def count(self) -> int:
|
||||
with self._lock:
|
||||
@@ -604,6 +639,22 @@ class SessionManager:
|
||||
# last close(). The next set_state() call syncs it
|
||||
# naturally; writing 'idle' here could race a concurrent
|
||||
# close() that writes 'closed' under self._lock.
|
||||
#
|
||||
# Bump only ``updated`` (no state write) so this row's
|
||||
# timestamp is fresh against the orphan-reaper cutoff —
|
||||
# otherwise a concurrent close_idle pass-2 in this same
|
||||
# process could clobber a freshly-rehydrated row whose
|
||||
# ``updated`` is older than the cutoff. The pure-
|
||||
# timestamp write is safe against concurrent close()
|
||||
# because close still wins on the state column.
|
||||
try:
|
||||
self._storage.touch_workstream(ws_id)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"session_mgr.touch_workstream_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_rehydrated(ws)
|
||||
return ws
|
||||
@@ -773,9 +824,36 @@ class SessionManager:
|
||||
log.debug("session_mgr.state_update_failed ws=%s", ws_id[:8], exc_info=True)
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_state(ws, state)
|
||||
if self._on_state_change is not None:
|
||||
# Snapshot under the subscribers lock so concurrent
|
||||
# subscribe / unsubscribe can't shift the iterator's index
|
||||
# mid-dispatch (skipping or repeating callbacks). Iterate
|
||||
# the snapshot WITHOUT the lock so a slow callback doesn't
|
||||
# block subscribe / unsubscribe.
|
||||
with self._state_subscribers_lock:
|
||||
subscribers = list(self._state_subscribers)
|
||||
for callback in subscribers:
|
||||
with contextlib.suppress(Exception):
|
||||
self._on_state_change(ws_id, state)
|
||||
callback(ws_id, state)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State-change subscription
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def subscribe_to_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Register ``callback`` to fire on every workstream state change.
|
||||
|
||||
Multiple subscribers are supported and fire in registration order.
|
||||
Each callback is wrapped in exception-suppression so a failing
|
||||
subscriber doesn't block the others. Use
|
||||
:meth:`unsubscribe_from_state` to remove.
|
||||
"""
|
||||
with self._state_subscribers_lock:
|
||||
self._state_subscribers.append(callback)
|
||||
|
||||
def unsubscribe_from_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Remove a previously-registered state-change callback. No-op if absent."""
|
||||
with self._state_subscribers_lock, contextlib.suppress(ValueError):
|
||||
self._state_subscribers.remove(callback)
|
||||
|
||||
def cancel(self, ws_id: str) -> bool:
|
||||
"""Cancel in-flight generation and unblock any pending approval / plan.
|
||||
@@ -804,15 +882,52 @@ class SessionManager:
|
||||
def close_idle(self, max_age_seconds: float) -> list[str]:
|
||||
"""Close IDLE workstreams inactive for more than ``max_age_seconds``.
|
||||
|
||||
Returns the list of closed ws_ids. Unlike the old WSM version,
|
||||
this does NOT skip the last workstream — the default-startup
|
||||
relic is gone, callers can handle the 0-workstream case.
|
||||
Two-pass shape:
|
||||
|
||||
- Pass 1 (in-memory): close loaded ``IDLE`` rows whose
|
||||
``ws.last_active`` (monotonic) is past timeout. Closes only
|
||||
``IDLE`` so legitimately-attentive rows (waiting for user
|
||||
response) stay live.
|
||||
- Pass 2 (DB orphans): bulk-close DB rows of this manager's
|
||||
kind whose ``updated`` is past the wall-clock cutoff and
|
||||
which are not currently loaded. This catches workstreams
|
||||
left behind by prior process incarnations — a process crash
|
||||
/restart leaves rows in non-terminal states forever
|
||||
otherwise. Closes ``idle/thinking/attention/running``
|
||||
because any matching row is by definition not loaded by any
|
||||
live process and cannot be in a live interaction.
|
||||
|
||||
**Liveness scoping** (the rendezvous router's primitive
|
||||
since PR #384): when ``self._service_type`` resolves to a
|
||||
known service type — both production kinds do — pass 2
|
||||
calls ``storage.list_services`` to enumerate peer processes
|
||||
with recent heartbeats and protects rows whose ``node_id``
|
||||
matches a live ``service_id`` from reap, even when *this*
|
||||
manager is on a different node. This is essential for
|
||||
containerized deployments with dynamic hostnames: dead-pod
|
||||
rows fall out of the live set after the heartbeat window
|
||||
and become reapable; alive-pod rows stay protected as long
|
||||
as the owner heartbeats. A future kind with no service
|
||||
registration would resolve ``_service_type`` to ``None``
|
||||
and skip the live-services lookup (single-process / CLI).
|
||||
|
||||
**Conservative fallback**: if ``list_services`` raises,
|
||||
pass 2 is skipped entirely this tick — never reap when
|
||||
liveness state is unknown. Pass 1 still runs. Next tick
|
||||
retries the lookup.
|
||||
|
||||
Returns the combined list of closed ws_ids (in-memory first,
|
||||
then DB orphans). Pass 1 emits ``ws_closed``; pass 2 does
|
||||
not, because never-loaded rows have no SSE listeners
|
||||
expecting them.
|
||||
|
||||
Atomic pop per victim under ``self._lock`` (bug-5): a pending
|
||||
tool result can flip state IDLE→RUNNING between the snapshot
|
||||
and the close, so the state test + pop must run together.
|
||||
Batches every pop under one ``self._lock`` acquisition (perf-5)
|
||||
rather than locking once per victim.
|
||||
rather than locking once per victim. The DB pass runs OUTSIDE
|
||||
``self._lock`` — only a brief lock to snapshot loaded keys —
|
||||
so a slow UPDATE doesn't block create/get/set_state.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
popped: list[Workstream] = []
|
||||
@@ -853,6 +968,63 @@ class SessionManager:
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_closed(ws.id, name=ws.name)
|
||||
closed_ids.append(ws.id)
|
||||
|
||||
# Pass 2: reap DB orphans of this kind older than the cutoff.
|
||||
# Snapshot loaded keys under self._lock briefly so a concurrent
|
||||
# create/load doesn't get its row clobbered by the UPDATE; release
|
||||
# before the DB call.
|
||||
#
|
||||
# Liveness scoping uses ``services.last_heartbeat`` — the same
|
||||
# primitive the rendezvous router (PR #384) uses for routing. A
|
||||
# row's ``node_id`` is stamped at create time and never updated;
|
||||
# in containerized deployments with dynamic hostnames the dead
|
||||
# pod's ``node_id`` points at a service that's no longer
|
||||
# heartbeating, so the row falls through to reap. Conversely,
|
||||
# rows whose ``node_id`` matches a heartbeating service are
|
||||
# protected even when *this* manager is on a different node —
|
||||
# the alive peer may legitimately have them loaded.
|
||||
with self._lock:
|
||||
loaded = list(self._workstreams.keys())
|
||||
cutoff = (datetime.now(UTC) - timedelta(seconds=max_age_seconds)).strftime(
|
||||
"%Y-%m-%dT%H:%M:%S"
|
||||
)
|
||||
live_node_ids: list[str] | None = None
|
||||
skip_pass_2 = False
|
||||
if self._service_type is not None:
|
||||
try:
|
||||
live_services = self._storage.list_services(self._service_type)
|
||||
live_node_ids = [
|
||||
str(svc["service_id"]) for svc in live_services if svc.get("service_id")
|
||||
]
|
||||
except Exception:
|
||||
# Conservative fallback: skip pass 2 entirely this tick
|
||||
# so we can't accidentally reap rows whose owners we
|
||||
# failed to enumerate. Next tick retries.
|
||||
log.debug(
|
||||
"session_mgr.list_services_failed kind=%s",
|
||||
self.kind.value,
|
||||
exc_info=True,
|
||||
)
|
||||
skip_pass_2 = True
|
||||
orphans: list[str] = []
|
||||
if not skip_pass_2:
|
||||
try:
|
||||
orphans = self._storage.bulk_close_stale_orphans(
|
||||
self.kind, cutoff, loaded, live_node_ids=live_node_ids
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"session_mgr.bulk_close_orphans_failed kind=%s",
|
||||
self.kind.value,
|
||||
exc_info=True,
|
||||
)
|
||||
if orphans:
|
||||
log.info(
|
||||
"session_mgr.bulk_close_orphans count=%d kind=%s",
|
||||
len(orphans),
|
||||
self.kind.value,
|
||||
)
|
||||
closed_ids.extend(orphans)
|
||||
return closed_ids
|
||||
|
||||
def _close_if_idle_locked(self, ws_id: str) -> Workstream | None:
|
||||
|
||||
@@ -394,6 +394,15 @@ class SessionEndpointConfig:
|
||||
# separate ``/history`` endpoint and doesn't render the per-tab
|
||||
# status bar). Kinds that don't need pre-replay wire ``None``.
|
||||
events_replay: EventsReplay | None = None
|
||||
# async (ws, ui, request) -> None. Kind-specific async pre-step
|
||||
# the lifted ``events`` body awaits BEFORE iterating
|
||||
# ``events_replay``. Lets a kind move blocking storage I/O off
|
||||
# the event loop (via ``asyncio.to_thread``) and stash results
|
||||
# on ``request.state`` for the sync replay generator to read.
|
||||
# Interactive uses it to pre-load intent_verdicts +
|
||||
# output_assessments so ``_build_history``'s decoration stays
|
||||
# off the hot path. Coord wires ``None``.
|
||||
events_replay_prepare: Callable[..., Any] | None = None
|
||||
# (request) -> Executor for the SSE live-loop's blocking
|
||||
# ``queue.get`` wait. Interactive returns the dedicated
|
||||
# ``request.app.state.sse_executor`` (200-thread pool) so SSE
|
||||
@@ -1203,6 +1212,8 @@ def make_open_handler(
|
||||
"""
|
||||
|
||||
async def open_ws(request: Request) -> Response:
|
||||
import asyncio
|
||||
|
||||
if cfg.permission_gate is not None:
|
||||
err = cfg.permission_gate(request)
|
||||
if err is not None:
|
||||
@@ -1304,7 +1315,14 @@ def make_open_handler(
|
||||
# emit_rehydrated path).
|
||||
if cfg.open_post_load is not None:
|
||||
try:
|
||||
cfg.open_post_load(request, ws)
|
||||
# Off-loop: interactive's post_load runs the sync
|
||||
# ``_build_history`` (storage I/O for verdict
|
||||
# indexes + message reconstruction) — without the
|
||||
# to_thread wrap this blocks the event loop on every
|
||||
# workstream open, mirroring the SSE replay path
|
||||
# that's already protected via
|
||||
# ``events_replay_prepare``.
|
||||
await asyncio.to_thread(cfg.open_post_load, request, ws)
|
||||
except Exception:
|
||||
# Post-load is observational — never let a hook bug
|
||||
# block the open. Log + continue.
|
||||
@@ -1454,6 +1472,20 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler:
|
||||
# 500-slot cap on a chatty mid-generation workstream)
|
||||
# while replay was being built.
|
||||
if replay_cb is not None:
|
||||
# Kind-specific async prep — runs before the sync
|
||||
# replay generator iterates so blocking storage
|
||||
# I/O lands in the executor pool rather than the
|
||||
# event loop's hot path. Interactive uses this
|
||||
# to pre-load verdict indexes; coord skips.
|
||||
if cfg.events_replay_prepare is not None:
|
||||
try:
|
||||
await cfg.events_replay_prepare(ws, ui, request)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"ws.events.replay_prepare_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
for ev in replay_cb(ws, ui, request):
|
||||
yield {"data": json.dumps(ev)}
|
||||
@@ -2240,6 +2272,34 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler:
|
||||
messages = await asyncio.to_thread(storage.load_messages, ws_id, limit=limit)
|
||||
except Exception:
|
||||
log.debug("ws.history.load_failed ws=%s", ws_id[:8], exc_info=True)
|
||||
# Audit-trail decoration — attach persisted intent_verdict and
|
||||
# output_assessment data to each assistant.tool_calls entry so
|
||||
# the dashboard's history replay paints the same verdict pills
|
||||
# / output-warning bubbles the live SSE path shows. Both
|
||||
# storage queries are off-loop via ``to_thread``. Best-effort:
|
||||
# any failure leaves messages undecorated — replay degrades to
|
||||
# the pre-decoration shape rather than 500-ing.
|
||||
if messages:
|
||||
try:
|
||||
from turnstone.core.history_decoration import (
|
||||
decorate_history_messages,
|
||||
load_verdict_indexes,
|
||||
)
|
||||
|
||||
indexes = await asyncio.to_thread(load_verdict_indexes, ws_id)
|
||||
decorate_history_messages(messages, indexes[0], indexes[1])
|
||||
except Exception:
|
||||
# Operationally interesting: a persistent decoration
|
||||
# failure (missing migration, driver mismatch, schema
|
||||
# drift) silently strips verdict pills + output
|
||||
# warnings from every reload of every workstream.
|
||||
# Log at warning so it surfaces in normal log review
|
||||
# rather than only when DEBUG is on.
|
||||
log.warning(
|
||||
"ws.history.decoration_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
return JSONResponse({"ws_id": ws_id, "messages": messages})
|
||||
|
||||
return history
|
||||
|
||||
@@ -41,6 +41,7 @@ log = get_logger(__name__)
|
||||
# from bloating memory.
|
||||
_DEFAULT_LISTENER_QUEUE_MAX = 500
|
||||
|
||||
|
||||
# Cap on the per-turn assistant content accumulator. The accumulator
|
||||
# is piggybacked onto the ``ws_state:idle`` broadcast payload so the
|
||||
# cluster collector / dashboard can render the freshly-emitted assistant
|
||||
@@ -327,6 +328,11 @@ class SessionUIBase:
|
||||
"always": bool(always),
|
||||
}
|
||||
)
|
||||
# Kind-specific cross-stream broadcast — ConsoleCoordinatorUI
|
||||
# overrides to push onto the cluster bus so a coord parent's
|
||||
# tree UI clears the pending-approval pill in lockstep with
|
||||
# the actual decision. Stage 3 Step 4.
|
||||
self._broadcast_approval_resolved(approved, feedback, always=always)
|
||||
self._approval_event.set()
|
||||
|
||||
@staticmethod
|
||||
@@ -579,6 +585,20 @@ class SessionUIBase:
|
||||
"judge_pending": judge_pending,
|
||||
}
|
||||
self._enqueue(self._pending_approval)
|
||||
# Cross-stream broadcast — push the items via the cluster bus
|
||||
# so a coord parent's tree UI can render the inline approve/deny
|
||||
# block without waiting for a bulk fetch. Without this, the
|
||||
# bulk fetch races with this assignment: the state transition
|
||||
# to ATTENTION fires upstream BEFORE approve_tools runs (see
|
||||
# session.py:_emit_state("attention") preceding ui.approve_tools),
|
||||
# so a bulk fetch landing in the ~50-200ms window between
|
||||
# _emit_state and this point sees ``_pending_approval=None``
|
||||
# and returns ``pending_approval_detail: null``. The 5s TTL
|
||||
# then locks the coord row on a "loading" placeholder until
|
||||
# the next state event triggers a refresh — which never comes
|
||||
# while parked on _approval_event.wait. The push path
|
||||
# eliminates the race.
|
||||
self._broadcast_approve_request(self._pending_approval)
|
||||
if not self._approval_event.wait(timeout=self._APPROVAL_WAIT_TIMEOUT):
|
||||
# Approval timed out (e.g., user disconnected). Deny via
|
||||
# resolve_approval so verdicts and state are updated consistently.
|
||||
@@ -632,6 +652,12 @@ class SessionUIBase:
|
||||
del self._llm_verdicts[oldest_key]
|
||||
self._llm_verdicts[call_id] = verdict
|
||||
self._enqueue({"type": "intent_verdict", **verdict})
|
||||
# Kind-specific cross-stream broadcast — ConsoleCoordinatorUI
|
||||
# overrides to push onto the cluster bus so a coord parent's
|
||||
# tree UI sees the verdict without polling. Default is no-op
|
||||
# (the per-ws ``_enqueue`` above already covers WebUI's own
|
||||
# SSE listeners). Stage 3 Step 4.
|
||||
self._broadcast_intent_verdict(verdict)
|
||||
self._persist_intent_verdict(verdict)
|
||||
# Decision check + either queue or flag-for-persist happen
|
||||
# under ONE lock acquisition so resolve_approval can't swap-
|
||||
@@ -1293,6 +1319,42 @@ class SessionUIBase:
|
||||
def on_error(self, message: str) -> None:
|
||||
self._enqueue({"type": "error", "message": message})
|
||||
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
"""Surface a metacognitive user-channel nudge as its own UI
|
||||
element.
|
||||
|
||||
Reminders live on the user message dict's ``_reminders``
|
||||
side-channel and are spliced into ``content`` only at the
|
||||
provider boundary; this event is what lets every connected
|
||||
SSE consumer (other browser tabs, CLI mirrors, future channel
|
||||
adapters) render the reminder bubble in lockstep with the
|
||||
originating tab. The history-replay path surfaces the same
|
||||
shape via ``_build_history`` so a tab reconnecting later
|
||||
renders the same bubble.
|
||||
"""
|
||||
self._enqueue({"type": "user_reminder", "reminders": reminders})
|
||||
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None:
|
||||
"""Surface a metacognitive tool-channel nudge (``tool_error`` /
|
||||
``repeat``) as its own UI element below the tool result that
|
||||
triggered it.
|
||||
|
||||
Tool-channel reminders ride the same ``_reminders``
|
||||
side-channel pattern as the user channel — kept out of
|
||||
``content`` so compaction / title-gen / channel adapters never
|
||||
see the nudge text, spliced into the wire only via
|
||||
``_apply_reminders_for_provider``. ``tool_call_id`` is the
|
||||
anchor the frontend uses to render the bubble below the
|
||||
specific tool result that triggered the batch's reminder.
|
||||
"""
|
||||
self._enqueue(
|
||||
{
|
||||
"type": "tool_reminder",
|
||||
"reminders": reminders,
|
||||
"tool_call_id": tool_call_id,
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Broadcast hooks — kind-specific transport.
|
||||
#
|
||||
@@ -1315,6 +1377,42 @@ class SessionUIBase:
|
||||
Default: no-op. Subclasses override.
|
||||
"""
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None: # noqa: ARG002 — hook stub
|
||||
"""Fan an LLM intent-judge verdict out to the kind's transport.
|
||||
|
||||
Default: no-op. ``ConsoleCoordinatorUI`` overrides to push a
|
||||
``intent_verdict`` event onto the cluster bus so the parent
|
||||
coordinator's tree UI can render the risk pill + verdict
|
||||
result without polling. Stage 3 Step 4: hook only — the
|
||||
cluster-bus event class lands in Step 5.
|
||||
"""
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool, # noqa: ARG002 — hook stub
|
||||
feedback: str | None = None, # noqa: ARG002 — hook stub
|
||||
*,
|
||||
always: bool = False, # noqa: ARG002 — hook stub
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision out to the kind's transport.
|
||||
|
||||
Default: no-op. ``ConsoleCoordinatorUI`` overrides to push to
|
||||
the cluster bus so the parent coordinator's tree UI can clear
|
||||
the pending-approval pill in sync with the actual decision.
|
||||
"""
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None: # noqa: ARG002 — hook stub
|
||||
"""Fan an ``approve_request`` payload out to the kind's transport.
|
||||
|
||||
Default: no-op. ``WebUI`` and ``ConsoleCoordinatorUI`` override
|
||||
to push the items list (the same dict that landed in
|
||||
``_pending_approval``) onto their respective transports. The
|
||||
push path eliminates the bulk-fetch race that otherwise
|
||||
leaves coord rows stuck on a loading placeholder when the
|
||||
bulk fetch lands in the gap between the state transition to
|
||||
ATTENTION and ``_pending_approval`` being set.
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State-broadcast snapshot helper
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -97,7 +97,7 @@ from turnstone.core.storage._utils import sanitize_text
|
||||
from turnstone.core.storage._utils import (
|
||||
scan_skill_content as _scan_skill_content,
|
||||
)
|
||||
from turnstone.core.workstream import WorkstreamKind
|
||||
from turnstone.core.workstream import BULK_CLOSE_STATE_VALUES, WorkstreamKind
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
@@ -595,6 +595,57 @@ class PostgreSQLBackend:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
norm_kind = WorkstreamKind(kind).value
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
stmt = (
|
||||
sa.update(workstreams)
|
||||
.where(
|
||||
workstreams.c.kind == norm_kind,
|
||||
workstreams.c.state.in_(BULK_CLOSE_STATE_VALUES),
|
||||
workstreams.c.updated < cutoff,
|
||||
)
|
||||
.values(state="closed", updated=now)
|
||||
.returning(workstreams.c.ws_id)
|
||||
)
|
||||
# Protect rows whose owning process is still heartbeating in the
|
||||
# services table (rendezvous router's liveness primitive). NULL
|
||||
# node_id rows have no owner identity — always eligible. The
|
||||
# ``and live_node_ids`` short-circuits both ``None`` (skip the
|
||||
# filter entirely — single-process / operator backfill) and ``[]``
|
||||
# (no nodes alive — every row unprotected, no extra predicate
|
||||
# needed since absence equals match-all).
|
||||
if live_node_ids is not None and live_node_ids:
|
||||
stmt = stmt.where(
|
||||
sa.or_(
|
||||
workstreams.c.node_id.is_(None),
|
||||
~workstreams.c.node_id.in_(live_node_ids),
|
||||
)
|
||||
)
|
||||
if exclude_ws_ids:
|
||||
# Skip ``NOT IN ()`` when nothing to exclude — keeps the SQL clean
|
||||
# and avoids SQLAlchemy's empty-collection warning.
|
||||
stmt = stmt.where(~workstreams.c.ws_id.in_(exclude_ws_ids))
|
||||
with self._conn() as conn:
|
||||
result = conn.execute(stmt)
|
||||
ids = [row[0] for row in result]
|
||||
conn.commit()
|
||||
return ids
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
|
||||
@@ -431,6 +431,68 @@ class StorageBackend(Protocol):
|
||||
"""Update a workstream's state and bump updated timestamp."""
|
||||
...
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Close DB-side workstream rows of *kind* whose state is in
|
||||
``BULK_CLOSE_STATE_VALUES`` and whose ``updated`` is lex-older than
|
||||
*cutoff*, excluding rows currently loaded in memory. Sets
|
||||
``state='closed'`` and bumps ``updated``. Returns the list of ws_ids
|
||||
actually transitioned.
|
||||
|
||||
``cutoff`` is a UTC ``YYYY-MM-DDTHH:MM:SS`` string matching the on-disk
|
||||
format ``update_workstream_state`` writes — lex compare is safe for
|
||||
same-offset timestamps. Empty ``exclude_ws_ids`` means no exclusion.
|
||||
|
||||
``live_node_ids`` is the set of ``services.service_id`` values whose
|
||||
``last_heartbeat`` is recent (i.e. owning processes still alive);
|
||||
rows whose ``node_id`` matches one of these are protected because
|
||||
their owning process may legitimately have them loaded on another
|
||||
worker. ``None`` skips the filter entirely (single-process / tests
|
||||
/ operator backfill). Empty list ``[]`` treats every node as dead —
|
||||
useful when operator scripts want to reap regardless of liveness.
|
||||
|
||||
Rows with ``NULL`` ``node_id`` are always eligible: they have no
|
||||
meaningful owner identity, so age alone gates the reap.
|
||||
|
||||
Liveness scoping replaces an earlier ``node_id == self`` heuristic.
|
||||
That heuristic broke in the post-rendezvous-routing world (PR #384):
|
||||
``workstreams.node_id`` is stamped at create time and never updated,
|
||||
so dead-pod orphans in containerized deployments with dynamic
|
||||
hostnames couldn't be reclaimed. ``services.last_heartbeat`` is the
|
||||
rendezvous router's authoritative liveness primitive — using it here
|
||||
keeps reap scoping aligned with routing.
|
||||
|
||||
Asymmetric with ``SessionManager.close_idle``'s in-memory pass on
|
||||
purpose: that pass closes only ``IDLE`` (legitimately-attentive rows
|
||||
stay), this method closes the broader ``BULK_CLOSE_STATE_VALUES`` set
|
||||
because any row matching here is by definition not loaded by any
|
||||
live process and cannot be in a live interaction.
|
||||
"""
|
||||
...
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
"""Bump a workstream row's ``updated`` timestamp without touching its
|
||||
state.
|
||||
|
||||
Used by ``SessionManager.open()`` on cold rehydrate so a freshly-
|
||||
loaded row's ``updated`` can't be older than the orphan-reaper cutoff
|
||||
— protects against a same-process race where a parallel
|
||||
``close_idle`` pass-2 snapshots loaded keys after the storage read
|
||||
but before the in-memory install. Distinct from
|
||||
``update_workstream_state(ws_id, current_state)`` because the
|
||||
rehydrate path explicitly avoids a state write (see the
|
||||
``open()`` no-DB-state-flip-on-resurrect comment): a state write
|
||||
could race a concurrent ``close()`` and resurrect a closed row.
|
||||
Bumping only ``updated`` is safe — close still wins on the state
|
||||
column.
|
||||
"""
|
||||
...
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
"""Update a workstream's display name."""
|
||||
...
|
||||
|
||||
@@ -97,7 +97,7 @@ from turnstone.core.storage._utils import sanitize_text
|
||||
from turnstone.core.storage._utils import (
|
||||
scan_skill_content as _scan_skill_content,
|
||||
)
|
||||
from turnstone.core.workstream import WorkstreamKind
|
||||
from turnstone.core.workstream import BULK_CLOSE_STATE_VALUES, WorkstreamKind
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
@@ -707,6 +707,87 @@ class SQLiteBackend:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
norm_kind = WorkstreamKind(kind).value
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
# SQLite has no RETURNING precedent in this file — do SELECT-then-
|
||||
# UPDATE in one transaction, with the SAME WHERE predicates re-applied
|
||||
# to the UPDATE. Re-application defends against a same-process race:
|
||||
# ``SessionManager.open()`` calls ``touch_workstream`` between the
|
||||
# SELECT and the UPDATE could have bumped a row's ``updated`` past
|
||||
# ``cutoff`` (or ``set_state`` could have flipped its state out of
|
||||
# the bulk-close set). Without the re-applied WHERE the UPDATE
|
||||
# closes those rows anyway; with it, the UPDATE skips rows that
|
||||
# became ineligible after the SELECT and the row stays open.
|
||||
# Chunked through ``_in_chunks`` so the ``IN`` clause never exceeds
|
||||
# SQLite's bind-parameter limit (default 999) on a large reap.
|
||||
candidate_conditions = [
|
||||
workstreams.c.kind == norm_kind,
|
||||
workstreams.c.state.in_(BULK_CLOSE_STATE_VALUES),
|
||||
workstreams.c.updated < cutoff,
|
||||
]
|
||||
if live_node_ids is not None and live_node_ids:
|
||||
# Protect rows owned by heartbeating services. NULL node_id is
|
||||
# always eligible. Empty list means "no nodes alive" — every
|
||||
# row is unprotected; the absence of this predicate is
|
||||
# equivalent to "match all," so we just skip it.
|
||||
candidate_conditions.append(
|
||||
sa.or_(
|
||||
workstreams.c.node_id.is_(None),
|
||||
~workstreams.c.node_id.in_(live_node_ids),
|
||||
)
|
||||
)
|
||||
if exclude_ws_ids:
|
||||
candidate_conditions.append(~workstreams.c.ws_id.in_(exclude_ws_ids))
|
||||
select_stmt = sa.select(workstreams.c.ws_id).where(*candidate_conditions)
|
||||
closed: list[str] = []
|
||||
# Match the chunk size used by ``prune_workstreams`` (line 453) — keeps
|
||||
# ``IN`` clauses well below SQLite's default 999-bind-param limit even
|
||||
# on very large reaps.
|
||||
chunk_size = 500
|
||||
with self._conn() as conn:
|
||||
candidate_ids = [row[0] for row in conn.execute(select_stmt)]
|
||||
for i in range(0, len(candidate_ids), chunk_size):
|
||||
chunk = candidate_ids[i : i + chunk_size]
|
||||
# Re-apply the eligibility predicates on the UPDATE so a row
|
||||
# that became fresh between the SELECT and the UPDATE is not
|
||||
# clobbered. Then SELECT back by ``state='closed' AND updated=now``
|
||||
# to determine which rows actually transitioned this commit —
|
||||
# the returned list reflects reality even when re-application
|
||||
# filters out some candidates.
|
||||
conn.execute(
|
||||
sa.update(workstreams)
|
||||
.where(workstreams.c.ws_id.in_(chunk), *candidate_conditions)
|
||||
.values(state="closed", updated=now)
|
||||
)
|
||||
actually_closed = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
sa.select(workstreams.c.ws_id).where(
|
||||
workstreams.c.ws_id.in_(chunk),
|
||||
workstreams.c.state == "closed",
|
||||
workstreams.c.updated == now,
|
||||
)
|
||||
)
|
||||
]
|
||||
closed.extend(actually_closed)
|
||||
conn.commit()
|
||||
return closed
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Add a partial composite index for the orphan-reaper query.
|
||||
|
||||
``StorageBackend.bulk_close_stale_orphans`` (introduced alongside the
|
||||
workstream-lifecycle leak fix) runs every ``min(300s, idle_timeout/4)``
|
||||
on every server and console process. Its WHERE shape is:
|
||||
|
||||
WHERE kind = ?
|
||||
AND state IN ('idle', 'thinking', 'attention', 'running')
|
||||
AND updated < ?
|
||||
AND (node_id IS NULL OR node_id NOT IN (alive_service_ids))
|
||||
|
||||
At current scale (low-thousands of workstream rows) the existing single-
|
||||
column indexes are sufficient — ``idx_workstreams_state`` prunes to the
|
||||
non-closed subset, and the planner filters the rest sequentially. At
|
||||
100k+ rows that filter becomes a tablescan-shaped cost on the reaper's
|
||||
periodic run.
|
||||
|
||||
A **partial** index covering only ``BULK_CLOSE_STATE_VALUES`` rows
|
||||
matches the reaper's query exactly while staying tiny — closed rows
|
||||
(typically 95%+ of the table per empirical diagnosis) and ``error``
|
||||
rows are excluded, so the index is roughly 5% the size a full multi-
|
||||
column index would be. Write amplification only kicks in for
|
||||
transitions that touch one of the four covered states.
|
||||
|
||||
Column order ``(kind, updated)``:
|
||||
|
||||
- ``kind`` first because the reaper always supplies it as an equality
|
||||
predicate; partitions the partial index into interactive vs
|
||||
coordinator subtrees.
|
||||
- ``updated`` last so the range comparison rides the trailing column —
|
||||
classic composite-index pattern for ``WHERE eq AND range``.
|
||||
|
||||
``node_id`` is intentionally NOT in the index. The reaper's predicate
|
||||
on it is ``NOT IN (small list)`` against an unbounded-cardinality
|
||||
column, which planners don't index well; including it would just add
|
||||
write cost for negligible read benefit.
|
||||
|
||||
PostgreSQL uses ``CREATE INDEX CONCURRENTLY`` so the build is
|
||||
non-blocking on a live system; SQLite has no concurrent concept and
|
||||
the table-level write lock already serializes, so a plain
|
||||
``CREATE INDEX`` is fine.
|
||||
|
||||
Revision ID: 048
|
||||
Revises: 047
|
||||
Create Date: 2026-04-30
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "048"
|
||||
down_revision = "047"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_REAPER_PARTIAL_WHERE = "state IN ('idle', 'thinking', 'attention', 'running')"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
with op.get_context().autocommit_block():
|
||||
op.execute(
|
||||
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_workstreams_reaper "
|
||||
"ON workstreams (kind, updated) "
|
||||
f"WHERE {_REAPER_PARTIAL_WHERE}"
|
||||
)
|
||||
else:
|
||||
op.create_index(
|
||||
"idx_workstreams_reaper",
|
||||
"workstreams",
|
||||
["kind", "updated"],
|
||||
sqlite_where=sa.text(_REAPER_PARTIAL_WHERE),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
with op.get_context().autocommit_block():
|
||||
op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_workstreams_reaper")
|
||||
else:
|
||||
op.drop_index("idx_workstreams_reaper", table_name="workstreams")
|
||||
@@ -77,6 +77,25 @@ class WorkstreamState(enum.Enum):
|
||||
ERROR = "error" # last operation failed
|
||||
|
||||
|
||||
# States the orphan reaper (``SessionManager.close_idle`` pass 2 +
|
||||
# ``StorageBackend.bulk_close_stale_orphans``) is allowed to flip to
|
||||
# ``closed`` for rows past the staleness cutoff. Excludes ``ERROR``
|
||||
# deliberately — error rows are user-investigatable and shouldn't be
|
||||
# auto-reaped — and excludes ``CLOSED`` (terminal). Centralized here
|
||||
# so the storage backends and FakeStorage all agree; if a new transient
|
||||
# state is added to ``WorkstreamState``, deciding whether it joins
|
||||
# this set is part of the change rather than an after-the-fact
|
||||
# audit across three files.
|
||||
BULK_CLOSE_STATE_VALUES: frozenset[str] = frozenset(
|
||||
{
|
||||
WorkstreamState.IDLE.value,
|
||||
WorkstreamState.THINKING.value,
|
||||
WorkstreamState.RUNNING.value,
|
||||
WorkstreamState.ATTENTION.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workstream dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -139,6 +139,12 @@ class NullUI:
|
||||
def on_error(self, message: str) -> None:
|
||||
pass
|
||||
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
pass
|
||||
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None:
|
||||
pass
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
You are a coordinator on a small, focused infrastructure team. Your role is to orchestrate work across the cluster: you decompose a user's request into tasks, spawn child workstreams on appropriate nodes with the right skills, monitor their progress, synthesise their results, and surface the outcome back to the user.
|
||||
|
||||
You do not edit files, run shell commands, browse the web, or manipulate the codebase directly. Children do that. Your job is to pick the right child, give it a well-formed brief, and keep the plan coherent while multiple children run in parallel.
|
||||
You do not edit files, run shells, or browse the web — children do. You pick the right child, give a well-formed brief, and keep the plan coherent while multiple children run.
|
||||
|
||||
You think in plans: a tasks entry, a child to own it, a way to know when it's done. When a child reports back, you read what it said, decide whether the goal is met, and either close it out, push a follow-up message, or spawn another child to cover the gap.
|
||||
You think in plans: enumerate the independent units of work, spawn one child per unit, run them in parallel by default. Sequential only when one child's output feeds the next. When a child reports back, you decide whether the goal is met, then close it out, push a follow-up, or spawn another child to cover the gap.
|
||||
|
||||
You are precise about what you delegate. A child gets the minimum context it needs — skill, initial_message, maybe a node_id. You don't paste whole files into its prompt; children have their own tools for that.
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
TOOL PATTERNS:
|
||||
|
||||
You are a coordinator. You do not edit files, run shell commands, or browse the web directly. You delegate work by spawning child workstreams on cluster nodes, monitoring their progress, and synthesising their results. Every tool below is in your schema; nothing else is.
|
||||
|
||||
Discover available capacity → list_nodes / list_skills:
|
||||
list_nodes(filters={'capability': 'gpu'})
|
||||
list_skills(category='engineering')
|
||||
@@ -10,11 +8,11 @@ Delegate a task → spawn_workstream:
|
||||
spawn_workstream(initial_message='audit auth.py for CSRF handling', name='csrf-audit')
|
||||
spawn_workstream(initial_message='compare FastAPI vs Starlette for async websockets', target_node='flat-blck-io_43a3')
|
||||
|
||||
Fan out to multiple children in one approval → spawn_batch (up to 10):
|
||||
Fan out across independent inputs → spawn_batch:
|
||||
spawn_batch(children=[
|
||||
{'initial_message': 'benchmark A'},
|
||||
{'initial_message': 'benchmark B'},
|
||||
{'initial_message': 'prototype the winner'},
|
||||
{'initial_message': 'top stories on Hacker News'},
|
||||
{'initial_message': 'top stories on Lobsters'},
|
||||
{'initial_message': 'top stories on r/programming'},
|
||||
])
|
||||
|
||||
Check on a child → inspect_workstream:
|
||||
@@ -24,8 +22,9 @@ Wait for spawned children to finish → wait_for_workstream (PREFER over busy-po
|
||||
wait_for_workstream(ws_ids=['a1b2c3d4'], timeout=120)
|
||||
wait_for_workstream(ws_ids=['a1b2c3d4', 'e5f6g7h8', 'i9j0k1l2'], mode='all', timeout=300)
|
||||
|
||||
Push a follow-up message to a running child → send_to_workstream:
|
||||
Push a follow-up message to a child → send_to_workstream (mid-run nudge, or course-correct a child that drifted off-brief):
|
||||
send_to_workstream(ws_id='a1b2c3d4', message='also capture the test-coverage delta')
|
||||
send_to_workstream(ws_id='a1b2c3d4', message='stop — you are editing auth_legacy.py, the active path is auth.py')
|
||||
|
||||
List what you've spawned → list_workstreams:
|
||||
list_workstreams()
|
||||
@@ -38,19 +37,10 @@ Wind a child down → close_workstream (soft; session stops, storage kept) or de
|
||||
close_workstream(ws_id='a1b2c3d4', reason='task complete')
|
||||
delete_workstream(ws_id='a1b2c3d4')
|
||||
|
||||
Wind all direct children down at once → close_all_children (soft-close cascade, single approval):
|
||||
Wind all direct children down at once → close_all_children (soft-close cascade):
|
||||
close_all_children(reason='batch complete, synthesising results')
|
||||
|
||||
Plan and track work → tasks (your scratchpad; children don't see it):
|
||||
tasks(action='add', title='audit auth.py for CSRF')
|
||||
tasks(action='update', task_id='t_03', status='in_progress')
|
||||
tasks(action='list')
|
||||
tasks(action='remove', task_id='t_03')
|
||||
|
||||
## Workflow shape
|
||||
|
||||
Prefer: tasks to plan → spawn_workstream to delegate → wait_for_workstream to block on completion → inspect_workstream to read the final message → synthesise → close_workstream.
|
||||
|
||||
Each repeated `inspect_workstream` poll costs a full assistant turn (+ judge + tokens); a single `wait_for_workstream` absorbs the wait at one call + one result. The cost gap widens fast on fan-outs of 3+ children.
|
||||
|
||||
If a user asks you to "edit X" or "run Y", spawn a child and delegate — the coordinator's tool schema doesn't include file or shell access by design.
|
||||
|
||||
+241
-16
@@ -53,6 +53,15 @@ from turnstone.core.auth import (
|
||||
_DenyFilter,
|
||||
jwt_version_slot,
|
||||
)
|
||||
from turnstone.core.history_decoration import (
|
||||
TOOL_RESULT_STORAGE_CAP,
|
||||
)
|
||||
from turnstone.core.history_decoration import (
|
||||
decorate_tool_call as _decorate_tool_call,
|
||||
)
|
||||
from turnstone.core.history_decoration import (
|
||||
load_verdict_indexes as _load_verdict_indexes,
|
||||
)
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.metrics import metrics as _metrics
|
||||
from turnstone.core.ratelimit import resolve_client_ip
|
||||
@@ -194,6 +203,14 @@ class WebUI(SessionUIBase):
|
||||
}
|
||||
if state == "idle":
|
||||
event["content"] = payload["content"]
|
||||
# ``pending_approval_detail`` is NO LONGER piggybacked on
|
||||
# state-change events (Stage 3 cleanup). Symmetric event
|
||||
# flow now: initial approval items arrive via bulk fetch
|
||||
# triggered by the ``activity_state="approval"`` transition,
|
||||
# individual verdicts via the explicit
|
||||
# ``intent_verdict`` event class, and resolution via
|
||||
# ``approval_resolved``. Reducer no longer has to dedupe
|
||||
# the piggyback path against the explicit one.
|
||||
try:
|
||||
WebUI._global_queue.put_nowait(event)
|
||||
except queue.Full:
|
||||
@@ -218,6 +235,73 @@ class WebUI(SessionUIBase):
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
"""Send an LLM intent-judge verdict to the global SSE channel.
|
||||
|
||||
Stage 3 Step 5 — the cluster collector's ``_apply_delta``
|
||||
forwards this verbatim to the cluster bus, where coord
|
||||
adapters dispatch it as ``child_ws_intent_verdict`` for the
|
||||
owning parent's tree UI. Unlike the existing
|
||||
``pending_approval_detail`` piggyback on ``ws_state``, this
|
||||
fires WHENEVER a verdict lands — including the common case
|
||||
where the judge daemon writes during ``attention`` with no
|
||||
state transition to ride along on.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": self.ws_id,
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool,
|
||||
feedback: str | None = None,
|
||||
*,
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Send an ``approval_resolved`` decision to the global SSE channel.
|
||||
|
||||
Clears the parent's pending-approval pill in lockstep with
|
||||
the actual decision rather than waiting for the next
|
||||
state-change piggyback.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": self.ws_id,
|
||||
"approved": approved,
|
||||
"feedback": feedback or "",
|
||||
"always": bool(always),
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None:
|
||||
"""Send an ``approve_request`` payload to the global SSE channel.
|
||||
|
||||
Push path for the initial approval items so a coord parent's
|
||||
tree UI can render the inline approve/deny block immediately
|
||||
without waiting for a bulk-fetch round-trip. The bulk fetch
|
||||
races with ``_pending_approval`` being set inside
|
||||
``approve_tools`` (the state transition to ATTENTION fires
|
||||
upstream first); the push path eliminates that race entirely.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": self.ws_id,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
# --- SessionUI protocol ---
|
||||
#
|
||||
# ``on_thinking_start`` / ``on_thinking_stop`` / ``on_reasoning_token``
|
||||
@@ -333,8 +417,20 @@ class WebUI(SessionUIBase):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Verdict + output-assessment decoration helpers (``_decorate_tool_call``,
|
||||
# ``_load_verdict_indexes``) are imported at module top alongside the
|
||||
# rest of ``turnstone.core.*``. Both this builder and
|
||||
# :func:`make_history_handler` (the /history REST endpoint coord uses
|
||||
# as its primary history loader) share them so the two surfaces don't
|
||||
# drift on the wire shape they emit.
|
||||
|
||||
|
||||
def _build_history(
|
||||
session: ChatSession, has_pending_approval: bool = False
|
||||
session: ChatSession,
|
||||
has_pending_approval: bool = False,
|
||||
*,
|
||||
verdicts: dict[str, dict[str, Any]] | None = None,
|
||||
assessments: dict[str, dict[str, Any]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Build a history replay list from ChatSession messages.
|
||||
|
||||
@@ -346,7 +442,35 @@ def _build_history(
|
||||
``"denied": True``, and the corresponding assistant entry that
|
||||
issued the tool calls is also marked ``"denied": True`` so the
|
||||
client can render the correct badge.
|
||||
|
||||
``verdicts`` and ``assessments`` are optional pre-loaded
|
||||
``{call_id → row}`` dicts (see :func:`_load_verdict_indexes`).
|
||||
Async callers should pre-load via ``asyncio.to_thread`` and pass
|
||||
them in to avoid blocking the event loop on storage I/O. When
|
||||
omitted, the storage call runs inline (sync call sites).
|
||||
"""
|
||||
# Metacognitive nudges live on the message dict's ``_reminders``
|
||||
# side-channel — user messages carry user-channel nudges
|
||||
# (correction / denial / resume / start / completion), tool
|
||||
# messages carry tool-channel nudges (tool_error / repeat). Both
|
||||
# are surfaced separately on each entry so the UI can render them
|
||||
# as their own bubble (live via ``user_reminder`` /
|
||||
# ``tool_reminder`` SSE events; replay via this propagation).
|
||||
# ``content`` never carries the ``<system-reminder>`` envelope —
|
||||
# that splice is transient, applied to a wire-bound copy in
|
||||
# ``ChatSession._apply_reminders_for_provider``.
|
||||
#
|
||||
# Verdict + output-assessment lookup tables — populated either
|
||||
# inline (sync call sites) or pre-loaded by an async caller via
|
||||
# asyncio.to_thread (see _load_verdict_indexes). Pre-loading is
|
||||
# what keeps _build_history off the event loop's hot path on the
|
||||
# SSE replay generator path.
|
||||
if verdicts is not None and assessments is not None:
|
||||
verdicts_by_call_id = verdicts
|
||||
assessments_by_call_id = assessments
|
||||
else:
|
||||
ws_id = getattr(session, "_ws_id", "") or ""
|
||||
verdicts_by_call_id, assessments_by_call_id = _load_verdict_indexes(ws_id)
|
||||
history = []
|
||||
for msg in session.messages:
|
||||
content = msg.get("content")
|
||||
@@ -392,18 +516,64 @@ def _build_history(
|
||||
entry = {"role": msg["role"], "content": content}
|
||||
if attachments_meta:
|
||||
entry["attachments"] = attachments_meta
|
||||
# Surface the ``_reminders`` side-channel so a tab reconnecting
|
||||
# via /history renders the same metacognitive nudge bubble the
|
||||
# originating tab saw live (user-channel reminders via
|
||||
# ``user_reminder`` SSE; tool-channel via ``tool_reminder``).
|
||||
# Reminders are in-memory only (not persisted to DB), so this
|
||||
# only fires for the originating session.
|
||||
reminders = msg.get("_reminders")
|
||||
if isinstance(reminders, list):
|
||||
# Filter first so an all-malformed _reminders doesn't set the
|
||||
# field to []; absent vs. empty-list should mean the same
|
||||
# thing on the wire.
|
||||
clean_reminders = [
|
||||
{"type": str(r.get("type") or ""), "text": str(r.get("text") or "")}
|
||||
for r in reminders
|
||||
if isinstance(r, dict)
|
||||
]
|
||||
if clean_reminders:
|
||||
entry["reminders"] = clean_reminders
|
||||
if msg.get("tool_calls"):
|
||||
entry["tool_calls"] = [
|
||||
{
|
||||
"id": tc.get("id", ""),
|
||||
tc_entries: list[dict[str, Any]] = []
|
||||
for tc in msg["tool_calls"]:
|
||||
tc_entry: dict[str, Any] = {
|
||||
"id": tc.get("id", "") or "",
|
||||
"name": tc["function"]["name"],
|
||||
"arguments": tc["function"].get("arguments", ""),
|
||||
}
|
||||
for tc in msg["tool_calls"]
|
||||
]
|
||||
# Decorate with persisted verdict + output_assessment
|
||||
# via the shared helper (also used by
|
||||
# ``make_history_handler``). Skips unflagged
|
||||
# ("risk_level == 'none'") rows so the wire stays
|
||||
# tight; ships only the fields the UI renders.
|
||||
_decorate_tool_call(
|
||||
tc_entry,
|
||||
verdicts_by_call_id,
|
||||
assessments_by_call_id,
|
||||
)
|
||||
tc_entries.append(tc_entry)
|
||||
entry["tool_calls"] = tc_entries
|
||||
# Detect denied/blocked/errored tool results by their content prefix.
|
||||
if msg.get("role") == "tool":
|
||||
content = msg.get("content", "")
|
||||
# Propagate tool_call_id so replayHistory can anchor the
|
||||
# rendered output to the specific .ts-approval-tool element
|
||||
# by data-call-id (mirrors the live appendToolOutput path).
|
||||
# Without this, multi-tool batches render every result at
|
||||
# the bottom of the block rather than under each header.
|
||||
result_call_id = msg.get("tool_call_id")
|
||||
if result_call_id:
|
||||
entry["tool_call_id"] = str(result_call_id)
|
||||
# Tool results are clamped to TOOL_RESULT_STORAGE_CAP
|
||||
# chars per row at storage time (session.py). Surface
|
||||
# that on replay so the user knows the visible output is
|
||||
# a clipped view of what the live session saw, rather
|
||||
# than the full result. Reference the shared constant
|
||||
# rather than a literal so the UI pill logic can't
|
||||
# silently desync if the cap ever changes.
|
||||
if isinstance(content, str) and len(content) >= TOOL_RESULT_STORAGE_CAP:
|
||||
entry["truncated"] = True
|
||||
if isinstance(content, str):
|
||||
if content.startswith("Denied by user") or content.startswith("Blocked"):
|
||||
entry["denied"] = True
|
||||
@@ -644,6 +814,31 @@ def _audit_close_workstream(
|
||||
)
|
||||
|
||||
|
||||
async def _interactive_events_replay_prepare(ws: Workstream, ui: Any, request: Request) -> None:
|
||||
"""Async pre-step run before ``_interactive_events_replay`` iterates.
|
||||
|
||||
Loads ``intent_verdicts`` + ``output_assessments`` for the
|
||||
workstream off the event loop (via ``asyncio.to_thread``) and
|
||||
stashes the result on ``request.state.verdict_indexes``. The sync
|
||||
replay generator reads from there and passes the dicts into
|
||||
``_build_history`` so the storage I/O never blocks the event loop
|
||||
on the SSE replay path.
|
||||
|
||||
Best-effort: if the workstream has no session or no ws_id, leaves
|
||||
``request.state.verdict_indexes`` unset and ``_build_history``
|
||||
falls back to the inline storage call (sync path).
|
||||
"""
|
||||
del ui # not needed; lookup is keyed on ws.session._ws_id
|
||||
session = ws.session
|
||||
if session is None:
|
||||
return
|
||||
ws_id = getattr(session, "_ws_id", "") or ""
|
||||
if not ws_id:
|
||||
return
|
||||
indexes = await asyncio.to_thread(_load_verdict_indexes, ws_id)
|
||||
request.state.verdict_indexes = indexes
|
||||
|
||||
|
||||
def _interactive_events_replay(
|
||||
ws: Workstream, ui: Any, request: Request
|
||||
) -> Iterable[dict[str, Any]]:
|
||||
@@ -661,7 +856,6 @@ def _interactive_events_replay(
|
||||
|
||||
Pure read — never mutates ``ws`` / ``ui`` / ``session``.
|
||||
"""
|
||||
del request # not needed; replay reads ws/ui/session state
|
||||
session = ws.session
|
||||
if session is None:
|
||||
# Defensive — the lifted body's UI presence check guarantees
|
||||
@@ -676,9 +870,22 @@ def _interactive_events_replay(
|
||||
|
||||
# History replay — pending-approval flag rides on the last
|
||||
# assistant entry's tool_calls so the client renders them as
|
||||
# awaiting approval rather than already approved.
|
||||
# awaiting approval rather than already approved. Verdict /
|
||||
# assessment indexes were pre-loaded off the event loop by
|
||||
# _interactive_events_replay_prepare; passing them in here keeps
|
||||
# _build_history's storage I/O out of the sync generator path.
|
||||
pending_approval = getattr(ui, "_pending_approval", None)
|
||||
history = _build_history(session, has_pending_approval=pending_approval is not None)
|
||||
cached_indexes = getattr(request.state, "verdict_indexes", None)
|
||||
if isinstance(cached_indexes, tuple) and len(cached_indexes) == 2:
|
||||
verdicts, assessments = cached_indexes
|
||||
else:
|
||||
verdicts, assessments = None, None
|
||||
history = _build_history(
|
||||
session,
|
||||
has_pending_approval=pending_approval is not None,
|
||||
verdicts=verdicts,
|
||||
assessments=assessments,
|
||||
)
|
||||
if history:
|
||||
yield {"type": "history", "messages": history}
|
||||
|
||||
@@ -823,6 +1030,16 @@ def _build_node_snapshot(app_state: Any) -> dict[str, Any]:
|
||||
title = ""
|
||||
if ws.session:
|
||||
title = get_workstream_display_name(ws.session.ws_id) or ""
|
||||
# ``pending_approval_detail`` mirrors the dashboard handler's
|
||||
# projection so the console collector's reconnect-via-snapshot
|
||||
# path (``_reconcile_node``) can carry the rich approval payload
|
||||
# across reconnects — without it, a child sitting in approval-
|
||||
# pending across a console restart or network blip would render
|
||||
# with no buttons until the next state change. Same data, same
|
||||
# ``read`` scope as ``/v1/api/dashboard``.
|
||||
approval_detail: dict[str, Any] | None = None
|
||||
if ui is not None and hasattr(ui, "serialize_pending_approval_detail"):
|
||||
approval_detail = ui.serialize_pending_approval_detail()
|
||||
ws_list.append(
|
||||
{
|
||||
"id": ws.id,
|
||||
@@ -839,6 +1056,7 @@ def _build_node_snapshot(app_state: Any) -> dict[str, Any]:
|
||||
"kind": ws.kind,
|
||||
"parent_ws_id": ws.parent_ws_id,
|
||||
"user_id": ws.user_id,
|
||||
"pending_approval_detail": approval_detail,
|
||||
}
|
||||
)
|
||||
return {
|
||||
@@ -1342,13 +1560,13 @@ async def command(request: Request) -> JSONResponse:
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
elif cmd_word == "/resume":
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
history = _build_history(ws.session)
|
||||
history = await asyncio.to_thread(_build_history, ws.session)
|
||||
if history:
|
||||
ui._enqueue({"type": "history", "messages": history})
|
||||
elif cmd_word in ("/rewind", "/retry"):
|
||||
# Refresh frontend with truncated history
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
history = _build_history(ws.session)
|
||||
history = await asyncio.to_thread(_build_history, ws.session)
|
||||
if history:
|
||||
ui._enqueue({"type": "history", "messages": history})
|
||||
# Audit trail
|
||||
@@ -1823,7 +2041,7 @@ async def _interactive_create_post_install(
|
||||
ui = ws.ui
|
||||
if isinstance(ui, WebUI):
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
history = _build_history(ws.session)
|
||||
history = await asyncio.to_thread(_build_history, ws.session)
|
||||
if history:
|
||||
ui._enqueue({"type": "history", "messages": history})
|
||||
with contextlib.suppress(queue.Full):
|
||||
@@ -3319,12 +3537,14 @@ def create_app(
|
||||
open_resolve_alias=_resolve_workstream_alias,
|
||||
open_post_load=_interactive_open_post_load,
|
||||
events_replay=_interactive_events_replay,
|
||||
events_replay_prepare=_interactive_events_replay_prepare,
|
||||
# Pre-lift ``events_sse`` used the dedicated 200-thread
|
||||
# ``sse_executor`` so SSE polling stayed isolated from
|
||||
# every other ``asyncio.to_thread`` caller in the process
|
||||
# (storage, router, audit). Restore that isolation under
|
||||
# the lifted contract — coord wires ``None`` and falls
|
||||
# back to the default executor.
|
||||
# the lifted contract. The console's coord endpoint wires
|
||||
# its own ``coord_sse_executor`` on the same lookup hook —
|
||||
# see ``turnstone/console/server.py``.
|
||||
sse_executor_lookup=lambda request: request.app.state.sse_executor,
|
||||
create_supports_attachments=True,
|
||||
create_supports_user_id_override=True,
|
||||
@@ -3544,6 +3764,11 @@ def main() -> None:
|
||||
default=8080,
|
||||
help="Port to listen on (default: 8080)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-permissions",
|
||||
action="store_true",
|
||||
help="Auto-approve all tool calls (no confirmation prompts)",
|
||||
)
|
||||
# MCP config path is bootstrap-critical (needed before ConfigStore for tool loading)
|
||||
parser.add_argument(
|
||||
"--mcp-config",
|
||||
@@ -3971,7 +4196,7 @@ def main() -> None:
|
||||
ws = manager.create(user_id="", name="resumed")
|
||||
if not isinstance(ws.ui, WebUI):
|
||||
raise TypeError(f"Expected WebUI, got {type(ws.ui).__name__}")
|
||||
if config_store.get("tools.skip_permissions"):
|
||||
if args.skip_permissions or config_store.get("tools.skip_permissions"):
|
||||
ws.ui.auto_approve = True
|
||||
assert ws.session is not None
|
||||
ws.session.set_watch_runner(
|
||||
@@ -4009,7 +4234,7 @@ def main() -> None:
|
||||
_advertise_host = args.host if args.host not in ("0.0.0.0", "::") else socket.gethostname()
|
||||
_advertise_url = f"http://{_advertise_host}:{args.port}"
|
||||
|
||||
_skip_perms = config_store.get("tools.skip_permissions")
|
||||
_skip_perms = args.skip_permissions or config_store.get("tools.skip_permissions")
|
||||
app = create_app(
|
||||
workstreams=manager,
|
||||
global_queue=global_queue,
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
console/static (Saved Coordinators). Single source of truth so the two
|
||||
surfaces don't drift on hover affordance, padding, or typography.
|
||||
==========================================================================
|
||||
Class names match the original ui/static rules they replaced; the
|
||||
delete-mode subset stays in ui/static/style.css until coordinator gets
|
||||
the same UX (then it can move here too).
|
||||
Class names match the original ui/static rules they replaced. Delete-mode
|
||||
selectors live here too so console (Saved Coordinators) and ui/static
|
||||
(Saved Workstreams) share one card + delete affordance.
|
||||
========================================================================== */
|
||||
|
||||
.dashboard-cards {
|
||||
@@ -76,3 +76,283 @@
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
/* ==========================================================================
|
||||
Delete UX — section-level "Delete" toggle, per-card checkboxes, bottom
|
||||
toolbar, and confirmation modal. Moved out of ui/static/style.css when
|
||||
the console grew the same multi-select delete on Saved Coordinators.
|
||||
========================================================================== */
|
||||
|
||||
.ws-delete-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
color 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
.ws-delete-btn:hover {
|
||||
color: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
|
||||
/* Delete mode */
|
||||
.dashboard-card.ws-delete-mode {
|
||||
cursor: pointer;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.04);
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected {
|
||||
cursor: default;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode:hover {
|
||||
background: rgba(220, 38, 38, 0.04);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-card-check {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: var(--red);
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
opacity: 0;
|
||||
animation: ws-check-fadein 0.2s ease-out forwards;
|
||||
}
|
||||
.ws-card-check:focus-visible {
|
||||
/* Card click-handler proxies space/enter to the checkbox, so focus
|
||||
usually rests on the card; if a screen reader / power user tabs
|
||||
directly onto the checkbox the UA outline can be killed by
|
||||
adjacent rules — this guarantees a visible affordance. */
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
@keyframes ws-check-fadein {
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
.dashboard-card.ws-selected {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-selected {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-delete-bar {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
display: flex;
|
||||
animation: ws-bar-slide 0.2s ease-out;
|
||||
}
|
||||
@keyframes ws-bar-slide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ws-card-check {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
/* Below ~700px the four-pill bar's preferred width (~480-520px) starts
|
||||
eating into hit-targets and risking horizontal overflow. Wrap onto
|
||||
two rows: count + Cancel + Select All on the first, the destructive
|
||||
Delete Selected on its own full-width row underneath — also a better
|
||||
thumb-target separation than the desktop layout. */
|
||||
@media (max-width: 700px) {
|
||||
.ws-delete-bar {
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: 0;
|
||||
flex: 1 1 100%;
|
||||
order: 99;
|
||||
}
|
||||
}
|
||||
.ws-delete-bar .ws-delete-count-label {
|
||||
font-size: 12px;
|
||||
color: var(--fg-dim);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: auto;
|
||||
/* Dark-theme --red (#f87171) on #fff is only 3.0:1 — below WCAG AA
|
||||
for normal text on a destructive button. Use the deeper red
|
||||
(#dc2626 → 4.85:1) on the filled state so the button label clears
|
||||
AA in the default theme. Light theme already uses --red (#b91c1c,
|
||||
5.9:1) and stays put — the override below pins it. */
|
||||
background: #dc2626;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 16px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-bar .ws-delete-bar-btn {
|
||||
background: var(--red);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:hover:not(:disabled) {
|
||||
filter: brightness(1.1);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn:hover {
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
|
||||
/* Delete modal — id-scoped so the surface that owns it controls visibility.
|
||||
Both ui/static (#ws-delete-overlay) and console/static
|
||||
(#coord-delete-overlay) share the same shape via the .ws-delete-modal
|
||||
class hooks below. */
|
||||
.ws-delete-modal-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
.ws-delete-modal-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 24px;
|
||||
max-width: 480px;
|
||||
width: 90%;
|
||||
}
|
||||
.ws-delete-modal-box h3 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 16px;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
.ws-delete-modal-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
margin: 12px 0;
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item {
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
color: var(--fg-bright);
|
||||
border-bottom: 1px solid var(--border);
|
||||
/* Long aliases / raw ws_ids in the confirm + results list shouldn't
|
||||
punch out of the modal at narrow viewports. */
|
||||
word-break: break-word;
|
||||
}
|
||||
/* Modal alert region — only painted when the controller writes a
|
||||
message. Both close paths clear it, so :not(:empty) keeps the box
|
||||
invisible at rest and avoids an empty-frame artefact. */
|
||||
.ws-delete-modal-box [role="alert"]:not(:empty) {
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
border: 1px solid var(--red);
|
||||
color: var(--red);
|
||||
border-radius: var(--radius);
|
||||
padding: 8px 10px;
|
||||
font-size: 12px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-modal-box [role="alert"]:not(:empty) {
|
||||
background: rgba(220, 38, 38, 0.06);
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item.ws-delete-error {
|
||||
color: var(--red);
|
||||
}
|
||||
.ws-delete-modal-buttons {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
.ws-delete-modal-buttons button {
|
||||
padding: 8px 20px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
.ws-delete-modal-buttons button.ws-delete-confirm {
|
||||
/* Mirror .ws-delete-bar-btn's contrast bump — same destructive
|
||||
filled-button treatment, same dark-theme AA fix. */
|
||||
background: #dc2626;
|
||||
color: #fff;
|
||||
border-color: #dc2626;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-modal-buttons button.ws-delete-confirm {
|
||||
background: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
/* `.ws-delete-close` is a state-marker, not a colour rule — the
|
||||
controller drops `.ws-delete-confirm` when the modal transitions to
|
||||
the post-delete "Close" state, and the default
|
||||
`.ws-delete-modal-buttons button` rule above already provides the
|
||||
transparent / fg-bright / border styling. The class itself is useful
|
||||
for DOM inspection and as a future hook. */
|
||||
|
||||
@@ -77,3 +77,433 @@ function renderSessionCard(sess, opts) {
|
||||
card.appendChild(metaEl);
|
||||
return card;
|
||||
}
|
||||
|
||||
/* createSavedCardsController — shared multi-select-delete behaviour for
|
||||
the dashboard / home "saved cards" surfaces. ui/static (Saved
|
||||
Workstreams) and console/static (Saved Coordinators) both instantiate
|
||||
one of these; the controller owns:
|
||||
|
||||
- delete-mode state (active flag + selected ws_id set)
|
||||
- card decoration (checkbox + key/click overrides)
|
||||
- the bottom toolbar wiring (count, Select All, Delete Selected)
|
||||
- the confirmation modal (focus trap, batch fan-out, results view)
|
||||
|
||||
It does NOT own how cards get fetched or rendered — the caller's
|
||||
render() is invoked when the controller needs the list redrawn (mode
|
||||
transitions, Select-All toggles).
|
||||
|
||||
Required opts:
|
||||
idPrefix — DOM-id prefix shared by the toolbar + modal
|
||||
(e.g. "ws-delete" / "coord-delete"). The DOM
|
||||
must already contain `${idPrefix}-bar`,
|
||||
`${idPrefix}-bar-count`, `${idPrefix}-bar-delete`,
|
||||
`${idPrefix}-bar-select-all`, `${idPrefix}-overlay`,
|
||||
`${idPrefix}-box`, `${idPrefix}-error`,
|
||||
`${idPrefix}-count`, `${idPrefix}-list`,
|
||||
`${idPrefix}-confirm-btn`, `${idPrefix}-cancel-btn`.
|
||||
buttonId — id of the section's start/cancel toggle button.
|
||||
noun — singular display word for the item kind, e.g.
|
||||
"workstream" / "coordinator". Used in toast +
|
||||
modal copy.
|
||||
activateLabel — sess => string; aria-label for the card when NOT
|
||||
in delete mode (e.g. "Resume: foo").
|
||||
buildDeleteRequest — wsId => { url, options }; what authFetch should
|
||||
send to delete one item.
|
||||
render — () => void; redraw the visible cards. Called by
|
||||
the controller on mode start/cancel and Select-
|
||||
All toggle. Caller is responsible for calling
|
||||
setItems(items) + decorateCard() inside it.
|
||||
onClose — optional () => void; called once after the user
|
||||
closes the post-delete results modal. Typical
|
||||
use: re-fetch the saved list.
|
||||
*/
|
||||
function createSavedCardsController(opts) {
|
||||
var state = { mode: false, selected: {}, items: [] };
|
||||
var batchTrap = null;
|
||||
/* Element that owned focus when the modal opened — restored in
|
||||
closeModal() so keyboard users land back on the toggle button (or
|
||||
wherever they came from) instead of <body>. WCAG 2.4.3. */
|
||||
var prevFocus = null;
|
||||
|
||||
function $(id) {
|
||||
return document.getElementById(opts.idPrefix + "-" + id);
|
||||
}
|
||||
|
||||
/* Replace the toggle button's content with a glyph + label, keeping
|
||||
the glyph in an aria-hidden span so screen readers only read the
|
||||
label. Built from DOM nodes (no innerHTML) — same shape as the
|
||||
section-header markup the JS replaces. */
|
||||
function setIconButton(btn, glyph, label) {
|
||||
btn.replaceChildren();
|
||||
var span = document.createElement("span");
|
||||
span.setAttribute("aria-hidden", "true");
|
||||
span.textContent = glyph;
|
||||
btn.appendChild(span);
|
||||
btn.appendChild(document.createTextNode(" " + label));
|
||||
}
|
||||
|
||||
function setItems(items) {
|
||||
state.items = items;
|
||||
/* Drop any selections whose ws_id is no longer on the visible page —
|
||||
SSE-driven re-renders or pagination jumps shouldn't leave ghost
|
||||
entries inflating the count and 404-ing on confirm. */
|
||||
if (state.mode) {
|
||||
var byId = {};
|
||||
items.forEach(function (s) {
|
||||
byId[s.ws_id] = true;
|
||||
});
|
||||
Object.keys(state.selected).forEach(function (id) {
|
||||
if (!byId[id]) delete state.selected[id];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function inMode() {
|
||||
return state.mode;
|
||||
}
|
||||
|
||||
function blockActivate() {
|
||||
return state.mode;
|
||||
}
|
||||
|
||||
function isSelected(wsId) {
|
||||
return !!state.selected[wsId];
|
||||
}
|
||||
|
||||
function ariaLabel(sess) {
|
||||
var label = sess.alias || sess.title || sess.name || sess.ws_id;
|
||||
if (state.mode) return "Select " + opts.noun + ": " + label;
|
||||
return typeof opts.activateLabel === "function"
|
||||
? opts.activateLabel(sess)
|
||||
: "Activate: " + label;
|
||||
}
|
||||
|
||||
/* Decorate an already-rendered .dashboard-card with the checkbox +
|
||||
event overrides used in delete mode. Idempotent guard: only acts
|
||||
when the controller is active. */
|
||||
function decorateCard(card, sess) {
|
||||
if (!state.mode) return;
|
||||
card.classList.add("ws-delete-mode");
|
||||
card.removeAttribute("role");
|
||||
var chk = document.createElement("input");
|
||||
chk.type = "checkbox";
|
||||
chk.className = "ws-card-check";
|
||||
chk.checked = !!state.selected[sess.ws_id];
|
||||
var label = sess.alias || sess.title || sess.name || sess.ws_id;
|
||||
chk.setAttribute("aria-label", "Select " + label + " for deletion");
|
||||
chk.onclick = function (e) {
|
||||
e.stopPropagation();
|
||||
if (chk.checked) state.selected[sess.ws_id] = true;
|
||||
else delete state.selected[sess.ws_id];
|
||||
card.classList.toggle("ws-selected", chk.checked);
|
||||
refreshBar();
|
||||
};
|
||||
card.insertBefore(chk, card.firstChild);
|
||||
card.onclick = function (e) {
|
||||
if (e.target === chk) return;
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
};
|
||||
card.onkeydown = function (e) {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
}
|
||||
};
|
||||
if (state.selected[sess.ws_id]) card.classList.add("ws-selected");
|
||||
}
|
||||
|
||||
function refreshBar() {
|
||||
var count = Object.keys(state.selected).length;
|
||||
var label = $("bar-count");
|
||||
if (label) label.textContent = count + " selected";
|
||||
var delBtn = $("bar-delete");
|
||||
if (delBtn) delBtn.disabled = count === 0;
|
||||
var selBtn = $("bar-select-all");
|
||||
if (selBtn) {
|
||||
var allSelected = count === state.items.length && state.items.length > 0;
|
||||
selBtn.textContent = allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (!state.items.length) {
|
||||
if (typeof showToast === "function") {
|
||||
showToast("No saved " + opts.noun + "s to delete");
|
||||
}
|
||||
return;
|
||||
}
|
||||
state.mode = true;
|
||||
state.selected = {};
|
||||
opts.render();
|
||||
var btn = document.getElementById(opts.buttonId);
|
||||
if (btn) {
|
||||
setIconButton(btn, "✕", "Cancel");
|
||||
btn.onclick = cancel;
|
||||
}
|
||||
var bar = $("bar");
|
||||
if (bar) bar.classList.add("visible");
|
||||
refreshBar();
|
||||
}
|
||||
|
||||
function cancel() {
|
||||
state.mode = false;
|
||||
state.selected = {};
|
||||
opts.render();
|
||||
var btn = document.getElementById(opts.buttonId);
|
||||
if (btn) {
|
||||
setIconButton(btn, "\u{1f5d1}", "Delete");
|
||||
btn.onclick = start;
|
||||
}
|
||||
var bar = $("bar");
|
||||
if (bar) bar.classList.remove("visible");
|
||||
}
|
||||
|
||||
function toggleAll() {
|
||||
var allSelected =
|
||||
Object.keys(state.selected).length === state.items.length &&
|
||||
state.items.length > 0;
|
||||
if (allSelected) {
|
||||
state.selected = {};
|
||||
} else {
|
||||
state.items.forEach(function (s) {
|
||||
state.selected[s.ws_id] = true;
|
||||
});
|
||||
}
|
||||
opts.render();
|
||||
refreshBar();
|
||||
}
|
||||
|
||||
function _byId() {
|
||||
/* Single-pass index over the visible items so the modal + fan-out
|
||||
paths don't repeat O(N) `find` calls per selection. */
|
||||
var map = {};
|
||||
state.items.forEach(function (s) {
|
||||
map[s.ws_id] = s;
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
function confirmSelection() {
|
||||
var selected = Object.keys(state.selected);
|
||||
if (!selected.length) {
|
||||
if (typeof showToast === "function") {
|
||||
showToast("No " + opts.noun + "s selected");
|
||||
}
|
||||
return;
|
||||
}
|
||||
var byId = _byId();
|
||||
var overlay = $("overlay");
|
||||
var countEl = $("count");
|
||||
var listEl = $("list");
|
||||
var errorEl = $("error");
|
||||
if (errorEl) errorEl.textContent = "";
|
||||
if (countEl) {
|
||||
countEl.textContent =
|
||||
selected.length + " " + opts.noun + "(s) will be permanently deleted:";
|
||||
}
|
||||
if (listEl) {
|
||||
listEl.replaceChildren();
|
||||
selected.forEach(function (wsId) {
|
||||
var item = byId[wsId];
|
||||
var name = item ? item.alias || item.title || item.name || wsId : wsId;
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item";
|
||||
div.textContent = name;
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
}
|
||||
var delBtn = $("confirm-btn");
|
||||
if (delBtn) {
|
||||
delBtn.textContent = "Delete";
|
||||
delBtn.disabled = false;
|
||||
delBtn.classList.remove("ws-delete-close");
|
||||
delBtn.classList.add("ws-delete-confirm");
|
||||
delBtn.onclick = confirm;
|
||||
}
|
||||
var cancelBtn = $("cancel-btn");
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
if (overlay) overlay.style.display = "flex";
|
||||
|
||||
if (batchTrap) document.removeEventListener("keydown", batchTrap);
|
||||
batchTrap = function (e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
closeModal();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Tab") {
|
||||
var box = $("box");
|
||||
if (!box) return;
|
||||
var focusable = box.querySelectorAll("button:not(:disabled)");
|
||||
if (!focusable.length) return;
|
||||
var first = focusable[0];
|
||||
var last = focusable[focusable.length - 1];
|
||||
if (e.shiftKey && document.activeElement === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && document.activeElement === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", batchTrap);
|
||||
/* Snapshot the pre-modal focus owner so closeModal() can return to
|
||||
it. Captured before we move focus into the dialog so the
|
||||
restore-target is the caller, not the dialog itself. */
|
||||
prevFocus = document.activeElement;
|
||||
if (cancelBtn) cancelBtn.focus();
|
||||
}
|
||||
|
||||
function closeModal() {
|
||||
var overlay = $("overlay");
|
||||
if (overlay) overlay.style.display = "none";
|
||||
if (batchTrap) {
|
||||
document.removeEventListener("keydown", batchTrap);
|
||||
batchTrap = null;
|
||||
}
|
||||
/* Pick the most useful focus target:
|
||||
1. prevFocus (where the user came from), if it's still in the
|
||||
DOM and visible. Esc / Cancel paths land here — the bar is
|
||||
still on screen, so focus returns to "Delete Selected".
|
||||
2. The section toggle button — always present, semantic exit
|
||||
point for the flow. Used when prevFocus has been hidden by
|
||||
cancel() (post-delete Close path: cancel() ran first and
|
||||
put `.ws-delete-bar` at display:none, so the bar's button
|
||||
is no longer focusable). */
|
||||
var target = prevFocus;
|
||||
if (!target || target.offsetParent === null) {
|
||||
target = document.getElementById(opts.buttonId);
|
||||
}
|
||||
if (target && typeof target.focus === "function") {
|
||||
try {
|
||||
target.focus();
|
||||
} catch (_) {
|
||||
/* node detached between open and close — give up silently */
|
||||
}
|
||||
}
|
||||
prevFocus = null;
|
||||
}
|
||||
|
||||
function confirm() {
|
||||
var selected = Object.keys(state.selected);
|
||||
if (!selected.length) return;
|
||||
var byId = _byId();
|
||||
var errorEl = $("error");
|
||||
var listEl = $("list");
|
||||
var countEl = $("count");
|
||||
var delBtn = $("confirm-btn");
|
||||
var cancelBtn = $("cancel-btn");
|
||||
if (errorEl) errorEl.textContent = "";
|
||||
if (delBtn) {
|
||||
delBtn.disabled = true;
|
||||
delBtn.textContent = "Deleting...";
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = true;
|
||||
|
||||
var results = [];
|
||||
var promises = selected.map(function (wsId) {
|
||||
var shortId = wsId.substring(0, 8);
|
||||
var item = byId[wsId];
|
||||
var name = item ? item.alias || item.title || item.name || wsId : wsId;
|
||||
var req = opts.buildDeleteRequest(wsId);
|
||||
return authFetch(req.url, req.options)
|
||||
.then(function (r) {
|
||||
var status = r.status;
|
||||
var contentType = r.headers.get("content-type") || "";
|
||||
if (r.ok) {
|
||||
results.push({ name: name, shortId: shortId, ok: true });
|
||||
return;
|
||||
}
|
||||
return r.text().then(function (body) {
|
||||
var errMsg = shortId + ": HTTP " + status;
|
||||
if (contentType.includes("json")) {
|
||||
try {
|
||||
var j = JSON.parse(body);
|
||||
if (j.error) errMsg = shortId + ": " + j.error;
|
||||
} catch (_) {
|
||||
/* fall through */
|
||||
}
|
||||
} else if (body) {
|
||||
errMsg = shortId + ": " + body.substring(0, 200);
|
||||
}
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: errMsg,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(function (err) {
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: shortId + ": " + err.message,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Promise.all(promises).then(function () {
|
||||
if (listEl) {
|
||||
listEl.replaceChildren();
|
||||
results.forEach(function (r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item" + (r.ok ? "" : " ws-delete-error");
|
||||
div.textContent =
|
||||
(r.ok ? "✓ " : "✗ ") + r.name + (r.error ? " — " + r.error : "");
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
}
|
||||
var okCount = results.filter(function (r) {
|
||||
return r.ok;
|
||||
}).length;
|
||||
var failCount = results.filter(function (r) {
|
||||
return !r.ok;
|
||||
}).length;
|
||||
if (countEl) {
|
||||
countEl.textContent = okCount + " deleted, " + failCount + " failed";
|
||||
}
|
||||
if (delBtn) {
|
||||
delBtn.disabled = false;
|
||||
delBtn.textContent = "Close";
|
||||
/* Swap modifier classes so styling is intent-driven instead of
|
||||
cascade-positional: the Close button picks up the default
|
||||
".ws-delete-modal-buttons button" rule once .ws-delete-confirm
|
||||
is removed. */
|
||||
delBtn.classList.remove("ws-delete-confirm");
|
||||
delBtn.classList.add("ws-delete-close");
|
||||
delBtn.onclick = function () {
|
||||
/* Order matters: cancel() reshapes the toggle button via
|
||||
setIconButton(), which preserves the element identity but
|
||||
swaps its subtree. closeModal() then focuses prevFocus —
|
||||
which IS that toggle button — landing on a freshly rebuilt
|
||||
"Delete" affordance instead of <body>. */
|
||||
cancel();
|
||||
closeModal();
|
||||
if (typeof opts.onClose === "function") opts.onClose();
|
||||
};
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
setItems: setItems,
|
||||
inMode: inMode,
|
||||
blockActivate: blockActivate,
|
||||
isSelected: isSelected,
|
||||
ariaLabel: ariaLabel,
|
||||
decorateCard: decorateCard,
|
||||
refreshBar: refreshBar,
|
||||
start: start,
|
||||
cancel: cancel,
|
||||
toggleAll: toggleAll,
|
||||
confirmSelection: confirmSelection,
|
||||
closeModal: closeModal,
|
||||
confirm: confirm,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -885,6 +885,27 @@
|
||||
.msg.user {
|
||||
border-left-color: var(--accent);
|
||||
}
|
||||
/* Metacognitive reminder — slotted directly below the message it
|
||||
advises (user message for correction/denial/etc., tool result for
|
||||
tool_error/repeat). Yellow accent reads as "advisory metadata"
|
||||
against the amber-ish user colour and the cyan tool cards;
|
||||
deliberately quieter than the surrounding bubbles so it doesn't
|
||||
compete for attention. Lives in the shared stylesheet so both
|
||||
the interactive UI and the console coord viewer render the same
|
||||
themed bubble. */
|
||||
.msg.user-reminder {
|
||||
border-left-color: var(--yellow);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 6px 10px;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
.msg.user-reminder .msg-user-reminder-label {
|
||||
color: var(--yellow);
|
||||
font-weight: 600;
|
||||
margin-right: 6px;
|
||||
text-transform: lowercase;
|
||||
}
|
||||
.msg.assistant {
|
||||
border-left-color: var(--hair-2);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "wait_for_workstream",
|
||||
"description": "Block until one or all named child workstreams reach a terminal state. Prefer this over busy-polling inspect_workstream after a fan-out: the tool absorbs the wait, so you get one call + one result regardless of duration. Returns `{results: {ws_id: {state, tokens, updated, message, truncated}}, elapsed, complete, mode}` — `results` is keyed by ws_id (not top-level), `complete` is true when the wait condition fired before the timeout. `message` is the child's last assistant text for `idle`/`error`, a short sentinel for `closed`/`denied`, and `null` for non-terminal rows so a follow-up read knows which children still need work; capped at 6 KiB UTF-8, with `truncated=true` when the cap fires (call inspect_workstream for the rest). Real terminal states: `idle`, `error`, `closed`, `deleted` (the last is unobservable since hard-delete cascades the row out of storage). `mode='any'` returns when the first child hits a real terminal — a `denied` id alone never satisfies the condition, so a typo'd / foreign / nonexistent id can't false-positive a wait. `mode='all'` returns once every id has settled (real terminal OR denied). Cross-tenant guard: only the coordinator's own children (or itself) are visible; everything else is reported `state='denied'`. Capped at 32 ws_ids and 600s; both overflows error rather than silently truncating.",
|
||||
"description": "Block until one or all named child workstreams reach a terminal state. Prefer this over busy-polling inspect_workstream after a fan-out: the tool absorbs the wait, so you get one call + one result regardless of duration. Returns `{results: {ws_id: {state, tokens, updated, message, truncated}}, elapsed, complete, mode}` — `results` is keyed by ws_id (not top-level), `complete` is true when the wait condition fired before the timeout. `message` is the child's last assistant text for `idle`/`error`, a short sentinel for `closed`/`denied`, and `null` for non-terminal rows so a follow-up read knows which children still need work; capped at 10 KiB UTF-8, with `truncated=true` when the cap fires (call inspect_workstream for the rest). Real terminal states: `idle`, `error`, `closed`, `deleted` (the last is unobservable since hard-delete cascades the row out of storage). `mode='any'` returns when the first child hits a real terminal — a `denied` id alone never satisfies the condition, so a typo'd / foreign / nonexistent id can't false-positive a wait. `mode='all'` returns once every id has settled (real terminal OR denied). Cross-tenant guard: only the coordinator's own children (or itself) are visible; everything else is reported `state='denied'`. Capped at 32 ws_ids and 600s; both overflows error rather than silently truncating.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
+404
-304
@@ -557,6 +557,44 @@ Pane.prototype.handleEvent = function (evt) {
|
||||
this.addErrorMessage(evt.message);
|
||||
break;
|
||||
|
||||
case "user_reminder":
|
||||
// Metacognitive nudges — render as their own bubble below the
|
||||
// user message they advise (semantically: a hint to the model
|
||||
// right before its turn). The originating tab's optimistic
|
||||
// addUserMessage already ran when the user clicked send, so by
|
||||
// the time this SSE event arrives the just-sent user bubble is
|
||||
// at the bottom of messagesEl and addUserReminder's "anchor to
|
||||
// most recent .msg.user" lookup finds it correctly; the
|
||||
// insertAdjacentElement('afterend', el) call drops the bubble
|
||||
// immediately below.
|
||||
//
|
||||
// Multi-tab caveat: the server emits no user_message SSE event
|
||||
// today, so a non-originating tab open on the same workstream
|
||||
// sees the reminder without a paired user-message render — the
|
||||
// anchor falls on a stale prior user bubble, mis-positioning
|
||||
// the reminder. The next /history reload corrects it (the
|
||||
// entry["reminders"] propagation in _build_history is
|
||||
// anchor-stable because replayHistory runs addUserMessage first
|
||||
// for every turn). Acceptable cost for stage 1; closing the
|
||||
// gap is a follow-up that adds a user_message SSE event.
|
||||
if (Array.isArray(evt.reminders) && evt.reminders.length) {
|
||||
this.addUserReminder(evt.reminders);
|
||||
}
|
||||
break;
|
||||
|
||||
case "tool_reminder":
|
||||
// Metacognitive tool-channel nudge (tool_error / repeat) —
|
||||
// render as the same yellow themed bubble used for user-channel
|
||||
// reminders, anchored below the .ts-approval block whose tool
|
||||
// result triggered the batch's reminder. evt.tool_call_id
|
||||
// identifies the specific tool element; addToolReminder walks
|
||||
// up to its parent approval block and inserts the bubble
|
||||
// immediately after.
|
||||
if (Array.isArray(evt.reminders) && evt.reminders.length) {
|
||||
this.addToolReminder(evt.reminders, evt.tool_call_id || "");
|
||||
}
|
||||
break;
|
||||
|
||||
case "message_queued":
|
||||
// Confirmation from server that a queued message was accepted.
|
||||
// The UI already showed the message optimistically in addQueuedMessage.
|
||||
@@ -669,6 +707,97 @@ Pane.prototype.removeThinkingIndicator = function () {
|
||||
if (el) el.remove();
|
||||
};
|
||||
|
||||
Pane.prototype.addUserReminder = function (reminders) {
|
||||
// Render each metacognitive reminder as its own bubble immediately
|
||||
// BELOW the user message it advises — semantically the reminder is
|
||||
// a hint to the model right before the assistant turn. Always
|
||||
// called AFTER the corresponding addUserMessage (live: optimistic
|
||||
// local render ran before the SSE event arrived; replay:
|
||||
// replayHistory renders the user message first), so "most recent
|
||||
// .msg.user" is always THIS turn's bubble — insertAdjacentElement
|
||||
// afterend drops the reminder directly below it. When no .msg.user
|
||||
// exists at all (e.g. a non-originating tab receiving a reminder
|
||||
// before any user turn has rendered) we append; the next /history
|
||||
// reload corrects any anchor anomaly.
|
||||
this.removeEmptyState();
|
||||
var userBubbles = this.messagesEl.querySelectorAll(".msg.user");
|
||||
var anchor = userBubbles.length ? userBubbles[userBubbles.length - 1] : null;
|
||||
for (var i = 0; i < reminders.length; i++) {
|
||||
var r = reminders[i] || {};
|
||||
var el = document.createElement("div");
|
||||
el.className = "msg user-reminder";
|
||||
var labelEl = document.createElement("span");
|
||||
labelEl.className = "msg-user-reminder-label";
|
||||
labelEl.textContent =
|
||||
"metacognition" + (r.type ? " · " + String(r.type) : "");
|
||||
var textEl = document.createElement("span");
|
||||
textEl.className = "msg-user-reminder-text";
|
||||
textEl.textContent = r.text || "";
|
||||
el.appendChild(labelEl);
|
||||
el.appendChild(textEl);
|
||||
if (anchor) {
|
||||
anchor.insertAdjacentElement("afterend", el);
|
||||
// Anchor advances so multiple reminders stack below the user
|
||||
// message in queued order (rather than each landing
|
||||
// immediately-after the user msg, which would reverse them).
|
||||
anchor = el;
|
||||
} else {
|
||||
this.messagesEl.appendChild(el);
|
||||
}
|
||||
}
|
||||
this.scrollToBottom(true);
|
||||
};
|
||||
|
||||
Pane.prototype.addToolReminder = function (reminders, toolCallId) {
|
||||
// Render each metacognitive tool-channel reminder (tool_error /
|
||||
// repeat) as the same yellow themed bubble used for user-channel
|
||||
// reminders, anchored below the .ts-approval block that produced
|
||||
// the tool result. toolCallId is the live-path anchor (SSE event
|
||||
// carries it); during replay it's an empty string and we fall back
|
||||
// to "last .ts-approval block in messagesEl", which is correct
|
||||
// because messages render in order — the assistant block carrying
|
||||
// the tool batch is always the most recent approval block by the
|
||||
// time we hit the tool message that owns the reminder.
|
||||
this.removeEmptyState();
|
||||
var anchor = null;
|
||||
if (toolCallId) {
|
||||
var escapedId = CSS.escape(toolCallId);
|
||||
var toolEl = this.messagesEl.querySelector(
|
||||
'.ts-approval-tool[data-call-id="' + escapedId + '"]',
|
||||
);
|
||||
if (toolEl) {
|
||||
anchor = toolEl.closest(".ts-approval");
|
||||
}
|
||||
}
|
||||
if (!anchor) {
|
||||
var blocks = this.messagesEl.querySelectorAll(".ts-approval");
|
||||
if (blocks.length) anchor = blocks[blocks.length - 1];
|
||||
}
|
||||
for (var i = 0; i < reminders.length; i++) {
|
||||
var r = reminders[i] || {};
|
||||
var el = document.createElement("div");
|
||||
// Same .msg.user-reminder class — visual treatment is shared
|
||||
// across user and tool channels (both are metacog nudges).
|
||||
el.className = "msg user-reminder";
|
||||
var labelEl = document.createElement("span");
|
||||
labelEl.className = "msg-user-reminder-label";
|
||||
labelEl.textContent =
|
||||
"metacognition" + (r.type ? " · " + String(r.type) : "");
|
||||
var textEl = document.createElement("span");
|
||||
textEl.className = "msg-user-reminder-text";
|
||||
textEl.textContent = r.text || "";
|
||||
el.appendChild(labelEl);
|
||||
el.appendChild(textEl);
|
||||
if (anchor) {
|
||||
anchor.insertAdjacentElement("afterend", el);
|
||||
anchor = el;
|
||||
} else {
|
||||
this.messagesEl.appendChild(el);
|
||||
}
|
||||
}
|
||||
this.scrollToBottom(true);
|
||||
};
|
||||
|
||||
Pane.prototype.addUserMessage = function (text, attachments) {
|
||||
this.removeEmptyState();
|
||||
var el = document.createElement("div");
|
||||
@@ -907,13 +1036,54 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
this.showEmptyState();
|
||||
return;
|
||||
}
|
||||
// Suppress the polite live region while we batch-build the replay
|
||||
// — messagesEl is aria-live="polite" so a fresh replay would otherwise
|
||||
// queue an announcement for every approved/denied/verdict pill we
|
||||
// insert. Restored after the loop so live SSE updates announce
|
||||
// normally. WCAG 4.1.3 — historical content should not behave like
|
||||
// real-time updates.
|
||||
this.messagesEl.setAttribute("aria-busy", "true");
|
||||
// pendingAssessments[call_id] = output_assessment dict. Populated
|
||||
// from the assistant branch, consumed by the role==="tool" branch
|
||||
// (or after the loop, for legacy rows missing tool_call_id).
|
||||
// Replaces a JSON.stringify→dataset→JSON.parse round-trip with an
|
||||
// in-memory map keyed by call_id.
|
||||
var pendingAssessments = {};
|
||||
var lastToolBlock = null;
|
||||
for (var i = 0; i < messages.length; i++) {
|
||||
var msg = messages[i];
|
||||
if (msg.role === "user") {
|
||||
// addUserMessage first so addUserReminder's "anchor to most
|
||||
// recent .msg.user" lookup finds THIS message's bubble (not the
|
||||
// previous user message's, which would associate the reminder
|
||||
// with the wrong turn). addUserReminder then drops the bubble
|
||||
// immediately below the just-rendered user message via
|
||||
// insertAdjacentElement('afterend', el).
|
||||
this.addUserMessage(msg.content || "", msg.attachments || null);
|
||||
if (Array.isArray(msg.reminders) && msg.reminders.length) {
|
||||
this.addUserReminder(msg.reminders);
|
||||
}
|
||||
lastToolBlock = null;
|
||||
} else if (msg.role === "assistant") {
|
||||
// Render content BEFORE the tool block so the visual order
|
||||
// matches the live SSE flow (stream_text streams content first,
|
||||
// then tool_info / approve_request paints the tool block, then
|
||||
// tool_result fills it in). Order also matters structurally:
|
||||
// the tool-result message in the NEXT iteration anchors via
|
||||
// lastToolBlock, which the tool-block branch sets last — so
|
||||
// content must run first to avoid clobbering that anchor.
|
||||
if (msg.content) {
|
||||
var el = document.createElement("div");
|
||||
el.className = "msg assistant";
|
||||
var bodyEl = document.createElement("div");
|
||||
bodyEl.className = "msg-body";
|
||||
var rendered = renderMarkdown(msg.content);
|
||||
bodyEl.innerHTML = rendered;
|
||||
el.appendChild(bodyEl);
|
||||
postRenderMarkdown(el);
|
||||
self.messagesEl.appendChild(el);
|
||||
lastToolBlock = null;
|
||||
}
|
||||
if (msg.tool_calls && msg.tool_calls.length) {
|
||||
if (msg.pending) {
|
||||
lastToolBlock = null;
|
||||
@@ -958,7 +1128,35 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
cmd.textContent = tc.arguments.substring(0, 100);
|
||||
}
|
||||
div.appendChild(cmd);
|
||||
// Verdict badge — anchor to THIS tool's row (div) rather
|
||||
// than the whole block, so a multi-tool batch with one
|
||||
// flagged call doesn't drift the badge above unrelated
|
||||
// calls. Same renderVerdictBadge helper as live; pass
|
||||
// judgePending=false because any verdict on replay is
|
||||
// final — no spinner.
|
||||
if (tc.verdict) {
|
||||
div.insertAdjacentHTML(
|
||||
"beforeend",
|
||||
renderVerdictBadge(tc.verdict, false),
|
||||
);
|
||||
}
|
||||
block.appendChild(div);
|
||||
// Output-guard finding — defer insertion until the tool
|
||||
// result lands so the warning anchors under the output
|
||||
// (mirrors live showOutputWarning placement). Stash in
|
||||
// a function-local map keyed by call_id so the
|
||||
// role==="tool" branch below can pick it up; legacy rows
|
||||
// missing tool_call_id are flushed at end-of-replay.
|
||||
if (
|
||||
tc.output_assessment &&
|
||||
tc.output_assessment.risk_level &&
|
||||
tc.output_assessment.risk_level !== "none"
|
||||
) {
|
||||
pendingAssessments[tc.id || ""] = {
|
||||
assessment: tc.output_assessment,
|
||||
toolDiv: div,
|
||||
};
|
||||
}
|
||||
});
|
||||
var badge = document.createElement("div");
|
||||
badge.setAttribute("role", "status");
|
||||
@@ -974,18 +1172,6 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
lastToolBlock = block;
|
||||
}
|
||||
}
|
||||
if (msg.content) {
|
||||
var el = document.createElement("div");
|
||||
el.className = "msg assistant";
|
||||
var bodyEl = document.createElement("div");
|
||||
bodyEl.className = "msg-body";
|
||||
var rendered = renderMarkdown(msg.content);
|
||||
bodyEl.innerHTML = rendered;
|
||||
el.appendChild(bodyEl);
|
||||
postRenderMarkdown(el);
|
||||
self.messagesEl.appendChild(el);
|
||||
lastToolBlock = null;
|
||||
}
|
||||
} else if (msg.role === "tool") {
|
||||
if (lastToolBlock) {
|
||||
var stripped = stripAnsi(msg.content || "").trim();
|
||||
@@ -994,34 +1180,156 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
/^Denied by user/.test(stripped) ||
|
||||
/^Blocked/.test(stripped);
|
||||
var isToolError = !!msg.is_error;
|
||||
// Anchor the rendered output to the specific .ts-approval-tool
|
||||
// element matching this result's tool_call_id — mirrors the
|
||||
// live appendToolOutput path so multi-tool batches show
|
||||
// [hdr A][out A][hdr B][out B] rather than [A][B][out A][out B].
|
||||
// Falls back to "before badge" when tool_call_id is absent
|
||||
// (legacy rows pre-dating the wire-format addition).
|
||||
var resultTarget = null;
|
||||
if (msg.tool_call_id) {
|
||||
resultTarget = lastToolBlock.querySelector(
|
||||
'.ts-approval-tool[data-call-id="' +
|
||||
CSS.escape(msg.tool_call_id) +
|
||||
'"]',
|
||||
);
|
||||
}
|
||||
// Cursor-style append: cursor advances after each insert so
|
||||
// the next sibling lands AFTER the previous one. Fixes the
|
||||
// bug where calling resultTarget.after(node) twice put the
|
||||
// second node BETWEEN resultTarget and the first (the second
|
||||
// .after call was always relative to the same anchor).
|
||||
// Resulting order with all three present:
|
||||
// [tool div][output][truncation pill][output-warning]
|
||||
var insertCursor = resultTarget;
|
||||
var insertChained = function (node) {
|
||||
if (insertCursor) {
|
||||
insertCursor.after(node);
|
||||
insertCursor = node;
|
||||
} else {
|
||||
var bdg = lastToolBlock.querySelector(".ts-approval-badge");
|
||||
if (bdg) lastToolBlock.insertBefore(node, bdg);
|
||||
else lastToolBlock.appendChild(node);
|
||||
}
|
||||
};
|
||||
if (stripped && !isDenied) {
|
||||
var media = !isToolError ? tryParseMedia(stripped) : null;
|
||||
if (media) {
|
||||
var embed = buildMediaEmbed(media, stripped);
|
||||
var bdg = lastToolBlock.querySelector(".ts-approval-badge");
|
||||
if (bdg) lastToolBlock.insertBefore(embed, bdg);
|
||||
else lastToolBlock.appendChild(embed);
|
||||
insertChained(buildMediaEmbed(media, stripped));
|
||||
} else {
|
||||
var out = renderToolOutput(stripped, isToolError);
|
||||
if (out.textContent.split("\n").length > 10) {
|
||||
makeCollapsible(out);
|
||||
}
|
||||
var bdg = lastToolBlock.querySelector(".ts-approval-badge");
|
||||
if (bdg) lastToolBlock.insertBefore(out, bdg);
|
||||
else lastToolBlock.appendChild(out);
|
||||
insertChained(out);
|
||||
}
|
||||
// Truncation pill — server marks this when the stored row
|
||||
// hit the 2000-char cap. Live tool_result events carry full
|
||||
// output so they don't need the indicator.
|
||||
if (msg.truncated) {
|
||||
var pill = document.createElement("span");
|
||||
pill.className = "tool-output-truncated";
|
||||
pill.textContent = "… truncated in storage";
|
||||
pill.title =
|
||||
"The full tool output was sent to the model live; only the first 10000 characters are persisted to the conversation row.";
|
||||
insertChained(pill);
|
||||
}
|
||||
}
|
||||
if (isToolError && !lastToolBlock.classList.contains("denied")) {
|
||||
lastToolBlock.classList.add("error");
|
||||
appendToolErrorBadge(lastToolBlock);
|
||||
}
|
||||
// Output-guard warning — pull the assessment out of the
|
||||
// function-local pendingAssessments map (populated in the
|
||||
// assistant branch). Skip when the tool result was denied —
|
||||
// the ✗ denied badge already signals the deny path.
|
||||
if (!isDenied && msg.tool_call_id) {
|
||||
var pending = pendingAssessments[msg.tool_call_id];
|
||||
if (pending) {
|
||||
insertChained(_buildOutputWarningEl(pending.assessment));
|
||||
delete pendingAssessments[msg.tool_call_id];
|
||||
}
|
||||
}
|
||||
}
|
||||
// Tool-channel metacog reminders (tool_error / repeat) attach
|
||||
// to the LAST tool message in a batch; on replay we render the
|
||||
// bubble immediately below the .ts-approval block that owns
|
||||
// the tool result. addToolReminder's empty-toolCallId fallback
|
||||
// resolves to "last .ts-approval block" — which is exactly
|
||||
// lastToolBlock here.
|
||||
if (Array.isArray(msg.reminders) && msg.reminders.length) {
|
||||
this.addToolReminder(msg.reminders, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
// Flush any output_assessments left in the map — these correspond
|
||||
// to assistant tool_calls whose tool result row didn't carry a
|
||||
// tool_call_id (legacy / migrated rows pre-dating the wire-format
|
||||
// addition). Render the warning under the tool div itself rather
|
||||
// than dropping the safety information silently.
|
||||
var leftoverIds = Object.keys(pendingAssessments);
|
||||
for (var p = 0; p < leftoverIds.length; p++) {
|
||||
var leftover = pendingAssessments[leftoverIds[p]];
|
||||
if (!leftover) continue;
|
||||
leftover.toolDiv.insertAdjacentElement(
|
||||
"afterend",
|
||||
_buildOutputWarningEl(leftover.assessment),
|
||||
);
|
||||
}
|
||||
this._attachRetryToLastAssistant();
|
||||
this.scrollToBottom();
|
||||
// Focus the input so keyboard users land on the next-action target
|
||||
// after replay finishes — but only when this is the focused pane,
|
||||
// there's no pending approval competing for focus, and an input
|
||||
// element actually exists. Skipping when not the focused pane
|
||||
// avoids stealing focus from another tab the user is interacting
|
||||
// with while a background replay completes.
|
||||
if (
|
||||
this.id === focusedPaneId &&
|
||||
!this.pendingApproval &&
|
||||
this.inputEl &&
|
||||
!this.busy
|
||||
) {
|
||||
try {
|
||||
this.inputEl.focus({ preventScroll: true });
|
||||
} catch (_) {
|
||||
this.inputEl.focus();
|
||||
}
|
||||
}
|
||||
// Restore live-region semantics now that the batch build is done.
|
||||
this.messagesEl.removeAttribute("aria-busy");
|
||||
};
|
||||
|
||||
// Shared output-warning DOM builder — used by both replayHistory
|
||||
// (saved-workstream rendering) and the live appendToolOutput path
|
||||
// via showOutputWarning. Single source of truth keeps the two
|
||||
// surfaces from drifting on role / class / escape semantics.
|
||||
function _buildOutputWarningEl(assessment) {
|
||||
var risk = (assessment && assessment.risk_level) || "medium";
|
||||
var flags = (assessment && assessment.flags) || [];
|
||||
var warning = document.createElement("div");
|
||||
warning.className = "output-warning output-warning-" + risk;
|
||||
// role="status" (polite) rather than "alert" (assertive) — these
|
||||
// are findings, not emergencies; the assertive announcement live
|
||||
// would interrupt the user mid-typing on a high-risk match, which
|
||||
// is more disruptive than informative.
|
||||
warning.setAttribute("role", "status");
|
||||
var labelEl = document.createElement("span");
|
||||
labelEl.className = "output-warning-label";
|
||||
labelEl.textContent = "⚠ " + String(risk).toUpperCase();
|
||||
warning.appendChild(labelEl);
|
||||
if (flags.length) {
|
||||
warning.appendChild(document.createTextNode(" " + flags.join(", ")));
|
||||
}
|
||||
if (assessment && assessment.redacted) {
|
||||
var redacted = document.createElement("span");
|
||||
redacted.className = "output-warning-redacted";
|
||||
redacted.textContent = " (credentials redacted)";
|
||||
warning.appendChild(redacted);
|
||||
}
|
||||
return warning;
|
||||
}
|
||||
|
||||
Pane.prototype._attachRetryToLastAssistant = function () {
|
||||
// Remove any previous retry buttons
|
||||
var old = this.messagesEl.querySelectorAll(".msg.assistant .msg-actions");
|
||||
@@ -1029,6 +1337,21 @@ Pane.prototype._attachRetryToLastAssistant = function () {
|
||||
// Find the last assistant message with content and add retry.
|
||||
// Reasoning blocks emit as .msg.reasoning (distinct modifier) so the
|
||||
// .msg.assistant selector already excludes them — no extra guard needed.
|
||||
//
|
||||
// Skip retry attachment when the most recent semantic turn is
|
||||
// tool-only — last DOM child is a .ts-approval block. Walk back
|
||||
// past .user-reminder bubbles (added via addToolReminder /
|
||||
// addUserReminder AFTER the .ts-approval block they advise) so the
|
||||
// guard fires correctly even when the tool turn carried a metacog
|
||||
// reminder. Without this skip, retry lands on a stale prior
|
||||
// assistant content bubble belonging to an earlier turn.
|
||||
var lastChild = this.messagesEl.lastElementChild;
|
||||
while (lastChild && lastChild.classList.contains("user-reminder")) {
|
||||
lastChild = lastChild.previousElementSibling;
|
||||
}
|
||||
if (lastChild && lastChild.classList.contains("ts-approval")) {
|
||||
return;
|
||||
}
|
||||
var assistants = this.messagesEl.querySelectorAll(".msg.assistant");
|
||||
if (assistants.length) {
|
||||
this._addRetryAction(assistants[assistants.length - 1]);
|
||||
@@ -1318,6 +1641,19 @@ Pane.prototype.appendToolOutput = function (callId, name, output, isError) {
|
||||
var stripped = stripAnsi(output || "").trim();
|
||||
if (!stripped) return;
|
||||
|
||||
// Skip rendering for denied/blocked tool results — the ✗ denied
|
||||
// badge from resolveApproval already shows the denial reason; the
|
||||
// SSE tool_result event would otherwise duplicate the text. Mirror
|
||||
// the guard in the history-replay path (the live path used to be
|
||||
// safe because no tool_result event was ever emitted for denied
|
||||
// items, but we now emit one so _tool_error_flags gets set).
|
||||
var parentBlock = target.closest(".ts-approval");
|
||||
var isDenied =
|
||||
(parentBlock && parentBlock.classList.contains("denied")) ||
|
||||
/^Denied by user/.test(stripped) ||
|
||||
/^Blocked/.test(stripped);
|
||||
if (isDenied) return;
|
||||
|
||||
// Detect structured media output and render interactive embed
|
||||
if (!isError) {
|
||||
var media = tryParseMedia(stripped);
|
||||
@@ -1332,12 +1668,9 @@ Pane.prototype.appendToolOutput = function (callId, name, output, isError) {
|
||||
var out = renderToolOutput(stripped, isError);
|
||||
|
||||
// Mark the parent approval block as errored
|
||||
if (isError) {
|
||||
var parentBlock = target.closest(".ts-approval");
|
||||
if (parentBlock && !parentBlock.classList.contains("denied")) {
|
||||
parentBlock.classList.add("error");
|
||||
appendToolErrorBadge(parentBlock);
|
||||
}
|
||||
if (isError && parentBlock && !parentBlock.classList.contains("denied")) {
|
||||
parentBlock.classList.add("error");
|
||||
appendToolErrorBadge(parentBlock);
|
||||
}
|
||||
|
||||
if (out.textContent.split("\n").length > 10) {
|
||||
@@ -1355,20 +1688,14 @@ Pane.prototype.showOutputWarning = function (evt) {
|
||||
'.ts-approval-tool[data-call-id="' + escapedId + '"]',
|
||||
);
|
||||
if (!toolDiv) return;
|
||||
var risk = evt.risk_level || "medium";
|
||||
var flags = evt.flags || [];
|
||||
var warning = document.createElement("div");
|
||||
warning.className = "output-warning output-warning-" + risk;
|
||||
warning.setAttribute("role", "alert");
|
||||
warning.innerHTML =
|
||||
'<span class="output-warning-label">\u26a0 ' +
|
||||
escapeHtml(risk.toUpperCase()) +
|
||||
"</span> " +
|
||||
flags.map(escapeHtml).join(", ");
|
||||
if (evt.redacted) {
|
||||
warning.innerHTML +=
|
||||
' <span class="output-warning-redacted">(credentials redacted)</span>';
|
||||
}
|
||||
// Shared DOM-builder with replayHistory \u2014 single source of truth for
|
||||
// role / class / escape semantics. Argument shape mirrors the
|
||||
// server-side output_assessment dict (risk_level / flags / redacted).
|
||||
var warning = _buildOutputWarningEl({
|
||||
risk_level: evt.risk_level,
|
||||
flags: evt.flags,
|
||||
redacted: evt.redacted,
|
||||
});
|
||||
var nextEl = toolDiv.nextElementSibling;
|
||||
if (nextEl && nextEl.classList.contains("tool-output")) {
|
||||
nextEl.insertAdjacentElement("afterend", warning);
|
||||
@@ -3511,12 +3838,35 @@ function updateDashFooter(agg) {
|
||||
}
|
||||
}
|
||||
|
||||
var _wsDeleteMode = false;
|
||||
var _wsDeleteSelected = {};
|
||||
// Saved Workstreams cache + multi-select delete controller. The
|
||||
// controller (from /shared/cards.js) owns mode state, checkbox
|
||||
// decoration, the toolbar wiring, and the confirmation modal — see
|
||||
// createSavedCardsController for the shared bits.
|
||||
var _wsSavedItems = [];
|
||||
var _wsDeleteController = createSavedCardsController({
|
||||
idPrefix: "ws-delete",
|
||||
buttonId: "ws-delete-btn",
|
||||
noun: "workstream",
|
||||
activateLabel: function (s) {
|
||||
return "Resume: " + (s.alias || s.title || s.ws_id);
|
||||
},
|
||||
buildDeleteRequest: function (wsId) {
|
||||
return {
|
||||
url: "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/delete",
|
||||
options: { method: "POST" },
|
||||
};
|
||||
},
|
||||
render: function () {
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
},
|
||||
onClose: function () {
|
||||
loadDashboard();
|
||||
},
|
||||
});
|
||||
|
||||
function renderSavedWorkstreams(items) {
|
||||
_wsSavedItems = items;
|
||||
_wsDeleteController.setItems(items);
|
||||
var c = document.getElementById("dashboard-saved-cards");
|
||||
c.replaceChildren();
|
||||
if (!items.length) {
|
||||
@@ -3527,289 +3877,39 @@ function renderSavedWorkstreams(items) {
|
||||
return;
|
||||
}
|
||||
items.forEach(function (sess) {
|
||||
// Default card shape (title + meta + wsid + Resume click) comes from
|
||||
// the shared /shared/cards.js helper so console (Saved Coordinators)
|
||||
// and ui/static (Saved Workstreams) stay in lock-step. Delete mode
|
||||
// is interactive-only; we layer the checkbox + selection wiring on
|
||||
// top of the shared card after construction.
|
||||
var card = renderSessionCard(sess, {
|
||||
ariaLabel: function (s) {
|
||||
var label = s.alias || s.title || s.ws_id;
|
||||
return _wsDeleteMode ? "Select: " + label : "Resume: " + label;
|
||||
},
|
||||
ariaLabel: _wsDeleteController.ariaLabel,
|
||||
onActivate: function (s) {
|
||||
// Suppressed in delete mode \u2014 the layered checkbox handler below
|
||||
// owns clicks while delete-mode is active.
|
||||
if (_wsDeleteMode) return;
|
||||
if (_wsDeleteController.blockActivate()) return;
|
||||
dashboardResumeSession(s.ws_id);
|
||||
},
|
||||
});
|
||||
|
||||
if (_wsDeleteMode) {
|
||||
card.classList.add("ws-delete-mode");
|
||||
card.removeAttribute("role"); // becomes a checkbox host, not a button
|
||||
var chk = document.createElement("input");
|
||||
chk.type = "checkbox";
|
||||
chk.className = "ws-card-check";
|
||||
chk.checked = !!_wsDeleteSelected[sess.ws_id];
|
||||
var label = sess.alias || sess.title || sess.ws_id;
|
||||
chk.setAttribute("aria-label", "Select " + label + " for deletion");
|
||||
chk.onclick = function (e) {
|
||||
e.stopPropagation();
|
||||
if (chk.checked) _wsDeleteSelected[sess.ws_id] = true;
|
||||
else delete _wsDeleteSelected[sess.ws_id];
|
||||
card.classList.toggle("ws-selected", chk.checked);
|
||||
updateWsDeleteBar();
|
||||
};
|
||||
card.insertBefore(chk, card.firstChild);
|
||||
// Override the shared helper's onclick/onkeydown \u2014 in delete mode
|
||||
// a card click toggles the checkbox instead of activating Resume.
|
||||
card.onclick = function (e) {
|
||||
if (e.target === chk) return;
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
};
|
||||
card.onkeydown = function (e) {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
}
|
||||
};
|
||||
if (_wsDeleteSelected[sess.ws_id]) card.classList.add("ws-selected");
|
||||
}
|
||||
|
||||
_wsDeleteController.decorateCard(card, sess);
|
||||
c.appendChild(card);
|
||||
});
|
||||
if (_wsDeleteController.inMode()) _wsDeleteController.refreshBar();
|
||||
}
|
||||
|
||||
// HTML inline-onclick wrappers — keep the global names the existing
|
||||
// markup binds to (`onclick="startWsDeleteMode()"` etc.) and forward
|
||||
// to the controller.
|
||||
function startWsDeleteMode() {
|
||||
_wsDeleteMode = true;
|
||||
_wsDeleteSelected = {};
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
var btn = document.getElementById("ws-delete-btn");
|
||||
if (btn) {
|
||||
btn.textContent = "\u2715 Cancel";
|
||||
btn.onclick = cancelWsDeleteMode;
|
||||
}
|
||||
var bar = document.getElementById("ws-delete-bar");
|
||||
if (bar) bar.classList.add("visible");
|
||||
_wsDeleteController.start();
|
||||
}
|
||||
|
||||
function cancelWsDeleteMode() {
|
||||
_wsDeleteMode = false;
|
||||
_wsDeleteSelected = {};
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
var btn = document.getElementById("ws-delete-btn");
|
||||
if (btn) {
|
||||
btn.innerHTML = "🗑 Delete";
|
||||
btn.onclick = startWsDeleteMode;
|
||||
}
|
||||
var bar = document.getElementById("ws-delete-bar");
|
||||
if (bar) bar.classList.remove("visible");
|
||||
_wsDeleteController.cancel();
|
||||
}
|
||||
|
||||
function updateWsDeleteBar() {
|
||||
var count = Object.keys(_wsDeleteSelected).length;
|
||||
var label = document.getElementById("ws-delete-bar-count");
|
||||
if (label) label.textContent = count + " selected";
|
||||
var delBtn = document.getElementById("ws-delete-bar-delete");
|
||||
if (delBtn) delBtn.disabled = count === 0;
|
||||
var selBtn = document.getElementById("ws-delete-bar-select-all");
|
||||
if (selBtn) {
|
||||
var allSelected =
|
||||
count === _wsSavedItems.length && _wsSavedItems.length > 0;
|
||||
selBtn.textContent = allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
}
|
||||
|
||||
function toggleSelectAll() {
|
||||
var allSelected =
|
||||
Object.keys(_wsDeleteSelected).length === _wsSavedItems.length &&
|
||||
_wsSavedItems.length > 0;
|
||||
if (allSelected) {
|
||||
_wsDeleteSelected = {};
|
||||
} else {
|
||||
_wsSavedItems.forEach(function (s) {
|
||||
_wsDeleteSelected[s.ws_id] = true;
|
||||
});
|
||||
}
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
updateWsDeleteBar();
|
||||
_wsDeleteController.toggleAll();
|
||||
}
|
||||
|
||||
var _wsDeleteBatchTrap = null;
|
||||
|
||||
function confirmWsDeleteSelection() {
|
||||
var selected = Object.keys(_wsDeleteSelected);
|
||||
if (!selected.length) {
|
||||
showToast("No workstreams selected", "warning");
|
||||
return;
|
||||
}
|
||||
var overlay = document.getElementById("ws-delete-overlay");
|
||||
var countEl = document.getElementById("ws-delete-count");
|
||||
var listEl = document.getElementById("ws-delete-list");
|
||||
var errorEl = document.getElementById("ws-delete-error");
|
||||
errorEl.textContent = "";
|
||||
countEl.textContent =
|
||||
selected.length + " workstream(s) will be permanently deleted:";
|
||||
listEl.innerHTML = "";
|
||||
selected.forEach(function (wsId) {
|
||||
var item = _wsSavedItems.find(function (s) {
|
||||
return s.ws_id === wsId;
|
||||
});
|
||||
var name = item ? item.alias || item.title || wsId : wsId;
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item";
|
||||
div.textContent = name;
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
// Reset confirm button handler (may have been overwritten to "Close" by previous run)
|
||||
var delBtn = document.getElementById("ws-delete-confirm-btn");
|
||||
if (delBtn) {
|
||||
delBtn.textContent = "Delete";
|
||||
delBtn.disabled = false;
|
||||
delBtn.classList.remove("ws-delete-close");
|
||||
delBtn.onclick = confirmWsDelete;
|
||||
}
|
||||
var cancelBtn = document.getElementById("ws-delete-cancel-btn");
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
overlay.style.display = "flex";
|
||||
|
||||
// Focus trap + Escape
|
||||
if (_wsDeleteBatchTrap)
|
||||
document.removeEventListener("keydown", _wsDeleteBatchTrap);
|
||||
_wsDeleteBatchTrap = function (e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
cancelWsDelete();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Tab") {
|
||||
var box = document.getElementById("ws-delete-box");
|
||||
var focusable = box.querySelectorAll("button:not(:disabled)");
|
||||
var first = focusable[0];
|
||||
var last = focusable[focusable.length - 1];
|
||||
if (e.shiftKey && document.activeElement === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && document.activeElement === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", _wsDeleteBatchTrap);
|
||||
if (cancelBtn) cancelBtn.focus();
|
||||
_wsDeleteController.confirmSelection();
|
||||
}
|
||||
|
||||
function cancelWsDelete() {
|
||||
document.getElementById("ws-delete-overlay").style.display = "none";
|
||||
if (_wsDeleteBatchTrap) {
|
||||
document.removeEventListener("keydown", _wsDeleteBatchTrap);
|
||||
_wsDeleteBatchTrap = null;
|
||||
}
|
||||
_wsDeleteController.closeModal();
|
||||
}
|
||||
|
||||
function confirmWsDelete() {
|
||||
var selected = Object.keys(_wsDeleteSelected);
|
||||
if (!selected.length) return;
|
||||
var overlay = document.getElementById("ws-delete-overlay");
|
||||
var errorEl = document.getElementById("ws-delete-error");
|
||||
var listEl = document.getElementById("ws-delete-list");
|
||||
var countEl = document.getElementById("ws-delete-count");
|
||||
var delBtn = document.getElementById("ws-delete-confirm-btn");
|
||||
var cancelBtn = document.getElementById("ws-delete-cancel-btn");
|
||||
errorEl.textContent = "";
|
||||
|
||||
// Disable buttons during deletion
|
||||
if (delBtn) {
|
||||
delBtn.disabled = true;
|
||||
delBtn.textContent = "Deleting...";
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = true;
|
||||
|
||||
var results = [];
|
||||
var promises = selected.map(function (wsId) {
|
||||
var shortId = wsId.substring(0, 8);
|
||||
var item = _wsSavedItems.find(function (s) {
|
||||
return s.ws_id === wsId;
|
||||
});
|
||||
var name = item ? item.alias || item.title || wsId : wsId;
|
||||
var url = "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/delete";
|
||||
|
||||
return authFetch(url, { method: "POST" })
|
||||
.then(function (r) {
|
||||
var status = r.status;
|
||||
var contentType = r.headers.get("content-type") || "";
|
||||
if (r.ok) {
|
||||
results.push({ name: name, shortId: shortId, ok: true });
|
||||
return;
|
||||
}
|
||||
// Read body as text first to avoid JSON parse errors
|
||||
return r.text().then(function (body) {
|
||||
var errMsg = shortId + ": HTTP " + status;
|
||||
if (contentType.includes("json")) {
|
||||
try {
|
||||
var j = JSON.parse(body);
|
||||
if (j.error) errMsg = shortId + ": " + j.error;
|
||||
} catch (_) {
|
||||
/* fall through */
|
||||
}
|
||||
} else if (body) {
|
||||
errMsg = shortId + ": " + body.substring(0, 200);
|
||||
}
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: errMsg,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(function (err) {
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: shortId + ": " + err.message,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Promise.all(promises).then(function () {
|
||||
// Rebuild the list with results
|
||||
listEl.innerHTML = "";
|
||||
results.forEach(function (r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item" + (r.ok ? "" : " ws-delete-error");
|
||||
div.textContent =
|
||||
(r.ok ? "\u2713 " : "\u2717 ") +
|
||||
r.name +
|
||||
(r.error ? " — " + r.error : "");
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
|
||||
var okCount = results.filter(function (r) {
|
||||
return r.ok;
|
||||
}).length;
|
||||
var failCount = results.filter(function (r) {
|
||||
return !r.ok;
|
||||
}).length;
|
||||
countEl.textContent = okCount + " deleted, " + failCount + " failed";
|
||||
|
||||
if (delBtn) {
|
||||
delBtn.disabled = false;
|
||||
delBtn.textContent = "Close";
|
||||
delBtn.classList.add("ws-delete-close");
|
||||
delBtn.onclick = function () {
|
||||
cancelWsDelete();
|
||||
cancelWsDeleteMode();
|
||||
loadDashboard();
|
||||
};
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
});
|
||||
_wsDeleteController.confirm();
|
||||
}
|
||||
|
||||
// --- Workstream title management ---
|
||||
|
||||
@@ -363,17 +363,18 @@
|
||||
<!-- Delete workstreams confirmation modal (batch) -->
|
||||
<div
|
||||
id="ws-delete-overlay"
|
||||
class="ws-delete-modal-overlay"
|
||||
style="display: none"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="ws-delete-title"
|
||||
>
|
||||
<div id="ws-delete-box">
|
||||
<div id="ws-delete-box" class="ws-delete-modal-box">
|
||||
<h3 id="ws-delete-title">Delete Workstreams</h3>
|
||||
<div id="ws-delete-error" role="alert" aria-live="assertive"></div>
|
||||
<p id="ws-delete-count"></p>
|
||||
<div id="ws-delete-list"></div>
|
||||
<div id="ws-delete-buttons">
|
||||
<div id="ws-delete-list" class="ws-delete-modal-list"></div>
|
||||
<div id="ws-delete-buttons" class="ws-delete-modal-buttons">
|
||||
<button
|
||||
id="ws-delete-cancel-btn"
|
||||
type="button"
|
||||
@@ -383,6 +384,7 @@
|
||||
</button>
|
||||
<button
|
||||
id="ws-delete-confirm-btn"
|
||||
class="ws-delete-confirm"
|
||||
type="button"
|
||||
onclick="confirmWsDelete()"
|
||||
>
|
||||
|
||||
+85
-250
@@ -642,6 +642,9 @@
|
||||
.msg.user {
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
/* .msg.user-reminder lives in shared_static/chat.css so both the
|
||||
interactive UI and the console coord viewer pick up the same
|
||||
yellow themed bubble. */
|
||||
/* .msg.assistant / .msg.info / .msg.error alignment + baseline visuals
|
||||
come from shared_static/chat.css. Interactive UI adds a pre-wrap
|
||||
override for info messages and a tightened tool-message shape with
|
||||
@@ -1628,6 +1631,77 @@ body {
|
||||
.ts-approval-tool .tool-diff .diff-warn {
|
||||
color: var(--yellow);
|
||||
}
|
||||
/* memory/recall calls are background metadata — the audit trail is
|
||||
valuable but they crowd the narrative when a workstream contains
|
||||
dozens of them. Dim by default; full opacity on hover/focus so
|
||||
they remain inspectable without permanently competing for
|
||||
attention. General-sibling combinator (~) extends the fade past
|
||||
any verdict-badge or output-warning sitting between the tool row
|
||||
and its output, so the whole sub-tree fades together rather than
|
||||
leaving a full-opacity badge stranded next to a dim row. */
|
||||
.ts-approval-tool[data-func-name="memory"],
|
||||
.ts-approval-tool[data-func-name="recall"] {
|
||||
opacity: 0.55;
|
||||
transition: opacity 120ms ease-out;
|
||||
}
|
||||
.ts-approval-tool[data-func-name="memory"]:hover,
|
||||
.ts-approval-tool[data-func-name="memory"]:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"]:hover,
|
||||
.ts-approval-tool[data-func-name="recall"]:focus-within {
|
||||
opacity: 1;
|
||||
}
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .media-embed,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .media-embed,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .output-warning,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .output-warning,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output-truncated,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output-truncated {
|
||||
opacity: 0.55;
|
||||
transition: opacity 120ms ease-out;
|
||||
}
|
||||
/* Reveal on hover OR focus-within across the entire dimmed
|
||||
subtree. Without :focus-within on the siblings, a keyboard user
|
||||
tabbing into a link or collapsible toggle inside .tool-output
|
||||
sees the content remain dimmed — a11y regression. Cover the
|
||||
warning + truncation pills too so they fully reveal alongside
|
||||
the result they decorate. */
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output:focus-within,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .media-embed:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .media-embed:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .media-embed:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .media-embed:focus-within,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .output-warning:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .output-warning:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .output-warning:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .output-warning:focus-within,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output-truncated:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output-truncated:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output-truncated:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output-truncated:focus-within {
|
||||
opacity: 1;
|
||||
}
|
||||
/* Truncation indicator — the persisted tool result is clamped at
|
||||
2000 chars per row in storage; surface that on replay so users
|
||||
know they're seeing a clipped view rather than the full output
|
||||
the live session saw. Aligns with .output-warning's left gutter
|
||||
(margin-left: 16px) and uses transparent background + dim border
|
||||
so it reads as quiet metadata rather than a foreign element. */
|
||||
.tool-output-truncated {
|
||||
display: inline-block;
|
||||
margin-top: 4px;
|
||||
margin-left: 16px;
|
||||
padding: 1px 6px;
|
||||
font-size: 10px;
|
||||
color: var(--fg-dim);
|
||||
background: transparent;
|
||||
border-radius: 3px;
|
||||
border: 1px solid var(--fg-dim);
|
||||
}
|
||||
/* .ts-approval (chat.css) stacks its children with flex gap, so a
|
||||
border-top on the body would float above a strip of container
|
||||
background instead of sitting flush against the previous tool row.
|
||||
@@ -2363,224 +2437,10 @@ audio.media-player {
|
||||
color: var(--accent);
|
||||
margin: 0;
|
||||
}
|
||||
.ws-delete-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
color 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
.ws-delete-btn:hover {
|
||||
color: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
/* .dashboard-cards / .dashboard-card / .card-title / .card-meta moved to
|
||||
/shared/cards.css so console (Saved Coordinators) and ui/static (Saved
|
||||
Workstreams) share one source of truth for the basic card primitive.
|
||||
Delete-mode rules stay below — they're ui/static-only until coordinator
|
||||
gets the same UX. */
|
||||
|
||||
/* Delete mode */
|
||||
.dashboard-card.ws-delete-mode {
|
||||
cursor: pointer;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.04);
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected {
|
||||
cursor: default;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode:hover {
|
||||
background: rgba(220, 38, 38, 0.04);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-card-check {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: var(--red);
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
opacity: 0;
|
||||
animation: ws-check-fadein 0.2s ease-out forwards;
|
||||
}
|
||||
@keyframes ws-check-fadein {
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
.dashboard-card.ws-selected {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-selected {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-delete-bar {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
display: flex;
|
||||
animation: ws-bar-slide 0.2s ease-out;
|
||||
}
|
||||
@keyframes ws-bar-slide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ws-card-check {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
.ws-delete-bar .ws-delete-count-label {
|
||||
font-size: 12px;
|
||||
color: var(--fg-dim);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: auto;
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 16px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:hover:not(:disabled) {
|
||||
filter: brightness(1.1);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn:hover {
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
|
||||
/* Delete modal */
|
||||
#ws-delete-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
#ws-delete-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 24px;
|
||||
max-width: 480px;
|
||||
width: 90%;
|
||||
}
|
||||
#ws-delete-box h3 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 16px;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
#ws-delete-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
margin: 12px 0;
|
||||
}
|
||||
#ws-delete-list .ws-delete-item {
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
color: var(--fg-bright);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
#ws-delete-list .ws-delete-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
#ws-delete-list .ws-delete-item.ws-delete-error {
|
||||
color: var(--red);
|
||||
}
|
||||
#ws-delete-buttons {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
#ws-delete-buttons button {
|
||||
padding: 8px 20px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
#ws-delete-buttons button:last-child {
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
border-color: var(--red);
|
||||
}
|
||||
#ws-delete-buttons button.ws-delete-close {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border);
|
||||
}
|
||||
/* .dashboard-cards / .dashboard-card / .card-title / .card-meta and the
|
||||
delete-mode + modal rules are owned by /shared/cards.css so console
|
||||
(Saved Coordinators) and ui/static (Saved Workstreams) share one source
|
||||
of truth. */
|
||||
|
||||
/* Server dashboard row — clickable */
|
||||
.dash-row {
|
||||
@@ -2686,38 +2546,13 @@ audio.media-player {
|
||||
}
|
||||
}
|
||||
|
||||
/* ==========================================================================
|
||||
Verdict badges (intent judge)
|
||||
========================================================================== */
|
||||
.verdict-badge {
|
||||
padding: 4px 10px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-top: 2px;
|
||||
/* No top separator — .ts-verdict-badge (chat.css) shrinks to
|
||||
max-content width, and a 1px border-top would extend only under
|
||||
the badge text and read as a truncated line. */
|
||||
}
|
||||
.verdict-low {
|
||||
color: var(--green);
|
||||
border-left: 3px solid var(--green);
|
||||
}
|
||||
.verdict-medium {
|
||||
color: var(--yellow);
|
||||
border-left: 3px solid var(--yellow);
|
||||
}
|
||||
.verdict-high {
|
||||
color: var(--red);
|
||||
border-left: 3px solid var(--red);
|
||||
}
|
||||
.verdict-critical {
|
||||
color: var(--red);
|
||||
border-left: 3px solid var(--red);
|
||||
background: rgba(255, 80, 80, 0.05);
|
||||
}
|
||||
/* Verdict-badge styling lives in the color-mix block further down
|
||||
in this file (.verdict-badge.verdict-{low,medium,high,critical}).
|
||||
The earlier flat-palette duplicate that lived here was removed —
|
||||
two competing .verdict-badge rule sets caused subtle cascade drift
|
||||
(the color-mix block won for backgrounds, the flat one won for the
|
||||
bare .verdict-low/medium/high/critical class names) which made
|
||||
tweaks fragile. Single source of truth now. */
|
||||
|
||||
.verdict-detail {
|
||||
padding: 6px 12px;
|
||||
|
||||
Reference in New Issue
Block a user