mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 15:32:24 -06:00
Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bd9f780b21 | |||
| 5d14b5f675 | |||
| 4693fa95f1 | |||
| c3423d6606 | |||
| 53f1222c22 | |||
| 802d87a57f | |||
| 8349d9994d | |||
| ac1fd67137 | |||
| 1b40ae79f9 | |||
| b078ddccf0 | |||
| 4b6c93a0e9 | |||
| 9d283e951f | |||
| 4e407e7d4f | |||
| 7ab24e500b | |||
| 5bcbcb73b9 | |||
| af6749421a | |||
| 4d6cb77075 | |||
| 5c225ef39b | |||
| f5a843f44a | |||
| cf44841624 | |||
| 7ffab6a272 | |||
| ba3bc9d989 |
+1
-1
@@ -8,7 +8,7 @@ FROM python:3.14-slim
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.8 /uv /usr/local/bin/uv
|
||||
|
||||
# Remove the slim image's man page exclusion so man-db has actual content
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/docker
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "1.5.1"
|
||||
version = "1.5.5"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
|
||||
@@ -35,11 +35,10 @@ def _seed_children(
|
||||
The production path populates the registry via the cluster-event
|
||||
fan-out thread observing ``ws_created`` events. These tests just
|
||||
need a known-children set for the endpoint handlers to iterate —
|
||||
inject directly under ``_children_lock`` rather than spinning up
|
||||
the collector + fan-out plumbing.
|
||||
inject directly via the registry's bulk-merge surface rather than
|
||||
spinning up the collector + fan-out plumbing.
|
||||
"""
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked(coord_ws_id, child_ws_ids)
|
||||
adapter._registry.merge_children(coord_ws_id, child_ws_ids)
|
||||
|
||||
|
||||
class _AuthMiddleware(BaseHTTPMiddleware):
|
||||
|
||||
@@ -92,3 +92,71 @@ def test_tool_error_does_not_overwrite_approval_badge() -> None:
|
||||
"badge instead so the approval verdict stays visible alongside "
|
||||
"the error."
|
||||
)
|
||||
|
||||
|
||||
def test_replay_history_renders_content_before_tool_block() -> None:
|
||||
"""In ``replayHistory``'s ``role === "assistant"`` branch, the
|
||||
``msg.content`` render must precede the ``msg.tool_calls`` render.
|
||||
|
||||
Two reasons, both load-bearing:
|
||||
|
||||
1. **Structural** — the next loop iteration's ``role === "tool"``
|
||||
message anchors to ``lastToolBlock``. The tool-block branch sets
|
||||
that anchor; the content branch clears it. If content runs after
|
||||
the tool block, the clear silently drops the upcoming tool
|
||||
result. Pre-fix, every interactive tool result was missing from
|
||||
saved-workstream replays whenever the assistant turn carried
|
||||
both narration and tool calls (very common output shape).
|
||||
|
||||
2. **Visual** — the live SSE path renders content first
|
||||
(``stream_text`` streams before ``tool_info`` /
|
||||
``approve_request``), so replay should match.
|
||||
|
||||
The test pins the order via the offsets of the ``msg.content`` and
|
||||
``msg.tool_calls`` branch headers inside the function body."""
|
||||
body = _APP_JS.read_text(encoding="utf-8")
|
||||
start = body.index("Pane.prototype.replayHistory = function")
|
||||
end = body.index("Pane.prototype._attachRetryToLastAssistant", start)
|
||||
fn = body[start:end]
|
||||
# Locate the assistant branch and bound the search to its body —
|
||||
# the function also handles user / tool roles which would otherwise
|
||||
# confuse the offset comparison.
|
||||
asst_start = fn.index('msg.role === "assistant"')
|
||||
asst_end = fn.index('msg.role === "tool"', asst_start)
|
||||
asst = fn[asst_start:asst_end]
|
||||
content_idx = asst.index("if (msg.content)")
|
||||
tool_calls_idx = asst.index("if (msg.tool_calls && msg.tool_calls.length)")
|
||||
assert content_idx < tool_calls_idx, (
|
||||
"replayHistory must render msg.content BEFORE msg.tool_calls "
|
||||
"inside the assistant branch — otherwise the lastToolBlock "
|
||||
"anchor is clobbered before the next iteration's tool result "
|
||||
"can attach to it (and the visual order also drifts from the "
|
||||
"live SSE flow)."
|
||||
)
|
||||
|
||||
|
||||
def test_replay_history_renders_persisted_verdict_badge() -> None:
|
||||
"""Saved-workstream replays must paint the persisted intent verdict
|
||||
next to each tool div, using the same ``renderVerdictBadge`` helper
|
||||
the live ``showInlineToolBlock`` path uses. Pre-fix the audit trail
|
||||
was complete in storage (``intent_verdicts`` table) but never
|
||||
surfaced on replay — operators reviewing a saved workstream
|
||||
couldn't see what the heuristic / LLM judge thought of any tool
|
||||
call. This test pins the call site so a refactor that drops the
|
||||
decoration regresses the audit surface."""
|
||||
body = _APP_JS.read_text(encoding="utf-8")
|
||||
start = body.index("Pane.prototype.replayHistory = function")
|
||||
end = body.index("Pane.prototype._attachRetryToLastAssistant", start)
|
||||
fn = body[start:end]
|
||||
# Match a `renderVerdictBadge(<something>.verdict, ...)` call inside
|
||||
# the replay loop. Loose on whitespace + identifier so a future
|
||||
# rename of the iteration variable doesn't trip CI.
|
||||
badge_call_re = re.compile(
|
||||
r"renderVerdictBadge\(\s*\w+\.verdict\b",
|
||||
)
|
||||
assert badge_call_re.search(fn), (
|
||||
"replayHistory must call renderVerdictBadge(tc.verdict, ...) "
|
||||
"when a persisted verdict is attached to a tool_call entry — "
|
||||
"otherwise the audit-trail data persisted to intent_verdicts "
|
||||
"doesn't surface on saved-workstream replays."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Unit tests for :mod:`turnstone.core.child_source`.
|
||||
|
||||
Covers both strategies in isolation against fakes — no live collector,
|
||||
no live SessionManager. Adapter-level integration coverage continues to
|
||||
live in ``test_coordinator_adapter.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.core.child_source import ClusterChildSource, SameNodeChildSource
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import queue
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SameNodeChildSource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeManager:
|
||||
"""Minimal SessionManager stand-in implementing the subscribe API."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.subscribers: list[Any] = []
|
||||
|
||||
def subscribe_to_state(self, callback: Any) -> None:
|
||||
self.subscribers.append(callback)
|
||||
|
||||
def unsubscribe_from_state(self, callback: Any) -> None:
|
||||
with contextlib.suppress(ValueError):
|
||||
self.subscribers.remove(callback)
|
||||
|
||||
def fire(self, ws_id: str, state: WorkstreamState) -> None:
|
||||
for cb in self.subscribers:
|
||||
cb(ws_id, state)
|
||||
|
||||
|
||||
class TestSameNodeChildSource:
|
||||
def test_start_subscribes_to_manager(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
assert len(mgr.subscribers) == 1
|
||||
|
||||
def test_state_change_for_known_child_pushes_to_sink(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
registry.add_child("p1", "c1")
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
|
||||
mgr.fire("c1", WorkstreamState.RUNNING)
|
||||
|
||||
assert len(sink_calls) == 1
|
||||
ev = sink_calls[0]
|
||||
assert ev["type"] == "cluster_state"
|
||||
assert ev["ws_id"] == "c1"
|
||||
assert ev["state"] == "running"
|
||||
|
||||
def test_state_change_for_unknown_workstream_is_dropped(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
|
||||
# No registry entry — pre-filter drops the event without
|
||||
# invoking the sink.
|
||||
mgr.fire("ws-unknown", WorkstreamState.IDLE)
|
||||
assert sink_calls == []
|
||||
|
||||
def test_shutdown_unsubscribes(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(mgr.subscribers) == 1
|
||||
src.shutdown()
|
||||
assert mgr.subscribers == []
|
||||
|
||||
def test_start_is_idempotent(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
src.start(sink=lambda ev: None)
|
||||
src.start(sink=lambda ev: None)
|
||||
# Second start is a no-op; only one subscription.
|
||||
assert len(mgr.subscribers) == 1
|
||||
|
||||
def test_sink_exception_does_not_propagate(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
registry.add_child("p1", "c1")
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
|
||||
def bad_sink(ev: dict[str, Any]) -> None:
|
||||
raise RuntimeError("sink boom")
|
||||
|
||||
src.start(sink=bad_sink)
|
||||
# Should not raise — the strategy catches sink failures and logs.
|
||||
mgr.fire("c1", WorkstreamState.RUNNING)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClusterChildSource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeCollector:
|
||||
"""Minimal ClusterCollector stand-in providing the listener API."""
|
||||
|
||||
def __init__(self, snapshot: dict[str, Any] | None = None) -> None:
|
||||
self._snapshot = snapshot or {"nodes": []}
|
||||
self.queues: list[queue.Queue[dict[str, Any]]] = []
|
||||
self.unregistered: list[queue.Queue[dict[str, Any]]] = []
|
||||
|
||||
def get_snapshot_and_register(self, q: queue.Queue[dict[str, Any]]) -> dict[str, Any]:
|
||||
self.queues.append(q)
|
||||
return self._snapshot
|
||||
|
||||
def unregister_listener(self, q: queue.Queue[dict[str, Any]]) -> None:
|
||||
self.unregistered.append(q)
|
||||
|
||||
def emit(self, event: dict[str, Any]) -> None:
|
||||
"""Push an event to all registered listener queues."""
|
||||
for q in self.queues:
|
||||
q.put(event)
|
||||
|
||||
|
||||
class TestClusterChildSource:
|
||||
def test_start_subscribes_to_collector(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(coll.queues) == 1
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_start_primes_registry_from_snapshot(self) -> None:
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
{
|
||||
"workstreams": [
|
||||
{"id": "c1", "parent_ws_id": "p1"},
|
||||
{"id": "c2", "parent_ws_id": "p1"},
|
||||
# Unknown parent — dropped
|
||||
{"id": "x", "parent_ws_id": "p-unknown"},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
coll = _FakeCollector(snapshot)
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=lambda: ["p1"],
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
assert registry.parent_for("x") is None
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_event_dispatched_to_sink(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
src.start(sink=sink_calls.append)
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c1", "state": "running"})
|
||||
# Daemon thread loop has 1.0s queue timeout; poll briefly.
|
||||
for _ in range(20):
|
||||
if sink_calls:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(sink_calls) == 1
|
||||
assert sink_calls[0]["ws_id"] == "c1"
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_shutdown_unregisters_and_joins_thread(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
src.start(sink=lambda ev: None)
|
||||
src.shutdown()
|
||||
assert coll.unregistered == coll.queues
|
||||
# Second shutdown is a no-op (idempotent).
|
||||
src.shutdown()
|
||||
|
||||
def test_start_is_idempotent(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(coll.queues) == 1
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_sink_exception_does_not_kill_thread(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
survived_calls: list[dict[str, Any]] = []
|
||||
call_count = [0]
|
||||
|
||||
def flaky_sink(ev: dict[str, Any]) -> None:
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
raise RuntimeError("first one boom")
|
||||
survived_calls.append(ev)
|
||||
|
||||
try:
|
||||
src.start(sink=flaky_sink)
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c1", "state": "x"})
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c2", "state": "y"})
|
||||
for _ in range(40):
|
||||
if survived_calls:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(survived_calls) == 1
|
||||
assert survived_calls[0]["ws_id"] == "c2"
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
|
||||
# Multi-subscriber observer tests for ``SessionManager.subscribe_to_state``
|
||||
# / ``unsubscribe_from_state`` live in ``test_session_manager.py`` where
|
||||
# the proper FakeAdapter / FakeStorage construction helpers already exist.
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Unit tests for :class:`turnstone.core.children_registry.ChildrenRegistry`.
|
||||
|
||||
The registry was lifted from ``CoordinatorAdapter`` in Stage 3 Step 1.
|
||||
Adapter-level coverage for the integrated behavior already lives in
|
||||
``test_coordinator_adapter.py``; this file pins the data structure
|
||||
invariants in isolation so the registry can be reused by future
|
||||
``ChildSource`` strategies (Step 2) without re-deriving the behavior
|
||||
from the adapter test surface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
|
||||
|
||||
class _Sentinel:
|
||||
"""Lightweight UI stand-in; identity-comparable, no behavior."""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry() -> ChildrenRegistry:
|
||||
return ChildrenRegistry()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# install / uninstall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInstallUninstall:
|
||||
def test_install_seeds_empty_child_set_and_presence(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.children_of("p1") == []
|
||||
assert registry.ui_for("p1") is ui
|
||||
assert registry.parents() == ["p1"]
|
||||
|
||||
def test_install_is_idempotent_repoints_ui_keeps_children(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui_a = _Sentinel()
|
||||
ui_b = _Sentinel()
|
||||
registry.install("p1", ui_a)
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
registry.install("p1", ui_b)
|
||||
assert registry.ui_for("p1") is ui_b
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
|
||||
def test_uninstall_clears_forward_reverse_and_presence(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
registry.uninstall("p1")
|
||||
assert registry.children_of("p1") == []
|
||||
assert registry.ui_for("p1") is None
|
||||
assert registry.parents() == []
|
||||
assert registry.parent_for("c1") is None
|
||||
assert registry.parent_for("c2") is None
|
||||
|
||||
def test_uninstall_unknown_parent_is_noop(self, registry: ChildrenRegistry) -> None:
|
||||
registry.uninstall("never-installed") # must not raise
|
||||
|
||||
def test_uninstall_does_not_clobber_other_parents(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
registry.install("p2", _Sentinel())
|
||||
registry.merge_children("p1", ["c1"])
|
||||
registry.merge_children("p2", ["c2"])
|
||||
registry.uninstall("p1")
|
||||
assert registry.parent_for("c1") is None
|
||||
assert registry.parent_for("c2") == "p2"
|
||||
assert registry.parents() == ["p2"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# add_child — atomic check-and-route
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAddChild:
|
||||
def test_add_child_returns_ui_on_success(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.add_child("p1", "c1") is ui
|
||||
assert registry.parent_for("c1") == "p1"
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
def test_add_child_returns_none_when_parent_not_installed(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
assert registry.add_child("absent", "c1") is None
|
||||
assert registry.parent_for("c1") is None
|
||||
|
||||
def test_add_child_returns_none_on_duplicate(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.add_child("p1", "c1") is ui
|
||||
# second add for same child returns None — caller must not
|
||||
# double-dispatch.
|
||||
assert registry.add_child("p1", "c1") is None
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_children — bulk seeding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergeChildren:
|
||||
def test_merge_seeds_forward_and_reverse(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["c1", "c2", "c3"])
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2", "c3"}
|
||||
for cid in ("c1", "c2", "c3"):
|
||||
assert registry.parent_for(cid) == "p1"
|
||||
|
||||
def test_merge_is_idempotent(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["c1"])
|
||||
registry.merge_children("p1", ["c1"])
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
def test_merge_skips_empty_or_falsy_ids(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["", "c1", "", "c2"])
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
|
||||
def test_merge_does_not_require_install(self, registry: ChildrenRegistry) -> None:
|
||||
# Snapshot-priming may run before the parent's install fires —
|
||||
# the merge still seeds the forward set so the install picks
|
||||
# the children up. (Storage-seeded rebuild relies on this.)
|
||||
registry.merge_children("p1", ["c1"])
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
# ui_for is still None because install hasn't run
|
||||
assert registry.ui_for("p1") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lookups — return copies, not live refs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLookups:
|
||||
def test_children_of_returns_copy(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
snap = registry.children_of("p1")
|
||||
snap.append("c3-injected")
|
||||
assert "c3-injected" not in registry.children_of("p1")
|
||||
|
||||
def test_children_of_unknown_parent_returns_empty(self, registry: ChildrenRegistry) -> None:
|
||||
assert registry.children_of("absent") == []
|
||||
|
||||
def test_parent_for_unknown_child_returns_none(self, registry: ChildrenRegistry) -> None:
|
||||
assert registry.parent_for("absent") is None
|
||||
|
||||
def test_parents_returns_copy(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
snap = registry.parents()
|
||||
snap.append("p2-injected")
|
||||
assert "p2-injected" not in registry.parents()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concurrency — concurrent add_child must not exceed the unique-set
|
||||
# invariant or leave a half-installed reverse-index entry.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConcurrency:
|
||||
def test_concurrent_add_child_returns_ui_exactly_once_per_unique(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
results: list[object] = []
|
||||
results_lock = threading.Lock()
|
||||
|
||||
def attempt_add(child_id: str) -> None:
|
||||
r = registry.add_child("p1", child_id)
|
||||
with results_lock:
|
||||
results.append(r)
|
||||
|
||||
threads = [threading.Thread(target=attempt_add, args=("c1",)) for _ in range(20)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Exactly one thread sees the UI; the remaining 19 see None
|
||||
# (duplicate). The forward + reverse indexes carry exactly one
|
||||
# entry for c1.
|
||||
successes = [r for r in results if r is ui]
|
||||
nones = [r for r in results if r is None]
|
||||
assert len(successes) == 1
|
||||
assert len(nones) == 19
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
assert registry.parent_for("c1") == "p1"
|
||||
|
||||
def test_concurrent_install_and_add_child_no_resurrect(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
# add_child racing with uninstall: either lands first (registry
|
||||
# populated) or the parent is gone (returns None). Must NOT
|
||||
# leave a forward-set entry without presence — that would be
|
||||
# the "resurrected after close" leak the locked dispatch path
|
||||
# was guarding against.
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
|
||||
outcomes: list[object] = []
|
||||
|
||||
def adder() -> None:
|
||||
outcomes.append(registry.add_child("p1", "c1"))
|
||||
|
||||
def uninstaller() -> None:
|
||||
registry.uninstall("p1")
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=adder),
|
||||
threading.Thread(target=uninstaller),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# If add_child landed first: c1 is in the forward set, then
|
||||
# uninstall clears everything. End state: nothing.
|
||||
# If uninstall landed first: add_child sees no presence,
|
||||
# returns None, no entry added. End state: nothing.
|
||||
# Either way, the leak invariant holds: child set is empty or
|
||||
# parent is gone, never "child set populated but no presence".
|
||||
children = registry.children_of("p1")
|
||||
ui_present = registry.ui_for("p1") is not None
|
||||
if children:
|
||||
assert ui_present, "registry leaked: children set without presence"
|
||||
+151
-24
@@ -314,12 +314,14 @@ class TestCollectorSnapshot:
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["state"] == "running"
|
||||
|
||||
def test_apply_snapshot_state_change_forwards_pending_approval_detail(self):
|
||||
"""Reconnect-via-snapshot is the resync path after every console
|
||||
restart or network blip. Without forwarding the field here,
|
||||
a child sitting in approval-pending across the gap renders as
|
||||
``activity_state=approval`` with no buttons until the next
|
||||
state change — broken UX during the most common re-sync event."""
|
||||
def test_apply_snapshot_state_change_does_not_carry_pending_approval_detail(self):
|
||||
"""Stage 3 cleanup — the snapshot-resync cluster_state event no
|
||||
longer piggybacks ``pending_approval_detail`` (the field is
|
||||
gone from cluster_state entirely). On reconnect the browser's
|
||||
bulk fetch — triggered by the ``activity_state="approval"``
|
||||
transition in the reducer — pulls the items directly from
|
||||
``ui.serialize_pending_approval_detail()`` via the dashboard
|
||||
endpoint."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -329,10 +331,6 @@ class TestCollectorSnapshot:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
@@ -344,7 +342,6 @@ class TestCollectorSnapshot:
|
||||
"name": "same",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
}
|
||||
],
|
||||
"health": {},
|
||||
@@ -354,7 +351,8 @@ class TestCollectorSnapshot:
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["pending_approval_detail"] == detail
|
||||
assert event["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in event
|
||||
|
||||
def test_apply_snapshot_skips_empty_id_workstream(self):
|
||||
c = _make_collector()
|
||||
@@ -401,12 +399,12 @@ class TestCollectorDelta:
|
||||
# Verify in-memory state was updated
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["state"] == "running"
|
||||
|
||||
def test_apply_delta_ws_state_forwards_pending_approval_detail(self):
|
||||
"""The rich approval payload now travels on the cluster bus so
|
||||
coord tabs can render inline approve/deny buttons in lockstep
|
||||
with the activity_state transition. Collector must forward
|
||||
the field verbatim — the adapter does the child-routing on
|
||||
top, but the bus carries the data."""
|
||||
def test_apply_delta_ws_state_does_not_carry_pending_approval_detail(self):
|
||||
"""Stage 3 cleanup — ``cluster_state`` no longer carries the
|
||||
``pending_approval_detail`` piggyback. Approval items now arrive
|
||||
via bulk fetch on activity_state transition; verdicts via the
|
||||
explicit ``intent_verdict`` event class. Symmetric event flow,
|
||||
no piggyback to dedupe against."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -416,10 +414,6 @@ class TestCollectorDelta:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
@@ -427,13 +421,13 @@ class TestCollectorDelta:
|
||||
"ws_id": "ws1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["pending_approval_detail"] == detail
|
||||
assert event["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in event
|
||||
|
||||
def test_apply_delta_ws_created(self):
|
||||
c = _make_collector()
|
||||
@@ -480,6 +474,139 @@ class TestCollectorDelta:
|
||||
assert event["name"] == "new-name"
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["name"] == "new-name"
|
||||
|
||||
def test_apply_delta_intent_verdict_forwards_verbatim(self):
|
||||
"""Stage 3 Step 5 — node-emitted intent_verdict events flow
|
||||
through _apply_delta to cluster fan-out so coord adapters can
|
||||
re-emit as child_ws_intent_verdict on the parent's SSE."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.9,
|
||||
"recommendation": "approve",
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "intent_verdict", "ws_id": "ws1", "verdict": verdict},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "intent_verdict"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["verdict"] == verdict
|
||||
|
||||
def test_apply_delta_intent_verdict_drops_when_ws_id_missing(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "intent_verdict", "verdict": {}})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_delta_approval_resolved_forwards_verbatim(self):
|
||||
"""Stage 3 Step 5 — paired with intent_verdict; clears the
|
||||
coord tree's pending-approval pill in lockstep with the
|
||||
actual decision rather than waiting for the state-change
|
||||
piggyback."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "ws1",
|
||||
"approved": True,
|
||||
"feedback": "lgtm",
|
||||
"always": False,
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "approval_resolved"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["approved"] is True
|
||||
assert event["feedback"] == "lgtm"
|
||||
assert event["always"] is False
|
||||
|
||||
def test_apply_delta_approve_request_forwards_detail(self):
|
||||
"""Push path for the initial approval items — eliminates the
|
||||
bulk-fetch race that left the coord row stuck on a loading
|
||||
placeholder when the bulk fetch landed in the gap between
|
||||
_emit_state(ATTENTION) and approve_tools setting _pending_approval."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "approve_request", "ws_id": "ws1", "detail": detail},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "approve_request"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["detail"] == detail
|
||||
|
||||
def test_apply_delta_approve_request_drops_when_ws_id_missing(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "approve_request", "detail": {}})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_delta_approval_resolved_coerces_missing_fields(self):
|
||||
"""Defensive: ``approved`` / ``always`` / ``feedback`` may be
|
||||
omitted by older nodes mid-rolling-upgrade; collector coerces
|
||||
to safe defaults."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "approval_resolved", "ws_id": "ws1"})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["approved"] is False
|
||||
assert event["feedback"] == ""
|
||||
assert event["always"] is False
|
||||
|
||||
def test_apply_delta_health_changed(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Tests for the console's coordinator idle-cleanup thread helper.
|
||||
|
||||
The helper itself is a tiny loop wrapping ``mgr.close_idle``; the heavy
|
||||
lifting is in ``SessionManager.close_idle`` (covered in
|
||||
``test_session_manager.py``) and ``bulk_close_stale_orphans`` (covered
|
||||
in ``test_storage_sqlite.py``). These tests verify the glue:
|
||||
|
||||
- the helper runs an initial sweep BEFORE its first sleep (cold-start
|
||||
cleanup without blocking the lifespan),
|
||||
- the helper swallows exceptions so a transient DB blip can't kill the
|
||||
daemon thread,
|
||||
- the helper exits cleanly when ``stop_event`` is set.
|
||||
|
||||
The ``stop_event`` parameter is exclusively for tests — production
|
||||
callers pass ``None`` and the daemon runs for process lifetime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
from turnstone.console.server import _coord_idle_cleanup_thread
|
||||
|
||||
|
||||
class _StubMgr:
|
||||
def __init__(
|
||||
self, *, stop_event: threading.Event, expected_calls: int, raise_after: int = -1
|
||||
) -> None:
|
||||
self.calls: list[float] = []
|
||||
self.sleep_calls_at_each_close: list[int] = []
|
||||
self._stop_event = stop_event
|
||||
self._expected = expected_calls
|
||||
self._raise_after = raise_after
|
||||
self._sleep_count = 0
|
||||
|
||||
def close_idle(self, timeout_sec: float) -> list[str]:
|
||||
# Snapshot how many sleeps preceded this close — lets the
|
||||
# "initial sweep" test verify the first close_idle ran with
|
||||
# zero preceding sleeps.
|
||||
self.sleep_calls_at_each_close.append(self._sleep_count)
|
||||
self.calls.append(timeout_sec)
|
||||
try:
|
||||
if 0 <= self._raise_after < len(self.calls):
|
||||
raise RuntimeError("simulated DB blip")
|
||||
finally:
|
||||
# Set stop after the helper has been exercised enough,
|
||||
# regardless of whether this call raised.
|
||||
if len(self.calls) >= self._expected:
|
||||
self._stop_event.set()
|
||||
return []
|
||||
|
||||
def record_sleep(self, _seconds: float) -> None:
|
||||
self._sleep_count += 1
|
||||
|
||||
|
||||
def _run_until_done(mgr: _StubMgr, stop_event: threading.Event, timeout_sec: float) -> None:
|
||||
with patch("turnstone.console.server.time.sleep", mgr.record_sleep):
|
||||
thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(mgr, timeout_sec, stop_event),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
thread.join(timeout=2.0)
|
||||
assert not thread.is_alive(), "helper failed to exit on stop_event"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_runs_initial_sweep_before_sleep() -> None:
|
||||
"""The first close_idle call must happen BEFORE the first time.sleep —
|
||||
otherwise cold-start orphans wait one ``check_every`` interval (~30 min
|
||||
on default 2h timeout) for the first reap. Crucial because the
|
||||
lifespan no longer does a synchronous initial sweep."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert mgr.sleep_calls_at_each_close == [0], "first close_idle should run before any sleep"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_calls_close_idle_each_tick() -> None:
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=3)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert len(mgr.calls) == 3
|
||||
assert all(t == 120.0 for t in mgr.calls)
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_survives_close_idle_exceptions() -> None:
|
||||
"""A transient DB error must not kill the daemon thread — the next
|
||||
tick should still fire close_idle. Without the try/except, a single
|
||||
blip would silently leak orphans forever."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=4, raise_after=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
# All four calls must have fired despite calls 2-4 raising.
|
||||
assert len(mgr.calls) == 4
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_exits_cleanly_on_stop_event() -> None:
|
||||
"""The stop_event mechanism is the test contract; verify the thread
|
||||
actually exits when the event is set, without needing exceptions or
|
||||
daemon-process termination."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=2)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert stop_event.is_set()
|
||||
@@ -464,3 +464,148 @@ def test_coord_budget_override_survives_wildcard_allow_policy() -> None:
|
||||
assert approved is True
|
||||
types = [e.get("type") for e in captured_events]
|
||||
assert "approve_request" in types, "Wildcard allow must not strip the budget-override prompt"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cluster-bus broadcast hooks — _broadcast_intent_verdict / _approval_resolved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBroadcastIntentVerdict:
|
||||
"""``ConsoleCoordinatorUI._broadcast_intent_verdict`` overrides the
|
||||
no-op base hook to push the verdict onto the cluster bus via
|
||||
``ClusterCollector.emit_console_ws_intent_verdict``. The far more
|
||||
common path is the per-node ``WebUI`` override (covered in
|
||||
test_webui_content.py); this lights up the rare coord-self path
|
||||
(a coord that runs its own LLM judge).
|
||||
"""
|
||||
|
||||
def test_calls_collector_emit_with_ws_id_and_verdict(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "high",
|
||||
"confidence": 0.91,
|
||||
}
|
||||
ui._broadcast_intent_verdict(verdict)
|
||||
collector.emit_console_ws_intent_verdict.assert_called_once_with(
|
||||
"coord-a",
|
||||
verdict,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_intent_verdict.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise — collector failures are observational only.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
class TestBroadcastApprovalResolved:
|
||||
"""``ConsoleCoordinatorUI._broadcast_approval_resolved`` overrides
|
||||
the base hook to push the resolution onto the cluster bus via
|
||||
``ClusterCollector.emit_console_ws_approval_resolved``."""
|
||||
|
||||
def test_calls_collector_with_decision_fields(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
ui._broadcast_approval_resolved(True, "lgtm", always=True)
|
||||
collector.emit_console_ws_approval_resolved.assert_called_once_with(
|
||||
"coord-a",
|
||||
approved=True,
|
||||
feedback="lgtm",
|
||||
always=True,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_normalises_none_feedback_to_empty_string(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
ui._broadcast_approval_resolved(False, None)
|
||||
collector.emit_console_ws_approval_resolved.assert_called_once_with(
|
||||
"coord-a",
|
||||
approved=False,
|
||||
feedback="",
|
||||
always=False,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, None)
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_approval_resolved.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, "ok")
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
class TestBroadcastApproveRequest:
|
||||
"""Coord-side override for the approve_request push. Same rationale
|
||||
as the WebUI override — the coord-self path is rare today, but
|
||||
parity keeps the override symmetric with the rest of the broadcast
|
||||
family."""
|
||||
|
||||
def test_calls_collector_emit_with_ws_id_and_detail(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
ui._broadcast_approve_request(detail)
|
||||
collector.emit_console_ws_approve_request.assert_called_once_with(
|
||||
"coord-a",
|
||||
detail,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_approve_request.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
@@ -378,13 +378,21 @@ class TestCoordinatorAdapterWorkerDispatch:
|
||||
|
||||
|
||||
class TestCoordinatorAdapterChildrenRegistry:
|
||||
def test_emit_created_seeds_empty_children_set(self) -> None:
|
||||
"""Adapter-level integration with :class:`ChildrenRegistry`.
|
||||
|
||||
Pure-registry invariants (forward/reverse consistency, idempotent
|
||||
merge, locking) live in ``test_children_registry.py``. These
|
||||
tests cover the adapter's wiring: that ``emit_*`` paths drive the
|
||||
registry correctly and that the snapshot-priming bridge between
|
||||
a collector snapshot and the registry preserves merge semantics.
|
||||
"""
|
||||
|
||||
def test_emit_created_installs_parent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.emit_created(ws)
|
||||
assert ws.id in adapter._children
|
||||
assert adapter._children[ws.id] == set()
|
||||
assert adapter._active_coords[ws.id] is ws.ui
|
||||
assert adapter._registry.children_of(ws.id) == []
|
||||
assert adapter._registry.ui_for(ws.id) is ws.ui
|
||||
|
||||
def test_emit_rehydrated_calls_rebuild(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
@@ -398,42 +406,36 @@ class TestCoordinatorAdapterChildrenRegistry:
|
||||
adapter.emit_rehydrated(ws)
|
||||
assert calls == [ws.id]
|
||||
|
||||
def test_emit_closed_clears_forward_and_reverse_indexes(self) -> None:
|
||||
def test_emit_closed_uninstalls_parent_and_clears_children(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._merge_child_ids_locked("coord-b", ["child-b1"])
|
||||
adapter._active_coords["coord-a"] = object()
|
||||
adapter._active_coords["coord-b"] = object()
|
||||
adapter._registry.install("coord-a", object())
|
||||
adapter._registry.install("coord-b", object())
|
||||
adapter._registry.merge_children("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._registry.merge_children("coord-b", ["child-b1"])
|
||||
|
||||
adapter.emit_closed("coord-a")
|
||||
|
||||
assert "coord-a" not in adapter._children
|
||||
assert "coord-a" not in adapter._active_coords
|
||||
assert "child-a1" not in adapter._child_to_coord
|
||||
assert "child-a2" not in adapter._child_to_coord
|
||||
assert adapter._registry.ui_for("coord-a") is None
|
||||
assert adapter._registry.children_of("coord-a") == []
|
||||
assert adapter._registry.parent_for("child-a1") is None
|
||||
assert adapter._registry.parent_for("child-a2") is None
|
||||
# coord-b untouched
|
||||
assert adapter._child_to_coord["child-b1"] == "coord-b"
|
||||
assert "coord-b" in adapter._children
|
||||
|
||||
def test_merge_child_ids_locked_is_idempotent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
assert adapter._children["coord-a"] == {"child-1"}
|
||||
assert adapter._child_to_coord == {"child-1": "coord-a"}
|
||||
assert adapter._registry.parent_for("child-b1") == "coord-b"
|
||||
assert adapter._registry.ui_for("coord-b") is not None
|
||||
|
||||
def test_prime_children_from_snapshot_merges_without_overwriting(self) -> None:
|
||||
# Snapshot priming now lives on ClusterChildSource (production
|
||||
# path). The adapter no longer carries its own duplicate copy.
|
||||
from turnstone.core.child_source import ClusterChildSource
|
||||
|
||||
adapter, _ = _make_adapter()
|
||||
# Seed one in-memory coord + one existing child
|
||||
coord_ws = _make_ws()
|
||||
coord_ws.id = "coord-a"
|
||||
mgr = MagicMock()
|
||||
mgr.list_all.return_value = [coord_ws]
|
||||
adapter.attach(mgr)
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
|
||||
source = ClusterChildSource(
|
||||
collector=MagicMock(),
|
||||
registry=adapter._registry,
|
||||
parents_provider=lambda: ["coord-a"],
|
||||
)
|
||||
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
@@ -448,10 +450,13 @@ class TestCoordinatorAdapterChildrenRegistry:
|
||||
},
|
||||
],
|
||||
}
|
||||
adapter._prime_children_from_snapshot(snapshot)
|
||||
assert adapter._children["coord-a"] == {"child-a1", "child-a2"}
|
||||
assert adapter._child_to_coord["child-a2"] == "coord-a"
|
||||
assert "child-x" not in adapter._child_to_coord
|
||||
source._prime_from_snapshot(snapshot)
|
||||
assert set(adapter._registry.children_of("coord-a")) == {
|
||||
"child-a1",
|
||||
"child-a2",
|
||||
}
|
||||
assert adapter._registry.parent_for("child-a2") == "coord-a"
|
||||
assert adapter._registry.parent_for("child-x") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -478,9 +483,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
coord_ws.id = coord_id
|
||||
recorder = _UIRecorder()
|
||||
coord_ws.ui = recorder # type: ignore[assignment]
|
||||
with adapter._children_lock:
|
||||
adapter._children.setdefault(coord_id, set())
|
||||
adapter._active_coords[coord_id] = recorder
|
||||
adapter._registry.install(coord_id, recorder)
|
||||
adapter.attach(_StubManager(coord_ws)) # type: ignore[arg-type]
|
||||
return adapter, recorder, coord_ws
|
||||
|
||||
@@ -510,12 +513,11 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
# Reverse index updated for subsequent cluster_state events.
|
||||
assert adapter._child_to_coord["child-a1"] == "coord-a"
|
||||
assert adapter._registry.parent_for("child-a1") == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_routes_via_reverse_index(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -533,8 +535,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
|
||||
def test_dispatch_ws_closed_routes_to_parent_coord(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{"type": "ws_closed", "ws_id": "child-a1", "reason": "evicted"}
|
||||
)
|
||||
@@ -548,8 +549,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
"""perf-6: _enqueue_on_ui mutates the payload dict in place with
|
||||
the coord's ws_id so the browser can discriminate child events."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -559,53 +559,160 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
)
|
||||
assert recorder.enqueued[0]["ws_id"] == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_forwards_pending_approval_detail(self) -> None:
|
||||
"""The rich approval payload now rides on child_ws_state directly so
|
||||
the browser can mutate liveBadgeCache without a separate live-bulk
|
||||
fetch. Drift here means the inline approve/deny buttons would
|
||||
regress to chasing the dashboard cache (the load-storm pattern
|
||||
Shape A is unwinding)."""
|
||||
def test_dispatch_cluster_state_does_not_carry_pending_approval_detail(
|
||||
self,
|
||||
) -> None:
|
||||
"""Stage 3 cleanup — the ``pending_approval_detail`` piggyback
|
||||
on ``cluster_state`` is gone. Approval items now arrive via
|
||||
bulk fetch (triggered by ``activity_state="approval"`` in the
|
||||
browser); verdicts via ``child_ws_intent_verdict``; resolution
|
||||
via ``child_ws_approval_resolved``. The state event carries
|
||||
only state + activity_state — no detail field."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_state"
|
||||
assert payload["activity_state"] == "approval"
|
||||
assert payload["pending_approval_detail"] == detail
|
||||
assert "pending_approval_detail" not in payload
|
||||
|
||||
def test_dispatch_cluster_state_pending_approval_detail_none_passes_through(
|
||||
self,
|
||||
) -> None:
|
||||
"""Missing pending_approval_detail (no approval pending, or pre-fix
|
||||
node mid-rolling-upgrade) must forward as None — not raise, not
|
||||
omit — so the browser's handleChildState treats it as "no SSE-
|
||||
supplied detail, fall back to cached value"."""
|
||||
def test_dispatch_intent_verdict_emits_child_ws_intent_verdict(self) -> None:
|
||||
"""Stage 3 Step 6 — explicit verdict events are re-emitted as
|
||||
child_ws_intent_verdict on the parent's SSE so the tree UI
|
||||
renders the risk pill without polling."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.92,
|
||||
"recommendation": "approve",
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"type": "intent_verdict",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "tool",
|
||||
"node_id": "node-1",
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert "pending_approval_detail" in payload
|
||||
assert payload["pending_approval_detail"] is None
|
||||
assert payload["type"] == "child_ws_intent_verdict"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["node_id"] == "node-1"
|
||||
assert payload["verdict"] == verdict
|
||||
|
||||
def test_dispatch_intent_verdict_unknown_child_drops(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": "ws-orphan",
|
||||
"verdict": {"call_id": "c1"},
|
||||
}
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_approval_resolved_emits_child_ws_approval_resolved(
|
||||
self,
|
||||
) -> None:
|
||||
"""Stage 3 Step 6 — paired with intent_verdict; clears the
|
||||
pending-approval pill on the parent's tree UI in lockstep
|
||||
with the actual decision."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"approved": True,
|
||||
"feedback": "lgtm",
|
||||
"always": False,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_approval_resolved"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["approved"] is True
|
||||
assert payload["feedback"] == "lgtm"
|
||||
assert payload["always"] is False
|
||||
|
||||
def test_dispatch_approval_resolved_coerces_missing_fields(self) -> None:
|
||||
"""Older nodes mid-rolling-upgrade may omit approved / always /
|
||||
feedback; dispatch coerces to safe defaults."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event({"type": "approval_resolved", "ws_id": "child-a1"})
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["approved"] is False
|
||||
assert payload["feedback"] == ""
|
||||
assert payload["always"] is False
|
||||
|
||||
def test_dispatch_approval_resolved_unknown_child_drops(self) -> None:
|
||||
"""Symmetric to the intent_verdict drop test — events for
|
||||
ws_ids the registry doesn't know about silently drop instead
|
||||
of fanning out to a parent that has no business seeing them."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "ws-orphan",
|
||||
"approved": True,
|
||||
},
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_approve_request_emits_child_ws_approve_request(
|
||||
self,
|
||||
) -> None:
|
||||
"""Push path for the initial approval items — eliminates the
|
||||
bulk-fetch race that left the coord row stuck on a loading
|
||||
placeholder when the bulk fetch landed in the gap between
|
||||
_emit_state(ATTENTION) and approve_tools setting _pending_approval."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"detail": detail,
|
||||
},
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_approve_request"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["node_id"] == "node-1"
|
||||
assert payload["detail"] == detail
|
||||
|
||||
def test_dispatch_approve_request_unknown_child_drops(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": "ws-orphan",
|
||||
"detail": {"items": []},
|
||||
},
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
@@ -79,9 +79,10 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
assert "function submitChildApproval" in body or "submitChildApproval(" in body
|
||||
# The shared approve POST helper (parameterized for child ws_ids)
|
||||
assert "function approveWorkstream" in body or "approveWorkstream(" in body
|
||||
# The urgent live-bulk fetch option that fires on activity_state
|
||||
# transitions in/out of "approval"
|
||||
assert "{ urgent: true }" in body or "urgent: true" in body
|
||||
# The 409 stale-call_id retry path uses invalidateLiveBadge +
|
||||
# scheduleLiveFetch (Stage 3 cleanup removed the urgent flag —
|
||||
# cache invalidation makes the TTL gate fall through naturally).
|
||||
assert "invalidateLiveBadge(targetWsId)" in body
|
||||
# Server-side payload field — drift here means the JS reads stale keys
|
||||
assert "pending_approval_detail" in body
|
||||
# Reconnect parity (chunk 4): the SSE re-open handler must drop
|
||||
@@ -90,10 +91,10 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# can't render zombie approve/deny buttons on a row whose
|
||||
# approval was resolved during the gap. The implementation
|
||||
# iterates the cache and deletes only !permanent entries —
|
||||
# asserting the literal Map iteration form keeps a refactor
|
||||
# back to liveBadgeCache.clear() (which would re-pay 403s on
|
||||
# every reconnect for denied ids) from sneaking in.
|
||||
assert "liveBadgeCache.delete" in body
|
||||
# asserting the literal helper call keeps a refactor back to
|
||||
# _liveBadgeCacheClear() (which would re-pay 403s on every
|
||||
# reconnect for denied ids) from sneaking in.
|
||||
assert "_liveBadgeCacheDelete" in body
|
||||
# Edge-case matrix sentinel labels — POLICY-BLOCKED renders when
|
||||
# an item has error set + needs_approval=False (server-side
|
||||
# tool policy already blocked the call); "(judge unavailable)"
|
||||
@@ -113,15 +114,19 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# coord-self ws_id (the coord lives on the console process).
|
||||
# Children live on cluster nodes and 404 without the prefix.
|
||||
assert "/v1/api/route/workstreams/" in body
|
||||
# Late-judge polling — the LLM judge runs async on the child
|
||||
# node and never pushes a signal that reaches the coord, so
|
||||
# the row's pending_approval_detail with judge_pending=true
|
||||
# would freeze on heuristic verdicts forever without this
|
||||
# poll loop. The poller is GLOBAL (not per-row) so off-screen
|
||||
# rows still refresh — a per-row poller's scheduleLiveFetch
|
||||
# call short-circuits on non-visible rows, leaving them stuck.
|
||||
assert "_maybeStartJudgePoll" in body
|
||||
assert "_judgePollTick" in body
|
||||
# Late-arriving LLM judge verdicts — Stage 3 Step 5 promoted
|
||||
# ``intent_verdict`` and ``approval_resolved`` to first-class
|
||||
# cluster-bus event types, so the coord adapter dispatches them
|
||||
# as ``child_ws_intent_verdict`` / ``child_ws_approval_resolved``
|
||||
# on the parent's SSE stream. The browser handlers write
|
||||
# directly to liveBadgeCache (bypassing scheduleLiveFetch's
|
||||
# visibility gate cleanly) so off-screen rows pick up verdicts
|
||||
# without polling. Replaced the old ``_judgePollTick`` 90-second
|
||||
# global poll loop and its visibility-gate-bypass workaround.
|
||||
assert "handleChildIntentVerdict" in body
|
||||
assert "handleChildApprovalResolved" in body
|
||||
assert "child_ws_intent_verdict" in body
|
||||
assert "child_ws_approval_resolved" in body
|
||||
# Reload parity for the coord-self approval gate: init() must
|
||||
# consume the authoritative GET /workstreams snapshot's
|
||||
# pending_approval_detail so a freshly opened tab can render
|
||||
@@ -154,18 +159,19 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
assert "callOutcomes" in body
|
||||
|
||||
|
||||
def test_coordinator_js_handle_child_state_reads_sse_pending_approval_detail():
|
||||
"""Lock the Shape A behavior change: child_ws_state SSE events now
|
||||
carry ``pending_approval_detail`` directly so the browser mutates
|
||||
``liveBadgeCache`` without firing an urgent live-bulk fetch on
|
||||
every activity_state transition into/out of approval. A refactor
|
||||
that re-introduces the urgent-fetch path on routine transitions
|
||||
(or drops the SSE-source merge guard in flushLiveFetches) would
|
||||
re-open the load-storm pattern this PR is fixing.
|
||||
def test_coordinator_js_handle_child_state_no_longer_reads_sse_pending_approval_detail():
|
||||
"""Stage 3 cleanup — ``pending_approval_detail`` is no longer
|
||||
piggybacked on child_ws_state events. Approval items now arrive
|
||||
via bulk fetch on the activity_state="approval" transition;
|
||||
verdicts via the explicit ``child_ws_intent_verdict`` event class;
|
||||
resolution via ``child_ws_approval_resolved``. A refactor that
|
||||
re-introduces the piggyback would silently re-open the
|
||||
duplicate-path race the dedicated event classes were added to
|
||||
eliminate.
|
||||
|
||||
Structural assertions (regex against multi-line source) — symbol-
|
||||
presence alone wouldn't catch a guard that keeps the names but
|
||||
inverts the comparison or drops the ``prev.live`` check. This
|
||||
inverts the comparison or drops the ``prev.live`` check. This
|
||||
codebase has no JS test framework, so locking the guard's shape
|
||||
here is the next-best thing to a behavioral test."""
|
||||
import re
|
||||
@@ -176,26 +182,46 @@ def test_coordinator_js_handle_child_state_reads_sse_pending_approval_detail():
|
||||
)
|
||||
body = coord_js.read_text(encoding="utf-8")
|
||||
|
||||
# handleChildState now reads the SSE-supplied detail.
|
||||
assert "ev.pending_approval_detail" in body
|
||||
# The pre-fix urgent-fetch on activity_state transitions is
|
||||
# gone (the 409 retry path keeps its own ``{ urgent: true }``
|
||||
# for stale-call_id refresh — that's a different scenario).
|
||||
# The piggyback read is gone from handleChildState. (The string
|
||||
# may still appear elsewhere — e.g. handleChildIntentVerdict
|
||||
# reading from cache, or comments — but never as ``ev.pending_approval_detail``.)
|
||||
assert "ev.pending_approval_detail" not in body
|
||||
# The pre-fix urgent-fetch on activity_state transitions is gone.
|
||||
assert "enteredApproval" not in body
|
||||
assert "leftApproval" not in body
|
||||
# ``pendingApproval`` flag derivation must check BOTH state and
|
||||
# activity_state. The worker thread can fire the state transition
|
||||
# to "attention" before approve_tools updates activity_state, so
|
||||
# checking only activity_state misses children that legitimately
|
||||
# need approval. Pin the disjunction so the regression doesn't
|
||||
# silently re-introduce.
|
||||
assert re.search(
|
||||
r'existing\.state\s*===\s*"attention"\s*\|\|\s*'
|
||||
r'existing\.activity_state\s*===\s*"approval"',
|
||||
body,
|
||||
), (
|
||||
"handleChildState must derive pendingApproval from "
|
||||
"(state==='attention' || activity_state==='approval')"
|
||||
)
|
||||
|
||||
# SSE-authoritative window constant is defined and used.
|
||||
assert re.search(r"\bconst\s+SSE_AUTHORITATIVE_MS\s*=\s*\d+", body), (
|
||||
"SSE_AUTHORITATIVE_MS constant must be defined as a numeric literal"
|
||||
)
|
||||
|
||||
# handleChildState writes sseUpdatedAt = Date.now() into the cache
|
||||
# entry it sets. This is the SSE-source tag; without it, the
|
||||
# merge guard in flushLiveFetches has nothing to gate on.
|
||||
# SSE writers tag entries with sseUpdatedAt: Date.now() so the
|
||||
# merge guard in flushLiveFetches preserves them against stale
|
||||
# bulk-fetch responses. handleChildState only stamps when it
|
||||
# AUTHORITATIVELY clears the detail (off-approval transition);
|
||||
# writers that stamp unconditionally are intent_verdict (verdict
|
||||
# stamp), approval_resolved (clear), and the optimistic-clear
|
||||
# path in submitChildApproval. Pinning the literal Date.now()
|
||||
# call keeps a refactor that drops the SSE-source tag entirely
|
||||
# from sneaking in.
|
||||
assert re.search(
|
||||
r"sseUpdatedAt:\s*Date\.now\(\)",
|
||||
body,
|
||||
), "handleChildState must write sseUpdatedAt: Date.now() onto liveBadgeCache entries"
|
||||
), "Critical SSE writers must stamp sseUpdatedAt: Date.now()"
|
||||
|
||||
# flushLiveFetches' merge guard structure: SSE-set pending_approval
|
||||
# / _detail wins over a stale bulk-poll snapshot when (live) AND
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Unit tests for ``turnstone.core.history_decoration``.
|
||||
|
||||
The decoration helpers are shared between two surfaces — interactive's
|
||||
SSE replay (``_build_history``) and the lifted ``/history`` REST
|
||||
endpoint (``make_history_handler``, used by both interactive and
|
||||
coord). Pinning the wire shape here lets a future schema/projection
|
||||
change land in one file rather than spread across the two surfaces.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from turnstone.core.history_decoration import (
|
||||
build_output_assessment_payload,
|
||||
build_verdict_payload,
|
||||
decorate_history_messages,
|
||||
decorate_tool_call,
|
||||
)
|
||||
|
||||
|
||||
class TestBuildVerdictPayload:
|
||||
"""The wire-shape projection that's the single source of truth for
|
||||
what intent_verdict fields ship to the client."""
|
||||
|
||||
def test_skips_unflagged_baseline(self) -> None:
|
||||
"""``risk_level`` "none" is the unflagged-tool baseline; the
|
||||
client filters those anyway, so projecting None at the wire
|
||||
layer keeps the payload tight on long workstreams."""
|
||||
row = {"risk_level": "none", "recommendation": "approve", "tier": "heuristic"}
|
||||
assert build_verdict_payload(row) is None
|
||||
|
||||
def test_drops_call_id_and_func_name(self) -> None:
|
||||
"""The client already has these on ``tc.id`` / ``tc.name``;
|
||||
re-shipping them per-tool_call would balloon long replays."""
|
||||
row = {
|
||||
"call_id": "call_abc",
|
||||
"func_name": "bash",
|
||||
"risk_level": "medium",
|
||||
"recommendation": "review",
|
||||
"confidence": 0.8,
|
||||
"intent_summary": "summary",
|
||||
"tier": "heuristic",
|
||||
}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "call_id" not in out
|
||||
assert "func_name" not in out
|
||||
# Sanity — the kept fields are the ones renderVerdictBadge reads.
|
||||
assert out["risk_level"] == "medium"
|
||||
assert out["recommendation"] == "review"
|
||||
assert out["confidence"] == 0.8
|
||||
assert out["intent_summary"] == "summary"
|
||||
assert out["tier"] == "heuristic"
|
||||
|
||||
def test_includes_reasoning_for_either_tier_when_present(self) -> None:
|
||||
"""Heuristic verdicts in this project emit structured
|
||||
rationales (one per matched pattern) — e.g.
|
||||
``policy.py`` writes a reasoning string per heuristic hit.
|
||||
Ship the field for either tier when it has content; only
|
||||
omit when the row didn't write one."""
|
||||
for tier in ("heuristic", "llm"):
|
||||
row = {
|
||||
"risk_level": "high",
|
||||
"tier": tier,
|
||||
"reasoning": "The command exfiltrates ~/.ssh/id_rsa over an external connection.",
|
||||
}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "id_rsa" in out["reasoning"]
|
||||
|
||||
def test_omits_reasoning_when_empty(self) -> None:
|
||||
"""An absent / empty reasoning string shouldn't ship as
|
||||
``reasoning: ""`` — the rationale ``<details>`` block on the
|
||||
client renders an empty disclosure when the field is present
|
||||
but empty."""
|
||||
row = {"risk_level": "high", "tier": "heuristic", "reasoning": ""}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "reasoning" not in out
|
||||
|
||||
def test_includes_judge_model_when_present(self) -> None:
|
||||
"""``judge_model`` rides through so the batch tier badge can
|
||||
render ``⚖ llm:claude-haiku-4`` on history-only replays
|
||||
rather than the bare ``⚖ llm`` label."""
|
||||
row = {"risk_level": "high", "tier": "llm", "judge_model": "claude-haiku-4"}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert out["judge_model"] == "claude-haiku-4"
|
||||
|
||||
def test_omits_judge_model_when_empty(self) -> None:
|
||||
row = {"risk_level": "medium", "tier": "heuristic", "judge_model": ""}
|
||||
out = build_verdict_payload(row)
|
||||
assert out is not None
|
||||
assert "judge_model" not in out
|
||||
|
||||
|
||||
class TestBuildOutputAssessmentPayload:
|
||||
"""Output-guard wire shape — flags decoded from JSON string at
|
||||
this layer so the client never has to parse twice."""
|
||||
|
||||
def test_skips_unflagged_baseline(self) -> None:
|
||||
row = {"risk_level": "none", "flags": "[]"}
|
||||
assert build_output_assessment_payload(row) is None
|
||||
|
||||
def test_decodes_flags_from_json(self) -> None:
|
||||
row = {"risk_level": "high", "flags": '["api_key","email"]', "redacted": 1}
|
||||
out = build_output_assessment_payload(row)
|
||||
assert out is not None
|
||||
assert out["flags"] == ["api_key", "email"]
|
||||
assert out["redacted"] is True
|
||||
assert out["risk_level"] == "high"
|
||||
|
||||
def test_handles_malformed_flags_json(self) -> None:
|
||||
"""Bad JSON in ``flags`` must not block the rest of the
|
||||
assessment from rendering — degrade to empty list."""
|
||||
row = {"risk_level": "medium", "flags": "not-json", "redacted": 0}
|
||||
out = build_output_assessment_payload(row)
|
||||
assert out is not None
|
||||
assert out["flags"] == []
|
||||
assert out["redacted"] is False
|
||||
|
||||
|
||||
class TestDecorateToolCall:
|
||||
"""In-place mutation of either OpenAI-format or flattened tool_call
|
||||
entries — both shapes carry ``id`` at the top level."""
|
||||
|
||||
def test_attaches_verdict_when_present(self) -> None:
|
||||
tc: dict[str, object] = {"id": "call_1", "function": {"name": "bash", "arguments": "{}"}}
|
||||
verdicts = {
|
||||
"call_1": {
|
||||
"risk_level": "medium",
|
||||
"recommendation": "review",
|
||||
"confidence": 0.7,
|
||||
"intent_summary": "summary",
|
||||
"tier": "heuristic",
|
||||
}
|
||||
}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" in tc
|
||||
assert tc["verdict"]["risk_level"] == "medium" # type: ignore[index]
|
||||
|
||||
def test_skips_when_no_call_id_match(self) -> None:
|
||||
tc: dict[str, object] = {"id": "call_other", "name": "bash"}
|
||||
verdicts = {
|
||||
"call_1": {"risk_level": "medium", "tier": "heuristic"},
|
||||
}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
def test_skips_unflagged_verdict(self) -> None:
|
||||
"""``build_verdict_payload`` returns None for unflagged rows;
|
||||
decorate_tool_call must not stamp ``verdict`` in that case."""
|
||||
tc: dict[str, object] = {"id": "call_1", "name": "bash"}
|
||||
verdicts = {"call_1": {"risk_level": "none", "tier": "heuristic"}}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
def test_handles_empty_id(self) -> None:
|
||||
"""A tool_call with no id can't be paired against the lookup
|
||||
table — must not raise (or stamp the wrong row's verdict)."""
|
||||
tc: dict[str, object] = {"id": "", "name": "bash"}
|
||||
verdicts = {"call_1": {"risk_level": "high", "tier": "heuristic"}}
|
||||
decorate_tool_call(tc, verdicts, {})
|
||||
assert "verdict" not in tc
|
||||
|
||||
|
||||
class TestDecorateHistoryMessages:
|
||||
"""End-to-end mutation of a /history-shaped message list — covers
|
||||
the full transform applied by ``make_history_handler``."""
|
||||
|
||||
def test_decorates_tool_calls_and_marks_truncated(self) -> None:
|
||||
verdicts = {
|
||||
"call_a": {
|
||||
"risk_level": "high",
|
||||
"recommendation": "deny",
|
||||
"confidence": 0.95,
|
||||
"intent_summary": "exfil",
|
||||
"tier": "llm",
|
||||
"reasoning": "ssh key access",
|
||||
}
|
||||
}
|
||||
assessments = {
|
||||
"call_a": {"risk_level": "high", "flags": '["secret"]', "redacted": 1},
|
||||
}
|
||||
# Tool result content of exactly TOOL_RESULT_STORAGE_CAP chars
|
||||
# hits the storage cap (longer is impossible — storage clamps
|
||||
# at the cap). Reference the constant rather than a literal so
|
||||
# this test stays correct if the cap moves again.
|
||||
from turnstone.core.history_decoration import TOOL_RESULT_STORAGE_CAP
|
||||
|
||||
truncated_content = "x" * TOOL_RESULT_STORAGE_CAP
|
||||
messages: list[dict[str, object]] = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "running",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_a",
|
||||
"function": {"name": "bash", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_a", "content": truncated_content},
|
||||
{"role": "tool", "tool_call_id": "call_b", "content": "short"},
|
||||
]
|
||||
decorate_history_messages(messages, verdicts, assessments)
|
||||
# Assistant tool_calls got both decorations.
|
||||
tc = messages[1]["tool_calls"][0] # type: ignore[index]
|
||||
assert tc["verdict"]["risk_level"] == "high"
|
||||
assert tc["verdict"]["tier"] == "llm"
|
||||
assert "reasoning" in tc["verdict"]
|
||||
assert tc["output_assessment"]["flags"] == ["secret"]
|
||||
assert tc["output_assessment"]["redacted"] is True
|
||||
# Truncated tool message got the flag; the short one did not.
|
||||
assert messages[2].get("truncated") is True
|
||||
assert "truncated" not in messages[3]
|
||||
|
||||
def test_no_op_on_empty_indexes(self) -> None:
|
||||
"""When neither table has rows for the workstream, the wire
|
||||
shape passes through unchanged — replay must degrade
|
||||
gracefully when verdict storage is empty / unavailable."""
|
||||
messages: list[dict[str, object]] = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "call_a", "function": {"name": "bash", "arguments": "{}"}}],
|
||||
},
|
||||
]
|
||||
decorate_history_messages(messages, {}, {})
|
||||
tc = messages[0]["tool_calls"][0] # type: ignore[index]
|
||||
assert "verdict" not in tc
|
||||
assert "output_assessment" not in tc
|
||||
@@ -8,6 +8,7 @@ from turnstone.core.metacognition import (
|
||||
NUDGE_RESUME,
|
||||
NUDGE_START,
|
||||
NUDGE_TOOL_ERROR,
|
||||
RepeatDetector,
|
||||
detect_completion,
|
||||
detect_correction,
|
||||
format_nudge,
|
||||
@@ -308,3 +309,70 @@ class TestRepeatNudge:
|
||||
"""Repeat nudge should fire even with zero memories."""
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("repeat", state, message_count=5, memory_count=0) is True
|
||||
|
||||
|
||||
class TestRepeatDetector:
|
||||
"""Repeat-detection streak machine — fires only when the same signature
|
||||
is recorded ``threshold`` times *consecutively* (default 3). Recording
|
||||
any different signature resets the streak, so an interrupted repeat
|
||||
isn't flagged as a stuck loop."""
|
||||
|
||||
def test_below_threshold_does_not_fire(self):
|
||||
det = RepeatDetector()
|
||||
assert det.record("a") is False
|
||||
assert det.record("a") is False # second call still under threshold
|
||||
|
||||
def test_at_threshold_fires(self):
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_continues_to_fire_past_threshold(self):
|
||||
# Caller is responsible for clearing after a fire — until they do,
|
||||
# subsequent identical calls keep returning True.
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("a") is True
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_clear_resets_count(self):
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
det.clear()
|
||||
assert det.record("a") is False # back to 1 after clear
|
||||
|
||||
def test_intervening_sig_resets_streak(self):
|
||||
# The streak is consecutive: recording any other sig mid-streak
|
||||
# discards the in-progress count. An alternating pattern like
|
||||
# [A, A, B, A, A] is two short streaks of 2, not a streak of 4.
|
||||
det = RepeatDetector()
|
||||
det.record("a")
|
||||
det.record("a")
|
||||
assert det.record("b") is False # b at count 1; a's streak is gone
|
||||
assert det.record("a") is False # a starts fresh at 1
|
||||
assert det.record("a") is False # a at 2
|
||||
assert det.record("a") is True # a hits 3 — fresh streak completes
|
||||
|
||||
def test_errored_signature_counts_toward_repeat(self):
|
||||
# Regression: when metacog was split out of the system message,
|
||||
# the error-output skip got reintroduced and stuck-loop detection
|
||||
# silently broke for tools that kept failing. Detector itself is
|
||||
# signature-only — error vs. success is the caller's policy.
|
||||
det = RepeatDetector()
|
||||
# Caller records an errored call's sig the same as a successful one;
|
||||
# the streak is what matters.
|
||||
for _ in range(3):
|
||||
last = det.record("bash:ls /nonexistent")
|
||||
assert last is True
|
||||
|
||||
def test_custom_threshold(self):
|
||||
det = RepeatDetector(threshold=2)
|
||||
assert det.record("a") is False
|
||||
assert det.record("a") is True
|
||||
|
||||
def test_threshold_one_fires_immediately(self):
|
||||
det = RepeatDetector(threshold=1)
|
||||
assert det.record("a") is True
|
||||
|
||||
@@ -224,6 +224,25 @@ class TestOpenAIProvider:
|
||||
sanitize_messages([original])
|
||||
assert original["content"] is None
|
||||
|
||||
def test_sanitize_messages_strips_underscore_sibling_keys(self) -> None:
|
||||
"""Internal sibling metadata (``_reminders``, ``_reminders_delivered``,
|
||||
``_attachments_meta``, ``_provider_content``) must be stripped
|
||||
before the wire — the OpenAI-compat APIs reject unknown fields."""
|
||||
msgs = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi",
|
||||
"_reminders": [{"type": "correction", "text": "watch"}],
|
||||
"_reminders_delivered": True,
|
||||
"_attachments_meta": [{"kind": "image"}],
|
||||
}
|
||||
]
|
||||
result = sanitize_messages(msgs)
|
||||
assert result == [{"role": "user", "content": "hi"}]
|
||||
assert "_reminders" not in result[0]
|
||||
assert "_reminders_delivered" not in result[0]
|
||||
assert "_attachments_meta" not in result[0]
|
||||
|
||||
# -- sanitize_messages: orphan detection -----------------------------------
|
||||
|
||||
def test_sanitize_orphaned_tool_call_synthesized(self) -> None:
|
||||
|
||||
+837
-114
File diff suppressed because it is too large
Load Diff
@@ -18,13 +18,22 @@ from __future__ import annotations
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.session_manager import SessionKindAdapter, SessionManager
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
from turnstone.core.workstream import (
|
||||
BULK_CLOSE_STATE_VALUES,
|
||||
Workstream,
|
||||
WorkstreamKind,
|
||||
WorkstreamState,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test fixtures
|
||||
@@ -156,6 +165,8 @@ class _Row:
|
||||
kind: str
|
||||
state: str = "idle"
|
||||
parent_ws_id: str | None = None
|
||||
updated: str = ""
|
||||
node_id: str | None = None
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
@@ -164,8 +175,19 @@ class FakeStorage:
|
||||
def __init__(self) -> None:
|
||||
self.rows: dict[str, _Row] = {}
|
||||
self.state_updates: list[tuple[str, str]] = []
|
||||
self.touch_calls: list[str] = []
|
||||
self.register_raises = False
|
||||
self.lock = threading.Lock()
|
||||
# Live-services lookup target for close_idle pass 2. Map
|
||||
# service_type → list of live service_ids. Tests that exercise
|
||||
# liveness scoping populate this directly; default empty means
|
||||
# "no peers alive" (every row unprotected by liveness).
|
||||
self.live_services: dict[str, list[str]] = {}
|
||||
self.list_services_raises = False
|
||||
|
||||
@staticmethod
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
def register_workstream(
|
||||
self,
|
||||
@@ -178,6 +200,8 @@ class FakeStorage:
|
||||
parent_ws_id: str | None = None,
|
||||
skill_id: str = "",
|
||||
skill_version: int = 0,
|
||||
state: str = "idle",
|
||||
updated: str | None = None,
|
||||
) -> None:
|
||||
if self.register_raises:
|
||||
raise RuntimeError("register forced failure")
|
||||
@@ -188,14 +212,66 @@ class FakeStorage:
|
||||
user_id=user_id or "",
|
||||
name=name,
|
||||
kind=kind_str,
|
||||
state=state,
|
||||
parent_ws_id=parent_ws_id,
|
||||
updated=updated if updated is not None else self._now_iso(),
|
||||
node_id=node_id,
|
||||
)
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
with self.lock:
|
||||
self.touch_calls.append(ws_id)
|
||||
if ws_id in self.rows:
|
||||
self.rows[ws_id].updated = self._now_iso()
|
||||
|
||||
def update_workstream_state(self, ws_id: str, state: str) -> None:
|
||||
with self.lock:
|
||||
self.state_updates.append((ws_id, state))
|
||||
if ws_id in self.rows:
|
||||
self.rows[ws_id].state = state
|
||||
self.rows[ws_id].updated = self._now_iso()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
kind_str = kind.value if isinstance(kind, WorkstreamKind) else str(kind)
|
||||
excluded = set(exclude_ws_ids)
|
||||
live_set = set(live_node_ids) if live_node_ids else set()
|
||||
now = self._now_iso()
|
||||
closed: list[str] = []
|
||||
with self.lock:
|
||||
for ws_id, row in self.rows.items():
|
||||
if (
|
||||
row.kind == kind_str
|
||||
and row.state in BULK_CLOSE_STATE_VALUES
|
||||
and row.updated < cutoff
|
||||
and ws_id not in excluded
|
||||
):
|
||||
# Liveness gate: when live_node_ids was provided AND
|
||||
# non-empty, protect rows whose owner is in the live
|
||||
# set. NULL node_id is always eligible. When
|
||||
# live_node_ids is None or empty, no protection
|
||||
# (mirror of the real backends).
|
||||
if live_node_ids and row.node_id is not None and row.node_id in live_set:
|
||||
continue
|
||||
row.state = "closed"
|
||||
row.updated = now
|
||||
self.state_updates.append((ws_id, "closed"))
|
||||
closed.append(ws_id)
|
||||
return closed
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
if self.list_services_raises:
|
||||
raise RuntimeError("list_services forced failure")
|
||||
with self.lock:
|
||||
return [
|
||||
{"service_id": sid, "service_type": service_type}
|
||||
for sid in self.live_services.get(service_type, [])
|
||||
]
|
||||
|
||||
def get_workstream(self, ws_id: str) -> dict[str, Any] | None:
|
||||
with self.lock:
|
||||
@@ -228,6 +304,7 @@ def _make_manager(
|
||||
max_active: int = 5,
|
||||
storage: FakeStorage | None = None,
|
||||
event_emitter: Any = _EMITTER_DEFAULT,
|
||||
node_id: str | None = None,
|
||||
) -> tuple[SessionManager, FakeAdapter, FakeStorage]:
|
||||
"""Build a SessionManager wired to a FakeAdapter for both Protocols.
|
||||
|
||||
@@ -246,6 +323,7 @@ def _make_manager(
|
||||
storage=storage,
|
||||
max_active=max_active,
|
||||
event_emitter=emitter,
|
||||
node_id=node_id,
|
||||
)
|
||||
return mgr, adapter, storage
|
||||
|
||||
@@ -579,6 +657,24 @@ def test_open_resurrects_closed_state() -> None:
|
||||
assert ws_id in [e.ws_id for e in adapter.events_of("rehydrated")]
|
||||
|
||||
|
||||
def test_open_touches_workstream_on_rehydrate() -> None:
|
||||
"""Rehydrating a workstream must bump its ``updated`` so a concurrent
|
||||
close_idle pass-2 in this same process can't clobber the freshly-loaded
|
||||
row to ``closed`` because its DB ``updated`` is older than the cutoff.
|
||||
The touch is best-effort (try/except in open()) but must fire on the
|
||||
happy path."""
|
||||
mgr, _, storage = _make_manager()
|
||||
ws = mgr.create(user_id="u1")
|
||||
ws_id = ws.id
|
||||
mgr.close(ws_id)
|
||||
storage.touch_calls.clear() # only care about touches from rehydrate
|
||||
|
||||
reopened = mgr.open(ws_id)
|
||||
|
||||
assert reopened is not None
|
||||
assert ws_id in storage.touch_calls
|
||||
|
||||
|
||||
def test_open_ignores_owner_mismatch() -> None:
|
||||
# Turnstone is a trusted-team tool; row-level ownership is
|
||||
# metadata, not an access boundary. ``open`` no longer cares
|
||||
@@ -827,6 +923,197 @@ def test_close_idle_on_empty_manager_returns_empty_list() -> None:
|
||||
assert mgr.close_idle(max_age_seconds=1.0) == []
|
||||
|
||||
|
||||
def test_close_idle_runs_db_orphan_pass() -> None:
|
||||
"""DB rows of this kind that aren't loaded into the manager get
|
||||
bulk-closed when their ``updated`` is older than the cutoff. Catches
|
||||
the orphan-after-process-restart case the original close_idle missed."""
|
||||
mgr, _, storage = _make_manager()
|
||||
# Orphan rows live in storage but were never loaded via mgr.create.
|
||||
storage.register_workstream(
|
||||
"orphan-1",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"orphan-2",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
state="thinking",
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert set(closed) == {"orphan-1", "orphan-2"}
|
||||
assert ("orphan-1", "closed") in storage.state_updates
|
||||
assert ("orphan-2", "closed") in storage.state_updates
|
||||
assert storage.rows["orphan-1"].state == "closed"
|
||||
assert storage.rows["orphan-2"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_excludes_loaded_workstreams_from_db_pass() -> None:
|
||||
"""A workstream loaded into memory must NOT be reaped by the DB
|
||||
orphan pass even when its storage ``updated`` is stale — the
|
||||
in-memory pass owns those. Verifies the exclude_ws_ids plumbing."""
|
||||
mgr, _, storage = _make_manager()
|
||||
ws = mgr.create(user_id="u1")
|
||||
# Force the storage row's ``updated`` to look stale. In practice
|
||||
# ``set_state`` would bump it, but we're simulating a long-running
|
||||
# active workstream whose updated drifted older than the cutoff.
|
||||
storage.rows[ws.id].updated = "2020-01-01T00:00:00"
|
||||
|
||||
# Huge timeout so the in-memory IDLE pass skips it (stays loaded).
|
||||
closed = mgr.close_idle(max_age_seconds=10_000.0)
|
||||
|
||||
assert ws.id not in closed
|
||||
assert mgr.get(ws.id) is not None
|
||||
assert storage.rows[ws.id].state == "idle"
|
||||
|
||||
|
||||
def test_close_idle_filters_db_orphans_by_kind() -> None:
|
||||
"""An interactive manager's close_idle must not touch coordinator
|
||||
rows in storage and vice versa. Without this filter, both managers
|
||||
would race to close each other's rows."""
|
||||
mgr, _, storage = _make_manager() # interactive by default
|
||||
storage.register_workstream(
|
||||
"coord-orphan",
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"interactive-orphan",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert "interactive-orphan" in closed
|
||||
assert "coord-orphan" not in closed
|
||||
assert storage.rows["coord-orphan"].state == "idle"
|
||||
assert storage.rows["interactive-orphan"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_protects_rows_owned_by_live_services() -> None:
|
||||
"""Multi-node correctness: rows whose ``node_id`` matches a service
|
||||
with a recent heartbeat must NOT be reaped, even when *this* manager
|
||||
is on a different node — the alive peer may legitimately have them
|
||||
loaded. Liveness is the rendezvous router's primitive (post-PR-#384);
|
||||
using it here keeps reap scoping aligned with routing.
|
||||
|
||||
Default ``_make_manager`` uses an INTERACTIVE adapter, which derives
|
||||
``service_type='server'`` — so live_services seeded under "server"
|
||||
are what the manager queries."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.live_services["server"] = ["node-b"] # only node-b is alive
|
||||
storage.register_workstream(
|
||||
"ours-from-dead-node",
|
||||
node_id="node-a", # dead pod (not in live_services)
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"theirs-still-alive",
|
||||
node_id="node-b",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["ours-from-dead-node"]
|
||||
assert storage.rows["ours-from-dead-node"].state == "closed"
|
||||
assert storage.rows["theirs-still-alive"].state == "idle"
|
||||
|
||||
|
||||
def test_close_idle_protects_live_services_for_coordinator_kind() -> None:
|
||||
"""Coord-side parity: a coordinator manager derives
|
||||
``service_type='console'``, so live_services seeded under "console"
|
||||
are what gets queried. Mirrors the interactive test to ensure both
|
||||
halves of the production wiring are exercised."""
|
||||
coord_adapter = FakeAdapter(kind=WorkstreamKind.COORDINATOR)
|
||||
mgr, _, storage = _make_manager(coord_adapter)
|
||||
storage.live_services["console"] = ["console"] # console is alive
|
||||
storage.register_workstream(
|
||||
"alive-console-coord",
|
||||
node_id="console",
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"dead-console-coord",
|
||||
node_id="dead-console-instance", # not in live set
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["dead-console-coord"]
|
||||
assert storage.rows["alive-console-coord"].state == "idle"
|
||||
assert storage.rows["dead-console-coord"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_reaps_rows_with_null_node_id() -> None:
|
||||
"""A row with no ``node_id`` has no owner identity — age alone gates
|
||||
the reap. Defends against a NULL silently propagating through ``NOT
|
||||
IN (live)`` and protecting orphans forever."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.live_services["server"] = ["node-a"]
|
||||
storage.register_workstream(
|
||||
"no-owner",
|
||||
node_id=None,
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["no-owner"]
|
||||
|
||||
|
||||
def test_close_idle_reaps_all_orphans_when_no_peers_alive() -> None:
|
||||
"""When ``list_services`` returns an empty list (no heartbeating
|
||||
peers), every stale orphan is unprotected and gets reaped. This is
|
||||
the cold-start / single-process / dead-cluster-recovery case."""
|
||||
mgr, _, storage = _make_manager()
|
||||
# storage.live_services["server"] left empty — no peers heartbeating
|
||||
storage.register_workstream(
|
||||
"any-node-1",
|
||||
node_id="node-a",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"any-node-2",
|
||||
node_id="node-b",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert set(closed) == {"any-node-1", "any-node-2"}
|
||||
|
||||
|
||||
def test_close_idle_skips_pass_2_when_list_services_fails() -> None:
|
||||
"""Conservative fallback: if list_services fails we can't enumerate
|
||||
live owners safely, so pass 2 must skip rather than reap blind. Pass
|
||||
1 (in-memory IDLE) still runs."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.list_services_raises = True
|
||||
storage.register_workstream(
|
||||
"would-be-orphan",
|
||||
node_id="node-a",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == []
|
||||
assert storage.rows["would-be-orphan"].state == "idle"
|
||||
|
||||
|
||||
def test_list_all_returns_creation_order() -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
a = mgr.create(user_id="u1")
|
||||
@@ -1084,3 +1371,127 @@ class TestSessionManagerWithStateWriter:
|
||||
assert "running" not in ws_writes, (
|
||||
f"set_state after close enqueued through buffer: {ws_writes}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-subscriber observer — subscribe_to_state / unsubscribe_from_state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStateSubscribers:
|
||||
"""Multi-subscriber observer for ``set_state``.
|
||||
|
||||
Used by the CLI's background-attention notifier and by
|
||||
``SameNodeChildSource``. Subscribe / unsubscribe must be safe under
|
||||
concurrent dispatch, and dispatch must not skip / repeat callbacks
|
||||
when subscribers register or unregister mid-iteration.
|
||||
"""
|
||||
|
||||
def test_subscribe_fires_on_set_state(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
events: list[tuple[str, str]] = []
|
||||
|
||||
def cb(ws_id: str, state: WorkstreamState) -> None:
|
||||
events.append((ws_id, state.value))
|
||||
|
||||
mgr.subscribe_to_state(cb)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert events == [(ws.id, "running")]
|
||||
|
||||
def test_unsubscribe_stops_firing(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
events: list[str] = []
|
||||
|
||||
def cb(_ws_id: str, state: WorkstreamState) -> None:
|
||||
events.append(state.value)
|
||||
|
||||
mgr.subscribe_to_state(cb)
|
||||
mgr.unsubscribe_from_state(cb)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert events == []
|
||||
|
||||
def test_unsubscribe_unknown_is_noop(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
# Doesn't raise.
|
||||
mgr.unsubscribe_from_state(lambda *_: None)
|
||||
|
||||
def test_multiple_subscribers_fire_in_registration_order(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
order: list[int] = []
|
||||
|
||||
def make(i: int) -> Callable[[str, WorkstreamState], None]:
|
||||
def cb(_ws_id: str, _state: WorkstreamState) -> None:
|
||||
order.append(i)
|
||||
|
||||
return cb
|
||||
|
||||
mgr.subscribe_to_state(make(1))
|
||||
mgr.subscribe_to_state(make(2))
|
||||
mgr.subscribe_to_state(make(3))
|
||||
mgr.set_state(ws.id, WorkstreamState.IDLE)
|
||||
assert order == [1, 2, 3]
|
||||
|
||||
def test_subscriber_exception_does_not_block_others(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
survived: list[str] = []
|
||||
|
||||
def boom(*_: Any) -> None:
|
||||
raise RuntimeError("subscriber crash")
|
||||
|
||||
def good(_ws_id: str, state: WorkstreamState) -> None:
|
||||
survived.append(state.value)
|
||||
|
||||
mgr.subscribe_to_state(boom)
|
||||
mgr.subscribe_to_state(good)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert survived == ["running"]
|
||||
|
||||
@pytest.mark.parametrize("n_threads", [10, 50])
|
||||
def test_concurrent_subscribe(self, n_threads: int) -> None:
|
||||
"""Subscribe from many threads; all callbacks land in the list.
|
||||
|
||||
Validates the lock around mutation — without it the underlying
|
||||
list.append could lose entries under contention.
|
||||
"""
|
||||
mgr, _, _ = _make_manager()
|
||||
callbacks = [lambda *_, i=i: None for i in range(n_threads)]
|
||||
threads = [threading.Thread(target=mgr.subscribe_to_state, args=(cb,)) for cb in callbacks]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
# Snapshot under the lock to read the count safely.
|
||||
with mgr._state_subscribers_lock:
|
||||
assert len(mgr._state_subscribers) == n_threads
|
||||
|
||||
def test_subscribe_during_dispatch_does_not_corrupt_iteration(self) -> None:
|
||||
"""A subscriber that calls subscribe_to_state during its own
|
||||
callback must not affect the in-flight dispatch (snapshot
|
||||
isolation). This is the bug-1 invariant: mutation during
|
||||
iteration can't shift the iterator's index because dispatch
|
||||
iterates a snapshot, not the live list.
|
||||
"""
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
fired: list[str] = []
|
||||
|
||||
def late(_ws_id: str, state: WorkstreamState) -> None:
|
||||
fired.append("late:" + state.value)
|
||||
|
||||
def first(_ws_id: str, state: WorkstreamState) -> None:
|
||||
fired.append("first:" + state.value)
|
||||
mgr.subscribe_to_state(late) # mid-dispatch addition
|
||||
|
||||
mgr.subscribe_to_state(first)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
# ``late`` was added during dispatch but the snapshot was
|
||||
# already frozen — so it doesn't fire on this round.
|
||||
assert fired == ["first:running"]
|
||||
# Next round it does fire, in registration order after first.
|
||||
fired.clear()
|
||||
mgr.set_state(ws.id, WorkstreamState.IDLE)
|
||||
assert fired == ["first:idle", "late:idle"]
|
||||
|
||||
@@ -4,6 +4,10 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import workstreams
|
||||
|
||||
# -- Workstream registration ---------------------------------------------------
|
||||
|
||||
|
||||
@@ -664,6 +668,277 @@ class TestBatchPrimitives:
|
||||
assert result == {"never-seen": 0}
|
||||
|
||||
|
||||
# -- bulk_close_stale_orphans --------------------------------------------------
|
||||
|
||||
|
||||
def _force_updated(backend: Any, ws_id: str, updated: str) -> None:
|
||||
"""Stamp a workstream row's ``updated`` column directly.
|
||||
|
||||
The public surface only sets ``updated`` to ``now``, which makes it
|
||||
impossible to fabricate a stale row through register/update calls.
|
||||
Reaches into ``backend._engine`` — same access pattern conftest uses
|
||||
for cross-backend cleanup.
|
||||
"""
|
||||
with backend._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=updated)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
class TestBulkCloseStaleOrphans:
|
||||
def test_closes_stale_non_terminal_rows_of_kind(self, backend):
|
||||
backend.register_workstream("stale-idle", kind="interactive")
|
||||
backend.register_workstream("stale-thinking", kind="interactive")
|
||||
backend.update_workstream_state("stale-thinking", "thinking")
|
||||
backend.register_workstream("fresh-idle", kind="interactive")
|
||||
_force_updated(backend, "stale-idle", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "stale-thinking", "2020-01-01T00:00:00")
|
||||
# fresh-idle stays at registration time (effectively now)
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"stale-idle", "stale-thinking"}
|
||||
rows = backend.get_workstreams_batch(["stale-idle", "stale-thinking", "fresh-idle"])
|
||||
assert rows["stale-idle"]["state"] == "closed"
|
||||
assert rows["stale-thinking"]["state"] == "closed"
|
||||
assert rows["fresh-idle"]["state"] == "idle"
|
||||
|
||||
def test_skips_already_closed(self, backend):
|
||||
backend.register_workstream("already-closed", kind="interactive")
|
||||
backend.update_workstream_state("already-closed", "closed")
|
||||
_force_updated(backend, "already-closed", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == []
|
||||
|
||||
def test_filters_by_kind(self, backend):
|
||||
backend.register_workstream("interactive-stale", kind="interactive")
|
||||
backend.register_workstream("coord-stale", kind="coordinator")
|
||||
_force_updated(backend, "interactive-stale", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "coord-stale", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == ["interactive-stale"]
|
||||
rows = backend.get_workstreams_batch(["interactive-stale", "coord-stale"])
|
||||
assert rows["interactive-stale"]["state"] == "closed"
|
||||
assert rows["coord-stale"]["state"] == "idle"
|
||||
|
||||
def test_excludes_loaded_ws_ids(self, backend):
|
||||
backend.register_workstream("ws-keep", kind="interactive")
|
||||
backend.register_workstream("ws-close", kind="interactive")
|
||||
_force_updated(backend, "ws-keep", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "ws-close", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=["ws-keep"]
|
||||
)
|
||||
|
||||
assert closed == ["ws-close"]
|
||||
rows = backend.get_workstreams_batch(["ws-keep", "ws-close"])
|
||||
assert rows["ws-keep"]["state"] == "idle"
|
||||
assert rows["ws-close"]["state"] == "closed"
|
||||
|
||||
def test_empty_exclude_list_does_not_break_sql(self, backend):
|
||||
backend.register_workstream("orphan", kind="interactive")
|
||||
_force_updated(backend, "orphan", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == ["orphan"]
|
||||
|
||||
def test_no_orphans_returns_empty(self, backend):
|
||||
backend.register_workstream("fresh", kind="interactive")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == []
|
||||
|
||||
def test_closes_all_non_terminal_states(self, backend):
|
||||
for ws_id, state in [
|
||||
("o-idle", "idle"),
|
||||
("o-thinking", "thinking"),
|
||||
("o-attention", "attention"),
|
||||
("o-running", "running"),
|
||||
]:
|
||||
backend.register_workstream(ws_id, kind="interactive")
|
||||
if state != "idle":
|
||||
backend.update_workstream_state(ws_id, state)
|
||||
_force_updated(backend, ws_id, "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"o-idle", "o-thinking", "o-attention", "o-running"}
|
||||
|
||||
def test_bumps_updated_on_close(self, backend):
|
||||
stale_updated = "2020-01-01T00:00:00"
|
||||
backend.register_workstream("orphan", kind="interactive")
|
||||
_force_updated(backend, "orphan", stale_updated)
|
||||
|
||||
backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
# ``updated`` must change away from the forced stale value. Asserting
|
||||
# inequality from the seed (rather than ``> "2024-01-01..."``) keeps
|
||||
# the test independent of wall-clock date.
|
||||
with backend._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(workstreams.c.updated).where(workstreams.c.ws_id == "orphan")
|
||||
).one()
|
||||
assert row[0] != stale_updated
|
||||
|
||||
def test_protects_rows_owned_by_live_services(self, backend):
|
||||
"""Liveness scoping (post-#384 rendezvous-routing world): rows
|
||||
whose ``node_id`` matches a heartbeating service must NOT be
|
||||
reaped, because that owner may legitimately have them loaded on
|
||||
another worker. Rows whose ``node_id`` matches a dead service
|
||||
ARE eligible — that's how dead-pod orphans get reclaimed in
|
||||
containerized deployments with dynamic hostnames."""
|
||||
backend.register_workstream("dead-node", node_id="dead-pod-x4k2", kind="interactive")
|
||||
backend.register_workstream("alive-node", node_id="alive-pod-y9p3", kind="interactive")
|
||||
_force_updated(backend, "dead-node", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "alive-node", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=["alive-pod-y9p3"],
|
||||
)
|
||||
|
||||
assert closed == ["dead-node"]
|
||||
rows = backend.get_workstreams_batch(["dead-node", "alive-node"])
|
||||
assert rows["dead-node"]["state"] == "closed"
|
||||
assert rows["alive-node"]["state"] == "idle"
|
||||
|
||||
def test_null_node_id_always_eligible(self, backend):
|
||||
"""A row with NULL ``node_id`` has no owner identity — age alone
|
||||
gates the reap. Belt-and-suspenders against ``NULL NOT IN (...)``
|
||||
evaluating to NULL (not TRUE) and silently protecting orphans
|
||||
forever."""
|
||||
backend.register_workstream("no-owner", node_id=None, kind="interactive")
|
||||
_force_updated(backend, "no-owner", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=["some-other-node"],
|
||||
)
|
||||
|
||||
assert closed == ["no-owner"]
|
||||
|
||||
def test_live_node_ids_none_skips_filter(self, backend):
|
||||
"""``live_node_ids=None`` is the single-process / operator-backfill
|
||||
mode — all rows of *kind* are eligible regardless of node_id."""
|
||||
backend.register_workstream("node-a", node_id="node-a", kind="interactive")
|
||||
backend.register_workstream("node-b", node_id="node-b", kind="interactive")
|
||||
_force_updated(backend, "node-a", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "node-b", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"node-a", "node-b"}
|
||||
|
||||
def test_empty_live_node_ids_treats_all_as_dead(self, backend):
|
||||
"""Empty list ``live_node_ids=[]`` means "no nodes alive" — every
|
||||
row's owner is unprotected. Useful for operator scripts that
|
||||
want to reap regardless of liveness."""
|
||||
backend.register_workstream("any", node_id="node-a", kind="interactive")
|
||||
_force_updated(backend, "any", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=[],
|
||||
)
|
||||
|
||||
assert closed == ["any"]
|
||||
|
||||
def test_combines_live_node_ids_and_exclude_ws_ids(self, backend):
|
||||
"""Both filters stack as AND clauses on the UPDATE. Covers the
|
||||
full 2x2 matrix to catch a future edit that replaces an AND with
|
||||
an OR or drops one of the filters: only the (orphan + dead-node)
|
||||
cell should be reaped."""
|
||||
# All four registered with the same stale ``updated``.
|
||||
for ws_id, node in [
|
||||
("loaded-alive", "alive-node"),
|
||||
("loaded-dead", "dead-node"),
|
||||
("orphan-alive", "alive-node"),
|
||||
("orphan-dead", "dead-node"),
|
||||
]:
|
||||
backend.register_workstream(ws_id, node_id=node, kind="interactive")
|
||||
_force_updated(backend, ws_id, "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=["loaded-alive", "loaded-dead"],
|
||||
live_node_ids=["alive-node"],
|
||||
)
|
||||
|
||||
# Only orphan-dead is unprotected by both filters.
|
||||
assert closed == ["orphan-dead"]
|
||||
rows = backend.get_workstreams_batch(
|
||||
["loaded-alive", "loaded-dead", "orphan-alive", "orphan-dead"]
|
||||
)
|
||||
assert rows["loaded-alive"]["state"] == "idle"
|
||||
assert rows["loaded-dead"]["state"] == "idle"
|
||||
assert rows["orphan-alive"]["state"] == "idle"
|
||||
assert rows["orphan-dead"]["state"] == "closed"
|
||||
|
||||
|
||||
# -- touch_workstream ----------------------------------------------------------
|
||||
|
||||
|
||||
class TestTouchWorkstream:
|
||||
def test_bumps_updated_only(self, backend):
|
||||
"""Used by ``open()`` on rehydrate to defend against the orphan
|
||||
reaper clobbering a freshly-loaded row. Must not change ``state``
|
||||
(the open() path explicitly avoids state writes to dodge a race
|
||||
with concurrent close())."""
|
||||
stale_updated = "2020-01-01T00:00:00"
|
||||
backend.register_workstream("ws-touch", kind="interactive")
|
||||
backend.update_workstream_state("ws-touch", "closed") # simulate prior close
|
||||
_force_updated(backend, "ws-touch", stale_updated)
|
||||
|
||||
backend.touch_workstream("ws-touch")
|
||||
|
||||
with backend._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(workstreams.c.state, workstreams.c.updated).where(
|
||||
workstreams.c.ws_id == "ws-touch"
|
||||
)
|
||||
).one()
|
||||
assert row[0] == "closed", "state must not be modified by touch"
|
||||
# Compare against the forced stale value rather than a fixed calendar
|
||||
# date so the test is independent of wall-clock time.
|
||||
assert row[1] != stale_updated, "updated must be bumped"
|
||||
|
||||
def test_unknown_id_is_noop(self, backend):
|
||||
"""Touch on a missing id must not raise — open()'s exception
|
||||
handler is best-effort."""
|
||||
backend.touch_workstream("nonexistent") # must not raise
|
||||
|
||||
|
||||
# -- Lifecycle -----------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
+129
-37
@@ -157,20 +157,17 @@ class TestContentAccumulation:
|
||||
assert len(idle_events[0]["content"]) <= _MAX_TURN_CONTENT_CHARS + 1024
|
||||
|
||||
|
||||
class TestPendingApprovalDetailGate:
|
||||
"""The Shape A SSE plumbing carries ``pending_approval_detail`` on the
|
||||
``ws_state`` event so the coord tree UI can render inline approve/deny
|
||||
buttons in lockstep with the activity_state transition. The gate
|
||||
(``if self._pending_approval is not None``) keeps the per-broadcast
|
||||
serializer cost off the common no-approval-pending path — these tests
|
||||
lock both branches down."""
|
||||
class TestPendingApprovalDetailNotPiggybacked:
|
||||
"""Stage 3 cleanup — ``pending_approval_detail`` is no longer
|
||||
piggybacked on ``ws_state`` events. Approval items now arrive via
|
||||
bulk fetch when the coord tree's reducer sees the
|
||||
``activity_state="approval"`` transition; verdicts via the explicit
|
||||
``intent_verdict`` event class; resolution via
|
||||
``approval_resolved``. These tests lock the no-piggyback contract
|
||||
down so a future regression doesn't silently re-introduce the
|
||||
duplicated path."""
|
||||
|
||||
def test_state_broadcast_omits_field_when_no_approval_pending(self):
|
||||
"""Common case: no approval pending → field absent from event so the
|
||||
per-broadcast verdict-cache deepcopy in
|
||||
``serialize_pending_approval_detail`` never runs. A regression
|
||||
that drops the gate would silently 10x the cost of every state
|
||||
broadcast in the steady state."""
|
||||
ui = _make_ui()
|
||||
assert ui._pending_approval is None
|
||||
ui._broadcast_state("running")
|
||||
@@ -180,17 +177,12 @@ class TestPendingApprovalDetailGate:
|
||||
assert len(running_events) == 1
|
||||
assert "pending_approval_detail" not in running_events[0]
|
||||
|
||||
def test_state_broadcast_includes_field_when_approval_pending(self):
|
||||
"""When an approval is pending the broadcast must carry the rich
|
||||
payload — the coord tree UI reads it directly to render inline
|
||||
approve/deny buttons. Without this, a coord browser would have
|
||||
to chase a separate ``cluster/ws/live`` fetch on every
|
||||
activity_state transition (the load-storm pattern Shape A is
|
||||
unwinding)."""
|
||||
def test_state_broadcast_omits_field_even_when_approval_pending(self):
|
||||
"""The piggyback is gone: even when ``_pending_approval`` is set,
|
||||
the state broadcast must NOT carry ``pending_approval_detail``.
|
||||
The browser triggers a bulk fetch off the
|
||||
``activity_state="approval"`` transition to get the items."""
|
||||
ui = _make_ui()
|
||||
# Mirror the shape ``pause_for_approval`` writes (session_ui_base
|
||||
# lines 576-580) — items with call_id + header is the minimum
|
||||
# the serializer needs to project.
|
||||
ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
@@ -209,21 +201,9 @@ class TestPendingApprovalDetailGate:
|
||||
events = _drain_global()
|
||||
attn = [e for e in events if e.get("state") == "attention"]
|
||||
assert len(attn) == 1
|
||||
# Field present and structurally sound — the serializer's
|
||||
# full shape is covered by tests/test_session_ui_base.py;
|
||||
# here we only need to confirm the gate fires and the
|
||||
# serializer's output is what lands on the event.
|
||||
assert "pending_approval_detail" in attn[0]
|
||||
detail = attn[0]["pending_approval_detail"]
|
||||
assert detail is not None
|
||||
assert detail.get("items")
|
||||
assert detail["items"][0]["call_id"] == "c1"
|
||||
assert "pending_approval_detail" not in attn[0]
|
||||
|
||||
def test_field_cleared_after_approval_resolves(self):
|
||||
"""Once ``_pending_approval`` is cleared, subsequent state
|
||||
broadcasts must drop the field again — without this, the
|
||||
browser would render stale approve/deny buttons until the
|
||||
next bulk-poll TTL window expired."""
|
||||
def test_field_stays_absent_after_approval_resolves(self):
|
||||
ui = _make_ui()
|
||||
ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
@@ -231,7 +211,7 @@ class TestPendingApprovalDetailGate:
|
||||
"judge_pending": False,
|
||||
}
|
||||
ui._broadcast_state("attention")
|
||||
_drain_global() # discard the with-detail event
|
||||
_drain_global()
|
||||
|
||||
ui._pending_approval = None
|
||||
ui._broadcast_state("running")
|
||||
@@ -239,3 +219,115 @@ class TestPendingApprovalDetailGate:
|
||||
running = [e for e in events if e.get("state") == "running"]
|
||||
assert len(running) == 1
|
||||
assert "pending_approval_detail" not in running[0]
|
||||
|
||||
|
||||
class TestBroadcastIntentVerdict:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_intent_verdict``.
|
||||
|
||||
The collector-side test (``test_apply_delta_intent_verdict_*`` in
|
||||
test_console.py) covers consumption; this pins the event shape the
|
||||
producer puts on the global queue. A field rename or missed key
|
||||
here would slip past the consumer test because the consumer reads
|
||||
via ``data.get(...)``.
|
||||
"""
|
||||
|
||||
def test_pushes_intent_verdict_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.92,
|
||||
"recommendation": "approve",
|
||||
"reasoning": "tool reads only",
|
||||
}
|
||||
ui._broadcast_intent_verdict(verdict)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "intent_verdict"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["verdict"] == verdict
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
def test_queue_full_swallowed(self):
|
||||
# Force a tiny queue then fill it so the next put_nowait
|
||||
# raises queue.Full — the broadcast must absorb it without
|
||||
# propagating (matches _broadcast_state's queue.Full handling).
|
||||
WebUI._global_queue = queue.Queue(maxsize=1)
|
||||
WebUI._global_queue.put_nowait({"sentinel": True})
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
|
||||
class TestBroadcastApprovalResolved:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_approval_resolved``."""
|
||||
|
||||
def test_pushes_approval_resolved_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(True, "lgtm", always=False)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "approval_resolved"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["approved"] is True
|
||||
assert ev["feedback"] == "lgtm"
|
||||
assert ev["always"] is False
|
||||
|
||||
def test_normalises_none_feedback_to_empty_string(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(False, None)
|
||||
|
||||
events = _drain_global()
|
||||
assert events[0]["feedback"] == ""
|
||||
assert events[0]["approved"] is False
|
||||
assert events[0]["always"] is False
|
||||
|
||||
def test_always_kwarg_propagates(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(True, "ok", always=True)
|
||||
events = _drain_global()
|
||||
assert events[0]["always"] is True
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, None)
|
||||
|
||||
|
||||
class TestBroadcastApproveRequest:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_approve_request`` —
|
||||
push path for the initial approval items so a coord parent's tree
|
||||
UI can render the inline approve/deny block immediately without
|
||||
waiting for a bulk-fetch round-trip."""
|
||||
|
||||
def test_pushes_approve_request_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
ui._broadcast_approve_request(detail)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "approve_request"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["detail"] == detail
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
|
||||
@@ -899,6 +899,138 @@ class TestHistoryInteractive:
|
||||
assert client.get(base, params={"limit": 999}).status_code == 200
|
||||
|
||||
|
||||
class TestBuildHistoryReminderPropagation:
|
||||
"""``_build_history`` must surface the ``_reminders`` side-channel on
|
||||
each entry so a tab reconnecting via ``/history`` renders the same
|
||||
metacognitive nudge bubble the originating tab saw via the live
|
||||
``user_reminder`` SSE event.
|
||||
"""
|
||||
|
||||
def _session_with_messages(self, messages: list[dict]) -> MagicMock:
|
||||
session = MagicMock()
|
||||
session.messages = messages
|
||||
return session
|
||||
|
||||
def test_reminders_sidechannel_surfaces_on_entry(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "ah no",
|
||||
"_reminders": [{"type": "correction", "text": "watch out"}],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == "ah no"
|
||||
assert history[0]["reminders"] == [{"type": "correction", "text": "watch out"}]
|
||||
|
||||
def test_no_reminders_key_when_sidechannel_absent(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages([{"role": "user", "content": "just a message"}])
|
||||
history = _build_history(session)
|
||||
assert "reminders" not in history[0]
|
||||
|
||||
def test_no_reminders_key_when_sidechannel_empty(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages([{"role": "user", "content": "hi", "_reminders": []}])
|
||||
history = _build_history(session)
|
||||
assert "reminders" not in history[0]
|
||||
|
||||
def test_multiple_reminders_preserved_in_order(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "x",
|
||||
"_reminders": [
|
||||
{"type": "denial", "text": "FIRST"},
|
||||
{"type": "correction", "text": "SECOND"},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["reminders"] == [
|
||||
{"type": "denial", "text": "FIRST"},
|
||||
{"type": "correction", "text": "SECOND"},
|
||||
]
|
||||
|
||||
def test_reminders_coexist_with_attachments(self):
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "data:..."}},
|
||||
],
|
||||
"_reminders": [{"type": "correction", "text": "watch"}],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == "look"
|
||||
assert history[0]["attachments"] == [{"kind": "image", "filename": "", "mime_type": ""}]
|
||||
assert history[0]["reminders"] == [{"type": "correction", "text": "watch"}]
|
||||
|
||||
def test_malformed_reminders_filtered_out(self):
|
||||
"""Defensive: a non-dict element in the list (corruption / bug)
|
||||
is dropped rather than crashing the history serialisation."""
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "x",
|
||||
"_reminders": [
|
||||
{"type": "correction", "text": "ok"},
|
||||
"not-a-dict",
|
||||
{"type": "denial"}, # missing text
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
history = _build_history(session)
|
||||
# Non-dicts dropped; missing-text fills with empty string.
|
||||
assert history[0]["reminders"] == [
|
||||
{"type": "correction", "text": "ok"},
|
||||
{"type": "denial", "text": ""},
|
||||
]
|
||||
|
||||
def test_clean_message_passes_through_unchanged(self):
|
||||
"""No reminders, plain content — _build_history is a no-op for the
|
||||
reminder field and ``content`` rides through verbatim."""
|
||||
from turnstone.server import _build_history
|
||||
|
||||
session = self._session_with_messages(
|
||||
[{"role": "user", "content": "just a normal message"}]
|
||||
)
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == "just a normal message"
|
||||
assert "reminders" not in history[0]
|
||||
|
||||
def test_assistant_content_with_literal_reminder_tag_unchanged(self):
|
||||
"""Assistant output may legitimately reference the tag (e.g. when
|
||||
the model is explaining the reminder system itself). No
|
||||
transformation should ever apply to assistant content."""
|
||||
from turnstone.server import _build_history
|
||||
|
||||
content = "Here is a <system-reminder> tag in assistant output."
|
||||
session = self._session_with_messages([{"role": "assistant", "content": content}])
|
||||
history = _build_history(session)
|
||||
assert history[0]["content"] == content
|
||||
|
||||
|
||||
class TestDetailInteractive:
|
||||
"""Interactive parity for the lifted ``GET /v1/api/workstreams/{ws_id}``.
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "1.5.1"
|
||||
__version__ = "1.5.5"
|
||||
|
||||
+24
-1
@@ -312,6 +312,29 @@ class TerminalUI(SessionUI):
|
||||
sys.stdout.write(f"{RED}{message}{RESET}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
def _print_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
"""Render a metacognitive reminder list as ``[metacognition · type] text``
|
||||
lines in the terminal — the CLI's equivalent of the web UI's
|
||||
yellow themed bubble. Used by both ``on_user_reminder`` and
|
||||
``on_tool_reminder``; the rendering is identical because
|
||||
terminal output is anchor-by-flow rather than DOM-by-anchor.
|
||||
"""
|
||||
for r in reminders:
|
||||
nt = str(r.get("type", "") or "")
|
||||
text = str(r.get("text", "") or "")
|
||||
label = "metacognition" + (f" · {nt}" if nt else "")
|
||||
sys.stdout.write(f"{YELLOW}[{label}]{RESET} {text}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
self._print_reminder(reminders)
|
||||
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None:
|
||||
# tool_call_id ignored — the CLI anchors by output sequence
|
||||
# (the line lands directly after the tool result that
|
||||
# triggered the batch's reminder).
|
||||
self._print_reminder(reminders)
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
pass # base TerminalUI ignores state changes
|
||||
|
||||
@@ -1219,7 +1242,7 @@ def main() -> None:
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
manager._on_state_change = _bg_attention_notify
|
||||
manager.subscribe_to_state(_bg_attention_notify)
|
||||
|
||||
# Print banner
|
||||
print(f"\n{bold('Chat')} with {cyan(model)}")
|
||||
|
||||
+121
-14
@@ -498,7 +498,6 @@ class ClusterCollector:
|
||||
"kind": WorkstreamKind.from_raw(new_w.get("kind")),
|
||||
"parent_ws_id": new_w.get("parent_ws_id"),
|
||||
"activity_state": new_w.get("activity_state", ""),
|
||||
"pending_approval_detail": new_w.get("pending_approval_detail"),
|
||||
}
|
||||
)
|
||||
old_name = old_ws.get("title", "") or old_ws.get("name", "")
|
||||
@@ -558,18 +557,6 @@ class ClusterCollector:
|
||||
ws["kind"] = data["kind"]
|
||||
if "parent_ws_id" in data:
|
||||
ws["parent_ws_id"] = data["parent_ws_id"]
|
||||
# ``pending_approval_detail`` overwrites (no
|
||||
# ``ws.get`` fallback): the node's broadcast gate
|
||||
# on ``_pending_approval is not None`` means the
|
||||
# field is absent from ``data`` exactly when no
|
||||
# approval is pending — falling back to the cached
|
||||
# value would resurrect a stale detail after the
|
||||
# approval resolved. Without this assignment the
|
||||
# cached ``node.workstreams`` dict served by
|
||||
# ``get_node_detail`` / ``get_snapshot`` between
|
||||
# reconciliations would render stale approve/deny
|
||||
# buttons on closed approvals.
|
||||
ws["pending_approval_detail"] = data.get("pending_approval_detail")
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -581,7 +568,6 @@ class ClusterCollector:
|
||||
"kind": WorkstreamKind.from_raw(ws.get("kind")),
|
||||
"parent_ws_id": ws.get("parent_ws_id"),
|
||||
"activity_state": ws.get("activity_state", ""),
|
||||
"pending_approval_detail": data.get("pending_approval_detail"),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -648,6 +634,60 @@ class ClusterCollector:
|
||||
ws["name"] = name
|
||||
pending_events.append({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
elif etype == "intent_verdict":
|
||||
# Pure pass-through for LLM judge verdicts. The node's
|
||||
# WebUI._broadcast_intent_verdict pushes onto the
|
||||
# global queue (Stage 3 Step 5); the cluster fan-out
|
||||
# forwards to subscribers (e.g. CoordinatorAdapter,
|
||||
# which dispatches to the parent's coord SSE as
|
||||
# ``child_ws_intent_verdict``).
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"verdict": data.get("verdict") or {},
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "approval_resolved":
|
||||
# Pure pass-through for approve/deny resolutions. Pairs
|
||||
# with ``intent_verdict`` above so the parent
|
||||
# coordinator's tree UI can clear the pending-approval
|
||||
# pill the moment the user decides, rather than waiting
|
||||
# for the subsequent state-change piggyback.
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"approved": bool(data.get("approved", False)),
|
||||
"feedback": data.get("feedback", "") or "",
|
||||
"always": bool(data.get("always", False)),
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "approve_request":
|
||||
# Push path for the initial approval items. Eliminates
|
||||
# the bulk-fetch race that otherwise leaves the coord
|
||||
# tree stuck on a loading placeholder when the bulk
|
||||
# fetch lands in the gap between the state transition
|
||||
# to ATTENTION and ``_pending_approval`` being set.
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"detail": data.get("detail") or {},
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "health_changed":
|
||||
# Update the health dict's backend status in-place
|
||||
bstatus = data.get("backend_status", "")
|
||||
@@ -1217,3 +1257,70 @@ class ClusterCollector:
|
||||
return
|
||||
entry["name"] = name
|
||||
self._fanout({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
def emit_console_ws_intent_verdict(self, ws_id: str, verdict: dict[str, Any]) -> None:
|
||||
"""Fan an LLM intent-judge verdict for a console-pseudo-node ws.
|
||||
|
||||
Gives coord-spawned approval flows a first-class cluster-bus
|
||||
event so the parent's tree UI can render the risk pill +
|
||||
verdict result without polling.
|
||||
``CoordinatorAdapter._dispatch_child_event`` re-emits these as
|
||||
``child_ws_intent_verdict`` for the parent coordinator's SSE
|
||||
stream. The dispatch path filters by registry membership;
|
||||
downstream subscribers tolerate verdicts for ws_ids they don't
|
||||
own (silently drop), so we skip the membership pre-check that
|
||||
would otherwise add a lock acquisition per emit on a path
|
||||
that fires once per tool-call during heuristic+LLM judging.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
|
||||
def emit_console_ws_approval_resolved(
|
||||
self,
|
||||
ws_id: str,
|
||||
*,
|
||||
approved: bool,
|
||||
feedback: str = "",
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision for a console-pseudo-node ws.
|
||||
|
||||
Paired with :meth:`emit_console_ws_intent_verdict` so the
|
||||
coord tree UI clears the pending-approval pill in lockstep
|
||||
with the actual decision. Same lock-skip rationale as the
|
||||
intent-verdict emit above.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"approved": approved,
|
||||
"feedback": feedback,
|
||||
"always": always,
|
||||
}
|
||||
)
|
||||
|
||||
def emit_console_ws_approve_request(self, ws_id: str, detail: dict[str, Any]) -> None:
|
||||
"""Fan an ``approve_request`` payload for a console-pseudo-node ws.
|
||||
|
||||
Push path for the initial approval items so a coord parent's
|
||||
tree UI can render the inline approve/deny block immediately
|
||||
without a bulk-fetch round-trip. ``CoordinatorAdapter._dispatch_child_event``
|
||||
re-emits as ``child_ws_approve_request`` for the parent
|
||||
coordinator's SSE stream.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -15,17 +15,17 @@ for the storage-seeded children rebuild.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.core import session_worker
|
||||
from turnstone.core.adapters._ui_cleanup import cleanup_session_ui
|
||||
from turnstone.core.child_source import ClusterChildSource
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
@@ -56,38 +56,25 @@ class CoordinatorAdapter:
|
||||
# the manager to the adapter's ``__init__`` — break the cycle
|
||||
# with a setter called from console startup.
|
||||
self._manager: SessionManager | None = None
|
||||
# Per-coordinator known-child ws_id set. Populated lazily on
|
||||
# create/open from storage and updated live as the cluster fan-out
|
||||
# thread sees ws_created events with matching parent_ws_id.
|
||||
# Closed / deleted children stay in the registry so the tree UI
|
||||
# can keep rendering them grayed out; the authoritative render
|
||||
# path reads storage for state. Bounded by eventual coordinator
|
||||
# close/eviction.
|
||||
self._children: dict[str, set[str]] = {}
|
||||
# Reverse index for O(1) child → coord lookup on every cluster
|
||||
# event. Without this, every cluster event incurs a linear scan
|
||||
# over every coordinator's child set while holding the fan-out
|
||||
# lock — a hot-path tax that scales with both active coordinators
|
||||
# and their retained-history depth.
|
||||
self._child_to_coord: dict[str, str] = {}
|
||||
self._children_lock = threading.Lock()
|
||||
# Cluster-event fan-out: subscribes to the ClusterCollector's
|
||||
# listener channel, filters by known child ws_ids, and re-emits
|
||||
# child_ws_* events on the matching coordinator's UI. Configured
|
||||
# lazily via ``start_child_event_fanout(collector)`` from the
|
||||
# console lifespan once both the manager and collector exist.
|
||||
self._collector_queue: queue.Queue[dict[str, Any]] | None = None
|
||||
self._fanout_thread: threading.Thread | None = None
|
||||
self._fanout_stop = threading.Event()
|
||||
# Coord-ws-id → UI map for the fan-out dispatch path. Read
|
||||
# and written under ``self._children_lock`` alongside the
|
||||
# forward/reverse child maps. (Previously the value was
|
||||
# ``(user_id, ui)`` with a copy-on-write dict swap so the
|
||||
# dispatch could read it lock-free — but _dispatch_child_event
|
||||
# already re-validates the parent under _children_lock anyway,
|
||||
# so the lock-free snapshot was premature. The user_id half is
|
||||
# also dead after a46dab1 dropped row-level ownership gates.)
|
||||
self._active_coords: dict[str, Any] = {}
|
||||
# Children registry — universal parent → children + reverse
|
||||
# lookup primitive. Lifted from inline data on this adapter to
|
||||
# ``turnstone.core.children_registry.ChildrenRegistry`` (Stage 3
|
||||
# Step 1) so the same primitive can serve interactive
|
||||
# workstreams when they gain spawn capability. The dispatch
|
||||
# path (`_dispatch_child_event`) calls into the registry
|
||||
# atomically; everything else delegates through the legacy
|
||||
# method names which still exist as thin shims for the
|
||||
# cluster-routing + cleanup callers.
|
||||
self._registry = ChildrenRegistry()
|
||||
# Cross-node child events arrive via ``ClusterChildSource``
|
||||
# (Stage 3 Step 2): a strategy that subscribes to the
|
||||
# collector's listener channel and runs a daemon thread that
|
||||
# drains the queue, pushing each event to the sink. The sink
|
||||
# is :meth:`_dispatch_child_event` so the existing translation
|
||||
# logic (cluster_state → child_ws_state, etc.) stays in one
|
||||
# place. Constructed lazily by ``start_child_event_fanout`` so
|
||||
# the collector reference is available.
|
||||
self._child_source: ClusterChildSource | None = None
|
||||
|
||||
def attach(self, manager: SessionManager) -> None:
|
||||
"""Late-bind the owning :class:`SessionManager`.
|
||||
@@ -109,7 +96,7 @@ class CoordinatorAdapter:
|
||||
# needs the empty forward/presence entries so
|
||||
# ``_dispatch_child_event`` recognises this coordinator when
|
||||
# its first child is spawned.
|
||||
self._install_coord_registry(ws)
|
||||
self._registry.install(ws.id, ws.ui)
|
||||
self._fanout_console_ws_created(ws)
|
||||
|
||||
def emit_rehydrated(self, ws: Workstream) -> None:
|
||||
@@ -117,21 +104,10 @@ class CoordinatorAdapter:
|
||||
# it from storage after the registry seed so a ``ws_created``
|
||||
# for an already-spawned child that fires mid-rebuild merges
|
||||
# cleanly.
|
||||
self._install_coord_registry(ws)
|
||||
self._registry.install(ws.id, ws.ui)
|
||||
self._rebuild_children_registry(ws.id)
|
||||
self._fanout_console_ws_created(ws)
|
||||
|
||||
def _install_coord_registry(self, ws: Workstream) -> None:
|
||||
"""Seed the children registry + presence map for ``ws``.
|
||||
|
||||
Shared by ``emit_created`` and ``emit_rehydrated`` — the
|
||||
difference between the two is purely whether we then rebuild
|
||||
from storage.
|
||||
"""
|
||||
with self._children_lock:
|
||||
self._children.setdefault(ws.id, set())
|
||||
self._active_coords[ws.id] = ws.ui
|
||||
|
||||
def _fanout_console_ws_created(self, ws: Workstream) -> None:
|
||||
try:
|
||||
self._collector.emit_console_ws_created(
|
||||
@@ -197,14 +173,11 @@ class CoordinatorAdapter:
|
||||
# toast only fire for real-node (interactive) ws_closed events.
|
||||
del reason, name
|
||||
# Drop the coordinator's children-registry entries AND its
|
||||
# presence slot. Mirrors the eviction/close paths from the old
|
||||
# CoordinatorManager (which did the same under _children_lock
|
||||
# + _lock respectively). A plain _children.pop without clearing
|
||||
# the reverse index would leak every evicted coordinator's
|
||||
# child→parent pointers forever.
|
||||
with self._children_lock:
|
||||
self._pop_coord_registry_locked(ws_id)
|
||||
self._active_coords.pop(ws_id, None)
|
||||
# presence slot. A plain pop without clearing the reverse
|
||||
# index would leak every evicted coordinator's child→parent
|
||||
# pointers forever — :meth:`ChildrenRegistry.uninstall`
|
||||
# handles forward set + reverse entries + presence atomically.
|
||||
self._registry.uninstall(ws_id)
|
||||
try:
|
||||
self._collector.emit_console_ws_closed(ws_id)
|
||||
except Exception:
|
||||
@@ -356,25 +329,6 @@ class CoordinatorAdapter:
|
||||
# Children registry
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_child_ids_locked(self, coord_ws_id: str, child_ids: Iterable[str]) -> None:
|
||||
"""Merge ``child_ids`` into ``coord_ws_id``'s forward + reverse maps.
|
||||
|
||||
Caller MUST hold ``self._children_lock``. Idempotent — re-adding
|
||||
an existing child is a no-op (the reverse-index pointer is
|
||||
already correct). Empty / falsy entries in ``child_ids`` are
|
||||
skipped.
|
||||
|
||||
Sole write-path for bulk registry updates so
|
||||
``_rebuild_children_registry`` (storage-seeded) and
|
||||
``_prime_children_from_snapshot`` (collector-seeded) agree on
|
||||
ordering and reverse-index invariants.
|
||||
"""
|
||||
existing = self._children.setdefault(coord_ws_id, set())
|
||||
for cid in child_ids:
|
||||
if cid and cid not in existing:
|
||||
existing.add(cid)
|
||||
self._child_to_coord[cid] = coord_ws_id
|
||||
|
||||
def _rebuild_children_registry(self, coord_ws_id: str) -> None:
|
||||
"""Populate ``self._children[coord_ws_id]`` from storage.
|
||||
|
||||
@@ -445,174 +399,88 @@ class CoordinatorAdapter:
|
||||
if not child_id:
|
||||
continue
|
||||
child_ids.append(child_id)
|
||||
with self._children_lock:
|
||||
self._merge_child_ids_locked(coord_ws_id, child_ids)
|
||||
|
||||
def _coord_for_child(self, child_ws_id: str) -> str | None:
|
||||
"""Reverse-lookup: which coordinator owns this child ws_id?
|
||||
|
||||
O(1) via the ``_child_to_coord`` reverse index. Cluster events
|
||||
fire on every token tick across the cluster; a linear scan here
|
||||
turned into a hot-path tax as the retained-history set grew.
|
||||
"""
|
||||
with self._children_lock:
|
||||
return self._child_to_coord.get(child_ws_id)
|
||||
self._registry.merge_children(coord_ws_id, child_ids)
|
||||
|
||||
def children_snapshot(self, coord_ws_id: str) -> list[str]:
|
||||
"""Return a snapshot of the coordinator's direct child ws_ids.
|
||||
|
||||
Used by ``stop_cascade`` to iterate children without holding the
|
||||
registry lock during the per-child HTTP dispatch. A mutation
|
||||
racing with the snapshot (child spawned mid-cascade) either
|
||||
lands before the snapshot and gets cancelled, or lands after
|
||||
and is out of scope for this batch — both outcomes are safe.
|
||||
Returns an empty list for unknown coordinators.
|
||||
Used by ``stop_cascade`` to iterate children without holding
|
||||
the registry lock during the per-child HTTP dispatch. A
|
||||
mutation racing with the snapshot (child spawned mid-cascade)
|
||||
either lands before (cancelled) or after (out of scope for
|
||||
this batch) — both safe. Returns an empty list for unknown
|
||||
coordinators.
|
||||
"""
|
||||
with self._children_lock:
|
||||
child_set = self._children.get(coord_ws_id)
|
||||
return list(child_set) if child_set else []
|
||||
|
||||
def _pop_coord_registry_locked(self, coord_ws_id: str) -> None:
|
||||
"""Remove a coordinator's forward set + reverse-index entries.
|
||||
|
||||
Caller MUST hold ``self._children_lock``. Used by close /
|
||||
eviction paths so stale coordinators don't leak registry
|
||||
entries. No-op if the coordinator is unknown.
|
||||
"""
|
||||
child_set = self._children.pop(coord_ws_id, None)
|
||||
if child_set is None:
|
||||
return
|
||||
for cid in child_set:
|
||||
# Defensive: only clear the reverse entry if it still points
|
||||
# at THIS coordinator. If a child has since been reassigned
|
||||
# (unusual but possible on schema changes), we don't want to
|
||||
# orphan the new owner's entry.
|
||||
if self._child_to_coord.get(cid) == coord_ws_id:
|
||||
self._child_to_coord.pop(cid, None)
|
||||
return self._registry.children_of(coord_ws_id)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cluster-event fan-out thread
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def start_child_event_fanout(self, collector: ClusterCollector) -> None:
|
||||
"""Subscribe to cluster events and start the filter + re-emit thread.
|
||||
"""Subscribe to cluster events via :class:`ClusterChildSource`.
|
||||
|
||||
Idempotent — calling twice is a no-op (already-started fan-out
|
||||
thread stays). Called once from the console lifespan after both
|
||||
the collector and the session manager are constructed.
|
||||
Idempotent — already-started ChildSource stays. Called once
|
||||
from the console lifespan after both the collector and the
|
||||
session manager are constructed.
|
||||
"""
|
||||
if self._fanout_thread is not None and self._fanout_thread.is_alive():
|
||||
if self._child_source is not None:
|
||||
return
|
||||
self._collector = collector
|
||||
self._collector_queue = queue.Queue(maxsize=1000)
|
||||
# Ensure the "console" pseudo-node exists in the snapshot map so
|
||||
# emit_console_ws_* calls from create / close / open land on a
|
||||
# real node entry the snapshot will surface.
|
||||
# Coord-specific transport setup: the "console" pseudo-node
|
||||
# must exist in the snapshot map BEFORE any
|
||||
# ``emit_console_ws_*`` calls (and before the snapshot is
|
||||
# taken inside ``ChildSource.start``) so those emits land on
|
||||
# a real node entry the snapshot surfaces.
|
||||
collector.ensure_console_pseudo_node()
|
||||
# Register with the collector — use the existing listener channel
|
||||
# the browser SSE fan-out uses; the collector treats our queue as
|
||||
# just another subscriber.
|
||||
snapshot = collector.get_snapshot_and_register(self._collector_queue)
|
||||
# Prime the child registry from the snapshot so a coordinator
|
||||
# that opens right after a console restart sees already-live
|
||||
# children without waiting for the next ``ws_state`` tick to
|
||||
# discover them via the fan-out path.
|
||||
self._prime_children_from_snapshot(snapshot)
|
||||
# Seed the pseudo-node with any coordinators already loaded in
|
||||
# memory when the collector binds. Prevents a race where early
|
||||
# creates happened before the collector was wired up and their
|
||||
# rows never showed on the snapshot.
|
||||
mgr = self._manager
|
||||
if mgr is not None:
|
||||
for ws in mgr.list_all():
|
||||
try:
|
||||
collector.emit_console_ws_created(
|
||||
ws.id,
|
||||
name=ws.name,
|
||||
user_id=ws.user_id or "",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=ws.state.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_adapter.collector_seed_failed ws=%s",
|
||||
ws.id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
self._fanout_stop.clear()
|
||||
t = threading.Thread(
|
||||
target=self._fanout_loop,
|
||||
name="coord-adapter-child-fanout",
|
||||
daemon=True,
|
||||
)
|
||||
self._fanout_thread = t
|
||||
t.start()
|
||||
|
||||
def _prime_children_from_snapshot(self, snapshot: dict[str, Any]) -> None:
|
||||
"""Populate ``_children`` + ``_child_to_coord`` from a collector snapshot.
|
||||
|
||||
The snapshot's per-node workstreams carry ``parent_ws_id``. For
|
||||
every workstream whose parent is an in-memory coordinator,
|
||||
record the child so the fan-out filter sees it immediately.
|
||||
"""
|
||||
nodes = snapshot.get("nodes", []) if isinstance(snapshot, dict) else []
|
||||
if not nodes:
|
||||
return
|
||||
mgr = self._manager
|
||||
if mgr is None:
|
||||
raise RuntimeError(
|
||||
"CoordinatorAdapter: manager not attached — call attach(mgr) after construction"
|
||||
)
|
||||
by_parent: dict[str, list[str]] = {}
|
||||
known = {ws.id for ws in mgr.list_all()}
|
||||
for node in nodes:
|
||||
for entry in node.get("workstreams", []) or []:
|
||||
parent = entry.get("parent_ws_id") or ""
|
||||
child_id = entry.get("id") or ""
|
||||
if not parent or not child_id or parent not in known:
|
||||
continue
|
||||
by_parent.setdefault(parent, []).append(child_id)
|
||||
if not by_parent:
|
||||
return
|
||||
with self._children_lock:
|
||||
for parent, kids in by_parent.items():
|
||||
self._merge_child_ids_locked(parent, kids)
|
||||
# Build + start the strategy. The sink is
|
||||
# :meth:`_dispatch_child_event` so cluster events flow through
|
||||
# the same translation path that synthesises ``child_ws_*``
|
||||
# payloads for the parent's UI.
|
||||
source = ClusterChildSource(
|
||||
collector=collector,
|
||||
registry=self._registry,
|
||||
parents_provider=lambda: [ws.id for ws in mgr.list_all()],
|
||||
)
|
||||
source.start(sink=self._dispatch_child_event)
|
||||
self._child_source = source
|
||||
# Seed the pseudo-node with any coordinators already loaded in
|
||||
# memory when the collector binds. Prevents a race where early
|
||||
# creates happened before the collector was wired up and their
|
||||
# rows never showed on the snapshot. (Coord-specific — interactive
|
||||
# has no analogous pseudo-node.)
|
||||
for ws in mgr.list_all():
|
||||
try:
|
||||
collector.emit_console_ws_created(
|
||||
ws.id,
|
||||
name=ws.name,
|
||||
user_id=ws.user_id or "",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=ws.state.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_adapter.collector_seed_failed ws=%s",
|
||||
ws.id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the fan-out thread and unregister from the collector.
|
||||
"""Stop the ChildSource and unregister from the collector.
|
||||
|
||||
Safe to call multiple times; idempotent. Invoked from the
|
||||
console lifespan teardown so SSE listener queues don't leak.
|
||||
"""
|
||||
self._fanout_stop.set()
|
||||
t = self._fanout_thread
|
||||
q = self._collector_queue
|
||||
coll = self._collector
|
||||
self._fanout_thread = None
|
||||
self._collector_queue = None
|
||||
if coll is not None and q is not None:
|
||||
try:
|
||||
coll.unregister_listener(q)
|
||||
except Exception:
|
||||
log.debug("coord_adapter.unregister_listener_failed", exc_info=True)
|
||||
if t is not None:
|
||||
t.join(timeout=2.0)
|
||||
|
||||
def _fanout_loop(self) -> None:
|
||||
"""Drain collector events, filter by known children, dispatch."""
|
||||
q = self._collector_queue
|
||||
if q is None:
|
||||
return
|
||||
while not self._fanout_stop.is_set():
|
||||
try:
|
||||
event = q.get(timeout=1.0)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
self._dispatch_child_event(event)
|
||||
except Exception:
|
||||
log.debug("coord_adapter.fanout.dispatch_failed", exc_info=True)
|
||||
src = self._child_source
|
||||
self._child_source = None
|
||||
if src is not None:
|
||||
src.shutdown()
|
||||
|
||||
def _dispatch_child_event(self, event: dict[str, Any]) -> None:
|
||||
"""Match a cluster event to a coordinator and re-emit on its UI.
|
||||
@@ -622,10 +490,12 @@ class CoordinatorAdapter:
|
||||
- ``ws_created`` with ``parent_ws_id`` matching an in-memory
|
||||
coordinator → add to registry + re-emit as
|
||||
``child_ws_created``.
|
||||
- ``cluster_state`` / ``ws_closed`` / ``ws_rename`` whose
|
||||
``ws_id`` is in any coordinator's known-children registry →
|
||||
re-emit as ``child_ws_state`` / ``child_ws_closed`` /
|
||||
``child_ws_rename``.
|
||||
- ``cluster_state`` / ``ws_closed`` / ``ws_rename`` /
|
||||
``intent_verdict`` / ``approval_resolved`` whose ``ws_id``
|
||||
is in any coordinator's known-children registry → re-emit as
|
||||
``child_ws_state`` / ``child_ws_closed`` /
|
||||
``child_ws_rename`` / ``child_ws_intent_verdict`` /
|
||||
``child_ws_approval_resolved``.
|
||||
|
||||
Events for ws_ids we don't own silently drop — the filter lives
|
||||
on the server so each coordinator's SSE stream stays small.
|
||||
@@ -639,23 +509,17 @@ class CoordinatorAdapter:
|
||||
parent = event.get("parent_ws_id") or ""
|
||||
if not parent:
|
||||
return
|
||||
# Presence check + registry mutation under the same lock:
|
||||
# a concurrent close()/eviction can pop the entry between
|
||||
# the check and the mutation, after which a bare setdefault
|
||||
# would resurrect the entry — leaking the registry key and
|
||||
# enqueuing onto the closed coordinator's UI. Trusted-team
|
||||
# posture (#400 / a46dab1) means no per-event tenant gate
|
||||
# here; scope-level auth at the SSE endpoint is the only
|
||||
# boundary.
|
||||
with self._children_lock:
|
||||
coord_ui = self._active_coords.get(parent)
|
||||
if coord_ui is None:
|
||||
return
|
||||
existing = self._children.setdefault(parent, set())
|
||||
if ws_id in existing:
|
||||
return
|
||||
existing.add(ws_id)
|
||||
self._child_to_coord[ws_id] = parent
|
||||
# Atomic check-and-route under the registry's lock: a
|
||||
# concurrent close()/eviction can pop the parent's entry
|
||||
# between the presence check and the mutation, so the two
|
||||
# must happen together. ``add_child`` returns the parent's
|
||||
# UI on success or None on (a) parent not installed, or
|
||||
# (b) duplicate child. Trusted-team posture (#400 / a46dab1)
|
||||
# means no per-event tenant gate here; scope-level auth at
|
||||
# the SSE endpoint is the only boundary.
|
||||
coord_ui = self._registry.add_child(parent, ws_id)
|
||||
if coord_ui is None:
|
||||
return
|
||||
payload = {
|
||||
"type": "child_ws_created",
|
||||
"ws_id": ws_id,
|
||||
@@ -668,8 +532,15 @@ class CoordinatorAdapter:
|
||||
_enqueue_on_ui(coord_ui, parent, payload)
|
||||
return
|
||||
|
||||
if etype in ("cluster_state", "ws_closed", "ws_rename"):
|
||||
coord_id = self._coord_for_child(ws_id)
|
||||
if etype in (
|
||||
"cluster_state",
|
||||
"ws_closed",
|
||||
"ws_rename",
|
||||
"intent_verdict",
|
||||
"approval_resolved",
|
||||
"approve_request",
|
||||
):
|
||||
coord_id = self._registry.parent_for(ws_id)
|
||||
if coord_id is None:
|
||||
return
|
||||
mgr = self._manager
|
||||
@@ -684,14 +555,13 @@ class CoordinatorAdapter:
|
||||
"state": event.get("state", ""),
|
||||
"tokens": event.get("tokens", 0),
|
||||
"node_id": event.get("node_id", ""),
|
||||
# activity_state lets the JS detect approval-state
|
||||
# transitions; pending_approval_detail rides on
|
||||
# the same event so the browser can mutate
|
||||
# liveBadgeCache directly and render inline
|
||||
# approve/deny buttons in lockstep with the
|
||||
# transition, no separate dashboard refetch.
|
||||
# ``activity_state`` lets the JS detect approval
|
||||
# transitions and trigger a bulk fetch for the
|
||||
# initial detail. The detail itself no longer
|
||||
# piggybacks here (Stage 3 cleanup) — verdicts
|
||||
# arrive via the explicit ``intent_verdict`` event
|
||||
# class and resolution via ``approval_resolved``.
|
||||
"activity_state": event.get("activity_state", ""),
|
||||
"pending_approval_detail": event.get("pending_approval_detail"),
|
||||
}
|
||||
elif etype == "ws_closed":
|
||||
child_event = {
|
||||
@@ -700,13 +570,52 @@ class CoordinatorAdapter:
|
||||
"parent_ws_id": coord_id,
|
||||
"reason": event.get("reason", ""),
|
||||
}
|
||||
else: # ws_rename
|
||||
elif etype == "ws_rename":
|
||||
child_event = {
|
||||
"type": "child_ws_rename",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"name": event.get("name", ""),
|
||||
}
|
||||
elif etype == "intent_verdict":
|
||||
# Per-coord re-emit of an explicit verdict event so
|
||||
# the tree UI can render the risk pill + verdict
|
||||
# result without polling. The corresponding
|
||||
# ``cluster_state`` event no longer carries the
|
||||
# detail (the piggyback was removed end-to-end);
|
||||
# the bulk fetch on initial approval entry plus this
|
||||
# explicit event class are the only carriers.
|
||||
child_event = {
|
||||
"type": "child_ws_intent_verdict",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"verdict": event.get("verdict") or {},
|
||||
}
|
||||
elif etype == "approval_resolved":
|
||||
child_event = {
|
||||
"type": "child_ws_approval_resolved",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"approved": bool(event.get("approved", False)),
|
||||
"feedback": event.get("feedback", "") or "",
|
||||
"always": bool(event.get("always", False)),
|
||||
}
|
||||
else: # approve_request
|
||||
# Push path for the initial approval items —
|
||||
# eliminates the bulk-fetch race that previously left
|
||||
# the coord row stuck on a loading placeholder when
|
||||
# the bulk fetch landed in the gap between the state
|
||||
# transition to ATTENTION and ``_pending_approval``
|
||||
# being set inside ``approve_tools``.
|
||||
child_event = {
|
||||
"type": "child_ws_approve_request",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"detail": event.get("detail") or {},
|
||||
}
|
||||
_enqueue_on_ui(owning_ws.ui, coord_id, child_event)
|
||||
|
||||
|
||||
|
||||
@@ -181,6 +181,81 @@ class ConsoleCoordinatorUI(SessionUIBase):
|
||||
with self._ws_lock:
|
||||
self._last_broadcast_activity = current
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
"""Fan an LLM intent-judge verdict to the cluster collector.
|
||||
|
||||
Stage 3 Step 5 — overrides the no-op base hook so a coord
|
||||
workstream that produces its own verdict (rare — coord agents
|
||||
don't typically run the LLM judge) gets a first-class
|
||||
cluster-bus event for the parent's tree UI. The far more
|
||||
common path is a CHILD workstream firing its verdict on a
|
||||
real node; that path goes through ``WebUI._broadcast_intent_verdict``
|
||||
→ global SSE → collector ``_apply_delta``.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_intent_verdict(self.ws_id, verdict)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.intent_verdict_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool,
|
||||
feedback: str | None = None,
|
||||
*,
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision to the cluster collector.
|
||||
|
||||
Stage 3 Step 5 — paired with :meth:`_broadcast_intent_verdict`.
|
||||
Same rationale: coord-direct approvals are rare; the typical
|
||||
path is a child workstream resolving on its node, with that
|
||||
node's WebUI broadcasting through the global queue.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_approval_resolved(
|
||||
self.ws_id,
|
||||
approved=approved,
|
||||
feedback=feedback or "",
|
||||
always=always,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.approval_resolved_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None:
|
||||
"""Fan an ``approve_request`` payload to the cluster collector.
|
||||
|
||||
Push path for the initial approval items. Same rationale as
|
||||
the other two broadcast hooks: coord-self approvals are rare
|
||||
(the LLM judge isn't wired on the console coord today), but
|
||||
the override exists for symmetry and lights up the same path
|
||||
a future coord-self gate would use.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_approve_request(self.ws_id, detail)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.approve_request_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
# Flow state transitions through the unified SessionManager so
|
||||
# the storage write + adapter emit_state fan-out stay in lockstep
|
||||
|
||||
@@ -23,6 +23,7 @@ import queue
|
||||
import re
|
||||
import secrets
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -89,6 +90,7 @@ if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
|
||||
from turnstone.core.session import ChatSession
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
from turnstone.core.storage._protocol import StorageBackend
|
||||
|
||||
log = logging.getLogger("turnstone.console.server")
|
||||
@@ -844,8 +846,20 @@ async def _fetch_live_block(
|
||||
live = {k: entry.get(k) for k in _CLUSTER_WS_LIVE_KEYS if k in entry}
|
||||
# Derived field — kept in lockstep with
|
||||
# _coordinator_live_snapshot so both origins produce the
|
||||
# same keys.
|
||||
live["pending_approval"] = live.get("activity_state") == "approval"
|
||||
# same keys. ``state="attention"`` is the canonical signal;
|
||||
# ``activity_state="approval"`` is set inside approve_tools
|
||||
# AFTER the state transition fires, so a bulk fetch that
|
||||
# races with that window can see state=attention and
|
||||
# activity_state="" simultaneously. A non-null
|
||||
# ``pending_approval_detail`` is also a definitive signal
|
||||
# (the serializer only emits non-None when ``_pending_approval``
|
||||
# is set on the UI). Any of the three flips this true; the
|
||||
# frontend reducer mirrors the same disjunction.
|
||||
live["pending_approval"] = (
|
||||
live.get("activity_state") == "approval"
|
||||
or entry.get("state") == "attention"
|
||||
or live.get("pending_approval_detail") is not None
|
||||
)
|
||||
return live
|
||||
return None
|
||||
|
||||
@@ -3687,6 +3701,50 @@ async def _verify_collector_service_scope(app: Starlette, client: httpx.AsyncCli
|
||||
)
|
||||
|
||||
|
||||
def _coord_idle_cleanup_thread(
|
||||
mgr: SessionManager,
|
||||
timeout_sec: float,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> None:
|
||||
"""Periodically reap idle + DB-orphan coordinator workstreams.
|
||||
|
||||
Mirrors the regular server's ``_idle_cleanup_thread`` (turnstone/server.py)
|
||||
but skips the rate-limiter / global-queue arms — the console doesn't have
|
||||
those. ``mgr.close_idle`` does the work: closes loaded IDLE rows AND
|
||||
bulk-closes DB rows of this kind whose ``updated`` is past the cutoff
|
||||
and which aren't currently loaded. The latter pass catches coords left
|
||||
behind by prior console process incarnations.
|
||||
|
||||
Runs an initial sweep BEFORE the first sleep so cold-start orphans are
|
||||
reaped immediately rather than waiting one ``check_every`` interval (~30
|
||||
min on default 2h timeout). This intentionally diverges from the regular
|
||||
server pattern, which has no initial sweep — the regular server runs
|
||||
inside a normal request-handling lifecycle, the console-side coord pool
|
||||
is a small fixed-size cache where orphans dominate the row count after
|
||||
a cold boot.
|
||||
|
||||
``stop_event`` is for tests — when set, the thread exits cleanly after
|
||||
the next loop check. Production callers pass ``None`` (the daemon is
|
||||
process-lifetime).
|
||||
"""
|
||||
check_every = min(300.0, timeout_sec / 4)
|
||||
# Initial sweep — runs once before entering the sleep loop.
|
||||
try:
|
||||
mgr.close_idle(timeout_sec)
|
||||
except Exception:
|
||||
log.debug("console.coord_idle_cleanup_initial_failed", exc_info=True)
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
return
|
||||
time.sleep(check_every)
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
return
|
||||
try:
|
||||
mgr.close_idle(timeout_sec)
|
||||
except Exception:
|
||||
log.debug("console.coord_idle_cleanup_failed", exc_info=True)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
# Create async HTTP clients for proxy routes. Auth headers are NOT baked
|
||||
@@ -3992,6 +4050,27 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
coord_adapter.start_child_event_fanout(app.state.collector)
|
||||
except Exception:
|
||||
log.warning("console.coordinator_child_fanout_init_failed", exc_info=True)
|
||||
# Idle cleanup: closes loaded-but-stale coords AND DB orphans
|
||||
# left behind by prior console processes. The thread runs an
|
||||
# initial sweep on entry (no synchronous lifespan call needed —
|
||||
# see ``_coord_idle_cleanup_thread``) so cold-start cleanup
|
||||
# doesn't block startup. Reuses the regular-server
|
||||
# ``server.workstream_idle_timeout`` setting — the same cadence
|
||||
# makes sense for both kinds and avoids a redundant config knob.
|
||||
try:
|
||||
idle_minutes = int(config_store.get("server.workstream_idle_timeout"))
|
||||
except Exception:
|
||||
idle_minutes = 0
|
||||
if idle_minutes > 0:
|
||||
timeout_sec = float(idle_minutes * 60)
|
||||
cleanup_thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(coord_mgr, timeout_sec),
|
||||
name="coord-idle-cleanup",
|
||||
daemon=True,
|
||||
)
|
||||
cleanup_thread.start()
|
||||
app.state.coord_idle_cleanup_thread = cleanup_thread
|
||||
log.info(
|
||||
"console.coordinator_mgr_ready max_active=%s",
|
||||
config_store.get("coordinator.max_active"),
|
||||
|
||||
+160
-10
@@ -1851,6 +1851,16 @@ var _savedCoordsRetry = false;
|
||||
|
||||
function loadSavedCoordinators() {
|
||||
if (!_hasCoordPermission()) return;
|
||||
// Freeze the list while the user is multi-selecting — re-rendering
|
||||
// mid-mode would shuffle the visible page out from under them. The
|
||||
// delete-mode wrapper drains the retry flag on cancel/onClose.
|
||||
if (
|
||||
typeof _coordDeleteController !== "undefined" &&
|
||||
_coordDeleteController.inMode()
|
||||
) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
}
|
||||
if (_savedCoordsInFlight) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
@@ -1861,6 +1871,16 @@ function loadSavedCoordinators() {
|
||||
return r.ok ? r.json() : { workstreams: [] };
|
||||
})
|
||||
.then(function (data) {
|
||||
// Belt-and-braces: if the user entered delete mode while this
|
||||
// fetch was already in flight, defer the render — re-rendering
|
||||
// mid-selection would shuffle visible cards and reshape selections.
|
||||
if (
|
||||
typeof _coordDeleteController !== "undefined" &&
|
||||
_coordDeleteController.inMode()
|
||||
) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
}
|
||||
renderSavedCoordinators(data.workstreams || []);
|
||||
})
|
||||
.catch(function () {
|
||||
@@ -1878,7 +1898,52 @@ function loadSavedCoordinators() {
|
||||
});
|
||||
}
|
||||
|
||||
// Saved Coordinators: paginated card list + multi-select delete.
|
||||
// The shared controller (createSavedCardsController in /shared/cards.js)
|
||||
// owns mode state, checkbox decoration, the toolbar, and the modal.
|
||||
// Pagination caps Select-All fan-out at COORD_PAGE_SIZE — the controller
|
||||
// only ever sees the visible page, so a confirm-all batch is bounded to
|
||||
// COORD_PAGE_SIZE parallel POSTs against the routing proxy.
|
||||
var COORD_PAGE_SIZE = 24;
|
||||
var _coordPage = 0;
|
||||
var _coordSavedItems = [];
|
||||
var _coordDeleteController = createSavedCardsController({
|
||||
idPrefix: "coord-delete",
|
||||
buttonId: "coord-delete-btn",
|
||||
noun: "coordinator",
|
||||
activateLabel: function (s) {
|
||||
return "Resume coordinator: " + (s.alias || s.title || s.name || s.ws_id);
|
||||
},
|
||||
// Coordinators live on whichever node owns the ws_id, so we can't fire
|
||||
// a path-keyed delete the way ui/static does. The router proxy reads
|
||||
// ws_id from the body, resolves the owning node via the consistent-
|
||||
// hash ring, and forwards to that node's POST workstreams/{ws_id}/delete.
|
||||
buildDeleteRequest: function (wsId) {
|
||||
return {
|
||||
url: "/v1/api/route/workstreams/delete",
|
||||
options: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ ws_id: wsId }),
|
||||
},
|
||||
};
|
||||
},
|
||||
render: function () {
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
},
|
||||
onClose: function () {
|
||||
// Drain queued retries before the explicit reload — without this,
|
||||
// _savedCoordsRetry is still true from SSE events that arrived
|
||||
// during the freeze, so loadSavedCoordinators's .finally() would
|
||||
// re-fire a second fetch immediately after the first resolves.
|
||||
// Same idiom as cancelCoordDeleteMode below.
|
||||
_savedCoordsRetry = false;
|
||||
loadSavedCoordinators();
|
||||
},
|
||||
});
|
||||
|
||||
function renderSavedCoordinators(items) {
|
||||
_coordSavedItems = items;
|
||||
var section = document.getElementById("saved-coordinators");
|
||||
var cards = document.getElementById("saved-coord-cards");
|
||||
var countEl = document.getElementById("saved-coord-count");
|
||||
@@ -1887,24 +1952,31 @@ function renderSavedCoordinators(items) {
|
||||
section.style.display = "none";
|
||||
cards.replaceChildren();
|
||||
if (countEl) countEl.textContent = "";
|
||||
_coordPage = 0;
|
||||
if (_coordDeleteController.inMode()) _coordDeleteController.cancel();
|
||||
_renderCoordPagination();
|
||||
return;
|
||||
}
|
||||
// Clamp the page index after deletes (or upstream churn) shrink the list.
|
||||
var pages = Math.max(1, Math.ceil(items.length / COORD_PAGE_SIZE));
|
||||
if (_coordPage > pages - 1) _coordPage = pages - 1;
|
||||
if (_coordPage < 0) _coordPage = 0;
|
||||
var visible = items.slice(
|
||||
_coordPage * COORD_PAGE_SIZE,
|
||||
(_coordPage + 1) * COORD_PAGE_SIZE,
|
||||
);
|
||||
_coordDeleteController.setItems(visible);
|
||||
|
||||
section.style.display = "";
|
||||
if (countEl) countEl.textContent = "(" + items.length + ")";
|
||||
cards.replaceChildren();
|
||||
items.forEach(function (sess) {
|
||||
visible.forEach(function (sess) {
|
||||
var card = renderSessionCard(sess, {
|
||||
ariaLabel: function (s) {
|
||||
return (
|
||||
"Resume coordinator: " + (s.alias || s.title || s.name || s.ws_id)
|
||||
);
|
||||
},
|
||||
ariaLabel: _coordDeleteController.ariaLabel,
|
||||
onActivate: function (s, cardEl) {
|
||||
if (_coordDeleteController.blockActivate()) return;
|
||||
// POST /open BEFORE navigating so capacity issues surface as a
|
||||
// toast instead of a broken-looking detail page. The /open
|
||||
// endpoint calls the same lazy-rehydrate path the GET would,
|
||||
// but we get the status code synchronously so the user learns
|
||||
// "all slots in use" instead of staring at a 404.
|
||||
// toast instead of a broken-looking detail page.
|
||||
cardEl.classList.add("is-busy");
|
||||
authFetch(
|
||||
"/v1/api/workstreams/" + encodeURIComponent(s.ws_id) + "/open",
|
||||
@@ -1936,8 +2008,86 @@ function renderSavedCoordinators(items) {
|
||||
});
|
||||
},
|
||||
});
|
||||
_coordDeleteController.decorateCard(card, sess);
|
||||
cards.appendChild(card);
|
||||
});
|
||||
if (_coordDeleteController.inMode()) _coordDeleteController.refreshBar();
|
||||
_renderCoordPagination();
|
||||
}
|
||||
|
||||
function _renderCoordPagination() {
|
||||
var pag = document.getElementById("coord-pagination");
|
||||
if (!pag) return;
|
||||
var total = _coordSavedItems.length;
|
||||
var pages = Math.max(1, Math.ceil(total / COORD_PAGE_SIZE));
|
||||
// Single-page lists and delete-mode hide the controls — page changes
|
||||
// would invalidate the user's checkbox selections, so we lock them out.
|
||||
if (pages <= 1 || _coordDeleteController.inMode()) {
|
||||
pag.style.display = "none";
|
||||
return;
|
||||
}
|
||||
pag.style.display = "";
|
||||
var label = document.getElementById("coord-page-label");
|
||||
if (label) {
|
||||
/* Visible text uses the terse "X / Y" form to match the
|
||||
filtered-pagination control elsewhere in the console; the long
|
||||
form sits on the parent's aria-label so screen readers still get
|
||||
a full sentence. */
|
||||
label.textContent = _coordPage + 1 + " / " + pages;
|
||||
pag.setAttribute(
|
||||
"aria-label",
|
||||
"Saved coordinators pagination — page " +
|
||||
(_coordPage + 1) +
|
||||
" of " +
|
||||
pages,
|
||||
);
|
||||
}
|
||||
var prev = document.getElementById("coord-page-prev");
|
||||
if (prev) prev.disabled = _coordPage <= 0;
|
||||
var next = document.getElementById("coord-page-next");
|
||||
if (next) next.disabled = _coordPage >= pages - 1;
|
||||
}
|
||||
|
||||
function coordPagePrev() {
|
||||
if (_coordPage > 0) {
|
||||
_coordPage--;
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
}
|
||||
}
|
||||
|
||||
function coordPageNext() {
|
||||
var pages = Math.max(1, Math.ceil(_coordSavedItems.length / COORD_PAGE_SIZE));
|
||||
if (_coordPage < pages - 1) {
|
||||
_coordPage++;
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
}
|
||||
}
|
||||
|
||||
// HTML inline-onclick wrappers — keep the global names the markup binds
|
||||
// to and forward to the shared controller.
|
||||
function startCoordDeleteMode() {
|
||||
_coordDeleteController.start();
|
||||
}
|
||||
function cancelCoordDeleteMode() {
|
||||
_coordDeleteController.cancel();
|
||||
// The freeze gate (see loadSavedCoordinators) may have queued retries
|
||||
// while we were multi-selecting; drain them now that we're idle again.
|
||||
if (_savedCoordsRetry) {
|
||||
_savedCoordsRetry = false;
|
||||
loadSavedCoordinators();
|
||||
}
|
||||
}
|
||||
function toggleCoordSelectAll() {
|
||||
_coordDeleteController.toggleAll();
|
||||
}
|
||||
function confirmCoordDeleteSelection() {
|
||||
_coordDeleteController.confirmSelection();
|
||||
}
|
||||
function cancelCoordDelete() {
|
||||
_coordDeleteController.closeModal();
|
||||
}
|
||||
function confirmCoordDelete() {
|
||||
_coordDeleteController.confirm();
|
||||
}
|
||||
|
||||
// --- Init ---
|
||||
|
||||
@@ -396,6 +396,87 @@
|
||||
background: color-mix(in srgb, var(--err) 12%, var(--panel-2));
|
||||
}
|
||||
|
||||
/* Output-guard finding rendered under its specific .coord-tool-row.
|
||||
Stays anchored to the call that tripped the guard rather than
|
||||
floating into the chat log as a generic "[output guard]" line —
|
||||
matches interactive's `.output-warning` placement convention.
|
||||
Severity drives the hue (matches .verdict-badge.verdict-* palette
|
||||
so an operator scanning a workstream reads risk consistently
|
||||
across both surfaces). */
|
||||
.coord-tool-row-warning {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
margin-top: 4px;
|
||||
padding: 2px 8px;
|
||||
font-size: 11px;
|
||||
font-family: var(--font-mono);
|
||||
border: 1px solid var(--hair);
|
||||
border-radius: 3px;
|
||||
border-left-width: 3px;
|
||||
background: var(--panel-2);
|
||||
color: var(--ink-2);
|
||||
max-width: max-content;
|
||||
}
|
||||
.coord-tool-row-warning--low {
|
||||
color: color-mix(in srgb, var(--ok) 70%, var(--ink-2));
|
||||
border-left-color: var(--ok);
|
||||
background: color-mix(in srgb, var(--ok) 12%, var(--panel-2));
|
||||
}
|
||||
.coord-tool-row-warning--medium {
|
||||
color: color-mix(in srgb, var(--warn) 70%, var(--ink-2));
|
||||
border-left-color: var(--warn);
|
||||
background: var(--warn-tint);
|
||||
}
|
||||
.coord-tool-row-warning--high,
|
||||
.coord-tool-row-warning--critical {
|
||||
color: color-mix(in srgb, var(--err) 70%, var(--ink-2));
|
||||
border-left-color: var(--err);
|
||||
background: color-mix(in srgb, var(--err) 12%, var(--panel-2));
|
||||
}
|
||||
.coord-tool-row-warning-redacted {
|
||||
color: var(--ink-3);
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
/* Storage-truncation indicator — same convention as the interactive
|
||||
UI's `.tool-output-truncated` pill (transparent bg, dim border,
|
||||
small font) so the operator reads the affordance the same way on
|
||||
both surfaces. Sibling node next to .coord-tool-row-result rather
|
||||
than text-in-content so a future "best-effort JSON repair" pass
|
||||
on the result body doesn't have to strip a marker string. */
|
||||
.coord-tool-truncated {
|
||||
display: inline-block;
|
||||
margin-top: 4px;
|
||||
margin-left: 6px;
|
||||
padding: 1px 6px;
|
||||
font-size: 10px;
|
||||
font-family: var(--font-mono);
|
||||
color: var(--ink-3);
|
||||
background: transparent;
|
||||
border: 1px solid var(--ink-3);
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
/* memory/recall calls are background metadata — the audit trail is
|
||||
useful but they crowd the tree on workstreams with heavy memory
|
||||
usage. Dim the row by default; full opacity on hover so they
|
||||
stay inspectable. Mirrors the interactive UI's metacog dim rule
|
||||
(style.css `.ts-approval-tool[data-func-name="memory"]`). The
|
||||
row stamps `data-tool-name` from item.func_name in coordinator.js
|
||||
so this selector has something to match. */
|
||||
.coord-tool-row[data-tool-name="memory"],
|
||||
.coord-tool-row[data-tool-name="recall"] {
|
||||
opacity: 0.55;
|
||||
transition: opacity 120ms ease-out;
|
||||
}
|
||||
.coord-tool-row[data-tool-name="memory"]:hover,
|
||||
.coord-tool-row[data-tool-name="memory"]:focus-within,
|
||||
.coord-tool-row[data-tool-name="recall"]:hover,
|
||||
.coord-tool-row[data-tool-name="recall"]:focus-within {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Tool result block — paired under its row, mono pre-block. Capped
|
||||
at 240px with internal scroll so a long tool output doesn't push
|
||||
the rest of the chat off-screen. The interactive UI uses a
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -182,6 +182,36 @@
|
||||
font-size: 11px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
/* Loading-state placeholder rendered while the bulk fetch is
|
||||
in-flight. Same outer container as the real block so the row
|
||||
height is stable when the real content swaps in. */
|
||||
.ch-row .approval-block-loading {
|
||||
color: var(--ink-3);
|
||||
}
|
||||
.ch-row .approval-loading-spin {
|
||||
display: inline-block;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
border: 1.5px solid var(--accent);
|
||||
border-top-color: transparent;
|
||||
margin-right: 5px;
|
||||
animation: ts-spin 0.9s linear infinite;
|
||||
vertical-align: middle;
|
||||
}
|
||||
/* Inline status note shown when the operator clicks Approve/Deny
|
||||
but the call_id is stale (already resolved on another channel).
|
||||
Quieter than a toast — the row is about to be replaced wholesale
|
||||
by the refresh, so this only needs to bridge ~350ms. */
|
||||
.ch-row .approval-stale-note {
|
||||
font-size: 11px;
|
||||
color: var(--ink-3);
|
||||
font-style: italic;
|
||||
margin-top: 4px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ch-row .approval-loading-spin { animation: none; }
|
||||
}
|
||||
/* Auto-approved pill — same 22px indent as .approval-block / .meta
|
||||
so a child row showing both stacks cleanly. Lower visual weight
|
||||
than the live approve/deny block (this is informational, not
|
||||
|
||||
@@ -165,6 +165,14 @@
|
||||
<h2 class="home-section-title">
|
||||
<span>Saved Coordinators</span>
|
||||
<span id="saved-coord-count" class="home-section-count"></span>
|
||||
<button
|
||||
id="coord-delete-btn"
|
||||
class="ws-delete-btn home-section-action"
|
||||
onclick="startCoordDeleteMode()"
|
||||
title="Delete coordinators"
|
||||
>
|
||||
<span aria-hidden="true">🗑</span> Delete
|
||||
</button>
|
||||
</h2>
|
||||
<div
|
||||
id="saved-coord-cards"
|
||||
@@ -172,6 +180,61 @@
|
||||
role="list"
|
||||
aria-live="polite"
|
||||
></div>
|
||||
<div
|
||||
id="coord-pagination"
|
||||
class="pagination"
|
||||
style="display: none"
|
||||
role="navigation"
|
||||
aria-label="Saved coordinators pagination"
|
||||
>
|
||||
<button
|
||||
id="coord-page-prev"
|
||||
type="button"
|
||||
onclick="coordPagePrev()"
|
||||
>
|
||||
◄ Prev
|
||||
</button>
|
||||
<span id="coord-page-label" aria-live="polite" aria-atomic="true">
|
||||
</span>
|
||||
<button
|
||||
id="coord-page-next"
|
||||
type="button"
|
||||
onclick="coordPageNext()"
|
||||
>
|
||||
Next ►
|
||||
</button>
|
||||
</div>
|
||||
<div id="coord-delete-bar" class="ws-delete-bar">
|
||||
<span
|
||||
class="ws-delete-count-label"
|
||||
id="coord-delete-bar-count"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
aria-atomic="true"
|
||||
>0 selected</span
|
||||
>
|
||||
<button
|
||||
class="ws-delete-cancel-btn"
|
||||
onclick="cancelCoordDeleteMode()"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
class="ws-delete-selectall-btn"
|
||||
id="coord-delete-bar-select-all"
|
||||
onclick="toggleCoordSelectAll()"
|
||||
>
|
||||
Select All
|
||||
</button>
|
||||
<button
|
||||
class="ws-delete-bar-btn"
|
||||
id="coord-delete-bar-delete"
|
||||
onclick="confirmCoordDeleteSelection()"
|
||||
disabled
|
||||
>
|
||||
Delete Selected
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Cluster details — node list, always visible. The list is
|
||||
@@ -3907,6 +3970,40 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Delete coordinators confirmation modal (batch) -->
|
||||
<div
|
||||
id="coord-delete-overlay"
|
||||
class="ws-delete-modal-overlay"
|
||||
style="display: none"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="coord-delete-title"
|
||||
>
|
||||
<div id="coord-delete-box" class="ws-delete-modal-box">
|
||||
<h3 id="coord-delete-title">Delete Coordinators</h3>
|
||||
<div id="coord-delete-error" role="alert" aria-live="assertive"></div>
|
||||
<p id="coord-delete-count"></p>
|
||||
<div id="coord-delete-list" class="ws-delete-modal-list"></div>
|
||||
<div id="coord-delete-buttons" class="ws-delete-modal-buttons">
|
||||
<button
|
||||
id="coord-delete-cancel-btn"
|
||||
type="button"
|
||||
onclick="cancelCoordDelete()"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
id="coord-delete-confirm-btn"
|
||||
class="ws-delete-confirm"
|
||||
type="button"
|
||||
onclick="confirmCoordDelete()"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="/static/admin.js"></script>
|
||||
<script src="/static/governance.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
|
||||
@@ -133,6 +133,17 @@
|
||||
font-size: 11px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
/* Right-align action buttons (e.g. Saved Coordinators "Delete") inside
|
||||
.home-section-title without breaking the count's natural left position. */
|
||||
.home-section-title .home-section-action {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
/* Saved Coordinators reuses the existing .pagination control (see the
|
||||
"Pagination" block below) — the visible page is capped at
|
||||
COORD_PAGE_SIZE so Select-All fan-out is bounded. Pagination is
|
||||
hidden in delete mode and when there's only one page (see
|
||||
_renderCoordPagination). */
|
||||
|
||||
.home-coord-list {
|
||||
border: 1px solid var(--border);
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Strategy interface for delivering child workstream lifecycle events.
|
||||
|
||||
Two implementations bind to the unified :class:`ChildrenRegistry`:
|
||||
|
||||
- :class:`SameNodeChildSource` — subscribes to a :class:`SessionManager`'s
|
||||
state-change callbacks. In-process, no transport. For interactive
|
||||
workstreams that spawn children locally (no cluster routing).
|
||||
- :class:`ClusterChildSource` — subscribes to a :class:`ClusterCollector`'s
|
||||
listener channel and runs a daemon thread that drains the queue and
|
||||
pushes events to the sink. For coordinator workstreams whose
|
||||
children are routed across the cluster by hash bucket.
|
||||
|
||||
The strategy doesn't translate events into UI-shaped payloads; that's
|
||||
the sink's job. This split keeps the strategy generic across kinds and
|
||||
lets the consumer (e.g. ``CoordinatorAdapter._dispatch_child_event``)
|
||||
own the per-kind translation.
|
||||
|
||||
Sink signature: ``Callable[[dict[str, Any]], None]``. The sink is
|
||||
responsible for filtering by registry membership; strategies push raw
|
||||
events without registry-side filtering so the sink can decide whether
|
||||
to act based on its own state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class ChildSource(Protocol):
|
||||
"""Subscription strategy for child workstream lifecycle events."""
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
"""Begin delivering events to ``sink``. Idempotent."""
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the strategy. Idempotent; safe to call multiple times."""
|
||||
|
||||
|
||||
class _CollectorProtocol(Protocol):
|
||||
"""Subset of :class:`ClusterCollector` that :class:`ClusterChildSource` consumes.
|
||||
|
||||
Defined here (not imported) to keep ``turnstone/core/`` free of
|
||||
``turnstone/console/`` imports AND to let test fakes satisfy the
|
||||
type signature without subclassing the real collector.
|
||||
"""
|
||||
|
||||
def get_snapshot_and_register(self, q: queue.Queue[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Register ``q`` and return the current snapshot."""
|
||||
|
||||
def unregister_listener(self, q: queue.Queue[dict[str, Any]]) -> None:
|
||||
"""Drop ``q`` from the listener set."""
|
||||
|
||||
|
||||
class _ManagerProtocol(Protocol):
|
||||
"""Subset of :class:`SessionManager` that :class:`SameNodeChildSource` consumes.
|
||||
|
||||
Lets test fakes participate in the strategy's typed surface
|
||||
without forcing a full SessionManager construction.
|
||||
"""
|
||||
|
||||
def subscribe_to_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Register ``callback`` for state-change events."""
|
||||
|
||||
def unsubscribe_from_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Remove a previously-registered ``callback``."""
|
||||
|
||||
|
||||
class SameNodeChildSource:
|
||||
"""In-process child events via :class:`SessionManager` state observer.
|
||||
|
||||
Subscribes to the manager's state-change callbacks (registered via
|
||||
:meth:`SessionManager.subscribe_to_state`). For each transition on
|
||||
a workstream that's a known child (per the
|
||||
:class:`ChildrenRegistry` reverse index), synthesises a
|
||||
cluster-state-shaped event and pushes it to the sink.
|
||||
|
||||
Used by interactive workstreams when they gain spawn capability —
|
||||
children live on the same node as the parent, so no cluster routing
|
||||
is needed and event fan-out is in-process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
manager: _ManagerProtocol,
|
||||
registry: ChildrenRegistry,
|
||||
) -> None:
|
||||
self._manager = manager
|
||||
self._registry = registry
|
||||
self._sink: Callable[[dict[str, Any]], None] | None = None
|
||||
self._callback: Callable[[str, WorkstreamState], None] | None = None
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
if self._callback is not None:
|
||||
return # idempotent — already started
|
||||
self._sink = sink
|
||||
|
||||
def _on_state(ws_id: str, state: WorkstreamState) -> None:
|
||||
sink_fn = self._sink
|
||||
if sink_fn is None:
|
||||
return
|
||||
# Cheap pre-filter: skip dispatch for transitions on
|
||||
# workstreams that aren't children of any in-memory parent.
|
||||
# ``has_children`` is a lock-free dict-truthiness read; the
|
||||
# ``parent_for`` call below would otherwise acquire the
|
||||
# registry lock on every state change even when no
|
||||
# children exist (the steady state for an interactive
|
||||
# manager). The cluster strategy can't pre-filter because
|
||||
# the collector queue carries all events; here we have the
|
||||
# information to skip the synthesis entirely.
|
||||
if not self._registry.has_children():
|
||||
return
|
||||
if self._registry.parent_for(ws_id) is None:
|
||||
return
|
||||
# ``pending_approval_detail`` deliberately omitted — the
|
||||
# field was removed from cluster_state end-to-end in the
|
||||
# Stage 3 cleanup pass. Approval items arrive via bulk
|
||||
# fetch; verdicts via the explicit intent_verdict event;
|
||||
# resolution via approval_resolved.
|
||||
event = {
|
||||
"type": "cluster_state",
|
||||
"ws_id": ws_id,
|
||||
"state": state.value,
|
||||
"node_id": "",
|
||||
"tokens": 0,
|
||||
"activity_state": "",
|
||||
}
|
||||
try:
|
||||
sink_fn(event)
|
||||
except Exception:
|
||||
log.debug("same_node_child_source.sink_failed", exc_info=True)
|
||||
|
||||
self._callback = _on_state
|
||||
self._manager.subscribe_to_state(_on_state)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
cb = self._callback
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
self._manager.unsubscribe_from_state(cb)
|
||||
except Exception:
|
||||
log.debug("same_node_child_source.unsubscribe_failed", exc_info=True)
|
||||
self._callback = None
|
||||
self._sink = None
|
||||
|
||||
|
||||
class ClusterChildSource:
|
||||
"""Cross-node child events via :class:`ClusterCollector` subscription.
|
||||
|
||||
Refactor of the existing fan-out machinery from
|
||||
``CoordinatorAdapter`` (was ``_collector_queue`` +
|
||||
``_fanout_thread`` + ``_fanout_loop``). Subscribes as a listener on
|
||||
the collector's broadcast channel and runs a daemon thread that
|
||||
drains the queue, pushing each event to the sink.
|
||||
|
||||
On :meth:`start`, also primes the registry from the collector's
|
||||
snapshot so a parent that re-installs after a console restart sees
|
||||
its already-live children without waiting for the next state tick.
|
||||
The ``parents_provider`` callback returns the set of in-memory
|
||||
parent ws_ids for snapshot filtering — only children whose parent
|
||||
is currently installed get merged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
collector: _CollectorProtocol,
|
||||
registry: ChildrenRegistry,
|
||||
*,
|
||||
parents_provider: Callable[[], Iterable[str]],
|
||||
) -> None:
|
||||
self._collector = collector
|
||||
self._registry = registry
|
||||
self._parents_provider = parents_provider
|
||||
self._sink: Callable[[dict[str, Any]], None] | None = None
|
||||
self._queue: queue.Queue[dict[str, Any]] | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return # idempotent — already started
|
||||
self._sink = sink
|
||||
self._queue = queue.Queue(maxsize=1000)
|
||||
snapshot = self._collector.get_snapshot_and_register(self._queue)
|
||||
self._prime_from_snapshot(snapshot)
|
||||
self._stop.clear()
|
||||
t = threading.Thread(
|
||||
target=self._loop,
|
||||
name="cluster-child-source",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread = t
|
||||
t.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._stop.set()
|
||||
t = self._thread
|
||||
q = self._queue
|
||||
coll = self._collector
|
||||
self._thread = None
|
||||
self._queue = None
|
||||
if coll is not None and q is not None:
|
||||
try:
|
||||
coll.unregister_listener(q)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"cluster_child_source.unregister_listener_failed",
|
||||
exc_info=True,
|
||||
)
|
||||
if t is not None:
|
||||
t.join(timeout=2.0)
|
||||
self._sink = None
|
||||
|
||||
def _loop(self) -> None:
|
||||
q = self._queue
|
||||
if q is None:
|
||||
return
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
event = q.get(timeout=1.0)
|
||||
except queue.Empty:
|
||||
continue
|
||||
sink = self._sink
|
||||
if sink is None:
|
||||
continue
|
||||
try:
|
||||
sink(event)
|
||||
except Exception:
|
||||
log.debug("cluster_child_source.dispatch_failed", exc_info=True)
|
||||
|
||||
def _prime_from_snapshot(self, snapshot: dict[str, Any]) -> None:
|
||||
"""Populate the registry from a collector snapshot.
|
||||
|
||||
For every workstream in the snapshot whose ``parent_ws_id``
|
||||
names a currently-installed parent (per ``parents_provider``),
|
||||
merge it into the registry. Caller-installed parents that
|
||||
appear in the snapshot are seeded; unknown parents are skipped
|
||||
— they'll be picked up by the live fan-out path once their
|
||||
``ws_created`` event arrives.
|
||||
"""
|
||||
nodes = snapshot.get("nodes", []) if isinstance(snapshot, dict) else []
|
||||
if not nodes:
|
||||
return
|
||||
known_parents = set(self._parents_provider())
|
||||
if not known_parents:
|
||||
return
|
||||
by_parent: dict[str, list[str]] = {}
|
||||
for node in nodes:
|
||||
for entry in node.get("workstreams", []) or []:
|
||||
parent = entry.get("parent_ws_id") or ""
|
||||
child_id = entry.get("id") or ""
|
||||
if not parent or not child_id or parent not in known_parents:
|
||||
continue
|
||||
by_parent.setdefault(parent, []).append(child_id)
|
||||
for parent, kids in by_parent.items():
|
||||
self._registry.merge_children(parent, kids)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Universal parent → children registry for SessionManager.
|
||||
|
||||
Pure data + lookups; no IO, no transport. Lifted from
|
||||
:class:`turnstone.console.coordinator_adapter.CoordinatorAdapter` where
|
||||
it lived bound to the coordinator kind. The lift is what lets the
|
||||
``ChildSource`` strategies (Step 2) plug into a single shared primitive
|
||||
regardless of whether children are local (interactive) or cluster-routed
|
||||
(coordinator).
|
||||
|
||||
Storage rebuild and snapshot priming happen in the caller — typically
|
||||
the ``ChildSource`` implementation that owns the relevant transport.
|
||||
The registry exposes :meth:`merge_children` for bulk seeding so callers
|
||||
that compute child id lists from any source can feed them in without
|
||||
the registry needing to know about storage shapes or collector
|
||||
snapshots.
|
||||
|
||||
Threading: every public method is internally locked. Helpers suffixed
|
||||
``_locked`` require the caller to already hold :attr:`_lock`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class ChildrenRegistry:
|
||||
"""Tracks parent → children + reverse lookup for in-memory parents.
|
||||
|
||||
The forward index (``_children``) is parent_ws_id → set of child ws_ids.
|
||||
The reverse index (``_child_to_parent``) is child_ws_id → parent_ws_id.
|
||||
The presence map (``_active``) is parent_ws_id → UI ref, used by the
|
||||
dispatch path to atomically check-and-route in one lock acquisition.
|
||||
|
||||
Closed / deleted children stay in the registry until their owning
|
||||
parent is uninstalled — the tree UI keeps rendering them grayed out;
|
||||
state authority lives in storage, not here.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._children: dict[str, set[str]] = {}
|
||||
self._child_to_parent: dict[str, str] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._active: dict[str, Any] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle — install / uninstall a parent
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def install(self, parent_ws_id: str, ui: Any) -> None:
|
||||
"""Seed the forward set + presence map for a new parent.
|
||||
|
||||
Idempotent — re-installing re-points the UI but leaves the
|
||||
existing child set intact. Mirrors the original
|
||||
``_install_coord_registry`` semantics so a coordinator that
|
||||
rehydrates after a crash doesn't lose its known-children.
|
||||
"""
|
||||
with self._lock:
|
||||
self._children.setdefault(parent_ws_id, set())
|
||||
self._active[parent_ws_id] = ui
|
||||
|
||||
def uninstall(self, parent_ws_id: str) -> None:
|
||||
"""Drop a parent: forward set, reverse-index entries, presence.
|
||||
|
||||
No-op if the parent is unknown. Used by close / eviction paths.
|
||||
"""
|
||||
with self._lock:
|
||||
self._uninstall_locked(parent_ws_id)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Mutation — register children under a parent
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_child(self, parent_ws_id: str, child_ws_id: str) -> Any | None:
|
||||
"""Register a child under a parent. Returns parent's UI or None.
|
||||
|
||||
Returns the parent's UI on success (so the dispatch path can
|
||||
atomically check-and-route in one lock acquisition). Returns
|
||||
``None`` if the parent isn't installed (concurrent close /
|
||||
eviction) or if the child is already registered (duplicate
|
||||
ws_created from the cluster fan-out).
|
||||
"""
|
||||
with self._lock:
|
||||
ui = self._active.get(parent_ws_id)
|
||||
if ui is None:
|
||||
return None
|
||||
existing = self._children.setdefault(parent_ws_id, set())
|
||||
if child_ws_id in existing:
|
||||
return None
|
||||
existing.add(child_ws_id)
|
||||
self._child_to_parent[child_ws_id] = parent_ws_id
|
||||
return ui
|
||||
|
||||
def merge_children(self, parent_ws_id: str, child_ws_ids: Iterable[str]) -> None:
|
||||
"""Bulk-merge child_ids under a parent. Idempotent.
|
||||
|
||||
Sole bulk write-path — used by both storage-seeded rebuilds and
|
||||
snapshot-seeded priming so reverse-index ordering invariants
|
||||
hold regardless of which seed source races first.
|
||||
"""
|
||||
with self._lock:
|
||||
self._merge_locked(parent_ws_id, child_ws_ids)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lookups
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def parent_for(self, child_ws_id: str) -> str | None:
|
||||
"""Reverse lookup: which parent owns this child? O(1)."""
|
||||
with self._lock:
|
||||
return self._child_to_parent.get(child_ws_id)
|
||||
|
||||
def has_children(self) -> bool:
|
||||
"""Lock-free fast path: any child registered under any parent?
|
||||
|
||||
Reads ``bool(self._child_to_parent)`` without taking the lock.
|
||||
Dict-truthiness is a single GIL-atomic read, so callers on
|
||||
the hot state-broadcast path can short-circuit without paying
|
||||
the lock acquisition when the registry is empty (the steady
|
||||
state for an interactive manager today). The answer is best-
|
||||
effort — if a child is added concurrently with the read the
|
||||
caller may falsely return ``False``, but the next state event
|
||||
will pick up the change correctly.
|
||||
"""
|
||||
return bool(self._child_to_parent)
|
||||
|
||||
def children_of(self, parent_ws_id: str) -> list[str]:
|
||||
"""Snapshot copy of the parent's child ws_ids.
|
||||
|
||||
Returned list is a copy so callers can iterate without holding
|
||||
the registry lock during per-child work. A mutation racing with
|
||||
the snapshot either lands before (included) or after (excluded)
|
||||
— both outcomes are safe for cascade-style dispatch.
|
||||
"""
|
||||
with self._lock:
|
||||
child_set = self._children.get(parent_ws_id)
|
||||
return list(child_set) if child_set else []
|
||||
|
||||
def ui_for(self, parent_ws_id: str) -> Any | None:
|
||||
"""Look up the UI registered for a parent."""
|
||||
with self._lock:
|
||||
return self._active.get(parent_ws_id)
|
||||
|
||||
def parents(self) -> list[str]:
|
||||
"""Snapshot copy of installed parent ws_ids."""
|
||||
with self._lock:
|
||||
return list(self._active)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Locked helpers — caller must hold ``self._lock``
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_locked(self, parent_ws_id: str, child_ws_ids: Iterable[str]) -> None:
|
||||
"""Idempotent merge under caller's lock. Empty/falsy ids skipped."""
|
||||
existing = self._children.setdefault(parent_ws_id, set())
|
||||
for cid in child_ws_ids:
|
||||
if cid and cid not in existing:
|
||||
existing.add(cid)
|
||||
self._child_to_parent[cid] = parent_ws_id
|
||||
|
||||
def _uninstall_locked(self, parent_ws_id: str) -> None:
|
||||
"""Pop forward set + presence + own reverse-index entries.
|
||||
|
||||
Defensive: only clears reverse entries that still point at
|
||||
``parent_ws_id``. Schema-shaped reassignments (rare but
|
||||
possible) shouldn't orphan the new owner's entry.
|
||||
"""
|
||||
child_set = self._children.pop(parent_ws_id, None)
|
||||
self._active.pop(parent_ws_id, None)
|
||||
if child_set is None:
|
||||
return
|
||||
for cid in child_set:
|
||||
if self._child_to_parent.get(cid) == parent_ws_id:
|
||||
self._child_to_parent.pop(cid, None)
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Shared history-replay decoration helpers.
|
||||
|
||||
Both surfaces that build a history wire payload — interactive's SSE
|
||||
``_build_history`` and the lifted ``make_history_handler`` REST
|
||||
endpoint — need the same audit-trail data attached to each
|
||||
``tool_calls`` entry: the persisted intent verdict (``intent_verdicts``
|
||||
table) and the output-guard assessment (``output_assessments`` table).
|
||||
|
||||
Centralising the lookup + decoration here keeps the two surfaces from
|
||||
drifting on which fields ship to the client and how they're shaped.
|
||||
The shared helpers also let us project only the fields the UI actually
|
||||
renders, dropping redundant ones (``call_id``/``func_name`` already
|
||||
carried on ``tc.id``/``tc.name``) so the wire payload stays tight.
|
||||
|
||||
All functions are pure I/O or pure transforms — safe to call from
|
||||
either an async caller (via ``asyncio.to_thread``) or a sync hook.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# Tool results are clamped at this length per row at storage time
|
||||
# (see ``session.py``'s ``store_text = raw_output[:TOOL_RESULT_STORAGE_CAP]``).
|
||||
# Keeping the constant here lets the truncation flag detection in
|
||||
# ``decorate_history_messages`` stay in sync without a magic number
|
||||
# duplicated across server.py / session.py.
|
||||
#
|
||||
# Raised from 2000 → 10000 because a 2000-char clip routinely cut
|
||||
# the body of a single grep / file read mid-line, leaving the
|
||||
# historical record useless for retrospective debugging. FTS5
|
||||
# index + row size grow proportionally; the per-tool upper bound is
|
||||
# still bounded upstream by ``_truncate_output``'s context-budget
|
||||
# clamp (so a single huge result can't blow past the live context
|
||||
# window).
|
||||
TOOL_RESULT_STORAGE_CAP = 10000
|
||||
|
||||
|
||||
def load_verdict_indexes(
|
||||
ws_id: str,
|
||||
) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]:
|
||||
"""Bulk-load intent verdicts and output assessments for a workstream.
|
||||
|
||||
Returns ``(verdicts_by_call_id, assessments_by_call_id)``. Both
|
||||
tables are indexed by ws_id so the queries are O(rows-for-ws); the
|
||||
DESC ordering plus first-seen-wins dedupe leaves the newest
|
||||
verdict per call_id (LLM upgrade beats heuristic when both exist).
|
||||
|
||||
Pure storage I/O — safe to run in ``asyncio.to_thread`` from an
|
||||
async caller. Returns empty dicts when storage is unavailable or
|
||||
the lookup raises (best-effort: replay must never block on
|
||||
audit-trail decoration).
|
||||
"""
|
||||
verdicts_by_call_id: dict[str, dict[str, Any]] = {}
|
||||
assessments_by_call_id: dict[str, dict[str, Any]] = {}
|
||||
if not ws_id:
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
try:
|
||||
from turnstone.core.storage._registry import get_storage
|
||||
|
||||
storage = get_storage()
|
||||
if storage is None:
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
for v in storage.list_intent_verdicts(ws_id=ws_id, limit=10000):
|
||||
cid = v.get("call_id") or ""
|
||||
if cid and cid not in verdicts_by_call_id:
|
||||
verdicts_by_call_id[cid] = v
|
||||
for a in storage.list_output_assessments(ws_id=ws_id, limit=10000):
|
||||
cid = a.get("call_id") or ""
|
||||
if cid and cid not in assessments_by_call_id:
|
||||
assessments_by_call_id[cid] = a
|
||||
except Exception:
|
||||
# Missing storage / migration drift / driver error must not
|
||||
# block replay — degrade to an unannotated history.
|
||||
log.debug(
|
||||
"verdict/assessment lookup failed; replay continues unannotated",
|
||||
exc_info=True,
|
||||
)
|
||||
return verdicts_by_call_id, assessments_by_call_id
|
||||
|
||||
|
||||
def build_verdict_payload(vrow: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Project a stored ``intent_verdicts`` row into the wire shape.
|
||||
|
||||
Returns ``None`` when the verdict is the unflagged baseline
|
||||
(``risk_level == "none"``) — the client's ``renderVerdictBadge``
|
||||
helper would suppress those anyway, so skipping at the wire layer
|
||||
keeps the payload tight on long workstreams.
|
||||
|
||||
Drops ``call_id`` and ``func_name`` from the wire payload — they're
|
||||
already carried on the parent ``tc.id`` / ``tc.name`` fields.
|
||||
Ships ``reasoning`` for either tier when the row has non-empty
|
||||
prose (heuristic rules in this project DO write meaningful
|
||||
rationales — e.g. ``policy.py`` emits structured reasoning per
|
||||
matched pattern). ``judge_model`` rides through so the batch tier
|
||||
badge can render ``⚖ llm:claude-haiku-4`` on history-only batches
|
||||
rather than the bare ``⚖ llm`` label.
|
||||
"""
|
||||
if (vrow.get("risk_level") or "none") == "none":
|
||||
return None
|
||||
payload: dict[str, Any] = {
|
||||
"risk_level": vrow.get("risk_level", "medium"),
|
||||
"recommendation": vrow.get("recommendation", "review"),
|
||||
"confidence": vrow.get("confidence", 0.0),
|
||||
"intent_summary": vrow.get("intent_summary", ""),
|
||||
"tier": vrow.get("tier", "heuristic"),
|
||||
}
|
||||
if vrow.get("reasoning"):
|
||||
payload["reasoning"] = vrow.get("reasoning", "")
|
||||
judge_model = vrow.get("judge_model") or ""
|
||||
if judge_model:
|
||||
payload["judge_model"] = judge_model
|
||||
return payload
|
||||
|
||||
|
||||
def build_output_assessment_payload(arow: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Project a stored ``output_assessments`` row into the wire shape.
|
||||
|
||||
Returns ``None`` when the assessment is the unflagged baseline
|
||||
(``risk_level == "none"``) — same skip-on-clean pattern as
|
||||
:func:`build_verdict_payload`.
|
||||
|
||||
Decodes ``flags`` from its JSON string form here so the client
|
||||
never has to parse twice. Falls back to an empty list on bad JSON
|
||||
rather than raising — the rest of the assessment is still useful.
|
||||
"""
|
||||
if (arow.get("risk_level") or "none") == "none":
|
||||
return None
|
||||
flags_raw = arow.get("flags") or "[]"
|
||||
try:
|
||||
flags = json.loads(flags_raw) if isinstance(flags_raw, str) else flags_raw
|
||||
except (ValueError, TypeError):
|
||||
flags = []
|
||||
return {
|
||||
"risk_level": arow.get("risk_level", "none"),
|
||||
"flags": flags if isinstance(flags, list) else [],
|
||||
"redacted": bool(arow.get("redacted", 0)),
|
||||
}
|
||||
|
||||
|
||||
def decorate_tool_call(
|
||||
tc: dict[str, Any],
|
||||
verdicts_by_call_id: dict[str, dict[str, Any]],
|
||||
assessments_by_call_id: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Mutate ``tc`` in place, attaching ``verdict`` / ``output_assessment``.
|
||||
|
||||
Works on either tool_call shape:
|
||||
- OpenAI format (``{id, function: {name, arguments}}``) — used by
|
||||
``/history`` REST.
|
||||
- Flattened format (``{id, name, arguments}``) — used by SSE replay.
|
||||
|
||||
Both carry ``id`` at the top level, which is the only field this
|
||||
helper reads. No-ops cleanly when the call_id has no matching
|
||||
row (unflagged tools stay clean).
|
||||
"""
|
||||
call_id = tc.get("id", "") or ""
|
||||
if not call_id:
|
||||
return
|
||||
vrow = verdicts_by_call_id.get(call_id)
|
||||
if vrow is not None:
|
||||
verdict = build_verdict_payload(vrow)
|
||||
if verdict is not None:
|
||||
tc["verdict"] = verdict
|
||||
arow = assessments_by_call_id.get(call_id)
|
||||
if arow is not None:
|
||||
assessment = build_output_assessment_payload(arow)
|
||||
if assessment is not None:
|
||||
tc["output_assessment"] = assessment
|
||||
|
||||
|
||||
def decorate_history_messages(
|
||||
messages: list[dict[str, Any]],
|
||||
verdicts_by_call_id: dict[str, dict[str, Any]],
|
||||
assessments_by_call_id: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Mutate a list of OpenAI-format messages, decorating tool_calls.
|
||||
|
||||
Used by the ``/history`` REST endpoint after ``load_messages``
|
||||
returns. For each assistant message with ``tool_calls``, runs
|
||||
:func:`decorate_tool_call` on every entry. For each tool message
|
||||
whose content hits the storage cap, sets ``truncated: True`` so
|
||||
the client can render the "… truncated in storage" pill.
|
||||
|
||||
Pure transform — no I/O. Async callers should pre-load the
|
||||
indexes via :func:`load_verdict_indexes` (in ``to_thread``) and
|
||||
pass them in.
|
||||
"""
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
if role == "assistant":
|
||||
tcs = msg.get("tool_calls")
|
||||
if isinstance(tcs, list):
|
||||
for tc in tcs:
|
||||
if isinstance(tc, dict):
|
||||
decorate_tool_call(tc, verdicts_by_call_id, assessments_by_call_id)
|
||||
elif role == "tool":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str) and len(content) >= TOOL_RESULT_STORAGE_CAP:
|
||||
msg["truncated"] = True
|
||||
@@ -5,7 +5,50 @@ from __future__ import annotations
|
||||
import re
|
||||
import time
|
||||
|
||||
_COOLDOWN_SECS = 300 # 5 minutes between nudges of the same type
|
||||
# Default cooldown (s) between nudges of the same type. Production
|
||||
# paths pass ``cooldown_secs`` explicitly from
|
||||
# ``MemoryConfig.nudge_cooldown`` (config-store ``memory.nudge_cooldown``,
|
||||
# default 300); this constant is the fallback for tests and unit-style
|
||||
# callers without a ``MemoryConfig`` and is kept aligned with that
|
||||
# canonical default so both paths behave the same.
|
||||
_COOLDOWN_SECS = 300
|
||||
|
||||
# Repeat-detection threshold — number of *consecutive* identical tool
|
||||
# calls (same name + same arguments) before a repeat warning fires.
|
||||
# Two-in-a-row is too noisy because legitimate retries on transient
|
||||
# failures look identical; three-in-a-row is the cheapest signal that
|
||||
# the model is stuck on the same call.
|
||||
_REPEAT_THRESHOLD = 3
|
||||
|
||||
|
||||
class RepeatDetector:
|
||||
"""Detect a streak of identical tool-call signatures.
|
||||
|
||||
``record(sig)`` returns ``True`` once *sig* has been recorded
|
||||
``threshold`` times in a row (default 3). Recording a different
|
||||
signature resets the streak — interleaved tool calls aren't a
|
||||
stuck loop, only repeated identical ones are. After a fire, the
|
||||
caller is expected to call ``clear()`` to start a fresh streak.
|
||||
"""
|
||||
|
||||
def __init__(self, threshold: int = _REPEAT_THRESHOLD) -> None:
|
||||
self._threshold = threshold
|
||||
self._sig: str | None = None
|
||||
self._count = 0
|
||||
|
||||
def record(self, sig: str) -> bool:
|
||||
"""Record *sig*; return ``True`` when the streak hits the threshold."""
|
||||
if sig == self._sig:
|
||||
self._count += 1
|
||||
else:
|
||||
self._sig = sig
|
||||
self._count = 1
|
||||
return self._count >= self._threshold
|
||||
|
||||
def clear(self) -> None:
|
||||
self._sig = None
|
||||
self._count = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nudge messages (brief, model-facing hints)
|
||||
|
||||
+412
-206
@@ -44,6 +44,7 @@ from turnstone.core.attachments import (
|
||||
)
|
||||
from turnstone.core.config import get_tavily_key
|
||||
from turnstone.core.edit import find_occurrences, pick_nearest
|
||||
from turnstone.core.history_decoration import TOOL_RESULT_STORAGE_CAP
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.memory import (
|
||||
count_structured_memories,
|
||||
@@ -81,6 +82,7 @@ from turnstone.core.memory_relevance import (
|
||||
score_memories,
|
||||
)
|
||||
from turnstone.core.metacognition import (
|
||||
RepeatDetector,
|
||||
detect_completion,
|
||||
detect_correction,
|
||||
format_nudge,
|
||||
@@ -90,6 +92,7 @@ from turnstone.core.providers import create_provider
|
||||
from turnstone.core.safety import is_command_blocked, sanitize_command
|
||||
from turnstone.core.sandbox import execute_math_sandboxed
|
||||
from turnstone.core.storage._registry import get_storage
|
||||
from turnstone.core.tool_advisory import escape_wrapper_tags, render_system_reminder
|
||||
from turnstone.core.tool_search import ToolSearchManager
|
||||
from turnstone.core.tools import (
|
||||
AGENT_AUTO_TOOLS,
|
||||
@@ -275,6 +278,8 @@ class SessionUI(Protocol):
|
||||
def on_plan_review(self, content: str) -> str: ...
|
||||
def on_info(self, message: str) -> None: ...
|
||||
def on_error(self, message: str) -> None: ...
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None: ...
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None: ...
|
||||
def on_state_change(self, state: str) -> None: ...
|
||||
def on_rename(self, name: str) -> None: ...
|
||||
def on_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
@@ -474,8 +479,12 @@ class ChatSession:
|
||||
collections.OrderedDict()
|
||||
)
|
||||
self._queued_lock = threading.Lock()
|
||||
# Repeat detection: track recent tool call signatures
|
||||
self._recent_tool_sigs: set[str] = set()
|
||||
# Repeat detection: streak counter over tool-call signatures.
|
||||
# Fires when a (name, args) signature has been seen N times in
|
||||
# a row; recording any different signature resets the streak.
|
||||
# Also cleared after a write tool succeeds (state changed) or
|
||||
# after a warning fires (clean slate, re-fire on the next streak).
|
||||
self._repeat_detector = RepeatDetector()
|
||||
# Tool error tracking: call_id → is_error for message persistence
|
||||
self._tool_error_flags: dict[str, bool] = {}
|
||||
# Cooperative cancellation: set from outside to stop generation
|
||||
@@ -1293,7 +1302,7 @@ class ChatSession:
|
||||
self._ws_id = ws_id
|
||||
self.messages = messages
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
self._last_usage = None
|
||||
self._calibrated_msg_count = 0
|
||||
self._title_generated = True # don't re-title resumed workstreams
|
||||
@@ -1638,6 +1647,124 @@ class ChatSession:
|
||||
"""System messages + conversation history."""
|
||||
return self.system_messages + self.messages
|
||||
|
||||
def _apply_reminders_for_provider(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return a transient copy of *messages* with ``_reminders`` rendered
|
||||
inline for the model.
|
||||
|
||||
Metacognitive nudges live on the message dict's ``_reminders``
|
||||
side-channel regardless of role — user messages carry
|
||||
user-channel nudges (correction / denial / resume / start /
|
||||
completion), tool messages carry tool-channel nudges
|
||||
(tool_error / repeat). Both ride the same side-channel so
|
||||
``self.messages`` and every downstream consumer (UI replay,
|
||||
compaction, title gen, channel adapters, DB) see clean
|
||||
``content``; only the wire-bound copy here carries the
|
||||
rendered reminder.
|
||||
|
||||
For each message that has ``_reminders`` AND has not yet been
|
||||
flagged delivered, build a shallow copy and splice the reminders
|
||||
as ``<system-reminder>`` blocks onto the trailing edge of the
|
||||
copy's ``content`` — string content gets a tail block, list
|
||||
content gets the block on the trailing text part (or a new text
|
||||
part if there isn't one). Messages without ``_reminders`` (or
|
||||
already delivered) pass through unchanged (same object
|
||||
reference) so the common case is allocation-free.
|
||||
|
||||
**Reminder lifecycle.** After a successful provider stream
|
||||
``_mark_reminders_delivered`` flips ``_reminders_delivered`` to
|
||||
``True`` on every message (user or tool) that carried reminders
|
||||
into that call, so subsequent provider calls skip them — the
|
||||
model sees each reminder once, the turn it advised. The
|
||||
``_reminders`` key itself stays on the message dict for the
|
||||
lifetime of the in-memory session so ``/history`` (reconnecting
|
||||
tabs, multi-tab live mirrors) still renders the same nudge
|
||||
bubbles the originating tab saw; only the wire-side replay is
|
||||
suppressed. Compaction is the natural full drain (it replaces
|
||||
``self.messages`` wholesale).
|
||||
|
||||
``sanitize_messages`` later drops both leading-underscore sibling
|
||||
keys (``_reminders`` and ``_reminders_delivered``) on the way to
|
||||
the wire, so the provider sees only ``content`` with the
|
||||
reminder spliced in.
|
||||
|
||||
**Read-only contract on the returned list.** The pass-through
|
||||
path returns the original ``msg`` by reference; callers must
|
||||
not mutate the returned dicts in place (today's only callers —
|
||||
``sanitize_messages`` + provider conversion — construct new
|
||||
dicts, so the contract holds). Mutations on the spliced copy
|
||||
are safe; mutations on a pass-through reference would bleed
|
||||
back into ``self.messages``.
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for msg in messages:
|
||||
raw_reminders = msg.get("_reminders")
|
||||
if not raw_reminders or msg.get("_reminders_delivered"):
|
||||
out.append(msg)
|
||||
continue
|
||||
# Defensive filter — only dict entries are valid; a string /
|
||||
# None / other shape from corruption or partial state must
|
||||
# not abort the whole send via an AttributeError on .get().
|
||||
# Mirrors the same filter ``_build_history`` applies on the
|
||||
# wire-out side.
|
||||
reminders = [r for r in raw_reminders if isinstance(r, dict)]
|
||||
if not reminders:
|
||||
out.append(msg)
|
||||
continue
|
||||
block = "\n\n" + "\n\n".join(
|
||||
render_system_reminder(r.get("text", "")) for r in reminders
|
||||
)
|
||||
copy = dict(msg)
|
||||
content = copy.get("content")
|
||||
if isinstance(content, str):
|
||||
copy["content"] = escape_wrapper_tags(content) + block
|
||||
elif isinstance(content, list):
|
||||
# Shallow-copy the parts list and any text parts we'll
|
||||
# mutate so the original list/dicts in self.messages stay
|
||||
# untouched.
|
||||
new_parts = [
|
||||
dict(p) if isinstance(p, dict) and p.get("type") == "text" else p
|
||||
for p in content
|
||||
]
|
||||
text_parts = [
|
||||
p for p in new_parts if isinstance(p, dict) and p.get("type") == "text"
|
||||
]
|
||||
for part in text_parts:
|
||||
part["text"] = escape_wrapper_tags(part.get("text", ""))
|
||||
if text_parts:
|
||||
text_parts[-1]["text"] = text_parts[-1]["text"] + block
|
||||
else:
|
||||
new_parts.append({"type": "text", "text": block})
|
||||
copy["content"] = new_parts
|
||||
else:
|
||||
# Unexpected shape (None, etc.) — attach as a text-only
|
||||
# content rather than dropping the reminder silently.
|
||||
copy["content"] = block.lstrip()
|
||||
out.append(copy)
|
||||
return out
|
||||
|
||||
def _mark_reminders_delivered(self) -> None:
|
||||
"""Flag every message's ``_reminders`` as delivered.
|
||||
|
||||
Role-agnostic — both user-channel reminders (set by
|
||||
``_attach_pending_user_reminders`` on user messages) and
|
||||
tool-channel reminders (set by the per-result loop on tool
|
||||
messages) ride the same ``_reminders`` side-channel and the
|
||||
same delivered flag. Called after a successful provider
|
||||
stream; subsequent calls to ``_apply_reminders_for_provider``
|
||||
skip messages with this flag, so the model sees each reminder
|
||||
exactly once (the turn it advised). The flag is a sibling key
|
||||
like ``_reminders`` itself; ``sanitize_messages`` strips both
|
||||
before the wire and ``_build_history`` ignores the delivered
|
||||
flag entirely so UI replay parity is preserved across
|
||||
reconnects.
|
||||
"""
|
||||
for msg in self.messages:
|
||||
if msg.get("_reminders") and not msg.get("_reminders_delivered"):
|
||||
msg["_reminders_delivered"] = True
|
||||
|
||||
def _emit_state(self, state: str) -> None:
|
||||
"""Notify UI of a workstream state transition.
|
||||
|
||||
@@ -2113,12 +2240,13 @@ class ChatSession:
|
||||
# Metacognitive user-channel drain: any nudges queued via
|
||||
# _queue_user_advisory (correction/start/completion from this
|
||||
# turn, denial from the previous tool batch, resume from
|
||||
# rehydrate) splice in as <system-reminder> blocks at the
|
||||
# trailing edge of the user content. The DB row stores
|
||||
# ``user_input`` only (line below) so these blocks stay
|
||||
# ephemeral — they advise the next assistant turn and do not
|
||||
# persist across reloads.
|
||||
self._splice_pending_user_advisories(user_msg)
|
||||
# rehydrate) attach to the user message dict's ``_reminders``
|
||||
# side-channel — content stays clean. The wire-side splice
|
||||
# happens later in _apply_reminders_for_provider against a
|
||||
# transient copy. The DB row stores ``user_input`` only (line
|
||||
# below) so reminders stay in-memory only and don't persist
|
||||
# across reloads.
|
||||
self._attach_pending_user_reminders(user_msg)
|
||||
self.messages.append(user_msg)
|
||||
self._msg_tokens.append(max(1, int(self._msg_char_count(user_msg) / self._chars_per_token)))
|
||||
# DB row stores the raw text only; attachments are joined back in
|
||||
@@ -2198,7 +2326,7 @@ class ChatSession:
|
||||
try:
|
||||
while True:
|
||||
self._check_cancelled(my_generation)
|
||||
msgs = self._full_messages()
|
||||
msgs = self._apply_reminders_for_provider(self._full_messages())
|
||||
|
||||
if self.debug:
|
||||
self._debug_print_request(msgs)
|
||||
@@ -2235,7 +2363,7 @@ class ChatSession:
|
||||
self.ui.on_thinking_stop()
|
||||
try:
|
||||
self._compact_messages(auto=True)
|
||||
msgs = self._full_messages()
|
||||
msgs = self._apply_reminders_for_provider(self._full_messages())
|
||||
self.ui.on_thinking_start()
|
||||
stream = self._create_stream_with_retry(msgs)
|
||||
except Exception:
|
||||
@@ -2257,7 +2385,19 @@ class ChatSession:
|
||||
if self._generation != my_generation:
|
||||
return
|
||||
|
||||
self._update_token_table(assistant_msg)
|
||||
# Reuse the wire-bound ``msgs`` we already built for the
|
||||
# stream call instead of re-applying the reminder splice
|
||||
# (perf-2). After mark-delivered runs below, a fresh
|
||||
# _apply_reminders_for_provider would skip the
|
||||
# just-rendered reminders and undercount; passing the
|
||||
# already-rendered list keeps calibration char count
|
||||
# aligned with what the provider actually counted.
|
||||
self._update_token_table(assistant_msg, msgs=msgs)
|
||||
# Reminders that rode this stream have now reached the
|
||||
# model; flag delivered so the next provider call skips
|
||||
# them (one-shot semantics for the wire; UI replay still
|
||||
# surfaces ``_reminders`` for reconnect parity).
|
||||
self._mark_reminders_delivered()
|
||||
self._print_status_line() # Report usage for EVERY API call
|
||||
self.messages.append(assistant_msg)
|
||||
self._msg_tokens.append(
|
||||
@@ -2275,16 +2415,7 @@ class ChatSession:
|
||||
if assistant_msg.get("_provider_content"):
|
||||
provider_data = json.dumps(assistant_msg["_provider_content"])
|
||||
|
||||
# Build tool_calls JSON (excluding memory tools)
|
||||
tool_calls_json: str | None = None
|
||||
if tc:
|
||||
filtered_tc = [
|
||||
call
|
||||
for call in tc
|
||||
if call.get("function", {}).get("name", "") not in ("memory", "recall")
|
||||
]
|
||||
if filtered_tc:
|
||||
tool_calls_json = json.dumps(filtered_tc)
|
||||
tool_calls_json: str | None = json.dumps(tc) if tc else None
|
||||
|
||||
# Save assistant message atomically (content + tool_calls in one row)
|
||||
if content or provider_data is not None or tool_calls_json:
|
||||
@@ -2332,92 +2463,10 @@ class ChatSession:
|
||||
if self._generation != my_generation:
|
||||
return
|
||||
|
||||
# Repeat detection: warn when a tool is called with identical args.
|
||||
# Skip error outputs — retrying a failed tool is valid.
|
||||
# Skip JSON outputs (MCP structured results) — appending
|
||||
# text would corrupt the payload.
|
||||
_tc_by_id = {c["id"]: c for c in tool_calls}
|
||||
_repeat_detected = False
|
||||
_error_prefixes = (
|
||||
"Error",
|
||||
"JSON parse error",
|
||||
"Unknown tool",
|
||||
"Command timed out",
|
||||
"Blocked:",
|
||||
"Denied",
|
||||
)
|
||||
|
||||
# Clear dedup sigs when a write tool executed successfully —
|
||||
# the state has changed so re-running a read tool is valid.
|
||||
_write_tools = frozenset({"write_file", "edit_file", "bash"})
|
||||
if any(
|
||||
tc["function"]["name"] in _write_tools
|
||||
and not any(
|
||||
cid == tc["id"] and isinstance(out, str) and out.startswith(_error_prefixes)
|
||||
for cid, out in results
|
||||
)
|
||||
for tc in tool_calls
|
||||
):
|
||||
self._recent_tool_sigs.clear()
|
||||
for i, (tc_id, output) in enumerate(results):
|
||||
tc = _tc_by_id.get(tc_id)
|
||||
if tc and isinstance(output, str) and not output.startswith(_error_prefixes):
|
||||
raw = tc["function"]["name"] + ":" + tc["function"]["arguments"]
|
||||
sig = hashlib.sha256(raw.encode()).hexdigest()
|
||||
is_json = output.lstrip().startswith(("{", "["))
|
||||
if sig in self._recent_tool_sigs:
|
||||
_repeat_detected = True
|
||||
if not is_json:
|
||||
output += (
|
||||
"\n\n⚠ Warning: this is an identical repeat of a "
|
||||
"previous tool call. The result is the same. "
|
||||
"Try a different approach."
|
||||
)
|
||||
results[i] = (tc_id, output)
|
||||
self.ui.on_info(
|
||||
f"{GRAY}[repeat: {tc['function']['name']}() "
|
||||
f"called with same arguments]{RESET}"
|
||||
)
|
||||
self._recent_tool_sigs.add(sig)
|
||||
if _repeat_detected:
|
||||
# Reset so the model gets a clean slate after the warning.
|
||||
# If it repeats again, a new warning fires.
|
||||
self._recent_tool_sigs.clear()
|
||||
if self._mem_cfg.nudges and should_nudge(
|
||||
"repeat",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
):
|
||||
self._queue_tool_advisory("repeat", format_nudge("repeat"))
|
||||
|
||||
# Tool-error nudge — checked here (pre-iteration) so the
|
||||
# MetacognitiveAdvisory rides the same _collect_advisories
|
||||
# drain pass that handles guard findings and user
|
||||
# interjections. Cooldown gating in should_nudge keeps
|
||||
# this to one nudge per batch even with many failing
|
||||
# tools.
|
||||
if (
|
||||
self._mem_cfg.nudges
|
||||
and any(
|
||||
isinstance(out, str)
|
||||
and (
|
||||
out.startswith("Error")
|
||||
or " error: " in out[:50]
|
||||
or out.startswith("Command timed out")
|
||||
or out.startswith("Unknown tool:")
|
||||
)
|
||||
for _, out in results
|
||||
)
|
||||
and should_nudge(
|
||||
"tool_error",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
memory_count=self._visible_memory_count(),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
)
|
||||
):
|
||||
self._queue_tool_advisory("tool_error", format_nudge("tool_error"))
|
||||
# Repeat-detection + tool-error nudge. Mutates *results*
|
||||
# in place to inject inline warning text on identical
|
||||
# repeats; queues advisories for the next drain pass.
|
||||
self._apply_post_execute_advisories(tool_calls, results)
|
||||
|
||||
# Map tool_call_id → tool name for logging
|
||||
from turnstone.core.tool_advisory import wrap_tool_result
|
||||
@@ -2456,19 +2505,30 @@ class ChatSession:
|
||||
# Capture raw output for DB storage before advisory wrapping
|
||||
raw_output = output
|
||||
|
||||
# Advisory injection: wrap tool output with advisories
|
||||
# (output guard findings, queued user messages, etc.)
|
||||
advisories = self._collect_advisories(
|
||||
# Advisory injection: persistent advisories (output
|
||||
# guard findings, queued user interjections) wrap
|
||||
# into the tool-result envelope and stay in
|
||||
# self.messages. Metacognitive tool-channel
|
||||
# reminders (tool_error / repeat) ride a side-channel
|
||||
# — never inside content — so the model sees the
|
||||
# splice only at the wire boundary via
|
||||
# _apply_reminders_for_provider, while UI/replay
|
||||
# surfaces them as a themed bubble below the tool
|
||||
# result.
|
||||
persistent_advisories, metacog_reminders = self._collect_advisories(
|
||||
assessment, _tc_names.get(tc_id, ""), _ri == _last_idx
|
||||
)
|
||||
if isinstance(output, str):
|
||||
output = wrap_tool_result(output, advisories)
|
||||
elif isinstance(output, list) and advisories:
|
||||
output = wrap_tool_result(output, persistent_advisories)
|
||||
elif isinstance(output, list) and persistent_advisories:
|
||||
# Structured/image output — append advisories as a
|
||||
# text part so they aren't silently dropped.
|
||||
output = [
|
||||
*output,
|
||||
{"type": "text", "text": wrap_tool_result("", advisories)},
|
||||
{
|
||||
"type": "text",
|
||||
"text": wrap_tool_result("", persistent_advisories),
|
||||
},
|
||||
]
|
||||
|
||||
tool_msg: dict[str, Any] = {
|
||||
@@ -2478,6 +2538,15 @@ class ChatSession:
|
||||
}
|
||||
if self._tool_error_flags.pop(tc_id, False):
|
||||
tool_msg["is_error"] = True
|
||||
if metacog_reminders:
|
||||
tool_msg["_reminders"] = metacog_reminders
|
||||
try:
|
||||
self.ui.on_tool_reminder(metacog_reminders, tc_id)
|
||||
except Exception:
|
||||
log.warning(
|
||||
"ui.on_tool_reminder failed; reminder still attached",
|
||||
exc_info=True,
|
||||
)
|
||||
self.messages.append(tool_msg)
|
||||
|
||||
# Token estimation — image content uses a fixed heuristic
|
||||
@@ -2494,27 +2563,27 @@ class ChatSession:
|
||||
tok_est = max(1, int(len(output) / self._chars_per_token))
|
||||
self._msg_tokens.append(tok_est)
|
||||
|
||||
# Log tool result (skip memory tools to avoid noise).
|
||||
# Use raw_output (pre-advisory-wrap) so DB stores clean
|
||||
# tool output without ephemeral advisory XML.
|
||||
# Log tool result. Use raw_output (pre-advisory-wrap)
|
||||
# so the DB stores clean tool output without ephemeral
|
||||
# advisory XML. memory/recall persist alongside every
|
||||
# other tool: replays show the full audit trail, and
|
||||
# output already passes through _truncate_output above
|
||||
# so size is bounded by the same budget every other
|
||||
# tool uses.
|
||||
_tname = _tc_names.get(tc_id, "")
|
||||
if _tname not in (
|
||||
"memory",
|
||||
"recall",
|
||||
):
|
||||
if isinstance(raw_output, list):
|
||||
store_text = " ".join(
|
||||
p.get("text", "") for p in raw_output if p.get("type") == "text"
|
||||
)[:2000]
|
||||
else:
|
||||
store_text = raw_output[:2000]
|
||||
save_message(
|
||||
self._ws_id,
|
||||
"tool",
|
||||
store_text,
|
||||
_tname,
|
||||
tool_call_id=tc_id,
|
||||
)
|
||||
if isinstance(raw_output, list):
|
||||
store_text = " ".join(
|
||||
p.get("text", "") for p in raw_output if p.get("type") == "text"
|
||||
)[:TOOL_RESULT_STORAGE_CAP]
|
||||
else:
|
||||
store_text = raw_output[:TOOL_RESULT_STORAGE_CAP]
|
||||
save_message(
|
||||
self._ws_id,
|
||||
"tool",
|
||||
store_text,
|
||||
_tname,
|
||||
tool_call_id=tc_id,
|
||||
)
|
||||
# Inject user feedback from approval prompt (e.g. "y, use full path")
|
||||
if user_feedback:
|
||||
self.messages.append({"role": "user", "content": user_feedback})
|
||||
@@ -2583,10 +2652,7 @@ class ChatSession:
|
||||
# Drain any queued user messages so they appear in the
|
||||
# conversation and are visible on the next send().
|
||||
self._flush_queued_messages()
|
||||
# Tool-channel nudges queued earlier in this generation
|
||||
# (tool_error, repeat) belong to the abandoned batch — drop
|
||||
# them so they don't bleed into the next send()'s tool loop.
|
||||
self._pending_tool_advisories.clear()
|
||||
self._drain_pending_advisories()
|
||||
# No need to clear _cancel_event — it's replaced per-generation
|
||||
# in send(), so this generation's event is simply discarded.
|
||||
self.ui.on_info("[Generation cancelled]")
|
||||
@@ -2596,15 +2662,29 @@ class ChatSession:
|
||||
except KeyboardInterrupt as exc:
|
||||
self._synthesize_cancelled_results("Interrupted by user.")
|
||||
self._flush_queued_messages()
|
||||
self._pending_tool_advisories.clear()
|
||||
self._drain_pending_advisories()
|
||||
self._record_fatal_error(exc)
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._flush_queued_messages()
|
||||
self._pending_tool_advisories.clear()
|
||||
self._drain_pending_advisories()
|
||||
self._record_fatal_error(exc)
|
||||
raise
|
||||
|
||||
def _drain_pending_advisories(self) -> None:
|
||||
"""Drop both advisory channels' pending buffers.
|
||||
|
||||
Both channels are scoped to the current generation: tool-channel
|
||||
nudges (``tool_error``, ``repeat``) queued earlier in this batch
|
||||
and user-channel nudges (``correction``, ``denial``, …) queued
|
||||
during ``_check_metacognitive_nudge`` but not yet drained. When
|
||||
a generation is abandoned (cancel, KeyboardInterrupt, unexpected
|
||||
exception) both must drop so they don't bleed into the next
|
||||
send's tool loop or next user turn.
|
||||
"""
|
||||
self._pending_tool_advisories.clear()
|
||||
self._pending_user_advisories.clear()
|
||||
|
||||
def _synthesize_cancelled_results(self, reason: str) -> None:
|
||||
"""Synthesize tool_result messages for orphaned tool_calls after cancel.
|
||||
|
||||
@@ -3137,8 +3217,24 @@ class ChatSession:
|
||||
text_chars, images, doc_chars = self._msg_text_chars(msg)
|
||||
return text_chars + doc_chars + int(images * self._IMAGE_TOKENS * self._chars_per_token)
|
||||
|
||||
def _update_token_table(self, assistant_msg: dict[str, Any]) -> None:
|
||||
"""Update per-message token estimates using API usage data."""
|
||||
def _update_token_table(
|
||||
self,
|
||||
assistant_msg: dict[str, Any],
|
||||
*,
|
||||
msgs: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
"""Update per-message token estimates using API usage data.
|
||||
|
||||
*msgs* (optional) is the wire-bound message list already built
|
||||
for the stream call — passing it avoids a redundant
|
||||
``_apply_reminders_for_provider`` walk and, more importantly,
|
||||
ensures the char count matches the bytes the provider counted
|
||||
even after ``_mark_reminders_delivered`` has flipped the flag
|
||||
on the reminders that rode the stream. When *msgs* is None the
|
||||
caller didn't pre-build (rare path) — fall back to applying
|
||||
the splice on the fly, but be aware the result will be
|
||||
reminder-free if delivered flags are already set.
|
||||
"""
|
||||
if not self._last_usage:
|
||||
return
|
||||
|
||||
@@ -3148,8 +3244,16 @@ class ChatSession:
|
||||
# Calibrate chars_per_token ratio from actual usage.
|
||||
# Images get a fixed token budget (subtracted). Documents
|
||||
# tokenize non-linearly depending on provider — excluded from
|
||||
# calibration so they don't skew the text ratio.
|
||||
all_msgs = self._full_messages() # system + self.messages (before append)
|
||||
# calibration so they don't skew the text ratio. ``all_msgs``
|
||||
# must reflect what the provider actually counted in
|
||||
# ``prompt_tokens``: when called from the loop with the
|
||||
# pre-built ``msgs``, that's exact; without it, fall back to
|
||||
# applying the splice fresh (post-mark-delivered the result
|
||||
# may undercount, but no caller currently takes this path
|
||||
# after a successful stream).
|
||||
all_msgs = (
|
||||
msgs if msgs is not None else self._apply_reminders_for_provider(self._full_messages())
|
||||
) # system + self.messages (before append)
|
||||
active_tools = self._get_active_tools() or []
|
||||
tool_def_chars = sum(len(json.dumps(t)) for t in active_tools)
|
||||
text_chars = 0
|
||||
@@ -3391,7 +3495,7 @@ class ChatSession:
|
||||
self.messages = [summary_user, summary_asst]
|
||||
# File contents are gone after compaction — force re-read before edit_file
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
|
||||
# Rebuild token table
|
||||
su_tok = max(1, int(self._msg_char_count(summary_user) / self._chars_per_token))
|
||||
@@ -3780,12 +3884,26 @@ class ChatSession:
|
||||
assessment: OutputAssessment | None,
|
||||
func_name: str,
|
||||
is_last_in_batch: bool,
|
||||
) -> list[ToolAdvisory]:
|
||||
) -> tuple[list[ToolAdvisory], list[dict[str, str]]]:
|
||||
"""Gather advisories to attach to a tool result message.
|
||||
|
||||
Returns an empty list when no advisories apply (common case).
|
||||
Guard advisories attach per-result; user messages drain on the
|
||||
last result in the batch only.
|
||||
Returns ``(persistent, metacog_reminders)``:
|
||||
|
||||
- ``persistent`` — guard findings + user interjections that ride
|
||||
inside the tool-result envelope via ``wrap_tool_result``.
|
||||
These are conversation history and must persist in
|
||||
``self.messages``.
|
||||
- ``metacog_reminders`` — list of ``{"type", "text"}`` dicts for
|
||||
``tool_error`` / ``repeat`` nudges that the caller attaches to
|
||||
the tool message dict's ``_reminders`` side-channel. Like
|
||||
user-channel reminders, they are spliced into ``content`` only
|
||||
at the wire boundary by ``_apply_reminders_for_provider`` and
|
||||
surfaced separately on the UI as a themed bubble below the
|
||||
tool result.
|
||||
|
||||
Both lists are empty when no advisories apply (common case).
|
||||
Guard advisories attach per-result; user messages and
|
||||
metacognitive nudges drain on the last result in the batch only.
|
||||
"""
|
||||
from turnstone.core.tool_advisory import GuardAdvisory, UserInterjection
|
||||
|
||||
@@ -3801,26 +3919,25 @@ class ChatSession:
|
||||
if is_last_in_batch:
|
||||
self._pending_tool_advisories.clear()
|
||||
self._flush_queued_messages()
|
||||
return []
|
||||
return [], []
|
||||
|
||||
advisories: list[ToolAdvisory] = []
|
||||
persistent: list[ToolAdvisory] = []
|
||||
metacog_reminders: list[dict[str, str]] = []
|
||||
|
||||
# Output guard advisory
|
||||
# Output guard advisory — persists with the tool result.
|
||||
if assessment is not None:
|
||||
advisories.append(GuardAdvisory(assessment=assessment, func_name=func_name))
|
||||
persistent.append(GuardAdvisory(assessment=assessment, func_name=func_name))
|
||||
|
||||
# Metacognitive tool-channel drain — fires once per batch on the
|
||||
# last result. Queued by _queue_tool_advisory from the
|
||||
# tool_error and repeat detection paths just before this loop.
|
||||
# last result. Queued by _queue_tool_advisory from the
|
||||
# tool_error / repeat detection paths just before this loop.
|
||||
# Lands on the tool message dict's ``_reminders`` side-channel
|
||||
# (caller's responsibility) so it stays out of persisted content
|
||||
# and rides the wire only via the transient-copy splice.
|
||||
if is_last_in_batch and self._pending_tool_advisories:
|
||||
from turnstone.core.tool_advisory import MetacognitiveAdvisory
|
||||
|
||||
drained = list(self._pending_tool_advisories)
|
||||
self._pending_tool_advisories.clear()
|
||||
advisories.extend(
|
||||
MetacognitiveAdvisory(nudge_type=nt, message=text) for nt, text in drained
|
||||
)
|
||||
self._emit_nudge_ping(nt for nt, _ in drained)
|
||||
metacog_reminders.extend({"type": nt, "text": text} for nt, text in drained)
|
||||
|
||||
# Drain queued user messages on the last result in the batch.
|
||||
# Attachment-bearing items fall back to a full multipart user
|
||||
@@ -3834,7 +3951,7 @@ class ChatSession:
|
||||
if att_ids:
|
||||
attachment_items.append((queue_msg_id, msg, priority, att_ids))
|
||||
else:
|
||||
advisories.append(UserInterjection(message=msg, priority=priority))
|
||||
persistent.append(UserInterjection(message=msg, priority=priority))
|
||||
if attachment_items:
|
||||
from turnstone.core.tool_advisory import PRIORITY_IMPORTANT
|
||||
|
||||
@@ -3845,7 +3962,7 @@ class ChatSession:
|
||||
)
|
||||
self._append_user_turn(text, resolved, send_id=queue_msg_id)
|
||||
|
||||
return advisories
|
||||
return persistent, metacog_reminders
|
||||
|
||||
# -- Two-phase tool execution -----------------------------------------------
|
||||
#
|
||||
@@ -3974,7 +4091,14 @@ class ChatSession:
|
||||
)
|
||||
return item["call_id"], item["error"]
|
||||
if item.get("denied"):
|
||||
return item["call_id"], item.get("denial_msg", "Denied by user")
|
||||
msg = item.get("denial_msg", "Denied by user")
|
||||
self._report_tool_result(
|
||||
item["call_id"],
|
||||
item.get("func_name", "unknown"),
|
||||
msg,
|
||||
is_error=True,
|
||||
)
|
||||
return item["call_id"], msg
|
||||
try:
|
||||
result: tuple[str, str | list[dict[str, Any]]] = item["execute"](item)
|
||||
return result
|
||||
@@ -5377,59 +5501,52 @@ class ChatSession:
|
||||
def _queue_user_advisory(self, nudge_type: str, text: str) -> None:
|
||||
"""Queue a metacognitive nudge for the next user turn.
|
||||
|
||||
Drains in ``_append_user_turn`` as a ``<system-reminder>`` block
|
||||
appended to the user message body. Used for nudges that respond
|
||||
to user behaviour: ``correction``, ``denial``, ``resume``,
|
||||
Drains in ``_append_user_turn`` onto the user message dict's
|
||||
``_reminders`` side-channel. Used for nudges that respond to
|
||||
user behaviour: ``correction``, ``denial``, ``resume``,
|
||||
``start``, ``completion``.
|
||||
"""
|
||||
self._pending_user_advisories.append((nudge_type, text))
|
||||
|
||||
def _splice_pending_user_advisories(self, user_msg: dict[str, Any]) -> None:
|
||||
"""Drain ``_pending_user_advisories`` into *user_msg*'s content.
|
||||
def _attach_pending_user_reminders(self, user_msg: dict[str, Any]) -> None:
|
||||
"""Drain ``_pending_user_advisories`` onto *user_msg*'s ``_reminders``
|
||||
sibling key (a side-channel, never inside ``content``).
|
||||
|
||||
Mutates *user_msg* in place — the caller appends it after.
|
||||
Renders each queued nudge as a ``<system-reminder>`` block
|
||||
(same envelope as ``wrap_tool_result``) and attaches them to
|
||||
the trailing edge of the user content. Every text segment in
|
||||
the user content is passed through ``escape_wrapper_tags``
|
||||
first so a user typing literal ``<system-reminder>`` cannot
|
||||
fabricate an envelope the model would treat as a
|
||||
Turnstone-issued reminder. For attachment-bearing turns the
|
||||
blocks land on the trailing text part so they stay glued to
|
||||
the same multipart turn.
|
||||
Mutates *user_msg* in place — the caller appends it after. The
|
||||
rendered ``<system-reminder>`` envelope is built later in
|
||||
``_apply_reminders_for_provider`` against a transient copy, so
|
||||
the model still sees the reminder spliced into ``content`` at
|
||||
the wire boundary while ``self.messages`` and every downstream
|
||||
consumer (UI replay, compaction, title gen, channel adapters,
|
||||
DB) see clean user text.
|
||||
|
||||
``_reminders`` rides the leading-underscore convention used by
|
||||
other internal sibling metadata (``_attachments_meta``,
|
||||
``_provider_content``); ``sanitize_messages`` strips it before
|
||||
the wire on its own pass.
|
||||
|
||||
Also fires the live ``on_user_reminder`` UI hook so any open
|
||||
SSE consumers (other browser tabs, CLI mirrors, eventual
|
||||
channel adapters) can render the reminder bubble in lockstep
|
||||
with the originating tab's optimistic render. Hook failures
|
||||
are logged and swallowed: the side-channel write is the
|
||||
load-bearing op, and a UI implementation throwing here must
|
||||
not abort the user's send (which would otherwise drop both the
|
||||
user message and the queued nudges).
|
||||
"""
|
||||
if not self._pending_user_advisories:
|
||||
return
|
||||
from turnstone.core.tool_advisory import escape_wrapper_tags, render_system_reminder
|
||||
|
||||
items = list(self._pending_user_advisories)
|
||||
self._pending_user_advisories.clear()
|
||||
|
||||
block = "\n\n" + "\n\n".join(render_system_reminder(text) for _, text in items)
|
||||
content = user_msg["content"]
|
||||
if isinstance(content, str):
|
||||
user_msg["content"] = escape_wrapper_tags(content) + block
|
||||
else:
|
||||
text_parts = [p for p in content if isinstance(p, dict) and p.get("type") == "text"]
|
||||
for part in text_parts:
|
||||
part["text"] = escape_wrapper_tags(part.get("text", ""))
|
||||
if text_parts:
|
||||
text_parts[-1]["text"] = text_parts[-1]["text"] + block
|
||||
else:
|
||||
content.append({"type": "text", "text": block})
|
||||
reminders = [{"type": nudge_type, "text": text} for nudge_type, text in items]
|
||||
user_msg["_reminders"] = reminders
|
||||
|
||||
self._emit_nudge_ping(nudge_type for nudge_type, _ in items)
|
||||
|
||||
def _emit_nudge_ping(self, types: Iterable[str]) -> None:
|
||||
"""Surface the ``[metacognition: nudge injected — …]`` UI line.
|
||||
|
||||
Centralised so both drain sites (tool channel via
|
||||
``_collect_advisories``, user channel via
|
||||
``_splice_pending_user_advisories``) emit the same wording.
|
||||
"""
|
||||
joined = ", ".join(types)
|
||||
if joined:
|
||||
self.ui.on_info(f"{GRAY}[metacognition: nudge injected — {joined}]{RESET}")
|
||||
try:
|
||||
self.ui.on_user_reminder(reminders)
|
||||
except Exception:
|
||||
log.warning("ui.on_user_reminder failed; reminder still attached", exc_info=True)
|
||||
|
||||
def _queue_tool_advisory(self, nudge_type: str, text: str) -> None:
|
||||
"""Queue a metacognitive nudge for the next tool-result batch.
|
||||
@@ -5441,6 +5558,95 @@ class ChatSession:
|
||||
"""
|
||||
self._pending_tool_advisories.append((nudge_type, text))
|
||||
|
||||
def _apply_post_execute_advisories(
|
||||
self,
|
||||
tool_calls: list[dict[str, Any]],
|
||||
results: list[tuple[str, str | list[dict[str, Any]]]],
|
||||
) -> None:
|
||||
"""Run repeat detection + tool-error nudge over a freshly-executed batch.
|
||||
|
||||
Mutates *results* in place when an identical-repeat warning is
|
||||
appended to a tool's text output. Updates ``self._repeat_detector``,
|
||||
``self._pending_tool_advisories``, and ``self._metacog_state``
|
||||
(cooldown timestamp via ``should_nudge``). The operator-visible
|
||||
signal is the themed ``tool_reminder`` bubble below the tool
|
||||
block — emitted by the per-result loop downstream when the
|
||||
drained metacog reminders attach to the tool message dict's
|
||||
``_reminders`` side-channel.
|
||||
|
||||
Repeat detection's job is to nudge a flaky local model out of a
|
||||
loop where it keeps making the same tool call ("``bash(cmd='echo
|
||||
test')`` × 3" being the canonical example). It fires on the
|
||||
consecutive-streak signal alone, with no regard for the tool's
|
||||
success / failure / output content — same (name, args) for N
|
||||
turns in a row is by definition stuck. ``RepeatDetector.record``
|
||||
already resets the streak on any different signature, so an
|
||||
intervening tool call (read, write, anything different) breaks
|
||||
the streak naturally without an explicit clear here.
|
||||
|
||||
``_tool_error_flags`` is the authoritative is_error signal —
|
||||
consumed below for the tool-error nudge gate; the per-result
|
||||
loop in ``_run_loop`` ``.pop``s it after this returns.
|
||||
"""
|
||||
# Repeat detection: warn when a tool is called with identical
|
||||
# args N times in a row. Independent of success/failure — the
|
||||
# stuck-loop pattern is sig-driven, not state-driven. JSON
|
||||
# outputs (MCP structured results) are tracked but exempt from
|
||||
# the inline warning text (appending text would corrupt the
|
||||
# payload).
|
||||
_tc_by_id = {c["id"]: c for c in tool_calls}
|
||||
_repeat_detected = False
|
||||
|
||||
for i, (tc_id, output) in enumerate(results):
|
||||
tc = _tc_by_id.get(tc_id)
|
||||
if tc and isinstance(output, str):
|
||||
raw = tc["function"]["name"] + ":" + tc["function"]["arguments"]
|
||||
sig = hashlib.sha256(raw.encode()).hexdigest()
|
||||
is_json = output.lstrip().startswith(("{", "["))
|
||||
if self._repeat_detector.record(sig):
|
||||
_repeat_detected = True
|
||||
if not is_json:
|
||||
output += (
|
||||
"\n\n⚠ Warning: this is an identical repeat of a "
|
||||
"previous tool call. The result is the same. "
|
||||
"Try a different approach."
|
||||
)
|
||||
results[i] = (tc_id, output)
|
||||
# The themed ``tool_reminder`` bubble below the tool
|
||||
# block carries the operator-visible signal; the
|
||||
# tool-name context comes from the visible tool
|
||||
# block immediately above the bubble, so a separate
|
||||
# diagnostic info line would just duplicate it.
|
||||
|
||||
if _repeat_detected:
|
||||
# Reset so the model gets a clean slate after the warning.
|
||||
# If it repeats again, a new warning fires.
|
||||
self._repeat_detector.clear()
|
||||
if self._mem_cfg.nudges and should_nudge(
|
||||
"repeat",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
):
|
||||
self._queue_tool_advisory("repeat", format_nudge("repeat"))
|
||||
|
||||
# Tool-error nudge — queued so the MetacognitiveAdvisory rides
|
||||
# the same _collect_advisories drain pass as guard findings and
|
||||
# user interjections. Cooldown gating in should_nudge keeps
|
||||
# this to one nudge per batch even with many failing tools.
|
||||
if (
|
||||
self._mem_cfg.nudges
|
||||
and any(self._tool_error_flags.get(tc_id) for tc_id, _ in results)
|
||||
and should_nudge(
|
||||
"tool_error",
|
||||
self._metacog_state,
|
||||
message_count=len(self.messages),
|
||||
memory_count=self._visible_memory_count(),
|
||||
cooldown_secs=self._mem_cfg.nudge_cooldown,
|
||||
)
|
||||
):
|
||||
self._queue_tool_advisory("tool_error", format_nudge("tool_error"))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Coordinator tools — reachable only when ``kind == "coordinator"``.
|
||||
# All six dispatch through ``self._coord_client`` which is None when
|
||||
@@ -9203,7 +9409,7 @@ class ChatSession:
|
||||
elif cmd == "/clear":
|
||||
self.messages.clear()
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
self._last_usage = None
|
||||
self._calibrated_msg_count = 0
|
||||
self._msg_tokens = []
|
||||
@@ -9214,7 +9420,7 @@ class ChatSession:
|
||||
|
||||
self.messages.clear()
|
||||
self._read_files.clear()
|
||||
self._recent_tool_sigs.clear()
|
||||
self._repeat_detector.clear()
|
||||
self._last_usage = None
|
||||
self._calibrated_msg_count = 0
|
||||
self._msg_tokens = []
|
||||
|
||||
@@ -13,6 +13,7 @@ import contextlib
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
@@ -28,6 +29,22 @@ if TYPE_CHECKING:
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# Maps each workstream kind to the ``services.service_type`` its hosting
|
||||
# process registers under. Used by ``SessionManager.close_idle`` pass 2
|
||||
# to enumerate live peer processes for orphan-reaper liveness scoping.
|
||||
# Server processes register as ``("server", node_id, ...)`` (see
|
||||
# ``turnstone/server.py``); the console process as ``("console",
|
||||
# "console", ...)`` (see ``turnstone/console/server.py``). Deriving from
|
||||
# kind here removes a duplicated-config footgun: any caller that builds
|
||||
# a ``SessionManager`` automatically gets the correct service_type for
|
||||
# its kind, with no risk of miswiring INTERACTIVE→"console" or vice
|
||||
# versa.
|
||||
_KIND_SERVICE_TYPE: dict[WorkstreamKind, str] = {
|
||||
WorkstreamKind.INTERACTIVE: "server",
|
||||
WorkstreamKind.COORDINATOR: "console",
|
||||
}
|
||||
|
||||
|
||||
class SessionKindAdapter(Protocol):
|
||||
"""Per-kind construction + cleanup policies the shared ``SessionManager`` delegates to.
|
||||
|
||||
@@ -197,13 +214,21 @@ class SessionManager:
|
||||
# manager never reads them.
|
||||
self._active_id: str | None = None
|
||||
self._eviction_count: int = 0
|
||||
# Optional state-change observer. The CLI sets this to a
|
||||
# callback that prints a background-attention notification
|
||||
# when a non-focused workstream transitions to ATTENTION.
|
||||
# Web/coord paths use the event_emitter's emit_state for their
|
||||
# own fan-out; this is a second, manager-level hook for callers
|
||||
# that don't consume SSE.
|
||||
self._on_state_change: Callable[[str, WorkstreamState], None] | None = None
|
||||
# State-change subscribers. Multi-subscriber to support the
|
||||
# CLI's background-attention notification AND the in-process
|
||||
# ``SameNodeChildSource`` strategy that delivers child
|
||||
# workstream state changes to a parent's UI without going
|
||||
# through the cluster bus. Each callback fires under
|
||||
# exception-suppression so one failing subscriber doesn't
|
||||
# block the others. Subscribers register via
|
||||
# :meth:`subscribe_to_state`. ``_state_subscribers_lock``
|
||||
# guards mutation + snapshot — set_state copies the list
|
||||
# under the lock then iterates the snapshot unlocked so a
|
||||
# slow subscriber doesn't block subscribe/unsubscribe (and
|
||||
# so concurrent subscribe/unsubscribe during a state event
|
||||
# can't shift the iterator's index — caught by /review bug-1).
|
||||
self._state_subscribers: list[Callable[[str, WorkstreamState], None]] = []
|
||||
self._state_subscribers_lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Properties
|
||||
@@ -217,6 +242,16 @@ class SessionManager:
|
||||
def kind(self) -> WorkstreamKind:
|
||||
return self._adapter.kind
|
||||
|
||||
@property
|
||||
def _service_type(self) -> str | None:
|
||||
"""``services.service_type`` this manager's hosting process registers
|
||||
under, derived from its ``kind``. Used by ``close_idle`` pass 2 to
|
||||
enumerate live peer processes. Returns ``None`` for kinds that have
|
||||
no production service mapping (only the two existing kinds map
|
||||
today; ``None`` would be a marker for a future kind without a
|
||||
clustered hosting model)."""
|
||||
return _KIND_SERVICE_TYPE.get(self.kind)
|
||||
|
||||
@property
|
||||
def count(self) -> int:
|
||||
with self._lock:
|
||||
@@ -604,6 +639,22 @@ class SessionManager:
|
||||
# last close(). The next set_state() call syncs it
|
||||
# naturally; writing 'idle' here could race a concurrent
|
||||
# close() that writes 'closed' under self._lock.
|
||||
#
|
||||
# Bump only ``updated`` (no state write) so this row's
|
||||
# timestamp is fresh against the orphan-reaper cutoff —
|
||||
# otherwise a concurrent close_idle pass-2 in this same
|
||||
# process could clobber a freshly-rehydrated row whose
|
||||
# ``updated`` is older than the cutoff. The pure-
|
||||
# timestamp write is safe against concurrent close()
|
||||
# because close still wins on the state column.
|
||||
try:
|
||||
self._storage.touch_workstream(ws_id)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"session_mgr.touch_workstream_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_rehydrated(ws)
|
||||
return ws
|
||||
@@ -773,9 +824,36 @@ class SessionManager:
|
||||
log.debug("session_mgr.state_update_failed ws=%s", ws_id[:8], exc_info=True)
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_state(ws, state)
|
||||
if self._on_state_change is not None:
|
||||
# Snapshot under the subscribers lock so concurrent
|
||||
# subscribe / unsubscribe can't shift the iterator's index
|
||||
# mid-dispatch (skipping or repeating callbacks). Iterate
|
||||
# the snapshot WITHOUT the lock so a slow callback doesn't
|
||||
# block subscribe / unsubscribe.
|
||||
with self._state_subscribers_lock:
|
||||
subscribers = list(self._state_subscribers)
|
||||
for callback in subscribers:
|
||||
with contextlib.suppress(Exception):
|
||||
self._on_state_change(ws_id, state)
|
||||
callback(ws_id, state)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State-change subscription
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def subscribe_to_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Register ``callback`` to fire on every workstream state change.
|
||||
|
||||
Multiple subscribers are supported and fire in registration order.
|
||||
Each callback is wrapped in exception-suppression so a failing
|
||||
subscriber doesn't block the others. Use
|
||||
:meth:`unsubscribe_from_state` to remove.
|
||||
"""
|
||||
with self._state_subscribers_lock:
|
||||
self._state_subscribers.append(callback)
|
||||
|
||||
def unsubscribe_from_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Remove a previously-registered state-change callback. No-op if absent."""
|
||||
with self._state_subscribers_lock, contextlib.suppress(ValueError):
|
||||
self._state_subscribers.remove(callback)
|
||||
|
||||
def cancel(self, ws_id: str) -> bool:
|
||||
"""Cancel in-flight generation and unblock any pending approval / plan.
|
||||
@@ -804,15 +882,52 @@ class SessionManager:
|
||||
def close_idle(self, max_age_seconds: float) -> list[str]:
|
||||
"""Close IDLE workstreams inactive for more than ``max_age_seconds``.
|
||||
|
||||
Returns the list of closed ws_ids. Unlike the old WSM version,
|
||||
this does NOT skip the last workstream — the default-startup
|
||||
relic is gone, callers can handle the 0-workstream case.
|
||||
Two-pass shape:
|
||||
|
||||
- Pass 1 (in-memory): close loaded ``IDLE`` rows whose
|
||||
``ws.last_active`` (monotonic) is past timeout. Closes only
|
||||
``IDLE`` so legitimately-attentive rows (waiting for user
|
||||
response) stay live.
|
||||
- Pass 2 (DB orphans): bulk-close DB rows of this manager's
|
||||
kind whose ``updated`` is past the wall-clock cutoff and
|
||||
which are not currently loaded. This catches workstreams
|
||||
left behind by prior process incarnations — a process crash
|
||||
/restart leaves rows in non-terminal states forever
|
||||
otherwise. Closes ``idle/thinking/attention/running``
|
||||
because any matching row is by definition not loaded by any
|
||||
live process and cannot be in a live interaction.
|
||||
|
||||
**Liveness scoping** (the rendezvous router's primitive
|
||||
since PR #384): when ``self._service_type`` resolves to a
|
||||
known service type — both production kinds do — pass 2
|
||||
calls ``storage.list_services`` to enumerate peer processes
|
||||
with recent heartbeats and protects rows whose ``node_id``
|
||||
matches a live ``service_id`` from reap, even when *this*
|
||||
manager is on a different node. This is essential for
|
||||
containerized deployments with dynamic hostnames: dead-pod
|
||||
rows fall out of the live set after the heartbeat window
|
||||
and become reapable; alive-pod rows stay protected as long
|
||||
as the owner heartbeats. A future kind with no service
|
||||
registration would resolve ``_service_type`` to ``None``
|
||||
and skip the live-services lookup (single-process / CLI).
|
||||
|
||||
**Conservative fallback**: if ``list_services`` raises,
|
||||
pass 2 is skipped entirely this tick — never reap when
|
||||
liveness state is unknown. Pass 1 still runs. Next tick
|
||||
retries the lookup.
|
||||
|
||||
Returns the combined list of closed ws_ids (in-memory first,
|
||||
then DB orphans). Pass 1 emits ``ws_closed``; pass 2 does
|
||||
not, because never-loaded rows have no SSE listeners
|
||||
expecting them.
|
||||
|
||||
Atomic pop per victim under ``self._lock`` (bug-5): a pending
|
||||
tool result can flip state IDLE→RUNNING between the snapshot
|
||||
and the close, so the state test + pop must run together.
|
||||
Batches every pop under one ``self._lock`` acquisition (perf-5)
|
||||
rather than locking once per victim.
|
||||
rather than locking once per victim. The DB pass runs OUTSIDE
|
||||
``self._lock`` — only a brief lock to snapshot loaded keys —
|
||||
so a slow UPDATE doesn't block create/get/set_state.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
popped: list[Workstream] = []
|
||||
@@ -853,6 +968,63 @@ class SessionManager:
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_closed(ws.id, name=ws.name)
|
||||
closed_ids.append(ws.id)
|
||||
|
||||
# Pass 2: reap DB orphans of this kind older than the cutoff.
|
||||
# Snapshot loaded keys under self._lock briefly so a concurrent
|
||||
# create/load doesn't get its row clobbered by the UPDATE; release
|
||||
# before the DB call.
|
||||
#
|
||||
# Liveness scoping uses ``services.last_heartbeat`` — the same
|
||||
# primitive the rendezvous router (PR #384) uses for routing. A
|
||||
# row's ``node_id`` is stamped at create time and never updated;
|
||||
# in containerized deployments with dynamic hostnames the dead
|
||||
# pod's ``node_id`` points at a service that's no longer
|
||||
# heartbeating, so the row falls through to reap. Conversely,
|
||||
# rows whose ``node_id`` matches a heartbeating service are
|
||||
# protected even when *this* manager is on a different node —
|
||||
# the alive peer may legitimately have them loaded.
|
||||
with self._lock:
|
||||
loaded = list(self._workstreams.keys())
|
||||
cutoff = (datetime.now(UTC) - timedelta(seconds=max_age_seconds)).strftime(
|
||||
"%Y-%m-%dT%H:%M:%S"
|
||||
)
|
||||
live_node_ids: list[str] | None = None
|
||||
skip_pass_2 = False
|
||||
if self._service_type is not None:
|
||||
try:
|
||||
live_services = self._storage.list_services(self._service_type)
|
||||
live_node_ids = [
|
||||
str(svc["service_id"]) for svc in live_services if svc.get("service_id")
|
||||
]
|
||||
except Exception:
|
||||
# Conservative fallback: skip pass 2 entirely this tick
|
||||
# so we can't accidentally reap rows whose owners we
|
||||
# failed to enumerate. Next tick retries.
|
||||
log.debug(
|
||||
"session_mgr.list_services_failed kind=%s",
|
||||
self.kind.value,
|
||||
exc_info=True,
|
||||
)
|
||||
skip_pass_2 = True
|
||||
orphans: list[str] = []
|
||||
if not skip_pass_2:
|
||||
try:
|
||||
orphans = self._storage.bulk_close_stale_orphans(
|
||||
self.kind, cutoff, loaded, live_node_ids=live_node_ids
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"session_mgr.bulk_close_orphans_failed kind=%s",
|
||||
self.kind.value,
|
||||
exc_info=True,
|
||||
)
|
||||
if orphans:
|
||||
log.info(
|
||||
"session_mgr.bulk_close_orphans count=%d kind=%s",
|
||||
len(orphans),
|
||||
self.kind.value,
|
||||
)
|
||||
closed_ids.extend(orphans)
|
||||
return closed_ids
|
||||
|
||||
def _close_if_idle_locked(self, ws_id: str) -> Workstream | None:
|
||||
|
||||
@@ -394,6 +394,15 @@ class SessionEndpointConfig:
|
||||
# separate ``/history`` endpoint and doesn't render the per-tab
|
||||
# status bar). Kinds that don't need pre-replay wire ``None``.
|
||||
events_replay: EventsReplay | None = None
|
||||
# async (ws, ui, request) -> None. Kind-specific async pre-step
|
||||
# the lifted ``events`` body awaits BEFORE iterating
|
||||
# ``events_replay``. Lets a kind move blocking storage I/O off
|
||||
# the event loop (via ``asyncio.to_thread``) and stash results
|
||||
# on ``request.state`` for the sync replay generator to read.
|
||||
# Interactive uses it to pre-load intent_verdicts +
|
||||
# output_assessments so ``_build_history``'s decoration stays
|
||||
# off the hot path. Coord wires ``None``.
|
||||
events_replay_prepare: Callable[..., Any] | None = None
|
||||
# (request) -> Executor for the SSE live-loop's blocking
|
||||
# ``queue.get`` wait. Interactive returns the dedicated
|
||||
# ``request.app.state.sse_executor`` (200-thread pool) so SSE
|
||||
@@ -1203,6 +1212,8 @@ def make_open_handler(
|
||||
"""
|
||||
|
||||
async def open_ws(request: Request) -> Response:
|
||||
import asyncio
|
||||
|
||||
if cfg.permission_gate is not None:
|
||||
err = cfg.permission_gate(request)
|
||||
if err is not None:
|
||||
@@ -1304,7 +1315,14 @@ def make_open_handler(
|
||||
# emit_rehydrated path).
|
||||
if cfg.open_post_load is not None:
|
||||
try:
|
||||
cfg.open_post_load(request, ws)
|
||||
# Off-loop: interactive's post_load runs the sync
|
||||
# ``_build_history`` (storage I/O for verdict
|
||||
# indexes + message reconstruction) — without the
|
||||
# to_thread wrap this blocks the event loop on every
|
||||
# workstream open, mirroring the SSE replay path
|
||||
# that's already protected via
|
||||
# ``events_replay_prepare``.
|
||||
await asyncio.to_thread(cfg.open_post_load, request, ws)
|
||||
except Exception:
|
||||
# Post-load is observational — never let a hook bug
|
||||
# block the open. Log + continue.
|
||||
@@ -1454,6 +1472,20 @@ def make_events_handler(cfg: SessionEndpointConfig) -> Handler:
|
||||
# 500-slot cap on a chatty mid-generation workstream)
|
||||
# while replay was being built.
|
||||
if replay_cb is not None:
|
||||
# Kind-specific async prep — runs before the sync
|
||||
# replay generator iterates so blocking storage
|
||||
# I/O lands in the executor pool rather than the
|
||||
# event loop's hot path. Interactive uses this
|
||||
# to pre-load verdict indexes; coord skips.
|
||||
if cfg.events_replay_prepare is not None:
|
||||
try:
|
||||
await cfg.events_replay_prepare(ws, ui, request)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"ws.events.replay_prepare_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
for ev in replay_cb(ws, ui, request):
|
||||
yield {"data": json.dumps(ev)}
|
||||
@@ -2240,6 +2272,34 @@ def make_history_handler(cfg: SessionEndpointConfig) -> Handler:
|
||||
messages = await asyncio.to_thread(storage.load_messages, ws_id, limit=limit)
|
||||
except Exception:
|
||||
log.debug("ws.history.load_failed ws=%s", ws_id[:8], exc_info=True)
|
||||
# Audit-trail decoration — attach persisted intent_verdict and
|
||||
# output_assessment data to each assistant.tool_calls entry so
|
||||
# the dashboard's history replay paints the same verdict pills
|
||||
# / output-warning bubbles the live SSE path shows. Both
|
||||
# storage queries are off-loop via ``to_thread``. Best-effort:
|
||||
# any failure leaves messages undecorated — replay degrades to
|
||||
# the pre-decoration shape rather than 500-ing.
|
||||
if messages:
|
||||
try:
|
||||
from turnstone.core.history_decoration import (
|
||||
decorate_history_messages,
|
||||
load_verdict_indexes,
|
||||
)
|
||||
|
||||
indexes = await asyncio.to_thread(load_verdict_indexes, ws_id)
|
||||
decorate_history_messages(messages, indexes[0], indexes[1])
|
||||
except Exception:
|
||||
# Operationally interesting: a persistent decoration
|
||||
# failure (missing migration, driver mismatch, schema
|
||||
# drift) silently strips verdict pills + output
|
||||
# warnings from every reload of every workstream.
|
||||
# Log at warning so it surfaces in normal log review
|
||||
# rather than only when DEBUG is on.
|
||||
log.warning(
|
||||
"ws.history.decoration_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
return JSONResponse({"ws_id": ws_id, "messages": messages})
|
||||
|
||||
return history
|
||||
|
||||
@@ -41,6 +41,7 @@ log = get_logger(__name__)
|
||||
# from bloating memory.
|
||||
_DEFAULT_LISTENER_QUEUE_MAX = 500
|
||||
|
||||
|
||||
# Cap on the per-turn assistant content accumulator. The accumulator
|
||||
# is piggybacked onto the ``ws_state:idle`` broadcast payload so the
|
||||
# cluster collector / dashboard can render the freshly-emitted assistant
|
||||
@@ -327,6 +328,11 @@ class SessionUIBase:
|
||||
"always": bool(always),
|
||||
}
|
||||
)
|
||||
# Kind-specific cross-stream broadcast — ConsoleCoordinatorUI
|
||||
# overrides to push onto the cluster bus so a coord parent's
|
||||
# tree UI clears the pending-approval pill in lockstep with
|
||||
# the actual decision. Stage 3 Step 4.
|
||||
self._broadcast_approval_resolved(approved, feedback, always=always)
|
||||
self._approval_event.set()
|
||||
|
||||
@staticmethod
|
||||
@@ -579,6 +585,20 @@ class SessionUIBase:
|
||||
"judge_pending": judge_pending,
|
||||
}
|
||||
self._enqueue(self._pending_approval)
|
||||
# Cross-stream broadcast — push the items via the cluster bus
|
||||
# so a coord parent's tree UI can render the inline approve/deny
|
||||
# block without waiting for a bulk fetch. Without this, the
|
||||
# bulk fetch races with this assignment: the state transition
|
||||
# to ATTENTION fires upstream BEFORE approve_tools runs (see
|
||||
# session.py:_emit_state("attention") preceding ui.approve_tools),
|
||||
# so a bulk fetch landing in the ~50-200ms window between
|
||||
# _emit_state and this point sees ``_pending_approval=None``
|
||||
# and returns ``pending_approval_detail: null``. The 5s TTL
|
||||
# then locks the coord row on a "loading" placeholder until
|
||||
# the next state event triggers a refresh — which never comes
|
||||
# while parked on _approval_event.wait. The push path
|
||||
# eliminates the race.
|
||||
self._broadcast_approve_request(self._pending_approval)
|
||||
if not self._approval_event.wait(timeout=self._APPROVAL_WAIT_TIMEOUT):
|
||||
# Approval timed out (e.g., user disconnected). Deny via
|
||||
# resolve_approval so verdicts and state are updated consistently.
|
||||
@@ -632,6 +652,12 @@ class SessionUIBase:
|
||||
del self._llm_verdicts[oldest_key]
|
||||
self._llm_verdicts[call_id] = verdict
|
||||
self._enqueue({"type": "intent_verdict", **verdict})
|
||||
# Kind-specific cross-stream broadcast — ConsoleCoordinatorUI
|
||||
# overrides to push onto the cluster bus so a coord parent's
|
||||
# tree UI sees the verdict without polling. Default is no-op
|
||||
# (the per-ws ``_enqueue`` above already covers WebUI's own
|
||||
# SSE listeners). Stage 3 Step 4.
|
||||
self._broadcast_intent_verdict(verdict)
|
||||
self._persist_intent_verdict(verdict)
|
||||
# Decision check + either queue or flag-for-persist happen
|
||||
# under ONE lock acquisition so resolve_approval can't swap-
|
||||
@@ -1293,6 +1319,42 @@ class SessionUIBase:
|
||||
def on_error(self, message: str) -> None:
|
||||
self._enqueue({"type": "error", "message": message})
|
||||
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
"""Surface a metacognitive user-channel nudge as its own UI
|
||||
element.
|
||||
|
||||
Reminders live on the user message dict's ``_reminders``
|
||||
side-channel and are spliced into ``content`` only at the
|
||||
provider boundary; this event is what lets every connected
|
||||
SSE consumer (other browser tabs, CLI mirrors, future channel
|
||||
adapters) render the reminder bubble in lockstep with the
|
||||
originating tab. The history-replay path surfaces the same
|
||||
shape via ``_build_history`` so a tab reconnecting later
|
||||
renders the same bubble.
|
||||
"""
|
||||
self._enqueue({"type": "user_reminder", "reminders": reminders})
|
||||
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None:
|
||||
"""Surface a metacognitive tool-channel nudge (``tool_error`` /
|
||||
``repeat``) as its own UI element below the tool result that
|
||||
triggered it.
|
||||
|
||||
Tool-channel reminders ride the same ``_reminders``
|
||||
side-channel pattern as the user channel — kept out of
|
||||
``content`` so compaction / title-gen / channel adapters never
|
||||
see the nudge text, spliced into the wire only via
|
||||
``_apply_reminders_for_provider``. ``tool_call_id`` is the
|
||||
anchor the frontend uses to render the bubble below the
|
||||
specific tool result that triggered the batch's reminder.
|
||||
"""
|
||||
self._enqueue(
|
||||
{
|
||||
"type": "tool_reminder",
|
||||
"reminders": reminders,
|
||||
"tool_call_id": tool_call_id,
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Broadcast hooks — kind-specific transport.
|
||||
#
|
||||
@@ -1315,6 +1377,42 @@ class SessionUIBase:
|
||||
Default: no-op. Subclasses override.
|
||||
"""
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None: # noqa: ARG002 — hook stub
|
||||
"""Fan an LLM intent-judge verdict out to the kind's transport.
|
||||
|
||||
Default: no-op. ``ConsoleCoordinatorUI`` overrides to push a
|
||||
``intent_verdict`` event onto the cluster bus so the parent
|
||||
coordinator's tree UI can render the risk pill + verdict
|
||||
result without polling. Stage 3 Step 4: hook only — the
|
||||
cluster-bus event class lands in Step 5.
|
||||
"""
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool, # noqa: ARG002 — hook stub
|
||||
feedback: str | None = None, # noqa: ARG002 — hook stub
|
||||
*,
|
||||
always: bool = False, # noqa: ARG002 — hook stub
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision out to the kind's transport.
|
||||
|
||||
Default: no-op. ``ConsoleCoordinatorUI`` overrides to push to
|
||||
the cluster bus so the parent coordinator's tree UI can clear
|
||||
the pending-approval pill in sync with the actual decision.
|
||||
"""
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None: # noqa: ARG002 — hook stub
|
||||
"""Fan an ``approve_request`` payload out to the kind's transport.
|
||||
|
||||
Default: no-op. ``WebUI`` and ``ConsoleCoordinatorUI`` override
|
||||
to push the items list (the same dict that landed in
|
||||
``_pending_approval``) onto their respective transports. The
|
||||
push path eliminates the bulk-fetch race that otherwise
|
||||
leaves coord rows stuck on a loading placeholder when the
|
||||
bulk fetch lands in the gap between the state transition to
|
||||
ATTENTION and ``_pending_approval`` being set.
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State-broadcast snapshot helper
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -97,7 +97,7 @@ from turnstone.core.storage._utils import sanitize_text
|
||||
from turnstone.core.storage._utils import (
|
||||
scan_skill_content as _scan_skill_content,
|
||||
)
|
||||
from turnstone.core.workstream import WorkstreamKind
|
||||
from turnstone.core.workstream import BULK_CLOSE_STATE_VALUES, WorkstreamKind
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
@@ -595,6 +595,57 @@ class PostgreSQLBackend:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
norm_kind = WorkstreamKind(kind).value
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
stmt = (
|
||||
sa.update(workstreams)
|
||||
.where(
|
||||
workstreams.c.kind == norm_kind,
|
||||
workstreams.c.state.in_(BULK_CLOSE_STATE_VALUES),
|
||||
workstreams.c.updated < cutoff,
|
||||
)
|
||||
.values(state="closed", updated=now)
|
||||
.returning(workstreams.c.ws_id)
|
||||
)
|
||||
# Protect rows whose owning process is still heartbeating in the
|
||||
# services table (rendezvous router's liveness primitive). NULL
|
||||
# node_id rows have no owner identity — always eligible. The
|
||||
# ``and live_node_ids`` short-circuits both ``None`` (skip the
|
||||
# filter entirely — single-process / operator backfill) and ``[]``
|
||||
# (no nodes alive — every row unprotected, no extra predicate
|
||||
# needed since absence equals match-all).
|
||||
if live_node_ids is not None and live_node_ids:
|
||||
stmt = stmt.where(
|
||||
sa.or_(
|
||||
workstreams.c.node_id.is_(None),
|
||||
~workstreams.c.node_id.in_(live_node_ids),
|
||||
)
|
||||
)
|
||||
if exclude_ws_ids:
|
||||
# Skip ``NOT IN ()`` when nothing to exclude — keeps the SQL clean
|
||||
# and avoids SQLAlchemy's empty-collection warning.
|
||||
stmt = stmt.where(~workstreams.c.ws_id.in_(exclude_ws_ids))
|
||||
with self._conn() as conn:
|
||||
result = conn.execute(stmt)
|
||||
ids = [row[0] for row in result]
|
||||
conn.commit()
|
||||
return ids
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
|
||||
@@ -431,6 +431,68 @@ class StorageBackend(Protocol):
|
||||
"""Update a workstream's state and bump updated timestamp."""
|
||||
...
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Close DB-side workstream rows of *kind* whose state is in
|
||||
``BULK_CLOSE_STATE_VALUES`` and whose ``updated`` is lex-older than
|
||||
*cutoff*, excluding rows currently loaded in memory. Sets
|
||||
``state='closed'`` and bumps ``updated``. Returns the list of ws_ids
|
||||
actually transitioned.
|
||||
|
||||
``cutoff`` is a UTC ``YYYY-MM-DDTHH:MM:SS`` string matching the on-disk
|
||||
format ``update_workstream_state`` writes — lex compare is safe for
|
||||
same-offset timestamps. Empty ``exclude_ws_ids`` means no exclusion.
|
||||
|
||||
``live_node_ids`` is the set of ``services.service_id`` values whose
|
||||
``last_heartbeat`` is recent (i.e. owning processes still alive);
|
||||
rows whose ``node_id`` matches one of these are protected because
|
||||
their owning process may legitimately have them loaded on another
|
||||
worker. ``None`` skips the filter entirely (single-process / tests
|
||||
/ operator backfill). Empty list ``[]`` treats every node as dead —
|
||||
useful when operator scripts want to reap regardless of liveness.
|
||||
|
||||
Rows with ``NULL`` ``node_id`` are always eligible: they have no
|
||||
meaningful owner identity, so age alone gates the reap.
|
||||
|
||||
Liveness scoping replaces an earlier ``node_id == self`` heuristic.
|
||||
That heuristic broke in the post-rendezvous-routing world (PR #384):
|
||||
``workstreams.node_id`` is stamped at create time and never updated,
|
||||
so dead-pod orphans in containerized deployments with dynamic
|
||||
hostnames couldn't be reclaimed. ``services.last_heartbeat`` is the
|
||||
rendezvous router's authoritative liveness primitive — using it here
|
||||
keeps reap scoping aligned with routing.
|
||||
|
||||
Asymmetric with ``SessionManager.close_idle``'s in-memory pass on
|
||||
purpose: that pass closes only ``IDLE`` (legitimately-attentive rows
|
||||
stay), this method closes the broader ``BULK_CLOSE_STATE_VALUES`` set
|
||||
because any row matching here is by definition not loaded by any
|
||||
live process and cannot be in a live interaction.
|
||||
"""
|
||||
...
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
"""Bump a workstream row's ``updated`` timestamp without touching its
|
||||
state.
|
||||
|
||||
Used by ``SessionManager.open()`` on cold rehydrate so a freshly-
|
||||
loaded row's ``updated`` can't be older than the orphan-reaper cutoff
|
||||
— protects against a same-process race where a parallel
|
||||
``close_idle`` pass-2 snapshots loaded keys after the storage read
|
||||
but before the in-memory install. Distinct from
|
||||
``update_workstream_state(ws_id, current_state)`` because the
|
||||
rehydrate path explicitly avoids a state write (see the
|
||||
``open()`` no-DB-state-flip-on-resurrect comment): a state write
|
||||
could race a concurrent ``close()`` and resurrect a closed row.
|
||||
Bumping only ``updated`` is safe — close still wins on the state
|
||||
column.
|
||||
"""
|
||||
...
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
"""Update a workstream's display name."""
|
||||
...
|
||||
|
||||
@@ -97,7 +97,7 @@ from turnstone.core.storage._utils import sanitize_text
|
||||
from turnstone.core.storage._utils import (
|
||||
scan_skill_content as _scan_skill_content,
|
||||
)
|
||||
from turnstone.core.workstream import WorkstreamKind
|
||||
from turnstone.core.workstream import BULK_CLOSE_STATE_VALUES, WorkstreamKind
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
@@ -707,6 +707,87 @@ class SQLiteBackend:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
norm_kind = WorkstreamKind(kind).value
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
# SQLite has no RETURNING precedent in this file — do SELECT-then-
|
||||
# UPDATE in one transaction, with the SAME WHERE predicates re-applied
|
||||
# to the UPDATE. Re-application defends against a same-process race:
|
||||
# ``SessionManager.open()`` calls ``touch_workstream`` between the
|
||||
# SELECT and the UPDATE could have bumped a row's ``updated`` past
|
||||
# ``cutoff`` (or ``set_state`` could have flipped its state out of
|
||||
# the bulk-close set). Without the re-applied WHERE the UPDATE
|
||||
# closes those rows anyway; with it, the UPDATE skips rows that
|
||||
# became ineligible after the SELECT and the row stays open.
|
||||
# Chunked through ``_in_chunks`` so the ``IN`` clause never exceeds
|
||||
# SQLite's bind-parameter limit (default 999) on a large reap.
|
||||
candidate_conditions = [
|
||||
workstreams.c.kind == norm_kind,
|
||||
workstreams.c.state.in_(BULK_CLOSE_STATE_VALUES),
|
||||
workstreams.c.updated < cutoff,
|
||||
]
|
||||
if live_node_ids is not None and live_node_ids:
|
||||
# Protect rows owned by heartbeating services. NULL node_id is
|
||||
# always eligible. Empty list means "no nodes alive" — every
|
||||
# row is unprotected; the absence of this predicate is
|
||||
# equivalent to "match all," so we just skip it.
|
||||
candidate_conditions.append(
|
||||
sa.or_(
|
||||
workstreams.c.node_id.is_(None),
|
||||
~workstreams.c.node_id.in_(live_node_ids),
|
||||
)
|
||||
)
|
||||
if exclude_ws_ids:
|
||||
candidate_conditions.append(~workstreams.c.ws_id.in_(exclude_ws_ids))
|
||||
select_stmt = sa.select(workstreams.c.ws_id).where(*candidate_conditions)
|
||||
closed: list[str] = []
|
||||
# Match the chunk size used by ``prune_workstreams`` (line 453) — keeps
|
||||
# ``IN`` clauses well below SQLite's default 999-bind-param limit even
|
||||
# on very large reaps.
|
||||
chunk_size = 500
|
||||
with self._conn() as conn:
|
||||
candidate_ids = [row[0] for row in conn.execute(select_stmt)]
|
||||
for i in range(0, len(candidate_ids), chunk_size):
|
||||
chunk = candidate_ids[i : i + chunk_size]
|
||||
# Re-apply the eligibility predicates on the UPDATE so a row
|
||||
# that became fresh between the SELECT and the UPDATE is not
|
||||
# clobbered. Then SELECT back by ``state='closed' AND updated=now``
|
||||
# to determine which rows actually transitioned this commit —
|
||||
# the returned list reflects reality even when re-application
|
||||
# filters out some candidates.
|
||||
conn.execute(
|
||||
sa.update(workstreams)
|
||||
.where(workstreams.c.ws_id.in_(chunk), *candidate_conditions)
|
||||
.values(state="closed", updated=now)
|
||||
)
|
||||
actually_closed = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
sa.select(workstreams.c.ws_id).where(
|
||||
workstreams.c.ws_id.in_(chunk),
|
||||
workstreams.c.state == "closed",
|
||||
workstreams.c.updated == now,
|
||||
)
|
||||
)
|
||||
]
|
||||
closed.extend(actually_closed)
|
||||
conn.commit()
|
||||
return closed
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Add a partial composite index for the orphan-reaper query.
|
||||
|
||||
``StorageBackend.bulk_close_stale_orphans`` (introduced alongside the
|
||||
workstream-lifecycle leak fix) runs every ``min(300s, idle_timeout/4)``
|
||||
on every server and console process. Its WHERE shape is:
|
||||
|
||||
WHERE kind = ?
|
||||
AND state IN ('idle', 'thinking', 'attention', 'running')
|
||||
AND updated < ?
|
||||
AND (node_id IS NULL OR node_id NOT IN (alive_service_ids))
|
||||
|
||||
At current scale (low-thousands of workstream rows) the existing single-
|
||||
column indexes are sufficient — ``idx_workstreams_state`` prunes to the
|
||||
non-closed subset, and the planner filters the rest sequentially. At
|
||||
100k+ rows that filter becomes a tablescan-shaped cost on the reaper's
|
||||
periodic run.
|
||||
|
||||
A **partial** index covering only ``BULK_CLOSE_STATE_VALUES`` rows
|
||||
matches the reaper's query exactly while staying tiny — closed rows
|
||||
(typically 95%+ of the table per empirical diagnosis) and ``error``
|
||||
rows are excluded, so the index is roughly 5% the size a full multi-
|
||||
column index would be. Write amplification only kicks in for
|
||||
transitions that touch one of the four covered states.
|
||||
|
||||
Column order ``(kind, updated)``:
|
||||
|
||||
- ``kind`` first because the reaper always supplies it as an equality
|
||||
predicate; partitions the partial index into interactive vs
|
||||
coordinator subtrees.
|
||||
- ``updated`` last so the range comparison rides the trailing column —
|
||||
classic composite-index pattern for ``WHERE eq AND range``.
|
||||
|
||||
``node_id`` is intentionally NOT in the index. The reaper's predicate
|
||||
on it is ``NOT IN (small list)`` against an unbounded-cardinality
|
||||
column, which planners don't index well; including it would just add
|
||||
write cost for negligible read benefit.
|
||||
|
||||
PostgreSQL uses ``CREATE INDEX CONCURRENTLY`` so the build is
|
||||
non-blocking on a live system; SQLite has no concurrent concept and
|
||||
the table-level write lock already serializes, so a plain
|
||||
``CREATE INDEX`` is fine.
|
||||
|
||||
Revision ID: 048
|
||||
Revises: 047
|
||||
Create Date: 2026-04-30
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "048"
|
||||
down_revision = "047"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_REAPER_PARTIAL_WHERE = "state IN ('idle', 'thinking', 'attention', 'running')"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
with op.get_context().autocommit_block():
|
||||
op.execute(
|
||||
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_workstreams_reaper "
|
||||
"ON workstreams (kind, updated) "
|
||||
f"WHERE {_REAPER_PARTIAL_WHERE}"
|
||||
)
|
||||
else:
|
||||
op.create_index(
|
||||
"idx_workstreams_reaper",
|
||||
"workstreams",
|
||||
["kind", "updated"],
|
||||
sqlite_where=sa.text(_REAPER_PARTIAL_WHERE),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
with op.get_context().autocommit_block():
|
||||
op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_workstreams_reaper")
|
||||
else:
|
||||
op.drop_index("idx_workstreams_reaper", table_name="workstreams")
|
||||
@@ -77,6 +77,25 @@ class WorkstreamState(enum.Enum):
|
||||
ERROR = "error" # last operation failed
|
||||
|
||||
|
||||
# States the orphan reaper (``SessionManager.close_idle`` pass 2 +
|
||||
# ``StorageBackend.bulk_close_stale_orphans``) is allowed to flip to
|
||||
# ``closed`` for rows past the staleness cutoff. Excludes ``ERROR``
|
||||
# deliberately — error rows are user-investigatable and shouldn't be
|
||||
# auto-reaped — and excludes ``CLOSED`` (terminal). Centralized here
|
||||
# so the storage backends and FakeStorage all agree; if a new transient
|
||||
# state is added to ``WorkstreamState``, deciding whether it joins
|
||||
# this set is part of the change rather than an after-the-fact
|
||||
# audit across three files.
|
||||
BULK_CLOSE_STATE_VALUES: frozenset[str] = frozenset(
|
||||
{
|
||||
WorkstreamState.IDLE.value,
|
||||
WorkstreamState.THINKING.value,
|
||||
WorkstreamState.RUNNING.value,
|
||||
WorkstreamState.ATTENTION.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workstream dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -139,6 +139,12 @@ class NullUI:
|
||||
def on_error(self, message: str) -> None:
|
||||
pass
|
||||
|
||||
def on_user_reminder(self, reminders: list[dict[str, str]]) -> None:
|
||||
pass
|
||||
|
||||
def on_tool_reminder(self, reminders: list[dict[str, str]], tool_call_id: str) -> None:
|
||||
pass
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
+220
-24
@@ -53,6 +53,15 @@ from turnstone.core.auth import (
|
||||
_DenyFilter,
|
||||
jwt_version_slot,
|
||||
)
|
||||
from turnstone.core.history_decoration import (
|
||||
TOOL_RESULT_STORAGE_CAP,
|
||||
)
|
||||
from turnstone.core.history_decoration import (
|
||||
decorate_tool_call as _decorate_tool_call,
|
||||
)
|
||||
from turnstone.core.history_decoration import (
|
||||
load_verdict_indexes as _load_verdict_indexes,
|
||||
)
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.metrics import metrics as _metrics
|
||||
from turnstone.core.ratelimit import resolve_client_ip
|
||||
@@ -194,18 +203,14 @@ class WebUI(SessionUIBase):
|
||||
}
|
||||
if state == "idle":
|
||||
event["content"] = payload["content"]
|
||||
# Coord tree-UI renders inline approve/deny buttons off
|
||||
# ``pending_approval_detail``; carrying it on the
|
||||
# state-change broadcast lets the cluster bus update those
|
||||
# buttons in lockstep with ``activity_state`` instead of
|
||||
# forcing the browser to chase a separate dashboard fetch.
|
||||
# Gated on existence so we don't pay the serializer's
|
||||
# per-broadcast verdict-cache deepcopy on the common
|
||||
# no-approval-pending path.
|
||||
if self._pending_approval is not None:
|
||||
detail = self.serialize_pending_approval_detail()
|
||||
if detail is not None:
|
||||
event["pending_approval_detail"] = detail
|
||||
# ``pending_approval_detail`` is NO LONGER piggybacked on
|
||||
# state-change events (Stage 3 cleanup). Symmetric event
|
||||
# flow now: initial approval items arrive via bulk fetch
|
||||
# triggered by the ``activity_state="approval"`` transition,
|
||||
# individual verdicts via the explicit
|
||||
# ``intent_verdict`` event class, and resolution via
|
||||
# ``approval_resolved``. Reducer no longer has to dedupe
|
||||
# the piggyback path against the explicit one.
|
||||
try:
|
||||
WebUI._global_queue.put_nowait(event)
|
||||
except queue.Full:
|
||||
@@ -230,6 +235,73 @@ class WebUI(SessionUIBase):
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
"""Send an LLM intent-judge verdict to the global SSE channel.
|
||||
|
||||
Stage 3 Step 5 — the cluster collector's ``_apply_delta``
|
||||
forwards this verbatim to the cluster bus, where coord
|
||||
adapters dispatch it as ``child_ws_intent_verdict`` for the
|
||||
owning parent's tree UI. Unlike the existing
|
||||
``pending_approval_detail`` piggyback on ``ws_state``, this
|
||||
fires WHENEVER a verdict lands — including the common case
|
||||
where the judge daemon writes during ``attention`` with no
|
||||
state transition to ride along on.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": self.ws_id,
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool,
|
||||
feedback: str | None = None,
|
||||
*,
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Send an ``approval_resolved`` decision to the global SSE channel.
|
||||
|
||||
Clears the parent's pending-approval pill in lockstep with
|
||||
the actual decision rather than waiting for the next
|
||||
state-change piggyback.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": self.ws_id,
|
||||
"approved": approved,
|
||||
"feedback": feedback or "",
|
||||
"always": bool(always),
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None:
|
||||
"""Send an ``approve_request`` payload to the global SSE channel.
|
||||
|
||||
Push path for the initial approval items so a coord parent's
|
||||
tree UI can render the inline approve/deny block immediately
|
||||
without waiting for a bulk-fetch round-trip. The bulk fetch
|
||||
races with ``_pending_approval`` being set inside
|
||||
``approve_tools`` (the state transition to ATTENTION fires
|
||||
upstream first); the push path eliminates that race entirely.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": self.ws_id,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
# --- SessionUI protocol ---
|
||||
#
|
||||
# ``on_thinking_start`` / ``on_thinking_stop`` / ``on_reasoning_token``
|
||||
@@ -345,8 +417,20 @@ class WebUI(SessionUIBase):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Verdict + output-assessment decoration helpers (``_decorate_tool_call``,
|
||||
# ``_load_verdict_indexes``) are imported at module top alongside the
|
||||
# rest of ``turnstone.core.*``. Both this builder and
|
||||
# :func:`make_history_handler` (the /history REST endpoint coord uses
|
||||
# as its primary history loader) share them so the two surfaces don't
|
||||
# drift on the wire shape they emit.
|
||||
|
||||
|
||||
def _build_history(
|
||||
session: ChatSession, has_pending_approval: bool = False
|
||||
session: ChatSession,
|
||||
has_pending_approval: bool = False,
|
||||
*,
|
||||
verdicts: dict[str, dict[str, Any]] | None = None,
|
||||
assessments: dict[str, dict[str, Any]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Build a history replay list from ChatSession messages.
|
||||
|
||||
@@ -358,7 +442,35 @@ def _build_history(
|
||||
``"denied": True``, and the corresponding assistant entry that
|
||||
issued the tool calls is also marked ``"denied": True`` so the
|
||||
client can render the correct badge.
|
||||
|
||||
``verdicts`` and ``assessments`` are optional pre-loaded
|
||||
``{call_id → row}`` dicts (see :func:`_load_verdict_indexes`).
|
||||
Async callers should pre-load via ``asyncio.to_thread`` and pass
|
||||
them in to avoid blocking the event loop on storage I/O. When
|
||||
omitted, the storage call runs inline (sync call sites).
|
||||
"""
|
||||
# Metacognitive nudges live on the message dict's ``_reminders``
|
||||
# side-channel — user messages carry user-channel nudges
|
||||
# (correction / denial / resume / start / completion), tool
|
||||
# messages carry tool-channel nudges (tool_error / repeat). Both
|
||||
# are surfaced separately on each entry so the UI can render them
|
||||
# as their own bubble (live via ``user_reminder`` /
|
||||
# ``tool_reminder`` SSE events; replay via this propagation).
|
||||
# ``content`` never carries the ``<system-reminder>`` envelope —
|
||||
# that splice is transient, applied to a wire-bound copy in
|
||||
# ``ChatSession._apply_reminders_for_provider``.
|
||||
#
|
||||
# Verdict + output-assessment lookup tables — populated either
|
||||
# inline (sync call sites) or pre-loaded by an async caller via
|
||||
# asyncio.to_thread (see _load_verdict_indexes). Pre-loading is
|
||||
# what keeps _build_history off the event loop's hot path on the
|
||||
# SSE replay generator path.
|
||||
if verdicts is not None and assessments is not None:
|
||||
verdicts_by_call_id = verdicts
|
||||
assessments_by_call_id = assessments
|
||||
else:
|
||||
ws_id = getattr(session, "_ws_id", "") or ""
|
||||
verdicts_by_call_id, assessments_by_call_id = _load_verdict_indexes(ws_id)
|
||||
history = []
|
||||
for msg in session.messages:
|
||||
content = msg.get("content")
|
||||
@@ -404,18 +516,64 @@ def _build_history(
|
||||
entry = {"role": msg["role"], "content": content}
|
||||
if attachments_meta:
|
||||
entry["attachments"] = attachments_meta
|
||||
# Surface the ``_reminders`` side-channel so a tab reconnecting
|
||||
# via /history renders the same metacognitive nudge bubble the
|
||||
# originating tab saw live (user-channel reminders via
|
||||
# ``user_reminder`` SSE; tool-channel via ``tool_reminder``).
|
||||
# Reminders are in-memory only (not persisted to DB), so this
|
||||
# only fires for the originating session.
|
||||
reminders = msg.get("_reminders")
|
||||
if isinstance(reminders, list):
|
||||
# Filter first so an all-malformed _reminders doesn't set the
|
||||
# field to []; absent vs. empty-list should mean the same
|
||||
# thing on the wire.
|
||||
clean_reminders = [
|
||||
{"type": str(r.get("type") or ""), "text": str(r.get("text") or "")}
|
||||
for r in reminders
|
||||
if isinstance(r, dict)
|
||||
]
|
||||
if clean_reminders:
|
||||
entry["reminders"] = clean_reminders
|
||||
if msg.get("tool_calls"):
|
||||
entry["tool_calls"] = [
|
||||
{
|
||||
"id": tc.get("id", ""),
|
||||
tc_entries: list[dict[str, Any]] = []
|
||||
for tc in msg["tool_calls"]:
|
||||
tc_entry: dict[str, Any] = {
|
||||
"id": tc.get("id", "") or "",
|
||||
"name": tc["function"]["name"],
|
||||
"arguments": tc["function"].get("arguments", ""),
|
||||
}
|
||||
for tc in msg["tool_calls"]
|
||||
]
|
||||
# Decorate with persisted verdict + output_assessment
|
||||
# via the shared helper (also used by
|
||||
# ``make_history_handler``). Skips unflagged
|
||||
# ("risk_level == 'none'") rows so the wire stays
|
||||
# tight; ships only the fields the UI renders.
|
||||
_decorate_tool_call(
|
||||
tc_entry,
|
||||
verdicts_by_call_id,
|
||||
assessments_by_call_id,
|
||||
)
|
||||
tc_entries.append(tc_entry)
|
||||
entry["tool_calls"] = tc_entries
|
||||
# Detect denied/blocked/errored tool results by their content prefix.
|
||||
if msg.get("role") == "tool":
|
||||
content = msg.get("content", "")
|
||||
# Propagate tool_call_id so replayHistory can anchor the
|
||||
# rendered output to the specific .ts-approval-tool element
|
||||
# by data-call-id (mirrors the live appendToolOutput path).
|
||||
# Without this, multi-tool batches render every result at
|
||||
# the bottom of the block rather than under each header.
|
||||
result_call_id = msg.get("tool_call_id")
|
||||
if result_call_id:
|
||||
entry["tool_call_id"] = str(result_call_id)
|
||||
# Tool results are clamped to TOOL_RESULT_STORAGE_CAP
|
||||
# chars per row at storage time (session.py). Surface
|
||||
# that on replay so the user knows the visible output is
|
||||
# a clipped view of what the live session saw, rather
|
||||
# than the full result. Reference the shared constant
|
||||
# rather than a literal so the UI pill logic can't
|
||||
# silently desync if the cap ever changes.
|
||||
if isinstance(content, str) and len(content) >= TOOL_RESULT_STORAGE_CAP:
|
||||
entry["truncated"] = True
|
||||
if isinstance(content, str):
|
||||
if content.startswith("Denied by user") or content.startswith("Blocked"):
|
||||
entry["denied"] = True
|
||||
@@ -656,6 +814,31 @@ def _audit_close_workstream(
|
||||
)
|
||||
|
||||
|
||||
async def _interactive_events_replay_prepare(ws: Workstream, ui: Any, request: Request) -> None:
|
||||
"""Async pre-step run before ``_interactive_events_replay`` iterates.
|
||||
|
||||
Loads ``intent_verdicts`` + ``output_assessments`` for the
|
||||
workstream off the event loop (via ``asyncio.to_thread``) and
|
||||
stashes the result on ``request.state.verdict_indexes``. The sync
|
||||
replay generator reads from there and passes the dicts into
|
||||
``_build_history`` so the storage I/O never blocks the event loop
|
||||
on the SSE replay path.
|
||||
|
||||
Best-effort: if the workstream has no session or no ws_id, leaves
|
||||
``request.state.verdict_indexes`` unset and ``_build_history``
|
||||
falls back to the inline storage call (sync path).
|
||||
"""
|
||||
del ui # not needed; lookup is keyed on ws.session._ws_id
|
||||
session = ws.session
|
||||
if session is None:
|
||||
return
|
||||
ws_id = getattr(session, "_ws_id", "") or ""
|
||||
if not ws_id:
|
||||
return
|
||||
indexes = await asyncio.to_thread(_load_verdict_indexes, ws_id)
|
||||
request.state.verdict_indexes = indexes
|
||||
|
||||
|
||||
def _interactive_events_replay(
|
||||
ws: Workstream, ui: Any, request: Request
|
||||
) -> Iterable[dict[str, Any]]:
|
||||
@@ -673,7 +856,6 @@ def _interactive_events_replay(
|
||||
|
||||
Pure read — never mutates ``ws`` / ``ui`` / ``session``.
|
||||
"""
|
||||
del request # not needed; replay reads ws/ui/session state
|
||||
session = ws.session
|
||||
if session is None:
|
||||
# Defensive — the lifted body's UI presence check guarantees
|
||||
@@ -688,9 +870,22 @@ def _interactive_events_replay(
|
||||
|
||||
# History replay — pending-approval flag rides on the last
|
||||
# assistant entry's tool_calls so the client renders them as
|
||||
# awaiting approval rather than already approved.
|
||||
# awaiting approval rather than already approved. Verdict /
|
||||
# assessment indexes were pre-loaded off the event loop by
|
||||
# _interactive_events_replay_prepare; passing them in here keeps
|
||||
# _build_history's storage I/O out of the sync generator path.
|
||||
pending_approval = getattr(ui, "_pending_approval", None)
|
||||
history = _build_history(session, has_pending_approval=pending_approval is not None)
|
||||
cached_indexes = getattr(request.state, "verdict_indexes", None)
|
||||
if isinstance(cached_indexes, tuple) and len(cached_indexes) == 2:
|
||||
verdicts, assessments = cached_indexes
|
||||
else:
|
||||
verdicts, assessments = None, None
|
||||
history = _build_history(
|
||||
session,
|
||||
has_pending_approval=pending_approval is not None,
|
||||
verdicts=verdicts,
|
||||
assessments=assessments,
|
||||
)
|
||||
if history:
|
||||
yield {"type": "history", "messages": history}
|
||||
|
||||
@@ -1365,13 +1560,13 @@ async def command(request: Request) -> JSONResponse:
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
elif cmd_word == "/resume":
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
history = _build_history(ws.session)
|
||||
history = await asyncio.to_thread(_build_history, ws.session)
|
||||
if history:
|
||||
ui._enqueue({"type": "history", "messages": history})
|
||||
elif cmd_word in ("/rewind", "/retry"):
|
||||
# Refresh frontend with truncated history
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
history = _build_history(ws.session)
|
||||
history = await asyncio.to_thread(_build_history, ws.session)
|
||||
if history:
|
||||
ui._enqueue({"type": "history", "messages": history})
|
||||
# Audit trail
|
||||
@@ -1846,7 +2041,7 @@ async def _interactive_create_post_install(
|
||||
ui = ws.ui
|
||||
if isinstance(ui, WebUI):
|
||||
ui._enqueue({"type": "clear_ui"})
|
||||
history = _build_history(ws.session)
|
||||
history = await asyncio.to_thread(_build_history, ws.session)
|
||||
if history:
|
||||
ui._enqueue({"type": "history", "messages": history})
|
||||
with contextlib.suppress(queue.Full):
|
||||
@@ -3342,6 +3537,7 @@ def create_app(
|
||||
open_resolve_alias=_resolve_workstream_alias,
|
||||
open_post_load=_interactive_open_post_load,
|
||||
events_replay=_interactive_events_replay,
|
||||
events_replay_prepare=_interactive_events_replay_prepare,
|
||||
# Pre-lift ``events_sse`` used the dedicated 200-thread
|
||||
# ``sse_executor`` so SSE polling stayed isolated from
|
||||
# every other ``asyncio.to_thread`` caller in the process
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
console/static (Saved Coordinators). Single source of truth so the two
|
||||
surfaces don't drift on hover affordance, padding, or typography.
|
||||
==========================================================================
|
||||
Class names match the original ui/static rules they replaced; the
|
||||
delete-mode subset stays in ui/static/style.css until coordinator gets
|
||||
the same UX (then it can move here too).
|
||||
Class names match the original ui/static rules they replaced. Delete-mode
|
||||
selectors live here too so console (Saved Coordinators) and ui/static
|
||||
(Saved Workstreams) share one card + delete affordance.
|
||||
========================================================================== */
|
||||
|
||||
.dashboard-cards {
|
||||
@@ -76,3 +76,283 @@
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
/* ==========================================================================
|
||||
Delete UX — section-level "Delete" toggle, per-card checkboxes, bottom
|
||||
toolbar, and confirmation modal. Moved out of ui/static/style.css when
|
||||
the console grew the same multi-select delete on Saved Coordinators.
|
||||
========================================================================== */
|
||||
|
||||
.ws-delete-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
color 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
.ws-delete-btn:hover {
|
||||
color: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
|
||||
/* Delete mode */
|
||||
.dashboard-card.ws-delete-mode {
|
||||
cursor: pointer;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.04);
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected {
|
||||
cursor: default;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode:hover {
|
||||
background: rgba(220, 38, 38, 0.04);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-card-check {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: var(--red);
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
opacity: 0;
|
||||
animation: ws-check-fadein 0.2s ease-out forwards;
|
||||
}
|
||||
.ws-card-check:focus-visible {
|
||||
/* Card click-handler proxies space/enter to the checkbox, so focus
|
||||
usually rests on the card; if a screen reader / power user tabs
|
||||
directly onto the checkbox the UA outline can be killed by
|
||||
adjacent rules — this guarantees a visible affordance. */
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
@keyframes ws-check-fadein {
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
.dashboard-card.ws-selected {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-selected {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-delete-bar {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
display: flex;
|
||||
animation: ws-bar-slide 0.2s ease-out;
|
||||
}
|
||||
@keyframes ws-bar-slide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ws-card-check {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
/* Below ~700px the four-pill bar's preferred width (~480-520px) starts
|
||||
eating into hit-targets and risking horizontal overflow. Wrap onto
|
||||
two rows: count + Cancel + Select All on the first, the destructive
|
||||
Delete Selected on its own full-width row underneath — also a better
|
||||
thumb-target separation than the desktop layout. */
|
||||
@media (max-width: 700px) {
|
||||
.ws-delete-bar {
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: 0;
|
||||
flex: 1 1 100%;
|
||||
order: 99;
|
||||
}
|
||||
}
|
||||
.ws-delete-bar .ws-delete-count-label {
|
||||
font-size: 12px;
|
||||
color: var(--fg-dim);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: auto;
|
||||
/* Dark-theme --red (#f87171) on #fff is only 3.0:1 — below WCAG AA
|
||||
for normal text on a destructive button. Use the deeper red
|
||||
(#dc2626 → 4.85:1) on the filled state so the button label clears
|
||||
AA in the default theme. Light theme already uses --red (#b91c1c,
|
||||
5.9:1) and stays put — the override below pins it. */
|
||||
background: #dc2626;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 16px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-bar .ws-delete-bar-btn {
|
||||
background: var(--red);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:hover:not(:disabled) {
|
||||
filter: brightness(1.1);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn:hover {
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
|
||||
/* Delete modal — id-scoped so the surface that owns it controls visibility.
|
||||
Both ui/static (#ws-delete-overlay) and console/static
|
||||
(#coord-delete-overlay) share the same shape via the .ws-delete-modal
|
||||
class hooks below. */
|
||||
.ws-delete-modal-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
.ws-delete-modal-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 24px;
|
||||
max-width: 480px;
|
||||
width: 90%;
|
||||
}
|
||||
.ws-delete-modal-box h3 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 16px;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
.ws-delete-modal-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
margin: 12px 0;
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item {
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
color: var(--fg-bright);
|
||||
border-bottom: 1px solid var(--border);
|
||||
/* Long aliases / raw ws_ids in the confirm + results list shouldn't
|
||||
punch out of the modal at narrow viewports. */
|
||||
word-break: break-word;
|
||||
}
|
||||
/* Modal alert region — only painted when the controller writes a
|
||||
message. Both close paths clear it, so :not(:empty) keeps the box
|
||||
invisible at rest and avoids an empty-frame artefact. */
|
||||
.ws-delete-modal-box [role="alert"]:not(:empty) {
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
border: 1px solid var(--red);
|
||||
color: var(--red);
|
||||
border-radius: var(--radius);
|
||||
padding: 8px 10px;
|
||||
font-size: 12px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-modal-box [role="alert"]:not(:empty) {
|
||||
background: rgba(220, 38, 38, 0.06);
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item.ws-delete-error {
|
||||
color: var(--red);
|
||||
}
|
||||
.ws-delete-modal-buttons {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
.ws-delete-modal-buttons button {
|
||||
padding: 8px 20px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
.ws-delete-modal-buttons button.ws-delete-confirm {
|
||||
/* Mirror .ws-delete-bar-btn's contrast bump — same destructive
|
||||
filled-button treatment, same dark-theme AA fix. */
|
||||
background: #dc2626;
|
||||
color: #fff;
|
||||
border-color: #dc2626;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-modal-buttons button.ws-delete-confirm {
|
||||
background: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
/* `.ws-delete-close` is a state-marker, not a colour rule — the
|
||||
controller drops `.ws-delete-confirm` when the modal transitions to
|
||||
the post-delete "Close" state, and the default
|
||||
`.ws-delete-modal-buttons button` rule above already provides the
|
||||
transparent / fg-bright / border styling. The class itself is useful
|
||||
for DOM inspection and as a future hook. */
|
||||
|
||||
@@ -77,3 +77,433 @@ function renderSessionCard(sess, opts) {
|
||||
card.appendChild(metaEl);
|
||||
return card;
|
||||
}
|
||||
|
||||
/* createSavedCardsController — shared multi-select-delete behaviour for
|
||||
the dashboard / home "saved cards" surfaces. ui/static (Saved
|
||||
Workstreams) and console/static (Saved Coordinators) both instantiate
|
||||
one of these; the controller owns:
|
||||
|
||||
- delete-mode state (active flag + selected ws_id set)
|
||||
- card decoration (checkbox + key/click overrides)
|
||||
- the bottom toolbar wiring (count, Select All, Delete Selected)
|
||||
- the confirmation modal (focus trap, batch fan-out, results view)
|
||||
|
||||
It does NOT own how cards get fetched or rendered — the caller's
|
||||
render() is invoked when the controller needs the list redrawn (mode
|
||||
transitions, Select-All toggles).
|
||||
|
||||
Required opts:
|
||||
idPrefix — DOM-id prefix shared by the toolbar + modal
|
||||
(e.g. "ws-delete" / "coord-delete"). The DOM
|
||||
must already contain `${idPrefix}-bar`,
|
||||
`${idPrefix}-bar-count`, `${idPrefix}-bar-delete`,
|
||||
`${idPrefix}-bar-select-all`, `${idPrefix}-overlay`,
|
||||
`${idPrefix}-box`, `${idPrefix}-error`,
|
||||
`${idPrefix}-count`, `${idPrefix}-list`,
|
||||
`${idPrefix}-confirm-btn`, `${idPrefix}-cancel-btn`.
|
||||
buttonId — id of the section's start/cancel toggle button.
|
||||
noun — singular display word for the item kind, e.g.
|
||||
"workstream" / "coordinator". Used in toast +
|
||||
modal copy.
|
||||
activateLabel — sess => string; aria-label for the card when NOT
|
||||
in delete mode (e.g. "Resume: foo").
|
||||
buildDeleteRequest — wsId => { url, options }; what authFetch should
|
||||
send to delete one item.
|
||||
render — () => void; redraw the visible cards. Called by
|
||||
the controller on mode start/cancel and Select-
|
||||
All toggle. Caller is responsible for calling
|
||||
setItems(items) + decorateCard() inside it.
|
||||
onClose — optional () => void; called once after the user
|
||||
closes the post-delete results modal. Typical
|
||||
use: re-fetch the saved list.
|
||||
*/
|
||||
function createSavedCardsController(opts) {
|
||||
var state = { mode: false, selected: {}, items: [] };
|
||||
var batchTrap = null;
|
||||
/* Element that owned focus when the modal opened — restored in
|
||||
closeModal() so keyboard users land back on the toggle button (or
|
||||
wherever they came from) instead of <body>. WCAG 2.4.3. */
|
||||
var prevFocus = null;
|
||||
|
||||
function $(id) {
|
||||
return document.getElementById(opts.idPrefix + "-" + id);
|
||||
}
|
||||
|
||||
/* Replace the toggle button's content with a glyph + label, keeping
|
||||
the glyph in an aria-hidden span so screen readers only read the
|
||||
label. Built from DOM nodes (no innerHTML) — same shape as the
|
||||
section-header markup the JS replaces. */
|
||||
function setIconButton(btn, glyph, label) {
|
||||
btn.replaceChildren();
|
||||
var span = document.createElement("span");
|
||||
span.setAttribute("aria-hidden", "true");
|
||||
span.textContent = glyph;
|
||||
btn.appendChild(span);
|
||||
btn.appendChild(document.createTextNode(" " + label));
|
||||
}
|
||||
|
||||
function setItems(items) {
|
||||
state.items = items;
|
||||
/* Drop any selections whose ws_id is no longer on the visible page —
|
||||
SSE-driven re-renders or pagination jumps shouldn't leave ghost
|
||||
entries inflating the count and 404-ing on confirm. */
|
||||
if (state.mode) {
|
||||
var byId = {};
|
||||
items.forEach(function (s) {
|
||||
byId[s.ws_id] = true;
|
||||
});
|
||||
Object.keys(state.selected).forEach(function (id) {
|
||||
if (!byId[id]) delete state.selected[id];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function inMode() {
|
||||
return state.mode;
|
||||
}
|
||||
|
||||
function blockActivate() {
|
||||
return state.mode;
|
||||
}
|
||||
|
||||
function isSelected(wsId) {
|
||||
return !!state.selected[wsId];
|
||||
}
|
||||
|
||||
function ariaLabel(sess) {
|
||||
var label = sess.alias || sess.title || sess.name || sess.ws_id;
|
||||
if (state.mode) return "Select " + opts.noun + ": " + label;
|
||||
return typeof opts.activateLabel === "function"
|
||||
? opts.activateLabel(sess)
|
||||
: "Activate: " + label;
|
||||
}
|
||||
|
||||
/* Decorate an already-rendered .dashboard-card with the checkbox +
|
||||
event overrides used in delete mode. Idempotent guard: only acts
|
||||
when the controller is active. */
|
||||
function decorateCard(card, sess) {
|
||||
if (!state.mode) return;
|
||||
card.classList.add("ws-delete-mode");
|
||||
card.removeAttribute("role");
|
||||
var chk = document.createElement("input");
|
||||
chk.type = "checkbox";
|
||||
chk.className = "ws-card-check";
|
||||
chk.checked = !!state.selected[sess.ws_id];
|
||||
var label = sess.alias || sess.title || sess.name || sess.ws_id;
|
||||
chk.setAttribute("aria-label", "Select " + label + " for deletion");
|
||||
chk.onclick = function (e) {
|
||||
e.stopPropagation();
|
||||
if (chk.checked) state.selected[sess.ws_id] = true;
|
||||
else delete state.selected[sess.ws_id];
|
||||
card.classList.toggle("ws-selected", chk.checked);
|
||||
refreshBar();
|
||||
};
|
||||
card.insertBefore(chk, card.firstChild);
|
||||
card.onclick = function (e) {
|
||||
if (e.target === chk) return;
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
};
|
||||
card.onkeydown = function (e) {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
}
|
||||
};
|
||||
if (state.selected[sess.ws_id]) card.classList.add("ws-selected");
|
||||
}
|
||||
|
||||
function refreshBar() {
|
||||
var count = Object.keys(state.selected).length;
|
||||
var label = $("bar-count");
|
||||
if (label) label.textContent = count + " selected";
|
||||
var delBtn = $("bar-delete");
|
||||
if (delBtn) delBtn.disabled = count === 0;
|
||||
var selBtn = $("bar-select-all");
|
||||
if (selBtn) {
|
||||
var allSelected = count === state.items.length && state.items.length > 0;
|
||||
selBtn.textContent = allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (!state.items.length) {
|
||||
if (typeof showToast === "function") {
|
||||
showToast("No saved " + opts.noun + "s to delete");
|
||||
}
|
||||
return;
|
||||
}
|
||||
state.mode = true;
|
||||
state.selected = {};
|
||||
opts.render();
|
||||
var btn = document.getElementById(opts.buttonId);
|
||||
if (btn) {
|
||||
setIconButton(btn, "✕", "Cancel");
|
||||
btn.onclick = cancel;
|
||||
}
|
||||
var bar = $("bar");
|
||||
if (bar) bar.classList.add("visible");
|
||||
refreshBar();
|
||||
}
|
||||
|
||||
function cancel() {
|
||||
state.mode = false;
|
||||
state.selected = {};
|
||||
opts.render();
|
||||
var btn = document.getElementById(opts.buttonId);
|
||||
if (btn) {
|
||||
setIconButton(btn, "\u{1f5d1}", "Delete");
|
||||
btn.onclick = start;
|
||||
}
|
||||
var bar = $("bar");
|
||||
if (bar) bar.classList.remove("visible");
|
||||
}
|
||||
|
||||
function toggleAll() {
|
||||
var allSelected =
|
||||
Object.keys(state.selected).length === state.items.length &&
|
||||
state.items.length > 0;
|
||||
if (allSelected) {
|
||||
state.selected = {};
|
||||
} else {
|
||||
state.items.forEach(function (s) {
|
||||
state.selected[s.ws_id] = true;
|
||||
});
|
||||
}
|
||||
opts.render();
|
||||
refreshBar();
|
||||
}
|
||||
|
||||
function _byId() {
|
||||
/* Single-pass index over the visible items so the modal + fan-out
|
||||
paths don't repeat O(N) `find` calls per selection. */
|
||||
var map = {};
|
||||
state.items.forEach(function (s) {
|
||||
map[s.ws_id] = s;
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
function confirmSelection() {
|
||||
var selected = Object.keys(state.selected);
|
||||
if (!selected.length) {
|
||||
if (typeof showToast === "function") {
|
||||
showToast("No " + opts.noun + "s selected");
|
||||
}
|
||||
return;
|
||||
}
|
||||
var byId = _byId();
|
||||
var overlay = $("overlay");
|
||||
var countEl = $("count");
|
||||
var listEl = $("list");
|
||||
var errorEl = $("error");
|
||||
if (errorEl) errorEl.textContent = "";
|
||||
if (countEl) {
|
||||
countEl.textContent =
|
||||
selected.length + " " + opts.noun + "(s) will be permanently deleted:";
|
||||
}
|
||||
if (listEl) {
|
||||
listEl.replaceChildren();
|
||||
selected.forEach(function (wsId) {
|
||||
var item = byId[wsId];
|
||||
var name = item ? item.alias || item.title || item.name || wsId : wsId;
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item";
|
||||
div.textContent = name;
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
}
|
||||
var delBtn = $("confirm-btn");
|
||||
if (delBtn) {
|
||||
delBtn.textContent = "Delete";
|
||||
delBtn.disabled = false;
|
||||
delBtn.classList.remove("ws-delete-close");
|
||||
delBtn.classList.add("ws-delete-confirm");
|
||||
delBtn.onclick = confirm;
|
||||
}
|
||||
var cancelBtn = $("cancel-btn");
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
if (overlay) overlay.style.display = "flex";
|
||||
|
||||
if (batchTrap) document.removeEventListener("keydown", batchTrap);
|
||||
batchTrap = function (e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
closeModal();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Tab") {
|
||||
var box = $("box");
|
||||
if (!box) return;
|
||||
var focusable = box.querySelectorAll("button:not(:disabled)");
|
||||
if (!focusable.length) return;
|
||||
var first = focusable[0];
|
||||
var last = focusable[focusable.length - 1];
|
||||
if (e.shiftKey && document.activeElement === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && document.activeElement === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", batchTrap);
|
||||
/* Snapshot the pre-modal focus owner so closeModal() can return to
|
||||
it. Captured before we move focus into the dialog so the
|
||||
restore-target is the caller, not the dialog itself. */
|
||||
prevFocus = document.activeElement;
|
||||
if (cancelBtn) cancelBtn.focus();
|
||||
}
|
||||
|
||||
function closeModal() {
|
||||
var overlay = $("overlay");
|
||||
if (overlay) overlay.style.display = "none";
|
||||
if (batchTrap) {
|
||||
document.removeEventListener("keydown", batchTrap);
|
||||
batchTrap = null;
|
||||
}
|
||||
/* Pick the most useful focus target:
|
||||
1. prevFocus (where the user came from), if it's still in the
|
||||
DOM and visible. Esc / Cancel paths land here — the bar is
|
||||
still on screen, so focus returns to "Delete Selected".
|
||||
2. The section toggle button — always present, semantic exit
|
||||
point for the flow. Used when prevFocus has been hidden by
|
||||
cancel() (post-delete Close path: cancel() ran first and
|
||||
put `.ws-delete-bar` at display:none, so the bar's button
|
||||
is no longer focusable). */
|
||||
var target = prevFocus;
|
||||
if (!target || target.offsetParent === null) {
|
||||
target = document.getElementById(opts.buttonId);
|
||||
}
|
||||
if (target && typeof target.focus === "function") {
|
||||
try {
|
||||
target.focus();
|
||||
} catch (_) {
|
||||
/* node detached between open and close — give up silently */
|
||||
}
|
||||
}
|
||||
prevFocus = null;
|
||||
}
|
||||
|
||||
function confirm() {
|
||||
var selected = Object.keys(state.selected);
|
||||
if (!selected.length) return;
|
||||
var byId = _byId();
|
||||
var errorEl = $("error");
|
||||
var listEl = $("list");
|
||||
var countEl = $("count");
|
||||
var delBtn = $("confirm-btn");
|
||||
var cancelBtn = $("cancel-btn");
|
||||
if (errorEl) errorEl.textContent = "";
|
||||
if (delBtn) {
|
||||
delBtn.disabled = true;
|
||||
delBtn.textContent = "Deleting...";
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = true;
|
||||
|
||||
var results = [];
|
||||
var promises = selected.map(function (wsId) {
|
||||
var shortId = wsId.substring(0, 8);
|
||||
var item = byId[wsId];
|
||||
var name = item ? item.alias || item.title || item.name || wsId : wsId;
|
||||
var req = opts.buildDeleteRequest(wsId);
|
||||
return authFetch(req.url, req.options)
|
||||
.then(function (r) {
|
||||
var status = r.status;
|
||||
var contentType = r.headers.get("content-type") || "";
|
||||
if (r.ok) {
|
||||
results.push({ name: name, shortId: shortId, ok: true });
|
||||
return;
|
||||
}
|
||||
return r.text().then(function (body) {
|
||||
var errMsg = shortId + ": HTTP " + status;
|
||||
if (contentType.includes("json")) {
|
||||
try {
|
||||
var j = JSON.parse(body);
|
||||
if (j.error) errMsg = shortId + ": " + j.error;
|
||||
} catch (_) {
|
||||
/* fall through */
|
||||
}
|
||||
} else if (body) {
|
||||
errMsg = shortId + ": " + body.substring(0, 200);
|
||||
}
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: errMsg,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(function (err) {
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: shortId + ": " + err.message,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Promise.all(promises).then(function () {
|
||||
if (listEl) {
|
||||
listEl.replaceChildren();
|
||||
results.forEach(function (r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item" + (r.ok ? "" : " ws-delete-error");
|
||||
div.textContent =
|
||||
(r.ok ? "✓ " : "✗ ") + r.name + (r.error ? " — " + r.error : "");
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
}
|
||||
var okCount = results.filter(function (r) {
|
||||
return r.ok;
|
||||
}).length;
|
||||
var failCount = results.filter(function (r) {
|
||||
return !r.ok;
|
||||
}).length;
|
||||
if (countEl) {
|
||||
countEl.textContent = okCount + " deleted, " + failCount + " failed";
|
||||
}
|
||||
if (delBtn) {
|
||||
delBtn.disabled = false;
|
||||
delBtn.textContent = "Close";
|
||||
/* Swap modifier classes so styling is intent-driven instead of
|
||||
cascade-positional: the Close button picks up the default
|
||||
".ws-delete-modal-buttons button" rule once .ws-delete-confirm
|
||||
is removed. */
|
||||
delBtn.classList.remove("ws-delete-confirm");
|
||||
delBtn.classList.add("ws-delete-close");
|
||||
delBtn.onclick = function () {
|
||||
/* Order matters: cancel() reshapes the toggle button via
|
||||
setIconButton(), which preserves the element identity but
|
||||
swaps its subtree. closeModal() then focuses prevFocus —
|
||||
which IS that toggle button — landing on a freshly rebuilt
|
||||
"Delete" affordance instead of <body>. */
|
||||
cancel();
|
||||
closeModal();
|
||||
if (typeof opts.onClose === "function") opts.onClose();
|
||||
};
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
setItems: setItems,
|
||||
inMode: inMode,
|
||||
blockActivate: blockActivate,
|
||||
isSelected: isSelected,
|
||||
ariaLabel: ariaLabel,
|
||||
decorateCard: decorateCard,
|
||||
refreshBar: refreshBar,
|
||||
start: start,
|
||||
cancel: cancel,
|
||||
toggleAll: toggleAll,
|
||||
confirmSelection: confirmSelection,
|
||||
closeModal: closeModal,
|
||||
confirm: confirm,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -885,6 +885,27 @@
|
||||
.msg.user {
|
||||
border-left-color: var(--accent);
|
||||
}
|
||||
/* Metacognitive reminder — slotted directly below the message it
|
||||
advises (user message for correction/denial/etc., tool result for
|
||||
tool_error/repeat). Yellow accent reads as "advisory metadata"
|
||||
against the amber-ish user colour and the cyan tool cards;
|
||||
deliberately quieter than the surrounding bubbles so it doesn't
|
||||
compete for attention. Lives in the shared stylesheet so both
|
||||
the interactive UI and the console coord viewer render the same
|
||||
themed bubble. */
|
||||
.msg.user-reminder {
|
||||
border-left-color: var(--yellow);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 6px 10px;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
.msg.user-reminder .msg-user-reminder-label {
|
||||
color: var(--yellow);
|
||||
font-weight: 600;
|
||||
margin-right: 6px;
|
||||
text-transform: lowercase;
|
||||
}
|
||||
.msg.assistant {
|
||||
border-left-color: var(--hair-2);
|
||||
}
|
||||
|
||||
+404
-304
@@ -557,6 +557,44 @@ Pane.prototype.handleEvent = function (evt) {
|
||||
this.addErrorMessage(evt.message);
|
||||
break;
|
||||
|
||||
case "user_reminder":
|
||||
// Metacognitive nudges — render as their own bubble below the
|
||||
// user message they advise (semantically: a hint to the model
|
||||
// right before its turn). The originating tab's optimistic
|
||||
// addUserMessage already ran when the user clicked send, so by
|
||||
// the time this SSE event arrives the just-sent user bubble is
|
||||
// at the bottom of messagesEl and addUserReminder's "anchor to
|
||||
// most recent .msg.user" lookup finds it correctly; the
|
||||
// insertAdjacentElement('afterend', el) call drops the bubble
|
||||
// immediately below.
|
||||
//
|
||||
// Multi-tab caveat: the server emits no user_message SSE event
|
||||
// today, so a non-originating tab open on the same workstream
|
||||
// sees the reminder without a paired user-message render — the
|
||||
// anchor falls on a stale prior user bubble, mis-positioning
|
||||
// the reminder. The next /history reload corrects it (the
|
||||
// entry["reminders"] propagation in _build_history is
|
||||
// anchor-stable because replayHistory runs addUserMessage first
|
||||
// for every turn). Acceptable cost for stage 1; closing the
|
||||
// gap is a follow-up that adds a user_message SSE event.
|
||||
if (Array.isArray(evt.reminders) && evt.reminders.length) {
|
||||
this.addUserReminder(evt.reminders);
|
||||
}
|
||||
break;
|
||||
|
||||
case "tool_reminder":
|
||||
// Metacognitive tool-channel nudge (tool_error / repeat) —
|
||||
// render as the same yellow themed bubble used for user-channel
|
||||
// reminders, anchored below the .ts-approval block whose tool
|
||||
// result triggered the batch's reminder. evt.tool_call_id
|
||||
// identifies the specific tool element; addToolReminder walks
|
||||
// up to its parent approval block and inserts the bubble
|
||||
// immediately after.
|
||||
if (Array.isArray(evt.reminders) && evt.reminders.length) {
|
||||
this.addToolReminder(evt.reminders, evt.tool_call_id || "");
|
||||
}
|
||||
break;
|
||||
|
||||
case "message_queued":
|
||||
// Confirmation from server that a queued message was accepted.
|
||||
// The UI already showed the message optimistically in addQueuedMessage.
|
||||
@@ -669,6 +707,97 @@ Pane.prototype.removeThinkingIndicator = function () {
|
||||
if (el) el.remove();
|
||||
};
|
||||
|
||||
Pane.prototype.addUserReminder = function (reminders) {
|
||||
// Render each metacognitive reminder as its own bubble immediately
|
||||
// BELOW the user message it advises — semantically the reminder is
|
||||
// a hint to the model right before the assistant turn. Always
|
||||
// called AFTER the corresponding addUserMessage (live: optimistic
|
||||
// local render ran before the SSE event arrived; replay:
|
||||
// replayHistory renders the user message first), so "most recent
|
||||
// .msg.user" is always THIS turn's bubble — insertAdjacentElement
|
||||
// afterend drops the reminder directly below it. When no .msg.user
|
||||
// exists at all (e.g. a non-originating tab receiving a reminder
|
||||
// before any user turn has rendered) we append; the next /history
|
||||
// reload corrects any anchor anomaly.
|
||||
this.removeEmptyState();
|
||||
var userBubbles = this.messagesEl.querySelectorAll(".msg.user");
|
||||
var anchor = userBubbles.length ? userBubbles[userBubbles.length - 1] : null;
|
||||
for (var i = 0; i < reminders.length; i++) {
|
||||
var r = reminders[i] || {};
|
||||
var el = document.createElement("div");
|
||||
el.className = "msg user-reminder";
|
||||
var labelEl = document.createElement("span");
|
||||
labelEl.className = "msg-user-reminder-label";
|
||||
labelEl.textContent =
|
||||
"metacognition" + (r.type ? " · " + String(r.type) : "");
|
||||
var textEl = document.createElement("span");
|
||||
textEl.className = "msg-user-reminder-text";
|
||||
textEl.textContent = r.text || "";
|
||||
el.appendChild(labelEl);
|
||||
el.appendChild(textEl);
|
||||
if (anchor) {
|
||||
anchor.insertAdjacentElement("afterend", el);
|
||||
// Anchor advances so multiple reminders stack below the user
|
||||
// message in queued order (rather than each landing
|
||||
// immediately-after the user msg, which would reverse them).
|
||||
anchor = el;
|
||||
} else {
|
||||
this.messagesEl.appendChild(el);
|
||||
}
|
||||
}
|
||||
this.scrollToBottom(true);
|
||||
};
|
||||
|
||||
Pane.prototype.addToolReminder = function (reminders, toolCallId) {
|
||||
// Render each metacognitive tool-channel reminder (tool_error /
|
||||
// repeat) as the same yellow themed bubble used for user-channel
|
||||
// reminders, anchored below the .ts-approval block that produced
|
||||
// the tool result. toolCallId is the live-path anchor (SSE event
|
||||
// carries it); during replay it's an empty string and we fall back
|
||||
// to "last .ts-approval block in messagesEl", which is correct
|
||||
// because messages render in order — the assistant block carrying
|
||||
// the tool batch is always the most recent approval block by the
|
||||
// time we hit the tool message that owns the reminder.
|
||||
this.removeEmptyState();
|
||||
var anchor = null;
|
||||
if (toolCallId) {
|
||||
var escapedId = CSS.escape(toolCallId);
|
||||
var toolEl = this.messagesEl.querySelector(
|
||||
'.ts-approval-tool[data-call-id="' + escapedId + '"]',
|
||||
);
|
||||
if (toolEl) {
|
||||
anchor = toolEl.closest(".ts-approval");
|
||||
}
|
||||
}
|
||||
if (!anchor) {
|
||||
var blocks = this.messagesEl.querySelectorAll(".ts-approval");
|
||||
if (blocks.length) anchor = blocks[blocks.length - 1];
|
||||
}
|
||||
for (var i = 0; i < reminders.length; i++) {
|
||||
var r = reminders[i] || {};
|
||||
var el = document.createElement("div");
|
||||
// Same .msg.user-reminder class — visual treatment is shared
|
||||
// across user and tool channels (both are metacog nudges).
|
||||
el.className = "msg user-reminder";
|
||||
var labelEl = document.createElement("span");
|
||||
labelEl.className = "msg-user-reminder-label";
|
||||
labelEl.textContent =
|
||||
"metacognition" + (r.type ? " · " + String(r.type) : "");
|
||||
var textEl = document.createElement("span");
|
||||
textEl.className = "msg-user-reminder-text";
|
||||
textEl.textContent = r.text || "";
|
||||
el.appendChild(labelEl);
|
||||
el.appendChild(textEl);
|
||||
if (anchor) {
|
||||
anchor.insertAdjacentElement("afterend", el);
|
||||
anchor = el;
|
||||
} else {
|
||||
this.messagesEl.appendChild(el);
|
||||
}
|
||||
}
|
||||
this.scrollToBottom(true);
|
||||
};
|
||||
|
||||
Pane.prototype.addUserMessage = function (text, attachments) {
|
||||
this.removeEmptyState();
|
||||
var el = document.createElement("div");
|
||||
@@ -907,13 +1036,54 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
this.showEmptyState();
|
||||
return;
|
||||
}
|
||||
// Suppress the polite live region while we batch-build the replay
|
||||
// — messagesEl is aria-live="polite" so a fresh replay would otherwise
|
||||
// queue an announcement for every approved/denied/verdict pill we
|
||||
// insert. Restored after the loop so live SSE updates announce
|
||||
// normally. WCAG 4.1.3 — historical content should not behave like
|
||||
// real-time updates.
|
||||
this.messagesEl.setAttribute("aria-busy", "true");
|
||||
// pendingAssessments[call_id] = output_assessment dict. Populated
|
||||
// from the assistant branch, consumed by the role==="tool" branch
|
||||
// (or after the loop, for legacy rows missing tool_call_id).
|
||||
// Replaces a JSON.stringify→dataset→JSON.parse round-trip with an
|
||||
// in-memory map keyed by call_id.
|
||||
var pendingAssessments = {};
|
||||
var lastToolBlock = null;
|
||||
for (var i = 0; i < messages.length; i++) {
|
||||
var msg = messages[i];
|
||||
if (msg.role === "user") {
|
||||
// addUserMessage first so addUserReminder's "anchor to most
|
||||
// recent .msg.user" lookup finds THIS message's bubble (not the
|
||||
// previous user message's, which would associate the reminder
|
||||
// with the wrong turn). addUserReminder then drops the bubble
|
||||
// immediately below the just-rendered user message via
|
||||
// insertAdjacentElement('afterend', el).
|
||||
this.addUserMessage(msg.content || "", msg.attachments || null);
|
||||
if (Array.isArray(msg.reminders) && msg.reminders.length) {
|
||||
this.addUserReminder(msg.reminders);
|
||||
}
|
||||
lastToolBlock = null;
|
||||
} else if (msg.role === "assistant") {
|
||||
// Render content BEFORE the tool block so the visual order
|
||||
// matches the live SSE flow (stream_text streams content first,
|
||||
// then tool_info / approve_request paints the tool block, then
|
||||
// tool_result fills it in). Order also matters structurally:
|
||||
// the tool-result message in the NEXT iteration anchors via
|
||||
// lastToolBlock, which the tool-block branch sets last — so
|
||||
// content must run first to avoid clobbering that anchor.
|
||||
if (msg.content) {
|
||||
var el = document.createElement("div");
|
||||
el.className = "msg assistant";
|
||||
var bodyEl = document.createElement("div");
|
||||
bodyEl.className = "msg-body";
|
||||
var rendered = renderMarkdown(msg.content);
|
||||
bodyEl.innerHTML = rendered;
|
||||
el.appendChild(bodyEl);
|
||||
postRenderMarkdown(el);
|
||||
self.messagesEl.appendChild(el);
|
||||
lastToolBlock = null;
|
||||
}
|
||||
if (msg.tool_calls && msg.tool_calls.length) {
|
||||
if (msg.pending) {
|
||||
lastToolBlock = null;
|
||||
@@ -958,7 +1128,35 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
cmd.textContent = tc.arguments.substring(0, 100);
|
||||
}
|
||||
div.appendChild(cmd);
|
||||
// Verdict badge — anchor to THIS tool's row (div) rather
|
||||
// than the whole block, so a multi-tool batch with one
|
||||
// flagged call doesn't drift the badge above unrelated
|
||||
// calls. Same renderVerdictBadge helper as live; pass
|
||||
// judgePending=false because any verdict on replay is
|
||||
// final — no spinner.
|
||||
if (tc.verdict) {
|
||||
div.insertAdjacentHTML(
|
||||
"beforeend",
|
||||
renderVerdictBadge(tc.verdict, false),
|
||||
);
|
||||
}
|
||||
block.appendChild(div);
|
||||
// Output-guard finding — defer insertion until the tool
|
||||
// result lands so the warning anchors under the output
|
||||
// (mirrors live showOutputWarning placement). Stash in
|
||||
// a function-local map keyed by call_id so the
|
||||
// role==="tool" branch below can pick it up; legacy rows
|
||||
// missing tool_call_id are flushed at end-of-replay.
|
||||
if (
|
||||
tc.output_assessment &&
|
||||
tc.output_assessment.risk_level &&
|
||||
tc.output_assessment.risk_level !== "none"
|
||||
) {
|
||||
pendingAssessments[tc.id || ""] = {
|
||||
assessment: tc.output_assessment,
|
||||
toolDiv: div,
|
||||
};
|
||||
}
|
||||
});
|
||||
var badge = document.createElement("div");
|
||||
badge.setAttribute("role", "status");
|
||||
@@ -974,18 +1172,6 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
lastToolBlock = block;
|
||||
}
|
||||
}
|
||||
if (msg.content) {
|
||||
var el = document.createElement("div");
|
||||
el.className = "msg assistant";
|
||||
var bodyEl = document.createElement("div");
|
||||
bodyEl.className = "msg-body";
|
||||
var rendered = renderMarkdown(msg.content);
|
||||
bodyEl.innerHTML = rendered;
|
||||
el.appendChild(bodyEl);
|
||||
postRenderMarkdown(el);
|
||||
self.messagesEl.appendChild(el);
|
||||
lastToolBlock = null;
|
||||
}
|
||||
} else if (msg.role === "tool") {
|
||||
if (lastToolBlock) {
|
||||
var stripped = stripAnsi(msg.content || "").trim();
|
||||
@@ -994,34 +1180,156 @@ Pane.prototype.replayHistory = function (messages) {
|
||||
/^Denied by user/.test(stripped) ||
|
||||
/^Blocked/.test(stripped);
|
||||
var isToolError = !!msg.is_error;
|
||||
// Anchor the rendered output to the specific .ts-approval-tool
|
||||
// element matching this result's tool_call_id — mirrors the
|
||||
// live appendToolOutput path so multi-tool batches show
|
||||
// [hdr A][out A][hdr B][out B] rather than [A][B][out A][out B].
|
||||
// Falls back to "before badge" when tool_call_id is absent
|
||||
// (legacy rows pre-dating the wire-format addition).
|
||||
var resultTarget = null;
|
||||
if (msg.tool_call_id) {
|
||||
resultTarget = lastToolBlock.querySelector(
|
||||
'.ts-approval-tool[data-call-id="' +
|
||||
CSS.escape(msg.tool_call_id) +
|
||||
'"]',
|
||||
);
|
||||
}
|
||||
// Cursor-style append: cursor advances after each insert so
|
||||
// the next sibling lands AFTER the previous one. Fixes the
|
||||
// bug where calling resultTarget.after(node) twice put the
|
||||
// second node BETWEEN resultTarget and the first (the second
|
||||
// .after call was always relative to the same anchor).
|
||||
// Resulting order with all three present:
|
||||
// [tool div][output][truncation pill][output-warning]
|
||||
var insertCursor = resultTarget;
|
||||
var insertChained = function (node) {
|
||||
if (insertCursor) {
|
||||
insertCursor.after(node);
|
||||
insertCursor = node;
|
||||
} else {
|
||||
var bdg = lastToolBlock.querySelector(".ts-approval-badge");
|
||||
if (bdg) lastToolBlock.insertBefore(node, bdg);
|
||||
else lastToolBlock.appendChild(node);
|
||||
}
|
||||
};
|
||||
if (stripped && !isDenied) {
|
||||
var media = !isToolError ? tryParseMedia(stripped) : null;
|
||||
if (media) {
|
||||
var embed = buildMediaEmbed(media, stripped);
|
||||
var bdg = lastToolBlock.querySelector(".ts-approval-badge");
|
||||
if (bdg) lastToolBlock.insertBefore(embed, bdg);
|
||||
else lastToolBlock.appendChild(embed);
|
||||
insertChained(buildMediaEmbed(media, stripped));
|
||||
} else {
|
||||
var out = renderToolOutput(stripped, isToolError);
|
||||
if (out.textContent.split("\n").length > 10) {
|
||||
makeCollapsible(out);
|
||||
}
|
||||
var bdg = lastToolBlock.querySelector(".ts-approval-badge");
|
||||
if (bdg) lastToolBlock.insertBefore(out, bdg);
|
||||
else lastToolBlock.appendChild(out);
|
||||
insertChained(out);
|
||||
}
|
||||
// Truncation pill — server marks this when the stored row
|
||||
// hit the 2000-char cap. Live tool_result events carry full
|
||||
// output so they don't need the indicator.
|
||||
if (msg.truncated) {
|
||||
var pill = document.createElement("span");
|
||||
pill.className = "tool-output-truncated";
|
||||
pill.textContent = "… truncated in storage";
|
||||
pill.title =
|
||||
"The full tool output was sent to the model live; only the first 10000 characters are persisted to the conversation row.";
|
||||
insertChained(pill);
|
||||
}
|
||||
}
|
||||
if (isToolError && !lastToolBlock.classList.contains("denied")) {
|
||||
lastToolBlock.classList.add("error");
|
||||
appendToolErrorBadge(lastToolBlock);
|
||||
}
|
||||
// Output-guard warning — pull the assessment out of the
|
||||
// function-local pendingAssessments map (populated in the
|
||||
// assistant branch). Skip when the tool result was denied —
|
||||
// the ✗ denied badge already signals the deny path.
|
||||
if (!isDenied && msg.tool_call_id) {
|
||||
var pending = pendingAssessments[msg.tool_call_id];
|
||||
if (pending) {
|
||||
insertChained(_buildOutputWarningEl(pending.assessment));
|
||||
delete pendingAssessments[msg.tool_call_id];
|
||||
}
|
||||
}
|
||||
}
|
||||
// Tool-channel metacog reminders (tool_error / repeat) attach
|
||||
// to the LAST tool message in a batch; on replay we render the
|
||||
// bubble immediately below the .ts-approval block that owns
|
||||
// the tool result. addToolReminder's empty-toolCallId fallback
|
||||
// resolves to "last .ts-approval block" — which is exactly
|
||||
// lastToolBlock here.
|
||||
if (Array.isArray(msg.reminders) && msg.reminders.length) {
|
||||
this.addToolReminder(msg.reminders, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
// Flush any output_assessments left in the map — these correspond
|
||||
// to assistant tool_calls whose tool result row didn't carry a
|
||||
// tool_call_id (legacy / migrated rows pre-dating the wire-format
|
||||
// addition). Render the warning under the tool div itself rather
|
||||
// than dropping the safety information silently.
|
||||
var leftoverIds = Object.keys(pendingAssessments);
|
||||
for (var p = 0; p < leftoverIds.length; p++) {
|
||||
var leftover = pendingAssessments[leftoverIds[p]];
|
||||
if (!leftover) continue;
|
||||
leftover.toolDiv.insertAdjacentElement(
|
||||
"afterend",
|
||||
_buildOutputWarningEl(leftover.assessment),
|
||||
);
|
||||
}
|
||||
this._attachRetryToLastAssistant();
|
||||
this.scrollToBottom();
|
||||
// Focus the input so keyboard users land on the next-action target
|
||||
// after replay finishes — but only when this is the focused pane,
|
||||
// there's no pending approval competing for focus, and an input
|
||||
// element actually exists. Skipping when not the focused pane
|
||||
// avoids stealing focus from another tab the user is interacting
|
||||
// with while a background replay completes.
|
||||
if (
|
||||
this.id === focusedPaneId &&
|
||||
!this.pendingApproval &&
|
||||
this.inputEl &&
|
||||
!this.busy
|
||||
) {
|
||||
try {
|
||||
this.inputEl.focus({ preventScroll: true });
|
||||
} catch (_) {
|
||||
this.inputEl.focus();
|
||||
}
|
||||
}
|
||||
// Restore live-region semantics now that the batch build is done.
|
||||
this.messagesEl.removeAttribute("aria-busy");
|
||||
};
|
||||
|
||||
// Shared output-warning DOM builder — used by both replayHistory
|
||||
// (saved-workstream rendering) and the live appendToolOutput path
|
||||
// via showOutputWarning. Single source of truth keeps the two
|
||||
// surfaces from drifting on role / class / escape semantics.
|
||||
function _buildOutputWarningEl(assessment) {
|
||||
var risk = (assessment && assessment.risk_level) || "medium";
|
||||
var flags = (assessment && assessment.flags) || [];
|
||||
var warning = document.createElement("div");
|
||||
warning.className = "output-warning output-warning-" + risk;
|
||||
// role="status" (polite) rather than "alert" (assertive) — these
|
||||
// are findings, not emergencies; the assertive announcement live
|
||||
// would interrupt the user mid-typing on a high-risk match, which
|
||||
// is more disruptive than informative.
|
||||
warning.setAttribute("role", "status");
|
||||
var labelEl = document.createElement("span");
|
||||
labelEl.className = "output-warning-label";
|
||||
labelEl.textContent = "⚠ " + String(risk).toUpperCase();
|
||||
warning.appendChild(labelEl);
|
||||
if (flags.length) {
|
||||
warning.appendChild(document.createTextNode(" " + flags.join(", ")));
|
||||
}
|
||||
if (assessment && assessment.redacted) {
|
||||
var redacted = document.createElement("span");
|
||||
redacted.className = "output-warning-redacted";
|
||||
redacted.textContent = " (credentials redacted)";
|
||||
warning.appendChild(redacted);
|
||||
}
|
||||
return warning;
|
||||
}
|
||||
|
||||
Pane.prototype._attachRetryToLastAssistant = function () {
|
||||
// Remove any previous retry buttons
|
||||
var old = this.messagesEl.querySelectorAll(".msg.assistant .msg-actions");
|
||||
@@ -1029,6 +1337,21 @@ Pane.prototype._attachRetryToLastAssistant = function () {
|
||||
// Find the last assistant message with content and add retry.
|
||||
// Reasoning blocks emit as .msg.reasoning (distinct modifier) so the
|
||||
// .msg.assistant selector already excludes them — no extra guard needed.
|
||||
//
|
||||
// Skip retry attachment when the most recent semantic turn is
|
||||
// tool-only — last DOM child is a .ts-approval block. Walk back
|
||||
// past .user-reminder bubbles (added via addToolReminder /
|
||||
// addUserReminder AFTER the .ts-approval block they advise) so the
|
||||
// guard fires correctly even when the tool turn carried a metacog
|
||||
// reminder. Without this skip, retry lands on a stale prior
|
||||
// assistant content bubble belonging to an earlier turn.
|
||||
var lastChild = this.messagesEl.lastElementChild;
|
||||
while (lastChild && lastChild.classList.contains("user-reminder")) {
|
||||
lastChild = lastChild.previousElementSibling;
|
||||
}
|
||||
if (lastChild && lastChild.classList.contains("ts-approval")) {
|
||||
return;
|
||||
}
|
||||
var assistants = this.messagesEl.querySelectorAll(".msg.assistant");
|
||||
if (assistants.length) {
|
||||
this._addRetryAction(assistants[assistants.length - 1]);
|
||||
@@ -1318,6 +1641,19 @@ Pane.prototype.appendToolOutput = function (callId, name, output, isError) {
|
||||
var stripped = stripAnsi(output || "").trim();
|
||||
if (!stripped) return;
|
||||
|
||||
// Skip rendering for denied/blocked tool results — the ✗ denied
|
||||
// badge from resolveApproval already shows the denial reason; the
|
||||
// SSE tool_result event would otherwise duplicate the text. Mirror
|
||||
// the guard in the history-replay path (the live path used to be
|
||||
// safe because no tool_result event was ever emitted for denied
|
||||
// items, but we now emit one so _tool_error_flags gets set).
|
||||
var parentBlock = target.closest(".ts-approval");
|
||||
var isDenied =
|
||||
(parentBlock && parentBlock.classList.contains("denied")) ||
|
||||
/^Denied by user/.test(stripped) ||
|
||||
/^Blocked/.test(stripped);
|
||||
if (isDenied) return;
|
||||
|
||||
// Detect structured media output and render interactive embed
|
||||
if (!isError) {
|
||||
var media = tryParseMedia(stripped);
|
||||
@@ -1332,12 +1668,9 @@ Pane.prototype.appendToolOutput = function (callId, name, output, isError) {
|
||||
var out = renderToolOutput(stripped, isError);
|
||||
|
||||
// Mark the parent approval block as errored
|
||||
if (isError) {
|
||||
var parentBlock = target.closest(".ts-approval");
|
||||
if (parentBlock && !parentBlock.classList.contains("denied")) {
|
||||
parentBlock.classList.add("error");
|
||||
appendToolErrorBadge(parentBlock);
|
||||
}
|
||||
if (isError && parentBlock && !parentBlock.classList.contains("denied")) {
|
||||
parentBlock.classList.add("error");
|
||||
appendToolErrorBadge(parentBlock);
|
||||
}
|
||||
|
||||
if (out.textContent.split("\n").length > 10) {
|
||||
@@ -1355,20 +1688,14 @@ Pane.prototype.showOutputWarning = function (evt) {
|
||||
'.ts-approval-tool[data-call-id="' + escapedId + '"]',
|
||||
);
|
||||
if (!toolDiv) return;
|
||||
var risk = evt.risk_level || "medium";
|
||||
var flags = evt.flags || [];
|
||||
var warning = document.createElement("div");
|
||||
warning.className = "output-warning output-warning-" + risk;
|
||||
warning.setAttribute("role", "alert");
|
||||
warning.innerHTML =
|
||||
'<span class="output-warning-label">\u26a0 ' +
|
||||
escapeHtml(risk.toUpperCase()) +
|
||||
"</span> " +
|
||||
flags.map(escapeHtml).join(", ");
|
||||
if (evt.redacted) {
|
||||
warning.innerHTML +=
|
||||
' <span class="output-warning-redacted">(credentials redacted)</span>';
|
||||
}
|
||||
// Shared DOM-builder with replayHistory \u2014 single source of truth for
|
||||
// role / class / escape semantics. Argument shape mirrors the
|
||||
// server-side output_assessment dict (risk_level / flags / redacted).
|
||||
var warning = _buildOutputWarningEl({
|
||||
risk_level: evt.risk_level,
|
||||
flags: evt.flags,
|
||||
redacted: evt.redacted,
|
||||
});
|
||||
var nextEl = toolDiv.nextElementSibling;
|
||||
if (nextEl && nextEl.classList.contains("tool-output")) {
|
||||
nextEl.insertAdjacentElement("afterend", warning);
|
||||
@@ -3511,12 +3838,35 @@ function updateDashFooter(agg) {
|
||||
}
|
||||
}
|
||||
|
||||
var _wsDeleteMode = false;
|
||||
var _wsDeleteSelected = {};
|
||||
// Saved Workstreams cache + multi-select delete controller. The
|
||||
// controller (from /shared/cards.js) owns mode state, checkbox
|
||||
// decoration, the toolbar wiring, and the confirmation modal — see
|
||||
// createSavedCardsController for the shared bits.
|
||||
var _wsSavedItems = [];
|
||||
var _wsDeleteController = createSavedCardsController({
|
||||
idPrefix: "ws-delete",
|
||||
buttonId: "ws-delete-btn",
|
||||
noun: "workstream",
|
||||
activateLabel: function (s) {
|
||||
return "Resume: " + (s.alias || s.title || s.ws_id);
|
||||
},
|
||||
buildDeleteRequest: function (wsId) {
|
||||
return {
|
||||
url: "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/delete",
|
||||
options: { method: "POST" },
|
||||
};
|
||||
},
|
||||
render: function () {
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
},
|
||||
onClose: function () {
|
||||
loadDashboard();
|
||||
},
|
||||
});
|
||||
|
||||
function renderSavedWorkstreams(items) {
|
||||
_wsSavedItems = items;
|
||||
_wsDeleteController.setItems(items);
|
||||
var c = document.getElementById("dashboard-saved-cards");
|
||||
c.replaceChildren();
|
||||
if (!items.length) {
|
||||
@@ -3527,289 +3877,39 @@ function renderSavedWorkstreams(items) {
|
||||
return;
|
||||
}
|
||||
items.forEach(function (sess) {
|
||||
// Default card shape (title + meta + wsid + Resume click) comes from
|
||||
// the shared /shared/cards.js helper so console (Saved Coordinators)
|
||||
// and ui/static (Saved Workstreams) stay in lock-step. Delete mode
|
||||
// is interactive-only; we layer the checkbox + selection wiring on
|
||||
// top of the shared card after construction.
|
||||
var card = renderSessionCard(sess, {
|
||||
ariaLabel: function (s) {
|
||||
var label = s.alias || s.title || s.ws_id;
|
||||
return _wsDeleteMode ? "Select: " + label : "Resume: " + label;
|
||||
},
|
||||
ariaLabel: _wsDeleteController.ariaLabel,
|
||||
onActivate: function (s) {
|
||||
// Suppressed in delete mode \u2014 the layered checkbox handler below
|
||||
// owns clicks while delete-mode is active.
|
||||
if (_wsDeleteMode) return;
|
||||
if (_wsDeleteController.blockActivate()) return;
|
||||
dashboardResumeSession(s.ws_id);
|
||||
},
|
||||
});
|
||||
|
||||
if (_wsDeleteMode) {
|
||||
card.classList.add("ws-delete-mode");
|
||||
card.removeAttribute("role"); // becomes a checkbox host, not a button
|
||||
var chk = document.createElement("input");
|
||||
chk.type = "checkbox";
|
||||
chk.className = "ws-card-check";
|
||||
chk.checked = !!_wsDeleteSelected[sess.ws_id];
|
||||
var label = sess.alias || sess.title || sess.ws_id;
|
||||
chk.setAttribute("aria-label", "Select " + label + " for deletion");
|
||||
chk.onclick = function (e) {
|
||||
e.stopPropagation();
|
||||
if (chk.checked) _wsDeleteSelected[sess.ws_id] = true;
|
||||
else delete _wsDeleteSelected[sess.ws_id];
|
||||
card.classList.toggle("ws-selected", chk.checked);
|
||||
updateWsDeleteBar();
|
||||
};
|
||||
card.insertBefore(chk, card.firstChild);
|
||||
// Override the shared helper's onclick/onkeydown \u2014 in delete mode
|
||||
// a card click toggles the checkbox instead of activating Resume.
|
||||
card.onclick = function (e) {
|
||||
if (e.target === chk) return;
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
};
|
||||
card.onkeydown = function (e) {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
}
|
||||
};
|
||||
if (_wsDeleteSelected[sess.ws_id]) card.classList.add("ws-selected");
|
||||
}
|
||||
|
||||
_wsDeleteController.decorateCard(card, sess);
|
||||
c.appendChild(card);
|
||||
});
|
||||
if (_wsDeleteController.inMode()) _wsDeleteController.refreshBar();
|
||||
}
|
||||
|
||||
// HTML inline-onclick wrappers — keep the global names the existing
|
||||
// markup binds to (`onclick="startWsDeleteMode()"` etc.) and forward
|
||||
// to the controller.
|
||||
function startWsDeleteMode() {
|
||||
_wsDeleteMode = true;
|
||||
_wsDeleteSelected = {};
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
var btn = document.getElementById("ws-delete-btn");
|
||||
if (btn) {
|
||||
btn.textContent = "\u2715 Cancel";
|
||||
btn.onclick = cancelWsDeleteMode;
|
||||
}
|
||||
var bar = document.getElementById("ws-delete-bar");
|
||||
if (bar) bar.classList.add("visible");
|
||||
_wsDeleteController.start();
|
||||
}
|
||||
|
||||
function cancelWsDeleteMode() {
|
||||
_wsDeleteMode = false;
|
||||
_wsDeleteSelected = {};
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
var btn = document.getElementById("ws-delete-btn");
|
||||
if (btn) {
|
||||
btn.innerHTML = "🗑 Delete";
|
||||
btn.onclick = startWsDeleteMode;
|
||||
}
|
||||
var bar = document.getElementById("ws-delete-bar");
|
||||
if (bar) bar.classList.remove("visible");
|
||||
_wsDeleteController.cancel();
|
||||
}
|
||||
|
||||
function updateWsDeleteBar() {
|
||||
var count = Object.keys(_wsDeleteSelected).length;
|
||||
var label = document.getElementById("ws-delete-bar-count");
|
||||
if (label) label.textContent = count + " selected";
|
||||
var delBtn = document.getElementById("ws-delete-bar-delete");
|
||||
if (delBtn) delBtn.disabled = count === 0;
|
||||
var selBtn = document.getElementById("ws-delete-bar-select-all");
|
||||
if (selBtn) {
|
||||
var allSelected =
|
||||
count === _wsSavedItems.length && _wsSavedItems.length > 0;
|
||||
selBtn.textContent = allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
}
|
||||
|
||||
function toggleSelectAll() {
|
||||
var allSelected =
|
||||
Object.keys(_wsDeleteSelected).length === _wsSavedItems.length &&
|
||||
_wsSavedItems.length > 0;
|
||||
if (allSelected) {
|
||||
_wsDeleteSelected = {};
|
||||
} else {
|
||||
_wsSavedItems.forEach(function (s) {
|
||||
_wsDeleteSelected[s.ws_id] = true;
|
||||
});
|
||||
}
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
updateWsDeleteBar();
|
||||
_wsDeleteController.toggleAll();
|
||||
}
|
||||
|
||||
var _wsDeleteBatchTrap = null;
|
||||
|
||||
function confirmWsDeleteSelection() {
|
||||
var selected = Object.keys(_wsDeleteSelected);
|
||||
if (!selected.length) {
|
||||
showToast("No workstreams selected", "warning");
|
||||
return;
|
||||
}
|
||||
var overlay = document.getElementById("ws-delete-overlay");
|
||||
var countEl = document.getElementById("ws-delete-count");
|
||||
var listEl = document.getElementById("ws-delete-list");
|
||||
var errorEl = document.getElementById("ws-delete-error");
|
||||
errorEl.textContent = "";
|
||||
countEl.textContent =
|
||||
selected.length + " workstream(s) will be permanently deleted:";
|
||||
listEl.innerHTML = "";
|
||||
selected.forEach(function (wsId) {
|
||||
var item = _wsSavedItems.find(function (s) {
|
||||
return s.ws_id === wsId;
|
||||
});
|
||||
var name = item ? item.alias || item.title || wsId : wsId;
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item";
|
||||
div.textContent = name;
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
// Reset confirm button handler (may have been overwritten to "Close" by previous run)
|
||||
var delBtn = document.getElementById("ws-delete-confirm-btn");
|
||||
if (delBtn) {
|
||||
delBtn.textContent = "Delete";
|
||||
delBtn.disabled = false;
|
||||
delBtn.classList.remove("ws-delete-close");
|
||||
delBtn.onclick = confirmWsDelete;
|
||||
}
|
||||
var cancelBtn = document.getElementById("ws-delete-cancel-btn");
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
overlay.style.display = "flex";
|
||||
|
||||
// Focus trap + Escape
|
||||
if (_wsDeleteBatchTrap)
|
||||
document.removeEventListener("keydown", _wsDeleteBatchTrap);
|
||||
_wsDeleteBatchTrap = function (e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
cancelWsDelete();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Tab") {
|
||||
var box = document.getElementById("ws-delete-box");
|
||||
var focusable = box.querySelectorAll("button:not(:disabled)");
|
||||
var first = focusable[0];
|
||||
var last = focusable[focusable.length - 1];
|
||||
if (e.shiftKey && document.activeElement === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && document.activeElement === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", _wsDeleteBatchTrap);
|
||||
if (cancelBtn) cancelBtn.focus();
|
||||
_wsDeleteController.confirmSelection();
|
||||
}
|
||||
|
||||
function cancelWsDelete() {
|
||||
document.getElementById("ws-delete-overlay").style.display = "none";
|
||||
if (_wsDeleteBatchTrap) {
|
||||
document.removeEventListener("keydown", _wsDeleteBatchTrap);
|
||||
_wsDeleteBatchTrap = null;
|
||||
}
|
||||
_wsDeleteController.closeModal();
|
||||
}
|
||||
|
||||
function confirmWsDelete() {
|
||||
var selected = Object.keys(_wsDeleteSelected);
|
||||
if (!selected.length) return;
|
||||
var overlay = document.getElementById("ws-delete-overlay");
|
||||
var errorEl = document.getElementById("ws-delete-error");
|
||||
var listEl = document.getElementById("ws-delete-list");
|
||||
var countEl = document.getElementById("ws-delete-count");
|
||||
var delBtn = document.getElementById("ws-delete-confirm-btn");
|
||||
var cancelBtn = document.getElementById("ws-delete-cancel-btn");
|
||||
errorEl.textContent = "";
|
||||
|
||||
// Disable buttons during deletion
|
||||
if (delBtn) {
|
||||
delBtn.disabled = true;
|
||||
delBtn.textContent = "Deleting...";
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = true;
|
||||
|
||||
var results = [];
|
||||
var promises = selected.map(function (wsId) {
|
||||
var shortId = wsId.substring(0, 8);
|
||||
var item = _wsSavedItems.find(function (s) {
|
||||
return s.ws_id === wsId;
|
||||
});
|
||||
var name = item ? item.alias || item.title || wsId : wsId;
|
||||
var url = "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/delete";
|
||||
|
||||
return authFetch(url, { method: "POST" })
|
||||
.then(function (r) {
|
||||
var status = r.status;
|
||||
var contentType = r.headers.get("content-type") || "";
|
||||
if (r.ok) {
|
||||
results.push({ name: name, shortId: shortId, ok: true });
|
||||
return;
|
||||
}
|
||||
// Read body as text first to avoid JSON parse errors
|
||||
return r.text().then(function (body) {
|
||||
var errMsg = shortId + ": HTTP " + status;
|
||||
if (contentType.includes("json")) {
|
||||
try {
|
||||
var j = JSON.parse(body);
|
||||
if (j.error) errMsg = shortId + ": " + j.error;
|
||||
} catch (_) {
|
||||
/* fall through */
|
||||
}
|
||||
} else if (body) {
|
||||
errMsg = shortId + ": " + body.substring(0, 200);
|
||||
}
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: errMsg,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(function (err) {
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: shortId + ": " + err.message,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Promise.all(promises).then(function () {
|
||||
// Rebuild the list with results
|
||||
listEl.innerHTML = "";
|
||||
results.forEach(function (r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item" + (r.ok ? "" : " ws-delete-error");
|
||||
div.textContent =
|
||||
(r.ok ? "\u2713 " : "\u2717 ") +
|
||||
r.name +
|
||||
(r.error ? " — " + r.error : "");
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
|
||||
var okCount = results.filter(function (r) {
|
||||
return r.ok;
|
||||
}).length;
|
||||
var failCount = results.filter(function (r) {
|
||||
return !r.ok;
|
||||
}).length;
|
||||
countEl.textContent = okCount + " deleted, " + failCount + " failed";
|
||||
|
||||
if (delBtn) {
|
||||
delBtn.disabled = false;
|
||||
delBtn.textContent = "Close";
|
||||
delBtn.classList.add("ws-delete-close");
|
||||
delBtn.onclick = function () {
|
||||
cancelWsDelete();
|
||||
cancelWsDeleteMode();
|
||||
loadDashboard();
|
||||
};
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
});
|
||||
_wsDeleteController.confirm();
|
||||
}
|
||||
|
||||
// --- Workstream title management ---
|
||||
|
||||
@@ -363,17 +363,18 @@
|
||||
<!-- Delete workstreams confirmation modal (batch) -->
|
||||
<div
|
||||
id="ws-delete-overlay"
|
||||
class="ws-delete-modal-overlay"
|
||||
style="display: none"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="ws-delete-title"
|
||||
>
|
||||
<div id="ws-delete-box">
|
||||
<div id="ws-delete-box" class="ws-delete-modal-box">
|
||||
<h3 id="ws-delete-title">Delete Workstreams</h3>
|
||||
<div id="ws-delete-error" role="alert" aria-live="assertive"></div>
|
||||
<p id="ws-delete-count"></p>
|
||||
<div id="ws-delete-list"></div>
|
||||
<div id="ws-delete-buttons">
|
||||
<div id="ws-delete-list" class="ws-delete-modal-list"></div>
|
||||
<div id="ws-delete-buttons" class="ws-delete-modal-buttons">
|
||||
<button
|
||||
id="ws-delete-cancel-btn"
|
||||
type="button"
|
||||
@@ -383,6 +384,7 @@
|
||||
</button>
|
||||
<button
|
||||
id="ws-delete-confirm-btn"
|
||||
class="ws-delete-confirm"
|
||||
type="button"
|
||||
onclick="confirmWsDelete()"
|
||||
>
|
||||
|
||||
+85
-250
@@ -642,6 +642,9 @@
|
||||
.msg.user {
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
/* .msg.user-reminder lives in shared_static/chat.css so both the
|
||||
interactive UI and the console coord viewer pick up the same
|
||||
yellow themed bubble. */
|
||||
/* .msg.assistant / .msg.info / .msg.error alignment + baseline visuals
|
||||
come from shared_static/chat.css. Interactive UI adds a pre-wrap
|
||||
override for info messages and a tightened tool-message shape with
|
||||
@@ -1628,6 +1631,77 @@ body {
|
||||
.ts-approval-tool .tool-diff .diff-warn {
|
||||
color: var(--yellow);
|
||||
}
|
||||
/* memory/recall calls are background metadata — the audit trail is
|
||||
valuable but they crowd the narrative when a workstream contains
|
||||
dozens of them. Dim by default; full opacity on hover/focus so
|
||||
they remain inspectable without permanently competing for
|
||||
attention. General-sibling combinator (~) extends the fade past
|
||||
any verdict-badge or output-warning sitting between the tool row
|
||||
and its output, so the whole sub-tree fades together rather than
|
||||
leaving a full-opacity badge stranded next to a dim row. */
|
||||
.ts-approval-tool[data-func-name="memory"],
|
||||
.ts-approval-tool[data-func-name="recall"] {
|
||||
opacity: 0.55;
|
||||
transition: opacity 120ms ease-out;
|
||||
}
|
||||
.ts-approval-tool[data-func-name="memory"]:hover,
|
||||
.ts-approval-tool[data-func-name="memory"]:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"]:hover,
|
||||
.ts-approval-tool[data-func-name="recall"]:focus-within {
|
||||
opacity: 1;
|
||||
}
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .media-embed,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .media-embed,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .output-warning,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .output-warning,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output-truncated,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output-truncated {
|
||||
opacity: 0.55;
|
||||
transition: opacity 120ms ease-out;
|
||||
}
|
||||
/* Reveal on hover OR focus-within across the entire dimmed
|
||||
subtree. Without :focus-within on the siblings, a keyboard user
|
||||
tabbing into a link or collapsible toggle inside .tool-output
|
||||
sees the content remain dimmed — a11y regression. Cover the
|
||||
warning + truncation pills too so they fully reveal alongside
|
||||
the result they decorate. */
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output:focus-within,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .media-embed:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .media-embed:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .media-embed:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .media-embed:focus-within,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .output-warning:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .output-warning:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .output-warning:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .output-warning:focus-within,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output-truncated:hover,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output-truncated:hover,
|
||||
.ts-approval-tool[data-func-name="memory"] ~ .tool-output-truncated:focus-within,
|
||||
.ts-approval-tool[data-func-name="recall"] ~ .tool-output-truncated:focus-within {
|
||||
opacity: 1;
|
||||
}
|
||||
/* Truncation indicator — the persisted tool result is clamped at
|
||||
2000 chars per row in storage; surface that on replay so users
|
||||
know they're seeing a clipped view rather than the full output
|
||||
the live session saw. Aligns with .output-warning's left gutter
|
||||
(margin-left: 16px) and uses transparent background + dim border
|
||||
so it reads as quiet metadata rather than a foreign element. */
|
||||
.tool-output-truncated {
|
||||
display: inline-block;
|
||||
margin-top: 4px;
|
||||
margin-left: 16px;
|
||||
padding: 1px 6px;
|
||||
font-size: 10px;
|
||||
color: var(--fg-dim);
|
||||
background: transparent;
|
||||
border-radius: 3px;
|
||||
border: 1px solid var(--fg-dim);
|
||||
}
|
||||
/* .ts-approval (chat.css) stacks its children with flex gap, so a
|
||||
border-top on the body would float above a strip of container
|
||||
background instead of sitting flush against the previous tool row.
|
||||
@@ -2363,224 +2437,10 @@ audio.media-player {
|
||||
color: var(--accent);
|
||||
margin: 0;
|
||||
}
|
||||
.ws-delete-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
color 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
.ws-delete-btn:hover {
|
||||
color: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
/* .dashboard-cards / .dashboard-card / .card-title / .card-meta moved to
|
||||
/shared/cards.css so console (Saved Coordinators) and ui/static (Saved
|
||||
Workstreams) share one source of truth for the basic card primitive.
|
||||
Delete-mode rules stay below — they're ui/static-only until coordinator
|
||||
gets the same UX. */
|
||||
|
||||
/* Delete mode */
|
||||
.dashboard-card.ws-delete-mode {
|
||||
cursor: pointer;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.04);
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected {
|
||||
cursor: default;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode:hover {
|
||||
background: rgba(220, 38, 38, 0.04);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-card-check {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: var(--red);
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
opacity: 0;
|
||||
animation: ws-check-fadein 0.2s ease-out forwards;
|
||||
}
|
||||
@keyframes ws-check-fadein {
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
.dashboard-card.ws-selected {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-selected {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-delete-bar {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
display: flex;
|
||||
animation: ws-bar-slide 0.2s ease-out;
|
||||
}
|
||||
@keyframes ws-bar-slide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ws-card-check {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
.ws-delete-bar .ws-delete-count-label {
|
||||
font-size: 12px;
|
||||
color: var(--fg-dim);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: auto;
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 16px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:hover:not(:disabled) {
|
||||
filter: brightness(1.1);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn:hover {
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
|
||||
/* Delete modal */
|
||||
#ws-delete-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
#ws-delete-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 24px;
|
||||
max-width: 480px;
|
||||
width: 90%;
|
||||
}
|
||||
#ws-delete-box h3 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 16px;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
#ws-delete-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
margin: 12px 0;
|
||||
}
|
||||
#ws-delete-list .ws-delete-item {
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
color: var(--fg-bright);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
#ws-delete-list .ws-delete-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
#ws-delete-list .ws-delete-item.ws-delete-error {
|
||||
color: var(--red);
|
||||
}
|
||||
#ws-delete-buttons {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
#ws-delete-buttons button {
|
||||
padding: 8px 20px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
#ws-delete-buttons button:last-child {
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
border-color: var(--red);
|
||||
}
|
||||
#ws-delete-buttons button.ws-delete-close {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border);
|
||||
}
|
||||
/* .dashboard-cards / .dashboard-card / .card-title / .card-meta and the
|
||||
delete-mode + modal rules are owned by /shared/cards.css so console
|
||||
(Saved Coordinators) and ui/static (Saved Workstreams) share one source
|
||||
of truth. */
|
||||
|
||||
/* Server dashboard row — clickable */
|
||||
.dash-row {
|
||||
@@ -2686,38 +2546,13 @@ audio.media-player {
|
||||
}
|
||||
}
|
||||
|
||||
/* ==========================================================================
|
||||
Verdict badges (intent judge)
|
||||
========================================================================== */
|
||||
.verdict-badge {
|
||||
padding: 4px 10px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-top: 2px;
|
||||
/* No top separator — .ts-verdict-badge (chat.css) shrinks to
|
||||
max-content width, and a 1px border-top would extend only under
|
||||
the badge text and read as a truncated line. */
|
||||
}
|
||||
.verdict-low {
|
||||
color: var(--green);
|
||||
border-left: 3px solid var(--green);
|
||||
}
|
||||
.verdict-medium {
|
||||
color: var(--yellow);
|
||||
border-left: 3px solid var(--yellow);
|
||||
}
|
||||
.verdict-high {
|
||||
color: var(--red);
|
||||
border-left: 3px solid var(--red);
|
||||
}
|
||||
.verdict-critical {
|
||||
color: var(--red);
|
||||
border-left: 3px solid var(--red);
|
||||
background: rgba(255, 80, 80, 0.05);
|
||||
}
|
||||
/* Verdict-badge styling lives in the color-mix block further down
|
||||
in this file (.verdict-badge.verdict-{low,medium,high,critical}).
|
||||
The earlier flat-palette duplicate that lived here was removed —
|
||||
two competing .verdict-badge rule sets caused subtle cascade drift
|
||||
(the color-mix block won for backgrounds, the flat one won for the
|
||||
bare .verdict-low/medium/high/critical class names) which made
|
||||
tweaks fragile. Single source of truth now. */
|
||||
|
||||
.verdict-detail {
|
||||
padding: 6px 12px;
|
||||
|
||||
Reference in New Issue
Block a user