mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 07:22:24 -06:00
Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4693fa95f1 | |||
| c3423d6606 | |||
| 53f1222c22 | |||
| 802d87a57f | |||
| 8349d9994d | |||
| ac1fd67137 | |||
| 1b40ae79f9 | |||
| b078ddccf0 | |||
| 4b6c93a0e9 | |||
| 9d283e951f | |||
| 4e407e7d4f | |||
| 7ab24e500b |
+1
-1
@@ -8,7 +8,7 @@ FROM python:3.14-slim
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.8 /uv /usr/local/bin/uv
|
||||
|
||||
# Remove the slim image's man page exclusion so man-db has actual content
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/docker
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "1.5.2"
|
||||
version = "1.5.4"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
|
||||
@@ -35,11 +35,10 @@ def _seed_children(
|
||||
The production path populates the registry via the cluster-event
|
||||
fan-out thread observing ``ws_created`` events. These tests just
|
||||
need a known-children set for the endpoint handlers to iterate —
|
||||
inject directly under ``_children_lock`` rather than spinning up
|
||||
the collector + fan-out plumbing.
|
||||
inject directly via the registry's bulk-merge surface rather than
|
||||
spinning up the collector + fan-out plumbing.
|
||||
"""
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked(coord_ws_id, child_ws_ids)
|
||||
adapter._registry.merge_children(coord_ws_id, child_ws_ids)
|
||||
|
||||
|
||||
class _AuthMiddleware(BaseHTTPMiddleware):
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Unit tests for :mod:`turnstone.core.child_source`.
|
||||
|
||||
Covers both strategies in isolation against fakes — no live collector,
|
||||
no live SessionManager. Adapter-level integration coverage continues to
|
||||
live in ``test_coordinator_adapter.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.core.child_source import ClusterChildSource, SameNodeChildSource
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import queue
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SameNodeChildSource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeManager:
|
||||
"""Minimal SessionManager stand-in implementing the subscribe API."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.subscribers: list[Any] = []
|
||||
|
||||
def subscribe_to_state(self, callback: Any) -> None:
|
||||
self.subscribers.append(callback)
|
||||
|
||||
def unsubscribe_from_state(self, callback: Any) -> None:
|
||||
with contextlib.suppress(ValueError):
|
||||
self.subscribers.remove(callback)
|
||||
|
||||
def fire(self, ws_id: str, state: WorkstreamState) -> None:
|
||||
for cb in self.subscribers:
|
||||
cb(ws_id, state)
|
||||
|
||||
|
||||
class TestSameNodeChildSource:
|
||||
def test_start_subscribes_to_manager(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
assert len(mgr.subscribers) == 1
|
||||
|
||||
def test_state_change_for_known_child_pushes_to_sink(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
registry.add_child("p1", "c1")
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
|
||||
mgr.fire("c1", WorkstreamState.RUNNING)
|
||||
|
||||
assert len(sink_calls) == 1
|
||||
ev = sink_calls[0]
|
||||
assert ev["type"] == "cluster_state"
|
||||
assert ev["ws_id"] == "c1"
|
||||
assert ev["state"] == "running"
|
||||
|
||||
def test_state_change_for_unknown_workstream_is_dropped(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
src.start(sink=sink_calls.append)
|
||||
|
||||
# No registry entry — pre-filter drops the event without
|
||||
# invoking the sink.
|
||||
mgr.fire("ws-unknown", WorkstreamState.IDLE)
|
||||
assert sink_calls == []
|
||||
|
||||
def test_shutdown_unsubscribes(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(mgr.subscribers) == 1
|
||||
src.shutdown()
|
||||
assert mgr.subscribers == []
|
||||
|
||||
def test_start_is_idempotent(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
src.start(sink=lambda ev: None)
|
||||
src.start(sink=lambda ev: None)
|
||||
# Second start is a no-op; only one subscription.
|
||||
assert len(mgr.subscribers) == 1
|
||||
|
||||
def test_sink_exception_does_not_propagate(self) -> None:
|
||||
mgr = _FakeManager()
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
registry.add_child("p1", "c1")
|
||||
src = SameNodeChildSource(mgr, registry)
|
||||
|
||||
def bad_sink(ev: dict[str, Any]) -> None:
|
||||
raise RuntimeError("sink boom")
|
||||
|
||||
src.start(sink=bad_sink)
|
||||
# Should not raise — the strategy catches sink failures and logs.
|
||||
mgr.fire("c1", WorkstreamState.RUNNING)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClusterChildSource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeCollector:
|
||||
"""Minimal ClusterCollector stand-in providing the listener API."""
|
||||
|
||||
def __init__(self, snapshot: dict[str, Any] | None = None) -> None:
|
||||
self._snapshot = snapshot or {"nodes": []}
|
||||
self.queues: list[queue.Queue[dict[str, Any]]] = []
|
||||
self.unregistered: list[queue.Queue[dict[str, Any]]] = []
|
||||
|
||||
def get_snapshot_and_register(self, q: queue.Queue[dict[str, Any]]) -> dict[str, Any]:
|
||||
self.queues.append(q)
|
||||
return self._snapshot
|
||||
|
||||
def unregister_listener(self, q: queue.Queue[dict[str, Any]]) -> None:
|
||||
self.unregistered.append(q)
|
||||
|
||||
def emit(self, event: dict[str, Any]) -> None:
|
||||
"""Push an event to all registered listener queues."""
|
||||
for q in self.queues:
|
||||
q.put(event)
|
||||
|
||||
|
||||
class TestClusterChildSource:
|
||||
def test_start_subscribes_to_collector(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(coll.queues) == 1
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_start_primes_registry_from_snapshot(self) -> None:
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
{
|
||||
"workstreams": [
|
||||
{"id": "c1", "parent_ws_id": "p1"},
|
||||
{"id": "c2", "parent_ws_id": "p1"},
|
||||
# Unknown parent — dropped
|
||||
{"id": "x", "parent_ws_id": "p-unknown"},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
coll = _FakeCollector(snapshot)
|
||||
registry = ChildrenRegistry()
|
||||
registry.install("p1", object())
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=lambda: ["p1"],
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
assert registry.parent_for("x") is None
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_event_dispatched_to_sink(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
sink_calls: list[dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
src.start(sink=sink_calls.append)
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c1", "state": "running"})
|
||||
# Daemon thread loop has 1.0s queue timeout; poll briefly.
|
||||
for _ in range(20):
|
||||
if sink_calls:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(sink_calls) == 1
|
||||
assert sink_calls[0]["ws_id"] == "c1"
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_shutdown_unregisters_and_joins_thread(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
src.start(sink=lambda ev: None)
|
||||
src.shutdown()
|
||||
assert coll.unregistered == coll.queues
|
||||
# Second shutdown is a no-op (idempotent).
|
||||
src.shutdown()
|
||||
|
||||
def test_start_is_idempotent(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
try:
|
||||
src.start(sink=lambda ev: None)
|
||||
src.start(sink=lambda ev: None)
|
||||
assert len(coll.queues) == 1
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
def test_sink_exception_does_not_kill_thread(self) -> None:
|
||||
coll = _FakeCollector()
|
||||
registry = ChildrenRegistry()
|
||||
src = ClusterChildSource(
|
||||
collector=coll,
|
||||
registry=registry,
|
||||
parents_provider=list,
|
||||
)
|
||||
survived_calls: list[dict[str, Any]] = []
|
||||
call_count = [0]
|
||||
|
||||
def flaky_sink(ev: dict[str, Any]) -> None:
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
raise RuntimeError("first one boom")
|
||||
survived_calls.append(ev)
|
||||
|
||||
try:
|
||||
src.start(sink=flaky_sink)
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c1", "state": "x"})
|
||||
coll.emit({"type": "cluster_state", "ws_id": "c2", "state": "y"})
|
||||
for _ in range(40):
|
||||
if survived_calls:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(survived_calls) == 1
|
||||
assert survived_calls[0]["ws_id"] == "c2"
|
||||
finally:
|
||||
src.shutdown()
|
||||
|
||||
|
||||
# Multi-subscriber observer tests for ``SessionManager.subscribe_to_state``
|
||||
# / ``unsubscribe_from_state`` live in ``test_session_manager.py`` where
|
||||
# the proper FakeAdapter / FakeStorage construction helpers already exist.
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Unit tests for :class:`turnstone.core.children_registry.ChildrenRegistry`.
|
||||
|
||||
The registry was lifted from ``CoordinatorAdapter`` in Stage 3 Step 1.
|
||||
Adapter-level coverage for the integrated behavior already lives in
|
||||
``test_coordinator_adapter.py``; this file pins the data structure
|
||||
invariants in isolation so the registry can be reused by future
|
||||
``ChildSource`` strategies (Step 2) without re-deriving the behavior
|
||||
from the adapter test surface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
|
||||
|
||||
class _Sentinel:
|
||||
"""Lightweight UI stand-in; identity-comparable, no behavior."""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry() -> ChildrenRegistry:
|
||||
return ChildrenRegistry()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# install / uninstall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInstallUninstall:
|
||||
def test_install_seeds_empty_child_set_and_presence(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.children_of("p1") == []
|
||||
assert registry.ui_for("p1") is ui
|
||||
assert registry.parents() == ["p1"]
|
||||
|
||||
def test_install_is_idempotent_repoints_ui_keeps_children(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui_a = _Sentinel()
|
||||
ui_b = _Sentinel()
|
||||
registry.install("p1", ui_a)
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
registry.install("p1", ui_b)
|
||||
assert registry.ui_for("p1") is ui_b
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
|
||||
def test_uninstall_clears_forward_reverse_and_presence(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
registry.uninstall("p1")
|
||||
assert registry.children_of("p1") == []
|
||||
assert registry.ui_for("p1") is None
|
||||
assert registry.parents() == []
|
||||
assert registry.parent_for("c1") is None
|
||||
assert registry.parent_for("c2") is None
|
||||
|
||||
def test_uninstall_unknown_parent_is_noop(self, registry: ChildrenRegistry) -> None:
|
||||
registry.uninstall("never-installed") # must not raise
|
||||
|
||||
def test_uninstall_does_not_clobber_other_parents(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
registry.install("p2", _Sentinel())
|
||||
registry.merge_children("p1", ["c1"])
|
||||
registry.merge_children("p2", ["c2"])
|
||||
registry.uninstall("p1")
|
||||
assert registry.parent_for("c1") is None
|
||||
assert registry.parent_for("c2") == "p2"
|
||||
assert registry.parents() == ["p2"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# add_child — atomic check-and-route
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAddChild:
|
||||
def test_add_child_returns_ui_on_success(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.add_child("p1", "c1") is ui
|
||||
assert registry.parent_for("c1") == "p1"
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
def test_add_child_returns_none_when_parent_not_installed(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
assert registry.add_child("absent", "c1") is None
|
||||
assert registry.parent_for("c1") is None
|
||||
|
||||
def test_add_child_returns_none_on_duplicate(self, registry: ChildrenRegistry) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
assert registry.add_child("p1", "c1") is ui
|
||||
# second add for same child returns None — caller must not
|
||||
# double-dispatch.
|
||||
assert registry.add_child("p1", "c1") is None
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_children — bulk seeding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergeChildren:
|
||||
def test_merge_seeds_forward_and_reverse(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["c1", "c2", "c3"])
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2", "c3"}
|
||||
for cid in ("c1", "c2", "c3"):
|
||||
assert registry.parent_for(cid) == "p1"
|
||||
|
||||
def test_merge_is_idempotent(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["c1"])
|
||||
registry.merge_children("p1", ["c1"])
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
|
||||
def test_merge_skips_empty_or_falsy_ids(self, registry: ChildrenRegistry) -> None:
|
||||
registry.merge_children("p1", ["", "c1", "", "c2"])
|
||||
assert set(registry.children_of("p1")) == {"c1", "c2"}
|
||||
|
||||
def test_merge_does_not_require_install(self, registry: ChildrenRegistry) -> None:
|
||||
# Snapshot-priming may run before the parent's install fires —
|
||||
# the merge still seeds the forward set so the install picks
|
||||
# the children up. (Storage-seeded rebuild relies on this.)
|
||||
registry.merge_children("p1", ["c1"])
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
# ui_for is still None because install hasn't run
|
||||
assert registry.ui_for("p1") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lookups — return copies, not live refs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLookups:
|
||||
def test_children_of_returns_copy(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
registry.merge_children("p1", ["c1", "c2"])
|
||||
snap = registry.children_of("p1")
|
||||
snap.append("c3-injected")
|
||||
assert "c3-injected" not in registry.children_of("p1")
|
||||
|
||||
def test_children_of_unknown_parent_returns_empty(self, registry: ChildrenRegistry) -> None:
|
||||
assert registry.children_of("absent") == []
|
||||
|
||||
def test_parent_for_unknown_child_returns_none(self, registry: ChildrenRegistry) -> None:
|
||||
assert registry.parent_for("absent") is None
|
||||
|
||||
def test_parents_returns_copy(self, registry: ChildrenRegistry) -> None:
|
||||
registry.install("p1", _Sentinel())
|
||||
snap = registry.parents()
|
||||
snap.append("p2-injected")
|
||||
assert "p2-injected" not in registry.parents()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concurrency — concurrent add_child must not exceed the unique-set
|
||||
# invariant or leave a half-installed reverse-index entry.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConcurrency:
|
||||
def test_concurrent_add_child_returns_ui_exactly_once_per_unique(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
results: list[object] = []
|
||||
results_lock = threading.Lock()
|
||||
|
||||
def attempt_add(child_id: str) -> None:
|
||||
r = registry.add_child("p1", child_id)
|
||||
with results_lock:
|
||||
results.append(r)
|
||||
|
||||
threads = [threading.Thread(target=attempt_add, args=("c1",)) for _ in range(20)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Exactly one thread sees the UI; the remaining 19 see None
|
||||
# (duplicate). The forward + reverse indexes carry exactly one
|
||||
# entry for c1.
|
||||
successes = [r for r in results if r is ui]
|
||||
nones = [r for r in results if r is None]
|
||||
assert len(successes) == 1
|
||||
assert len(nones) == 19
|
||||
assert registry.children_of("p1") == ["c1"]
|
||||
assert registry.parent_for("c1") == "p1"
|
||||
|
||||
def test_concurrent_install_and_add_child_no_resurrect(
|
||||
self, registry: ChildrenRegistry
|
||||
) -> None:
|
||||
# add_child racing with uninstall: either lands first (registry
|
||||
# populated) or the parent is gone (returns None). Must NOT
|
||||
# leave a forward-set entry without presence — that would be
|
||||
# the "resurrected after close" leak the locked dispatch path
|
||||
# was guarding against.
|
||||
ui = _Sentinel()
|
||||
registry.install("p1", ui)
|
||||
|
||||
outcomes: list[object] = []
|
||||
|
||||
def adder() -> None:
|
||||
outcomes.append(registry.add_child("p1", "c1"))
|
||||
|
||||
def uninstaller() -> None:
|
||||
registry.uninstall("p1")
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=adder),
|
||||
threading.Thread(target=uninstaller),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# If add_child landed first: c1 is in the forward set, then
|
||||
# uninstall clears everything. End state: nothing.
|
||||
# If uninstall landed first: add_child sees no presence,
|
||||
# returns None, no entry added. End state: nothing.
|
||||
# Either way, the leak invariant holds: child set is empty or
|
||||
# parent is gone, never "child set populated but no presence".
|
||||
children = registry.children_of("p1")
|
||||
ui_present = registry.ui_for("p1") is not None
|
||||
if children:
|
||||
assert ui_present, "registry leaked: children set without presence"
|
||||
+151
-24
@@ -314,12 +314,14 @@ class TestCollectorSnapshot:
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["state"] == "running"
|
||||
|
||||
def test_apply_snapshot_state_change_forwards_pending_approval_detail(self):
|
||||
"""Reconnect-via-snapshot is the resync path after every console
|
||||
restart or network blip. Without forwarding the field here,
|
||||
a child sitting in approval-pending across the gap renders as
|
||||
``activity_state=approval`` with no buttons until the next
|
||||
state change — broken UX during the most common re-sync event."""
|
||||
def test_apply_snapshot_state_change_does_not_carry_pending_approval_detail(self):
|
||||
"""Stage 3 cleanup — the snapshot-resync cluster_state event no
|
||||
longer piggybacks ``pending_approval_detail`` (the field is
|
||||
gone from cluster_state entirely). On reconnect the browser's
|
||||
bulk fetch — triggered by the ``activity_state="approval"``
|
||||
transition in the reducer — pulls the items directly from
|
||||
``ui.serialize_pending_approval_detail()`` via the dashboard
|
||||
endpoint."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -329,10 +331,6 @@ class TestCollectorSnapshot:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
c._apply_snapshot(
|
||||
"node-a",
|
||||
{
|
||||
@@ -344,7 +342,6 @@ class TestCollectorSnapshot:
|
||||
"name": "same",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
}
|
||||
],
|
||||
"health": {},
|
||||
@@ -354,7 +351,8 @@ class TestCollectorSnapshot:
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["pending_approval_detail"] == detail
|
||||
assert event["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in event
|
||||
|
||||
def test_apply_snapshot_skips_empty_id_workstream(self):
|
||||
c = _make_collector()
|
||||
@@ -401,12 +399,12 @@ class TestCollectorDelta:
|
||||
# Verify in-memory state was updated
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["state"] == "running"
|
||||
|
||||
def test_apply_delta_ws_state_forwards_pending_approval_detail(self):
|
||||
"""The rich approval payload now travels on the cluster bus so
|
||||
coord tabs can render inline approve/deny buttons in lockstep
|
||||
with the activity_state transition. Collector must forward
|
||||
the field verbatim — the adapter does the child-routing on
|
||||
top, but the bus carries the data."""
|
||||
def test_apply_delta_ws_state_does_not_carry_pending_approval_detail(self):
|
||||
"""Stage 3 cleanup — ``cluster_state`` no longer carries the
|
||||
``pending_approval_detail`` piggyback. Approval items now arrive
|
||||
via bulk fetch on activity_state transition; verdicts via the
|
||||
explicit ``intent_verdict`` event class. Symmetric event flow,
|
||||
no piggyback to dedupe against."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
@@ -416,10 +414,6 @@ class TestCollectorDelta:
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
@@ -427,13 +421,13 @@ class TestCollectorDelta:
|
||||
"ws_id": "ws1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "cluster_state"
|
||||
assert event["pending_approval_detail"] == detail
|
||||
assert event["activity_state"] == "approval"
|
||||
assert "pending_approval_detail" not in event
|
||||
|
||||
def test_apply_delta_ws_created(self):
|
||||
c = _make_collector()
|
||||
@@ -480,6 +474,139 @@ class TestCollectorDelta:
|
||||
assert event["name"] == "new-name"
|
||||
assert c._nodes["node-a"].workstreams["ws1"]["name"] == "new-name"
|
||||
|
||||
def test_apply_delta_intent_verdict_forwards_verbatim(self):
|
||||
"""Stage 3 Step 5 — node-emitted intent_verdict events flow
|
||||
through _apply_delta to cluster fan-out so coord adapters can
|
||||
re-emit as child_ws_intent_verdict on the parent's SSE."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.9,
|
||||
"recommendation": "approve",
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "intent_verdict", "ws_id": "ws1", "verdict": verdict},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "intent_verdict"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["verdict"] == verdict
|
||||
|
||||
def test_apply_delta_intent_verdict_drops_when_ws_id_missing(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "intent_verdict", "verdict": {}})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_delta_approval_resolved_forwards_verbatim(self):
|
||||
"""Stage 3 Step 5 — paired with intent_verdict; clears the
|
||||
coord tree's pending-approval pill in lockstep with the
|
||||
actual decision rather than waiting for the state-change
|
||||
piggyback."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "ws1",
|
||||
"approved": True,
|
||||
"feedback": "lgtm",
|
||||
"always": False,
|
||||
},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "approval_resolved"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["approved"] is True
|
||||
assert event["feedback"] == "lgtm"
|
||||
assert event["always"] is False
|
||||
|
||||
def test_apply_delta_approve_request_forwards_detail(self):
|
||||
"""Push path for the initial approval items — eliminates the
|
||||
bulk-fetch race that left the coord row stuck on a loading
|
||||
placeholder when the bulk fetch landed in the gap between
|
||||
_emit_state(ATTENTION) and approve_tools setting _pending_approval."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
c._apply_delta(
|
||||
"node-a",
|
||||
{"type": "approve_request", "ws_id": "ws1", "detail": detail},
|
||||
)
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["type"] == "approve_request"
|
||||
assert event["ws_id"] == "ws1"
|
||||
assert event["node_id"] == "node-a"
|
||||
assert event["detail"] == detail
|
||||
|
||||
def test_apply_delta_approve_request_drops_when_ws_id_missing(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(node_id="node-a", server_url="http://a:8080")
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "approve_request", "detail": {}})
|
||||
|
||||
assert q.empty()
|
||||
|
||||
def test_apply_delta_approval_resolved_coerces_missing_fields(self):
|
||||
"""Defensive: ``approved`` / ``always`` / ``feedback`` may be
|
||||
omitted by older nodes mid-rolling-upgrade; collector coerces
|
||||
to safe defaults."""
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
node_id="node-a",
|
||||
server_url="http://a:8080",
|
||||
workstreams={"ws1": {"id": "ws1", "name": "test", "state": "idle"}},
|
||||
)
|
||||
q: queue.Queue[dict] = queue.Queue()
|
||||
c.register_listener(q)
|
||||
|
||||
c._apply_delta("node-a", {"type": "approval_resolved", "ws_id": "ws1"})
|
||||
|
||||
event = q.get_nowait()
|
||||
assert event["approved"] is False
|
||||
assert event["feedback"] == ""
|
||||
assert event["always"] is False
|
||||
|
||||
def test_apply_delta_health_changed(self):
|
||||
c = _make_collector()
|
||||
c._nodes["node-a"] = NodeSnapshot(
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Tests for the console's coordinator idle-cleanup thread helper.
|
||||
|
||||
The helper itself is a tiny loop wrapping ``mgr.close_idle``; the heavy
|
||||
lifting is in ``SessionManager.close_idle`` (covered in
|
||||
``test_session_manager.py``) and ``bulk_close_stale_orphans`` (covered
|
||||
in ``test_storage_sqlite.py``). These tests verify the glue:
|
||||
|
||||
- the helper runs an initial sweep BEFORE its first sleep (cold-start
|
||||
cleanup without blocking the lifespan),
|
||||
- the helper swallows exceptions so a transient DB blip can't kill the
|
||||
daemon thread,
|
||||
- the helper exits cleanly when ``stop_event`` is set.
|
||||
|
||||
The ``stop_event`` parameter is exclusively for tests — production
|
||||
callers pass ``None`` and the daemon runs for process lifetime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
from turnstone.console.server import _coord_idle_cleanup_thread
|
||||
|
||||
|
||||
class _StubMgr:
|
||||
def __init__(
|
||||
self, *, stop_event: threading.Event, expected_calls: int, raise_after: int = -1
|
||||
) -> None:
|
||||
self.calls: list[float] = []
|
||||
self.sleep_calls_at_each_close: list[int] = []
|
||||
self._stop_event = stop_event
|
||||
self._expected = expected_calls
|
||||
self._raise_after = raise_after
|
||||
self._sleep_count = 0
|
||||
|
||||
def close_idle(self, timeout_sec: float) -> list[str]:
|
||||
# Snapshot how many sleeps preceded this close — lets the
|
||||
# "initial sweep" test verify the first close_idle ran with
|
||||
# zero preceding sleeps.
|
||||
self.sleep_calls_at_each_close.append(self._sleep_count)
|
||||
self.calls.append(timeout_sec)
|
||||
try:
|
||||
if 0 <= self._raise_after < len(self.calls):
|
||||
raise RuntimeError("simulated DB blip")
|
||||
finally:
|
||||
# Set stop after the helper has been exercised enough,
|
||||
# regardless of whether this call raised.
|
||||
if len(self.calls) >= self._expected:
|
||||
self._stop_event.set()
|
||||
return []
|
||||
|
||||
def record_sleep(self, _seconds: float) -> None:
|
||||
self._sleep_count += 1
|
||||
|
||||
|
||||
def _run_until_done(mgr: _StubMgr, stop_event: threading.Event, timeout_sec: float) -> None:
|
||||
with patch("turnstone.console.server.time.sleep", mgr.record_sleep):
|
||||
thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(mgr, timeout_sec, stop_event),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
thread.join(timeout=2.0)
|
||||
assert not thread.is_alive(), "helper failed to exit on stop_event"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_runs_initial_sweep_before_sleep() -> None:
|
||||
"""The first close_idle call must happen BEFORE the first time.sleep —
|
||||
otherwise cold-start orphans wait one ``check_every`` interval (~30 min
|
||||
on default 2h timeout) for the first reap. Crucial because the
|
||||
lifespan no longer does a synchronous initial sweep."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert mgr.sleep_calls_at_each_close == [0], "first close_idle should run before any sleep"
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_calls_close_idle_each_tick() -> None:
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=3)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert len(mgr.calls) == 3
|
||||
assert all(t == 120.0 for t in mgr.calls)
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_survives_close_idle_exceptions() -> None:
|
||||
"""A transient DB error must not kill the daemon thread — the next
|
||||
tick should still fire close_idle. Without the try/except, a single
|
||||
blip would silently leak orphans forever."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=4, raise_after=1)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
# All four calls must have fired despite calls 2-4 raising.
|
||||
assert len(mgr.calls) == 4
|
||||
|
||||
|
||||
def test_coord_idle_cleanup_exits_cleanly_on_stop_event() -> None:
|
||||
"""The stop_event mechanism is the test contract; verify the thread
|
||||
actually exits when the event is set, without needing exceptions or
|
||||
daemon-process termination."""
|
||||
stop_event = threading.Event()
|
||||
mgr = _StubMgr(stop_event=stop_event, expected_calls=2)
|
||||
_run_until_done(mgr, stop_event, timeout_sec=120.0)
|
||||
assert stop_event.is_set()
|
||||
@@ -464,3 +464,148 @@ def test_coord_budget_override_survives_wildcard_allow_policy() -> None:
|
||||
assert approved is True
|
||||
types = [e.get("type") for e in captured_events]
|
||||
assert "approve_request" in types, "Wildcard allow must not strip the budget-override prompt"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cluster-bus broadcast hooks — _broadcast_intent_verdict / _approval_resolved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBroadcastIntentVerdict:
|
||||
"""``ConsoleCoordinatorUI._broadcast_intent_verdict`` overrides the
|
||||
no-op base hook to push the verdict onto the cluster bus via
|
||||
``ClusterCollector.emit_console_ws_intent_verdict``. The far more
|
||||
common path is the per-node ``WebUI`` override (covered in
|
||||
test_webui_content.py); this lights up the rare coord-self path
|
||||
(a coord that runs its own LLM judge).
|
||||
"""
|
||||
|
||||
def test_calls_collector_emit_with_ws_id_and_verdict(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "high",
|
||||
"confidence": 0.91,
|
||||
}
|
||||
ui._broadcast_intent_verdict(verdict)
|
||||
collector.emit_console_ws_intent_verdict.assert_called_once_with(
|
||||
"coord-a",
|
||||
verdict,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_intent_verdict.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise — collector failures are observational only.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
class TestBroadcastApprovalResolved:
|
||||
"""``ConsoleCoordinatorUI._broadcast_approval_resolved`` overrides
|
||||
the base hook to push the resolution onto the cluster bus via
|
||||
``ClusterCollector.emit_console_ws_approval_resolved``."""
|
||||
|
||||
def test_calls_collector_with_decision_fields(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
ui._broadcast_approval_resolved(True, "lgtm", always=True)
|
||||
collector.emit_console_ws_approval_resolved.assert_called_once_with(
|
||||
"coord-a",
|
||||
approved=True,
|
||||
feedback="lgtm",
|
||||
always=True,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_normalises_none_feedback_to_empty_string(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
ui._broadcast_approval_resolved(False, None)
|
||||
collector.emit_console_ws_approval_resolved.assert_called_once_with(
|
||||
"coord-a",
|
||||
approved=False,
|
||||
feedback="",
|
||||
always=False,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, None)
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_approval_resolved.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, "ok")
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
|
||||
class TestBroadcastApproveRequest:
|
||||
"""Coord-side override for the approve_request push. Same rationale
|
||||
as the WebUI override — the coord-self path is rare today, but
|
||||
parity keeps the override symmetric with the rest of the broadcast
|
||||
family."""
|
||||
|
||||
def test_calls_collector_emit_with_ws_id_and_detail(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
ui._broadcast_approve_request(detail)
|
||||
collector.emit_console_ws_approve_request.assert_called_once_with(
|
||||
"coord-a",
|
||||
detail,
|
||||
)
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
def test_no_op_when_collector_unset(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
|
||||
def test_collector_exception_swallowed(self) -> None:
|
||||
ui = ConsoleCoordinatorUI(ws_id="coord-a", user_id="u1")
|
||||
collector = MagicMock()
|
||||
collector.emit_console_ws_approve_request.side_effect = RuntimeError("boom")
|
||||
ConsoleCoordinatorUI._collector = collector
|
||||
try:
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
finally:
|
||||
ConsoleCoordinatorUI._collector = None
|
||||
|
||||
@@ -378,13 +378,21 @@ class TestCoordinatorAdapterWorkerDispatch:
|
||||
|
||||
|
||||
class TestCoordinatorAdapterChildrenRegistry:
|
||||
def test_emit_created_seeds_empty_children_set(self) -> None:
|
||||
"""Adapter-level integration with :class:`ChildrenRegistry`.
|
||||
|
||||
Pure-registry invariants (forward/reverse consistency, idempotent
|
||||
merge, locking) live in ``test_children_registry.py``. These
|
||||
tests cover the adapter's wiring: that ``emit_*`` paths drive the
|
||||
registry correctly and that the snapshot-priming bridge between
|
||||
a collector snapshot and the registry preserves merge semantics.
|
||||
"""
|
||||
|
||||
def test_emit_created_installs_parent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
ws = _make_ws()
|
||||
adapter.emit_created(ws)
|
||||
assert ws.id in adapter._children
|
||||
assert adapter._children[ws.id] == set()
|
||||
assert adapter._active_coords[ws.id] is ws.ui
|
||||
assert adapter._registry.children_of(ws.id) == []
|
||||
assert adapter._registry.ui_for(ws.id) is ws.ui
|
||||
|
||||
def test_emit_rehydrated_calls_rebuild(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
@@ -398,42 +406,36 @@ class TestCoordinatorAdapterChildrenRegistry:
|
||||
adapter.emit_rehydrated(ws)
|
||||
assert calls == [ws.id]
|
||||
|
||||
def test_emit_closed_clears_forward_and_reverse_indexes(self) -> None:
|
||||
def test_emit_closed_uninstalls_parent_and_clears_children(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._merge_child_ids_locked("coord-b", ["child-b1"])
|
||||
adapter._active_coords["coord-a"] = object()
|
||||
adapter._active_coords["coord-b"] = object()
|
||||
adapter._registry.install("coord-a", object())
|
||||
adapter._registry.install("coord-b", object())
|
||||
adapter._registry.merge_children("coord-a", ["child-a1", "child-a2"])
|
||||
adapter._registry.merge_children("coord-b", ["child-b1"])
|
||||
|
||||
adapter.emit_closed("coord-a")
|
||||
|
||||
assert "coord-a" not in adapter._children
|
||||
assert "coord-a" not in adapter._active_coords
|
||||
assert "child-a1" not in adapter._child_to_coord
|
||||
assert "child-a2" not in adapter._child_to_coord
|
||||
assert adapter._registry.ui_for("coord-a") is None
|
||||
assert adapter._registry.children_of("coord-a") == []
|
||||
assert adapter._registry.parent_for("child-a1") is None
|
||||
assert adapter._registry.parent_for("child-a2") is None
|
||||
# coord-b untouched
|
||||
assert adapter._child_to_coord["child-b1"] == "coord-b"
|
||||
assert "coord-b" in adapter._children
|
||||
|
||||
def test_merge_child_ids_locked_is_idempotent(self) -> None:
|
||||
adapter, _ = _make_adapter()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-1"])
|
||||
assert adapter._children["coord-a"] == {"child-1"}
|
||||
assert adapter._child_to_coord == {"child-1": "coord-a"}
|
||||
assert adapter._registry.parent_for("child-b1") == "coord-b"
|
||||
assert adapter._registry.ui_for("coord-b") is not None
|
||||
|
||||
def test_prime_children_from_snapshot_merges_without_overwriting(self) -> None:
|
||||
# Snapshot priming now lives on ClusterChildSource (production
|
||||
# path). The adapter no longer carries its own duplicate copy.
|
||||
from turnstone.core.child_source import ClusterChildSource
|
||||
|
||||
adapter, _ = _make_adapter()
|
||||
# Seed one in-memory coord + one existing child
|
||||
coord_ws = _make_ws()
|
||||
coord_ws.id = "coord-a"
|
||||
mgr = MagicMock()
|
||||
mgr.list_all.return_value = [coord_ws]
|
||||
adapter.attach(mgr)
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
|
||||
source = ClusterChildSource(
|
||||
collector=MagicMock(),
|
||||
registry=adapter._registry,
|
||||
parents_provider=lambda: ["coord-a"],
|
||||
)
|
||||
|
||||
snapshot = {
|
||||
"nodes": [
|
||||
@@ -448,10 +450,13 @@ class TestCoordinatorAdapterChildrenRegistry:
|
||||
},
|
||||
],
|
||||
}
|
||||
adapter._prime_children_from_snapshot(snapshot)
|
||||
assert adapter._children["coord-a"] == {"child-a1", "child-a2"}
|
||||
assert adapter._child_to_coord["child-a2"] == "coord-a"
|
||||
assert "child-x" not in adapter._child_to_coord
|
||||
source._prime_from_snapshot(snapshot)
|
||||
assert set(adapter._registry.children_of("coord-a")) == {
|
||||
"child-a1",
|
||||
"child-a2",
|
||||
}
|
||||
assert adapter._registry.parent_for("child-a2") == "coord-a"
|
||||
assert adapter._registry.parent_for("child-x") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -478,9 +483,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
coord_ws.id = coord_id
|
||||
recorder = _UIRecorder()
|
||||
coord_ws.ui = recorder # type: ignore[assignment]
|
||||
with adapter._children_lock:
|
||||
adapter._children.setdefault(coord_id, set())
|
||||
adapter._active_coords[coord_id] = recorder
|
||||
adapter._registry.install(coord_id, recorder)
|
||||
adapter.attach(_StubManager(coord_ws)) # type: ignore[arg-type]
|
||||
return adapter, recorder, coord_ws
|
||||
|
||||
@@ -510,12 +513,11 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
# Reverse index updated for subsequent cluster_state events.
|
||||
assert adapter._child_to_coord["child-a1"] == "coord-a"
|
||||
assert adapter._registry.parent_for("child-a1") == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_routes_via_reverse_index(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -533,8 +535,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
|
||||
def test_dispatch_ws_closed_routes_to_parent_coord(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{"type": "ws_closed", "ws_id": "child-a1", "reason": "evicted"}
|
||||
)
|
||||
@@ -548,8 +549,7 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
"""perf-6: _enqueue_on_ui mutates the payload dict in place with
|
||||
the coord's ws_id so the browser can discriminate child events."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -559,53 +559,160 @@ class TestCoordinatorAdapterDispatchChildEvent:
|
||||
)
|
||||
assert recorder.enqueued[0]["ws_id"] == "coord-a"
|
||||
|
||||
def test_dispatch_cluster_state_forwards_pending_approval_detail(self) -> None:
|
||||
"""The rich approval payload now rides on child_ws_state directly so
|
||||
the browser can mutate liveBadgeCache without a separate live-bulk
|
||||
fetch. Drift here means the inline approve/deny buttons would
|
||||
regress to chasing the dashboard cache (the load-storm pattern
|
||||
Shape A is unwinding)."""
|
||||
def test_dispatch_cluster_state_does_not_carry_pending_approval_detail(
|
||||
self,
|
||||
) -> None:
|
||||
"""Stage 3 cleanup — the ``pending_approval_detail`` piggyback
|
||||
on ``cluster_state`` is gone. Approval items now arrive via
|
||||
bulk fetch (triggered by ``activity_state="approval"`` in the
|
||||
browser); verdicts via ``child_ws_intent_verdict``; resolution
|
||||
via ``child_ws_approval_resolved``. The state event carries
|
||||
only state + activity_state — no detail field."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
detail = {
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": False,
|
||||
}
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "approval",
|
||||
"pending_approval_detail": detail,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_state"
|
||||
assert payload["activity_state"] == "approval"
|
||||
assert payload["pending_approval_detail"] == detail
|
||||
assert "pending_approval_detail" not in payload
|
||||
|
||||
def test_dispatch_cluster_state_pending_approval_detail_none_passes_through(
|
||||
self,
|
||||
) -> None:
|
||||
"""Missing pending_approval_detail (no approval pending, or pre-fix
|
||||
node mid-rolling-upgrade) must forward as None — not raise, not
|
||||
omit — so the browser's handleChildState treats it as "no SSE-
|
||||
supplied detail, fall back to cached value"."""
|
||||
def test_dispatch_intent_verdict_emits_child_ws_intent_verdict(self) -> None:
|
||||
"""Stage 3 Step 6 — explicit verdict events are re-emitted as
|
||||
child_ws_intent_verdict on the parent's SSE so the tree UI
|
||||
renders the risk pill without polling."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
with adapter._children_lock:
|
||||
adapter._merge_child_ids_locked("coord-a", ["child-a1"])
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.92,
|
||||
"recommendation": "approve",
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
"type": "intent_verdict",
|
||||
"ws_id": "child-a1",
|
||||
"state": "running",
|
||||
"activity_state": "tool",
|
||||
"node_id": "node-1",
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert "pending_approval_detail" in payload
|
||||
assert payload["pending_approval_detail"] is None
|
||||
assert payload["type"] == "child_ws_intent_verdict"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["node_id"] == "node-1"
|
||||
assert payload["verdict"] == verdict
|
||||
|
||||
def test_dispatch_intent_verdict_unknown_child_drops(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": "ws-orphan",
|
||||
"verdict": {"call_id": "c1"},
|
||||
}
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_approval_resolved_emits_child_ws_approval_resolved(
|
||||
self,
|
||||
) -> None:
|
||||
"""Stage 3 Step 6 — paired with intent_verdict; clears the
|
||||
pending-approval pill on the parent's tree UI in lockstep
|
||||
with the actual decision."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"approved": True,
|
||||
"feedback": "lgtm",
|
||||
"always": False,
|
||||
}
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_approval_resolved"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["approved"] is True
|
||||
assert payload["feedback"] == "lgtm"
|
||||
assert payload["always"] is False
|
||||
|
||||
def test_dispatch_approval_resolved_coerces_missing_fields(self) -> None:
|
||||
"""Older nodes mid-rolling-upgrade may omit approved / always /
|
||||
feedback; dispatch coerces to safe defaults."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
adapter._dispatch_child_event({"type": "approval_resolved", "ws_id": "child-a1"})
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["approved"] is False
|
||||
assert payload["feedback"] == ""
|
||||
assert payload["always"] is False
|
||||
|
||||
def test_dispatch_approval_resolved_unknown_child_drops(self) -> None:
|
||||
"""Symmetric to the intent_verdict drop test — events for
|
||||
ws_ids the registry doesn't know about silently drop instead
|
||||
of fanning out to a parent that has no business seeing them."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": "ws-orphan",
|
||||
"approved": True,
|
||||
},
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
def test_dispatch_approve_request_emits_child_ws_approve_request(
|
||||
self,
|
||||
) -> None:
|
||||
"""Push path for the initial approval items — eliminates the
|
||||
bulk-fetch race that left the coord row stuck on a loading
|
||||
placeholder when the bulk fetch landed in the gap between
|
||||
_emit_state(ATTENTION) and approve_tools setting _pending_approval."""
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._registry.merge_children("coord-a", ["child-a1"])
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": "child-a1",
|
||||
"node_id": "node-1",
|
||||
"detail": detail,
|
||||
},
|
||||
)
|
||||
assert len(recorder.enqueued) == 1
|
||||
payload = recorder.enqueued[0]
|
||||
assert payload["type"] == "child_ws_approve_request"
|
||||
assert payload["child_ws_id"] == "child-a1"
|
||||
assert payload["parent_ws_id"] == "coord-a"
|
||||
assert payload["node_id"] == "node-1"
|
||||
assert payload["detail"] == detail
|
||||
|
||||
def test_dispatch_approve_request_unknown_child_drops(self) -> None:
|
||||
adapter, recorder, _ = self._setup()
|
||||
adapter._dispatch_child_event(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": "ws-orphan",
|
||||
"detail": {"items": []},
|
||||
},
|
||||
)
|
||||
assert recorder.enqueued == []
|
||||
|
||||
@@ -79,9 +79,10 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
assert "function submitChildApproval" in body or "submitChildApproval(" in body
|
||||
# The shared approve POST helper (parameterized for child ws_ids)
|
||||
assert "function approveWorkstream" in body or "approveWorkstream(" in body
|
||||
# The urgent live-bulk fetch option that fires on activity_state
|
||||
# transitions in/out of "approval"
|
||||
assert "{ urgent: true }" in body or "urgent: true" in body
|
||||
# The 409 stale-call_id retry path uses invalidateLiveBadge +
|
||||
# scheduleLiveFetch (Stage 3 cleanup removed the urgent flag —
|
||||
# cache invalidation makes the TTL gate fall through naturally).
|
||||
assert "invalidateLiveBadge(targetWsId)" in body
|
||||
# Server-side payload field — drift here means the JS reads stale keys
|
||||
assert "pending_approval_detail" in body
|
||||
# Reconnect parity (chunk 4): the SSE re-open handler must drop
|
||||
@@ -90,10 +91,10 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# can't render zombie approve/deny buttons on a row whose
|
||||
# approval was resolved during the gap. The implementation
|
||||
# iterates the cache and deletes only !permanent entries —
|
||||
# asserting the literal Map iteration form keeps a refactor
|
||||
# back to liveBadgeCache.clear() (which would re-pay 403s on
|
||||
# every reconnect for denied ids) from sneaking in.
|
||||
assert "liveBadgeCache.delete" in body
|
||||
# asserting the literal helper call keeps a refactor back to
|
||||
# _liveBadgeCacheClear() (which would re-pay 403s on every
|
||||
# reconnect for denied ids) from sneaking in.
|
||||
assert "_liveBadgeCacheDelete" in body
|
||||
# Edge-case matrix sentinel labels — POLICY-BLOCKED renders when
|
||||
# an item has error set + needs_approval=False (server-side
|
||||
# tool policy already blocked the call); "(judge unavailable)"
|
||||
@@ -113,15 +114,19 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
# coord-self ws_id (the coord lives on the console process).
|
||||
# Children live on cluster nodes and 404 without the prefix.
|
||||
assert "/v1/api/route/workstreams/" in body
|
||||
# Late-judge polling — the LLM judge runs async on the child
|
||||
# node and never pushes a signal that reaches the coord, so
|
||||
# the row's pending_approval_detail with judge_pending=true
|
||||
# would freeze on heuristic verdicts forever without this
|
||||
# poll loop. The poller is GLOBAL (not per-row) so off-screen
|
||||
# rows still refresh — a per-row poller's scheduleLiveFetch
|
||||
# call short-circuits on non-visible rows, leaving them stuck.
|
||||
assert "_maybeStartJudgePoll" in body
|
||||
assert "_judgePollTick" in body
|
||||
# Late-arriving LLM judge verdicts — Stage 3 Step 5 promoted
|
||||
# ``intent_verdict`` and ``approval_resolved`` to first-class
|
||||
# cluster-bus event types, so the coord adapter dispatches them
|
||||
# as ``child_ws_intent_verdict`` / ``child_ws_approval_resolved``
|
||||
# on the parent's SSE stream. The browser handlers write
|
||||
# directly to liveBadgeCache (bypassing scheduleLiveFetch's
|
||||
# visibility gate cleanly) so off-screen rows pick up verdicts
|
||||
# without polling. Replaced the old ``_judgePollTick`` 90-second
|
||||
# global poll loop and its visibility-gate-bypass workaround.
|
||||
assert "handleChildIntentVerdict" in body
|
||||
assert "handleChildApprovalResolved" in body
|
||||
assert "child_ws_intent_verdict" in body
|
||||
assert "child_ws_approval_resolved" in body
|
||||
# Reload parity for the coord-self approval gate: init() must
|
||||
# consume the authoritative GET /workstreams snapshot's
|
||||
# pending_approval_detail so a freshly opened tab can render
|
||||
@@ -154,18 +159,19 @@ def test_coordinator_js_exposes_inline_approval_helpers():
|
||||
assert "callOutcomes" in body
|
||||
|
||||
|
||||
def test_coordinator_js_handle_child_state_reads_sse_pending_approval_detail():
|
||||
"""Lock the Shape A behavior change: child_ws_state SSE events now
|
||||
carry ``pending_approval_detail`` directly so the browser mutates
|
||||
``liveBadgeCache`` without firing an urgent live-bulk fetch on
|
||||
every activity_state transition into/out of approval. A refactor
|
||||
that re-introduces the urgent-fetch path on routine transitions
|
||||
(or drops the SSE-source merge guard in flushLiveFetches) would
|
||||
re-open the load-storm pattern this PR is fixing.
|
||||
def test_coordinator_js_handle_child_state_no_longer_reads_sse_pending_approval_detail():
|
||||
"""Stage 3 cleanup — ``pending_approval_detail`` is no longer
|
||||
piggybacked on child_ws_state events. Approval items now arrive
|
||||
via bulk fetch on the activity_state="approval" transition;
|
||||
verdicts via the explicit ``child_ws_intent_verdict`` event class;
|
||||
resolution via ``child_ws_approval_resolved``. A refactor that
|
||||
re-introduces the piggyback would silently re-open the
|
||||
duplicate-path race the dedicated event classes were added to
|
||||
eliminate.
|
||||
|
||||
Structural assertions (regex against multi-line source) — symbol-
|
||||
presence alone wouldn't catch a guard that keeps the names but
|
||||
inverts the comparison or drops the ``prev.live`` check. This
|
||||
inverts the comparison or drops the ``prev.live`` check. This
|
||||
codebase has no JS test framework, so locking the guard's shape
|
||||
here is the next-best thing to a behavioral test."""
|
||||
import re
|
||||
@@ -176,26 +182,46 @@ def test_coordinator_js_handle_child_state_reads_sse_pending_approval_detail():
|
||||
)
|
||||
body = coord_js.read_text(encoding="utf-8")
|
||||
|
||||
# handleChildState now reads the SSE-supplied detail.
|
||||
assert "ev.pending_approval_detail" in body
|
||||
# The pre-fix urgent-fetch on activity_state transitions is
|
||||
# gone (the 409 retry path keeps its own ``{ urgent: true }``
|
||||
# for stale-call_id refresh — that's a different scenario).
|
||||
# The piggyback read is gone from handleChildState. (The string
|
||||
# may still appear elsewhere — e.g. handleChildIntentVerdict
|
||||
# reading from cache, or comments — but never as ``ev.pending_approval_detail``.)
|
||||
assert "ev.pending_approval_detail" not in body
|
||||
# The pre-fix urgent-fetch on activity_state transitions is gone.
|
||||
assert "enteredApproval" not in body
|
||||
assert "leftApproval" not in body
|
||||
# ``pendingApproval`` flag derivation must check BOTH state and
|
||||
# activity_state. The worker thread can fire the state transition
|
||||
# to "attention" before approve_tools updates activity_state, so
|
||||
# checking only activity_state misses children that legitimately
|
||||
# need approval. Pin the disjunction so the regression doesn't
|
||||
# silently re-introduce.
|
||||
assert re.search(
|
||||
r'existing\.state\s*===\s*"attention"\s*\|\|\s*'
|
||||
r'existing\.activity_state\s*===\s*"approval"',
|
||||
body,
|
||||
), (
|
||||
"handleChildState must derive pendingApproval from "
|
||||
"(state==='attention' || activity_state==='approval')"
|
||||
)
|
||||
|
||||
# SSE-authoritative window constant is defined and used.
|
||||
assert re.search(r"\bconst\s+SSE_AUTHORITATIVE_MS\s*=\s*\d+", body), (
|
||||
"SSE_AUTHORITATIVE_MS constant must be defined as a numeric literal"
|
||||
)
|
||||
|
||||
# handleChildState writes sseUpdatedAt = Date.now() into the cache
|
||||
# entry it sets. This is the SSE-source tag; without it, the
|
||||
# merge guard in flushLiveFetches has nothing to gate on.
|
||||
# SSE writers tag entries with sseUpdatedAt: Date.now() so the
|
||||
# merge guard in flushLiveFetches preserves them against stale
|
||||
# bulk-fetch responses. handleChildState only stamps when it
|
||||
# AUTHORITATIVELY clears the detail (off-approval transition);
|
||||
# writers that stamp unconditionally are intent_verdict (verdict
|
||||
# stamp), approval_resolved (clear), and the optimistic-clear
|
||||
# path in submitChildApproval. Pinning the literal Date.now()
|
||||
# call keeps a refactor that drops the SSE-source tag entirely
|
||||
# from sneaking in.
|
||||
assert re.search(
|
||||
r"sseUpdatedAt:\s*Date\.now\(\)",
|
||||
body,
|
||||
), "handleChildState must write sseUpdatedAt: Date.now() onto liveBadgeCache entries"
|
||||
), "Critical SSE writers must stamp sseUpdatedAt: Date.now()"
|
||||
|
||||
# flushLiveFetches' merge guard structure: SSE-set pending_approval
|
||||
# / _detail wins over a stale bulk-poll snapshot when (live) AND
|
||||
|
||||
@@ -18,13 +18,22 @@ from __future__ import annotations
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.session_manager import SessionKindAdapter, SessionManager
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
from turnstone.core.workstream import (
|
||||
BULK_CLOSE_STATE_VALUES,
|
||||
Workstream,
|
||||
WorkstreamKind,
|
||||
WorkstreamState,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test fixtures
|
||||
@@ -156,6 +165,8 @@ class _Row:
|
||||
kind: str
|
||||
state: str = "idle"
|
||||
parent_ws_id: str | None = None
|
||||
updated: str = ""
|
||||
node_id: str | None = None
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
@@ -164,8 +175,19 @@ class FakeStorage:
|
||||
def __init__(self) -> None:
|
||||
self.rows: dict[str, _Row] = {}
|
||||
self.state_updates: list[tuple[str, str]] = []
|
||||
self.touch_calls: list[str] = []
|
||||
self.register_raises = False
|
||||
self.lock = threading.Lock()
|
||||
# Live-services lookup target for close_idle pass 2. Map
|
||||
# service_type → list of live service_ids. Tests that exercise
|
||||
# liveness scoping populate this directly; default empty means
|
||||
# "no peers alive" (every row unprotected by liveness).
|
||||
self.live_services: dict[str, list[str]] = {}
|
||||
self.list_services_raises = False
|
||||
|
||||
@staticmethod
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
def register_workstream(
|
||||
self,
|
||||
@@ -178,6 +200,8 @@ class FakeStorage:
|
||||
parent_ws_id: str | None = None,
|
||||
skill_id: str = "",
|
||||
skill_version: int = 0,
|
||||
state: str = "idle",
|
||||
updated: str | None = None,
|
||||
) -> None:
|
||||
if self.register_raises:
|
||||
raise RuntimeError("register forced failure")
|
||||
@@ -188,14 +212,66 @@ class FakeStorage:
|
||||
user_id=user_id or "",
|
||||
name=name,
|
||||
kind=kind_str,
|
||||
state=state,
|
||||
parent_ws_id=parent_ws_id,
|
||||
updated=updated if updated is not None else self._now_iso(),
|
||||
node_id=node_id,
|
||||
)
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
with self.lock:
|
||||
self.touch_calls.append(ws_id)
|
||||
if ws_id in self.rows:
|
||||
self.rows[ws_id].updated = self._now_iso()
|
||||
|
||||
def update_workstream_state(self, ws_id: str, state: str) -> None:
|
||||
with self.lock:
|
||||
self.state_updates.append((ws_id, state))
|
||||
if ws_id in self.rows:
|
||||
self.rows[ws_id].state = state
|
||||
self.rows[ws_id].updated = self._now_iso()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
kind_str = kind.value if isinstance(kind, WorkstreamKind) else str(kind)
|
||||
excluded = set(exclude_ws_ids)
|
||||
live_set = set(live_node_ids) if live_node_ids else set()
|
||||
now = self._now_iso()
|
||||
closed: list[str] = []
|
||||
with self.lock:
|
||||
for ws_id, row in self.rows.items():
|
||||
if (
|
||||
row.kind == kind_str
|
||||
and row.state in BULK_CLOSE_STATE_VALUES
|
||||
and row.updated < cutoff
|
||||
and ws_id not in excluded
|
||||
):
|
||||
# Liveness gate: when live_node_ids was provided AND
|
||||
# non-empty, protect rows whose owner is in the live
|
||||
# set. NULL node_id is always eligible. When
|
||||
# live_node_ids is None or empty, no protection
|
||||
# (mirror of the real backends).
|
||||
if live_node_ids and row.node_id is not None and row.node_id in live_set:
|
||||
continue
|
||||
row.state = "closed"
|
||||
row.updated = now
|
||||
self.state_updates.append((ws_id, "closed"))
|
||||
closed.append(ws_id)
|
||||
return closed
|
||||
|
||||
def list_services(self, service_type: str, max_age_seconds: int = 120) -> list[dict[str, str]]:
|
||||
if self.list_services_raises:
|
||||
raise RuntimeError("list_services forced failure")
|
||||
with self.lock:
|
||||
return [
|
||||
{"service_id": sid, "service_type": service_type}
|
||||
for sid in self.live_services.get(service_type, [])
|
||||
]
|
||||
|
||||
def get_workstream(self, ws_id: str) -> dict[str, Any] | None:
|
||||
with self.lock:
|
||||
@@ -228,6 +304,7 @@ def _make_manager(
|
||||
max_active: int = 5,
|
||||
storage: FakeStorage | None = None,
|
||||
event_emitter: Any = _EMITTER_DEFAULT,
|
||||
node_id: str | None = None,
|
||||
) -> tuple[SessionManager, FakeAdapter, FakeStorage]:
|
||||
"""Build a SessionManager wired to a FakeAdapter for both Protocols.
|
||||
|
||||
@@ -246,6 +323,7 @@ def _make_manager(
|
||||
storage=storage,
|
||||
max_active=max_active,
|
||||
event_emitter=emitter,
|
||||
node_id=node_id,
|
||||
)
|
||||
return mgr, adapter, storage
|
||||
|
||||
@@ -579,6 +657,24 @@ def test_open_resurrects_closed_state() -> None:
|
||||
assert ws_id in [e.ws_id for e in adapter.events_of("rehydrated")]
|
||||
|
||||
|
||||
def test_open_touches_workstream_on_rehydrate() -> None:
|
||||
"""Rehydrating a workstream must bump its ``updated`` so a concurrent
|
||||
close_idle pass-2 in this same process can't clobber the freshly-loaded
|
||||
row to ``closed`` because its DB ``updated`` is older than the cutoff.
|
||||
The touch is best-effort (try/except in open()) but must fire on the
|
||||
happy path."""
|
||||
mgr, _, storage = _make_manager()
|
||||
ws = mgr.create(user_id="u1")
|
||||
ws_id = ws.id
|
||||
mgr.close(ws_id)
|
||||
storage.touch_calls.clear() # only care about touches from rehydrate
|
||||
|
||||
reopened = mgr.open(ws_id)
|
||||
|
||||
assert reopened is not None
|
||||
assert ws_id in storage.touch_calls
|
||||
|
||||
|
||||
def test_open_ignores_owner_mismatch() -> None:
|
||||
# Turnstone is a trusted-team tool; row-level ownership is
|
||||
# metadata, not an access boundary. ``open`` no longer cares
|
||||
@@ -827,6 +923,197 @@ def test_close_idle_on_empty_manager_returns_empty_list() -> None:
|
||||
assert mgr.close_idle(max_age_seconds=1.0) == []
|
||||
|
||||
|
||||
def test_close_idle_runs_db_orphan_pass() -> None:
|
||||
"""DB rows of this kind that aren't loaded into the manager get
|
||||
bulk-closed when their ``updated`` is older than the cutoff. Catches
|
||||
the orphan-after-process-restart case the original close_idle missed."""
|
||||
mgr, _, storage = _make_manager()
|
||||
# Orphan rows live in storage but were never loaded via mgr.create.
|
||||
storage.register_workstream(
|
||||
"orphan-1",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"orphan-2",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
state="thinking",
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert set(closed) == {"orphan-1", "orphan-2"}
|
||||
assert ("orphan-1", "closed") in storage.state_updates
|
||||
assert ("orphan-2", "closed") in storage.state_updates
|
||||
assert storage.rows["orphan-1"].state == "closed"
|
||||
assert storage.rows["orphan-2"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_excludes_loaded_workstreams_from_db_pass() -> None:
|
||||
"""A workstream loaded into memory must NOT be reaped by the DB
|
||||
orphan pass even when its storage ``updated`` is stale — the
|
||||
in-memory pass owns those. Verifies the exclude_ws_ids plumbing."""
|
||||
mgr, _, storage = _make_manager()
|
||||
ws = mgr.create(user_id="u1")
|
||||
# Force the storage row's ``updated`` to look stale. In practice
|
||||
# ``set_state`` would bump it, but we're simulating a long-running
|
||||
# active workstream whose updated drifted older than the cutoff.
|
||||
storage.rows[ws.id].updated = "2020-01-01T00:00:00"
|
||||
|
||||
# Huge timeout so the in-memory IDLE pass skips it (stays loaded).
|
||||
closed = mgr.close_idle(max_age_seconds=10_000.0)
|
||||
|
||||
assert ws.id not in closed
|
||||
assert mgr.get(ws.id) is not None
|
||||
assert storage.rows[ws.id].state == "idle"
|
||||
|
||||
|
||||
def test_close_idle_filters_db_orphans_by_kind() -> None:
|
||||
"""An interactive manager's close_idle must not touch coordinator
|
||||
rows in storage and vice versa. Without this filter, both managers
|
||||
would race to close each other's rows."""
|
||||
mgr, _, storage = _make_manager() # interactive by default
|
||||
storage.register_workstream(
|
||||
"coord-orphan",
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"interactive-orphan",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert "interactive-orphan" in closed
|
||||
assert "coord-orphan" not in closed
|
||||
assert storage.rows["coord-orphan"].state == "idle"
|
||||
assert storage.rows["interactive-orphan"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_protects_rows_owned_by_live_services() -> None:
|
||||
"""Multi-node correctness: rows whose ``node_id`` matches a service
|
||||
with a recent heartbeat must NOT be reaped, even when *this* manager
|
||||
is on a different node — the alive peer may legitimately have them
|
||||
loaded. Liveness is the rendezvous router's primitive (post-PR-#384);
|
||||
using it here keeps reap scoping aligned with routing.
|
||||
|
||||
Default ``_make_manager`` uses an INTERACTIVE adapter, which derives
|
||||
``service_type='server'`` — so live_services seeded under "server"
|
||||
are what the manager queries."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.live_services["server"] = ["node-b"] # only node-b is alive
|
||||
storage.register_workstream(
|
||||
"ours-from-dead-node",
|
||||
node_id="node-a", # dead pod (not in live_services)
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"theirs-still-alive",
|
||||
node_id="node-b",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["ours-from-dead-node"]
|
||||
assert storage.rows["ours-from-dead-node"].state == "closed"
|
||||
assert storage.rows["theirs-still-alive"].state == "idle"
|
||||
|
||||
|
||||
def test_close_idle_protects_live_services_for_coordinator_kind() -> None:
|
||||
"""Coord-side parity: a coordinator manager derives
|
||||
``service_type='console'``, so live_services seeded under "console"
|
||||
are what gets queried. Mirrors the interactive test to ensure both
|
||||
halves of the production wiring are exercised."""
|
||||
coord_adapter = FakeAdapter(kind=WorkstreamKind.COORDINATOR)
|
||||
mgr, _, storage = _make_manager(coord_adapter)
|
||||
storage.live_services["console"] = ["console"] # console is alive
|
||||
storage.register_workstream(
|
||||
"alive-console-coord",
|
||||
node_id="console",
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"dead-console-coord",
|
||||
node_id="dead-console-instance", # not in live set
|
||||
kind=WorkstreamKind.COORDINATOR,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["dead-console-coord"]
|
||||
assert storage.rows["alive-console-coord"].state == "idle"
|
||||
assert storage.rows["dead-console-coord"].state == "closed"
|
||||
|
||||
|
||||
def test_close_idle_reaps_rows_with_null_node_id() -> None:
|
||||
"""A row with no ``node_id`` has no owner identity — age alone gates
|
||||
the reap. Defends against a NULL silently propagating through ``NOT
|
||||
IN (live)`` and protecting orphans forever."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.live_services["server"] = ["node-a"]
|
||||
storage.register_workstream(
|
||||
"no-owner",
|
||||
node_id=None,
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == ["no-owner"]
|
||||
|
||||
|
||||
def test_close_idle_reaps_all_orphans_when_no_peers_alive() -> None:
|
||||
"""When ``list_services`` returns an empty list (no heartbeating
|
||||
peers), every stale orphan is unprotected and gets reaped. This is
|
||||
the cold-start / single-process / dead-cluster-recovery case."""
|
||||
mgr, _, storage = _make_manager()
|
||||
# storage.live_services["server"] left empty — no peers heartbeating
|
||||
storage.register_workstream(
|
||||
"any-node-1",
|
||||
node_id="node-a",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
storage.register_workstream(
|
||||
"any-node-2",
|
||||
node_id="node-b",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert set(closed) == {"any-node-1", "any-node-2"}
|
||||
|
||||
|
||||
def test_close_idle_skips_pass_2_when_list_services_fails() -> None:
|
||||
"""Conservative fallback: if list_services fails we can't enumerate
|
||||
live owners safely, so pass 2 must skip rather than reap blind. Pass
|
||||
1 (in-memory IDLE) still runs."""
|
||||
mgr, _, storage = _make_manager()
|
||||
storage.list_services_raises = True
|
||||
storage.register_workstream(
|
||||
"would-be-orphan",
|
||||
node_id="node-a",
|
||||
kind=WorkstreamKind.INTERACTIVE,
|
||||
updated="2020-01-01T00:00:00",
|
||||
)
|
||||
|
||||
closed = mgr.close_idle(max_age_seconds=0.0)
|
||||
|
||||
assert closed == []
|
||||
assert storage.rows["would-be-orphan"].state == "idle"
|
||||
|
||||
|
||||
def test_list_all_returns_creation_order() -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
a = mgr.create(user_id="u1")
|
||||
@@ -1084,3 +1371,127 @@ class TestSessionManagerWithStateWriter:
|
||||
assert "running" not in ws_writes, (
|
||||
f"set_state after close enqueued through buffer: {ws_writes}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-subscriber observer — subscribe_to_state / unsubscribe_from_state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStateSubscribers:
|
||||
"""Multi-subscriber observer for ``set_state``.
|
||||
|
||||
Used by the CLI's background-attention notifier and by
|
||||
``SameNodeChildSource``. Subscribe / unsubscribe must be safe under
|
||||
concurrent dispatch, and dispatch must not skip / repeat callbacks
|
||||
when subscribers register or unregister mid-iteration.
|
||||
"""
|
||||
|
||||
def test_subscribe_fires_on_set_state(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
events: list[tuple[str, str]] = []
|
||||
|
||||
def cb(ws_id: str, state: WorkstreamState) -> None:
|
||||
events.append((ws_id, state.value))
|
||||
|
||||
mgr.subscribe_to_state(cb)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert events == [(ws.id, "running")]
|
||||
|
||||
def test_unsubscribe_stops_firing(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
events: list[str] = []
|
||||
|
||||
def cb(_ws_id: str, state: WorkstreamState) -> None:
|
||||
events.append(state.value)
|
||||
|
||||
mgr.subscribe_to_state(cb)
|
||||
mgr.unsubscribe_from_state(cb)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert events == []
|
||||
|
||||
def test_unsubscribe_unknown_is_noop(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
# Doesn't raise.
|
||||
mgr.unsubscribe_from_state(lambda *_: None)
|
||||
|
||||
def test_multiple_subscribers_fire_in_registration_order(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
order: list[int] = []
|
||||
|
||||
def make(i: int) -> Callable[[str, WorkstreamState], None]:
|
||||
def cb(_ws_id: str, _state: WorkstreamState) -> None:
|
||||
order.append(i)
|
||||
|
||||
return cb
|
||||
|
||||
mgr.subscribe_to_state(make(1))
|
||||
mgr.subscribe_to_state(make(2))
|
||||
mgr.subscribe_to_state(make(3))
|
||||
mgr.set_state(ws.id, WorkstreamState.IDLE)
|
||||
assert order == [1, 2, 3]
|
||||
|
||||
def test_subscriber_exception_does_not_block_others(self) -> None:
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
survived: list[str] = []
|
||||
|
||||
def boom(*_: Any) -> None:
|
||||
raise RuntimeError("subscriber crash")
|
||||
|
||||
def good(_ws_id: str, state: WorkstreamState) -> None:
|
||||
survived.append(state.value)
|
||||
|
||||
mgr.subscribe_to_state(boom)
|
||||
mgr.subscribe_to_state(good)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
assert survived == ["running"]
|
||||
|
||||
@pytest.mark.parametrize("n_threads", [10, 50])
|
||||
def test_concurrent_subscribe(self, n_threads: int) -> None:
|
||||
"""Subscribe from many threads; all callbacks land in the list.
|
||||
|
||||
Validates the lock around mutation — without it the underlying
|
||||
list.append could lose entries under contention.
|
||||
"""
|
||||
mgr, _, _ = _make_manager()
|
||||
callbacks = [lambda *_, i=i: None for i in range(n_threads)]
|
||||
threads = [threading.Thread(target=mgr.subscribe_to_state, args=(cb,)) for cb in callbacks]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
# Snapshot under the lock to read the count safely.
|
||||
with mgr._state_subscribers_lock:
|
||||
assert len(mgr._state_subscribers) == n_threads
|
||||
|
||||
def test_subscribe_during_dispatch_does_not_corrupt_iteration(self) -> None:
|
||||
"""A subscriber that calls subscribe_to_state during its own
|
||||
callback must not affect the in-flight dispatch (snapshot
|
||||
isolation). This is the bug-1 invariant: mutation during
|
||||
iteration can't shift the iterator's index because dispatch
|
||||
iterates a snapshot, not the live list.
|
||||
"""
|
||||
mgr, _, _ = _make_manager()
|
||||
ws = mgr.create(user_id="u1", name="ws", skill=None)
|
||||
fired: list[str] = []
|
||||
|
||||
def late(_ws_id: str, state: WorkstreamState) -> None:
|
||||
fired.append("late:" + state.value)
|
||||
|
||||
def first(_ws_id: str, state: WorkstreamState) -> None:
|
||||
fired.append("first:" + state.value)
|
||||
mgr.subscribe_to_state(late) # mid-dispatch addition
|
||||
|
||||
mgr.subscribe_to_state(first)
|
||||
mgr.set_state(ws.id, WorkstreamState.RUNNING)
|
||||
# ``late`` was added during dispatch but the snapshot was
|
||||
# already frozen — so it doesn't fire on this round.
|
||||
assert fired == ["first:running"]
|
||||
# Next round it does fire, in registration order after first.
|
||||
fired.clear()
|
||||
mgr.set_state(ws.id, WorkstreamState.IDLE)
|
||||
assert fired == ["first:idle", "late:idle"]
|
||||
|
||||
@@ -4,6 +4,10 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import workstreams
|
||||
|
||||
# -- Workstream registration ---------------------------------------------------
|
||||
|
||||
|
||||
@@ -664,6 +668,277 @@ class TestBatchPrimitives:
|
||||
assert result == {"never-seen": 0}
|
||||
|
||||
|
||||
# -- bulk_close_stale_orphans --------------------------------------------------
|
||||
|
||||
|
||||
def _force_updated(backend: Any, ws_id: str, updated: str) -> None:
|
||||
"""Stamp a workstream row's ``updated`` column directly.
|
||||
|
||||
The public surface only sets ``updated`` to ``now``, which makes it
|
||||
impossible to fabricate a stale row through register/update calls.
|
||||
Reaches into ``backend._engine`` — same access pattern conftest uses
|
||||
for cross-backend cleanup.
|
||||
"""
|
||||
with backend._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=updated)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
class TestBulkCloseStaleOrphans:
|
||||
def test_closes_stale_non_terminal_rows_of_kind(self, backend):
|
||||
backend.register_workstream("stale-idle", kind="interactive")
|
||||
backend.register_workstream("stale-thinking", kind="interactive")
|
||||
backend.update_workstream_state("stale-thinking", "thinking")
|
||||
backend.register_workstream("fresh-idle", kind="interactive")
|
||||
_force_updated(backend, "stale-idle", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "stale-thinking", "2020-01-01T00:00:00")
|
||||
# fresh-idle stays at registration time (effectively now)
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"stale-idle", "stale-thinking"}
|
||||
rows = backend.get_workstreams_batch(["stale-idle", "stale-thinking", "fresh-idle"])
|
||||
assert rows["stale-idle"]["state"] == "closed"
|
||||
assert rows["stale-thinking"]["state"] == "closed"
|
||||
assert rows["fresh-idle"]["state"] == "idle"
|
||||
|
||||
def test_skips_already_closed(self, backend):
|
||||
backend.register_workstream("already-closed", kind="interactive")
|
||||
backend.update_workstream_state("already-closed", "closed")
|
||||
_force_updated(backend, "already-closed", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == []
|
||||
|
||||
def test_filters_by_kind(self, backend):
|
||||
backend.register_workstream("interactive-stale", kind="interactive")
|
||||
backend.register_workstream("coord-stale", kind="coordinator")
|
||||
_force_updated(backend, "interactive-stale", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "coord-stale", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == ["interactive-stale"]
|
||||
rows = backend.get_workstreams_batch(["interactive-stale", "coord-stale"])
|
||||
assert rows["interactive-stale"]["state"] == "closed"
|
||||
assert rows["coord-stale"]["state"] == "idle"
|
||||
|
||||
def test_excludes_loaded_ws_ids(self, backend):
|
||||
backend.register_workstream("ws-keep", kind="interactive")
|
||||
backend.register_workstream("ws-close", kind="interactive")
|
||||
_force_updated(backend, "ws-keep", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "ws-close", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=["ws-keep"]
|
||||
)
|
||||
|
||||
assert closed == ["ws-close"]
|
||||
rows = backend.get_workstreams_batch(["ws-keep", "ws-close"])
|
||||
assert rows["ws-keep"]["state"] == "idle"
|
||||
assert rows["ws-close"]["state"] == "closed"
|
||||
|
||||
def test_empty_exclude_list_does_not_break_sql(self, backend):
|
||||
backend.register_workstream("orphan", kind="interactive")
|
||||
_force_updated(backend, "orphan", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == ["orphan"]
|
||||
|
||||
def test_no_orphans_returns_empty(self, backend):
|
||||
backend.register_workstream("fresh", kind="interactive")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert closed == []
|
||||
|
||||
def test_closes_all_non_terminal_states(self, backend):
|
||||
for ws_id, state in [
|
||||
("o-idle", "idle"),
|
||||
("o-thinking", "thinking"),
|
||||
("o-attention", "attention"),
|
||||
("o-running", "running"),
|
||||
]:
|
||||
backend.register_workstream(ws_id, kind="interactive")
|
||||
if state != "idle":
|
||||
backend.update_workstream_state(ws_id, state)
|
||||
_force_updated(backend, ws_id, "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"o-idle", "o-thinking", "o-attention", "o-running"}
|
||||
|
||||
def test_bumps_updated_on_close(self, backend):
|
||||
stale_updated = "2020-01-01T00:00:00"
|
||||
backend.register_workstream("orphan", kind="interactive")
|
||||
_force_updated(backend, "orphan", stale_updated)
|
||||
|
||||
backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
# ``updated`` must change away from the forced stale value. Asserting
|
||||
# inequality from the seed (rather than ``> "2024-01-01..."``) keeps
|
||||
# the test independent of wall-clock date.
|
||||
with backend._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(workstreams.c.updated).where(workstreams.c.ws_id == "orphan")
|
||||
).one()
|
||||
assert row[0] != stale_updated
|
||||
|
||||
def test_protects_rows_owned_by_live_services(self, backend):
|
||||
"""Liveness scoping (post-#384 rendezvous-routing world): rows
|
||||
whose ``node_id`` matches a heartbeating service must NOT be
|
||||
reaped, because that owner may legitimately have them loaded on
|
||||
another worker. Rows whose ``node_id`` matches a dead service
|
||||
ARE eligible — that's how dead-pod orphans get reclaimed in
|
||||
containerized deployments with dynamic hostnames."""
|
||||
backend.register_workstream("dead-node", node_id="dead-pod-x4k2", kind="interactive")
|
||||
backend.register_workstream("alive-node", node_id="alive-pod-y9p3", kind="interactive")
|
||||
_force_updated(backend, "dead-node", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "alive-node", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=["alive-pod-y9p3"],
|
||||
)
|
||||
|
||||
assert closed == ["dead-node"]
|
||||
rows = backend.get_workstreams_batch(["dead-node", "alive-node"])
|
||||
assert rows["dead-node"]["state"] == "closed"
|
||||
assert rows["alive-node"]["state"] == "idle"
|
||||
|
||||
def test_null_node_id_always_eligible(self, backend):
|
||||
"""A row with NULL ``node_id`` has no owner identity — age alone
|
||||
gates the reap. Belt-and-suspenders against ``NULL NOT IN (...)``
|
||||
evaluating to NULL (not TRUE) and silently protecting orphans
|
||||
forever."""
|
||||
backend.register_workstream("no-owner", node_id=None, kind="interactive")
|
||||
_force_updated(backend, "no-owner", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=["some-other-node"],
|
||||
)
|
||||
|
||||
assert closed == ["no-owner"]
|
||||
|
||||
def test_live_node_ids_none_skips_filter(self, backend):
|
||||
"""``live_node_ids=None`` is the single-process / operator-backfill
|
||||
mode — all rows of *kind* are eligible regardless of node_id."""
|
||||
backend.register_workstream("node-a", node_id="node-a", kind="interactive")
|
||||
backend.register_workstream("node-b", node_id="node-b", kind="interactive")
|
||||
_force_updated(backend, "node-a", "2020-01-01T00:00:00")
|
||||
_force_updated(backend, "node-b", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive", cutoff="2024-01-01T00:00:00", exclude_ws_ids=[]
|
||||
)
|
||||
|
||||
assert set(closed) == {"node-a", "node-b"}
|
||||
|
||||
def test_empty_live_node_ids_treats_all_as_dead(self, backend):
|
||||
"""Empty list ``live_node_ids=[]`` means "no nodes alive" — every
|
||||
row's owner is unprotected. Useful for operator scripts that
|
||||
want to reap regardless of liveness."""
|
||||
backend.register_workstream("any", node_id="node-a", kind="interactive")
|
||||
_force_updated(backend, "any", "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=[],
|
||||
live_node_ids=[],
|
||||
)
|
||||
|
||||
assert closed == ["any"]
|
||||
|
||||
def test_combines_live_node_ids_and_exclude_ws_ids(self, backend):
|
||||
"""Both filters stack as AND clauses on the UPDATE. Covers the
|
||||
full 2x2 matrix to catch a future edit that replaces an AND with
|
||||
an OR or drops one of the filters: only the (orphan + dead-node)
|
||||
cell should be reaped."""
|
||||
# All four registered with the same stale ``updated``.
|
||||
for ws_id, node in [
|
||||
("loaded-alive", "alive-node"),
|
||||
("loaded-dead", "dead-node"),
|
||||
("orphan-alive", "alive-node"),
|
||||
("orphan-dead", "dead-node"),
|
||||
]:
|
||||
backend.register_workstream(ws_id, node_id=node, kind="interactive")
|
||||
_force_updated(backend, ws_id, "2020-01-01T00:00:00")
|
||||
|
||||
closed = backend.bulk_close_stale_orphans(
|
||||
"interactive",
|
||||
cutoff="2024-01-01T00:00:00",
|
||||
exclude_ws_ids=["loaded-alive", "loaded-dead"],
|
||||
live_node_ids=["alive-node"],
|
||||
)
|
||||
|
||||
# Only orphan-dead is unprotected by both filters.
|
||||
assert closed == ["orphan-dead"]
|
||||
rows = backend.get_workstreams_batch(
|
||||
["loaded-alive", "loaded-dead", "orphan-alive", "orphan-dead"]
|
||||
)
|
||||
assert rows["loaded-alive"]["state"] == "idle"
|
||||
assert rows["loaded-dead"]["state"] == "idle"
|
||||
assert rows["orphan-alive"]["state"] == "idle"
|
||||
assert rows["orphan-dead"]["state"] == "closed"
|
||||
|
||||
|
||||
# -- touch_workstream ----------------------------------------------------------
|
||||
|
||||
|
||||
class TestTouchWorkstream:
|
||||
def test_bumps_updated_only(self, backend):
|
||||
"""Used by ``open()`` on rehydrate to defend against the orphan
|
||||
reaper clobbering a freshly-loaded row. Must not change ``state``
|
||||
(the open() path explicitly avoids state writes to dodge a race
|
||||
with concurrent close())."""
|
||||
stale_updated = "2020-01-01T00:00:00"
|
||||
backend.register_workstream("ws-touch", kind="interactive")
|
||||
backend.update_workstream_state("ws-touch", "closed") # simulate prior close
|
||||
_force_updated(backend, "ws-touch", stale_updated)
|
||||
|
||||
backend.touch_workstream("ws-touch")
|
||||
|
||||
with backend._engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
sa.select(workstreams.c.state, workstreams.c.updated).where(
|
||||
workstreams.c.ws_id == "ws-touch"
|
||||
)
|
||||
).one()
|
||||
assert row[0] == "closed", "state must not be modified by touch"
|
||||
# Compare against the forced stale value rather than a fixed calendar
|
||||
# date so the test is independent of wall-clock time.
|
||||
assert row[1] != stale_updated, "updated must be bumped"
|
||||
|
||||
def test_unknown_id_is_noop(self, backend):
|
||||
"""Touch on a missing id must not raise — open()'s exception
|
||||
handler is best-effort."""
|
||||
backend.touch_workstream("nonexistent") # must not raise
|
||||
|
||||
|
||||
# -- Lifecycle -----------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
+129
-37
@@ -157,20 +157,17 @@ class TestContentAccumulation:
|
||||
assert len(idle_events[0]["content"]) <= _MAX_TURN_CONTENT_CHARS + 1024
|
||||
|
||||
|
||||
class TestPendingApprovalDetailGate:
|
||||
"""The Shape A SSE plumbing carries ``pending_approval_detail`` on the
|
||||
``ws_state`` event so the coord tree UI can render inline approve/deny
|
||||
buttons in lockstep with the activity_state transition. The gate
|
||||
(``if self._pending_approval is not None``) keeps the per-broadcast
|
||||
serializer cost off the common no-approval-pending path — these tests
|
||||
lock both branches down."""
|
||||
class TestPendingApprovalDetailNotPiggybacked:
|
||||
"""Stage 3 cleanup — ``pending_approval_detail`` is no longer
|
||||
piggybacked on ``ws_state`` events. Approval items now arrive via
|
||||
bulk fetch when the coord tree's reducer sees the
|
||||
``activity_state="approval"`` transition; verdicts via the explicit
|
||||
``intent_verdict`` event class; resolution via
|
||||
``approval_resolved``. These tests lock the no-piggyback contract
|
||||
down so a future regression doesn't silently re-introduce the
|
||||
duplicated path."""
|
||||
|
||||
def test_state_broadcast_omits_field_when_no_approval_pending(self):
|
||||
"""Common case: no approval pending → field absent from event so the
|
||||
per-broadcast verdict-cache deepcopy in
|
||||
``serialize_pending_approval_detail`` never runs. A regression
|
||||
that drops the gate would silently 10x the cost of every state
|
||||
broadcast in the steady state."""
|
||||
ui = _make_ui()
|
||||
assert ui._pending_approval is None
|
||||
ui._broadcast_state("running")
|
||||
@@ -180,17 +177,12 @@ class TestPendingApprovalDetailGate:
|
||||
assert len(running_events) == 1
|
||||
assert "pending_approval_detail" not in running_events[0]
|
||||
|
||||
def test_state_broadcast_includes_field_when_approval_pending(self):
|
||||
"""When an approval is pending the broadcast must carry the rich
|
||||
payload — the coord tree UI reads it directly to render inline
|
||||
approve/deny buttons. Without this, a coord browser would have
|
||||
to chase a separate ``cluster/ws/live`` fetch on every
|
||||
activity_state transition (the load-storm pattern Shape A is
|
||||
unwinding)."""
|
||||
def test_state_broadcast_omits_field_even_when_approval_pending(self):
|
||||
"""The piggyback is gone: even when ``_pending_approval`` is set,
|
||||
the state broadcast must NOT carry ``pending_approval_detail``.
|
||||
The browser triggers a bulk fetch off the
|
||||
``activity_state="approval"`` transition to get the items."""
|
||||
ui = _make_ui()
|
||||
# Mirror the shape ``pause_for_approval`` writes (session_ui_base
|
||||
# lines 576-580) — items with call_id + header is the minimum
|
||||
# the serializer needs to project.
|
||||
ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
@@ -209,21 +201,9 @@ class TestPendingApprovalDetailGate:
|
||||
events = _drain_global()
|
||||
attn = [e for e in events if e.get("state") == "attention"]
|
||||
assert len(attn) == 1
|
||||
# Field present and structurally sound — the serializer's
|
||||
# full shape is covered by tests/test_session_ui_base.py;
|
||||
# here we only need to confirm the gate fires and the
|
||||
# serializer's output is what lands on the event.
|
||||
assert "pending_approval_detail" in attn[0]
|
||||
detail = attn[0]["pending_approval_detail"]
|
||||
assert detail is not None
|
||||
assert detail.get("items")
|
||||
assert detail["items"][0]["call_id"] == "c1"
|
||||
assert "pending_approval_detail" not in attn[0]
|
||||
|
||||
def test_field_cleared_after_approval_resolves(self):
|
||||
"""Once ``_pending_approval`` is cleared, subsequent state
|
||||
broadcasts must drop the field again — without this, the
|
||||
browser would render stale approve/deny buttons until the
|
||||
next bulk-poll TTL window expired."""
|
||||
def test_field_stays_absent_after_approval_resolves(self):
|
||||
ui = _make_ui()
|
||||
ui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
@@ -231,7 +211,7 @@ class TestPendingApprovalDetailGate:
|
||||
"judge_pending": False,
|
||||
}
|
||||
ui._broadcast_state("attention")
|
||||
_drain_global() # discard the with-detail event
|
||||
_drain_global()
|
||||
|
||||
ui._pending_approval = None
|
||||
ui._broadcast_state("running")
|
||||
@@ -239,3 +219,115 @@ class TestPendingApprovalDetailGate:
|
||||
running = [e for e in events if e.get("state") == "running"]
|
||||
assert len(running) == 1
|
||||
assert "pending_approval_detail" not in running[0]
|
||||
|
||||
|
||||
class TestBroadcastIntentVerdict:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_intent_verdict``.
|
||||
|
||||
The collector-side test (``test_apply_delta_intent_verdict_*`` in
|
||||
test_console.py) covers consumption; this pins the event shape the
|
||||
producer puts on the global queue. A field rename or missed key
|
||||
here would slip past the consumer test because the consumer reads
|
||||
via ``data.get(...)``.
|
||||
"""
|
||||
|
||||
def test_pushes_intent_verdict_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
verdict = {
|
||||
"call_id": "c1",
|
||||
"risk_level": "low",
|
||||
"confidence": 0.92,
|
||||
"recommendation": "approve",
|
||||
"reasoning": "tool reads only",
|
||||
}
|
||||
ui._broadcast_intent_verdict(verdict)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "intent_verdict"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["verdict"] == verdict
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
def test_queue_full_swallowed(self):
|
||||
# Force a tiny queue then fill it so the next put_nowait
|
||||
# raises queue.Full — the broadcast must absorb it without
|
||||
# propagating (matches _broadcast_state's queue.Full handling).
|
||||
WebUI._global_queue = queue.Queue(maxsize=1)
|
||||
WebUI._global_queue.put_nowait({"sentinel": True})
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_intent_verdict({"call_id": "c1"})
|
||||
|
||||
|
||||
class TestBroadcastApprovalResolved:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_approval_resolved``."""
|
||||
|
||||
def test_pushes_approval_resolved_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(True, "lgtm", always=False)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "approval_resolved"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["approved"] is True
|
||||
assert ev["feedback"] == "lgtm"
|
||||
assert ev["always"] is False
|
||||
|
||||
def test_normalises_none_feedback_to_empty_string(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(False, None)
|
||||
|
||||
events = _drain_global()
|
||||
assert events[0]["feedback"] == ""
|
||||
assert events[0]["approved"] is False
|
||||
assert events[0]["always"] is False
|
||||
|
||||
def test_always_kwarg_propagates(self):
|
||||
ui = _make_ui()
|
||||
ui._broadcast_approval_resolved(True, "ok", always=True)
|
||||
events = _drain_global()
|
||||
assert events[0]["always"] is True
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approval_resolved(True, None)
|
||||
|
||||
|
||||
class TestBroadcastApproveRequest:
|
||||
"""Producer-side coverage for ``WebUI._broadcast_approve_request`` —
|
||||
push path for the initial approval items so a coord parent's tree
|
||||
UI can render the inline approve/deny block immediately without
|
||||
waiting for a bulk-fetch round-trip."""
|
||||
|
||||
def test_pushes_approve_request_event_to_global_queue(self):
|
||||
ui = _make_ui()
|
||||
detail = {
|
||||
"type": "approve_request",
|
||||
"items": [{"call_id": "c1", "header": "tool x"}],
|
||||
"judge_pending": True,
|
||||
}
|
||||
ui._broadcast_approve_request(detail)
|
||||
|
||||
events = _drain_global()
|
||||
assert len(events) == 1
|
||||
ev = events[0]
|
||||
assert ev["type"] == "approve_request"
|
||||
assert ev["ws_id"] == "ws-test"
|
||||
assert ev["detail"] == detail
|
||||
|
||||
def test_no_op_when_global_queue_unset(self):
|
||||
WebUI._global_queue = None
|
||||
ui = _make_ui()
|
||||
# Doesn't raise.
|
||||
ui._broadcast_approve_request({"items": []})
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "1.5.2"
|
||||
__version__ = "1.5.4"
|
||||
|
||||
+1
-1
@@ -1242,7 +1242,7 @@ def main() -> None:
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
manager._on_state_change = _bg_attention_notify
|
||||
manager.subscribe_to_state(_bg_attention_notify)
|
||||
|
||||
# Print banner
|
||||
print(f"\n{bold('Chat')} with {cyan(model)}")
|
||||
|
||||
+121
-14
@@ -498,7 +498,6 @@ class ClusterCollector:
|
||||
"kind": WorkstreamKind.from_raw(new_w.get("kind")),
|
||||
"parent_ws_id": new_w.get("parent_ws_id"),
|
||||
"activity_state": new_w.get("activity_state", ""),
|
||||
"pending_approval_detail": new_w.get("pending_approval_detail"),
|
||||
}
|
||||
)
|
||||
old_name = old_ws.get("title", "") or old_ws.get("name", "")
|
||||
@@ -558,18 +557,6 @@ class ClusterCollector:
|
||||
ws["kind"] = data["kind"]
|
||||
if "parent_ws_id" in data:
|
||||
ws["parent_ws_id"] = data["parent_ws_id"]
|
||||
# ``pending_approval_detail`` overwrites (no
|
||||
# ``ws.get`` fallback): the node's broadcast gate
|
||||
# on ``_pending_approval is not None`` means the
|
||||
# field is absent from ``data`` exactly when no
|
||||
# approval is pending — falling back to the cached
|
||||
# value would resurrect a stale detail after the
|
||||
# approval resolved. Without this assignment the
|
||||
# cached ``node.workstreams`` dict served by
|
||||
# ``get_node_detail`` / ``get_snapshot`` between
|
||||
# reconciliations would render stale approve/deny
|
||||
# buttons on closed approvals.
|
||||
ws["pending_approval_detail"] = data.get("pending_approval_detail")
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "cluster_state",
|
||||
@@ -581,7 +568,6 @@ class ClusterCollector:
|
||||
"kind": WorkstreamKind.from_raw(ws.get("kind")),
|
||||
"parent_ws_id": ws.get("parent_ws_id"),
|
||||
"activity_state": ws.get("activity_state", ""),
|
||||
"pending_approval_detail": data.get("pending_approval_detail"),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -648,6 +634,60 @@ class ClusterCollector:
|
||||
ws["name"] = name
|
||||
pending_events.append({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
elif etype == "intent_verdict":
|
||||
# Pure pass-through for LLM judge verdicts. The node's
|
||||
# WebUI._broadcast_intent_verdict pushes onto the
|
||||
# global queue (Stage 3 Step 5); the cluster fan-out
|
||||
# forwards to subscribers (e.g. CoordinatorAdapter,
|
||||
# which dispatches to the parent's coord SSE as
|
||||
# ``child_ws_intent_verdict``).
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"verdict": data.get("verdict") or {},
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "approval_resolved":
|
||||
# Pure pass-through for approve/deny resolutions. Pairs
|
||||
# with ``intent_verdict`` above so the parent
|
||||
# coordinator's tree UI can clear the pending-approval
|
||||
# pill the moment the user decides, rather than waiting
|
||||
# for the subsequent state-change piggyback.
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"approved": bool(data.get("approved", False)),
|
||||
"feedback": data.get("feedback", "") or "",
|
||||
"always": bool(data.get("always", False)),
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "approve_request":
|
||||
# Push path for the initial approval items. Eliminates
|
||||
# the bulk-fetch race that otherwise leaves the coord
|
||||
# tree stuck on a loading placeholder when the bulk
|
||||
# fetch lands in the gap between the state transition
|
||||
# to ATTENTION and ``_pending_approval`` being set.
|
||||
ws_id = data.get("ws_id", "")
|
||||
if ws_id:
|
||||
pending_events.append(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": ws_id,
|
||||
"node_id": node_id,
|
||||
"detail": data.get("detail") or {},
|
||||
}
|
||||
)
|
||||
|
||||
elif etype == "health_changed":
|
||||
# Update the health dict's backend status in-place
|
||||
bstatus = data.get("backend_status", "")
|
||||
@@ -1217,3 +1257,70 @@ class ClusterCollector:
|
||||
return
|
||||
entry["name"] = name
|
||||
self._fanout({"type": "ws_rename", "ws_id": ws_id, "name": name})
|
||||
|
||||
def emit_console_ws_intent_verdict(self, ws_id: str, verdict: dict[str, Any]) -> None:
|
||||
"""Fan an LLM intent-judge verdict for a console-pseudo-node ws.
|
||||
|
||||
Gives coord-spawned approval flows a first-class cluster-bus
|
||||
event so the parent's tree UI can render the risk pill +
|
||||
verdict result without polling.
|
||||
``CoordinatorAdapter._dispatch_child_event`` re-emits these as
|
||||
``child_ws_intent_verdict`` for the parent coordinator's SSE
|
||||
stream. The dispatch path filters by registry membership;
|
||||
downstream subscribers tolerate verdicts for ws_ids they don't
|
||||
own (silently drop), so we skip the membership pre-check that
|
||||
would otherwise add a lock acquisition per emit on a path
|
||||
that fires once per tool-call during heuristic+LLM judging.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
|
||||
def emit_console_ws_approval_resolved(
|
||||
self,
|
||||
ws_id: str,
|
||||
*,
|
||||
approved: bool,
|
||||
feedback: str = "",
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision for a console-pseudo-node ws.
|
||||
|
||||
Paired with :meth:`emit_console_ws_intent_verdict` so the
|
||||
coord tree UI clears the pending-approval pill in lockstep
|
||||
with the actual decision. Same lock-skip rationale as the
|
||||
intent-verdict emit above.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"approved": approved,
|
||||
"feedback": feedback,
|
||||
"always": always,
|
||||
}
|
||||
)
|
||||
|
||||
def emit_console_ws_approve_request(self, ws_id: str, detail: dict[str, Any]) -> None:
|
||||
"""Fan an ``approve_request`` payload for a console-pseudo-node ws.
|
||||
|
||||
Push path for the initial approval items so a coord parent's
|
||||
tree UI can render the inline approve/deny block immediately
|
||||
without a bulk-fetch round-trip. ``CoordinatorAdapter._dispatch_child_event``
|
||||
re-emits as ``child_ws_approve_request`` for the parent
|
||||
coordinator's SSE stream.
|
||||
"""
|
||||
self._fanout(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": ws_id,
|
||||
"node_id": self.CONSOLE_PSEUDO_NODE_ID,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -15,17 +15,17 @@ for the storage-seeded children rebuild.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.core import session_worker
|
||||
from turnstone.core.adapters._ui_cleanup import cleanup_session_ui
|
||||
from turnstone.core.child_source import ClusterChildSource
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.log import get_logger
|
||||
from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable
|
||||
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
from turnstone.console.coordinator_ui import ConsoleCoordinatorUI
|
||||
@@ -56,38 +56,25 @@ class CoordinatorAdapter:
|
||||
# the manager to the adapter's ``__init__`` — break the cycle
|
||||
# with a setter called from console startup.
|
||||
self._manager: SessionManager | None = None
|
||||
# Per-coordinator known-child ws_id set. Populated lazily on
|
||||
# create/open from storage and updated live as the cluster fan-out
|
||||
# thread sees ws_created events with matching parent_ws_id.
|
||||
# Closed / deleted children stay in the registry so the tree UI
|
||||
# can keep rendering them grayed out; the authoritative render
|
||||
# path reads storage for state. Bounded by eventual coordinator
|
||||
# close/eviction.
|
||||
self._children: dict[str, set[str]] = {}
|
||||
# Reverse index for O(1) child → coord lookup on every cluster
|
||||
# event. Without this, every cluster event incurs a linear scan
|
||||
# over every coordinator's child set while holding the fan-out
|
||||
# lock — a hot-path tax that scales with both active coordinators
|
||||
# and their retained-history depth.
|
||||
self._child_to_coord: dict[str, str] = {}
|
||||
self._children_lock = threading.Lock()
|
||||
# Cluster-event fan-out: subscribes to the ClusterCollector's
|
||||
# listener channel, filters by known child ws_ids, and re-emits
|
||||
# child_ws_* events on the matching coordinator's UI. Configured
|
||||
# lazily via ``start_child_event_fanout(collector)`` from the
|
||||
# console lifespan once both the manager and collector exist.
|
||||
self._collector_queue: queue.Queue[dict[str, Any]] | None = None
|
||||
self._fanout_thread: threading.Thread | None = None
|
||||
self._fanout_stop = threading.Event()
|
||||
# Coord-ws-id → UI map for the fan-out dispatch path. Read
|
||||
# and written under ``self._children_lock`` alongside the
|
||||
# forward/reverse child maps. (Previously the value was
|
||||
# ``(user_id, ui)`` with a copy-on-write dict swap so the
|
||||
# dispatch could read it lock-free — but _dispatch_child_event
|
||||
# already re-validates the parent under _children_lock anyway,
|
||||
# so the lock-free snapshot was premature. The user_id half is
|
||||
# also dead after a46dab1 dropped row-level ownership gates.)
|
||||
self._active_coords: dict[str, Any] = {}
|
||||
# Children registry — universal parent → children + reverse
|
||||
# lookup primitive. Lifted from inline data on this adapter to
|
||||
# ``turnstone.core.children_registry.ChildrenRegistry`` (Stage 3
|
||||
# Step 1) so the same primitive can serve interactive
|
||||
# workstreams when they gain spawn capability. The dispatch
|
||||
# path (`_dispatch_child_event`) calls into the registry
|
||||
# atomically; everything else delegates through the legacy
|
||||
# method names which still exist as thin shims for the
|
||||
# cluster-routing + cleanup callers.
|
||||
self._registry = ChildrenRegistry()
|
||||
# Cross-node child events arrive via ``ClusterChildSource``
|
||||
# (Stage 3 Step 2): a strategy that subscribes to the
|
||||
# collector's listener channel and runs a daemon thread that
|
||||
# drains the queue, pushing each event to the sink. The sink
|
||||
# is :meth:`_dispatch_child_event` so the existing translation
|
||||
# logic (cluster_state → child_ws_state, etc.) stays in one
|
||||
# place. Constructed lazily by ``start_child_event_fanout`` so
|
||||
# the collector reference is available.
|
||||
self._child_source: ClusterChildSource | None = None
|
||||
|
||||
def attach(self, manager: SessionManager) -> None:
|
||||
"""Late-bind the owning :class:`SessionManager`.
|
||||
@@ -109,7 +96,7 @@ class CoordinatorAdapter:
|
||||
# needs the empty forward/presence entries so
|
||||
# ``_dispatch_child_event`` recognises this coordinator when
|
||||
# its first child is spawned.
|
||||
self._install_coord_registry(ws)
|
||||
self._registry.install(ws.id, ws.ui)
|
||||
self._fanout_console_ws_created(ws)
|
||||
|
||||
def emit_rehydrated(self, ws: Workstream) -> None:
|
||||
@@ -117,21 +104,10 @@ class CoordinatorAdapter:
|
||||
# it from storage after the registry seed so a ``ws_created``
|
||||
# for an already-spawned child that fires mid-rebuild merges
|
||||
# cleanly.
|
||||
self._install_coord_registry(ws)
|
||||
self._registry.install(ws.id, ws.ui)
|
||||
self._rebuild_children_registry(ws.id)
|
||||
self._fanout_console_ws_created(ws)
|
||||
|
||||
def _install_coord_registry(self, ws: Workstream) -> None:
|
||||
"""Seed the children registry + presence map for ``ws``.
|
||||
|
||||
Shared by ``emit_created`` and ``emit_rehydrated`` — the
|
||||
difference between the two is purely whether we then rebuild
|
||||
from storage.
|
||||
"""
|
||||
with self._children_lock:
|
||||
self._children.setdefault(ws.id, set())
|
||||
self._active_coords[ws.id] = ws.ui
|
||||
|
||||
def _fanout_console_ws_created(self, ws: Workstream) -> None:
|
||||
try:
|
||||
self._collector.emit_console_ws_created(
|
||||
@@ -197,14 +173,11 @@ class CoordinatorAdapter:
|
||||
# toast only fire for real-node (interactive) ws_closed events.
|
||||
del reason, name
|
||||
# Drop the coordinator's children-registry entries AND its
|
||||
# presence slot. Mirrors the eviction/close paths from the old
|
||||
# CoordinatorManager (which did the same under _children_lock
|
||||
# + _lock respectively). A plain _children.pop without clearing
|
||||
# the reverse index would leak every evicted coordinator's
|
||||
# child→parent pointers forever.
|
||||
with self._children_lock:
|
||||
self._pop_coord_registry_locked(ws_id)
|
||||
self._active_coords.pop(ws_id, None)
|
||||
# presence slot. A plain pop without clearing the reverse
|
||||
# index would leak every evicted coordinator's child→parent
|
||||
# pointers forever — :meth:`ChildrenRegistry.uninstall`
|
||||
# handles forward set + reverse entries + presence atomically.
|
||||
self._registry.uninstall(ws_id)
|
||||
try:
|
||||
self._collector.emit_console_ws_closed(ws_id)
|
||||
except Exception:
|
||||
@@ -356,25 +329,6 @@ class CoordinatorAdapter:
|
||||
# Children registry
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_child_ids_locked(self, coord_ws_id: str, child_ids: Iterable[str]) -> None:
|
||||
"""Merge ``child_ids`` into ``coord_ws_id``'s forward + reverse maps.
|
||||
|
||||
Caller MUST hold ``self._children_lock``. Idempotent — re-adding
|
||||
an existing child is a no-op (the reverse-index pointer is
|
||||
already correct). Empty / falsy entries in ``child_ids`` are
|
||||
skipped.
|
||||
|
||||
Sole write-path for bulk registry updates so
|
||||
``_rebuild_children_registry`` (storage-seeded) and
|
||||
``_prime_children_from_snapshot`` (collector-seeded) agree on
|
||||
ordering and reverse-index invariants.
|
||||
"""
|
||||
existing = self._children.setdefault(coord_ws_id, set())
|
||||
for cid in child_ids:
|
||||
if cid and cid not in existing:
|
||||
existing.add(cid)
|
||||
self._child_to_coord[cid] = coord_ws_id
|
||||
|
||||
def _rebuild_children_registry(self, coord_ws_id: str) -> None:
|
||||
"""Populate ``self._children[coord_ws_id]`` from storage.
|
||||
|
||||
@@ -445,174 +399,88 @@ class CoordinatorAdapter:
|
||||
if not child_id:
|
||||
continue
|
||||
child_ids.append(child_id)
|
||||
with self._children_lock:
|
||||
self._merge_child_ids_locked(coord_ws_id, child_ids)
|
||||
|
||||
def _coord_for_child(self, child_ws_id: str) -> str | None:
|
||||
"""Reverse-lookup: which coordinator owns this child ws_id?
|
||||
|
||||
O(1) via the ``_child_to_coord`` reverse index. Cluster events
|
||||
fire on every token tick across the cluster; a linear scan here
|
||||
turned into a hot-path tax as the retained-history set grew.
|
||||
"""
|
||||
with self._children_lock:
|
||||
return self._child_to_coord.get(child_ws_id)
|
||||
self._registry.merge_children(coord_ws_id, child_ids)
|
||||
|
||||
def children_snapshot(self, coord_ws_id: str) -> list[str]:
|
||||
"""Return a snapshot of the coordinator's direct child ws_ids.
|
||||
|
||||
Used by ``stop_cascade`` to iterate children without holding the
|
||||
registry lock during the per-child HTTP dispatch. A mutation
|
||||
racing with the snapshot (child spawned mid-cascade) either
|
||||
lands before the snapshot and gets cancelled, or lands after
|
||||
and is out of scope for this batch — both outcomes are safe.
|
||||
Returns an empty list for unknown coordinators.
|
||||
Used by ``stop_cascade`` to iterate children without holding
|
||||
the registry lock during the per-child HTTP dispatch. A
|
||||
mutation racing with the snapshot (child spawned mid-cascade)
|
||||
either lands before (cancelled) or after (out of scope for
|
||||
this batch) — both safe. Returns an empty list for unknown
|
||||
coordinators.
|
||||
"""
|
||||
with self._children_lock:
|
||||
child_set = self._children.get(coord_ws_id)
|
||||
return list(child_set) if child_set else []
|
||||
|
||||
def _pop_coord_registry_locked(self, coord_ws_id: str) -> None:
|
||||
"""Remove a coordinator's forward set + reverse-index entries.
|
||||
|
||||
Caller MUST hold ``self._children_lock``. Used by close /
|
||||
eviction paths so stale coordinators don't leak registry
|
||||
entries. No-op if the coordinator is unknown.
|
||||
"""
|
||||
child_set = self._children.pop(coord_ws_id, None)
|
||||
if child_set is None:
|
||||
return
|
||||
for cid in child_set:
|
||||
# Defensive: only clear the reverse entry if it still points
|
||||
# at THIS coordinator. If a child has since been reassigned
|
||||
# (unusual but possible on schema changes), we don't want to
|
||||
# orphan the new owner's entry.
|
||||
if self._child_to_coord.get(cid) == coord_ws_id:
|
||||
self._child_to_coord.pop(cid, None)
|
||||
return self._registry.children_of(coord_ws_id)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cluster-event fan-out thread
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def start_child_event_fanout(self, collector: ClusterCollector) -> None:
|
||||
"""Subscribe to cluster events and start the filter + re-emit thread.
|
||||
"""Subscribe to cluster events via :class:`ClusterChildSource`.
|
||||
|
||||
Idempotent — calling twice is a no-op (already-started fan-out
|
||||
thread stays). Called once from the console lifespan after both
|
||||
the collector and the session manager are constructed.
|
||||
Idempotent — already-started ChildSource stays. Called once
|
||||
from the console lifespan after both the collector and the
|
||||
session manager are constructed.
|
||||
"""
|
||||
if self._fanout_thread is not None and self._fanout_thread.is_alive():
|
||||
if self._child_source is not None:
|
||||
return
|
||||
self._collector = collector
|
||||
self._collector_queue = queue.Queue(maxsize=1000)
|
||||
# Ensure the "console" pseudo-node exists in the snapshot map so
|
||||
# emit_console_ws_* calls from create / close / open land on a
|
||||
# real node entry the snapshot will surface.
|
||||
# Coord-specific transport setup: the "console" pseudo-node
|
||||
# must exist in the snapshot map BEFORE any
|
||||
# ``emit_console_ws_*`` calls (and before the snapshot is
|
||||
# taken inside ``ChildSource.start``) so those emits land on
|
||||
# a real node entry the snapshot surfaces.
|
||||
collector.ensure_console_pseudo_node()
|
||||
# Register with the collector — use the existing listener channel
|
||||
# the browser SSE fan-out uses; the collector treats our queue as
|
||||
# just another subscriber.
|
||||
snapshot = collector.get_snapshot_and_register(self._collector_queue)
|
||||
# Prime the child registry from the snapshot so a coordinator
|
||||
# that opens right after a console restart sees already-live
|
||||
# children without waiting for the next ``ws_state`` tick to
|
||||
# discover them via the fan-out path.
|
||||
self._prime_children_from_snapshot(snapshot)
|
||||
# Seed the pseudo-node with any coordinators already loaded in
|
||||
# memory when the collector binds. Prevents a race where early
|
||||
# creates happened before the collector was wired up and their
|
||||
# rows never showed on the snapshot.
|
||||
mgr = self._manager
|
||||
if mgr is not None:
|
||||
for ws in mgr.list_all():
|
||||
try:
|
||||
collector.emit_console_ws_created(
|
||||
ws.id,
|
||||
name=ws.name,
|
||||
user_id=ws.user_id or "",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=ws.state.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_adapter.collector_seed_failed ws=%s",
|
||||
ws.id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
self._fanout_stop.clear()
|
||||
t = threading.Thread(
|
||||
target=self._fanout_loop,
|
||||
name="coord-adapter-child-fanout",
|
||||
daemon=True,
|
||||
)
|
||||
self._fanout_thread = t
|
||||
t.start()
|
||||
|
||||
def _prime_children_from_snapshot(self, snapshot: dict[str, Any]) -> None:
|
||||
"""Populate ``_children`` + ``_child_to_coord`` from a collector snapshot.
|
||||
|
||||
The snapshot's per-node workstreams carry ``parent_ws_id``. For
|
||||
every workstream whose parent is an in-memory coordinator,
|
||||
record the child so the fan-out filter sees it immediately.
|
||||
"""
|
||||
nodes = snapshot.get("nodes", []) if isinstance(snapshot, dict) else []
|
||||
if not nodes:
|
||||
return
|
||||
mgr = self._manager
|
||||
if mgr is None:
|
||||
raise RuntimeError(
|
||||
"CoordinatorAdapter: manager not attached — call attach(mgr) after construction"
|
||||
)
|
||||
by_parent: dict[str, list[str]] = {}
|
||||
known = {ws.id for ws in mgr.list_all()}
|
||||
for node in nodes:
|
||||
for entry in node.get("workstreams", []) or []:
|
||||
parent = entry.get("parent_ws_id") or ""
|
||||
child_id = entry.get("id") or ""
|
||||
if not parent or not child_id or parent not in known:
|
||||
continue
|
||||
by_parent.setdefault(parent, []).append(child_id)
|
||||
if not by_parent:
|
||||
return
|
||||
with self._children_lock:
|
||||
for parent, kids in by_parent.items():
|
||||
self._merge_child_ids_locked(parent, kids)
|
||||
# Build + start the strategy. The sink is
|
||||
# :meth:`_dispatch_child_event` so cluster events flow through
|
||||
# the same translation path that synthesises ``child_ws_*``
|
||||
# payloads for the parent's UI.
|
||||
source = ClusterChildSource(
|
||||
collector=collector,
|
||||
registry=self._registry,
|
||||
parents_provider=lambda: [ws.id for ws in mgr.list_all()],
|
||||
)
|
||||
source.start(sink=self._dispatch_child_event)
|
||||
self._child_source = source
|
||||
# Seed the pseudo-node with any coordinators already loaded in
|
||||
# memory when the collector binds. Prevents a race where early
|
||||
# creates happened before the collector was wired up and their
|
||||
# rows never showed on the snapshot. (Coord-specific — interactive
|
||||
# has no analogous pseudo-node.)
|
||||
for ws in mgr.list_all():
|
||||
try:
|
||||
collector.emit_console_ws_created(
|
||||
ws.id,
|
||||
name=ws.name,
|
||||
user_id=ws.user_id or "",
|
||||
kind=WorkstreamKind.COORDINATOR.value,
|
||||
state=ws.state.value,
|
||||
parent_ws_id=None,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_adapter.collector_seed_failed ws=%s",
|
||||
ws.id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the fan-out thread and unregister from the collector.
|
||||
"""Stop the ChildSource and unregister from the collector.
|
||||
|
||||
Safe to call multiple times; idempotent. Invoked from the
|
||||
console lifespan teardown so SSE listener queues don't leak.
|
||||
"""
|
||||
self._fanout_stop.set()
|
||||
t = self._fanout_thread
|
||||
q = self._collector_queue
|
||||
coll = self._collector
|
||||
self._fanout_thread = None
|
||||
self._collector_queue = None
|
||||
if coll is not None and q is not None:
|
||||
try:
|
||||
coll.unregister_listener(q)
|
||||
except Exception:
|
||||
log.debug("coord_adapter.unregister_listener_failed", exc_info=True)
|
||||
if t is not None:
|
||||
t.join(timeout=2.0)
|
||||
|
||||
def _fanout_loop(self) -> None:
|
||||
"""Drain collector events, filter by known children, dispatch."""
|
||||
q = self._collector_queue
|
||||
if q is None:
|
||||
return
|
||||
while not self._fanout_stop.is_set():
|
||||
try:
|
||||
event = q.get(timeout=1.0)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
self._dispatch_child_event(event)
|
||||
except Exception:
|
||||
log.debug("coord_adapter.fanout.dispatch_failed", exc_info=True)
|
||||
src = self._child_source
|
||||
self._child_source = None
|
||||
if src is not None:
|
||||
src.shutdown()
|
||||
|
||||
def _dispatch_child_event(self, event: dict[str, Any]) -> None:
|
||||
"""Match a cluster event to a coordinator and re-emit on its UI.
|
||||
@@ -622,10 +490,12 @@ class CoordinatorAdapter:
|
||||
- ``ws_created`` with ``parent_ws_id`` matching an in-memory
|
||||
coordinator → add to registry + re-emit as
|
||||
``child_ws_created``.
|
||||
- ``cluster_state`` / ``ws_closed`` / ``ws_rename`` whose
|
||||
``ws_id`` is in any coordinator's known-children registry →
|
||||
re-emit as ``child_ws_state`` / ``child_ws_closed`` /
|
||||
``child_ws_rename``.
|
||||
- ``cluster_state`` / ``ws_closed`` / ``ws_rename`` /
|
||||
``intent_verdict`` / ``approval_resolved`` whose ``ws_id``
|
||||
is in any coordinator's known-children registry → re-emit as
|
||||
``child_ws_state`` / ``child_ws_closed`` /
|
||||
``child_ws_rename`` / ``child_ws_intent_verdict`` /
|
||||
``child_ws_approval_resolved``.
|
||||
|
||||
Events for ws_ids we don't own silently drop — the filter lives
|
||||
on the server so each coordinator's SSE stream stays small.
|
||||
@@ -639,23 +509,17 @@ class CoordinatorAdapter:
|
||||
parent = event.get("parent_ws_id") or ""
|
||||
if not parent:
|
||||
return
|
||||
# Presence check + registry mutation under the same lock:
|
||||
# a concurrent close()/eviction can pop the entry between
|
||||
# the check and the mutation, after which a bare setdefault
|
||||
# would resurrect the entry — leaking the registry key and
|
||||
# enqueuing onto the closed coordinator's UI. Trusted-team
|
||||
# posture (#400 / a46dab1) means no per-event tenant gate
|
||||
# here; scope-level auth at the SSE endpoint is the only
|
||||
# boundary.
|
||||
with self._children_lock:
|
||||
coord_ui = self._active_coords.get(parent)
|
||||
if coord_ui is None:
|
||||
return
|
||||
existing = self._children.setdefault(parent, set())
|
||||
if ws_id in existing:
|
||||
return
|
||||
existing.add(ws_id)
|
||||
self._child_to_coord[ws_id] = parent
|
||||
# Atomic check-and-route under the registry's lock: a
|
||||
# concurrent close()/eviction can pop the parent's entry
|
||||
# between the presence check and the mutation, so the two
|
||||
# must happen together. ``add_child`` returns the parent's
|
||||
# UI on success or None on (a) parent not installed, or
|
||||
# (b) duplicate child. Trusted-team posture (#400 / a46dab1)
|
||||
# means no per-event tenant gate here; scope-level auth at
|
||||
# the SSE endpoint is the only boundary.
|
||||
coord_ui = self._registry.add_child(parent, ws_id)
|
||||
if coord_ui is None:
|
||||
return
|
||||
payload = {
|
||||
"type": "child_ws_created",
|
||||
"ws_id": ws_id,
|
||||
@@ -668,8 +532,15 @@ class CoordinatorAdapter:
|
||||
_enqueue_on_ui(coord_ui, parent, payload)
|
||||
return
|
||||
|
||||
if etype in ("cluster_state", "ws_closed", "ws_rename"):
|
||||
coord_id = self._coord_for_child(ws_id)
|
||||
if etype in (
|
||||
"cluster_state",
|
||||
"ws_closed",
|
||||
"ws_rename",
|
||||
"intent_verdict",
|
||||
"approval_resolved",
|
||||
"approve_request",
|
||||
):
|
||||
coord_id = self._registry.parent_for(ws_id)
|
||||
if coord_id is None:
|
||||
return
|
||||
mgr = self._manager
|
||||
@@ -684,14 +555,13 @@ class CoordinatorAdapter:
|
||||
"state": event.get("state", ""),
|
||||
"tokens": event.get("tokens", 0),
|
||||
"node_id": event.get("node_id", ""),
|
||||
# activity_state lets the JS detect approval-state
|
||||
# transitions; pending_approval_detail rides on
|
||||
# the same event so the browser can mutate
|
||||
# liveBadgeCache directly and render inline
|
||||
# approve/deny buttons in lockstep with the
|
||||
# transition, no separate dashboard refetch.
|
||||
# ``activity_state`` lets the JS detect approval
|
||||
# transitions and trigger a bulk fetch for the
|
||||
# initial detail. The detail itself no longer
|
||||
# piggybacks here (Stage 3 cleanup) — verdicts
|
||||
# arrive via the explicit ``intent_verdict`` event
|
||||
# class and resolution via ``approval_resolved``.
|
||||
"activity_state": event.get("activity_state", ""),
|
||||
"pending_approval_detail": event.get("pending_approval_detail"),
|
||||
}
|
||||
elif etype == "ws_closed":
|
||||
child_event = {
|
||||
@@ -700,13 +570,52 @@ class CoordinatorAdapter:
|
||||
"parent_ws_id": coord_id,
|
||||
"reason": event.get("reason", ""),
|
||||
}
|
||||
else: # ws_rename
|
||||
elif etype == "ws_rename":
|
||||
child_event = {
|
||||
"type": "child_ws_rename",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"name": event.get("name", ""),
|
||||
}
|
||||
elif etype == "intent_verdict":
|
||||
# Per-coord re-emit of an explicit verdict event so
|
||||
# the tree UI can render the risk pill + verdict
|
||||
# result without polling. The corresponding
|
||||
# ``cluster_state`` event no longer carries the
|
||||
# detail (the piggyback was removed end-to-end);
|
||||
# the bulk fetch on initial approval entry plus this
|
||||
# explicit event class are the only carriers.
|
||||
child_event = {
|
||||
"type": "child_ws_intent_verdict",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"verdict": event.get("verdict") or {},
|
||||
}
|
||||
elif etype == "approval_resolved":
|
||||
child_event = {
|
||||
"type": "child_ws_approval_resolved",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"approved": bool(event.get("approved", False)),
|
||||
"feedback": event.get("feedback", "") or "",
|
||||
"always": bool(event.get("always", False)),
|
||||
}
|
||||
else: # approve_request
|
||||
# Push path for the initial approval items —
|
||||
# eliminates the bulk-fetch race that previously left
|
||||
# the coord row stuck on a loading placeholder when
|
||||
# the bulk fetch landed in the gap between the state
|
||||
# transition to ATTENTION and ``_pending_approval``
|
||||
# being set inside ``approve_tools``.
|
||||
child_event = {
|
||||
"type": "child_ws_approve_request",
|
||||
"child_ws_id": ws_id,
|
||||
"parent_ws_id": coord_id,
|
||||
"node_id": event.get("node_id", ""),
|
||||
"detail": event.get("detail") or {},
|
||||
}
|
||||
_enqueue_on_ui(owning_ws.ui, coord_id, child_event)
|
||||
|
||||
|
||||
|
||||
@@ -181,6 +181,81 @@ class ConsoleCoordinatorUI(SessionUIBase):
|
||||
with self._ws_lock:
|
||||
self._last_broadcast_activity = current
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
"""Fan an LLM intent-judge verdict to the cluster collector.
|
||||
|
||||
Stage 3 Step 5 — overrides the no-op base hook so a coord
|
||||
workstream that produces its own verdict (rare — coord agents
|
||||
don't typically run the LLM judge) gets a first-class
|
||||
cluster-bus event for the parent's tree UI. The far more
|
||||
common path is a CHILD workstream firing its verdict on a
|
||||
real node; that path goes through ``WebUI._broadcast_intent_verdict``
|
||||
→ global SSE → collector ``_apply_delta``.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_intent_verdict(self.ws_id, verdict)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.intent_verdict_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool,
|
||||
feedback: str | None = None,
|
||||
*,
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision to the cluster collector.
|
||||
|
||||
Stage 3 Step 5 — paired with :meth:`_broadcast_intent_verdict`.
|
||||
Same rationale: coord-direct approvals are rare; the typical
|
||||
path is a child workstream resolving on its node, with that
|
||||
node's WebUI broadcasting through the global queue.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_approval_resolved(
|
||||
self.ws_id,
|
||||
approved=approved,
|
||||
feedback=feedback or "",
|
||||
always=always,
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.approval_resolved_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None:
|
||||
"""Fan an ``approve_request`` payload to the cluster collector.
|
||||
|
||||
Push path for the initial approval items. Same rationale as
|
||||
the other two broadcast hooks: coord-self approvals are rare
|
||||
(the LLM judge isn't wired on the console coord today), but
|
||||
the override exists for symmetry and lights up the same path
|
||||
a future coord-self gate would use.
|
||||
"""
|
||||
collector = ConsoleCoordinatorUI._collector
|
||||
if collector is None:
|
||||
return
|
||||
try:
|
||||
collector.emit_console_ws_approve_request(self.ws_id, detail)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"coord_ui.approve_request_fanout_failed ws=%s",
|
||||
self.ws_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def on_state_change(self, state: str) -> None:
|
||||
# Flow state transitions through the unified SessionManager so
|
||||
# the storage write + adapter emit_state fan-out stay in lockstep
|
||||
|
||||
@@ -23,6 +23,7 @@ import queue
|
||||
import re
|
||||
import secrets
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -89,6 +90,7 @@ if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
|
||||
from turnstone.core.session import ChatSession
|
||||
from turnstone.core.session_manager import SessionManager
|
||||
from turnstone.core.storage._protocol import StorageBackend
|
||||
|
||||
log = logging.getLogger("turnstone.console.server")
|
||||
@@ -844,8 +846,20 @@ async def _fetch_live_block(
|
||||
live = {k: entry.get(k) for k in _CLUSTER_WS_LIVE_KEYS if k in entry}
|
||||
# Derived field — kept in lockstep with
|
||||
# _coordinator_live_snapshot so both origins produce the
|
||||
# same keys.
|
||||
live["pending_approval"] = live.get("activity_state") == "approval"
|
||||
# same keys. ``state="attention"`` is the canonical signal;
|
||||
# ``activity_state="approval"`` is set inside approve_tools
|
||||
# AFTER the state transition fires, so a bulk fetch that
|
||||
# races with that window can see state=attention and
|
||||
# activity_state="" simultaneously. A non-null
|
||||
# ``pending_approval_detail`` is also a definitive signal
|
||||
# (the serializer only emits non-None when ``_pending_approval``
|
||||
# is set on the UI). Any of the three flips this true; the
|
||||
# frontend reducer mirrors the same disjunction.
|
||||
live["pending_approval"] = (
|
||||
live.get("activity_state") == "approval"
|
||||
or entry.get("state") == "attention"
|
||||
or live.get("pending_approval_detail") is not None
|
||||
)
|
||||
return live
|
||||
return None
|
||||
|
||||
@@ -3687,6 +3701,50 @@ async def _verify_collector_service_scope(app: Starlette, client: httpx.AsyncCli
|
||||
)
|
||||
|
||||
|
||||
def _coord_idle_cleanup_thread(
|
||||
mgr: SessionManager,
|
||||
timeout_sec: float,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> None:
|
||||
"""Periodically reap idle + DB-orphan coordinator workstreams.
|
||||
|
||||
Mirrors the regular server's ``_idle_cleanup_thread`` (turnstone/server.py)
|
||||
but skips the rate-limiter / global-queue arms — the console doesn't have
|
||||
those. ``mgr.close_idle`` does the work: closes loaded IDLE rows AND
|
||||
bulk-closes DB rows of this kind whose ``updated`` is past the cutoff
|
||||
and which aren't currently loaded. The latter pass catches coords left
|
||||
behind by prior console process incarnations.
|
||||
|
||||
Runs an initial sweep BEFORE the first sleep so cold-start orphans are
|
||||
reaped immediately rather than waiting one ``check_every`` interval (~30
|
||||
min on default 2h timeout). This intentionally diverges from the regular
|
||||
server pattern, which has no initial sweep — the regular server runs
|
||||
inside a normal request-handling lifecycle, the console-side coord pool
|
||||
is a small fixed-size cache where orphans dominate the row count after
|
||||
a cold boot.
|
||||
|
||||
``stop_event`` is for tests — when set, the thread exits cleanly after
|
||||
the next loop check. Production callers pass ``None`` (the daemon is
|
||||
process-lifetime).
|
||||
"""
|
||||
check_every = min(300.0, timeout_sec / 4)
|
||||
# Initial sweep — runs once before entering the sleep loop.
|
||||
try:
|
||||
mgr.close_idle(timeout_sec)
|
||||
except Exception:
|
||||
log.debug("console.coord_idle_cleanup_initial_failed", exc_info=True)
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
return
|
||||
time.sleep(check_every)
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
return
|
||||
try:
|
||||
mgr.close_idle(timeout_sec)
|
||||
except Exception:
|
||||
log.debug("console.coord_idle_cleanup_failed", exc_info=True)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
# Create async HTTP clients for proxy routes. Auth headers are NOT baked
|
||||
@@ -3992,6 +4050,27 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
coord_adapter.start_child_event_fanout(app.state.collector)
|
||||
except Exception:
|
||||
log.warning("console.coordinator_child_fanout_init_failed", exc_info=True)
|
||||
# Idle cleanup: closes loaded-but-stale coords AND DB orphans
|
||||
# left behind by prior console processes. The thread runs an
|
||||
# initial sweep on entry (no synchronous lifespan call needed —
|
||||
# see ``_coord_idle_cleanup_thread``) so cold-start cleanup
|
||||
# doesn't block startup. Reuses the regular-server
|
||||
# ``server.workstream_idle_timeout`` setting — the same cadence
|
||||
# makes sense for both kinds and avoids a redundant config knob.
|
||||
try:
|
||||
idle_minutes = int(config_store.get("server.workstream_idle_timeout"))
|
||||
except Exception:
|
||||
idle_minutes = 0
|
||||
if idle_minutes > 0:
|
||||
timeout_sec = float(idle_minutes * 60)
|
||||
cleanup_thread = threading.Thread(
|
||||
target=_coord_idle_cleanup_thread,
|
||||
args=(coord_mgr, timeout_sec),
|
||||
name="coord-idle-cleanup",
|
||||
daemon=True,
|
||||
)
|
||||
cleanup_thread.start()
|
||||
app.state.coord_idle_cleanup_thread = cleanup_thread
|
||||
log.info(
|
||||
"console.coordinator_mgr_ready max_active=%s",
|
||||
config_store.get("coordinator.max_active"),
|
||||
|
||||
+160
-10
@@ -1851,6 +1851,16 @@ var _savedCoordsRetry = false;
|
||||
|
||||
function loadSavedCoordinators() {
|
||||
if (!_hasCoordPermission()) return;
|
||||
// Freeze the list while the user is multi-selecting — re-rendering
|
||||
// mid-mode would shuffle the visible page out from under them. The
|
||||
// delete-mode wrapper drains the retry flag on cancel/onClose.
|
||||
if (
|
||||
typeof _coordDeleteController !== "undefined" &&
|
||||
_coordDeleteController.inMode()
|
||||
) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
}
|
||||
if (_savedCoordsInFlight) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
@@ -1861,6 +1871,16 @@ function loadSavedCoordinators() {
|
||||
return r.ok ? r.json() : { workstreams: [] };
|
||||
})
|
||||
.then(function (data) {
|
||||
// Belt-and-braces: if the user entered delete mode while this
|
||||
// fetch was already in flight, defer the render — re-rendering
|
||||
// mid-selection would shuffle visible cards and reshape selections.
|
||||
if (
|
||||
typeof _coordDeleteController !== "undefined" &&
|
||||
_coordDeleteController.inMode()
|
||||
) {
|
||||
_savedCoordsRetry = true;
|
||||
return;
|
||||
}
|
||||
renderSavedCoordinators(data.workstreams || []);
|
||||
})
|
||||
.catch(function () {
|
||||
@@ -1878,7 +1898,52 @@ function loadSavedCoordinators() {
|
||||
});
|
||||
}
|
||||
|
||||
// Saved Coordinators: paginated card list + multi-select delete.
|
||||
// The shared controller (createSavedCardsController in /shared/cards.js)
|
||||
// owns mode state, checkbox decoration, the toolbar, and the modal.
|
||||
// Pagination caps Select-All fan-out at COORD_PAGE_SIZE — the controller
|
||||
// only ever sees the visible page, so a confirm-all batch is bounded to
|
||||
// COORD_PAGE_SIZE parallel POSTs against the routing proxy.
|
||||
var COORD_PAGE_SIZE = 24;
|
||||
var _coordPage = 0;
|
||||
var _coordSavedItems = [];
|
||||
var _coordDeleteController = createSavedCardsController({
|
||||
idPrefix: "coord-delete",
|
||||
buttonId: "coord-delete-btn",
|
||||
noun: "coordinator",
|
||||
activateLabel: function (s) {
|
||||
return "Resume coordinator: " + (s.alias || s.title || s.name || s.ws_id);
|
||||
},
|
||||
// Coordinators live on whichever node owns the ws_id, so we can't fire
|
||||
// a path-keyed delete the way ui/static does. The router proxy reads
|
||||
// ws_id from the body, resolves the owning node via the consistent-
|
||||
// hash ring, and forwards to that node's POST workstreams/{ws_id}/delete.
|
||||
buildDeleteRequest: function (wsId) {
|
||||
return {
|
||||
url: "/v1/api/route/workstreams/delete",
|
||||
options: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ ws_id: wsId }),
|
||||
},
|
||||
};
|
||||
},
|
||||
render: function () {
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
},
|
||||
onClose: function () {
|
||||
// Drain queued retries before the explicit reload — without this,
|
||||
// _savedCoordsRetry is still true from SSE events that arrived
|
||||
// during the freeze, so loadSavedCoordinators's .finally() would
|
||||
// re-fire a second fetch immediately after the first resolves.
|
||||
// Same idiom as cancelCoordDeleteMode below.
|
||||
_savedCoordsRetry = false;
|
||||
loadSavedCoordinators();
|
||||
},
|
||||
});
|
||||
|
||||
function renderSavedCoordinators(items) {
|
||||
_coordSavedItems = items;
|
||||
var section = document.getElementById("saved-coordinators");
|
||||
var cards = document.getElementById("saved-coord-cards");
|
||||
var countEl = document.getElementById("saved-coord-count");
|
||||
@@ -1887,24 +1952,31 @@ function renderSavedCoordinators(items) {
|
||||
section.style.display = "none";
|
||||
cards.replaceChildren();
|
||||
if (countEl) countEl.textContent = "";
|
||||
_coordPage = 0;
|
||||
if (_coordDeleteController.inMode()) _coordDeleteController.cancel();
|
||||
_renderCoordPagination();
|
||||
return;
|
||||
}
|
||||
// Clamp the page index after deletes (or upstream churn) shrink the list.
|
||||
var pages = Math.max(1, Math.ceil(items.length / COORD_PAGE_SIZE));
|
||||
if (_coordPage > pages - 1) _coordPage = pages - 1;
|
||||
if (_coordPage < 0) _coordPage = 0;
|
||||
var visible = items.slice(
|
||||
_coordPage * COORD_PAGE_SIZE,
|
||||
(_coordPage + 1) * COORD_PAGE_SIZE,
|
||||
);
|
||||
_coordDeleteController.setItems(visible);
|
||||
|
||||
section.style.display = "";
|
||||
if (countEl) countEl.textContent = "(" + items.length + ")";
|
||||
cards.replaceChildren();
|
||||
items.forEach(function (sess) {
|
||||
visible.forEach(function (sess) {
|
||||
var card = renderSessionCard(sess, {
|
||||
ariaLabel: function (s) {
|
||||
return (
|
||||
"Resume coordinator: " + (s.alias || s.title || s.name || s.ws_id)
|
||||
);
|
||||
},
|
||||
ariaLabel: _coordDeleteController.ariaLabel,
|
||||
onActivate: function (s, cardEl) {
|
||||
if (_coordDeleteController.blockActivate()) return;
|
||||
// POST /open BEFORE navigating so capacity issues surface as a
|
||||
// toast instead of a broken-looking detail page. The /open
|
||||
// endpoint calls the same lazy-rehydrate path the GET would,
|
||||
// but we get the status code synchronously so the user learns
|
||||
// "all slots in use" instead of staring at a 404.
|
||||
// toast instead of a broken-looking detail page.
|
||||
cardEl.classList.add("is-busy");
|
||||
authFetch(
|
||||
"/v1/api/workstreams/" + encodeURIComponent(s.ws_id) + "/open",
|
||||
@@ -1936,8 +2008,86 @@ function renderSavedCoordinators(items) {
|
||||
});
|
||||
},
|
||||
});
|
||||
_coordDeleteController.decorateCard(card, sess);
|
||||
cards.appendChild(card);
|
||||
});
|
||||
if (_coordDeleteController.inMode()) _coordDeleteController.refreshBar();
|
||||
_renderCoordPagination();
|
||||
}
|
||||
|
||||
function _renderCoordPagination() {
|
||||
var pag = document.getElementById("coord-pagination");
|
||||
if (!pag) return;
|
||||
var total = _coordSavedItems.length;
|
||||
var pages = Math.max(1, Math.ceil(total / COORD_PAGE_SIZE));
|
||||
// Single-page lists and delete-mode hide the controls — page changes
|
||||
// would invalidate the user's checkbox selections, so we lock them out.
|
||||
if (pages <= 1 || _coordDeleteController.inMode()) {
|
||||
pag.style.display = "none";
|
||||
return;
|
||||
}
|
||||
pag.style.display = "";
|
||||
var label = document.getElementById("coord-page-label");
|
||||
if (label) {
|
||||
/* Visible text uses the terse "X / Y" form to match the
|
||||
filtered-pagination control elsewhere in the console; the long
|
||||
form sits on the parent's aria-label so screen readers still get
|
||||
a full sentence. */
|
||||
label.textContent = _coordPage + 1 + " / " + pages;
|
||||
pag.setAttribute(
|
||||
"aria-label",
|
||||
"Saved coordinators pagination — page " +
|
||||
(_coordPage + 1) +
|
||||
" of " +
|
||||
pages,
|
||||
);
|
||||
}
|
||||
var prev = document.getElementById("coord-page-prev");
|
||||
if (prev) prev.disabled = _coordPage <= 0;
|
||||
var next = document.getElementById("coord-page-next");
|
||||
if (next) next.disabled = _coordPage >= pages - 1;
|
||||
}
|
||||
|
||||
function coordPagePrev() {
|
||||
if (_coordPage > 0) {
|
||||
_coordPage--;
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
}
|
||||
}
|
||||
|
||||
function coordPageNext() {
|
||||
var pages = Math.max(1, Math.ceil(_coordSavedItems.length / COORD_PAGE_SIZE));
|
||||
if (_coordPage < pages - 1) {
|
||||
_coordPage++;
|
||||
renderSavedCoordinators(_coordSavedItems);
|
||||
}
|
||||
}
|
||||
|
||||
// HTML inline-onclick wrappers — keep the global names the markup binds
|
||||
// to and forward to the shared controller.
|
||||
function startCoordDeleteMode() {
|
||||
_coordDeleteController.start();
|
||||
}
|
||||
function cancelCoordDeleteMode() {
|
||||
_coordDeleteController.cancel();
|
||||
// The freeze gate (see loadSavedCoordinators) may have queued retries
|
||||
// while we were multi-selecting; drain them now that we're idle again.
|
||||
if (_savedCoordsRetry) {
|
||||
_savedCoordsRetry = false;
|
||||
loadSavedCoordinators();
|
||||
}
|
||||
}
|
||||
function toggleCoordSelectAll() {
|
||||
_coordDeleteController.toggleAll();
|
||||
}
|
||||
function confirmCoordDeleteSelection() {
|
||||
_coordDeleteController.confirmSelection();
|
||||
}
|
||||
function cancelCoordDelete() {
|
||||
_coordDeleteController.closeModal();
|
||||
}
|
||||
function confirmCoordDelete() {
|
||||
_coordDeleteController.confirm();
|
||||
}
|
||||
|
||||
// --- Init ---
|
||||
|
||||
@@ -1730,7 +1730,7 @@
|
||||
// state) so a user lacking admin.cluster.inspect doesn't
|
||||
// pay one 403 per denied id on every reconnect.
|
||||
for (const [id, c] of liveBadgeCache) {
|
||||
if (!c || !c.permanent) liveBadgeCache.delete(id);
|
||||
if (!c || !c.permanent) _liveBadgeCacheDelete(id);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -2038,6 +2038,21 @@
|
||||
case "child_ws_rename":
|
||||
handleChildRename(ev);
|
||||
break;
|
||||
// Stage 3 Step 7 — explicit verdict + approval-resolved events
|
||||
// arrive without piggybacking on cluster_state, so verdicts that
|
||||
// land while a child is in `attention` (no state transition)
|
||||
// reach the parent's tree UI promptly. Reducer writes
|
||||
// liveBadgeCache directly (bypassing scheduleLiveFetch) so
|
||||
// off-screen rows update too.
|
||||
case "child_ws_intent_verdict":
|
||||
handleChildIntentVerdict(ev);
|
||||
break;
|
||||
case "child_ws_approval_resolved":
|
||||
handleChildApprovalResolved(ev);
|
||||
break;
|
||||
case "child_ws_approve_request":
|
||||
handleChildApproveRequest(ev);
|
||||
break;
|
||||
// wait_for_workstream observability (#14) — the worker thread
|
||||
// can block up to 600s inside the tool; these events drive a
|
||||
// sidebar indicator so operators see the coordinator is alive.
|
||||
@@ -2154,6 +2169,37 @@
|
||||
const childrenState = new Map();
|
||||
// ws_id -> {live: <dict>, fetched: <ms>} for the 5s TTL live-badge cache.
|
||||
const liveBadgeCache = new Map();
|
||||
// Incrementally-maintained set of ws_ids currently flagged
|
||||
// ``pending_approval`` in the cache. Updated at every cache mutation
|
||||
// via ``_liveBadgeCacheSet`` / ``_liveBadgeCacheDelete`` /
|
||||
// ``_liveBadgeCacheClear`` below so the sidebar pending-count read
|
||||
// is O(1) instead of an O(N) walk over the cache on every render
|
||||
// (caught by /review perf-1).
|
||||
const pendingApprovalIds = new Set();
|
||||
// The helpers reach into the Map's prototype directly so a future
|
||||
// edit can't accidentally rewrite the body's ``liveBadgeCache.set``
|
||||
// into the helper name and create infinite recursion (already
|
||||
// happened once when bulk-replacing call sites — caught at runtime
|
||||
// as "InternalError: too much recursion").
|
||||
const _mapSet = Map.prototype.set;
|
||||
const _mapDelete = Map.prototype.delete;
|
||||
const _mapClear = Map.prototype.clear;
|
||||
function _liveBadgeCacheSet(id, entry) {
|
||||
_mapSet.call(liveBadgeCache, id, entry);
|
||||
if (entry && entry.live && entry.live.pending_approval) {
|
||||
pendingApprovalIds.add(id);
|
||||
} else {
|
||||
pendingApprovalIds.delete(id);
|
||||
}
|
||||
}
|
||||
function _liveBadgeCacheDelete(id) {
|
||||
_mapDelete.call(liveBadgeCache, id);
|
||||
pendingApprovalIds.delete(id);
|
||||
}
|
||||
function _liveBadgeCacheClear() {
|
||||
_mapClear.call(liveBadgeCache);
|
||||
pendingApprovalIds.clear();
|
||||
}
|
||||
// ws_ids currently visible in the viewport — only these trigger
|
||||
// live-fetch on SSE state changes. Populated by an
|
||||
// IntersectionObserver attached to each rendered .ch-row so a
|
||||
@@ -2280,23 +2326,22 @@
|
||||
}
|
||||
}
|
||||
row.appendChild(meta);
|
||||
// Inline approve/deny block \u2014 shown only when the live block
|
||||
// carries pending_approval_detail (the rich payload added by the
|
||||
// server-side dashboard projection). A "\u2691 approval" badge alone
|
||||
// means the child is in attention state but the rich detail
|
||||
// hasn't arrived on the cache yet \u2014 a rare cross-version race
|
||||
// (e.g. a node mid-rolling-upgrade emitted ws_state without
|
||||
// pending_approval_detail before this PR landed). The next SSE
|
||||
// tick or the 5s TTL bulk-poll catches up and re-renders.
|
||||
if (cached && cached.live && cached.live.pending_approval_detail) {
|
||||
// Inline approve/deny block \u2014 the detail arrives via the bulk
|
||||
// fetch triggered by handleChildState off the activity_state="approval"
|
||||
// transition. Verdicts that land later arrive via
|
||||
// child_ws_intent_verdict and update the cached detail in place;
|
||||
// resolution arrives via child_ws_approval_resolved and clears it.
|
||||
// While the bulk fetch is in flight (~250ms debounce + ~100ms HTTP)
|
||||
// we render a loading placeholder so the row keeps its height
|
||||
// stable AND so a screen reader has a labelled region to land on
|
||||
// \u2014 without it the operator sees a "demand for action" badge
|
||||
// with no actionable content.
|
||||
if (cached && cached.live && cached.live.pending_approval) {
|
||||
const detail = cached.live.pending_approval_detail;
|
||||
const block = renderApprovalBlock(child, detail);
|
||||
const block = detail
|
||||
? renderApprovalBlock(child, detail)
|
||||
: renderApprovalPlaceholder(child);
|
||||
if (block) row.appendChild(block);
|
||||
// Late-arriving LLM judge: see comment above
|
||||
// _maybeStartJudgePoll for the why. Hooks into a single
|
||||
// global poller (not per-row) so off-screen rows still
|
||||
// refresh and one bulk request covers every pending row.
|
||||
_maybeStartJudgePoll();
|
||||
}
|
||||
// Recent auto-approves — tools that bypassed the operator gate
|
||||
// (skill ``allowed_tools`` allowlist / blanket / admin policy /
|
||||
@@ -2370,78 +2415,6 @@
|
||||
return pill;
|
||||
}
|
||||
|
||||
// Late-judge polling — single global timer driving a bulk
|
||||
// re-fetch of every row whose `pending_approval_detail.judge_pending`
|
||||
// is true and not every item has a `judge_verdict` yet. Necessary
|
||||
// because the LLM judge runs async on the child node and never
|
||||
// pushes a signal that reaches the coord; without polling the
|
||||
// operator would stare at a "heuristic"-tier pill forever.
|
||||
//
|
||||
// Why a global poller instead of per-row:
|
||||
// 1. Per-row would invalidate the cache and call scheduleLiveFetch
|
||||
// for off-screen rows, but scheduleLiveFetch returns early on
|
||||
// non-visible ids — leaving the cache empty AND no fetch in
|
||||
// flight. Result: off-screen rows stuck on stale heuristic.
|
||||
// 2. A single bulk request covers every pending row in one
|
||||
// round-trip; the per-row design would issue N microtask-
|
||||
// batched fetches that share the bulk path anyway.
|
||||
const JUDGE_POLL_INTERVAL_MS = 2000;
|
||||
// Cap by total wall-clock time, not attempts. Real LLM judges
|
||||
// (esp. with reasoning effort) can exceed 30s — a 6-attempt /
|
||||
// 12s cap was prematurely giving up.
|
||||
const JUDGE_POLL_MAX_DURATION_MS = 90_000;
|
||||
let judgePollTimer = null;
|
||||
let judgePollStartedAt = 0;
|
||||
|
||||
function _maybeStartJudgePoll() {
|
||||
if (judgePollTimer !== null) return; // already polling
|
||||
judgePollStartedAt = Date.now();
|
||||
judgePollTimer = setTimeout(_judgePollTick, JUDGE_POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function _judgePollTick() {
|
||||
judgePollTimer = null;
|
||||
if (Date.now() - judgePollStartedAt > JUDGE_POLL_MAX_DURATION_MS) {
|
||||
// Failed / timed-out judge — give up so we don't poll forever.
|
||||
// Operator can hit the Refresh button to force a fresh fetch.
|
||||
return;
|
||||
}
|
||||
// Walk the full childrenState — not just visible — so off-screen
|
||||
// rows still get refreshed. scheduleLiveFetch's visibility gate
|
||||
// exists to keep idle rows from burning round-trips; here we
|
||||
// explicitly want every pending-judge row in the next bulk
|
||||
// regardless of viewport.
|
||||
let stillPending = false;
|
||||
for (const [wsId, entry] of childrenState) {
|
||||
if (TERMINAL_CHILD_STATES.has(entry.state)) continue;
|
||||
const cached = liveBadgeCache.get(wsId);
|
||||
if (!cached || !cached.live) continue;
|
||||
const detail = cached.live.pending_approval_detail;
|
||||
if (!detail || !detail.judge_pending) continue;
|
||||
const items = Array.isArray(detail.items) ? detail.items : [];
|
||||
const allHaveJudge =
|
||||
items.length > 0 && items.every((it) => it.judge_verdict);
|
||||
if (allHaveJudge) continue;
|
||||
// Bypass scheduleLiveFetch entirely (skips visibility + TTL
|
||||
// gates) by adding to pendingLiveIds directly. flushLiveFetches
|
||||
// will batch every pending row into one request.
|
||||
if (WS_ID_RE.test(wsId)) {
|
||||
pendingLiveIds.add(wsId);
|
||||
stillPending = true;
|
||||
}
|
||||
}
|
||||
if (!stillPending) return; // every verdict landed — done
|
||||
// Cancel any debounce that's still pending so our flush runs
|
||||
// now instead of waiting for it. Then flush directly so the
|
||||
// bulk request fires before the next tick re-arms.
|
||||
if (liveBadgeFlushTimer !== null) {
|
||||
clearTimeout(liveBadgeFlushTimer);
|
||||
liveBadgeFlushTimer = null;
|
||||
}
|
||||
flushLiveFetches();
|
||||
judgePollTimer = setTimeout(_judgePollTick, JUDGE_POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
// Build the inline approval block: severity pill, intent summary +
|
||||
// judge reasoning, and approve/deny buttons. Returns a DOM node or
|
||||
// null if the detail is unusable (defensive \u2014 server is supposed to
|
||||
@@ -2534,6 +2507,37 @@
|
||||
return sub;
|
||||
}
|
||||
|
||||
// Loading-state placeholder rendered while the bulk fetch is
|
||||
// in-flight. Same outer ``.approval-block`` class so the row's
|
||||
// height stays stable when the real content swaps in (no shove of
|
||||
// sibling rows mid-mouse-movement). Carries an aria-label so a
|
||||
// screen reader announces the pending approval; the actual
|
||||
// assertive ``announceApproval`` call in handleChildState fires
|
||||
// once on the rising edge so the operator hears the demand even
|
||||
// before the bulk fetch lands.
|
||||
function renderApprovalPlaceholder(child) {
|
||||
const block = document.createElement("div");
|
||||
block.className = "approval-block approval-block-loading";
|
||||
block.setAttribute("role", "region");
|
||||
block.setAttribute(
|
||||
"aria-label",
|
||||
"Approval required for " +
|
||||
(child.name || child.ws_id || "child") +
|
||||
" — loading details",
|
||||
);
|
||||
const header = document.createElement("div");
|
||||
header.className = "approval-header";
|
||||
const pill = document.createElement("span");
|
||||
pill.className = "approval-pill approval-pill-pending";
|
||||
const spin = document.createElement("span");
|
||||
spin.className = "approval-loading-spin";
|
||||
pill.appendChild(spin);
|
||||
pill.appendChild(document.createTextNode(" loading…"));
|
||||
header.appendChild(pill);
|
||||
block.appendChild(header);
|
||||
return block;
|
||||
}
|
||||
|
||||
function renderApprovalBlock(child, detail) {
|
||||
if (!detail || !Array.isArray(detail.items) || detail.items.length === 0) {
|
||||
return null;
|
||||
@@ -2562,6 +2566,16 @@
|
||||
|
||||
const block = document.createElement("div");
|
||||
block.className = "approval-block";
|
||||
block.setAttribute("role", "region");
|
||||
block.setAttribute(
|
||||
"aria-label",
|
||||
"Approval required for " +
|
||||
(child.name || child.ws_id || "child") +
|
||||
" — " +
|
||||
(primary && primary.header
|
||||
? primary.header
|
||||
: items.length + " tool calls"),
|
||||
);
|
||||
|
||||
// Header line: pill + tool name(s) + tier:model
|
||||
const header = document.createElement("div");
|
||||
@@ -2603,6 +2617,22 @@
|
||||
const conf = verdict.confidence;
|
||||
const confStr = typeof conf === "number" ? " " + conf.toFixed(2) : "";
|
||||
pill.textContent = (verdict.risk_level || "").toUpperCase() + confStr;
|
||||
// SR-friendly label: spell out the risk level + optional
|
||||
// confidence so a screen reader doesn't read "LOW 0.85" as
|
||||
// a string of letters and a number. Visual text stays compact.
|
||||
const riskWord =
|
||||
riskCls === "crit"
|
||||
? "critical"
|
||||
: riskCls === "high"
|
||||
? "high"
|
||||
: riskCls === "med"
|
||||
? "medium"
|
||||
: "low";
|
||||
const confLabel =
|
||||
typeof conf === "number"
|
||||
? ", confidence " + Math.round(conf * 100) + "%"
|
||||
: "";
|
||||
pill.setAttribute("aria-label", riskWord + " risk" + confLabel);
|
||||
}
|
||||
header.appendChild(pill);
|
||||
|
||||
@@ -2803,17 +2833,33 @@
|
||||
// Stale call_id \u2014 server has rolled to a new round, or
|
||||
// (more commonly) the approval was already resolved on
|
||||
// another channel and this click raced. Keep both buttons
|
||||
// disabled until the urgent refresh lands and re-renders
|
||||
// the row: the row is about to be replaced wholesale, so
|
||||
// the disabled DOM is dropped along with it. Re-enabling
|
||||
// here was the bug \u2014 it opened a window where rapid clicks
|
||||
// hit the same already-resolved approval, each producing a
|
||||
// fresh 409, looping until the live-bulk eventually cleared
|
||||
// the row. On the rare path where the urgent refresh fails
|
||||
// entirely, the operator can hit the Refresh button on the
|
||||
// children panel to force a full reload.
|
||||
// disabled until the refresh lands and re-renders the row:
|
||||
// the row is about to be replaced wholesale, so the disabled
|
||||
// DOM is dropped along with it. Re-enabling here was the bug
|
||||
// \u2014 it opened a window where rapid clicks hit the same
|
||||
// already-resolved approval, each producing a fresh 409,
|
||||
// looping until the live-bulk eventually cleared the row.
|
||||
// On the rare path where the refresh fails entirely, the
|
||||
// operator can hit the Refresh button on the children panel
|
||||
// to force a full reload. invalidateLiveBadge clears the
|
||||
// cached entry so the next scheduleLiveFetch falls through
|
||||
// the TTL gate (no cached entry to compare against); the
|
||||
// standard 250ms debounce batches with any other in-flight
|
||||
// pending ids.
|
||||
// Inline note so the operator's "did my click work?" question
|
||||
// gets answered without a noisy toast. Stays in the row until
|
||||
// the refresh replaces the whole approval block.
|
||||
const block = denyBtn.closest(".approval-block");
|
||||
if (block && !block.querySelector(".approval-stale-note")) {
|
||||
const note = document.createElement("div");
|
||||
note.className = "approval-stale-note";
|
||||
note.setAttribute("role", "status");
|
||||
note.textContent =
|
||||
"\u21bb already resolved elsewhere \u2014 refreshing\u2026";
|
||||
block.appendChild(note);
|
||||
}
|
||||
invalidateLiveBadge(targetWsId);
|
||||
scheduleLiveFetch(targetWsId, { urgent: true });
|
||||
scheduleLiveFetch(targetWsId);
|
||||
// Quiet console-warn for diagnostics; no toast \u2014 the
|
||||
// disappearing buttons / fresh row IS the operator-facing
|
||||
// signal, and a toast on every rapid-click 409 would just
|
||||
@@ -2826,14 +2872,30 @@
|
||||
}
|
||||
// Optimistic clear \u2014 the next child_ws_state event will arrive
|
||||
// shortly and trigger a real refresh, but clearing locally
|
||||
// makes the buttons disappear immediately on click.
|
||||
// makes the buttons disappear immediately on click. The
|
||||
// ``sseUpdatedAt`` bump is load-bearing: ``flushLiveFetches``'s
|
||||
// merge guard preserves cleared pending_approval / _detail
|
||||
// against an in-flight bulk fetch only while
|
||||
// ``now - prev.sseUpdatedAt < SSE_AUTHORITATIVE_MS`` \u2014 without
|
||||
// bumping, a bulk fetch landing in the gap reverts the
|
||||
// cleared state from the upstream cache and the approve/deny
|
||||
// pill flickers back. (Caught by /review bug-4.)
|
||||
const cached = liveBadgeCache.get(targetWsId);
|
||||
if (cached && cached.live) {
|
||||
cached.live = Object.assign({}, cached.live, {
|
||||
pending_approval: false,
|
||||
pending_approval_detail: null,
|
||||
});
|
||||
liveBadgeCache.set(targetWsId, cached);
|
||||
cached.sseUpdatedAt = Date.now();
|
||||
_liveBadgeCacheSet(targetWsId, cached);
|
||||
}
|
||||
// Also clear the activity_state mirror so the row's "⚑ approval"
|
||||
// badge in .meta disappears immediately too — without this, the
|
||||
// row shows the badge with no buttons for ~50-150ms until the
|
||||
// child_ws_approval_resolved push or next state event lands.
|
||||
const childState = childrenState.get(targetWsId);
|
||||
if (childState && childState.activity_state === "approval") {
|
||||
childState.activity_state = "";
|
||||
}
|
||||
renderChildren();
|
||||
} catch (e) {
|
||||
@@ -2902,7 +2964,49 @@
|
||||
return _childObserver;
|
||||
}
|
||||
|
||||
// Focus preservation helpers shared by _renderChildrenNow and
|
||||
// _updateChildRow. ``replaceChildren()`` / ``replaceWith()`` blow
|
||||
// away the focused element silently — without restore, the operator
|
||||
// gets bounced to <body> mid-Tab whenever any state event fires for
|
||||
// a row in the children tree.
|
||||
function _captureRowFocusKey(scopeEl) {
|
||||
const active = document.activeElement;
|
||||
if (!active || !scopeEl || !scopeEl.contains(active)) return null;
|
||||
const row = active.closest(".ch-row");
|
||||
if (!row || !row.dataset.wsId) return null;
|
||||
return {
|
||||
wsId: row.dataset.wsId,
|
||||
marker: active.className || active.tagName,
|
||||
};
|
||||
}
|
||||
|
||||
function _restoreRowFocus(scopeEl, focusKey) {
|
||||
if (!focusKey || !scopeEl) return;
|
||||
const sel = '.ch-row[data-ws-id="' + cssEscape(focusKey.wsId) + '"]';
|
||||
const row = scopeEl.matches(sel) ? scopeEl : scopeEl.querySelector(sel);
|
||||
if (!row) return;
|
||||
let target = null;
|
||||
if (focusKey.marker) {
|
||||
// CSS.escape can't safely round-trip a class list with spaces,
|
||||
// so we walk focusables and string-compare. A future refactor
|
||||
// could swap to a stable ``data-focus-key`` attribute on each
|
||||
// focusable element to dodge class-string identity entirely
|
||||
// (see /review bug-2 — currently low risk because all
|
||||
// focusables in renderChildRow / renderApprovalBlock carry
|
||||
// single-class names).
|
||||
const candidates = row.querySelectorAll("button, [tabindex], a, summary");
|
||||
for (const el of candidates) {
|
||||
if ((el.className || el.tagName) === focusKey.marker) {
|
||||
target = el;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (target) target.focus({ preventScroll: true });
|
||||
}
|
||||
|
||||
function _renderChildrenNow() {
|
||||
const focusKey = _captureRowFocusKey(childrenTreeEl);
|
||||
childrenTreeEl.setAttribute("aria-busy", "false");
|
||||
const rows = Array.from(childrenState.values());
|
||||
// Sort: non-terminal states first, then by name.
|
||||
@@ -2934,7 +3038,42 @@
|
||||
else visibleChildIds.add(r.ws_id); // fallback: treat all visible
|
||||
});
|
||||
}
|
||||
childrenCountEl.textContent = rows.length ? "(" + rows.length + ")" : "";
|
||||
// Sidebar count: total + pending-approval annotation. The
|
||||
// pending count is maintained incrementally on cache mutations
|
||||
// (see ``pendingApprovalIds`` near the cache definition) so this
|
||||
// is O(1) per render rather than an O(N) walk over the cache.
|
||||
const pending = pendingApprovalIds.size;
|
||||
childrenCountEl.textContent = rows.length
|
||||
? "(" +
|
||||
rows.length +
|
||||
(pending > 0 ? " · " + pending + " pending" : "") +
|
||||
")"
|
||||
: "";
|
||||
_restoreRowFocus(childrenTreeEl, focusKey);
|
||||
}
|
||||
|
||||
// Targeted single-row update — used by handlers that only touch
|
||||
// one row's state (verdict landing, approval resolved). Avoids
|
||||
// the tree-wide rebuild ``_renderChildrenNow`` does so a 200-child
|
||||
// tree doesn't re-paint 199 unaffected rows on every verdict
|
||||
// arrival. Falls back to the full render if the row isn't in the
|
||||
// DOM yet (first-time render). Preserves keyboard focus across
|
||||
// the row swap — the same invariant ``_renderChildrenNow`` keeps
|
||||
// for tree-wide rebuilds (caught by /review bug-3).
|
||||
function _updateChildRow(childId) {
|
||||
const sel = '.ch-row[data-ws-id="' + cssEscape(childId) + '"]';
|
||||
const row = childrenTreeEl.querySelector(sel);
|
||||
const entry = childrenState.get(childId);
|
||||
if (row && entry) {
|
||||
const focusKey = _captureRowFocusKey(row);
|
||||
const replacement = renderChildRow(entry);
|
||||
row.replaceWith(replacement);
|
||||
const obs = _getChildObserver();
|
||||
if (obs) obs.observe(replacement);
|
||||
_restoreRowFocus(replacement, focusKey);
|
||||
} else {
|
||||
renderChildren();
|
||||
}
|
||||
}
|
||||
|
||||
function renderTaskRow(task) {
|
||||
@@ -3065,15 +3204,9 @@
|
||||
const LIVE_BADGE_BULK_FLUSH_MS = LIVE_BADGE_DEBOUNCE_MS;
|
||||
const pendingLiveIds = new Set();
|
||||
let liveBadgeFlushTimer = null;
|
||||
// Urgent-flush coalesce flag — N urgent calls in the same JS tick
|
||||
// would otherwise issue N single-id bulk fetches (bulk endpoint
|
||||
// accepts up to LIVE_BADGE_BULK_CAP ids per request). queueMicrotask
|
||||
// batches them into one request that drains pendingLiveIds.
|
||||
let urgentFlushScheduled = false;
|
||||
|
||||
function scheduleLiveFetch(childWsId, opts) {
|
||||
function scheduleLiveFetch(childWsId) {
|
||||
if (!childWsId) return;
|
||||
const urgent = !!(opts && opts.urgent);
|
||||
// Skip terminal-state children entirely — their live block will
|
||||
// never change again; fetching just burns a round-trip and caches
|
||||
// a stale value. Renderer already styles closed/deleted rows.
|
||||
@@ -3093,40 +3226,16 @@
|
||||
// change on any child triggers a fresh fetch → retry storm for
|
||||
// users who'll never have permission mid-session.
|
||||
if (cached.permanent) return;
|
||||
// Urgent fetches bypass the TTL — used when a child enters
|
||||
// approval state and the row needs the rich pending_approval_detail
|
||||
// payload (call_id, items, judge_verdict) to render inline buttons.
|
||||
// Waiting for the next 5s TTL window would leave the operator
|
||||
// staring at "⚑ approval" with no way to act.
|
||||
if (!urgent && Date.now() - cached.fetched < LIVE_BADGE_TTL_MS) return;
|
||||
// TTL gate — slower-moving fields (tokens, context_ratio) refresh
|
||||
// on this cadence. Approval state is event-driven (intent_verdict
|
||||
// / approval_resolved push, plus the bulk fetch on initial
|
||||
// approval entry); callers wanting a forced re-fetch
|
||||
// (e.g. 409 stale-call_id retry) call invalidateLiveBadge first
|
||||
// so the cache is empty and this gate falls through.
|
||||
if (Date.now() - cached.fetched < LIVE_BADGE_TTL_MS) return;
|
||||
}
|
||||
if (!WS_ID_RE.test(childWsId)) return;
|
||||
pendingLiveIds.add(childWsId);
|
||||
// Urgent: cancel the pending debounce and schedule a flush on
|
||||
// the next microtask so N urgent calls in the same tick coalesce
|
||||
// into one bulk request. Without the microtask hop, each urgent
|
||||
// caller would drain pendingLiveIds with a single id and fire a
|
||||
// separate fetch — defeating the bulk endpoint that accepts up
|
||||
// to LIVE_BADGE_BULK_CAP ids per request.
|
||||
if (urgent) {
|
||||
if (liveBadgeFlushTimer !== null) {
|
||||
clearTimeout(liveBadgeFlushTimer);
|
||||
liveBadgeFlushTimer = null;
|
||||
}
|
||||
if (!urgentFlushScheduled) {
|
||||
urgentFlushScheduled = true;
|
||||
const flush = () => {
|
||||
urgentFlushScheduled = false;
|
||||
flushLiveFetches();
|
||||
};
|
||||
if (typeof queueMicrotask === "function") {
|
||||
queueMicrotask(flush);
|
||||
} else {
|
||||
setTimeout(flush, 0);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (liveBadgeFlushTimer !== null) return;
|
||||
liveBadgeFlushTimer = setTimeout(() => {
|
||||
liveBadgeFlushTimer = null;
|
||||
@@ -3181,7 +3290,7 @@
|
||||
pending_approval_detail: prev.live.pending_approval_detail,
|
||||
});
|
||||
}
|
||||
liveBadgeCache.set(id, {
|
||||
_liveBadgeCacheSet(id, {
|
||||
live: mergedLive,
|
||||
fetched: now,
|
||||
// Denied ids are permission/identity misses — mark permanent
|
||||
@@ -3209,7 +3318,7 @@
|
||||
const now = Date.now();
|
||||
ids.forEach((id) => {
|
||||
const prev = liveBadgeCache.get(id);
|
||||
liveBadgeCache.set(id, {
|
||||
_liveBadgeCacheSet(id, {
|
||||
live: null,
|
||||
fetched: now,
|
||||
permanent: isPermanent,
|
||||
@@ -3221,7 +3330,7 @@
|
||||
}
|
||||
|
||||
function invalidateLiveBadge(childWsId) {
|
||||
liveBadgeCache.delete(childWsId);
|
||||
_liveBadgeCacheDelete(childWsId);
|
||||
}
|
||||
|
||||
// --- SSE handlers for child_ws_* events ----------------------------
|
||||
@@ -3261,44 +3370,54 @@
|
||||
if (ev.node_id) existing.node_id = ev.node_id;
|
||||
childrenState.set(childId, existing);
|
||||
_touchChild(childId);
|
||||
// pending_approval_detail rides on the ws_state event when an
|
||||
// approval is pending (see turnstone/server.py
|
||||
// WebUI._broadcast_state — gated on ``_pending_approval is not
|
||||
// None``, so absent on the steady state and possibly null on a
|
||||
// node mid-rolling-upgrade). When present we mutate
|
||||
// liveBadgeCache directly here — inline approve/deny buttons
|
||||
// render in lockstep with the activity_state transition without
|
||||
// a separate live-bulk fetch. The cache entry is tagged
|
||||
// ``sseUpdatedAt`` so a bulk-poll landing within
|
||||
// SSE_AUTHORITATIVE_MS preserves the SSE-supplied fields
|
||||
// (the upstream /dashboard cache's ~2s TTL would otherwise
|
||||
// clobber a fresh transition with pre-transition state).
|
||||
const pendingApproval = existing.activity_state === "approval";
|
||||
const evDetail =
|
||||
ev.pending_approval_detail !== undefined
|
||||
? ev.pending_approval_detail
|
||||
: null;
|
||||
// Track ``pending_approval`` flag from BOTH state and
|
||||
// activity_state. ``state="attention"`` is the canonical signal
|
||||
// that the workstream needs operator attention; ``activity_state
|
||||
// ="approval"`` is the secondary signal set by approve_tools.
|
||||
// We trust either: in practice the worker thread can fire the
|
||||
// state transition before approve_tools has updated activity_state
|
||||
// (the two writes happen on different lines under different
|
||||
// locks), so a state-attention event with empty activity_state
|
||||
// is a real and common case — checking only activity_state
|
||||
// misses 30+ children all sitting in attention. (Caught manual
|
||||
// testing: 30 children all state=attention rendered with no
|
||||
// approval blocks because pendingApproval was always false.)
|
||||
const pendingApproval =
|
||||
existing.state === "attention" || existing.activity_state === "approval";
|
||||
const cached = liveBadgeCache.get(childId);
|
||||
// When pending, prefer SSE-supplied detail. If SSE didn't
|
||||
// carry it (rare race; e.g. a node mid-rolling-upgrade), keep
|
||||
// any existing cached detail rather than blanking the row —
|
||||
// the next bulk-poll catches up. When not pending, hard-clear.
|
||||
let nextDetail;
|
||||
if (pendingApproval) {
|
||||
nextDetail =
|
||||
evDetail !== null
|
||||
? evDetail
|
||||
: cached && cached.live
|
||||
? cached.live.pending_approval_detail
|
||||
: null;
|
||||
} else {
|
||||
nextDetail = null;
|
||||
const cachedLive = (cached && cached.live) || {};
|
||||
// Rising-edge detection BEFORE we mutate the cache. The chat-pane
|
||||
// tool batches already announce assertively
|
||||
// (renderApprovalDock / appendToolBatch); the children-tree was
|
||||
// silent for SR users — fixing that here so a blind operator
|
||||
// hears the demand for action.
|
||||
const wasPendingApproval = cachedLive.pending_approval === true;
|
||||
if (pendingApproval && !wasPendingApproval) {
|
||||
_announceAssertive(
|
||||
"Approval required: " + (existing.name || childId.slice(0, 8)),
|
||||
);
|
||||
}
|
||||
const nextLive = Object.assign({}, (cached && cached.live) || {}, {
|
||||
const nextLive = Object.assign({}, cachedLive, {
|
||||
pending_approval: pendingApproval,
|
||||
pending_approval_detail: nextDetail,
|
||||
});
|
||||
liveBadgeCache.set(childId, {
|
||||
// Off-approval transition is the ONLY case here that
|
||||
// authoritatively writes a value the bulk fetch must not
|
||||
// resurrect (a stale bulk-fetch landing within
|
||||
// SSE_AUTHORITATIVE_MS would otherwise re-render the cleared
|
||||
// approval block). Setting ``pending_approval=true`` does NOT
|
||||
// claim cache authority — the bulk fetch is the source of
|
||||
// truth for ``pending_approval_detail``, and bumping
|
||||
// sseUpdatedAt here makes the merge guard in flushLiveFetches
|
||||
// preserve our stale (often null) detail over the bulk
|
||||
// fetch's actual data, leaving the row stuck on the loading
|
||||
// placeholder. (Caught when the screenshot showed buttons
|
||||
// briefly then loading replaced them.)
|
||||
const detailClearedAuthoritatively =
|
||||
!pendingApproval && cachedLive.pending_approval === true;
|
||||
if (!pendingApproval) {
|
||||
nextLive.pending_approval_detail = null;
|
||||
}
|
||||
_liveBadgeCacheSet(childId, {
|
||||
live: nextLive,
|
||||
// Preserve prior bulk-poll fetched timestamp so a fresh SSE
|
||||
// tick doesn't artificially extend the 5s TTL gate in
|
||||
@@ -3306,7 +3425,11 @@
|
||||
// moving fields (tokens, context_ratio) on its own schedule.
|
||||
fetched: cached ? cached.fetched : 0,
|
||||
permanent: !!(cached && cached.permanent),
|
||||
sseUpdatedAt: Date.now(),
|
||||
sseUpdatedAt: detailClearedAuthoritatively
|
||||
? Date.now()
|
||||
: cached
|
||||
? cached.sseUpdatedAt || 0
|
||||
: 0,
|
||||
});
|
||||
renderChildren();
|
||||
// Do NOT invalidateLiveBadge on routine state ticks — that
|
||||
@@ -3344,6 +3467,104 @@
|
||||
renderChildren();
|
||||
}
|
||||
|
||||
// Stage 3 Step 7 — verdict + approval reducer.
|
||||
//
|
||||
// Both write directly to liveBadgeCache and skip scheduleLiveFetch,
|
||||
// which sidesteps the visibility gate in that path so off-screen
|
||||
// rows pick up the new cache value the moment they scroll back in.
|
||||
// Both are idempotent — the bulk-fetch (single source of truth for
|
||||
// the items list) and the explicit intent_verdict event can deliver
|
||||
// overlapping data; receiving twice re-stamps the same value.
|
||||
//
|
||||
// Both call ``_updateChildRow`` (targeted single-row swap) instead
|
||||
// of ``renderChildren`` so a verdict landing on row 3 doesn't rebuild
|
||||
// every other row in a 200-child sidebar — which would also blow
|
||||
// away keyboard focus on whatever row the operator was tabbing to.
|
||||
|
||||
function handleChildIntentVerdict(ev) {
|
||||
const childId = ev.child_ws_id || ev.ws_id;
|
||||
if (!childId) return;
|
||||
const verdict = ev.verdict || {};
|
||||
const callId = verdict.call_id || "";
|
||||
if (!callId) return;
|
||||
const cached = liveBadgeCache.get(childId);
|
||||
const cachedLive = (cached && cached.live) || {};
|
||||
const detail = cachedLive.pending_approval_detail;
|
||||
if (!detail) {
|
||||
// No pending_approval_detail to stamp the verdict onto. The
|
||||
// verdict is still durable in storage; the next bulk fetch
|
||||
// will hydrate the detail and include the verdict via the
|
||||
// existing serialize path.
|
||||
return;
|
||||
}
|
||||
// Stamp on the matching item (UI render reads judge_verdict per
|
||||
// item) AND on the by-call_id map (matches the
|
||||
// serialize_pending_approval_detail shape).
|
||||
const items = Array.isArray(detail.items) ? detail.items : [];
|
||||
for (const item of items) {
|
||||
if (item && item.call_id === callId) {
|
||||
item.judge_verdict = verdict;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!detail.llm_verdicts) detail.llm_verdicts = {};
|
||||
detail.llm_verdicts[callId] = verdict;
|
||||
// judge_pending flips false once every item has a verdict —
|
||||
// matches the server-side serializer's logic.
|
||||
if (items.length > 0) {
|
||||
detail.judge_pending = !items.every((it) => it && it.judge_verdict);
|
||||
}
|
||||
_liveBadgeCacheSet(childId, {
|
||||
live: cachedLive,
|
||||
fetched: cached ? cached.fetched : 0,
|
||||
permanent: !!(cached && cached.permanent),
|
||||
sseUpdatedAt: Date.now(),
|
||||
});
|
||||
_updateChildRow(childId);
|
||||
}
|
||||
|
||||
function handleChildApprovalResolved(ev) {
|
||||
const childId = ev.child_ws_id || ev.ws_id;
|
||||
if (!childId) return;
|
||||
const cached = liveBadgeCache.get(childId);
|
||||
const cachedLive = (cached && cached.live) || {};
|
||||
cachedLive.pending_approval = false;
|
||||
cachedLive.pending_approval_detail = null;
|
||||
_liveBadgeCacheSet(childId, {
|
||||
live: cachedLive,
|
||||
fetched: cached ? cached.fetched : 0,
|
||||
permanent: !!(cached && cached.permanent),
|
||||
sseUpdatedAt: Date.now(),
|
||||
});
|
||||
_updateChildRow(childId);
|
||||
}
|
||||
|
||||
// Push path for the initial approval items — eliminates the
|
||||
// bulk-fetch race that previously left rows stuck on a loading
|
||||
// placeholder when the bulk fetch landed in the gap between the
|
||||
// state transition to ATTENTION and ``_pending_approval`` being
|
||||
// set inside ``approve_tools``. Stamps the items into the cache
|
||||
// directly; the bulk fetch remains as a reconnect / refresh
|
||||
// fallback. Idempotent — receiving the same approve_request twice
|
||||
// re-stamps the same detail.
|
||||
function handleChildApproveRequest(ev) {
|
||||
const childId = ev.child_ws_id || ev.ws_id;
|
||||
if (!childId) return;
|
||||
const detail = ev.detail || null;
|
||||
if (!detail) return;
|
||||
const cached = liveBadgeCache.get(childId);
|
||||
const cachedLive = (cached && cached.live) || {};
|
||||
cachedLive.pending_approval = true;
|
||||
cachedLive.pending_approval_detail = detail;
|
||||
_liveBadgeCacheSet(childId, {
|
||||
live: cachedLive,
|
||||
fetched: cached ? cached.fetched : 0,
|
||||
permanent: !!(cached && cached.permanent),
|
||||
sseUpdatedAt: Date.now(),
|
||||
});
|
||||
_updateChildRow(childId);
|
||||
}
|
||||
|
||||
// Periodic sweep of stale terminal rows. Operator tabs left open all
|
||||
// day would otherwise accumulate entries for every child the
|
||||
// coordinator ever spawned — rows the user can still see (state !=
|
||||
@@ -3359,7 +3580,7 @@
|
||||
if (terminal && now - lastSeen > CHILDREN_TERMINAL_GRACE_MS) {
|
||||
childrenState.delete(id);
|
||||
childrenLastSeen.delete(id);
|
||||
liveBadgeCache.delete(id);
|
||||
_liveBadgeCacheDelete(id);
|
||||
visibleChildIds.delete(id);
|
||||
removed += 1;
|
||||
}
|
||||
@@ -3375,7 +3596,7 @@
|
||||
const id = byAge[i][0];
|
||||
childrenState.delete(id);
|
||||
childrenLastSeen.delete(id);
|
||||
liveBadgeCache.delete(id);
|
||||
_liveBadgeCacheDelete(id);
|
||||
visibleChildIds.delete(id);
|
||||
removed += 1;
|
||||
}
|
||||
@@ -3388,7 +3609,7 @@
|
||||
|
||||
if (childrenRefreshBtn) {
|
||||
childrenRefreshBtn.addEventListener("click", () => {
|
||||
liveBadgeCache.clear();
|
||||
_liveBadgeCacheClear();
|
||||
// Explicit refresh wipes SSE-discovered rows the server no
|
||||
// longer knows about — the operator asked for a clean snapshot.
|
||||
loadChildren({ replace: true });
|
||||
|
||||
@@ -182,6 +182,36 @@
|
||||
font-size: 11px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
/* Loading-state placeholder rendered while the bulk fetch is
|
||||
in-flight. Same outer container as the real block so the row
|
||||
height is stable when the real content swaps in. */
|
||||
.ch-row .approval-block-loading {
|
||||
color: var(--ink-3);
|
||||
}
|
||||
.ch-row .approval-loading-spin {
|
||||
display: inline-block;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
border: 1.5px solid var(--accent);
|
||||
border-top-color: transparent;
|
||||
margin-right: 5px;
|
||||
animation: ts-spin 0.9s linear infinite;
|
||||
vertical-align: middle;
|
||||
}
|
||||
/* Inline status note shown when the operator clicks Approve/Deny
|
||||
but the call_id is stale (already resolved on another channel).
|
||||
Quieter than a toast — the row is about to be replaced wholesale
|
||||
by the refresh, so this only needs to bridge ~350ms. */
|
||||
.ch-row .approval-stale-note {
|
||||
font-size: 11px;
|
||||
color: var(--ink-3);
|
||||
font-style: italic;
|
||||
margin-top: 4px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ch-row .approval-loading-spin { animation: none; }
|
||||
}
|
||||
/* Auto-approved pill — same 22px indent as .approval-block / .meta
|
||||
so a child row showing both stacks cleanly. Lower visual weight
|
||||
than the live approve/deny block (this is informational, not
|
||||
|
||||
@@ -165,6 +165,14 @@
|
||||
<h2 class="home-section-title">
|
||||
<span>Saved Coordinators</span>
|
||||
<span id="saved-coord-count" class="home-section-count"></span>
|
||||
<button
|
||||
id="coord-delete-btn"
|
||||
class="ws-delete-btn home-section-action"
|
||||
onclick="startCoordDeleteMode()"
|
||||
title="Delete coordinators"
|
||||
>
|
||||
<span aria-hidden="true">🗑</span> Delete
|
||||
</button>
|
||||
</h2>
|
||||
<div
|
||||
id="saved-coord-cards"
|
||||
@@ -172,6 +180,61 @@
|
||||
role="list"
|
||||
aria-live="polite"
|
||||
></div>
|
||||
<div
|
||||
id="coord-pagination"
|
||||
class="pagination"
|
||||
style="display: none"
|
||||
role="navigation"
|
||||
aria-label="Saved coordinators pagination"
|
||||
>
|
||||
<button
|
||||
id="coord-page-prev"
|
||||
type="button"
|
||||
onclick="coordPagePrev()"
|
||||
>
|
||||
◄ Prev
|
||||
</button>
|
||||
<span id="coord-page-label" aria-live="polite" aria-atomic="true">
|
||||
</span>
|
||||
<button
|
||||
id="coord-page-next"
|
||||
type="button"
|
||||
onclick="coordPageNext()"
|
||||
>
|
||||
Next ►
|
||||
</button>
|
||||
</div>
|
||||
<div id="coord-delete-bar" class="ws-delete-bar">
|
||||
<span
|
||||
class="ws-delete-count-label"
|
||||
id="coord-delete-bar-count"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
aria-atomic="true"
|
||||
>0 selected</span
|
||||
>
|
||||
<button
|
||||
class="ws-delete-cancel-btn"
|
||||
onclick="cancelCoordDeleteMode()"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
class="ws-delete-selectall-btn"
|
||||
id="coord-delete-bar-select-all"
|
||||
onclick="toggleCoordSelectAll()"
|
||||
>
|
||||
Select All
|
||||
</button>
|
||||
<button
|
||||
class="ws-delete-bar-btn"
|
||||
id="coord-delete-bar-delete"
|
||||
onclick="confirmCoordDeleteSelection()"
|
||||
disabled
|
||||
>
|
||||
Delete Selected
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Cluster details — node list, always visible. The list is
|
||||
@@ -3907,6 +3970,40 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Delete coordinators confirmation modal (batch) -->
|
||||
<div
|
||||
id="coord-delete-overlay"
|
||||
class="ws-delete-modal-overlay"
|
||||
style="display: none"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="coord-delete-title"
|
||||
>
|
||||
<div id="coord-delete-box" class="ws-delete-modal-box">
|
||||
<h3 id="coord-delete-title">Delete Coordinators</h3>
|
||||
<div id="coord-delete-error" role="alert" aria-live="assertive"></div>
|
||||
<p id="coord-delete-count"></p>
|
||||
<div id="coord-delete-list" class="ws-delete-modal-list"></div>
|
||||
<div id="coord-delete-buttons" class="ws-delete-modal-buttons">
|
||||
<button
|
||||
id="coord-delete-cancel-btn"
|
||||
type="button"
|
||||
onclick="cancelCoordDelete()"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
id="coord-delete-confirm-btn"
|
||||
class="ws-delete-confirm"
|
||||
type="button"
|
||||
onclick="confirmCoordDelete()"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="/static/admin.js"></script>
|
||||
<script src="/static/governance.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
|
||||
@@ -133,6 +133,17 @@
|
||||
font-size: 11px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
/* Right-align action buttons (e.g. Saved Coordinators "Delete") inside
|
||||
.home-section-title without breaking the count's natural left position. */
|
||||
.home-section-title .home-section-action {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
/* Saved Coordinators reuses the existing .pagination control (see the
|
||||
"Pagination" block below) — the visible page is capped at
|
||||
COORD_PAGE_SIZE so Select-All fan-out is bounded. Pagination is
|
||||
hidden in delete mode and when there's only one page (see
|
||||
_renderCoordPagination). */
|
||||
|
||||
.home-coord-list {
|
||||
border: 1px solid var(--border);
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Strategy interface for delivering child workstream lifecycle events.
|
||||
|
||||
Two implementations bind to the unified :class:`ChildrenRegistry`:
|
||||
|
||||
- :class:`SameNodeChildSource` — subscribes to a :class:`SessionManager`'s
|
||||
state-change callbacks. In-process, no transport. For interactive
|
||||
workstreams that spawn children locally (no cluster routing).
|
||||
- :class:`ClusterChildSource` — subscribes to a :class:`ClusterCollector`'s
|
||||
listener channel and runs a daemon thread that drains the queue and
|
||||
pushes events to the sink. For coordinator workstreams whose
|
||||
children are routed across the cluster by hash bucket.
|
||||
|
||||
The strategy doesn't translate events into UI-shaped payloads; that's
|
||||
the sink's job. This split keeps the strategy generic across kinds and
|
||||
lets the consumer (e.g. ``CoordinatorAdapter._dispatch_child_event``)
|
||||
own the per-kind translation.
|
||||
|
||||
Sink signature: ``Callable[[dict[str, Any]], None]``. The sink is
|
||||
responsible for filtering by registry membership; strategies push raw
|
||||
events without registry-side filtering so the sink can decide whether
|
||||
to act based on its own state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from turnstone.core.children_registry import ChildrenRegistry
|
||||
from turnstone.core.workstream import WorkstreamState
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class ChildSource(Protocol):
|
||||
"""Subscription strategy for child workstream lifecycle events."""
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
"""Begin delivering events to ``sink``. Idempotent."""
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the strategy. Idempotent; safe to call multiple times."""
|
||||
|
||||
|
||||
class _CollectorProtocol(Protocol):
|
||||
"""Subset of :class:`ClusterCollector` that :class:`ClusterChildSource` consumes.
|
||||
|
||||
Defined here (not imported) to keep ``turnstone/core/`` free of
|
||||
``turnstone/console/`` imports AND to let test fakes satisfy the
|
||||
type signature without subclassing the real collector.
|
||||
"""
|
||||
|
||||
def get_snapshot_and_register(self, q: queue.Queue[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Register ``q`` and return the current snapshot."""
|
||||
|
||||
def unregister_listener(self, q: queue.Queue[dict[str, Any]]) -> None:
|
||||
"""Drop ``q`` from the listener set."""
|
||||
|
||||
|
||||
class _ManagerProtocol(Protocol):
|
||||
"""Subset of :class:`SessionManager` that :class:`SameNodeChildSource` consumes.
|
||||
|
||||
Lets test fakes participate in the strategy's typed surface
|
||||
without forcing a full SessionManager construction.
|
||||
"""
|
||||
|
||||
def subscribe_to_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Register ``callback`` for state-change events."""
|
||||
|
||||
def unsubscribe_from_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Remove a previously-registered ``callback``."""
|
||||
|
||||
|
||||
class SameNodeChildSource:
|
||||
"""In-process child events via :class:`SessionManager` state observer.
|
||||
|
||||
Subscribes to the manager's state-change callbacks (registered via
|
||||
:meth:`SessionManager.subscribe_to_state`). For each transition on
|
||||
a workstream that's a known child (per the
|
||||
:class:`ChildrenRegistry` reverse index), synthesises a
|
||||
cluster-state-shaped event and pushes it to the sink.
|
||||
|
||||
Used by interactive workstreams when they gain spawn capability —
|
||||
children live on the same node as the parent, so no cluster routing
|
||||
is needed and event fan-out is in-process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
manager: _ManagerProtocol,
|
||||
registry: ChildrenRegistry,
|
||||
) -> None:
|
||||
self._manager = manager
|
||||
self._registry = registry
|
||||
self._sink: Callable[[dict[str, Any]], None] | None = None
|
||||
self._callback: Callable[[str, WorkstreamState], None] | None = None
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
if self._callback is not None:
|
||||
return # idempotent — already started
|
||||
self._sink = sink
|
||||
|
||||
def _on_state(ws_id: str, state: WorkstreamState) -> None:
|
||||
sink_fn = self._sink
|
||||
if sink_fn is None:
|
||||
return
|
||||
# Cheap pre-filter: skip dispatch for transitions on
|
||||
# workstreams that aren't children of any in-memory parent.
|
||||
# ``has_children`` is a lock-free dict-truthiness read; the
|
||||
# ``parent_for`` call below would otherwise acquire the
|
||||
# registry lock on every state change even when no
|
||||
# children exist (the steady state for an interactive
|
||||
# manager). The cluster strategy can't pre-filter because
|
||||
# the collector queue carries all events; here we have the
|
||||
# information to skip the synthesis entirely.
|
||||
if not self._registry.has_children():
|
||||
return
|
||||
if self._registry.parent_for(ws_id) is None:
|
||||
return
|
||||
# ``pending_approval_detail`` deliberately omitted — the
|
||||
# field was removed from cluster_state end-to-end in the
|
||||
# Stage 3 cleanup pass. Approval items arrive via bulk
|
||||
# fetch; verdicts via the explicit intent_verdict event;
|
||||
# resolution via approval_resolved.
|
||||
event = {
|
||||
"type": "cluster_state",
|
||||
"ws_id": ws_id,
|
||||
"state": state.value,
|
||||
"node_id": "",
|
||||
"tokens": 0,
|
||||
"activity_state": "",
|
||||
}
|
||||
try:
|
||||
sink_fn(event)
|
||||
except Exception:
|
||||
log.debug("same_node_child_source.sink_failed", exc_info=True)
|
||||
|
||||
self._callback = _on_state
|
||||
self._manager.subscribe_to_state(_on_state)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
cb = self._callback
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
self._manager.unsubscribe_from_state(cb)
|
||||
except Exception:
|
||||
log.debug("same_node_child_source.unsubscribe_failed", exc_info=True)
|
||||
self._callback = None
|
||||
self._sink = None
|
||||
|
||||
|
||||
class ClusterChildSource:
|
||||
"""Cross-node child events via :class:`ClusterCollector` subscription.
|
||||
|
||||
Refactor of the existing fan-out machinery from
|
||||
``CoordinatorAdapter`` (was ``_collector_queue`` +
|
||||
``_fanout_thread`` + ``_fanout_loop``). Subscribes as a listener on
|
||||
the collector's broadcast channel and runs a daemon thread that
|
||||
drains the queue, pushing each event to the sink.
|
||||
|
||||
On :meth:`start`, also primes the registry from the collector's
|
||||
snapshot so a parent that re-installs after a console restart sees
|
||||
its already-live children without waiting for the next state tick.
|
||||
The ``parents_provider`` callback returns the set of in-memory
|
||||
parent ws_ids for snapshot filtering — only children whose parent
|
||||
is currently installed get merged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
collector: _CollectorProtocol,
|
||||
registry: ChildrenRegistry,
|
||||
*,
|
||||
parents_provider: Callable[[], Iterable[str]],
|
||||
) -> None:
|
||||
self._collector = collector
|
||||
self._registry = registry
|
||||
self._parents_provider = parents_provider
|
||||
self._sink: Callable[[dict[str, Any]], None] | None = None
|
||||
self._queue: queue.Queue[dict[str, Any]] | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self, sink: Callable[[dict[str, Any]], None]) -> None:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return # idempotent — already started
|
||||
self._sink = sink
|
||||
self._queue = queue.Queue(maxsize=1000)
|
||||
snapshot = self._collector.get_snapshot_and_register(self._queue)
|
||||
self._prime_from_snapshot(snapshot)
|
||||
self._stop.clear()
|
||||
t = threading.Thread(
|
||||
target=self._loop,
|
||||
name="cluster-child-source",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread = t
|
||||
t.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._stop.set()
|
||||
t = self._thread
|
||||
q = self._queue
|
||||
coll = self._collector
|
||||
self._thread = None
|
||||
self._queue = None
|
||||
if coll is not None and q is not None:
|
||||
try:
|
||||
coll.unregister_listener(q)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"cluster_child_source.unregister_listener_failed",
|
||||
exc_info=True,
|
||||
)
|
||||
if t is not None:
|
||||
t.join(timeout=2.0)
|
||||
self._sink = None
|
||||
|
||||
def _loop(self) -> None:
|
||||
q = self._queue
|
||||
if q is None:
|
||||
return
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
event = q.get(timeout=1.0)
|
||||
except queue.Empty:
|
||||
continue
|
||||
sink = self._sink
|
||||
if sink is None:
|
||||
continue
|
||||
try:
|
||||
sink(event)
|
||||
except Exception:
|
||||
log.debug("cluster_child_source.dispatch_failed", exc_info=True)
|
||||
|
||||
def _prime_from_snapshot(self, snapshot: dict[str, Any]) -> None:
|
||||
"""Populate the registry from a collector snapshot.
|
||||
|
||||
For every workstream in the snapshot whose ``parent_ws_id``
|
||||
names a currently-installed parent (per ``parents_provider``),
|
||||
merge it into the registry. Caller-installed parents that
|
||||
appear in the snapshot are seeded; unknown parents are skipped
|
||||
— they'll be picked up by the live fan-out path once their
|
||||
``ws_created`` event arrives.
|
||||
"""
|
||||
nodes = snapshot.get("nodes", []) if isinstance(snapshot, dict) else []
|
||||
if not nodes:
|
||||
return
|
||||
known_parents = set(self._parents_provider())
|
||||
if not known_parents:
|
||||
return
|
||||
by_parent: dict[str, list[str]] = {}
|
||||
for node in nodes:
|
||||
for entry in node.get("workstreams", []) or []:
|
||||
parent = entry.get("parent_ws_id") or ""
|
||||
child_id = entry.get("id") or ""
|
||||
if not parent or not child_id or parent not in known_parents:
|
||||
continue
|
||||
by_parent.setdefault(parent, []).append(child_id)
|
||||
for parent, kids in by_parent.items():
|
||||
self._registry.merge_children(parent, kids)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Universal parent → children registry for SessionManager.
|
||||
|
||||
Pure data + lookups; no IO, no transport. Lifted from
|
||||
:class:`turnstone.console.coordinator_adapter.CoordinatorAdapter` where
|
||||
it lived bound to the coordinator kind. The lift is what lets the
|
||||
``ChildSource`` strategies (Step 2) plug into a single shared primitive
|
||||
regardless of whether children are local (interactive) or cluster-routed
|
||||
(coordinator).
|
||||
|
||||
Storage rebuild and snapshot priming happen in the caller — typically
|
||||
the ``ChildSource`` implementation that owns the relevant transport.
|
||||
The registry exposes :meth:`merge_children` for bulk seeding so callers
|
||||
that compute child id lists from any source can feed them in without
|
||||
the registry needing to know about storage shapes or collector
|
||||
snapshots.
|
||||
|
||||
Threading: every public method is internally locked. Helpers suffixed
|
||||
``_locked`` require the caller to already hold :attr:`_lock`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class ChildrenRegistry:
|
||||
"""Tracks parent → children + reverse lookup for in-memory parents.
|
||||
|
||||
The forward index (``_children``) is parent_ws_id → set of child ws_ids.
|
||||
The reverse index (``_child_to_parent``) is child_ws_id → parent_ws_id.
|
||||
The presence map (``_active``) is parent_ws_id → UI ref, used by the
|
||||
dispatch path to atomically check-and-route in one lock acquisition.
|
||||
|
||||
Closed / deleted children stay in the registry until their owning
|
||||
parent is uninstalled — the tree UI keeps rendering them grayed out;
|
||||
state authority lives in storage, not here.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._children: dict[str, set[str]] = {}
|
||||
self._child_to_parent: dict[str, str] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._active: dict[str, Any] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle — install / uninstall a parent
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def install(self, parent_ws_id: str, ui: Any) -> None:
|
||||
"""Seed the forward set + presence map for a new parent.
|
||||
|
||||
Idempotent — re-installing re-points the UI but leaves the
|
||||
existing child set intact. Mirrors the original
|
||||
``_install_coord_registry`` semantics so a coordinator that
|
||||
rehydrates after a crash doesn't lose its known-children.
|
||||
"""
|
||||
with self._lock:
|
||||
self._children.setdefault(parent_ws_id, set())
|
||||
self._active[parent_ws_id] = ui
|
||||
|
||||
def uninstall(self, parent_ws_id: str) -> None:
|
||||
"""Drop a parent: forward set, reverse-index entries, presence.
|
||||
|
||||
No-op if the parent is unknown. Used by close / eviction paths.
|
||||
"""
|
||||
with self._lock:
|
||||
self._uninstall_locked(parent_ws_id)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Mutation — register children under a parent
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_child(self, parent_ws_id: str, child_ws_id: str) -> Any | None:
|
||||
"""Register a child under a parent. Returns parent's UI or None.
|
||||
|
||||
Returns the parent's UI on success (so the dispatch path can
|
||||
atomically check-and-route in one lock acquisition). Returns
|
||||
``None`` if the parent isn't installed (concurrent close /
|
||||
eviction) or if the child is already registered (duplicate
|
||||
ws_created from the cluster fan-out).
|
||||
"""
|
||||
with self._lock:
|
||||
ui = self._active.get(parent_ws_id)
|
||||
if ui is None:
|
||||
return None
|
||||
existing = self._children.setdefault(parent_ws_id, set())
|
||||
if child_ws_id in existing:
|
||||
return None
|
||||
existing.add(child_ws_id)
|
||||
self._child_to_parent[child_ws_id] = parent_ws_id
|
||||
return ui
|
||||
|
||||
def merge_children(self, parent_ws_id: str, child_ws_ids: Iterable[str]) -> None:
|
||||
"""Bulk-merge child_ids under a parent. Idempotent.
|
||||
|
||||
Sole bulk write-path — used by both storage-seeded rebuilds and
|
||||
snapshot-seeded priming so reverse-index ordering invariants
|
||||
hold regardless of which seed source races first.
|
||||
"""
|
||||
with self._lock:
|
||||
self._merge_locked(parent_ws_id, child_ws_ids)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lookups
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def parent_for(self, child_ws_id: str) -> str | None:
|
||||
"""Reverse lookup: which parent owns this child? O(1)."""
|
||||
with self._lock:
|
||||
return self._child_to_parent.get(child_ws_id)
|
||||
|
||||
def has_children(self) -> bool:
|
||||
"""Lock-free fast path: any child registered under any parent?
|
||||
|
||||
Reads ``bool(self._child_to_parent)`` without taking the lock.
|
||||
Dict-truthiness is a single GIL-atomic read, so callers on
|
||||
the hot state-broadcast path can short-circuit without paying
|
||||
the lock acquisition when the registry is empty (the steady
|
||||
state for an interactive manager today). The answer is best-
|
||||
effort — if a child is added concurrently with the read the
|
||||
caller may falsely return ``False``, but the next state event
|
||||
will pick up the change correctly.
|
||||
"""
|
||||
return bool(self._child_to_parent)
|
||||
|
||||
def children_of(self, parent_ws_id: str) -> list[str]:
|
||||
"""Snapshot copy of the parent's child ws_ids.
|
||||
|
||||
Returned list is a copy so callers can iterate without holding
|
||||
the registry lock during per-child work. A mutation racing with
|
||||
the snapshot either lands before (included) or after (excluded)
|
||||
— both outcomes are safe for cascade-style dispatch.
|
||||
"""
|
||||
with self._lock:
|
||||
child_set = self._children.get(parent_ws_id)
|
||||
return list(child_set) if child_set else []
|
||||
|
||||
def ui_for(self, parent_ws_id: str) -> Any | None:
|
||||
"""Look up the UI registered for a parent."""
|
||||
with self._lock:
|
||||
return self._active.get(parent_ws_id)
|
||||
|
||||
def parents(self) -> list[str]:
|
||||
"""Snapshot copy of installed parent ws_ids."""
|
||||
with self._lock:
|
||||
return list(self._active)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Locked helpers — caller must hold ``self._lock``
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_locked(self, parent_ws_id: str, child_ws_ids: Iterable[str]) -> None:
|
||||
"""Idempotent merge under caller's lock. Empty/falsy ids skipped."""
|
||||
existing = self._children.setdefault(parent_ws_id, set())
|
||||
for cid in child_ws_ids:
|
||||
if cid and cid not in existing:
|
||||
existing.add(cid)
|
||||
self._child_to_parent[cid] = parent_ws_id
|
||||
|
||||
def _uninstall_locked(self, parent_ws_id: str) -> None:
|
||||
"""Pop forward set + presence + own reverse-index entries.
|
||||
|
||||
Defensive: only clears reverse entries that still point at
|
||||
``parent_ws_id``. Schema-shaped reassignments (rare but
|
||||
possible) shouldn't orphan the new owner's entry.
|
||||
"""
|
||||
child_set = self._children.pop(parent_ws_id, None)
|
||||
self._active.pop(parent_ws_id, None)
|
||||
if child_set is None:
|
||||
return
|
||||
for cid in child_set:
|
||||
if self._child_to_parent.get(cid) == parent_ws_id:
|
||||
self._child_to_parent.pop(cid, None)
|
||||
@@ -13,6 +13,7 @@ import contextlib
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
@@ -28,6 +29,22 @@ if TYPE_CHECKING:
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# Maps each workstream kind to the ``services.service_type`` its hosting
|
||||
# process registers under. Used by ``SessionManager.close_idle`` pass 2
|
||||
# to enumerate live peer processes for orphan-reaper liveness scoping.
|
||||
# Server processes register as ``("server", node_id, ...)`` (see
|
||||
# ``turnstone/server.py``); the console process as ``("console",
|
||||
# "console", ...)`` (see ``turnstone/console/server.py``). Deriving from
|
||||
# kind here removes a duplicated-config footgun: any caller that builds
|
||||
# a ``SessionManager`` automatically gets the correct service_type for
|
||||
# its kind, with no risk of miswiring INTERACTIVE→"console" or vice
|
||||
# versa.
|
||||
_KIND_SERVICE_TYPE: dict[WorkstreamKind, str] = {
|
||||
WorkstreamKind.INTERACTIVE: "server",
|
||||
WorkstreamKind.COORDINATOR: "console",
|
||||
}
|
||||
|
||||
|
||||
class SessionKindAdapter(Protocol):
|
||||
"""Per-kind construction + cleanup policies the shared ``SessionManager`` delegates to.
|
||||
|
||||
@@ -197,13 +214,21 @@ class SessionManager:
|
||||
# manager never reads them.
|
||||
self._active_id: str | None = None
|
||||
self._eviction_count: int = 0
|
||||
# Optional state-change observer. The CLI sets this to a
|
||||
# callback that prints a background-attention notification
|
||||
# when a non-focused workstream transitions to ATTENTION.
|
||||
# Web/coord paths use the event_emitter's emit_state for their
|
||||
# own fan-out; this is a second, manager-level hook for callers
|
||||
# that don't consume SSE.
|
||||
self._on_state_change: Callable[[str, WorkstreamState], None] | None = None
|
||||
# State-change subscribers. Multi-subscriber to support the
|
||||
# CLI's background-attention notification AND the in-process
|
||||
# ``SameNodeChildSource`` strategy that delivers child
|
||||
# workstream state changes to a parent's UI without going
|
||||
# through the cluster bus. Each callback fires under
|
||||
# exception-suppression so one failing subscriber doesn't
|
||||
# block the others. Subscribers register via
|
||||
# :meth:`subscribe_to_state`. ``_state_subscribers_lock``
|
||||
# guards mutation + snapshot — set_state copies the list
|
||||
# under the lock then iterates the snapshot unlocked so a
|
||||
# slow subscriber doesn't block subscribe/unsubscribe (and
|
||||
# so concurrent subscribe/unsubscribe during a state event
|
||||
# can't shift the iterator's index — caught by /review bug-1).
|
||||
self._state_subscribers: list[Callable[[str, WorkstreamState], None]] = []
|
||||
self._state_subscribers_lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Properties
|
||||
@@ -217,6 +242,16 @@ class SessionManager:
|
||||
def kind(self) -> WorkstreamKind:
|
||||
return self._adapter.kind
|
||||
|
||||
@property
|
||||
def _service_type(self) -> str | None:
|
||||
"""``services.service_type`` this manager's hosting process registers
|
||||
under, derived from its ``kind``. Used by ``close_idle`` pass 2 to
|
||||
enumerate live peer processes. Returns ``None`` for kinds that have
|
||||
no production service mapping (only the two existing kinds map
|
||||
today; ``None`` would be a marker for a future kind without a
|
||||
clustered hosting model)."""
|
||||
return _KIND_SERVICE_TYPE.get(self.kind)
|
||||
|
||||
@property
|
||||
def count(self) -> int:
|
||||
with self._lock:
|
||||
@@ -604,6 +639,22 @@ class SessionManager:
|
||||
# last close(). The next set_state() call syncs it
|
||||
# naturally; writing 'idle' here could race a concurrent
|
||||
# close() that writes 'closed' under self._lock.
|
||||
#
|
||||
# Bump only ``updated`` (no state write) so this row's
|
||||
# timestamp is fresh against the orphan-reaper cutoff —
|
||||
# otherwise a concurrent close_idle pass-2 in this same
|
||||
# process could clobber a freshly-rehydrated row whose
|
||||
# ``updated`` is older than the cutoff. The pure-
|
||||
# timestamp write is safe against concurrent close()
|
||||
# because close still wins on the state column.
|
||||
try:
|
||||
self._storage.touch_workstream(ws_id)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"session_mgr.touch_workstream_failed ws=%s",
|
||||
ws_id[:8],
|
||||
exc_info=True,
|
||||
)
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_rehydrated(ws)
|
||||
return ws
|
||||
@@ -773,9 +824,36 @@ class SessionManager:
|
||||
log.debug("session_mgr.state_update_failed ws=%s", ws_id[:8], exc_info=True)
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_state(ws, state)
|
||||
if self._on_state_change is not None:
|
||||
# Snapshot under the subscribers lock so concurrent
|
||||
# subscribe / unsubscribe can't shift the iterator's index
|
||||
# mid-dispatch (skipping or repeating callbacks). Iterate
|
||||
# the snapshot WITHOUT the lock so a slow callback doesn't
|
||||
# block subscribe / unsubscribe.
|
||||
with self._state_subscribers_lock:
|
||||
subscribers = list(self._state_subscribers)
|
||||
for callback in subscribers:
|
||||
with contextlib.suppress(Exception):
|
||||
self._on_state_change(ws_id, state)
|
||||
callback(ws_id, state)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State-change subscription
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def subscribe_to_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Register ``callback`` to fire on every workstream state change.
|
||||
|
||||
Multiple subscribers are supported and fire in registration order.
|
||||
Each callback is wrapped in exception-suppression so a failing
|
||||
subscriber doesn't block the others. Use
|
||||
:meth:`unsubscribe_from_state` to remove.
|
||||
"""
|
||||
with self._state_subscribers_lock:
|
||||
self._state_subscribers.append(callback)
|
||||
|
||||
def unsubscribe_from_state(self, callback: Callable[[str, WorkstreamState], None]) -> None:
|
||||
"""Remove a previously-registered state-change callback. No-op if absent."""
|
||||
with self._state_subscribers_lock, contextlib.suppress(ValueError):
|
||||
self._state_subscribers.remove(callback)
|
||||
|
||||
def cancel(self, ws_id: str) -> bool:
|
||||
"""Cancel in-flight generation and unblock any pending approval / plan.
|
||||
@@ -804,15 +882,52 @@ class SessionManager:
|
||||
def close_idle(self, max_age_seconds: float) -> list[str]:
|
||||
"""Close IDLE workstreams inactive for more than ``max_age_seconds``.
|
||||
|
||||
Returns the list of closed ws_ids. Unlike the old WSM version,
|
||||
this does NOT skip the last workstream — the default-startup
|
||||
relic is gone, callers can handle the 0-workstream case.
|
||||
Two-pass shape:
|
||||
|
||||
- Pass 1 (in-memory): close loaded ``IDLE`` rows whose
|
||||
``ws.last_active`` (monotonic) is past timeout. Closes only
|
||||
``IDLE`` so legitimately-attentive rows (waiting for user
|
||||
response) stay live.
|
||||
- Pass 2 (DB orphans): bulk-close DB rows of this manager's
|
||||
kind whose ``updated`` is past the wall-clock cutoff and
|
||||
which are not currently loaded. This catches workstreams
|
||||
left behind by prior process incarnations — a process crash
|
||||
/restart leaves rows in non-terminal states forever
|
||||
otherwise. Closes ``idle/thinking/attention/running``
|
||||
because any matching row is by definition not loaded by any
|
||||
live process and cannot be in a live interaction.
|
||||
|
||||
**Liveness scoping** (the rendezvous router's primitive
|
||||
since PR #384): when ``self._service_type`` resolves to a
|
||||
known service type — both production kinds do — pass 2
|
||||
calls ``storage.list_services`` to enumerate peer processes
|
||||
with recent heartbeats and protects rows whose ``node_id``
|
||||
matches a live ``service_id`` from reap, even when *this*
|
||||
manager is on a different node. This is essential for
|
||||
containerized deployments with dynamic hostnames: dead-pod
|
||||
rows fall out of the live set after the heartbeat window
|
||||
and become reapable; alive-pod rows stay protected as long
|
||||
as the owner heartbeats. A future kind with no service
|
||||
registration would resolve ``_service_type`` to ``None``
|
||||
and skip the live-services lookup (single-process / CLI).
|
||||
|
||||
**Conservative fallback**: if ``list_services`` raises,
|
||||
pass 2 is skipped entirely this tick — never reap when
|
||||
liveness state is unknown. Pass 1 still runs. Next tick
|
||||
retries the lookup.
|
||||
|
||||
Returns the combined list of closed ws_ids (in-memory first,
|
||||
then DB orphans). Pass 1 emits ``ws_closed``; pass 2 does
|
||||
not, because never-loaded rows have no SSE listeners
|
||||
expecting them.
|
||||
|
||||
Atomic pop per victim under ``self._lock`` (bug-5): a pending
|
||||
tool result can flip state IDLE→RUNNING between the snapshot
|
||||
and the close, so the state test + pop must run together.
|
||||
Batches every pop under one ``self._lock`` acquisition (perf-5)
|
||||
rather than locking once per victim.
|
||||
rather than locking once per victim. The DB pass runs OUTSIDE
|
||||
``self._lock`` — only a brief lock to snapshot loaded keys —
|
||||
so a slow UPDATE doesn't block create/get/set_state.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
popped: list[Workstream] = []
|
||||
@@ -853,6 +968,63 @@ class SessionManager:
|
||||
if self._event_emitter is not None:
|
||||
self._event_emitter.emit_closed(ws.id, name=ws.name)
|
||||
closed_ids.append(ws.id)
|
||||
|
||||
# Pass 2: reap DB orphans of this kind older than the cutoff.
|
||||
# Snapshot loaded keys under self._lock briefly so a concurrent
|
||||
# create/load doesn't get its row clobbered by the UPDATE; release
|
||||
# before the DB call.
|
||||
#
|
||||
# Liveness scoping uses ``services.last_heartbeat`` — the same
|
||||
# primitive the rendezvous router (PR #384) uses for routing. A
|
||||
# row's ``node_id`` is stamped at create time and never updated;
|
||||
# in containerized deployments with dynamic hostnames the dead
|
||||
# pod's ``node_id`` points at a service that's no longer
|
||||
# heartbeating, so the row falls through to reap. Conversely,
|
||||
# rows whose ``node_id`` matches a heartbeating service are
|
||||
# protected even when *this* manager is on a different node —
|
||||
# the alive peer may legitimately have them loaded.
|
||||
with self._lock:
|
||||
loaded = list(self._workstreams.keys())
|
||||
cutoff = (datetime.now(UTC) - timedelta(seconds=max_age_seconds)).strftime(
|
||||
"%Y-%m-%dT%H:%M:%S"
|
||||
)
|
||||
live_node_ids: list[str] | None = None
|
||||
skip_pass_2 = False
|
||||
if self._service_type is not None:
|
||||
try:
|
||||
live_services = self._storage.list_services(self._service_type)
|
||||
live_node_ids = [
|
||||
str(svc["service_id"]) for svc in live_services if svc.get("service_id")
|
||||
]
|
||||
except Exception:
|
||||
# Conservative fallback: skip pass 2 entirely this tick
|
||||
# so we can't accidentally reap rows whose owners we
|
||||
# failed to enumerate. Next tick retries.
|
||||
log.debug(
|
||||
"session_mgr.list_services_failed kind=%s",
|
||||
self.kind.value,
|
||||
exc_info=True,
|
||||
)
|
||||
skip_pass_2 = True
|
||||
orphans: list[str] = []
|
||||
if not skip_pass_2:
|
||||
try:
|
||||
orphans = self._storage.bulk_close_stale_orphans(
|
||||
self.kind, cutoff, loaded, live_node_ids=live_node_ids
|
||||
)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"session_mgr.bulk_close_orphans_failed kind=%s",
|
||||
self.kind.value,
|
||||
exc_info=True,
|
||||
)
|
||||
if orphans:
|
||||
log.info(
|
||||
"session_mgr.bulk_close_orphans count=%d kind=%s",
|
||||
len(orphans),
|
||||
self.kind.value,
|
||||
)
|
||||
closed_ids.extend(orphans)
|
||||
return closed_ids
|
||||
|
||||
def _close_if_idle_locked(self, ws_id: str) -> Workstream | None:
|
||||
|
||||
@@ -41,6 +41,7 @@ log = get_logger(__name__)
|
||||
# from bloating memory.
|
||||
_DEFAULT_LISTENER_QUEUE_MAX = 500
|
||||
|
||||
|
||||
# Cap on the per-turn assistant content accumulator. The accumulator
|
||||
# is piggybacked onto the ``ws_state:idle`` broadcast payload so the
|
||||
# cluster collector / dashboard can render the freshly-emitted assistant
|
||||
@@ -327,6 +328,11 @@ class SessionUIBase:
|
||||
"always": bool(always),
|
||||
}
|
||||
)
|
||||
# Kind-specific cross-stream broadcast — ConsoleCoordinatorUI
|
||||
# overrides to push onto the cluster bus so a coord parent's
|
||||
# tree UI clears the pending-approval pill in lockstep with
|
||||
# the actual decision. Stage 3 Step 4.
|
||||
self._broadcast_approval_resolved(approved, feedback, always=always)
|
||||
self._approval_event.set()
|
||||
|
||||
@staticmethod
|
||||
@@ -579,6 +585,20 @@ class SessionUIBase:
|
||||
"judge_pending": judge_pending,
|
||||
}
|
||||
self._enqueue(self._pending_approval)
|
||||
# Cross-stream broadcast — push the items via the cluster bus
|
||||
# so a coord parent's tree UI can render the inline approve/deny
|
||||
# block without waiting for a bulk fetch. Without this, the
|
||||
# bulk fetch races with this assignment: the state transition
|
||||
# to ATTENTION fires upstream BEFORE approve_tools runs (see
|
||||
# session.py:_emit_state("attention") preceding ui.approve_tools),
|
||||
# so a bulk fetch landing in the ~50-200ms window between
|
||||
# _emit_state and this point sees ``_pending_approval=None``
|
||||
# and returns ``pending_approval_detail: null``. The 5s TTL
|
||||
# then locks the coord row on a "loading" placeholder until
|
||||
# the next state event triggers a refresh — which never comes
|
||||
# while parked on _approval_event.wait. The push path
|
||||
# eliminates the race.
|
||||
self._broadcast_approve_request(self._pending_approval)
|
||||
if not self._approval_event.wait(timeout=self._APPROVAL_WAIT_TIMEOUT):
|
||||
# Approval timed out (e.g., user disconnected). Deny via
|
||||
# resolve_approval so verdicts and state are updated consistently.
|
||||
@@ -632,6 +652,12 @@ class SessionUIBase:
|
||||
del self._llm_verdicts[oldest_key]
|
||||
self._llm_verdicts[call_id] = verdict
|
||||
self._enqueue({"type": "intent_verdict", **verdict})
|
||||
# Kind-specific cross-stream broadcast — ConsoleCoordinatorUI
|
||||
# overrides to push onto the cluster bus so a coord parent's
|
||||
# tree UI sees the verdict without polling. Default is no-op
|
||||
# (the per-ws ``_enqueue`` above already covers WebUI's own
|
||||
# SSE listeners). Stage 3 Step 4.
|
||||
self._broadcast_intent_verdict(verdict)
|
||||
self._persist_intent_verdict(verdict)
|
||||
# Decision check + either queue or flag-for-persist happen
|
||||
# under ONE lock acquisition so resolve_approval can't swap-
|
||||
@@ -1351,6 +1377,42 @@ class SessionUIBase:
|
||||
Default: no-op. Subclasses override.
|
||||
"""
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None: # noqa: ARG002 — hook stub
|
||||
"""Fan an LLM intent-judge verdict out to the kind's transport.
|
||||
|
||||
Default: no-op. ``ConsoleCoordinatorUI`` overrides to push a
|
||||
``intent_verdict`` event onto the cluster bus so the parent
|
||||
coordinator's tree UI can render the risk pill + verdict
|
||||
result without polling. Stage 3 Step 4: hook only — the
|
||||
cluster-bus event class lands in Step 5.
|
||||
"""
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool, # noqa: ARG002 — hook stub
|
||||
feedback: str | None = None, # noqa: ARG002 — hook stub
|
||||
*,
|
||||
always: bool = False, # noqa: ARG002 — hook stub
|
||||
) -> None:
|
||||
"""Fan an ``approval_resolved`` decision out to the kind's transport.
|
||||
|
||||
Default: no-op. ``ConsoleCoordinatorUI`` overrides to push to
|
||||
the cluster bus so the parent coordinator's tree UI can clear
|
||||
the pending-approval pill in sync with the actual decision.
|
||||
"""
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None: # noqa: ARG002 — hook stub
|
||||
"""Fan an ``approve_request`` payload out to the kind's transport.
|
||||
|
||||
Default: no-op. ``WebUI`` and ``ConsoleCoordinatorUI`` override
|
||||
to push the items list (the same dict that landed in
|
||||
``_pending_approval``) onto their respective transports. The
|
||||
push path eliminates the bulk-fetch race that otherwise
|
||||
leaves coord rows stuck on a loading placeholder when the
|
||||
bulk fetch lands in the gap between the state transition to
|
||||
ATTENTION and ``_pending_approval`` being set.
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State-broadcast snapshot helper
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -97,7 +97,7 @@ from turnstone.core.storage._utils import sanitize_text
|
||||
from turnstone.core.storage._utils import (
|
||||
scan_skill_content as _scan_skill_content,
|
||||
)
|
||||
from turnstone.core.workstream import WorkstreamKind
|
||||
from turnstone.core.workstream import BULK_CLOSE_STATE_VALUES, WorkstreamKind
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
@@ -595,6 +595,57 @@ class PostgreSQLBackend:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
norm_kind = WorkstreamKind(kind).value
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
stmt = (
|
||||
sa.update(workstreams)
|
||||
.where(
|
||||
workstreams.c.kind == norm_kind,
|
||||
workstreams.c.state.in_(BULK_CLOSE_STATE_VALUES),
|
||||
workstreams.c.updated < cutoff,
|
||||
)
|
||||
.values(state="closed", updated=now)
|
||||
.returning(workstreams.c.ws_id)
|
||||
)
|
||||
# Protect rows whose owning process is still heartbeating in the
|
||||
# services table (rendezvous router's liveness primitive). NULL
|
||||
# node_id rows have no owner identity — always eligible. The
|
||||
# ``and live_node_ids`` short-circuits both ``None`` (skip the
|
||||
# filter entirely — single-process / operator backfill) and ``[]``
|
||||
# (no nodes alive — every row unprotected, no extra predicate
|
||||
# needed since absence equals match-all).
|
||||
if live_node_ids is not None and live_node_ids:
|
||||
stmt = stmt.where(
|
||||
sa.or_(
|
||||
workstreams.c.node_id.is_(None),
|
||||
~workstreams.c.node_id.in_(live_node_ids),
|
||||
)
|
||||
)
|
||||
if exclude_ws_ids:
|
||||
# Skip ``NOT IN ()`` when nothing to exclude — keeps the SQL clean
|
||||
# and avoids SQLAlchemy's empty-collection warning.
|
||||
stmt = stmt.where(~workstreams.c.ws_id.in_(exclude_ws_ids))
|
||||
with self._conn() as conn:
|
||||
result = conn.execute(stmt)
|
||||
ids = [row[0] for row in result]
|
||||
conn.commit()
|
||||
return ids
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
|
||||
@@ -431,6 +431,68 @@ class StorageBackend(Protocol):
|
||||
"""Update a workstream's state and bump updated timestamp."""
|
||||
...
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Close DB-side workstream rows of *kind* whose state is in
|
||||
``BULK_CLOSE_STATE_VALUES`` and whose ``updated`` is lex-older than
|
||||
*cutoff*, excluding rows currently loaded in memory. Sets
|
||||
``state='closed'`` and bumps ``updated``. Returns the list of ws_ids
|
||||
actually transitioned.
|
||||
|
||||
``cutoff`` is a UTC ``YYYY-MM-DDTHH:MM:SS`` string matching the on-disk
|
||||
format ``update_workstream_state`` writes — lex compare is safe for
|
||||
same-offset timestamps. Empty ``exclude_ws_ids`` means no exclusion.
|
||||
|
||||
``live_node_ids`` is the set of ``services.service_id`` values whose
|
||||
``last_heartbeat`` is recent (i.e. owning processes still alive);
|
||||
rows whose ``node_id`` matches one of these are protected because
|
||||
their owning process may legitimately have them loaded on another
|
||||
worker. ``None`` skips the filter entirely (single-process / tests
|
||||
/ operator backfill). Empty list ``[]`` treats every node as dead —
|
||||
useful when operator scripts want to reap regardless of liveness.
|
||||
|
||||
Rows with ``NULL`` ``node_id`` are always eligible: they have no
|
||||
meaningful owner identity, so age alone gates the reap.
|
||||
|
||||
Liveness scoping replaces an earlier ``node_id == self`` heuristic.
|
||||
That heuristic broke in the post-rendezvous-routing world (PR #384):
|
||||
``workstreams.node_id`` is stamped at create time and never updated,
|
||||
so dead-pod orphans in containerized deployments with dynamic
|
||||
hostnames couldn't be reclaimed. ``services.last_heartbeat`` is the
|
||||
rendezvous router's authoritative liveness primitive — using it here
|
||||
keeps reap scoping aligned with routing.
|
||||
|
||||
Asymmetric with ``SessionManager.close_idle``'s in-memory pass on
|
||||
purpose: that pass closes only ``IDLE`` (legitimately-attentive rows
|
||||
stay), this method closes the broader ``BULK_CLOSE_STATE_VALUES`` set
|
||||
because any row matching here is by definition not loaded by any
|
||||
live process and cannot be in a live interaction.
|
||||
"""
|
||||
...
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
"""Bump a workstream row's ``updated`` timestamp without touching its
|
||||
state.
|
||||
|
||||
Used by ``SessionManager.open()`` on cold rehydrate so a freshly-
|
||||
loaded row's ``updated`` can't be older than the orphan-reaper cutoff
|
||||
— protects against a same-process race where a parallel
|
||||
``close_idle`` pass-2 snapshots loaded keys after the storage read
|
||||
but before the in-memory install. Distinct from
|
||||
``update_workstream_state(ws_id, current_state)`` because the
|
||||
rehydrate path explicitly avoids a state write (see the
|
||||
``open()`` no-DB-state-flip-on-resurrect comment): a state write
|
||||
could race a concurrent ``close()`` and resurrect a closed row.
|
||||
Bumping only ``updated`` is safe — close still wins on the state
|
||||
column.
|
||||
"""
|
||||
...
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
"""Update a workstream's display name."""
|
||||
...
|
||||
|
||||
@@ -97,7 +97,7 @@ from turnstone.core.storage._utils import sanitize_text
|
||||
from turnstone.core.storage._utils import (
|
||||
scan_skill_content as _scan_skill_content,
|
||||
)
|
||||
from turnstone.core.workstream import WorkstreamKind
|
||||
from turnstone.core.workstream import BULK_CLOSE_STATE_VALUES, WorkstreamKind
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
@@ -707,6 +707,87 @@ class SQLiteBackend:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def bulk_close_stale_orphans(
|
||||
self,
|
||||
kind: WorkstreamKind | str,
|
||||
cutoff: str,
|
||||
exclude_ws_ids: list[str],
|
||||
live_node_ids: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
norm_kind = WorkstreamKind(kind).value
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
# SQLite has no RETURNING precedent in this file — do SELECT-then-
|
||||
# UPDATE in one transaction, with the SAME WHERE predicates re-applied
|
||||
# to the UPDATE. Re-application defends against a same-process race:
|
||||
# ``SessionManager.open()`` calls ``touch_workstream`` between the
|
||||
# SELECT and the UPDATE could have bumped a row's ``updated`` past
|
||||
# ``cutoff`` (or ``set_state`` could have flipped its state out of
|
||||
# the bulk-close set). Without the re-applied WHERE the UPDATE
|
||||
# closes those rows anyway; with it, the UPDATE skips rows that
|
||||
# became ineligible after the SELECT and the row stays open.
|
||||
# Chunked through ``_in_chunks`` so the ``IN`` clause never exceeds
|
||||
# SQLite's bind-parameter limit (default 999) on a large reap.
|
||||
candidate_conditions = [
|
||||
workstreams.c.kind == norm_kind,
|
||||
workstreams.c.state.in_(BULK_CLOSE_STATE_VALUES),
|
||||
workstreams.c.updated < cutoff,
|
||||
]
|
||||
if live_node_ids is not None and live_node_ids:
|
||||
# Protect rows owned by heartbeating services. NULL node_id is
|
||||
# always eligible. Empty list means "no nodes alive" — every
|
||||
# row is unprotected; the absence of this predicate is
|
||||
# equivalent to "match all," so we just skip it.
|
||||
candidate_conditions.append(
|
||||
sa.or_(
|
||||
workstreams.c.node_id.is_(None),
|
||||
~workstreams.c.node_id.in_(live_node_ids),
|
||||
)
|
||||
)
|
||||
if exclude_ws_ids:
|
||||
candidate_conditions.append(~workstreams.c.ws_id.in_(exclude_ws_ids))
|
||||
select_stmt = sa.select(workstreams.c.ws_id).where(*candidate_conditions)
|
||||
closed: list[str] = []
|
||||
# Match the chunk size used by ``prune_workstreams`` (line 453) — keeps
|
||||
# ``IN`` clauses well below SQLite's default 999-bind-param limit even
|
||||
# on very large reaps.
|
||||
chunk_size = 500
|
||||
with self._conn() as conn:
|
||||
candidate_ids = [row[0] for row in conn.execute(select_stmt)]
|
||||
for i in range(0, len(candidate_ids), chunk_size):
|
||||
chunk = candidate_ids[i : i + chunk_size]
|
||||
# Re-apply the eligibility predicates on the UPDATE so a row
|
||||
# that became fresh between the SELECT and the UPDATE is not
|
||||
# clobbered. Then SELECT back by ``state='closed' AND updated=now``
|
||||
# to determine which rows actually transitioned this commit —
|
||||
# the returned list reflects reality even when re-application
|
||||
# filters out some candidates.
|
||||
conn.execute(
|
||||
sa.update(workstreams)
|
||||
.where(workstreams.c.ws_id.in_(chunk), *candidate_conditions)
|
||||
.values(state="closed", updated=now)
|
||||
)
|
||||
actually_closed = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
sa.select(workstreams.c.ws_id).where(
|
||||
workstreams.c.ws_id.in_(chunk),
|
||||
workstreams.c.state == "closed",
|
||||
workstreams.c.updated == now,
|
||||
)
|
||||
)
|
||||
]
|
||||
closed.extend(actually_closed)
|
||||
conn.commit()
|
||||
return closed
|
||||
|
||||
def touch_workstream(self, ws_id: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
sa.update(workstreams).where(workstreams.c.ws_id == ws_id).values(updated=now)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def update_workstream_name(self, ws_id: str, name: str) -> None:
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with self._conn() as conn:
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Add a partial composite index for the orphan-reaper query.
|
||||
|
||||
``StorageBackend.bulk_close_stale_orphans`` (introduced alongside the
|
||||
workstream-lifecycle leak fix) runs every ``min(300s, idle_timeout/4)``
|
||||
on every server and console process. Its WHERE shape is:
|
||||
|
||||
WHERE kind = ?
|
||||
AND state IN ('idle', 'thinking', 'attention', 'running')
|
||||
AND updated < ?
|
||||
AND (node_id IS NULL OR node_id NOT IN (alive_service_ids))
|
||||
|
||||
At current scale (low-thousands of workstream rows) the existing single-
|
||||
column indexes are sufficient — ``idx_workstreams_state`` prunes to the
|
||||
non-closed subset, and the planner filters the rest sequentially. At
|
||||
100k+ rows that filter becomes a tablescan-shaped cost on the reaper's
|
||||
periodic run.
|
||||
|
||||
A **partial** index covering only ``BULK_CLOSE_STATE_VALUES`` rows
|
||||
matches the reaper's query exactly while staying tiny — closed rows
|
||||
(typically 95%+ of the table per empirical diagnosis) and ``error``
|
||||
rows are excluded, so the index is roughly 5% the size a full multi-
|
||||
column index would be. Write amplification only kicks in for
|
||||
transitions that touch one of the four covered states.
|
||||
|
||||
Column order ``(kind, updated)``:
|
||||
|
||||
- ``kind`` first because the reaper always supplies it as an equality
|
||||
predicate; partitions the partial index into interactive vs
|
||||
coordinator subtrees.
|
||||
- ``updated`` last so the range comparison rides the trailing column —
|
||||
classic composite-index pattern for ``WHERE eq AND range``.
|
||||
|
||||
``node_id`` is intentionally NOT in the index. The reaper's predicate
|
||||
on it is ``NOT IN (small list)`` against an unbounded-cardinality
|
||||
column, which planners don't index well; including it would just add
|
||||
write cost for negligible read benefit.
|
||||
|
||||
PostgreSQL uses ``CREATE INDEX CONCURRENTLY`` so the build is
|
||||
non-blocking on a live system; SQLite has no concurrent concept and
|
||||
the table-level write lock already serializes, so a plain
|
||||
``CREATE INDEX`` is fine.
|
||||
|
||||
Revision ID: 048
|
||||
Revises: 047
|
||||
Create Date: 2026-04-30
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "048"
|
||||
down_revision = "047"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_REAPER_PARTIAL_WHERE = "state IN ('idle', 'thinking', 'attention', 'running')"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
with op.get_context().autocommit_block():
|
||||
op.execute(
|
||||
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_workstreams_reaper "
|
||||
"ON workstreams (kind, updated) "
|
||||
f"WHERE {_REAPER_PARTIAL_WHERE}"
|
||||
)
|
||||
else:
|
||||
op.create_index(
|
||||
"idx_workstreams_reaper",
|
||||
"workstreams",
|
||||
["kind", "updated"],
|
||||
sqlite_where=sa.text(_REAPER_PARTIAL_WHERE),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
with op.get_context().autocommit_block():
|
||||
op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_workstreams_reaper")
|
||||
else:
|
||||
op.drop_index("idx_workstreams_reaper", table_name="workstreams")
|
||||
@@ -77,6 +77,25 @@ class WorkstreamState(enum.Enum):
|
||||
ERROR = "error" # last operation failed
|
||||
|
||||
|
||||
# States the orphan reaper (``SessionManager.close_idle`` pass 2 +
|
||||
# ``StorageBackend.bulk_close_stale_orphans``) is allowed to flip to
|
||||
# ``closed`` for rows past the staleness cutoff. Excludes ``ERROR``
|
||||
# deliberately — error rows are user-investigatable and shouldn't be
|
||||
# auto-reaped — and excludes ``CLOSED`` (terminal). Centralized here
|
||||
# so the storage backends and FakeStorage all agree; if a new transient
|
||||
# state is added to ``WorkstreamState``, deciding whether it joins
|
||||
# this set is part of the change rather than an after-the-fact
|
||||
# audit across three files.
|
||||
BULK_CLOSE_STATE_VALUES: frozenset[str] = frozenset(
|
||||
{
|
||||
WorkstreamState.IDLE.value,
|
||||
WorkstreamState.THINKING.value,
|
||||
WorkstreamState.RUNNING.value,
|
||||
WorkstreamState.ATTENTION.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workstream dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+75
-12
@@ -194,18 +194,14 @@ class WebUI(SessionUIBase):
|
||||
}
|
||||
if state == "idle":
|
||||
event["content"] = payload["content"]
|
||||
# Coord tree-UI renders inline approve/deny buttons off
|
||||
# ``pending_approval_detail``; carrying it on the
|
||||
# state-change broadcast lets the cluster bus update those
|
||||
# buttons in lockstep with ``activity_state`` instead of
|
||||
# forcing the browser to chase a separate dashboard fetch.
|
||||
# Gated on existence so we don't pay the serializer's
|
||||
# per-broadcast verdict-cache deepcopy on the common
|
||||
# no-approval-pending path.
|
||||
if self._pending_approval is not None:
|
||||
detail = self.serialize_pending_approval_detail()
|
||||
if detail is not None:
|
||||
event["pending_approval_detail"] = detail
|
||||
# ``pending_approval_detail`` is NO LONGER piggybacked on
|
||||
# state-change events (Stage 3 cleanup). Symmetric event
|
||||
# flow now: initial approval items arrive via bulk fetch
|
||||
# triggered by the ``activity_state="approval"`` transition,
|
||||
# individual verdicts via the explicit
|
||||
# ``intent_verdict`` event class, and resolution via
|
||||
# ``approval_resolved``. Reducer no longer has to dedupe
|
||||
# the piggyback path against the explicit one.
|
||||
try:
|
||||
WebUI._global_queue.put_nowait(event)
|
||||
except queue.Full:
|
||||
@@ -230,6 +226,73 @@ class WebUI(SessionUIBase):
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_intent_verdict(self, verdict: dict[str, Any]) -> None:
|
||||
"""Send an LLM intent-judge verdict to the global SSE channel.
|
||||
|
||||
Stage 3 Step 5 — the cluster collector's ``_apply_delta``
|
||||
forwards this verbatim to the cluster bus, where coord
|
||||
adapters dispatch it as ``child_ws_intent_verdict`` for the
|
||||
owning parent's tree UI. Unlike the existing
|
||||
``pending_approval_detail`` piggyback on ``ws_state``, this
|
||||
fires WHENEVER a verdict lands — including the common case
|
||||
where the judge daemon writes during ``attention`` with no
|
||||
state transition to ride along on.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "intent_verdict",
|
||||
"ws_id": self.ws_id,
|
||||
"verdict": verdict,
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_approval_resolved(
|
||||
self,
|
||||
approved: bool,
|
||||
feedback: str | None = None,
|
||||
*,
|
||||
always: bool = False,
|
||||
) -> None:
|
||||
"""Send an ``approval_resolved`` decision to the global SSE channel.
|
||||
|
||||
Clears the parent's pending-approval pill in lockstep with
|
||||
the actual decision rather than waiting for the next
|
||||
state-change piggyback.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "approval_resolved",
|
||||
"ws_id": self.ws_id,
|
||||
"approved": approved,
|
||||
"feedback": feedback or "",
|
||||
"always": bool(always),
|
||||
}
|
||||
)
|
||||
|
||||
def _broadcast_approve_request(self, detail: dict[str, Any]) -> None:
|
||||
"""Send an ``approve_request`` payload to the global SSE channel.
|
||||
|
||||
Push path for the initial approval items so a coord parent's
|
||||
tree UI can render the inline approve/deny block immediately
|
||||
without waiting for a bulk-fetch round-trip. The bulk fetch
|
||||
races with ``_pending_approval`` being set inside
|
||||
``approve_tools`` (the state transition to ATTENTION fires
|
||||
upstream first); the push path eliminates that race entirely.
|
||||
"""
|
||||
if WebUI._global_queue is not None:
|
||||
with contextlib.suppress(queue.Full):
|
||||
WebUI._global_queue.put_nowait(
|
||||
{
|
||||
"type": "approve_request",
|
||||
"ws_id": self.ws_id,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
# --- SessionUI protocol ---
|
||||
#
|
||||
# ``on_thinking_start`` / ``on_thinking_stop`` / ``on_reasoning_token``
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
console/static (Saved Coordinators). Single source of truth so the two
|
||||
surfaces don't drift on hover affordance, padding, or typography.
|
||||
==========================================================================
|
||||
Class names match the original ui/static rules they replaced; the
|
||||
delete-mode subset stays in ui/static/style.css until coordinator gets
|
||||
the same UX (then it can move here too).
|
||||
Class names match the original ui/static rules they replaced. Delete-mode
|
||||
selectors live here too so console (Saved Coordinators) and ui/static
|
||||
(Saved Workstreams) share one card + delete affordance.
|
||||
========================================================================== */
|
||||
|
||||
.dashboard-cards {
|
||||
@@ -76,3 +76,283 @@
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
/* ==========================================================================
|
||||
Delete UX — section-level "Delete" toggle, per-card checkboxes, bottom
|
||||
toolbar, and confirmation modal. Moved out of ui/static/style.css when
|
||||
the console grew the same multi-select delete on Saved Coordinators.
|
||||
========================================================================== */
|
||||
|
||||
.ws-delete-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
color 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
.ws-delete-btn:hover {
|
||||
color: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
|
||||
/* Delete mode */
|
||||
.dashboard-card.ws-delete-mode {
|
||||
cursor: pointer;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.04);
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected {
|
||||
cursor: default;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode:hover {
|
||||
background: rgba(220, 38, 38, 0.04);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-card-check {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: var(--red);
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
opacity: 0;
|
||||
animation: ws-check-fadein 0.2s ease-out forwards;
|
||||
}
|
||||
.ws-card-check:focus-visible {
|
||||
/* Card click-handler proxies space/enter to the checkbox, so focus
|
||||
usually rests on the card; if a screen reader / power user tabs
|
||||
directly onto the checkbox the UA outline can be killed by
|
||||
adjacent rules — this guarantees a visible affordance. */
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
@keyframes ws-check-fadein {
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
.dashboard-card.ws-selected {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-selected {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-delete-bar {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
display: flex;
|
||||
animation: ws-bar-slide 0.2s ease-out;
|
||||
}
|
||||
@keyframes ws-bar-slide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ws-card-check {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
/* Below ~700px the four-pill bar's preferred width (~480-520px) starts
|
||||
eating into hit-targets and risking horizontal overflow. Wrap onto
|
||||
two rows: count + Cancel + Select All on the first, the destructive
|
||||
Delete Selected on its own full-width row underneath — also a better
|
||||
thumb-target separation than the desktop layout. */
|
||||
@media (max-width: 700px) {
|
||||
.ws-delete-bar {
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: 0;
|
||||
flex: 1 1 100%;
|
||||
order: 99;
|
||||
}
|
||||
}
|
||||
.ws-delete-bar .ws-delete-count-label {
|
||||
font-size: 12px;
|
||||
color: var(--fg-dim);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: auto;
|
||||
/* Dark-theme --red (#f87171) on #fff is only 3.0:1 — below WCAG AA
|
||||
for normal text on a destructive button. Use the deeper red
|
||||
(#dc2626 → 4.85:1) on the filled state so the button label clears
|
||||
AA in the default theme. Light theme already uses --red (#b91c1c,
|
||||
5.9:1) and stays put — the override below pins it. */
|
||||
background: #dc2626;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 16px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-bar .ws-delete-bar-btn {
|
||||
background: var(--red);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:hover:not(:disabled) {
|
||||
filter: brightness(1.1);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn:hover {
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
|
||||
/* Delete modal — id-scoped so the surface that owns it controls visibility.
|
||||
Both ui/static (#ws-delete-overlay) and console/static
|
||||
(#coord-delete-overlay) share the same shape via the .ws-delete-modal
|
||||
class hooks below. */
|
||||
.ws-delete-modal-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
.ws-delete-modal-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 24px;
|
||||
max-width: 480px;
|
||||
width: 90%;
|
||||
}
|
||||
.ws-delete-modal-box h3 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 16px;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
.ws-delete-modal-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
margin: 12px 0;
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item {
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
color: var(--fg-bright);
|
||||
border-bottom: 1px solid var(--border);
|
||||
/* Long aliases / raw ws_ids in the confirm + results list shouldn't
|
||||
punch out of the modal at narrow viewports. */
|
||||
word-break: break-word;
|
||||
}
|
||||
/* Modal alert region — only painted when the controller writes a
|
||||
message. Both close paths clear it, so :not(:empty) keeps the box
|
||||
invisible at rest and avoids an empty-frame artefact. */
|
||||
.ws-delete-modal-box [role="alert"]:not(:empty) {
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
border: 1px solid var(--red);
|
||||
color: var(--red);
|
||||
border-radius: var(--radius);
|
||||
padding: 8px 10px;
|
||||
font-size: 12px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-modal-box [role="alert"]:not(:empty) {
|
||||
background: rgba(220, 38, 38, 0.06);
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.ws-delete-modal-list .ws-delete-item.ws-delete-error {
|
||||
color: var(--red);
|
||||
}
|
||||
.ws-delete-modal-buttons {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
.ws-delete-modal-buttons button {
|
||||
padding: 8px 20px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
.ws-delete-modal-buttons button.ws-delete-confirm {
|
||||
/* Mirror .ws-delete-bar-btn's contrast bump — same destructive
|
||||
filled-button treatment, same dark-theme AA fix. */
|
||||
background: #dc2626;
|
||||
color: #fff;
|
||||
border-color: #dc2626;
|
||||
}
|
||||
[data-theme="light"] .ws-delete-modal-buttons button.ws-delete-confirm {
|
||||
background: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
/* `.ws-delete-close` is a state-marker, not a colour rule — the
|
||||
controller drops `.ws-delete-confirm` when the modal transitions to
|
||||
the post-delete "Close" state, and the default
|
||||
`.ws-delete-modal-buttons button` rule above already provides the
|
||||
transparent / fg-bright / border styling. The class itself is useful
|
||||
for DOM inspection and as a future hook. */
|
||||
|
||||
@@ -77,3 +77,433 @@ function renderSessionCard(sess, opts) {
|
||||
card.appendChild(metaEl);
|
||||
return card;
|
||||
}
|
||||
|
||||
/* createSavedCardsController — shared multi-select-delete behaviour for
|
||||
the dashboard / home "saved cards" surfaces. ui/static (Saved
|
||||
Workstreams) and console/static (Saved Coordinators) both instantiate
|
||||
one of these; the controller owns:
|
||||
|
||||
- delete-mode state (active flag + selected ws_id set)
|
||||
- card decoration (checkbox + key/click overrides)
|
||||
- the bottom toolbar wiring (count, Select All, Delete Selected)
|
||||
- the confirmation modal (focus trap, batch fan-out, results view)
|
||||
|
||||
It does NOT own how cards get fetched or rendered — the caller's
|
||||
render() is invoked when the controller needs the list redrawn (mode
|
||||
transitions, Select-All toggles).
|
||||
|
||||
Required opts:
|
||||
idPrefix — DOM-id prefix shared by the toolbar + modal
|
||||
(e.g. "ws-delete" / "coord-delete"). The DOM
|
||||
must already contain `${idPrefix}-bar`,
|
||||
`${idPrefix}-bar-count`, `${idPrefix}-bar-delete`,
|
||||
`${idPrefix}-bar-select-all`, `${idPrefix}-overlay`,
|
||||
`${idPrefix}-box`, `${idPrefix}-error`,
|
||||
`${idPrefix}-count`, `${idPrefix}-list`,
|
||||
`${idPrefix}-confirm-btn`, `${idPrefix}-cancel-btn`.
|
||||
buttonId — id of the section's start/cancel toggle button.
|
||||
noun — singular display word for the item kind, e.g.
|
||||
"workstream" / "coordinator". Used in toast +
|
||||
modal copy.
|
||||
activateLabel — sess => string; aria-label for the card when NOT
|
||||
in delete mode (e.g. "Resume: foo").
|
||||
buildDeleteRequest — wsId => { url, options }; what authFetch should
|
||||
send to delete one item.
|
||||
render — () => void; redraw the visible cards. Called by
|
||||
the controller on mode start/cancel and Select-
|
||||
All toggle. Caller is responsible for calling
|
||||
setItems(items) + decorateCard() inside it.
|
||||
onClose — optional () => void; called once after the user
|
||||
closes the post-delete results modal. Typical
|
||||
use: re-fetch the saved list.
|
||||
*/
|
||||
function createSavedCardsController(opts) {
|
||||
var state = { mode: false, selected: {}, items: [] };
|
||||
var batchTrap = null;
|
||||
/* Element that owned focus when the modal opened — restored in
|
||||
closeModal() so keyboard users land back on the toggle button (or
|
||||
wherever they came from) instead of <body>. WCAG 2.4.3. */
|
||||
var prevFocus = null;
|
||||
|
||||
function $(id) {
|
||||
return document.getElementById(opts.idPrefix + "-" + id);
|
||||
}
|
||||
|
||||
/* Replace the toggle button's content with a glyph + label, keeping
|
||||
the glyph in an aria-hidden span so screen readers only read the
|
||||
label. Built from DOM nodes (no innerHTML) — same shape as the
|
||||
section-header markup the JS replaces. */
|
||||
function setIconButton(btn, glyph, label) {
|
||||
btn.replaceChildren();
|
||||
var span = document.createElement("span");
|
||||
span.setAttribute("aria-hidden", "true");
|
||||
span.textContent = glyph;
|
||||
btn.appendChild(span);
|
||||
btn.appendChild(document.createTextNode(" " + label));
|
||||
}
|
||||
|
||||
function setItems(items) {
|
||||
state.items = items;
|
||||
/* Drop any selections whose ws_id is no longer on the visible page —
|
||||
SSE-driven re-renders or pagination jumps shouldn't leave ghost
|
||||
entries inflating the count and 404-ing on confirm. */
|
||||
if (state.mode) {
|
||||
var byId = {};
|
||||
items.forEach(function (s) {
|
||||
byId[s.ws_id] = true;
|
||||
});
|
||||
Object.keys(state.selected).forEach(function (id) {
|
||||
if (!byId[id]) delete state.selected[id];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function inMode() {
|
||||
return state.mode;
|
||||
}
|
||||
|
||||
function blockActivate() {
|
||||
return state.mode;
|
||||
}
|
||||
|
||||
function isSelected(wsId) {
|
||||
return !!state.selected[wsId];
|
||||
}
|
||||
|
||||
function ariaLabel(sess) {
|
||||
var label = sess.alias || sess.title || sess.name || sess.ws_id;
|
||||
if (state.mode) return "Select " + opts.noun + ": " + label;
|
||||
return typeof opts.activateLabel === "function"
|
||||
? opts.activateLabel(sess)
|
||||
: "Activate: " + label;
|
||||
}
|
||||
|
||||
/* Decorate an already-rendered .dashboard-card with the checkbox +
|
||||
event overrides used in delete mode. Idempotent guard: only acts
|
||||
when the controller is active. */
|
||||
function decorateCard(card, sess) {
|
||||
if (!state.mode) return;
|
||||
card.classList.add("ws-delete-mode");
|
||||
card.removeAttribute("role");
|
||||
var chk = document.createElement("input");
|
||||
chk.type = "checkbox";
|
||||
chk.className = "ws-card-check";
|
||||
chk.checked = !!state.selected[sess.ws_id];
|
||||
var label = sess.alias || sess.title || sess.name || sess.ws_id;
|
||||
chk.setAttribute("aria-label", "Select " + label + " for deletion");
|
||||
chk.onclick = function (e) {
|
||||
e.stopPropagation();
|
||||
if (chk.checked) state.selected[sess.ws_id] = true;
|
||||
else delete state.selected[sess.ws_id];
|
||||
card.classList.toggle("ws-selected", chk.checked);
|
||||
refreshBar();
|
||||
};
|
||||
card.insertBefore(chk, card.firstChild);
|
||||
card.onclick = function (e) {
|
||||
if (e.target === chk) return;
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
};
|
||||
card.onkeydown = function (e) {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
}
|
||||
};
|
||||
if (state.selected[sess.ws_id]) card.classList.add("ws-selected");
|
||||
}
|
||||
|
||||
function refreshBar() {
|
||||
var count = Object.keys(state.selected).length;
|
||||
var label = $("bar-count");
|
||||
if (label) label.textContent = count + " selected";
|
||||
var delBtn = $("bar-delete");
|
||||
if (delBtn) delBtn.disabled = count === 0;
|
||||
var selBtn = $("bar-select-all");
|
||||
if (selBtn) {
|
||||
var allSelected = count === state.items.length && state.items.length > 0;
|
||||
selBtn.textContent = allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (!state.items.length) {
|
||||
if (typeof showToast === "function") {
|
||||
showToast("No saved " + opts.noun + "s to delete");
|
||||
}
|
||||
return;
|
||||
}
|
||||
state.mode = true;
|
||||
state.selected = {};
|
||||
opts.render();
|
||||
var btn = document.getElementById(opts.buttonId);
|
||||
if (btn) {
|
||||
setIconButton(btn, "✕", "Cancel");
|
||||
btn.onclick = cancel;
|
||||
}
|
||||
var bar = $("bar");
|
||||
if (bar) bar.classList.add("visible");
|
||||
refreshBar();
|
||||
}
|
||||
|
||||
function cancel() {
|
||||
state.mode = false;
|
||||
state.selected = {};
|
||||
opts.render();
|
||||
var btn = document.getElementById(opts.buttonId);
|
||||
if (btn) {
|
||||
setIconButton(btn, "\u{1f5d1}", "Delete");
|
||||
btn.onclick = start;
|
||||
}
|
||||
var bar = $("bar");
|
||||
if (bar) bar.classList.remove("visible");
|
||||
}
|
||||
|
||||
function toggleAll() {
|
||||
var allSelected =
|
||||
Object.keys(state.selected).length === state.items.length &&
|
||||
state.items.length > 0;
|
||||
if (allSelected) {
|
||||
state.selected = {};
|
||||
} else {
|
||||
state.items.forEach(function (s) {
|
||||
state.selected[s.ws_id] = true;
|
||||
});
|
||||
}
|
||||
opts.render();
|
||||
refreshBar();
|
||||
}
|
||||
|
||||
function _byId() {
|
||||
/* Single-pass index over the visible items so the modal + fan-out
|
||||
paths don't repeat O(N) `find` calls per selection. */
|
||||
var map = {};
|
||||
state.items.forEach(function (s) {
|
||||
map[s.ws_id] = s;
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
function confirmSelection() {
|
||||
var selected = Object.keys(state.selected);
|
||||
if (!selected.length) {
|
||||
if (typeof showToast === "function") {
|
||||
showToast("No " + opts.noun + "s selected");
|
||||
}
|
||||
return;
|
||||
}
|
||||
var byId = _byId();
|
||||
var overlay = $("overlay");
|
||||
var countEl = $("count");
|
||||
var listEl = $("list");
|
||||
var errorEl = $("error");
|
||||
if (errorEl) errorEl.textContent = "";
|
||||
if (countEl) {
|
||||
countEl.textContent =
|
||||
selected.length + " " + opts.noun + "(s) will be permanently deleted:";
|
||||
}
|
||||
if (listEl) {
|
||||
listEl.replaceChildren();
|
||||
selected.forEach(function (wsId) {
|
||||
var item = byId[wsId];
|
||||
var name = item ? item.alias || item.title || item.name || wsId : wsId;
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item";
|
||||
div.textContent = name;
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
}
|
||||
var delBtn = $("confirm-btn");
|
||||
if (delBtn) {
|
||||
delBtn.textContent = "Delete";
|
||||
delBtn.disabled = false;
|
||||
delBtn.classList.remove("ws-delete-close");
|
||||
delBtn.classList.add("ws-delete-confirm");
|
||||
delBtn.onclick = confirm;
|
||||
}
|
||||
var cancelBtn = $("cancel-btn");
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
if (overlay) overlay.style.display = "flex";
|
||||
|
||||
if (batchTrap) document.removeEventListener("keydown", batchTrap);
|
||||
batchTrap = function (e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
closeModal();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Tab") {
|
||||
var box = $("box");
|
||||
if (!box) return;
|
||||
var focusable = box.querySelectorAll("button:not(:disabled)");
|
||||
if (!focusable.length) return;
|
||||
var first = focusable[0];
|
||||
var last = focusable[focusable.length - 1];
|
||||
if (e.shiftKey && document.activeElement === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && document.activeElement === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", batchTrap);
|
||||
/* Snapshot the pre-modal focus owner so closeModal() can return to
|
||||
it. Captured before we move focus into the dialog so the
|
||||
restore-target is the caller, not the dialog itself. */
|
||||
prevFocus = document.activeElement;
|
||||
if (cancelBtn) cancelBtn.focus();
|
||||
}
|
||||
|
||||
function closeModal() {
|
||||
var overlay = $("overlay");
|
||||
if (overlay) overlay.style.display = "none";
|
||||
if (batchTrap) {
|
||||
document.removeEventListener("keydown", batchTrap);
|
||||
batchTrap = null;
|
||||
}
|
||||
/* Pick the most useful focus target:
|
||||
1. prevFocus (where the user came from), if it's still in the
|
||||
DOM and visible. Esc / Cancel paths land here — the bar is
|
||||
still on screen, so focus returns to "Delete Selected".
|
||||
2. The section toggle button — always present, semantic exit
|
||||
point for the flow. Used when prevFocus has been hidden by
|
||||
cancel() (post-delete Close path: cancel() ran first and
|
||||
put `.ws-delete-bar` at display:none, so the bar's button
|
||||
is no longer focusable). */
|
||||
var target = prevFocus;
|
||||
if (!target || target.offsetParent === null) {
|
||||
target = document.getElementById(opts.buttonId);
|
||||
}
|
||||
if (target && typeof target.focus === "function") {
|
||||
try {
|
||||
target.focus();
|
||||
} catch (_) {
|
||||
/* node detached between open and close — give up silently */
|
||||
}
|
||||
}
|
||||
prevFocus = null;
|
||||
}
|
||||
|
||||
function confirm() {
|
||||
var selected = Object.keys(state.selected);
|
||||
if (!selected.length) return;
|
||||
var byId = _byId();
|
||||
var errorEl = $("error");
|
||||
var listEl = $("list");
|
||||
var countEl = $("count");
|
||||
var delBtn = $("confirm-btn");
|
||||
var cancelBtn = $("cancel-btn");
|
||||
if (errorEl) errorEl.textContent = "";
|
||||
if (delBtn) {
|
||||
delBtn.disabled = true;
|
||||
delBtn.textContent = "Deleting...";
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = true;
|
||||
|
||||
var results = [];
|
||||
var promises = selected.map(function (wsId) {
|
||||
var shortId = wsId.substring(0, 8);
|
||||
var item = byId[wsId];
|
||||
var name = item ? item.alias || item.title || item.name || wsId : wsId;
|
||||
var req = opts.buildDeleteRequest(wsId);
|
||||
return authFetch(req.url, req.options)
|
||||
.then(function (r) {
|
||||
var status = r.status;
|
||||
var contentType = r.headers.get("content-type") || "";
|
||||
if (r.ok) {
|
||||
results.push({ name: name, shortId: shortId, ok: true });
|
||||
return;
|
||||
}
|
||||
return r.text().then(function (body) {
|
||||
var errMsg = shortId + ": HTTP " + status;
|
||||
if (contentType.includes("json")) {
|
||||
try {
|
||||
var j = JSON.parse(body);
|
||||
if (j.error) errMsg = shortId + ": " + j.error;
|
||||
} catch (_) {
|
||||
/* fall through */
|
||||
}
|
||||
} else if (body) {
|
||||
errMsg = shortId + ": " + body.substring(0, 200);
|
||||
}
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: errMsg,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(function (err) {
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: shortId + ": " + err.message,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Promise.all(promises).then(function () {
|
||||
if (listEl) {
|
||||
listEl.replaceChildren();
|
||||
results.forEach(function (r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item" + (r.ok ? "" : " ws-delete-error");
|
||||
div.textContent =
|
||||
(r.ok ? "✓ " : "✗ ") + r.name + (r.error ? " — " + r.error : "");
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
}
|
||||
var okCount = results.filter(function (r) {
|
||||
return r.ok;
|
||||
}).length;
|
||||
var failCount = results.filter(function (r) {
|
||||
return !r.ok;
|
||||
}).length;
|
||||
if (countEl) {
|
||||
countEl.textContent = okCount + " deleted, " + failCount + " failed";
|
||||
}
|
||||
if (delBtn) {
|
||||
delBtn.disabled = false;
|
||||
delBtn.textContent = "Close";
|
||||
/* Swap modifier classes so styling is intent-driven instead of
|
||||
cascade-positional: the Close button picks up the default
|
||||
".ws-delete-modal-buttons button" rule once .ws-delete-confirm
|
||||
is removed. */
|
||||
delBtn.classList.remove("ws-delete-confirm");
|
||||
delBtn.classList.add("ws-delete-close");
|
||||
delBtn.onclick = function () {
|
||||
/* Order matters: cancel() reshapes the toggle button via
|
||||
setIconButton(), which preserves the element identity but
|
||||
swaps its subtree. closeModal() then focuses prevFocus —
|
||||
which IS that toggle button — landing on a freshly rebuilt
|
||||
"Delete" affordance instead of <body>. */
|
||||
cancel();
|
||||
closeModal();
|
||||
if (typeof opts.onClose === "function") opts.onClose();
|
||||
};
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
setItems: setItems,
|
||||
inMode: inMode,
|
||||
blockActivate: blockActivate,
|
||||
isSelected: isSelected,
|
||||
ariaLabel: ariaLabel,
|
||||
decorateCard: decorateCard,
|
||||
refreshBar: refreshBar,
|
||||
start: start,
|
||||
cancel: cancel,
|
||||
toggleAll: toggleAll,
|
||||
confirmSelection: confirmSelection,
|
||||
closeModal: closeModal,
|
||||
confirm: confirm,
|
||||
};
|
||||
}
|
||||
|
||||
+38
-265
@@ -3668,12 +3668,35 @@ function updateDashFooter(agg) {
|
||||
}
|
||||
}
|
||||
|
||||
var _wsDeleteMode = false;
|
||||
var _wsDeleteSelected = {};
|
||||
// Saved Workstreams cache + multi-select delete controller. The
|
||||
// controller (from /shared/cards.js) owns mode state, checkbox
|
||||
// decoration, the toolbar wiring, and the confirmation modal — see
|
||||
// createSavedCardsController for the shared bits.
|
||||
var _wsSavedItems = [];
|
||||
var _wsDeleteController = createSavedCardsController({
|
||||
idPrefix: "ws-delete",
|
||||
buttonId: "ws-delete-btn",
|
||||
noun: "workstream",
|
||||
activateLabel: function (s) {
|
||||
return "Resume: " + (s.alias || s.title || s.ws_id);
|
||||
},
|
||||
buildDeleteRequest: function (wsId) {
|
||||
return {
|
||||
url: "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/delete",
|
||||
options: { method: "POST" },
|
||||
};
|
||||
},
|
||||
render: function () {
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
},
|
||||
onClose: function () {
|
||||
loadDashboard();
|
||||
},
|
||||
});
|
||||
|
||||
function renderSavedWorkstreams(items) {
|
||||
_wsSavedItems = items;
|
||||
_wsDeleteController.setItems(items);
|
||||
var c = document.getElementById("dashboard-saved-cards");
|
||||
c.replaceChildren();
|
||||
if (!items.length) {
|
||||
@@ -3684,289 +3707,39 @@ function renderSavedWorkstreams(items) {
|
||||
return;
|
||||
}
|
||||
items.forEach(function (sess) {
|
||||
// Default card shape (title + meta + wsid + Resume click) comes from
|
||||
// the shared /shared/cards.js helper so console (Saved Coordinators)
|
||||
// and ui/static (Saved Workstreams) stay in lock-step. Delete mode
|
||||
// is interactive-only; we layer the checkbox + selection wiring on
|
||||
// top of the shared card after construction.
|
||||
var card = renderSessionCard(sess, {
|
||||
ariaLabel: function (s) {
|
||||
var label = s.alias || s.title || s.ws_id;
|
||||
return _wsDeleteMode ? "Select: " + label : "Resume: " + label;
|
||||
},
|
||||
ariaLabel: _wsDeleteController.ariaLabel,
|
||||
onActivate: function (s) {
|
||||
// Suppressed in delete mode \u2014 the layered checkbox handler below
|
||||
// owns clicks while delete-mode is active.
|
||||
if (_wsDeleteMode) return;
|
||||
if (_wsDeleteController.blockActivate()) return;
|
||||
dashboardResumeSession(s.ws_id);
|
||||
},
|
||||
});
|
||||
|
||||
if (_wsDeleteMode) {
|
||||
card.classList.add("ws-delete-mode");
|
||||
card.removeAttribute("role"); // becomes a checkbox host, not a button
|
||||
var chk = document.createElement("input");
|
||||
chk.type = "checkbox";
|
||||
chk.className = "ws-card-check";
|
||||
chk.checked = !!_wsDeleteSelected[sess.ws_id];
|
||||
var label = sess.alias || sess.title || sess.ws_id;
|
||||
chk.setAttribute("aria-label", "Select " + label + " for deletion");
|
||||
chk.onclick = function (e) {
|
||||
e.stopPropagation();
|
||||
if (chk.checked) _wsDeleteSelected[sess.ws_id] = true;
|
||||
else delete _wsDeleteSelected[sess.ws_id];
|
||||
card.classList.toggle("ws-selected", chk.checked);
|
||||
updateWsDeleteBar();
|
||||
};
|
||||
card.insertBefore(chk, card.firstChild);
|
||||
// Override the shared helper's onclick/onkeydown \u2014 in delete mode
|
||||
// a card click toggles the checkbox instead of activating Resume.
|
||||
card.onclick = function (e) {
|
||||
if (e.target === chk) return;
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
};
|
||||
card.onkeydown = function (e) {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
chk.checked = !chk.checked;
|
||||
chk.onclick(e);
|
||||
}
|
||||
};
|
||||
if (_wsDeleteSelected[sess.ws_id]) card.classList.add("ws-selected");
|
||||
}
|
||||
|
||||
_wsDeleteController.decorateCard(card, sess);
|
||||
c.appendChild(card);
|
||||
});
|
||||
if (_wsDeleteController.inMode()) _wsDeleteController.refreshBar();
|
||||
}
|
||||
|
||||
// HTML inline-onclick wrappers — keep the global names the existing
|
||||
// markup binds to (`onclick="startWsDeleteMode()"` etc.) and forward
|
||||
// to the controller.
|
||||
function startWsDeleteMode() {
|
||||
_wsDeleteMode = true;
|
||||
_wsDeleteSelected = {};
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
var btn = document.getElementById("ws-delete-btn");
|
||||
if (btn) {
|
||||
btn.textContent = "\u2715 Cancel";
|
||||
btn.onclick = cancelWsDeleteMode;
|
||||
}
|
||||
var bar = document.getElementById("ws-delete-bar");
|
||||
if (bar) bar.classList.add("visible");
|
||||
_wsDeleteController.start();
|
||||
}
|
||||
|
||||
function cancelWsDeleteMode() {
|
||||
_wsDeleteMode = false;
|
||||
_wsDeleteSelected = {};
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
var btn = document.getElementById("ws-delete-btn");
|
||||
if (btn) {
|
||||
btn.innerHTML = "🗑 Delete";
|
||||
btn.onclick = startWsDeleteMode;
|
||||
}
|
||||
var bar = document.getElementById("ws-delete-bar");
|
||||
if (bar) bar.classList.remove("visible");
|
||||
_wsDeleteController.cancel();
|
||||
}
|
||||
|
||||
function updateWsDeleteBar() {
|
||||
var count = Object.keys(_wsDeleteSelected).length;
|
||||
var label = document.getElementById("ws-delete-bar-count");
|
||||
if (label) label.textContent = count + " selected";
|
||||
var delBtn = document.getElementById("ws-delete-bar-delete");
|
||||
if (delBtn) delBtn.disabled = count === 0;
|
||||
var selBtn = document.getElementById("ws-delete-bar-select-all");
|
||||
if (selBtn) {
|
||||
var allSelected =
|
||||
count === _wsSavedItems.length && _wsSavedItems.length > 0;
|
||||
selBtn.textContent = allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
}
|
||||
|
||||
function toggleSelectAll() {
|
||||
var allSelected =
|
||||
Object.keys(_wsDeleteSelected).length === _wsSavedItems.length &&
|
||||
_wsSavedItems.length > 0;
|
||||
if (allSelected) {
|
||||
_wsDeleteSelected = {};
|
||||
} else {
|
||||
_wsSavedItems.forEach(function (s) {
|
||||
_wsDeleteSelected[s.ws_id] = true;
|
||||
});
|
||||
}
|
||||
renderSavedWorkstreams(_wsSavedItems);
|
||||
updateWsDeleteBar();
|
||||
_wsDeleteController.toggleAll();
|
||||
}
|
||||
|
||||
var _wsDeleteBatchTrap = null;
|
||||
|
||||
function confirmWsDeleteSelection() {
|
||||
var selected = Object.keys(_wsDeleteSelected);
|
||||
if (!selected.length) {
|
||||
showToast("No workstreams selected", "warning");
|
||||
return;
|
||||
}
|
||||
var overlay = document.getElementById("ws-delete-overlay");
|
||||
var countEl = document.getElementById("ws-delete-count");
|
||||
var listEl = document.getElementById("ws-delete-list");
|
||||
var errorEl = document.getElementById("ws-delete-error");
|
||||
errorEl.textContent = "";
|
||||
countEl.textContent =
|
||||
selected.length + " workstream(s) will be permanently deleted:";
|
||||
listEl.innerHTML = "";
|
||||
selected.forEach(function (wsId) {
|
||||
var item = _wsSavedItems.find(function (s) {
|
||||
return s.ws_id === wsId;
|
||||
});
|
||||
var name = item ? item.alias || item.title || wsId : wsId;
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item";
|
||||
div.textContent = name;
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
// Reset confirm button handler (may have been overwritten to "Close" by previous run)
|
||||
var delBtn = document.getElementById("ws-delete-confirm-btn");
|
||||
if (delBtn) {
|
||||
delBtn.textContent = "Delete";
|
||||
delBtn.disabled = false;
|
||||
delBtn.classList.remove("ws-delete-close");
|
||||
delBtn.onclick = confirmWsDelete;
|
||||
}
|
||||
var cancelBtn = document.getElementById("ws-delete-cancel-btn");
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
overlay.style.display = "flex";
|
||||
|
||||
// Focus trap + Escape
|
||||
if (_wsDeleteBatchTrap)
|
||||
document.removeEventListener("keydown", _wsDeleteBatchTrap);
|
||||
_wsDeleteBatchTrap = function (e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
cancelWsDelete();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Tab") {
|
||||
var box = document.getElementById("ws-delete-box");
|
||||
var focusable = box.querySelectorAll("button:not(:disabled)");
|
||||
var first = focusable[0];
|
||||
var last = focusable[focusable.length - 1];
|
||||
if (e.shiftKey && document.activeElement === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && document.activeElement === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", _wsDeleteBatchTrap);
|
||||
if (cancelBtn) cancelBtn.focus();
|
||||
_wsDeleteController.confirmSelection();
|
||||
}
|
||||
|
||||
function cancelWsDelete() {
|
||||
document.getElementById("ws-delete-overlay").style.display = "none";
|
||||
if (_wsDeleteBatchTrap) {
|
||||
document.removeEventListener("keydown", _wsDeleteBatchTrap);
|
||||
_wsDeleteBatchTrap = null;
|
||||
}
|
||||
_wsDeleteController.closeModal();
|
||||
}
|
||||
|
||||
function confirmWsDelete() {
|
||||
var selected = Object.keys(_wsDeleteSelected);
|
||||
if (!selected.length) return;
|
||||
var overlay = document.getElementById("ws-delete-overlay");
|
||||
var errorEl = document.getElementById("ws-delete-error");
|
||||
var listEl = document.getElementById("ws-delete-list");
|
||||
var countEl = document.getElementById("ws-delete-count");
|
||||
var delBtn = document.getElementById("ws-delete-confirm-btn");
|
||||
var cancelBtn = document.getElementById("ws-delete-cancel-btn");
|
||||
errorEl.textContent = "";
|
||||
|
||||
// Disable buttons during deletion
|
||||
if (delBtn) {
|
||||
delBtn.disabled = true;
|
||||
delBtn.textContent = "Deleting...";
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = true;
|
||||
|
||||
var results = [];
|
||||
var promises = selected.map(function (wsId) {
|
||||
var shortId = wsId.substring(0, 8);
|
||||
var item = _wsSavedItems.find(function (s) {
|
||||
return s.ws_id === wsId;
|
||||
});
|
||||
var name = item ? item.alias || item.title || wsId : wsId;
|
||||
var url = "/v1/api/workstreams/" + encodeURIComponent(wsId) + "/delete";
|
||||
|
||||
return authFetch(url, { method: "POST" })
|
||||
.then(function (r) {
|
||||
var status = r.status;
|
||||
var contentType = r.headers.get("content-type") || "";
|
||||
if (r.ok) {
|
||||
results.push({ name: name, shortId: shortId, ok: true });
|
||||
return;
|
||||
}
|
||||
// Read body as text first to avoid JSON parse errors
|
||||
return r.text().then(function (body) {
|
||||
var errMsg = shortId + ": HTTP " + status;
|
||||
if (contentType.includes("json")) {
|
||||
try {
|
||||
var j = JSON.parse(body);
|
||||
if (j.error) errMsg = shortId + ": " + j.error;
|
||||
} catch (_) {
|
||||
/* fall through */
|
||||
}
|
||||
} else if (body) {
|
||||
errMsg = shortId + ": " + body.substring(0, 200);
|
||||
}
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: errMsg,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(function (err) {
|
||||
results.push({
|
||||
name: name,
|
||||
shortId: shortId,
|
||||
ok: false,
|
||||
error: shortId + ": " + err.message,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Promise.all(promises).then(function () {
|
||||
// Rebuild the list with results
|
||||
listEl.innerHTML = "";
|
||||
results.forEach(function (r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "ws-delete-item" + (r.ok ? "" : " ws-delete-error");
|
||||
div.textContent =
|
||||
(r.ok ? "\u2713 " : "\u2717 ") +
|
||||
r.name +
|
||||
(r.error ? " — " + r.error : "");
|
||||
listEl.appendChild(div);
|
||||
});
|
||||
|
||||
var okCount = results.filter(function (r) {
|
||||
return r.ok;
|
||||
}).length;
|
||||
var failCount = results.filter(function (r) {
|
||||
return !r.ok;
|
||||
}).length;
|
||||
countEl.textContent = okCount + " deleted, " + failCount + " failed";
|
||||
|
||||
if (delBtn) {
|
||||
delBtn.disabled = false;
|
||||
delBtn.textContent = "Close";
|
||||
delBtn.classList.add("ws-delete-close");
|
||||
delBtn.onclick = function () {
|
||||
cancelWsDelete();
|
||||
cancelWsDeleteMode();
|
||||
loadDashboard();
|
||||
};
|
||||
}
|
||||
if (cancelBtn) cancelBtn.disabled = false;
|
||||
});
|
||||
_wsDeleteController.confirm();
|
||||
}
|
||||
|
||||
// --- Workstream title management ---
|
||||
|
||||
@@ -363,17 +363,18 @@
|
||||
<!-- Delete workstreams confirmation modal (batch) -->
|
||||
<div
|
||||
id="ws-delete-overlay"
|
||||
class="ws-delete-modal-overlay"
|
||||
style="display: none"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="ws-delete-title"
|
||||
>
|
||||
<div id="ws-delete-box">
|
||||
<div id="ws-delete-box" class="ws-delete-modal-box">
|
||||
<h3 id="ws-delete-title">Delete Workstreams</h3>
|
||||
<div id="ws-delete-error" role="alert" aria-live="assertive"></div>
|
||||
<p id="ws-delete-count"></p>
|
||||
<div id="ws-delete-list"></div>
|
||||
<div id="ws-delete-buttons">
|
||||
<div id="ws-delete-list" class="ws-delete-modal-list"></div>
|
||||
<div id="ws-delete-buttons" class="ws-delete-modal-buttons">
|
||||
<button
|
||||
id="ws-delete-cancel-btn"
|
||||
type="button"
|
||||
@@ -383,6 +384,7 @@
|
||||
</button>
|
||||
<button
|
||||
id="ws-delete-confirm-btn"
|
||||
class="ws-delete-confirm"
|
||||
type="button"
|
||||
onclick="confirmWsDelete()"
|
||||
>
|
||||
|
||||
@@ -2366,224 +2366,10 @@ audio.media-player {
|
||||
color: var(--accent);
|
||||
margin: 0;
|
||||
}
|
||||
.ws-delete-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
color: var(--fg-dim);
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
color 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
.ws-delete-btn:hover {
|
||||
color: var(--red);
|
||||
border-color: var(--red);
|
||||
}
|
||||
/* .dashboard-cards / .dashboard-card / .card-title / .card-meta moved to
|
||||
/shared/cards.css so console (Saved Coordinators) and ui/static (Saved
|
||||
Workstreams) share one source of truth for the basic card primitive.
|
||||
Delete-mode rules stay below — they're ui/static-only until coordinator
|
||||
gets the same UX. */
|
||||
|
||||
/* Delete mode */
|
||||
.dashboard-card.ws-delete-mode {
|
||||
cursor: pointer;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.04);
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected {
|
||||
cursor: default;
|
||||
}
|
||||
.dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode:hover {
|
||||
background: rgba(220, 38, 38, 0.04);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-delete-mode.ws-selected:hover {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-card-check {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
accent-color: var(--red);
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
opacity: 0;
|
||||
animation: ws-check-fadein 0.2s ease-out forwards;
|
||||
}
|
||||
@keyframes ws-check-fadein {
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
.dashboard-card.ws-selected {
|
||||
border-color: var(--red);
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
[data-theme="light"] .dashboard-card.ws-selected {
|
||||
background: rgba(220, 38, 38, 0.08);
|
||||
}
|
||||
.ws-delete-bar {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
display: flex;
|
||||
animation: ws-bar-slide 0.2s ease-out;
|
||||
}
|
||||
@keyframes ws-bar-slide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.ws-card-check {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
.ws-delete-bar.visible {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
.ws-delete-bar .ws-delete-count-label {
|
||||
font-size: 12px;
|
||||
color: var(--fg-dim);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn {
|
||||
margin-left: auto;
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 16px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:hover:not(:disabled) {
|
||||
filter: brightness(1.1);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-bar-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-cancel-btn:hover {
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
}
|
||||
.ws-delete-bar .ws-delete-selectall-btn:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--bg-highlight);
|
||||
}
|
||||
|
||||
/* Delete modal */
|
||||
#ws-delete-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
#ws-delete-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 24px;
|
||||
max-width: 480px;
|
||||
width: 90%;
|
||||
}
|
||||
#ws-delete-box h3 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 16px;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
#ws-delete-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
margin: 12px 0;
|
||||
}
|
||||
#ws-delete-list .ws-delete-item {
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
color: var(--fg-bright);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
#ws-delete-list .ws-delete-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
#ws-delete-list .ws-delete-item.ws-delete-error {
|
||||
color: var(--red);
|
||||
}
|
||||
#ws-delete-buttons {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
#ws-delete-buttons button {
|
||||
padding: 8px 20px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
}
|
||||
#ws-delete-buttons button:last-child {
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
border-color: var(--red);
|
||||
}
|
||||
#ws-delete-buttons button.ws-delete-close {
|
||||
background: transparent;
|
||||
color: var(--fg-bright);
|
||||
border-color: var(--border);
|
||||
}
|
||||
/* .dashboard-cards / .dashboard-card / .card-title / .card-meta and the
|
||||
delete-mode + modal rules are owned by /shared/cards.css so console
|
||||
(Saved Coordinators) and ui/static (Saved Workstreams) share one source
|
||||
of truth. */
|
||||
|
||||
/* Server dashboard row — clickable */
|
||||
.dash-row {
|
||||
|
||||
Reference in New Issue
Block a user