mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
e60c19befd
Wake path: - Denial metacog nudge moves to the tool channel so it drains with the denied tool batch instead of the next user-message seam. - wake_workstream_if_pending: shared wake gate for watch fires on already-idle workstreams (no IDLE transition for the watcher to observe), wired as wake_fn at every set_watch_runner site via the shared _watch_fire_wake_fn helper (closes over the Workstream OBJECT — after eviction+restore an id-keyed manager lookup would miss). - session_worker exit backstop re-runs the wake gate the moment worker ownership clears: IDLE fans out on the worker thread, so transition-time wakes always landed on the reuse path and no-op'd (the coordinator idle_children strand). - deliver_wake_nudge_from_queue contains GenerationCancelled — it is the wake worker's run() closure and only Exception is caught downstream. Watch delivery: - Terminal fires that cannot reach their workstream are HELD and redelivered on min(interval, 60s) without re-running the command, bounded by MAX_DELIVERY_ATTEMPTS per cycle and the watch's own max_polls across cycles; the poll charge commits durably at hold time so restarts stay budget-bounded. - Restore admission control: per-ws dedup + MAX_CONCURRENT_RESTORES cap, presence-only re-check under the lock, detection-only stall alerts (reclaiming a wedged admission would trade capped degradation for total poll-pool collapse). - Permanent-vs-transient restore taxonomy: corrupt persona stamp and genuinely-missing history (confirmed by a raising storage probe — the resume loader swallows read blips into []) deactivate the watch immediately; everything else holds and retries. - Cancel-race defense: delivery paths re-check is_watch_active before stashing/dispatching, cancel paths write the row BEFORE forget_terminal_dispatched, the HTTP cancel endpoint clears runner state, and a per-tick sweep bounds the residual stash-after-clear interleaving to one check_interval. - Abandon/exhaustion commits are write-then-clear so storage that can read but not write retries the row write instead of re-running the command every cycle; the fresh-fire unrestorable path stashes before its deactivation write for the same reason. Registry follows identity: - The dispatch registry is keyed by _ws_id at registration time; every rebind now moves it: non-fork resume() and /new go through _follow_watch_registration (new key live before the old is removed, never stealing a registration another live session holds), removals are owner-checked so tearing down a watch-restore shell or a resumed-away session cannot unregister a live pane, the restore shell yields to a registration that appears mid-restore, CLI --resume registers after the successful resume, and both the open path and the detail-GET lazy rehydrate wire the registration. Teardown gating and backpressure honesty: - cleanup_session_ui marks ws._closed FIRST under ws._lock — every teardown path (close, close_idle, evict, delete, discard) funnels through it — and session_worker.send re-checks under the same lock, so a wake can never spawn a worker on a torn-down workstream. - Create responses carry initial_message_status when the initial message could not be delivered (queue_full / refused_closed) instead of reading as success; staged attachments survive for the retry; /send surfaces a closed workstream as 404 rather than queue_full. Docs/spec: OpenAPI artifacts regenerated; api-reference documents the new create-response field; TS SDK type extended. Tests: ~30 new pins (cancel races, budget durability across restarts, owner-checked registry moves, teardown gating, stall alerts, backpressure surfaces, wait_until final re-check); wide subsystem sweep green (2353 passed).
1290 lines
50 KiB
Python
1290 lines
50 KiB
Python
"""Tests for the watch module — duration parsing, condition evaluation, WatchRunner."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import threading
|
||
import time
|
||
from datetime import UTC, datetime
|
||
from typing import Any
|
||
from unittest.mock import MagicMock
|
||
|
||
import pytest
|
||
|
||
from tests._helpers import wait_until
|
||
from turnstone.core.watch import (
|
||
WatchRunner,
|
||
build_watch_reminder,
|
||
evaluate_condition,
|
||
format_interval,
|
||
format_watch_message,
|
||
parse_duration,
|
||
validate_condition,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# parse_duration
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestParseDuration:
|
||
def test_seconds(self):
|
||
assert parse_duration("30s") == 30.0
|
||
|
||
def test_minutes(self):
|
||
assert parse_duration("5m") == 300.0
|
||
|
||
def test_hours(self):
|
||
assert parse_duration("1h") == 3600.0
|
||
|
||
def test_compound(self):
|
||
assert parse_duration("2h30m") == 9000.0
|
||
|
||
def test_bare_number(self):
|
||
assert parse_duration("90") == 90.0
|
||
|
||
def test_bare_float(self):
|
||
assert parse_duration("10.5") == 10.5
|
||
|
||
def test_whitespace(self):
|
||
assert parse_duration(" 5m ") == 300.0
|
||
|
||
def test_case_insensitive(self):
|
||
assert parse_duration("1H30M") == 5400.0
|
||
|
||
def test_empty_raises(self):
|
||
with pytest.raises(ValueError, match="empty"):
|
||
parse_duration("")
|
||
|
||
def test_invalid_raises(self):
|
||
with pytest.raises(ValueError, match="invalid duration"):
|
||
parse_duration("abc")
|
||
|
||
def test_negative_raises(self):
|
||
with pytest.raises(ValueError, match="positive"):
|
||
parse_duration("-5")
|
||
|
||
def test_zero_raises(self):
|
||
with pytest.raises(ValueError, match="positive"):
|
||
parse_duration("0")
|
||
|
||
def test_zero_duration_raises(self):
|
||
with pytest.raises(ValueError, match="positive"):
|
||
parse_duration("0s")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# validate_condition
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestValidateCondition:
|
||
def test_valid_expression(self):
|
||
assert validate_condition('data["state"] == "MERGED"') is None
|
||
|
||
def test_valid_simple(self):
|
||
assert validate_condition('"error" in output') is None
|
||
|
||
def test_valid_compound(self):
|
||
assert validate_condition('changed and "ready" in output.lower()') is None
|
||
|
||
def test_syntax_error(self):
|
||
result = validate_condition("if True:")
|
||
assert result is not None
|
||
assert "syntax" in result.lower()
|
||
|
||
def test_incomplete_expression(self):
|
||
result = validate_condition("==")
|
||
assert result is not None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# evaluate_condition
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestEvaluateCondition:
|
||
def test_none_first_poll_no_fire(self):
|
||
"""With stop_on=None, first poll (prev_output=None) should not fire."""
|
||
fired, reason = evaluate_condition(None, "hello", 0, None)
|
||
assert not fired
|
||
|
||
def test_none_change_detected(self):
|
||
fired, reason = evaluate_condition(None, "world", 0, "hello")
|
||
assert fired
|
||
assert "changed" in reason
|
||
|
||
def test_none_no_change(self):
|
||
fired, reason = evaluate_condition(None, "same", 0, "same")
|
||
assert not fired
|
||
|
||
def test_string_match(self):
|
||
fired, reason = evaluate_condition('"error" in output', "has error here", 0, None)
|
||
assert fired
|
||
|
||
def test_string_no_match(self):
|
||
fired, reason = evaluate_condition('"error" in output', "all good", 0, None)
|
||
assert not fired
|
||
|
||
def test_exit_code(self):
|
||
fired, reason = evaluate_condition("exit_code != 0", "fail", 1, None)
|
||
assert fired
|
||
|
||
def test_exit_code_zero(self):
|
||
fired, reason = evaluate_condition("exit_code != 0", "ok", 0, None)
|
||
assert not fired
|
||
|
||
def test_json_data(self):
|
||
output = '{"state": "MERGED"}'
|
||
fired, reason = evaluate_condition('data["state"] == "MERGED"', output, 0, None)
|
||
assert fired
|
||
|
||
def test_json_data_no_match(self):
|
||
output = '{"state": "OPEN"}'
|
||
fired, reason = evaluate_condition('data["state"] == "MERGED"', output, 0, None)
|
||
assert not fired
|
||
|
||
def test_json_data_none_for_non_json(self):
|
||
"""Non-JSON output should have data=None."""
|
||
fired, reason = evaluate_condition("data is None", "plain text", 0, None)
|
||
assert fired
|
||
|
||
def test_changed_variable(self):
|
||
fired, reason = evaluate_condition("changed", "new", 0, "old")
|
||
assert fired
|
||
|
||
def test_changed_false(self):
|
||
fired, reason = evaluate_condition("changed", "same", 0, "same")
|
||
assert not fired
|
||
|
||
def test_compound_condition(self):
|
||
fired, reason = evaluate_condition(
|
||
'changed and "ready" in output.lower()',
|
||
"System Ready",
|
||
0,
|
||
"System Starting",
|
||
)
|
||
assert fired
|
||
|
||
def test_invalid_expression_no_crash(self):
|
||
fired, reason = evaluate_condition("1/0", "hello", 0, None)
|
||
assert not fired
|
||
assert "error" in reason.lower()
|
||
|
||
def test_no_import_builtin(self):
|
||
"""__import__ should not be accessible."""
|
||
fired, reason = evaluate_condition("__import__('os')", "hello", 0, None)
|
||
assert not fired
|
||
assert "error" in reason.lower()
|
||
|
||
def test_no_open_builtin(self):
|
||
fired, reason = evaluate_condition("open('/etc/passwd')", "hello", 0, None)
|
||
assert not fired
|
||
assert "error" in reason.lower()
|
||
|
||
def test_no_exec_builtin(self):
|
||
fired, reason = evaluate_condition("exec('print(1)')", "hello", 0, None)
|
||
assert not fired
|
||
assert "error" in reason.lower()
|
||
|
||
def test_no_eval_builtin(self):
|
||
fired, reason = evaluate_condition("eval('1+1')", "hello", 0, None)
|
||
assert not fired
|
||
assert "error" in reason.lower()
|
||
|
||
def test_no_compile_builtin(self):
|
||
fired, reason = evaluate_condition("compile('1','','eval')", "hello", 0, None)
|
||
assert not fired
|
||
assert "error" in reason.lower()
|
||
|
||
def test_safe_len(self):
|
||
fired, reason = evaluate_condition("len(output) > 0", "hello", 0, None)
|
||
assert fired
|
||
|
||
def test_safe_sorted(self):
|
||
fired, reason = evaluate_condition("sorted([3,1,2]) == [1,2,3]", "x", 0, None)
|
||
assert fired
|
||
|
||
def test_data_get_method(self):
|
||
output = '{"mergedAt": "2024-01-15"}'
|
||
fired, reason = evaluate_condition('data.get("mergedAt") is not None', output, 0, None)
|
||
assert fired
|
||
|
||
def test_prev_output_available(self):
|
||
fired, reason = evaluate_condition(
|
||
"prev_output is not None and output != prev_output",
|
||
"new",
|
||
0,
|
||
"old",
|
||
)
|
||
assert fired
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# format_interval
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestFormatInterval:
|
||
def test_seconds(self):
|
||
assert format_interval(30) == "30s"
|
||
|
||
def test_exactly_60(self):
|
||
assert format_interval(60) == "1m"
|
||
|
||
def test_minutes(self):
|
||
assert format_interval(300) == "5m"
|
||
|
||
def test_exactly_3600(self):
|
||
assert format_interval(3600) == "1h"
|
||
|
||
def test_hours_and_minutes(self):
|
||
assert format_interval(5400) == "1h30m"
|
||
|
||
def test_hours_only(self):
|
||
assert format_interval(7200) == "2h"
|
||
|
||
def test_large_value(self):
|
||
assert format_interval(86400) == "24h"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# format_watch_message
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestFormatWatchMessage:
|
||
def test_basic(self):
|
||
msg = format_watch_message(
|
||
name="pr-review",
|
||
command="gh pr view --json state",
|
||
output='{"state": "MERGED"}',
|
||
poll_count=5,
|
||
max_polls=100,
|
||
elapsed_secs=1500,
|
||
stop_on='data["state"] == "MERGED"',
|
||
is_final=True,
|
||
reason='condition met: data["state"] == "MERGED"',
|
||
)
|
||
assert "pr-review" in msg
|
||
assert "poll #5/100" in msg
|
||
assert "25m" in msg
|
||
assert "gh pr view --json state" in msg
|
||
assert "MERGED" in msg
|
||
assert "auto-cancelled" in msg.lower()
|
||
# Model should see the condition it was waiting for
|
||
assert "condition:" in msg.lower()
|
||
|
||
def test_non_final(self):
|
||
msg = format_watch_message(
|
||
name="deploy",
|
||
command="curl -s http://localhost/health",
|
||
output="ok",
|
||
poll_count=3,
|
||
max_polls=50,
|
||
elapsed_secs=90,
|
||
stop_on=None,
|
||
is_final=False,
|
||
reason="",
|
||
)
|
||
assert "deploy" in msg
|
||
assert "auto-cancelled" not in msg.lower()
|
||
# Change-detection mode should be indicated
|
||
assert "output change" in msg.lower()
|
||
|
||
def test_max_polls_final(self):
|
||
msg = format_watch_message(
|
||
name="test",
|
||
command="echo hello",
|
||
output="hello",
|
||
poll_count=100,
|
||
max_polls=100,
|
||
elapsed_secs=6000,
|
||
stop_on=None,
|
||
is_final=True,
|
||
reason="",
|
||
)
|
||
assert "max polls" in msg.lower()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# build_watch_reminder
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestBuildWatchReminder:
|
||
"""The structured-reminder builder lifts ``format_watch_message``'s
|
||
args into a dict the dispatch closure can pass to
|
||
``WatchRunner._dispatch_result``. ``text`` matches the formatter's
|
||
output verbatim (so compaction / channel adapters / wire splice
|
||
keep their behaviour), and the optional fields ride alongside for
|
||
the frontend's ``.msg.watch-result`` card.
|
||
"""
|
||
|
||
def test_emits_text_body_and_fields(self):
|
||
kwargs = dict(
|
||
name="pr-review",
|
||
command="gh pr view --json state",
|
||
output='{"state": "MERGED"}',
|
||
poll_count=5,
|
||
max_polls=100,
|
||
elapsed_secs=1500,
|
||
stop_on='data["state"] == "MERGED"',
|
||
is_final=True,
|
||
reason='condition met: data["state"] == "MERGED"',
|
||
)
|
||
reminder = build_watch_reminder(**kwargs)
|
||
# Round-trip with format_watch_message — text is the same body
|
||
# the wire splice + channel adapters have always seen.
|
||
assert reminder["text"] == format_watch_message(**kwargs)
|
||
# Optional fields ride alongside.
|
||
assert reminder["type"] == "watch_triggered"
|
||
assert reminder["watch_name"] == "pr-review"
|
||
assert reminder["command"] == "gh pr view --json state"
|
||
# The raw shell output rides as its own field so the FE card body shows
|
||
# it alone (no header / command repeat); the wire ``text`` keeps the
|
||
# full prose for the model.
|
||
assert reminder["output"] == '{"state": "MERGED"}'
|
||
assert reminder["poll_count"] == 5
|
||
assert reminder["max_polls"] == 100
|
||
assert reminder["is_final"] is True
|
||
|
||
def test_non_final_carries_is_final_false(self):
|
||
reminder = build_watch_reminder(
|
||
name="deploy",
|
||
command="curl -s http://localhost/health",
|
||
output="ok",
|
||
poll_count=3,
|
||
max_polls=50,
|
||
elapsed_secs=90,
|
||
stop_on=None,
|
||
is_final=False,
|
||
reason="",
|
||
)
|
||
assert reminder["is_final"] is False
|
||
assert reminder["poll_count"] == 3
|
||
# No "auto-cancelled" body for non-final fires.
|
||
assert "auto-cancelled" not in reminder["text"].lower()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# WatchRunner
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestWatchRunner:
|
||
def _make_runner(self, storage=None, **kwargs):
|
||
if storage is None:
|
||
storage = MagicMock()
|
||
storage.list_due_watches.return_value = []
|
||
return WatchRunner(
|
||
storage=storage,
|
||
node_id="test-node",
|
||
check_interval=0.1,
|
||
tool_timeout=5,
|
||
**kwargs,
|
||
)
|
||
|
||
def test_start_stop(self):
|
||
runner = self._make_runner()
|
||
runner.start()
|
||
assert runner._thread is not None
|
||
assert runner._thread.is_alive()
|
||
runner.stop()
|
||
assert runner._thread is None
|
||
|
||
def test_tick_calls_list_due(self):
|
||
storage = MagicMock()
|
||
storage.list_due_watches.return_value = []
|
||
runner = self._make_runner(storage=storage)
|
||
runner._tick()
|
||
storage.list_due_watches.assert_called_once()
|
||
|
||
def test_poll_watch_runs_command(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage=storage)
|
||
dispatch_fn = MagicMock()
|
||
runner.set_dispatch_fn("ws-1", dispatch_fn)
|
||
|
||
watch_row = {
|
||
"watch_id": "abc123",
|
||
"ws_id": "ws-1",
|
||
"name": "test-watch",
|
||
"command": "echo hello",
|
||
"stop_on": '"hello" in output',
|
||
"max_polls": 100,
|
||
"poll_count": 0,
|
||
"last_output": None,
|
||
"interval_secs": 60,
|
||
"created": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S"),
|
||
}
|
||
runner._poll_watch(watch_row)
|
||
|
||
# Should update the watch in storage
|
||
storage.update_watch.assert_called_once()
|
||
call_kwargs = storage.update_watch.call_args
|
||
assert call_kwargs[0][0] == "abc123" # watch_id
|
||
assert call_kwargs[1]["poll_count"] == 1
|
||
# Condition should fire (output contains "hello")
|
||
assert call_kwargs[1]["active"] is False # deactivated
|
||
# Should dispatch result
|
||
dispatch_fn.assert_called_once()
|
||
|
||
def test_poll_watch_no_fire_on_first_change_detection(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage=storage)
|
||
dispatch_fn = MagicMock()
|
||
runner.set_dispatch_fn("ws-1", dispatch_fn)
|
||
|
||
watch_row = {
|
||
"watch_id": "abc123",
|
||
"ws_id": "ws-1",
|
||
"name": "test-watch",
|
||
"command": "echo hello",
|
||
"stop_on": None, # change detection
|
||
"max_polls": 100,
|
||
"poll_count": 0,
|
||
"last_output": None, # first poll
|
||
"interval_secs": 60,
|
||
"created": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S"),
|
||
}
|
||
runner._poll_watch(watch_row)
|
||
|
||
# First poll with change detection should not fire
|
||
dispatch_fn.assert_not_called()
|
||
call_kwargs = storage.update_watch.call_args
|
||
# Watch should remain active
|
||
assert "active" not in call_kwargs[1] or call_kwargs[1].get("active") is not False
|
||
|
||
def test_max_polls_deactivates(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage=storage)
|
||
dispatch_fn = MagicMock()
|
||
runner.set_dispatch_fn("ws-1", dispatch_fn)
|
||
|
||
watch_row = {
|
||
"watch_id": "abc123",
|
||
"ws_id": "ws-1",
|
||
"name": "test-watch",
|
||
"command": "echo hello",
|
||
"stop_on": '"never" in output', # won't fire
|
||
"max_polls": 5,
|
||
"poll_count": 4, # next is #5 = max
|
||
"last_output": "hello\n",
|
||
"interval_secs": 60,
|
||
"created": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S"),
|
||
}
|
||
runner._poll_watch(watch_row)
|
||
|
||
call_kwargs = storage.update_watch.call_args
|
||
assert call_kwargs[1]["active"] is False
|
||
assert call_kwargs[1]["poll_count"] == 5
|
||
dispatch_fn.assert_called_once()
|
||
|
||
def test_blocked_command_deactivates(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage=storage)
|
||
|
||
watch_row = {
|
||
"watch_id": "abc123",
|
||
"ws_id": "ws-1",
|
||
"name": "test-watch",
|
||
"command": "rm -rf /",
|
||
"stop_on": None,
|
||
"max_polls": 100,
|
||
"poll_count": 0,
|
||
"last_output": None,
|
||
"interval_secs": 60,
|
||
"created": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S"),
|
||
}
|
||
runner._poll_watch(watch_row)
|
||
|
||
storage.update_watch.assert_called_once()
|
||
call_kwargs = storage.update_watch.call_args
|
||
assert call_kwargs[0][0] == "abc123"
|
||
assert call_kwargs[1]["active"] is False
|
||
|
||
def test_dispatch_fn_registry(self):
|
||
runner = self._make_runner()
|
||
fn1 = MagicMock()
|
||
fn2 = MagicMock()
|
||
|
||
runner.set_dispatch_fn("ws-1", fn1)
|
||
runner.set_dispatch_fn("ws-2", fn2)
|
||
|
||
# ``_dispatch_result`` takes a structured reminder dict, not a
|
||
# bare string.
|
||
reminder1 = {"type": "watch_triggered", "text": "msg1"}
|
||
runner._dispatch_result("ws-1", reminder1, "watch-a")
|
||
fn1.assert_called_once_with(reminder1, "watch-a")
|
||
fn2.assert_not_called()
|
||
|
||
runner.remove_dispatch_fn("ws-1")
|
||
# After removal, dispatch should try restore_fn
|
||
reminder2 = {"type": "watch_triggered", "text": "msg2"}
|
||
runner._dispatch_result("ws-1", reminder2, "watch-b")
|
||
fn1.assert_called_once() # still just the one call
|
||
|
||
def test_restore_fn_called_for_evicted(self):
|
||
restored_fn = MagicMock()
|
||
restore_fn = MagicMock(return_value=restored_fn)
|
||
runner = self._make_runner(restore_fn=restore_fn)
|
||
|
||
reminder = {"type": "watch_triggered", "text": "hello"}
|
||
runner._dispatch_result("ws-evicted", reminder, "watch-x")
|
||
restore_fn.assert_called_once_with("ws-evicted")
|
||
restored_fn.assert_called_once_with(reminder, "watch-x")
|
||
|
||
def test_get_dispatch_fn_returns_registered_fn(self):
|
||
"""``get_dispatch_fn`` is the public accessor used by the
|
||
server-side restore path to retrieve the per-ws closure that
|
||
``set_watch_runner`` constructed during workstream rehydrate.
|
||
"""
|
||
runner = self._make_runner()
|
||
fn = MagicMock()
|
||
runner.set_dispatch_fn("ws-1", fn)
|
||
assert runner.get_dispatch_fn("ws-1") is fn
|
||
# Unknown ws → None.
|
||
assert runner.get_dispatch_fn("ws-missing") is None
|
||
# Owner-checked removal: a non-owner's teardown must not remove a
|
||
# still-live registration (restore shell vs reopened pane).
|
||
runner.remove_dispatch_fn("ws-1", owner=MagicMock())
|
||
assert runner.get_dispatch_fn("ws-1") is fn
|
||
# The owner (or a blind removal) does remove it.
|
||
runner.remove_dispatch_fn("ws-1", owner=fn)
|
||
assert runner.get_dispatch_fn("ws-1") is None
|
||
|
||
def test_run_command_success(self):
|
||
runner = self._make_runner()
|
||
output, code = runner._run_command("echo hello")
|
||
assert "hello" in output
|
||
assert code == 0
|
||
|
||
def test_run_command_failure(self):
|
||
runner = self._make_runner()
|
||
output, code = runner._run_command("exit 42")
|
||
assert code == 42
|
||
|
||
def test_run_command_timeout(self):
|
||
runner = self._make_runner()
|
||
runner._tool_timeout = 1
|
||
output, code = runner._run_command("sleep 30")
|
||
assert "timed out" in output.lower()
|
||
assert code == -1
|
||
|
||
|
||
def _watch_row(**over: Any) -> dict[str, Any]:
|
||
"""A firing watch row (condition matches ``echo hello``); override
|
||
fields per test."""
|
||
row: dict[str, Any] = {
|
||
"watch_id": "abc123",
|
||
"ws_id": "ws-1",
|
||
"name": "test-watch",
|
||
"command": "echo hello",
|
||
"stop_on": '"hello" in output',
|
||
"max_polls": 100,
|
||
"poll_count": 0,
|
||
"last_output": None,
|
||
"interval_secs": 60,
|
||
"created": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S"),
|
||
}
|
||
row.update(over)
|
||
return row
|
||
|
||
|
||
def _slow_restore(runner: WatchRunner, ws_id: str, calls: list[str], lock: threading.Lock) -> Any:
|
||
"""Restore_fn stand-in: record the call, register a dispatch fn (as the
|
||
real restore does via ``set_watch_runner``), and sleep briefly so a
|
||
concurrent second caller is guaranteed to be waiting on ``_restore_lock``
|
||
when we return.
|
||
"""
|
||
with lock:
|
||
calls.append(ws_id)
|
||
time.sleep(0.05)
|
||
fn = MagicMock()
|
||
runner.set_dispatch_fn(ws_id, fn)
|
||
return fn
|
||
|
||
|
||
class TestWatchRunnerDeliveryRetry:
|
||
"""Delivery failure HOLDS the built reminder and re-delivers it on a
|
||
later tick — never re-running the command, so a transient stop_on match
|
||
isn't lost — bounded by ``MAX_DELIVERY_ATTEMPTS``. Until delivery lands
|
||
the row commits only ``next_poll`` plus the fire's durable poll charge:
|
||
no baseline advance and no deactivation of a fire the model never saw,
|
||
while a restart mid-hold (which re-runs the command) stays bounded by
|
||
``max_polls``."""
|
||
|
||
def _make_runner(self, storage: Any, **kwargs: Any) -> WatchRunner:
|
||
return WatchRunner(
|
||
storage=storage,
|
||
node_id="test-node",
|
||
check_interval=0.1,
|
||
tool_timeout=5,
|
||
**kwargs,
|
||
)
|
||
|
||
def test_delivery_failure_holds_reminder_and_defers_row(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
# No dispatch fn registered, no restore_fn → delivery fails.
|
||
runner = self._make_runner(storage)
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
# Row commit is the retry cadence + this fire's durable poll charge
|
||
# — the fire stays fully retryable, and a restart mid-hold (which
|
||
# re-runs the command) stays bounded by max_polls.
|
||
storage.update_watch.assert_called_once()
|
||
args, kwargs = storage.update_watch.call_args
|
||
assert args[0] == "abc123"
|
||
assert set(kwargs) == {"next_poll", "poll_count"}
|
||
assert kwargs["poll_count"] == 1 # charged durably at hold time
|
||
assert kwargs["next_poll"] # advanced, not cleared
|
||
# The reminder is HELD for re-delivery; the row is NOT marked
|
||
# terminal-dispatched (the model never saw it).
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
assert runner._pending_delivery["abc123"]["attempts"] == 1
|
||
with runner._terminal_dispatched_lock:
|
||
assert "abc123" not in runner._terminal_dispatched
|
||
|
||
def test_redelivery_uses_held_reminder_without_rerunning_command(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage)
|
||
|
||
# Poll 1: fails → holds. Capture the exact held reminder object.
|
||
runner._poll_watch(_watch_row())
|
||
with runner._pending_delivery_lock:
|
||
held = runner._pending_delivery["abc123"]["reminder"]
|
||
|
||
# ws restored: register a fn, and make _run_command explode so the
|
||
# test proves re-delivery does NOT re-run the command.
|
||
dispatch_fn = MagicMock()
|
||
runner.set_dispatch_fn("ws-1", dispatch_fn)
|
||
runner._run_command = MagicMock( # type: ignore[method-assign]
|
||
side_effect=AssertionError("command must not re-run on re-delivery")
|
||
)
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
# Delivered the SAME held reminder; command untouched; committed
|
||
# terminal with the ORIGINAL fire's poll_count; hold cleared.
|
||
dispatch_fn.assert_called_once()
|
||
assert dispatch_fn.call_args[0][0] is held
|
||
runner._run_command.assert_not_called()
|
||
_a, kwargs = storage.update_watch.call_args
|
||
assert kwargs["active"] is False
|
||
assert kwargs["poll_count"] == 1 # retries consumed no ADDITIONAL budget
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
|
||
def test_transient_exhaustion_keeps_watch_active(self):
|
||
# A purely transient cause (no fn, no restore → returns False) that
|
||
# outlasts the attempt budget must NOT silently deactivate the watch
|
||
# — it drops the held reminder, charges ONE poll to the max_polls
|
||
# budget, and leaves the watch active to re-fire on its next
|
||
# interval. The baseline (last_output) stays uncommitted so a
|
||
# delta-style stop_on re-fires on the change the model never saw.
|
||
from turnstone.core.watch import MAX_DELIVERY_ATTEMPTS
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage) # always fails, transiently
|
||
|
||
# Poll 1 fires + holds (attempts=1); polls 2..MAX bump attempts, the
|
||
# MAX-th hitting the exhaustion ceiling.
|
||
for _ in range(MAX_DELIVERY_ATTEMPTS):
|
||
runner._poll_watch(_watch_row())
|
||
|
||
# Hold dropped, but the watch was NEVER deactivated — no active=False
|
||
# commit anywhere; the final commit charges the poll and re-schedules
|
||
# (no last_output → the fire re-detects next cycle).
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
deactivations = [
|
||
c for c in storage.update_watch.call_args_list if c.kwargs.get("active") is False
|
||
]
|
||
assert deactivations == []
|
||
_a, kwargs = storage.update_watch.call_args # last commit
|
||
assert set(kwargs) == {"poll_count", "next_poll"}
|
||
assert kwargs["poll_count"] == 1 # one poll charged to the budget
|
||
|
||
def test_transient_exhaustion_with_budget_spent_deactivates(self):
|
||
# The keep-alive-on-transient behavior is bounded by the watch's own
|
||
# max_polls budget: once poll_count reaches it, exhaustion commits
|
||
# the held (deactivating) update instead of re-running the command
|
||
# every interval forever against an unreachable workstream.
|
||
from turnstone.core.watch import MAX_DELIVERY_ATTEMPTS
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage) # always fails, transiently
|
||
|
||
for _ in range(MAX_DELIVERY_ATTEMPTS):
|
||
runner._poll_watch(_watch_row(max_polls=1)) # budget spent on fire 1
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
_a, kwargs = storage.update_watch.call_args # last commit
|
||
assert kwargs["active"] is False # deactivated: budget spent
|
||
assert kwargs["poll_count"] == 1
|
||
|
||
def test_held_delivery_retries_on_capped_cadence_not_interval(self):
|
||
# Re-delivery is a cheap in-memory dispatch — a daily watch whose
|
||
# fire hit a busy restore slot must retry within
|
||
# DELIVERY_RETRY_CAP_SECS, not sit on the reminder for 24 h.
|
||
from turnstone.core.watch import DELIVERY_RETRY_CAP_SECS
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage)
|
||
|
||
runner._poll_watch(_watch_row(interval_secs=86_400))
|
||
|
||
_a, kwargs = storage.update_watch.call_args
|
||
assert set(kwargs) == {"next_poll", "poll_count"}
|
||
retry_at = datetime.strptime(kwargs["next_poll"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC)
|
||
delta = (retry_at - datetime.now(UTC)).total_seconds()
|
||
assert 0 < delta <= DELIVERY_RETRY_CAP_SECS + 5 # capped, not 86400
|
||
|
||
def test_permanent_unrestorable_deactivates_immediately(self):
|
||
# A permanent failure (restore raises WatchWorkstreamUnrestorable,
|
||
# e.g. corrupt persona stamp) deactivates the watch on the FIRST
|
||
# fire — no held reminder, no waiting out the attempt budget.
|
||
from turnstone.core.watch import WatchWorkstreamUnrestorable
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_fn = MagicMock(side_effect=WatchWorkstreamUnrestorable("ws-1"))
|
||
runner = self._make_runner(storage, restore_fn=restore_fn)
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
restore_fn.assert_called_once_with("ws-1") # not retried 5×
|
||
_a, kwargs = storage.update_watch.call_args
|
||
assert kwargs["active"] is False # deactivated now
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery # nothing held
|
||
# The admission slot is released even on the raising path.
|
||
with runner._restore_lock:
|
||
assert "ws-1" not in runner._restoring
|
||
|
||
def test_pending_cleared_on_already_dispatched_retry(self):
|
||
# A re-delivery that succeeded but whose row-commit raised leaves
|
||
# the id in BOTH _terminal_dispatched and _pending_delivery. The
|
||
# next tick's already-dispatched branch must clear the hold too, or
|
||
# it leaks once the row deactivates.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage)
|
||
with runner._terminal_dispatched_lock:
|
||
runner._terminal_dispatched.add("abc123")
|
||
runner._stash_pending_delivery("abc123", {"text": "x"}, {"active": False}, attempts=1)
|
||
|
||
runner._poll_watch(_watch_row()) # hits the already_dispatched branch
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
with runner._terminal_dispatched_lock:
|
||
assert "abc123" not in runner._terminal_dispatched
|
||
|
||
def test_restore_capacity_full_defers_without_restoring(self):
|
||
# When MAX_CONCURRENT_RESTORES restores are already in flight, a
|
||
# new evicted-ws poll must DEFER (return False, hold) rather than
|
||
# block a poll slot — and must not start a restore.
|
||
from turnstone.core.watch import MAX_CONCURRENT_RESTORES
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_fn = MagicMock(return_value=MagicMock())
|
||
runner = self._make_runner(storage, restore_fn=restore_fn)
|
||
# Saturate the restore admission with other in-flight ws_ids.
|
||
with runner._restore_lock:
|
||
for i in range(MAX_CONCURRENT_RESTORES):
|
||
runner._restoring[f"other-{i}"] = time.monotonic()
|
||
|
||
result = runner._dispatch_result("ws-evicted", {"text": "x"}, "w1")
|
||
|
||
assert result is False # deferred
|
||
restore_fn.assert_not_called() # no restore admitted
|
||
|
||
def test_race_won_admission_delivers_outside_restore_lock(self):
|
||
# A dispatch fn registered between the fast-path miss and the
|
||
# admission check must be delivered WITHOUT holding _restore_lock:
|
||
# the closure can block (ws._lock, wake-thread spawn), and running
|
||
# it under the lock serialises every restore admission on the node
|
||
# behind one delivery.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_fn = MagicMock(return_value=None)
|
||
runner = self._make_runner(storage, restore_fn=restore_fn)
|
||
|
||
lock_free_during_dispatch: list[bool] = []
|
||
|
||
def probe(reminder: dict[str, Any], watch_id: str) -> None:
|
||
ok = runner._restore_lock.acquire(blocking=False)
|
||
lock_free_during_dispatch.append(ok)
|
||
if ok:
|
||
runner._restore_lock.release()
|
||
|
||
real_try = runner._try_dispatch_fn
|
||
calls = {"n": 0}
|
||
|
||
def fake_try(ws_id: str, reminder: dict[str, Any], watch_id: str) -> bool | None:
|
||
calls["n"] += 1
|
||
if calls["n"] == 1:
|
||
# Simulate a restore completing between the fast path and
|
||
# the admission check: the fn appears "while we waited".
|
||
runner.set_dispatch_fn("ws-1", probe)
|
||
return None
|
||
return real_try(ws_id, reminder, watch_id)
|
||
|
||
runner._try_dispatch_fn = fake_try # type: ignore[method-assign]
|
||
|
||
result = runner._dispatch_result("ws-1", {"text": "x"}, "w1")
|
||
|
||
assert result is True # race-won fn delivered
|
||
assert lock_free_during_dispatch == [True] # ...outside the lock
|
||
restore_fn.assert_not_called() # no restore admitted for a live fn
|
||
|
||
def test_restore_returning_none_holds(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_fn = MagicMock(return_value=None) # e.g. all slots active
|
||
runner = self._make_runner(storage, restore_fn=restore_fn)
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
restore_fn.assert_called_once_with("ws-1")
|
||
_a, kwargs = storage.update_watch.call_args
|
||
assert set(kwargs) == {"next_poll", "poll_count"}
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
|
||
def test_live_fn_raise_holds_without_restoring(self):
|
||
# A registered fn that RAISES means the ws is live; we must NOT fall
|
||
# through to restore (that would spawn a duplicate session on a live
|
||
# conversation). The reminder is held for re-delivery instead.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_fn = MagicMock()
|
||
runner = self._make_runner(storage, restore_fn=restore_fn)
|
||
runner.set_dispatch_fn("ws-1", MagicMock(side_effect=RuntimeError("stale closure")))
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
restore_fn.assert_not_called()
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
_a, kwargs = storage.update_watch.call_args
|
||
assert set(kwargs) == {"next_poll", "poll_count"}
|
||
|
||
def test_forget_terminal_dispatched_clears_held_reminder(self):
|
||
# User-cancel takes the row out of the due view; its held reminder
|
||
# must be dropped too or it would leak (never re-polled).
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage)
|
||
runner._poll_watch(_watch_row())
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
|
||
runner.forget_terminal_dispatched("abc123")
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
|
||
def test_abandon_write_failure_keeps_hold_for_write_retry(self):
|
||
# Reads-succeed/writes-fail storage (e.g. disk-full SQLite): a
|
||
# failed abandon commit must keep the hold so the next tick retries
|
||
# the WRITE via the redeliver path — the clear-first order let the
|
||
# row re-list into a fresh COMMAND RUN every attempt-budget cycle,
|
||
# forever, with the poll budget never advancing.
|
||
from turnstone.core.watch import WatchWorkstreamUnrestorable
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage)
|
||
runner._poll_watch(_watch_row()) # fire → hold (write still OK here)
|
||
with runner._pending_delivery_lock:
|
||
pending = dict(runner._pending_delivery["abc123"])
|
||
|
||
runner._restore_fn = MagicMock( # type: ignore[assignment]
|
||
side_effect=WatchWorkstreamUnrestorable("ws-1")
|
||
)
|
||
storage.update_watch.side_effect = RuntimeError("disk full")
|
||
|
||
with pytest.raises(RuntimeError):
|
||
runner._redeliver_pending(_watch_row(), pending)
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery # hold survived
|
||
|
||
def test_unrestorable_abandon_write_failure_keeps_hold(self):
|
||
# Fresh-fire permanent failure whose deactivation write fails must
|
||
# not strand the still-active row into a fresh command run every
|
||
# tick: the stash routes the next tick into the redeliver path,
|
||
# which retries the WRITE — never the command.
|
||
from turnstone.core.watch import WatchWorkstreamUnrestorable
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.side_effect = RuntimeError("disk full")
|
||
restore_fn = MagicMock(side_effect=WatchWorkstreamUnrestorable("ws-1"))
|
||
runner = self._make_runner(storage, restore_fn=restore_fn)
|
||
|
||
with pytest.raises(RuntimeError):
|
||
runner._poll_watch(_watch_row())
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery # hold survived
|
||
|
||
# Next tick: the write retries and lands; the command never re-runs.
|
||
runner._run_command = MagicMock( # type: ignore[method-assign]
|
||
side_effect=AssertionError("command must not re-run")
|
||
)
|
||
storage.update_watch.side_effect = None
|
||
storage.update_watch.return_value = True
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
runner._run_command.assert_not_called()
|
||
_a, kwargs = storage.update_watch.call_args
|
||
assert kwargs["active"] is False # deactivation landed on the retry
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
|
||
def test_exhaustion_write_failure_keeps_hold_for_write_retry(self):
|
||
# Same pathology on the transient-exhaustion branch: the charge
|
||
# commit failing must keep the hold (write retried next tick), not
|
||
# drop it into a fresh command cycle with the budget never durable.
|
||
from turnstone.core.watch import MAX_DELIVERY_ATTEMPTS
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage) # no fn, no restore → transient
|
||
runner._poll_watch(_watch_row()) # fire → hold
|
||
with runner._pending_delivery_lock:
|
||
runner._pending_delivery["abc123"]["attempts"] = MAX_DELIVERY_ATTEMPTS - 1
|
||
pending = dict(runner._pending_delivery["abc123"])
|
||
|
||
storage.update_watch.side_effect = RuntimeError("disk full")
|
||
|
||
with pytest.raises(RuntimeError):
|
||
runner._redeliver_pending(_watch_row(), pending)
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery # hold survived
|
||
|
||
|
||
class TestWatchRunnerCancelRace:
|
||
"""User-cancel racing the poll pool: the delivery paths re-check the
|
||
row's active state so a cancelled watch can neither deliver nor leak a
|
||
held reminder, and the tick sweep mops up the one interleaving the
|
||
point checks can't reach (a stash landing after the cancel path's
|
||
``forget_terminal_dispatched`` already cleared)."""
|
||
|
||
def _make_runner(self, storage: Any, **kwargs: Any) -> WatchRunner:
|
||
return WatchRunner(
|
||
storage=storage,
|
||
node_id="test-node",
|
||
check_interval=0.1,
|
||
tool_timeout=5,
|
||
**kwargs,
|
||
)
|
||
|
||
def test_hold_dropped_when_watch_cancelled_mid_fire(self):
|
||
# Cancel lands while the fire's command is running: the hold path
|
||
# re-checks the row and DROPS instead of stashing — an inactive row
|
||
# never re-lists, so a stash here would leak for the process
|
||
# lifetime with nothing ever retrying or clearing it.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
storage.is_watch_active.return_value = False
|
||
runner = self._make_runner(storage) # no fn, no restore → would hold
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
# No retry-cadence commit either — the row already left the view.
|
||
storage.update_watch.assert_not_called()
|
||
|
||
def test_redelivery_dropped_when_watch_cancelled(self):
|
||
# Cancel lands between the due listing and the redelivery dispatch:
|
||
# deliver nothing (the model must not act on — nor a restore be
|
||
# spawned for — a watch the user just cancelled) and drop the hold.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
runner = self._make_runner(storage)
|
||
runner._poll_watch(_watch_row()) # fails → holds (row still active)
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
|
||
storage.is_watch_active.return_value = False # user cancels
|
||
dispatch_fn = MagicMock()
|
||
runner.set_dispatch_fn("ws-1", dispatch_fn) # ws even came back live
|
||
|
||
runner._poll_watch(_watch_row())
|
||
|
||
dispatch_fn.assert_not_called()
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
# The only row write remains the initial hold's cadence commit —
|
||
# no terminal commit lands over the cancel's row state.
|
||
assert storage.update_watch.call_count == 1
|
||
|
||
def test_tick_sweeps_cancelled_holds(self):
|
||
# The residual interleaving: a stash that landed AFTER the cancel's
|
||
# forget_terminal_dispatched cleared (its active re-check passed
|
||
# just before the cancel's row write). The sweep drops it within
|
||
# one tick.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
storage.list_due_watches.return_value = []
|
||
runner = self._make_runner(storage)
|
||
runner._stash_pending_delivery("abc123", {"text": "x"}, {"active": False}, attempts=1)
|
||
storage.is_watch_active.return_value = False # row already cancelled
|
||
|
||
runner._tick()
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" not in runner._pending_delivery
|
||
|
||
def test_tick_sweep_keeps_active_holds(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
storage.list_due_watches.return_value = []
|
||
runner = self._make_runner(storage)
|
||
runner._stash_pending_delivery("abc123", {"text": "x"}, {"active": False}, attempts=1)
|
||
storage.is_watch_active.return_value = True
|
||
|
||
runner._tick()
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
|
||
def test_active_checks_bias_toward_delivery_on_storage_error(self):
|
||
# is_watch_active RAISING must not drop a fire: the sweep keeps the
|
||
# hold and the delivery paths proceed (bounded by their own attempt
|
||
# and poll budgets) — a storage blip is not a cancellation.
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
storage.list_due_watches.return_value = []
|
||
storage.is_watch_active.side_effect = RuntimeError("storage down")
|
||
runner = self._make_runner(storage)
|
||
runner._stash_pending_delivery("abc123", {"text": "x"}, {"active": False}, attempts=1)
|
||
|
||
runner._tick() # sweep: biased active → kept
|
||
|
||
with runner._pending_delivery_lock:
|
||
assert "abc123" in runner._pending_delivery
|
||
|
||
|
||
class TestWatchRunnerRestoreSerialization:
|
||
"""Two watches on ONE evicted workstream, polled concurrently, must
|
||
trigger the restore path at most once — otherwise each spawns a live
|
||
auto-approved session racing writes into one conversation history."""
|
||
|
||
def test_concurrent_same_ws_restores_once(self):
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_calls: list[str] = []
|
||
calls_lock = threading.Lock()
|
||
|
||
runner = WatchRunner(
|
||
storage=storage,
|
||
node_id="n",
|
||
check_interval=0.1,
|
||
tool_timeout=5,
|
||
restore_fn=lambda ws_id: _slow_restore(runner, ws_id, restore_calls, calls_lock),
|
||
)
|
||
|
||
reminder = {"type": "watch_triggered", "text": "x"}
|
||
barrier = threading.Barrier(2)
|
||
results: list[bool] = []
|
||
results_lock = threading.Lock()
|
||
|
||
def call(wid: str) -> None:
|
||
barrier.wait(timeout=2.0)
|
||
ok = runner._dispatch_result("ws-shared", reminder, wid)
|
||
with results_lock:
|
||
results.append(ok)
|
||
|
||
threads = [threading.Thread(target=call, args=(f"w{i}",)) for i in range(2)]
|
||
for t in threads:
|
||
t.start()
|
||
for t in threads:
|
||
t.join(timeout=3.0)
|
||
|
||
# Exactly one restore ran; the winner delivered (True) and the other
|
||
# DEFERRED (False, holds + re-delivers next tick) rather than blocking
|
||
# its poll slot on the in-flight restore or restoring a second time.
|
||
assert restore_calls == ["ws-shared"]
|
||
assert sorted(results) == [False, True]
|
||
# Admission slot released after the restore.
|
||
with runner._restore_lock:
|
||
assert "ws-shared" not in runner._restoring
|
||
|
||
def test_wedged_restore_admissions_defer_and_alert(self, caplog):
|
||
# Admission entries older than RESTORE_STALL_ALERT_SECS are alerted
|
||
# on but NEVER evicted: the wedged poll thread's pool slot is never
|
||
# released, so reclaiming its admission would just readmit a restore
|
||
# that can wedge another pool thread on the same cause — trading
|
||
# this capped degraded state (restores blocked, polling intact) for
|
||
# total poll-pool collapse. New restores keep deferring; the error
|
||
# log is the operator's restart signal.
|
||
from turnstone.core.watch import RESTORE_STALL_ALERT_SECS
|
||
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
restore_fn = MagicMock(return_value=MagicMock())
|
||
runner = WatchRunner(
|
||
storage=storage,
|
||
node_id="n",
|
||
check_interval=0.1,
|
||
tool_timeout=5,
|
||
restore_fn=restore_fn,
|
||
)
|
||
stalled_at = time.monotonic() - RESTORE_STALL_ALERT_SECS - 1
|
||
with runner._restore_lock:
|
||
runner._restoring["wedged-1"] = stalled_at
|
||
runner._restoring["wedged-2"] = stalled_at # both slots wedged
|
||
|
||
with caplog.at_level("ERROR"):
|
||
result = runner._dispatch_result("ws-new", {"text": "x"}, "w1")
|
||
|
||
assert result is False # wedged capacity stays consumed → defer
|
||
restore_fn.assert_not_called()
|
||
with runner._restore_lock:
|
||
assert "wedged-1" in runner._restoring
|
||
assert "wedged-2" in runner._restoring
|
||
assert any("watch_runner.restore_admission_wedged" in r.message for r in caplog.records)
|
||
|
||
|
||
class TestWatchRunnerConcurrency:
|
||
"""The tick thread only enumerates due rows; polls run on bounded
|
||
daemon threads. Pins: genuine concurrency, per-watch in-flight
|
||
dedup, saturation leaving rows due (not dropped), and ``stop``
|
||
draining in-flight polls."""
|
||
|
||
def _make_runner(self, rows: list[dict[str, Any]], **kwargs: Any) -> WatchRunner:
|
||
storage = MagicMock()
|
||
storage.update_watch.return_value = True
|
||
storage.list_due_watches.return_value = rows
|
||
return WatchRunner(
|
||
storage=storage,
|
||
node_id="test-node",
|
||
check_interval=0.1,
|
||
tool_timeout=5,
|
||
**kwargs,
|
||
)
|
||
|
||
@staticmethod
|
||
def _wait_in_flight_empty(runner: WatchRunner, timeout: float = 3.0) -> None:
|
||
def _drained() -> bool:
|
||
with runner._in_flight_lock:
|
||
return not runner._in_flight
|
||
|
||
wait_until(_drained, timeout=timeout)
|
||
|
||
def test_tick_polls_concurrently(self):
|
||
rows = [_watch_row(watch_id=f"w{i}", ws_id=f"ws-{i}") for i in range(3)]
|
||
runner = self._make_runner(rows)
|
||
|
||
all_in = threading.Event()
|
||
release = threading.Event()
|
||
barrier = threading.Barrier(3)
|
||
|
||
def fake_poll(_row: dict[str, Any]) -> None:
|
||
# All three poll threads must be inside simultaneously for the
|
||
# barrier to trip — serial execution would deadlock here (and
|
||
# fail via the barrier timeout instead).
|
||
barrier.wait(timeout=2.0)
|
||
all_in.set()
|
||
release.wait(timeout=2.0)
|
||
|
||
runner._poll_watch = fake_poll # type: ignore[method-assign]
|
||
runner._tick()
|
||
|
||
assert all_in.wait(timeout=2.0), "polls did not run concurrently"
|
||
release.set()
|
||
self._wait_in_flight_empty(runner)
|
||
|
||
def test_tick_skips_in_flight_watch(self):
|
||
rows = [_watch_row(watch_id="w0", ws_id="ws-0")]
|
||
runner = self._make_runner(rows)
|
||
polled: list[str] = []
|
||
runner._poll_watch = lambda row: polled.append(row["watch_id"]) # type: ignore[method-assign]
|
||
|
||
# Simulate a slow poll from a previous tick still running.
|
||
with runner._in_flight_lock:
|
||
runner._in_flight.add("w0")
|
||
|
||
runner._tick()
|
||
|
||
assert polled == []
|
||
# The foreign in-flight entry was not clobbered by the skip.
|
||
with runner._in_flight_lock:
|
||
assert "w0" in runner._in_flight
|
||
|
||
def test_tick_saturation_leaves_rows_due(self):
|
||
rows = [_watch_row(watch_id=f"w{i}", ws_id=f"ws-{i}") for i in range(2)]
|
||
runner = self._make_runner(rows, max_concurrent_polls=1)
|
||
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
polled: list[str] = []
|
||
|
||
def fake_poll(row: dict[str, Any]) -> None:
|
||
polled.append(row["watch_id"])
|
||
started.set()
|
||
release.wait(timeout=2.0)
|
||
|
||
runner._poll_watch = fake_poll # type: ignore[method-assign]
|
||
runner._tick()
|
||
assert started.wait(timeout=2.0)
|
||
|
||
# Only the first row got a slot this tick; the second stays due
|
||
# for the next tick rather than being dropped.
|
||
assert polled == ["w0"]
|
||
release.set()
|
||
self._wait_in_flight_empty(runner)
|
||
|
||
# Next tick (slot free again) picks up the remaining row.
|
||
runner._storage.list_due_watches.return_value = [rows[1]]
|
||
runner._tick()
|
||
self._wait_in_flight_empty(runner)
|
||
assert polled == ["w0", "w1"]
|
||
|
||
def test_stop_waits_for_in_flight_polls(self):
|
||
rows = [_watch_row(watch_id="w0", ws_id="ws-0")]
|
||
runner = self._make_runner(rows)
|
||
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
|
||
def fake_poll(_row: dict[str, Any]) -> None:
|
||
started.set()
|
||
release.wait(timeout=3.0)
|
||
|
||
runner._poll_watch = fake_poll # type: ignore[method-assign]
|
||
runner._tick()
|
||
assert started.wait(timeout=2.0)
|
||
|
||
stopper = threading.Thread(target=runner.stop, daemon=True)
|
||
stopper.start()
|
||
# stop() must be draining (poll still pinned), not returned.
|
||
time.sleep(0.15)
|
||
assert stopper.is_alive(), "stop() returned while a poll was in flight"
|
||
|
||
release.set()
|
||
stopper.join(timeout=3.0)
|
||
assert not stopper.is_alive()
|
||
with runner._in_flight_lock:
|
||
assert not runner._in_flight
|