mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
98d4be8ffe
WatchRunner._poll_watch committed active=False to the row BEFORE calling _dispatch_result for a terminal fire, and the dispatch closure registered by ChatSession.set_watch_runner enqueued each reminder with a valid_until=is_watch_active predicate that re-read the row at drain time. Since the runner already flipped active to 0, the predicate returned False for every dispatched fire and NudgeQueue.drain silently dropped the entry — the model never saw a watch result. Then a subsequent action=cancel call hit list_watches_for_ws (filters active==1), the now-inactive row was invisible, and the cancel returned 'Watch "X" not found.' regardless of whether the watch had actually run. Reorder _poll_watch to dispatch before the row write, drop the valid_until predicate from the watch closure (its only effect was the bug above), and add a _terminal_dispatched guard on the runner so a transient storage failure between dispatch and row-write doesn't re-fire the reminder on the next tick. Add WatchRunner.forget_terminal_dispatched and call it from the cancel path so an out-of-band deactivate (next_poll='') doesn't leak the watch_id from the runner's pending-retry set indefinitely. Cancel-by-name now routes through a new find_watch_by_name storage method that ignores the active filter and prefers active rows over newer-inactive same-name siblings. The session.py cancel branch distinguishes 'already completed (auto-cancelled)' from 'not found' so the model can tell apart 'this watch ran and finished' from 'no such watch.' Consolidate the two byte-identical _escape_like / _escape_ilike helpers in the storage backends into a single turnstone.core.storage._utils.escape_like and apply it to the new find_watch_by_name LIKE pattern so a model-supplied watch name containing % or _ can't redirect a cancel to a sibling watch. NudgeQueue.drain previously dropped predicate-failed entries without logging anything, which is what hid this bug for so long. Drain now emits nudge_queue.predicate_dropped: info for reason=predicate_false (the normal lifecycle case — idle_children when every active child finished between enqueue and drain), warning with exc_info for reason=predicate_raised (a misbehaving predicate). Tests: new test_poll_watch_terminal_fire_survives_drain (parametrized stop_on_fired + max_polls_reached) drives the real WatchRunner._poll_watch against a real tmp_db row and confirmed to fail against pristine main. test_poll_watch_retry_deactivate_after_update_watch_failure exercises the _terminal_dispatched retry-deactivate branch end to end. test_cancel_clears_pending_terminal_dispatched_entry covers the cancel- path leak case. test_find_by_name_prefers_active_over_newer_inactive catches the ordering regression. test_find_by_name_treats_percent_as_literal + test_find_by_name_treats_underscore_as_literal pin the LIKE escape.
224 lines
8.7 KiB
Python
224 lines
8.7 KiB
Python
"""Tests for watches storage CRUD."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from turnstone.core.storage._schema import watches as watches_table
|
|
|
|
|
|
def _make_watch_kwargs(**overrides):
|
|
"""Build default kwargs for create_watch."""
|
|
defaults = {
|
|
"watch_id": "watch_001",
|
|
"ws_id": "ws-abc",
|
|
"node_id": "node-1",
|
|
"name": "pr-review",
|
|
"command": "gh pr view --json state",
|
|
"interval_secs": 300.0,
|
|
"stop_on": 'data["state"] == "MERGED"',
|
|
"max_polls": 100,
|
|
"created_by": "model",
|
|
"next_poll": "2099-01-01T00:05:00",
|
|
}
|
|
defaults.update(overrides)
|
|
return defaults
|
|
|
|
|
|
class TestWatchCRUD:
|
|
def test_create_and_get(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
w = db.get_watch("watch_001")
|
|
assert w is not None
|
|
assert w["name"] == "pr-review"
|
|
assert w["command"] == "gh pr view --json state"
|
|
assert w["interval_secs"] == 300.0
|
|
assert w["active"] == 1
|
|
assert w["poll_count"] == 0
|
|
|
|
def test_get_nonexistent(self, db):
|
|
assert db.get_watch("nope") is None
|
|
|
|
def test_create_idempotent(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
db.create_watch(**_make_watch_kwargs()) # OR IGNORE
|
|
assert db.get_watch("watch_001") is not None
|
|
|
|
def test_update(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
updated = db.update_watch(
|
|
"watch_001",
|
|
poll_count=5,
|
|
last_output="hello",
|
|
last_exit_code=0,
|
|
)
|
|
assert updated is True
|
|
w = db.get_watch("watch_001")
|
|
assert w["poll_count"] == 5
|
|
assert w["last_output"] == "hello"
|
|
assert w["last_exit_code"] == 0
|
|
|
|
def test_update_nonexistent(self, db):
|
|
assert db.update_watch("nope", poll_count=1) is False
|
|
|
|
def test_update_active_flag(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
db.update_watch("watch_001", active=False)
|
|
w = db.get_watch("watch_001")
|
|
assert w["active"] == 0
|
|
|
|
def test_delete(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
assert db.delete_watch("watch_001") is True
|
|
assert db.get_watch("watch_001") is None
|
|
|
|
def test_delete_nonexistent(self, db):
|
|
assert db.delete_watch("nope") is False
|
|
|
|
|
|
class TestIsWatchActive:
|
|
def test_active_row_returns_true(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
assert db.is_watch_active("watch_001") is True
|
|
|
|
def test_inactive_row_returns_false(self, db):
|
|
db.create_watch(**_make_watch_kwargs())
|
|
db.update_watch("watch_001", active=False)
|
|
assert db.is_watch_active("watch_001") is False
|
|
|
|
def test_missing_row_returns_false(self, db):
|
|
assert db.is_watch_active("nope") is False
|
|
|
|
|
|
class TestWatchListQueries:
|
|
def test_list_for_ws(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1", name="a"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w2", ws_id="ws-1", name="b"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w3", ws_id="ws-2", name="c"))
|
|
|
|
ws1 = db.list_watches_for_ws("ws-1")
|
|
assert len(ws1) == 2
|
|
assert {w["name"] for w in ws1} == {"a", "b"}
|
|
|
|
def test_list_for_ws_excludes_inactive(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1"))
|
|
db.update_watch("w1", active=False)
|
|
assert db.list_watches_for_ws("ws-1") == []
|
|
|
|
def test_find_by_name_returns_inactive(self, db):
|
|
"""``find_watch_by_name`` ignores the active filter — that is
|
|
what lets the cancel-by-name UX distinguish 'already completed'
|
|
from 'no such watch.'
|
|
"""
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1", name="completed"))
|
|
db.update_watch("w1", active=False)
|
|
|
|
row = db.find_watch_by_name("ws-1", "completed")
|
|
assert row is not None
|
|
assert row["watch_id"] == "w1"
|
|
assert not row["active"]
|
|
|
|
def test_find_by_name_matches_watch_id_prefix(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="abcdef123", ws_id="ws-1", name="x"))
|
|
row = db.find_watch_by_name("ws-1", "abc")
|
|
assert row is not None
|
|
assert row["watch_id"] == "abcdef123"
|
|
|
|
def test_find_by_name_scoped_to_ws(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1", name="shared"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w2", ws_id="ws-2", name="shared"))
|
|
|
|
row = db.find_watch_by_name("ws-1", "shared")
|
|
assert row is not None
|
|
assert row["watch_id"] == "w1"
|
|
|
|
def test_find_by_name_returns_none_when_missing(self, db):
|
|
assert db.find_watch_by_name("ws-1", "ghost") is None
|
|
|
|
def test_find_by_name_empty_input_returns_none(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1", name="x"))
|
|
assert db.find_watch_by_name("ws-1", "") is None
|
|
|
|
def test_find_by_name_treats_percent_as_literal(self, db):
|
|
"""A model-supplied '%' must NOT match arbitrary watch_ids.
|
|
|
|
Pre-escape, ``watch_id.like(f"{name_or_prefix}%")`` would
|
|
interpret '%' as 'match anything' and pick up the first row in
|
|
the workstream regardless of name.
|
|
"""
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1", name="real-watch"))
|
|
assert db.find_watch_by_name("ws-1", "%") is None
|
|
|
|
def test_find_by_name_treats_underscore_as_literal(self, db):
|
|
"""Same as the '%' case for the single-char LIKE wildcard."""
|
|
db.create_watch(**_make_watch_kwargs(watch_id="abcd", ws_id="ws-1", name="real-watch"))
|
|
# '_' would otherwise match any single char, picking up
|
|
# watch_ids beginning with 'a', 'b', etc.
|
|
assert db.find_watch_by_name("ws-1", "_") is None
|
|
|
|
def test_find_by_name_prefers_active_over_newer_inactive(self, db):
|
|
"""If a same-name pair exists where the inactive row is NEWER
|
|
than the active row, find_watch_by_name must still return the
|
|
active row. Pre-fix the query was ``ORDER BY created DESC
|
|
LIMIT 1`` — which would return the newer inactive row and
|
|
cause the cancel UX to report 'already completed' for a name
|
|
whose live row is still polling.
|
|
|
|
Reachable in practice because storage allows out-of-band
|
|
writes (e.g. ``delete_watches_for_ws`` cleanup followed by
|
|
re-create, an admin manually flipping ``active``, or test
|
|
scaffolding) that bypass the create-time duplicate-name
|
|
guard.
|
|
"""
|
|
# Older active watch.
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w-active", ws_id="ws-1", name="recurring"))
|
|
# Newer inactive watch with the same name. ``create_watch``
|
|
# stamps ``created`` to ``now`` at second resolution, so we
|
|
# bypass the API to give the inactive row a deterministically
|
|
# later timestamp.
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w-inactive", ws_id="ws-1", name="recurring"))
|
|
with db._conn() as conn:
|
|
conn.execute(
|
|
sa.update(watches_table)
|
|
.where(watches_table.c.watch_id == "w-inactive")
|
|
.values(active=0, next_poll="", created="2099-01-01T00:00:00")
|
|
)
|
|
conn.commit()
|
|
|
|
row = db.find_watch_by_name("ws-1", "recurring")
|
|
assert row is not None
|
|
assert row["watch_id"] == "w-active"
|
|
assert row["active"]
|
|
|
|
def test_list_for_node(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", node_id="n1"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w2", node_id="n1"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w3", node_id="n2"))
|
|
|
|
n1 = db.list_watches_for_node("n1")
|
|
assert len(n1) == 2
|
|
|
|
def test_list_due(self, db):
|
|
# Due
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", next_poll="2020-01-01T00:00:00"))
|
|
# Not due (far future)
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w2", next_poll="2099-01-01T00:00:00"))
|
|
# Due but inactive
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w3", next_poll="2020-01-01T00:00:00"))
|
|
db.update_watch("w3", active=False)
|
|
|
|
due = db.list_due_watches("2025-01-01T00:00:00")
|
|
assert len(due) == 1
|
|
assert due[0]["watch_id"] == "w1"
|
|
|
|
def test_delete_for_ws(self, db):
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w1", ws_id="ws-1"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w2", ws_id="ws-1"))
|
|
db.create_watch(**_make_watch_kwargs(watch_id="w3", ws_id="ws-2"))
|
|
|
|
count = db.delete_watches_for_ws("ws-1")
|
|
assert count == 2
|
|
assert db.get_watch("w1") is None
|
|
assert db.get_watch("w2") is None
|
|
assert db.get_watch("w3") is not None
|