mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
6560f1ec4f
The judges and the regex ReDoS probe ran a blocking call on a ThreadPoolExecutor and abandoned the worker with shutdown(wait=False) on timeout or cancel. concurrent.futures joins every executor worker from an atexit hook regardless of wait=False, so a wedged call could pin interpreter exit — and hang the test suite at shutdown. Add turnstone/core/deadline.py::run_with_deadline: run a blocking callable on a daemon thread bounded by a wall-clock timeout and an optional cancel event. A daemon worker is never joined at exit, so abandoning one is safe. Migrate three sites onto it: - OutputGuardJudge.evaluate() - IntentJudge._evaluate_single / _run_judge — this also removes _ExecutorPoisonedError and the executor-restart dance: per-call daemon threads can't poison a shared single-slot pool, so a timeout now returns None and the caller delivers one fallback verdict. - console/server.py _validate_regex_pattern (regex ReDoS probe) Also: - Double the default judge LLM timeouts for slower local models: judge.timeout 60->120s and judge.output_guard_llm_timeout 30->60s (settings registry, JudgeConfig dataclass, --judge-timeout CLI default, class docstring, docs). Correct a stale doc that described the per-turn timeout as a total budget across turns. - Raise the regex probe bound 0.5->3.0s so a legitimately complex pattern isn't false-flagged as catastrophic backtracking. - CI: run pytest with -v instead of -q so a hang names the offending test instead of riding the job timeout. - Tests: cover deadline.py and the regex validator; move test_judge.py off fixed sleeps onto the existing _wait_for helper.
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
"""Tests for turnstone.core.deadline.run_with_deadline.
|
|
|
|
The load-bearing property is the daemon worker: on timeout or cancel the call
|
|
is abandoned, and the abandoned thread must be a daemon so it can never block
|
|
interpreter exit (the bug that motivated the helper — a non-daemon
|
|
ThreadPoolExecutor worker is joined by concurrent.futures' atexit hook).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from turnstone.core.deadline import (
|
|
DeadlineCancelledError,
|
|
DeadlineExceededError,
|
|
run_with_deadline,
|
|
)
|
|
|
|
|
|
def test_returns_result_on_success() -> None:
|
|
assert run_with_deadline(lambda: 42, timeout=1.0) == 42
|
|
|
|
|
|
def test_reraises_callable_exception() -> None:
|
|
def boom() -> None:
|
|
raise ValueError("upstream failed")
|
|
|
|
with pytest.raises(ValueError, match="upstream failed"):
|
|
run_with_deadline(boom, timeout=1.0)
|
|
|
|
|
|
def test_timeout_returns_promptly_and_abandons_a_daemon_worker() -> None:
|
|
# The worker sleeps far past the deadline; the call must return promptly
|
|
# via DeadlineExceededError, and the abandoned worker must be a daemon so
|
|
# it cannot pin interpreter exit.
|
|
start = time.monotonic()
|
|
with pytest.raises(DeadlineExceededError):
|
|
run_with_deadline(lambda: time.sleep(2.0), timeout=0.2, poll=0.05, thread_name="dl-timeout")
|
|
assert time.monotonic() - start < 1.0
|
|
stragglers = [t for t in threading.enumerate() if t.name == "dl-timeout" and not t.daemon]
|
|
assert stragglers == [], f"non-daemon worker survived: {stragglers}"
|
|
|
|
|
|
def test_cancel_returns_promptly() -> None:
|
|
cancel = threading.Event()
|
|
|
|
def _fire() -> None:
|
|
time.sleep(0.1)
|
|
cancel.set()
|
|
|
|
threading.Thread(target=_fire, daemon=True).start()
|
|
start = time.monotonic()
|
|
with pytest.raises(DeadlineCancelledError):
|
|
run_with_deadline(
|
|
lambda: time.sleep(2.0),
|
|
timeout=10.0,
|
|
cancel_event=cancel,
|
|
poll=0.05,
|
|
thread_name="dl-cancel",
|
|
)
|
|
assert time.monotonic() - start < 1.0
|
|
stragglers = [t for t in threading.enumerate() if t.name == "dl-cancel" and not t.daemon]
|
|
assert stragglers == [], f"non-daemon worker survived: {stragglers}"
|