mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
fix(tls): stub backoff via a _sleep seam, not the global asyncio.sleep
The test-postgres failure on test_init_retries_exhausted_raises surfaced the root cause: sleeps held 2275x 0.1 instead of [1.0, 2.0]. Those 0.1s came from a concurrent background poller doing asyncio.sleep(0.1) on anyio's shared (persistent) event loop — the tls retry tests patched the *global* asyncio.sleep, which intercepted that poller too. - Before: the stub didn't yield, so the poller busy-looped and monopolized the loop -> the test hung (the CI-only "after 92%" hang on 3.12+). - The earlier "make the stub yield" change converted the hang into this flood (the poller spins instead of blocking), which is what exposed it. Fix: route init()'s backoff through TLSClient._sleep so the tests stub that method in isolation and never touch the global asyncio.sleep. Tasks sharing the loop are no longer affected; schedule assertions are unchanged. The deeper fragility this exploited — a leaked, un-cancelled background poller surviving on the shared test loop — is left as a follow-up.
This commit is contained in:
@@ -96,10 +96,8 @@ def _make_flaky_client(monkeypatch, failures: int):
|
||||
"""TLSClient whose CA fetch fails ``failures`` times, then succeeds.
|
||||
|
||||
Returns (client, calls, sleeps) — mutable lists recording each CA-fetch
|
||||
attempt and each backoff delay (asyncio.sleep is stubbed out).
|
||||
attempt and each backoff delay (the client's backoff sleep is stubbed).
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from turnstone.core.tls import TLSClient
|
||||
|
||||
client = TLSClient(
|
||||
@@ -118,19 +116,15 @@ def _make_flaky_client(monkeypatch, failures: int):
|
||||
async def ok_request():
|
||||
pass
|
||||
|
||||
real_sleep = asyncio.sleep
|
||||
|
||||
async def fake_sleep(delay):
|
||||
# Record the backoff delay and skip the real wait, but still yield to
|
||||
# the loop. An async stub that returns without ever suspending lets the
|
||||
# whole retry run complete in a single event-loop step with no
|
||||
# checkpoint, which is fragile under the async test runner.
|
||||
# Stub the client's own _sleep seam, NOT the global asyncio.sleep:
|
||||
# patching the global also intercepts any concurrent task sharing the
|
||||
# event loop, which corrupted a background poller and hung CI.
|
||||
sleeps.append(delay)
|
||||
await real_sleep(0)
|
||||
|
||||
monkeypatch.setattr(client, "_fetch_ca_cert", flaky_fetch)
|
||||
monkeypatch.setattr(client, "_request_cert", ok_request)
|
||||
monkeypatch.setattr(asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(client, "_sleep", fake_sleep)
|
||||
return client, calls, sleeps
|
||||
|
||||
|
||||
@@ -182,8 +176,6 @@ async def test_init_retries_exhausted_raises(monkeypatch):
|
||||
@pytest.mark.anyio
|
||||
async def test_init_retries_discovery_failure(monkeypatch):
|
||||
"""Console discovery (not-yet-registered console) is retried too."""
|
||||
import asyncio
|
||||
|
||||
from turnstone.core.tls import TLSClient
|
||||
|
||||
client = TLSClient(storage=get_storage(), hostnames=["node-1"])
|
||||
@@ -195,18 +187,16 @@ async def test_init_retries_discovery_failure(monkeypatch):
|
||||
raise RuntimeError("No console service found in services table.")
|
||||
return "http://console:9999"
|
||||
|
||||
real_sleep = asyncio.sleep
|
||||
|
||||
async def ok():
|
||||
pass
|
||||
|
||||
async def fake_sleep(_delay):
|
||||
await real_sleep(0)
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(client, "_discover_console_url", flaky_discover)
|
||||
monkeypatch.setattr(client, "_fetch_ca_cert", ok)
|
||||
monkeypatch.setattr(client, "_request_cert", ok)
|
||||
monkeypatch.setattr(asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(client, "_sleep", fake_sleep)
|
||||
|
||||
await client.init(attempts=2)
|
||||
assert attempts == [1, 2]
|
||||
|
||||
+12
-3
@@ -294,8 +294,6 @@ class TLSClient:
|
||||
Discovery, CA fetch, and cert request are all idempotent, so the whole
|
||||
sequence is retried as a unit.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
if attempts < 1:
|
||||
# range(1, attempts + 1) would be empty: init() would return
|
||||
# "successfully" with no CA and no cert.
|
||||
@@ -321,7 +319,18 @@ class TLSClient:
|
||||
delay_seconds=delay,
|
||||
error=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
await self._sleep(delay)
|
||||
|
||||
async def _sleep(self, delay: float) -> None:
|
||||
"""Backoff sleep behind a seam so tests can stub it in isolation.
|
||||
|
||||
Patching the module-global ``asyncio.sleep`` would also intercept it
|
||||
for every other task sharing the event loop; routing the retry backoff
|
||||
through a method keeps test stubs from corrupting concurrent tasks.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
def _discover_console_url(self) -> str:
|
||||
"""Look up the console URL from the services table."""
|
||||
|
||||
Reference in New Issue
Block a user