mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
d820168f3f
Enabling mTLS broke the cluster in three layered ways: - Service certs were keyed on socket.gethostname() (the container ID) and never carried the advertised service name as a SAN, so every collector and routing-proxy handshake failed the hostname check. build_cert_hostnames() now puts the advertised host first: it becomes the cert's primary domain (hence a SAN) and a stable store key that survives container recreation. - lacme's RenewalManager renews everything in the store; with the store shared cluster-wide, every node renewed every other node's (and every dead container's) cert — an N×M renewal storm. _SingleDomainStore scopes each node's sweep to its own cert, and the console adds a periodic GC for the certs of long-departed nodes. - uvicorn loads its cert once at boot and never reloads, so renewed certs never reached the listener and the served cert expired mid-process. swap_context_cert() hot-swaps renewed material into the live SSL context (server listener and console client context) via load_cert_chain. Observability and browser access: - The collector logged connection/TLS failures at DEBUG, so a persistent mTLS-verify failure was invisible. It now logs the first failure per node (reachable->unreachable) at WARNING and stays at DEBUG on retries. - The console serves plain HTTP (it is the ACME bootstrap endpoint) and no longer rewrites its advertised URL to https://. Browser->console TLS is terminated by a reverse proxy: the cluster profile gains a caddy service (browser h2/HTTPS -> caddy -> console h1.1/HTTP) plus browser-TLS docs. Tests: tests/test_tls_san_renewal.py, tests/test_collector_reachability.py.
47 lines
1.7 KiB
Python
47 lines
1.7 KiB
Python
"""Collector reachability-transition semantics.
|
|
|
|
Regression cover for the observability gap where the collector logged TLS /
|
|
connection failures at DEBUG, so a persistent mTLS-verify failure was invisible
|
|
at the default log level. ``_mark_unreachable`` now reports the first
|
|
(reachable→unreachable) transition so the SSE loop can log it at WARNING and
|
|
stay quiet on subsequent retries.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from turnstone.console.collector import ClusterCollector, NodeSnapshot
|
|
|
|
|
|
def _collector() -> ClusterCollector:
|
|
return ClusterCollector(storage=MagicMock())
|
|
|
|
|
|
def test_first_failure_is_a_transition_then_quiet():
|
|
c = _collector()
|
|
c._nodes["node-1"] = NodeSnapshot(node_id="node-1", reachable=True)
|
|
|
|
# First failure flips reachable→unreachable → True (log at WARNING).
|
|
assert c._mark_unreachable("node-1", reason="SSLCertVerificationError") is True
|
|
assert c._nodes["node-1"].reachable is False
|
|
assert c._nodes["node-1"].reachable_reason == "SSLCertVerificationError"
|
|
|
|
# Still-down retries are not transitions → False (stay at DEBUG).
|
|
assert c._mark_unreachable("node-1", reason="SSLCertVerificationError") is False
|
|
|
|
|
|
def test_recovery_then_failure_is_a_new_transition():
|
|
c = _collector()
|
|
c._nodes["node-1"] = NodeSnapshot(node_id="node-1", reachable=True)
|
|
c._mark_unreachable("node-1", reason="ConnectError")
|
|
|
|
# Node comes back (as _apply_snapshot does), then fails again → new transition.
|
|
c._nodes["node-1"].reachable = True
|
|
assert c._mark_unreachable("node-1", reason="ConnectError") is True
|
|
|
|
|
|
def test_unknown_node_is_not_a_transition():
|
|
c = _collector()
|
|
assert c._mark_unreachable("ghost", reason="ConnectError") is False
|