diff --git a/compose.yaml b/compose.yaml index 66fbd5ac..e3c9cdbf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -23,6 +23,8 @@ volumes: turnstone-data: workspace: postgres-data: + caddy-data: + caddy-config: services: # ------------------------------------------------------------------- @@ -144,6 +146,27 @@ services: start_period: 10s restart: unless-stopped + # ------------------------------------------------------------------- + # caddy — browser TLS for the console dashboard (cluster/demo). + # Terminates HTTPS (Caddy's local CA, see deploy/Caddyfile) → console:8090. + # Dashboard: https://localhost:${CONSOLE_HTTPS_PORT:-8443} + # ------------------------------------------------------------------- + caddy: + image: caddy:2.11 + profiles: + - cluster + depends_on: + - console + ports: + - "${CONSOLE_HTTPS_PORT:-8443}:443" + volumes: + - ./deploy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data # persist Caddy's local CA across restarts + - caddy-config:/config + networks: + - turnstone-net + restart: unless-stopped + # ------------------------------------------------------------------- # turnstone-channel — Channel gateway (Discord, Slack, etc.) # Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 00000000..fea76e66 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,17 @@ +# Browser TLS for the console dashboard (cluster/demo profile): +# browser --h2/HTTPS--> caddy:443 --h1.1/HTTP--> console:8090 +# The console serves HTTP (it's the ACME bootstrap endpoint), so browser TLS is +# terminated here. `tls internal` uses Caddy's own local CA; trust its root once: +# docker compose exec caddy cat /data/caddy/pki/authorities/local/root.crt +# See docs/tls.md (incl. the acme_ca→console alternative and why it's not default). + +:443 { + # on_demand: a port-only site has no fixed name to pre-issue for; safe here + # because the issuer is Caddy's local CA, not a public one. + tls internal { + on_demand + } + reverse_proxy console:8090 { + flush_interval -1 # stream the dashboard SSE without buffering + } +} diff --git a/docs/tls.md b/docs/tls.md index 98305993..d203fd39 100644 --- a/docs/tls.md +++ b/docs/tls.md @@ -19,6 +19,48 @@ This: --- +## Browser access (dashboard HTTPS) + +The mTLS above secures **service-to-service** traffic (node↔node, collector and +routing proxy → nodes). The **console dashboard itself serves plain HTTP** — and +must, because it is the cluster's ACME bootstrap endpoint: new nodes fetch +`/acme/ca.pem` and provision their first cert over HTTP, before they have the CA +to verify TLS. So the console cannot be HTTPS-only on its port. + +To put the **browser → console** hop on HTTPS, terminate TLS at a reverse proxy +in front of the console. The `cluster` profile ships a `caddy` service that does +this: + +```bash +docker compose --profile cluster up +# dashboard: https://localhost:${CONSOLE_HTTPS_PORT:-8443} +``` + +``` +browser --h2 / HTTPS--> caddy:443 --h1.1 / HTTP--> console:8090 +``` + +Caddy uses its **own local CA** (`tls internal`, see `deploy/Caddyfile`), so the +setup is self-contained with no dependency on the console's ACME path. Trust the +local root once to silence the browser warning: + +```bash +docker compose exec caddy \ + cat /data/caddy/pki/authorities/local/root.crt # import into your OS/browser +``` + +**Can Caddy get its cert from the console's internal CA instead?** Technically +yes — the console exposes a real ACME directory (`/acme/directory`) with +auto-approval, so Caddy's `tls { ca http://console:8090/acme/directory }` would +mint a cert for any name. It's not recommended as the default: lacme's ACME +responder is built for turnstone's own client (interop with Caddy's client is +unverified), it couples Caddy startup to the console, and the browser must trust +a private CA either way — so it buys nothing over `tls internal`. For a publicly +trusted cert (no warning), point Caddy at Let's Encrypt with a real domain +instead. + +--- + ## Architecture ``` @@ -173,8 +215,15 @@ const client = new TurnstoneServer({ 1. Node starts, connects to shared database (plain connection) 2. Discovers console URL from `services` table 3. Fetches CA root cert from `http://console/acme/ca.pem` (plain HTTP, TOFU) -4. Requests service cert via ACME protocol (plain HTTP, JWS-signed) -5. Starts auto-renewal (24h interval, re-issues before expiry) +4. Requests a service cert via ACME (plain HTTP, JWS-signed). The cert's + primary domain / SAN is the node's **advertised host** (the host of + `TURNSTONE_ADVERTISE_URL`, e.g. `server-1`) — the name peers actually dial, + not the container hostname. This makes mTLS hostname verification succeed + and keys the cert by a stable name that survives container recreation. +5. Starts auto-renewal (24h interval, re-issues before expiry) **scoped to its + own certificate**. Each node renews only its own cert; the shared store is + never swept wholesale. Renewed certs are hot-swapped into the live HTTPS + listener with no restart. 6. All subsequent inter-service communication uses mTLS ### Console Startup Flow @@ -183,7 +232,9 @@ const client = new TurnstoneServer({ 2. Initialize CA (load from DB or generate new root key) 3. Mount ACME responder at `/acme` (serves `/ca.pem` natively) 4. Issue console certs (internal + optional frontend) -5. Start CA-direct auto-renewal (no network, signs directly) +5. Start CA-direct auto-renewal (no network, signs directly), scoped to the + console's own cert, plus a periodic GC that reclaims cert rows for + long-departed nodes 6. Register console URL in services table with heartbeat --- @@ -195,17 +246,31 @@ const client = new TurnstoneServer({ Certs are valid for 48 hours. If auto-renewal stopped (e.g. console was down), restart the service to re-request a cert. +### Collector/proxy can't reach a node (TLS hostname mismatch) + +mTLS verifies a node's advertised host against the cert's SANs. Each node's +cert is issued for the host in its `TURNSTONE_ADVERTISE_URL`, so that name is +always a SAN automatically — you do **not** need to set `TURNSTONE_TLS_SANS` +per node. Only set `TURNSTONE_TLS_SANS` to add *extra* names (e.g. a node +fronted under a second hostname). Symptom if this is wrong: the console +dashboard shows nodes as unreachable and `openssl s_client` reports the served +cert's SANs don't include the dialed name. + ### "No console service found" The console registers itself in the `services` table on startup. If the console hasn't started or the registration expired (1 hour TTL), nodes can't discover it. Use `--console-url` explicitly. -### Let's Encrypt for console frontend +### Browser HTTPS to the console -Set `tls.acme_directory` to `https://acme-v02.api.letsencrypt.org/directory` -in the admin Settings tab. The console will request a publicly trusted cert -for its HTTPS endpoint. Internal mTLS still uses the private CA. +The console serves plain HTTP (it's the ACME bootstrap endpoint — see +[Browser access](#browser-access-dashboard-https)). Put browser traffic on +HTTPS by terminating TLS at a reverse proxy; the `cluster` profile's `caddy` +service does this with Caddy's local CA. For a publicly trusted cert, front the +console with a proxy pointed at Let's Encrypt using a real domain. The +`tls.acme_directory` setting only governs the console's internal/frontend cert +material — it does **not** make the console listen on HTTPS itself. ### Verifying the cert chain diff --git a/tests/test_collector_reachability.py b/tests/test_collector_reachability.py new file mode 100644 index 00000000..26e5268f --- /dev/null +++ b/tests/test_collector_reachability.py @@ -0,0 +1,46 @@ +"""Collector reachability-transition semantics. + +Regression cover for the observability gap where the collector logged TLS / +connection failures at DEBUG, so a persistent mTLS-verify failure was invisible +at the default log level. ``_mark_unreachable`` now reports the first +(reachable→unreachable) transition so the SSE loop can log it at WARNING and +stay quiet on subsequent retries. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from turnstone.console.collector import ClusterCollector, NodeSnapshot + + +def _collector() -> ClusterCollector: + return ClusterCollector(storage=MagicMock()) + + +def test_first_failure_is_a_transition_then_quiet(): + c = _collector() + c._nodes["node-1"] = NodeSnapshot(node_id="node-1", reachable=True) + + # First failure flips reachable→unreachable → True (log at WARNING). + assert c._mark_unreachable("node-1", reason="SSLCertVerificationError") is True + assert c._nodes["node-1"].reachable is False + assert c._nodes["node-1"].reachable_reason == "SSLCertVerificationError" + + # Still-down retries are not transitions → False (stay at DEBUG). + assert c._mark_unreachable("node-1", reason="SSLCertVerificationError") is False + + +def test_recovery_then_failure_is_a_new_transition(): + c = _collector() + c._nodes["node-1"] = NodeSnapshot(node_id="node-1", reachable=True) + c._mark_unreachable("node-1", reason="ConnectError") + + # Node comes back (as _apply_snapshot does), then fails again → new transition. + c._nodes["node-1"].reachable = True + assert c._mark_unreachable("node-1", reason="ConnectError") is True + + +def test_unknown_node_is_not_a_transition(): + c = _collector() + assert c._mark_unreachable("ghost", reason="ConnectError") is False diff --git a/tests/test_tls_san_renewal.py b/tests/test_tls_san_renewal.py new file mode 100644 index 00000000..f931d480 --- /dev/null +++ b/tests/test_tls_san_renewal.py @@ -0,0 +1,298 @@ +"""Tests for the mTLS SAN/identity, renewal-scoping, and GC fixes. + +Regression coverage for the cluster-wide mTLS breakage where: + * service certs were keyed on ``socket.gethostname()`` (the container ID) + and never carried the advertised service name, so every collector/proxy + handshake failed the hostname check; and + * every node ran an unscoped ``RenewalManager`` over the *shared* store, + renewing every other node's cert (an N×M renewal storm). +""" + +from __future__ import annotations + +import socket + +import pytest + +from turnstone.core.storage import get_storage, init_storage, reset_storage + +lacme = pytest.importorskip("lacme") + + +@pytest.fixture(autouse=True) +def _storage(tmp_path): + """Initialize ephemeral SQLite storage for each test.""" + reset_storage() + init_storage("sqlite", path=str(tmp_path / "test.db")) + yield + reset_storage() + + +# ── build_cert_hostnames ────────────────────────────────────────────────────── + + +def test_advertised_host_is_primary(): + """The advertised host is first, so it becomes the cert's primary domain.""" + from turnstone.core.tls import build_cert_hostnames + + names = build_cert_hostnames("http://server-1:8080", bind_host="0.0.0.0") + assert names[0] == "server-1" + assert "localhost" in names + assert "127.0.0.1" in names + # 0.0.0.0 is a wildcard bind and must not become a SAN + assert "0.0.0.0" not in names + + +def test_strips_scheme_and_port(): + """Only the hostname is extracted from the advertise URL.""" + from turnstone.core.tls import build_cert_hostnames + + assert build_cert_hostnames("https://node-7:9999")[0] == "node-7" + + +def test_extra_sans_appended_and_deduped(): + """Env SANs are added once; duplicates collapse, order preserved.""" + from turnstone.core.tls import build_cert_hostnames + + names = build_cert_hostnames("http://server-1:8080", extra_sans="server-1, edge, edge") + assert names[0] == "server-1" + assert names.count("server-1") == 1 + assert names.count("edge") == 1 + + +def test_fallback_to_os_hostname_when_no_advertise_url(): + """Bare-metal fallback: OS hostname becomes primary when no URL is given.""" + from turnstone.core.tls import build_cert_hostnames + + assert build_cert_hostnames("")[0] == socket.gethostname() + + +def test_extra_sans_rejects_wildcard_and_unspecified(): + """A stray wildcard / unspecified-address SAN must not reach the cert.""" + from turnstone.core.tls import build_cert_hostnames + + names = build_cert_hostnames("http://server-1:8080", extra_sans="*, 0.0.0.0, ::, edge") + assert "*" not in names + assert "0.0.0.0" not in names + assert "::" not in names + assert "edge" in names + + +# ── _SingleDomainStore ──────────────────────────────────────────────────────── + + +def _san_values(cert_pem: bytes) -> list[str]: + from cryptography import x509 + + cert = x509.load_pem_x509_certificate(cert_pem) + san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + return [g.value for g in san] + + +@pytest.mark.anyio +async def test_single_domain_store_filters_and_delegates(): + """list_certs exposes only the wrapped domain; other ops delegate.""" + from turnstone.console.tls import TLSManager + from turnstone.core.tls import _SingleDomainStore + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + for dom in ("server-1", "server-2", "server-3"): + mgr._store.save_cert(mgr._ca.issue([dom])) + + wrapped = _SingleDomainStore(mgr._store, "server-2") + listed = wrapped.list_certs() + assert [b.domain for b in listed] == ["server-2"] + # __getattr__ delegation still reaches the real store + assert wrapped.load_cert("server-1") is not None + assert wrapped.delete_cert("server-3") is True + assert len(mgr._store.list_certs()) == 2 + # An empty domain (missing identity) matches nothing — the safe fallback + # that prevents an unscoped sweep of the whole shared store. + assert _SingleDomainStore(mgr._store, "").list_certs() == [] + + +# ── End-to-end SAN identity ─────────────────────────────────────────────────── + + +@pytest.mark.anyio +async def test_issued_cert_covers_advertised_host(): + """A cert issued from the helper's hostnames covers the dialed name.""" + from turnstone.console.tls import TLSManager + from turnstone.core.tls import build_cert_hostnames + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + hostnames = build_cert_hostnames("https://server-1:8080", extra_sans="server-1") + bundle = mgr._ca.issue(hostnames) + + # Stable, advertised-name store key (not the ephemeral container ID). + assert bundle.domain == "server-1" + assert "server-1" in _san_values(bundle.cert_pem) + + +# ── Renewal scoping (the storm fix) ─────────────────────────────────────────── + + +@pytest.mark.anyio +async def test_renewal_sweep_only_touches_own_domain(): + """A scoped sweep renews this node's cert and leaves siblings alone.""" + from turnstone.console.tls import TLSManager + from turnstone.core.tls import _SingleDomainStore + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + for dom in ("server-1", "server-2", "server-3"): + mgr._store.save_cert(mgr._ca.issue([dom])) + + # days_before_expiry is huge so every cert would be "due" — only scoping + # keeps the sweep from renewing siblings. + rm = lacme.RenewalManager( + ca=mgr._ca, + store=_SingleDomainStore(mgr._store, "server-1"), + days_before_expiry=99999, + ) + renewed = await rm.check_and_renew() + assert {b.domain for b in renewed} == {"server-1"} + + +# ── Orphan GC ───────────────────────────────────────────────────────────────── + + +@pytest.mark.anyio +async def test_gc_removes_only_long_expired_certs(): + """GC reclaims certs expired past the cutoff and keeps live ones.""" + from datetime import UTC, datetime, timedelta + + from turnstone.console.tls import TLSManager + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + live = mgr._ca.issue(["server-1"]) + mgr._store.save_cert(live) + + # A decommissioned node's row: reuse real PEMs but stamp it expired-long-ago. + dead = mgr._ca.issue(["dead-node"]) + old = (datetime.now(UTC) - timedelta(days=30)).isoformat() + get_storage().save_tls_cert( + domain="dead-node", + cert_pem=dead.cert_pem.decode(), + fullchain_pem=dead.fullchain_pem.decode(), + key_pem=dead.key_pem.decode(), + issued_at=old, + expires_at=old, + meta="{}", + ) + + removed = mgr.gc_expired_certs(max_age_days=7) + assert removed == 1 + domains = {b.domain for b in mgr._store.list_certs()} + assert domains == {"server-1"} + + +# ── Client-context caching + in-place reload ────────────────────────────────── + + +@pytest.mark.anyio +async def test_client_ctx_cached_and_reloaded_in_place(): + """The client context is cached and mutated in place on renewal.""" + from turnstone.console.tls import TLSManager + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + await mgr.issue_console_certs(["console"]) + + ctx1 = mgr.get_client_ssl_context() + ctx2 = mgr.get_client_ssl_context() + assert ctx1 is ctx2 # cached, not rebuilt per call + + # Reloading a renewed bundle must not raise and keeps the same object so + # httpx clients holding it pick up the new cert without a rebuild. + mgr._reload_client_ctx(mgr._ca.issue(["console"])) + assert mgr.get_client_ssl_context() is ctx1 + + +# ── Server-side renewal → reload-hook wiring ────────────────────────────────── + + +def test_renew_callback_updates_bundle_and_runs_reload_hook(): + """The renewal callback caches the new bundle and fires the reload hook.""" + from types import SimpleNamespace + + from turnstone.core.tls import TLSClient + + client = TLSClient(storage=get_storage(), hostnames=["server-1"]) + seen: list[object] = [] + client.set_cert_reload_hook(seen.append) + + bundle = SimpleNamespace(domain="server-1") + client._handle_renewed(bundle) + + assert client.bundle is bundle + assert seen == [bundle] + + +def test_renew_callback_swallows_reload_hook_errors(): + """A failing reload hook must not abort the renewal callback.""" + from types import SimpleNamespace + + from turnstone.core.tls import TLSClient + + client = TLSClient(storage=get_storage(), hostnames=["server-1"]) + + def _boom(_bundle: object) -> None: + raise RuntimeError("listener swap failed") + + client.set_cert_reload_hook(_boom) + bundle = SimpleNamespace(domain="server-1") + client._handle_renewed(bundle) # must not raise + assert client.bundle is bundle + + +# ── swap_context_cert (shared listener/client hot-swap) ─────────────────────── + + +def _tmp_pem_dirs() -> set[str]: + import glob + import tempfile + from pathlib import Path + + return set(glob.glob(str(Path(tempfile.gettempdir()) / "lacme-pem-*"))) + + +@pytest.mark.anyio +async def test_swap_context_cert_loads_and_leaves_no_temp_dir(): + """The hot-swap loads the renewed cert and reclaims its temp PEM dir.""" + import ssl + + from turnstone.console.tls import TLSManager + from turnstone.core.tls import swap_context_cert + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + bundle = mgr._ca.issue(["server-1"]) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + + before = _tmp_pem_dirs() + swap_context_cert(ctx, bundle, ca_pem=mgr.get_root_cert_pem()) + assert _tmp_pem_dirs() == before # no net leaked temp dir + + +@pytest.mark.anyio +async def test_swap_context_cert_cleans_up_on_failure(): + """A malformed bundle must not leave private-key material on disk.""" + import ssl + from types import SimpleNamespace + + from turnstone.console.tls import TLSManager + from turnstone.core.tls import swap_context_cert + + mgr = TLSManager(get_storage()) + await mgr.init_ca() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + bad = SimpleNamespace(domain="x", fullchain_pem=b"not a cert", key_pem=b"not a key") + + before = _tmp_pem_dirs() + with pytest.raises(ssl.SSLError): + swap_context_cert(ctx, bad, ca_pem=mgr.get_root_cert_pem()) + assert _tmp_pem_dirs() == before # temp dir removed even on load failure diff --git a/turnstone/console/collector.py b/turnstone/console/collector.py index 8cd6c082..ab652a7f 100644 --- a/turnstone/console/collector.py +++ b/turnstone/console/collector.py @@ -361,13 +361,20 @@ class ClusterCollector: except asyncio.CancelledError: raise except Exception as exc: - # Network / timeout / TLS errors — expected during brief - # node restarts. Keep at debug so the log doesn't flood - # on every backoff cycle; the warning above already - # covers configuration-level failures operators need to - # see. - log.debug("SSE error for node %s: %r", node_id, exc, exc_info=True) - self._mark_unreachable(node_id, reason=type(exc).__name__) + # Network / timeout / TLS errors. The FIRST failure + # (reachable→unreachable) is operator-actionable — a persistent + # TLS verify failure, refused connection, or DNS miss would + # otherwise be invisible — so surface it at WARNING. Subsequent + # retry failures drop to DEBUG to avoid flooding the log on + # every backoff cycle while the node stays down. + first_failure = self._mark_unreachable(node_id, reason=type(exc).__name__) + (log.warning if first_failure else log.debug)( + "SSE connection to node %s at %s failed: %r", + node_id, + url, + exc, + exc_info=first_failure, + ) await asyncio.sleep(min(backoff, 30) + random.random()) backoff = min(backoff * 2, 30) @@ -377,19 +384,25 @@ class ClusterCollector: node = self._nodes.get(node_id) return node.server_url if node else "" - def _mark_unreachable(self, node_id: str, reason: str = "") -> None: + def _mark_unreachable(self, node_id: str, reason: str = "") -> bool: """Mark a node as unreachable (thread-safe). ``reason`` is a short human-readable diagnostic (e.g. - ``"HTTP 403"``, ``"ConnectError"``) surfaced via the snapshot - + node endpoints so operators can see WHY a node is down. + ``"HTTP 403"``, ``"ConnectError"``, ``"SSLCertVerificationError"``) + surfaced via the snapshot + node endpoints so operators can see WHY a + node is down. Returns ``True`` when this is a reachable→unreachable + transition (the first failure), so callers can log it prominently and + stay quiet on subsequent retries. """ with self._lock: node = self._nodes.get(node_id) - if node: - node.reachable = False - if reason: - node.reachable_reason = reason + if not node: + return False + was_reachable = node.reachable + node.reachable = False + if reason: + node.reachable_reason = reason + return was_reachable # -- node discovery ------------------------------------------------------ diff --git a/turnstone/console/server.py b/turnstone/console/server.py index 570c5b0e..0a90b14f 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -4949,21 +4949,34 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: # TLS: init CA, issue console certs, start renewal tls_mgr = getattr(app.state, "tls_manager", None) if tls_mgr is not None: - import socket - try: if not tls_mgr.ca_initialized: await tls_mgr.init_ca() - hostname = socket.gethostname() - fqdn = socket.getfqdn() - cert_hostnames = [hostname, "localhost", "127.0.0.1"] - if fqdn != hostname: - cert_hostnames.append(fqdn) - extra_sans = os.environ.get("TURNSTONE_TLS_SANS", "") - if extra_sans: - cert_hostnames.extend(s.strip() for s in extra_sans.split(",") if s.strip()) + from turnstone.core.tls import build_cert_hostnames + + # Primary SAN = the console's advertised host so the cert is keyed + # by a stable name (not the container ID) and covers the name peers + # use to reach it. + cert_hostnames = build_cert_hostnames( + console_url, + extra_sans=os.environ.get("TURNSTONE_TLS_SANS", ""), + ) await tls_mgr.issue_console_certs(cert_hostnames) await tls_mgr.start_renewal() + # Reclaim cert rows for long-departed nodes; re-run periodically. + # Bind a stable non-None local so the closure keeps the narrowing. + gc_mgr = tls_mgr + gc_mgr.gc_expired_certs() + + async def _tls_gc_loop() -> None: + while True: + await asyncio.sleep(6 * 3600) + try: + gc_mgr.gc_expired_certs() + except Exception: + log.warning("tls.certs.gc_failed", exc_info=True) + + app.state.tls_gc_task = asyncio.create_task(_tls_gc_loop()) # Re-create proxy clients with mTLS context now that certs are ready client_ctx = tls_mgr.get_client_ssl_context() if client_ctx: @@ -5020,6 +5033,9 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: tls_mgr = getattr(app.state, "tls_manager", None) if tls_mgr is not None: await tls_mgr.stop_renewal() + _tls_gc_task = getattr(app.state, "tls_gc_task", None) + if _tls_gc_task is not None: + _tls_gc_task.cancel() if scheduler is not None: scheduler.stop() from turnstone.core.idle_nudge_watcher import shutdown_idle_nudge_watchers @@ -13420,9 +13436,11 @@ def main() -> None: import asyncio asyncio.run(tls_mgr.init_ca()) - # Upgrade scheme to https if no explicit URL was provided - if not _console_url_env: - console_url = console_url.replace("http://", "https://") + # console_url stays http://: the console serves HTTP (it's the + # ACME bootstrap endpoint nodes reach before trusting the CA). + # Browser HTTPS is terminated by a reverse proxy (caddy service + # / docs/tls.md); rewriting the scheme to https here would + # advertise an ACME URL nodes can't reach. log.info("TLS enabled") except ImportError: log.warning("TLS enabled but lacme not installed — pip install turnstone[tls]") diff --git a/turnstone/console/tls.py b/turnstone/console/tls.py index efe8a3ed..452b8601 100644 --- a/turnstone/console/tls.py +++ b/turnstone/console/tls.py @@ -70,6 +70,10 @@ class TLSManager: self._renewal_manager: Any | None = None self._internal_bundle: Any | None = None self._frontend_bundle: Any | None = None + # Cached mTLS client context, mutated in place on renewal so the + # proxy/collector httpx clients pick up the renewed client cert + # without being rebuilt. + self._client_ctx: ssl.SSLContext | None = None # Wire structlog to lacme events self._subscribe_events() @@ -278,17 +282,29 @@ class TLSManager: if self._ca is None: raise RuntimeError("CA not initialized") lacme = _require_lacme() + from turnstone.core.tls import _SingleDomainStore def _on_renewed(bundle: Any) -> None: # Update our cached bundles if the renewed domain matches if self._internal_bundle and bundle.domain == self._internal_bundle.domain: self._internal_bundle = bundle + # Swap the renewed material into the live mTLS client context + # so proxy/collector connections present the new cert. + self._reload_client_ctx(bundle) if self._frontend_bundle and bundle.domain == self._frontend_bundle.domain: self._frontend_bundle = bundle + # Scope the sweep to the console's own cert; the store is shared, so an + # unscoped CA-direct sweep would re-sign every node's cert. Empty domain + # → renew nothing (never the whole store). An external-ACME frontend + # cert has a different domain and is intentionally excluded (re-signing + # an externally-issued cert with the internal CA would break it). + own_domain = self._internal_bundle.domain if self._internal_bundle is not None else "" + renewal_store = _SingleDomainStore(self._store, own_domain) + self._renewal_manager = lacme.RenewalManager( ca=self._ca, - store=self._store, + store=renewal_store, interval_hours=_RENEW_INTERVAL_HOURS, days_before_expiry=_RENEW_BEFORE_EXPIRY_DAYS, on_renewed=_on_renewed, @@ -341,14 +357,47 @@ class TLSManager: """ if self._internal_bundle is None: return None - _require_lacme() - from lacme.mtls import client_ssl_context + if self._client_ctx is None: + _require_lacme() + from lacme.mtls import client_ssl_context - return client_ssl_context( # type: ignore[no-any-return,unused-ignore] - cert_pem=self._internal_bundle.cert_pem, - key_pem=self._internal_bundle.key_pem, - ca_cert_pem=self.get_root_cert_pem(), - ) + self._client_ctx = client_ssl_context( + cert_pem=self._internal_bundle.cert_pem, + key_pem=self._internal_bundle.key_pem, + ca_cert_pem=self.get_root_cert_pem(), + ) + return self._client_ctx + + def _reload_client_ctx(self, bundle: Any) -> None: + """Load a renewed bundle into the cached mTLS client context in place. + + httpx clients built with this context present the new client cert on + their next connection; existing keep-alive connections finish on the + old one. No-op until the context has been built. + """ + if self._client_ctx is None: + return + from turnstone.core.tls import swap_context_cert + + swap_context_cert(self._client_ctx, bundle) + + def gc_expired_certs(self, max_age_days: int = 7) -> int: + """Delete stored certs that expired more than ``max_age_days`` ago. + + A live node keeps its cert's ``expires_at`` in the future, so only + decommissioned-node (or legacy container-ID) rows go stale; deleting + them well past expiry is safe. Returns the number of rows removed. + """ + from datetime import UTC, datetime, timedelta + + cutoff = datetime.now(UTC) - timedelta(days=max_age_days) + removed = 0 + for bundle in self._store.list_certs(): + if bundle.expires_at < cutoff and self._store.delete_cert(bundle.domain): + removed += 1 + if removed: + log.info("tls.certs.gc", removed=removed, max_age_days=max_age_days) + return removed # -- Properties ------------------------------------------------------------ diff --git a/turnstone/core/tls.py b/turnstone/core/tls.py index 0eb06157..9eb16645 100644 --- a/turnstone/core/tls.py +++ b/turnstone/core/tls.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: import ssl + from collections.abc import Callable from turnstone.core.storage._protocol import StorageBackend @@ -38,6 +39,99 @@ def _require_lacme() -> Any: return lacme +def build_cert_hostnames( + advertise_url: str = "", + *, + bind_host: str = "", + extra_sans: str = "", +) -> list[str]: + """Build the ordered, de-duplicated SAN list for a service certificate. + + The advertised host (the name peers dial) goes **first**, becoming the + cert's primary domain. That makes it (a) a SAN, so mTLS hostname checks + pass — deriving SANs from ``gethostname()`` alone (the container ID) omits + it — and (b) a stable store key, so the cert reuses one row across + container recreations instead of orphaning one each time. Falls back to + ``gethostname()`` as primary only when no advertise URL is given. + """ + import socket + from urllib.parse import urlsplit + + names: list[str] = [] + if advertise_url: + host = urlsplit(advertise_url).hostname or "" + if host: + names.append(host) + # OS hostname (container ID under Docker) — keeps in-container self-dial + # working and provides a fallback primary on bare metal. + hostname = socket.gethostname() + names.append(hostname) + fqdn = socket.getfqdn() + if fqdn and fqdn != hostname: + names.append(fqdn) + names.extend(["localhost", "127.0.0.1"]) + if bind_host and bind_host not in ("0.0.0.0", "::", ""): + names.append(bind_host) + # Reject wildcard / unspecified-address SANs so a stray TURNSTONE_TLS_SANS + # can't mint an over-broad cert the internal CA would have peers trust. + for raw in extra_sans.split(","): + san = raw.strip() + if san and san not in ("0.0.0.0", "::", "*"): + names.append(san) + # De-duplicate, preserving first-seen order so the advertised host stays + # primary. + seen: set[str] = set() + ordered: list[str] = [] + for name in names: + if name and name not in seen: + seen.add(name) + ordered.append(name) + return ordered + + +class _SingleDomainStore: + """Store view exposing only one domain's cert to a renewal sweep. + + lacme's RenewalManager renews everything ``list_certs()`` returns. The + store is shared cluster-wide, so an unscoped manager on each node renews + every other node's (and every dead container's) cert — an N×M storm. This + wrapper limits the sweep to one domain; all other operations delegate to + the real store so renewed certs still persist to the shared database. + """ + + def __init__(self, inner: Any, domain: str) -> None: + self._inner = inner + self._domain = domain + + def list_certs(self) -> list[Any]: + cert = self._inner.load_cert(self._domain) + return [cert] if cert is not None else [] + + def __getattr__(self, name: str) -> Any: + return getattr(self._inner, name) + + +def swap_context_cert(ctx: ssl.SSLContext, bundle: Any, *, ca_pem: bytes | None = None) -> None: + """Hot-swap a renewed bundle into a live :class:`ssl.SSLContext`. + + Writes the bundle to short-lived PEM files, calls ``load_cert_chain`` (so + new handshakes use the renewed cert), then removes the temp dir — even on + failure, so a malformed bundle can't leave private-key material on disk. + Used by both the server listener context and the console client context. + """ + import contextlib + import shutil + + from lacme.mtls import write_pem_files + + paths = write_pem_files(bundle, ca_pem=ca_pem) + try: + ctx.load_cert_chain(str(paths.cert), str(paths.key)) + finally: + with contextlib.suppress(OSError): + shutil.rmtree(paths.cert.parent) + + class TLSClient: """TLS client for service nodes. @@ -72,6 +166,10 @@ class TLSClient: self._bundle: Any | None = None self._renewal_task: Any | None = None self._renewal_client: Any | None = None + # Optional hook invoked with each renewed bundle so the live HTTPS + # listener can swap in the new cert (uvicorn never reloads its SSL + # context on its own — see ``set_cert_reload_hook``). + self._cert_reload_hook: Callable[[Any], None] | None = None # Wire Prometheus metrics try: @@ -86,6 +184,25 @@ class TLSClient: else: raise + def set_cert_reload_hook(self, hook: Callable[[Any], None]) -> None: + """Register a callback that installs a renewed bundle into the listener. + + Renewal updates the DB + ``self._bundle`` but not the running uvicorn + listener, which keeps serving its boot cert until this hook swaps the + renewed cert into the live SSL context. + """ + self._cert_reload_hook = hook + + def _handle_renewed(self, bundle: Any) -> None: + """Renewal callback: cache the new bundle and run the reload hook.""" + self._bundle = bundle + log.info("tls.cert.renewed", domain=bundle.domain) + if self._cert_reload_hook is not None: + try: + self._cert_reload_hook(bundle) + except Exception: + log.warning("tls.cert.reload_hook_failed", exc_info=True) + async def init(self) -> None: """Fetch CA root cert and request a service certificate. @@ -169,10 +286,6 @@ class TLSClient: """Start background auto-renewal via the console's ACME endpoint.""" lacme = _require_lacme() - def _on_renewed(bundle: Any) -> None: - self._bundle = bundle - log.info("tls.cert.renewed", domain=bundle.domain) - from lacme.challenges.http01 import HTTP01Handler directory_url = f"{self._console_url}/acme/directory" @@ -185,12 +298,19 @@ class TLSClient: ) await client.__aenter__() + # Scope the renewal sweep to this node's own certificate. The store + # is shared cluster-wide; an unscoped RenewalManager would renew every + # node's cert on every node (see :class:`_SingleDomainStore`). An + # empty domain matches nothing, so a missing hostname renews nothing + # rather than falling back to re-signing the whole cluster. + own_domain = self._hostnames[0] if self._hostnames else "" + renewal_store = _SingleDomainStore(self._store, own_domain) manager = lacme.RenewalManager( client=client, - store=self._store, + store=renewal_store, interval_hours=_RENEW_INTERVAL_HOURS, days_before_expiry=_RENEW_BEFORE_EXPIRY_DAYS, - on_renewed=_on_renewed, + on_renewed=self._handle_renewed, event_dispatcher=self._event_dispatcher, ) self._renewal_task = manager.start() diff --git a/turnstone/server.py b/turnstone/server.py index 1937a91a..01deffcd 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -4734,20 +4734,18 @@ def main() -> None: try: import asyncio - from turnstone.core.tls import TLSClient + from turnstone.core.tls import TLSClient, build_cert_hostnames - hostname = socket.gethostname() - fqdn = socket.getfqdn() - hostnames = [hostname, "localhost", "127.0.0.1"] - if fqdn != hostname: - hostnames.append(fqdn) - # Only add bind host if it's a concrete address - if args.host not in ("0.0.0.0", "::", ""): - hostnames.append(args.host) - # Additional SANs from env (e.g. Docker service name) - extra_sans = os.environ.get("TURNSTONE_TLS_SANS", "") - if extra_sans: - hostnames.extend(s.strip() for s in extra_sans.split(",") if s.strip()) + # The advertised host (the name the collector + routing proxy dial) + # is placed first so it becomes the cert's primary domain / SAN and + # a stable store key. Deriving SANs from gethostname() alone (the + # container ID) omits the advertised name and breaks every mTLS + # handshake's hostname check. + hostnames = build_cert_hostnames( + _advertise_url, + bind_host=args.host, + extra_sans=os.environ.get("TURNSTONE_TLS_SANS", ""), + ) tls_client = TLSClient( storage=get_storage(), hostnames=hostnames, @@ -4769,6 +4767,24 @@ def main() -> None: # Store client on app state for lifespan renewal app.state.tls_client = tls_client + + def _reload_server_cert(new_bundle: Any) -> None: + """Swap a renewed cert into uvicorn's live SSL context. + + uvicorn loads its cert once at boot and never reloads, so + without this the served cert would expire mid-process and + break every mTLS peer. + """ + from turnstone.core.tls import swap_context_cert + + cfg = getattr(app.state, "uvicorn_config", None) + live_ctx = getattr(cfg, "ssl", None) if cfg is not None else None + if live_ctx is None: + return # listener not started yet — boot cert still valid + swap_context_cert(live_ctx, new_bundle, ca_pem=tls_client.ca_pem) + log.info("TLS cert reloaded into listener: %s", new_bundle.domain) + + tls_client.set_cert_reload_hook(_reload_server_cert) # Update advertise URL to HTTPS now that TLS is active if _advertise_url.startswith("http://"): app.state.advertise_url = _advertise_url.replace("http://", "https://", 1) @@ -4789,7 +4805,13 @@ def main() -> None: import uvicorn - uvicorn.run(app, host=args.host, port=args.port, log_level="warning", **ssl_kwargs) + uvicorn_config = uvicorn.Config( + app, host=args.host, port=args.port, log_level="warning", **ssl_kwargs + ) + # Expose the config so the TLS renewal hook can hot-swap the cert on the + # live SSL context (``config.ssl``); uvicorn has no built-in SSL reload. + app.state.uvicorn_config = uvicorn_config + uvicorn.Server(uvicorn_config).run() if __name__ == "__main__":