From a3ff07a86df7a4bd4f577eaa9cf54ff3179f50cd Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Tue, 9 Jun 2026 15:49:23 -0700 Subject: [PATCH] fix(tls): mTLS-aware container healthcheck + boot-time init retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A whole-stack restart races every node against the console for the CA fetch (compose re-enforces depends_on ordering only on `up`): losers logged one warning and served plain HTTP for their lifetime, while winners served mTLS that the plain-HTTP container healthcheck could never probe — leaving "healthy" plaintext nodes and "unhealthy" working ones. - TLSClient.init() grows attempts/base_delay retry (server passes 6 attempts, ~31 s backoff) absorbing the boot race; per-attempt CA-fetch failures log warning + debug traceback instead of error tracebacks. - healthcheck.py falls back to HTTPS when the plain probe fails, presenting the node's own cert as the client cert with the cluster CA pinned; dials localhost because the internal CA issues DNS SANs only. Default plain-HTTP deployments are unchanged. - The server writes boot PEMs under a fixed root (TURNSTONE_TLS_PEM_DIR, default /turnstone-tls) so the probe can find them; boot clears stale dirs and refuses a symlinked/foreign-owned root; renewal rewrites the PEM dir so the probe's client cert never outlives the served cert. - /health reports tls: "active"|"fallback" (absent when TLS is disabled) so a silently downgraded node is observable. --- docker/healthcheck.py | 86 +++++++++++-- docs/tls.md | 26 ++++ tests/test_docker_healthcheck.py | 213 +++++++++++++++++++++++++++++++ tests/test_health_tls_state.py | 65 ++++++++++ tests/test_tls_client.py | 181 ++++++++++++++++++++++++++ turnstone/core/tls.py | 118 ++++++++++++++++- turnstone/server.py | 40 +++++- 7 files changed, 707 insertions(+), 22 deletions(-) create mode 100644 tests/test_docker_healthcheck.py create mode 100644 tests/test_health_tls_state.py diff --git a/docker/healthcheck.py b/docker/healthcheck.py index 750ea4a0..3cd5d392 100644 --- a/docker/healthcheck.py +++ b/docker/healthcheck.py @@ -2,13 +2,69 @@ """Health check for turnstone containers. Usage: healthcheck.py -Exit 0 if the endpoint returns {"status": "ok"}, exit 1 otherwise. -Uses only stdlib — no pip dependencies required. +Exit 0 if the endpoint returns {"status": "ok"} or {"status": "degraded"}, +exit 1 otherwise. Uses only stdlib — no pip dependencies required. + +When the node serves mTLS (tls.enabled), a plain-HTTP probe is rejected at +the socket, so on failure this script retries over HTTPS, presenting the +node's own certificate as the client cert and pinning the cluster CA. The +PEM files are the ones the server writes at boot under +$TURNSTONE_TLS_PEM_DIR (default: /turnstone-tls). The host is +rewritten to "localhost" for the TLS attempt because the internal CA issues +DNS SANs only — certificate verification rejects a literal-IP dial. + +When mTLS is disabled (the default), the plain probe succeeds and nothing +here changes: the PEM directory is never consulted. """ import json +import os +import ssl import sys +import tempfile import urllib.request +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + + +def _check(url: str, context: ssl.SSLContext | None = None) -> None: + """Probe one URL; raise if unreachable or the payload is unhealthy.""" + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5, context=context) as resp: + data = json.loads(resp.read().decode()) + if data.get("status") not in ("ok", "degraded"): + raise RuntimeError(f"unhealthy payload: {data}") + + +def _pem_root() -> Path: + """PEM runtime root. + + Must mirror turnstone.core.tls.tls_pem_runtime_dir — this script is + standalone stdlib and cannot import turnstone; a drift-guard test in + tests/test_docker_healthcheck.py pins the two together. + """ + root_env = os.environ.get("TURNSTONE_TLS_PEM_DIR") + return Path(root_env) if root_env else Path(tempfile.gettempdir()) / "turnstone-tls" + + +def _find_pem_dir() -> Path | None: + """Locate the newest complete PEM dir written by the server at boot.""" + root = _pem_root() + candidates = [ + d + for d in root.glob("lacme-pem-*") + if all((d / name).is_file() for name in ("fullchain.pem", "key.pem", "ca.pem")) + ] + if not candidates: + return None + return max(candidates, key=lambda d: d.stat().st_mtime) + + +def _tls_url(url: str) -> str: + """Rewrite scheme to https and host to localhost, keeping port and path.""" + parts = urlsplit(url) + netloc = f"localhost:{parts.port}" if parts.port else "localhost" + return urlunsplit(("https", netloc, parts.path, parts.query, parts.fragment)) def main() -> None: @@ -18,16 +74,24 @@ def main() -> None: url = sys.argv[1] try: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - data = json.loads(resp.read().decode()) - if data.get("status") in ("ok", "degraded"): - sys.exit(0) - print(f"Unhealthy: {data}", file=sys.stderr) + _check(url) + sys.exit(0) + except Exception as plain_exc: + pem_dir = _find_pem_dir() + if pem_dir is None: + print(f"Health check failed: {plain_exc}", file=sys.stderr) + sys.exit(1) + try: + context = ssl.create_default_context(cafile=str(pem_dir / "ca.pem")) + context.load_cert_chain(str(pem_dir / "fullchain.pem"), str(pem_dir / "key.pem")) + _check(_tls_url(url), context=context) + sys.exit(0) + except Exception as tls_exc: + print( + f"Health check failed: plain: {plain_exc}; mtls: {tls_exc}", + file=sys.stderr, + ) sys.exit(1) - except Exception as exc: - print(f"Health check failed: {exc}", file=sys.stderr) - sys.exit(1) if __name__ == "__main__": diff --git a/docs/tls.md b/docs/tls.md index dfa81e7b..2d9a31bc 100644 --- a/docs/tls.md +++ b/docs/tls.md @@ -88,6 +88,32 @@ Console (CA + ACME Server) - **Frontend cert** (HTTPS): From an external ACME CA (e.g. Let's Encrypt) if `tls.acme_directory` is set, otherwise self-issued from the internal CA. +### Boot, retry, and fallback + +With `tls.enabled`, a node fetches the CA cert and requests its own cert +during startup, retrying with exponential backoff (6 attempts, ~31 s total) +— enough to absorb a whole-stack restart where every node races the console +for its listener. If all attempts fail, the node **falls back to plain +HTTP** (availability over confidentiality) and reports `"tls": "fallback"` +in `GET /health`; a node serving HTTPS reports `"tls": "active"`, and the +key is absent when TLS is disabled. Fallback persists until the next +restart — it is not upgraded in place. + +### Container healthcheck under mTLS + +An mTLS listener rejects plain-HTTP probes at the socket, so +`docker/healthcheck.py` falls back to HTTPS when the plain probe fails: +it presents the node's own cert as the client cert and pins the cluster +CA, using the PEM files the server writes at boot under +`$TURNSTONE_TLS_PEM_DIR` (default `/turnstone-tls`). The probe +dials `localhost` for the TLS attempt — the internal CA issues DNS SANs +only, so a literal-IP URL would fail verification. Cert renewal rewrites +the PEM dir alongside the live listener swap, so the probe's client cert +never outlives the served cert. With TLS disabled the plain probe succeeds +and the PEM directory is never consulted. On bare metal with multiple +nodes per host, set `TURNSTONE_TLS_PEM_DIR` per node (each boot clears +stale `lacme-pem-*` dirs under its root). + --- ## Configuration diff --git a/tests/test_docker_healthcheck.py b/tests/test_docker_healthcheck.py new file mode 100644 index 00000000..eea4d7c7 --- /dev/null +++ b/tests/test_docker_healthcheck.py @@ -0,0 +1,213 @@ +"""Tests for docker/healthcheck.py — the container health probe. + +Drives the real script via subprocess against real local listeners (plain +HTTP and mTLS with lacme-minted certs, the same CA path production uses), +mirroring how Docker invokes it. +""" + +from __future__ import annotations + +import http.server +import json +import os +import ssl +import subprocess +import sys +import threading +from pathlib import Path + +import pytest + +lacme = pytest.importorskip("lacme") + +SCRIPT = Path(__file__).parent.parent / "docker" / "healthcheck.py" + + +def run_healthcheck(url: str, pem_root: Path | None = None) -> subprocess.CompletedProcess: + env = dict(os.environ) + # Point the script at the test's PEM root — or at an empty dir to model + # a plain-HTTP node with no TLS material on disk. + env["TURNSTONE_TLS_PEM_DIR"] = str(pem_root) if pem_root else "/nonexistent" + return subprocess.run( + [sys.executable, str(SCRIPT), url], + capture_output=True, + text=True, + timeout=30, + env=env, + ) + + +class _Handler(http.server.BaseHTTPRequestHandler): + payload = {"status": "ok"} + + def do_GET(self): + body = json.dumps(self.payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +def _serve(handler_cls, ssl_context: ssl.SSLContext | None = None) -> int: + """Start a daemon-thread HTTP(S) server on an ephemeral port.""" + httpd = http.server.HTTPServer(("127.0.0.1", 0), handler_cls) + if ssl_context is not None: + httpd.socket = ssl_context.wrap_socket(httpd.socket, server_side=True) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd.server_address[1] + + +@pytest.fixture +def mtls_setup(tmp_path): + """Mint a CA + node cert exactly as the server does, write PEM files + under a runtime root, and build an mTLS server context requiring + client certs (mirrors uvicorn's ssl_cert_reqs=CERT_REQUIRED).""" + from lacme import CertificateAuthority, MemoryStore + from lacme.mtls import write_pem_files + + from turnstone.core.tls import build_cert_hostnames + + ca = CertificateAuthority(store=MemoryStore()) + ca.init() + bundle = ca.issue(build_cert_hostnames("http://node-1:8080", bind_host="0.0.0.0")) + + pem_root = tmp_path / "turnstone-tls" + pem_root.mkdir() + paths = write_pem_files(bundle, ca_pem=ca.root_cert_pem, directory=pem_root) + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(str(paths.cert), str(paths.key)) + server_ctx.load_verify_locations(str(paths.ca)) + server_ctx.verify_mode = ssl.CERT_REQUIRED + + return pem_root, server_ctx + + +# ── Plain HTTP (mTLS disabled — the default deployment) ───────────────────── + + +def test_plain_http_ok(): + """Default path: plain probe succeeds, PEM dir never consulted.""" + port = _serve(_Handler) + result = run_healthcheck(f"http://127.0.0.1:{port}/health") + assert result.returncode == 0, result.stderr + + +def test_plain_http_degraded_is_healthy(): + """'degraded' (backend down, server up) still counts as container-healthy.""" + + class Degraded(_Handler): + payload = {"status": "degraded"} + + port = _serve(Degraded) + result = run_healthcheck(f"http://127.0.0.1:{port}/health") + assert result.returncode == 0, result.stderr + + +def test_plain_http_bad_status_fails(): + class Bad(_Handler): + payload = {"status": "error"} + + port = _serve(Bad) + result = run_healthcheck(f"http://127.0.0.1:{port}/health") + assert result.returncode == 1 + assert "unhealthy payload" in result.stderr + + +def test_server_down_fails(): + """Nothing listening: fail, with or without PEM material around.""" + result = run_healthcheck("http://127.0.0.1:9/health") + assert result.returncode == 1 + assert "Health check failed" in result.stderr + + +# ── mTLS (tls.enabled) ─────────────────────────────────────────────────────── + + +def test_mtls_probe_with_pem_dir(mtls_setup): + """The regression case: mTLS node + plain-HTTP probe URL. + + The plain attempt is rejected at the socket; the script must fall back + to HTTPS with the node cert as client cert and report healthy.""" + pem_root, server_ctx = mtls_setup + port = _serve(_Handler, ssl_context=server_ctx) + result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=pem_root) + assert result.returncode == 0, result.stderr + + +def test_mtls_probe_without_pems_fails(mtls_setup): + """mTLS node but no PEM material on disk: the probe must fail.""" + _, server_ctx = mtls_setup + port = _serve(_Handler, ssl_context=server_ctx) + result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=None) + assert result.returncode == 1 + assert "Health check failed" in result.stderr + + +def test_mtls_unhealthy_payload_fails(mtls_setup): + """A reachable mTLS server with a bad payload is still unhealthy.""" + pem_root, server_ctx = mtls_setup + + class Bad(_Handler): + payload = {"status": "error"} + + port = _serve(Bad, ssl_context=server_ctx) + result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=pem_root) + assert result.returncode == 1 + assert "unhealthy payload" in result.stderr + + +def test_mtls_incomplete_pem_dir_fails(mtls_setup, tmp_path): + """A PEM dir missing the key is skipped, not half-used.""" + _, server_ctx = mtls_setup + incomplete = tmp_path / "incomplete-root" + d = incomplete / "lacme-pem-x" + d.mkdir(parents=True) + (d / "fullchain.pem").write_text("not a cert") + (d / "ca.pem").write_text("not a cert") + + port = _serve(_Handler, ssl_context=server_ctx) + result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=incomplete) + assert result.returncode == 1 + + +# ── Drift guards (script re-encodes contracts it cannot import) ────────────── + + +def _load_script_module(): + """Load healthcheck.py as a module — docker/ is not a package.""" + import importlib.util + + spec = importlib.util.spec_from_file_location("healthcheck_script", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_default_pem_root_matches_server(monkeypatch): + """Drift guard: the script's default PEM root equals the server's. + + The script cannot import turnstone (standalone stdlib), so the default + path literal is re-encoded; a rename on either side must fail here, not + silently break mTLS probing in production.""" + from turnstone.core.tls import tls_pem_runtime_dir + + monkeypatch.delenv("TURNSTONE_TLS_PEM_DIR", raising=False) + assert _load_script_module()._pem_root() == tls_pem_runtime_dir() + + +def test_find_pem_dir_accepts_real_pem_layout(monkeypatch, mtls_setup): + """Drift guard: lacme's on-disk layout is accepted by _find_pem_dir. + + Pins the lacme-pem-* dir prefix and the fullchain/key/ca filename + triplet against real write_pem_files output.""" + pem_root, _ = mtls_setup + monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(pem_root)) + found = _load_script_module()._find_pem_dir() + assert found is not None + assert found.parent == pem_root diff --git a/tests/test_health_tls_state.py b/tests/test_health_tls_state.py new file mode 100644 index 00000000..754efd23 --- /dev/null +++ b/tests/test_health_tls_state.py @@ -0,0 +1,65 @@ +"""/health surfaces the node's TLS state when tls.enabled is configured. + +A node that falls back to plain HTTP after a failed TLS init must be +observable (tls: "fallback"), and default plain-HTTP deployments must keep +an unchanged payload shape (no "tls" key). +""" + +from __future__ import annotations + +import queue +import threading +from unittest.mock import MagicMock + +import pytest + + +@pytest.fixture() +def make_client(): + from starlette.testclient import TestClient + + from turnstone.server import create_app + + clients = [] + + def _make(tls_state: str | None = None): + mock_mgr = MagicMock() + mock_mgr.list_all.return_value = [] + mock_mgr.max_active = 10 + app = create_app( + workstreams=mock_mgr, + global_queue=queue.Queue(), + global_listeners=[], + global_listeners_lock=threading.Lock(), + skip_permissions=False, + jwt_secret="test-jwt-secret-minimum-32-chars!", + ) + if tls_state is not None: + app.state.tls_state = tls_state + client = TestClient(app, raise_server_exceptions=False) + clients.append(client) + return client + + yield _make + for c in clients: + c.close() + + +def test_health_no_tls_key_by_default(make_client): + """mTLS disabled (default): payload shape unchanged — no tls key.""" + resp = make_client().get("/health") + assert resp.status_code == 200 + assert "tls" not in resp.json() + + +def test_health_tls_active(make_client): + resp = make_client(tls_state="active").get("/health") + assert resp.status_code == 200 + assert resp.json()["tls"] == "active" + + +def test_health_tls_fallback_visible(make_client): + """The silent-downgrade case must be observable in /health.""" + resp = make_client(tls_state="fallback").get("/health") + assert resp.status_code == 200 + assert resp.json()["tls"] == "fallback" diff --git a/tests/test_tls_client.py b/tests/test_tls_client.py index a570bf70..4cb44a52 100644 --- a/tests/test_tls_client.py +++ b/tests/test_tls_client.py @@ -2,6 +2,7 @@ from __future__ import annotations +from pathlib import Path from unittest.mock import MagicMock import pytest @@ -86,3 +87,183 @@ def test_collector_tls_defaults(): collector = ClusterCollector(storage=storage_mock) # Should store TLS settings for async client creation assert collector._tls_verify is True + + +# ── init() retry ───────────────────────────────────────────────────────────── + + +def _make_flaky_client(monkeypatch, failures: int): + """TLSClient whose CA fetch fails ``failures`` times, then succeeds. + + Returns (client, calls, sleeps) — mutable lists recording each CA-fetch + attempt and each backoff delay (asyncio.sleep is stubbed out). + """ + import asyncio + + from turnstone.core.tls import TLSClient + + client = TLSClient( + storage=get_storage(), + console_url="http://console:9999", + hostnames=["node-1"], + ) + calls: list[int] = [] + sleeps: list[float] = [] + + async def flaky_fetch(): + calls.append(len(calls) + 1) + if len(calls) <= failures: + raise ConnectionError("console not accepting connections yet") + + async def ok_request(): + pass + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(client, "_fetch_ca_cert", flaky_fetch) + monkeypatch.setattr(client, "_request_cert", ok_request) + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + return client, calls, sleeps + + +@pytest.mark.anyio +async def test_init_default_single_attempt(monkeypatch): + """Default init() keeps the old behavior: one attempt, no sleep.""" + client, calls, sleeps = _make_flaky_client(monkeypatch, failures=1) + with pytest.raises(ConnectionError): + await client.init() + assert calls == [1] + assert sleeps == [] + + +@pytest.mark.anyio +async def test_init_retries_transient_failure(monkeypatch): + """A transient console outage is absorbed by retries with backoff.""" + client, calls, sleeps = _make_flaky_client(monkeypatch, failures=2) + await client.init(attempts=6) + assert calls == [1, 2, 3] + assert sleeps == [1.0, 2.0] + + +@pytest.mark.anyio +async def test_init_retries_exhausted_raises(monkeypatch): + """When every attempt fails, the last error propagates.""" + client, calls, sleeps = _make_flaky_client(monkeypatch, failures=99) + with pytest.raises(ConnectionError): + await client.init(attempts=3) + assert calls == [1, 2, 3] + assert sleeps == [1.0, 2.0] + + +@pytest.mark.anyio +async def test_init_retries_discovery_failure(monkeypatch): + """Console discovery (not-yet-registered console) is retried too.""" + import asyncio + + from turnstone.core.tls import TLSClient + + client = TLSClient(storage=get_storage(), hostnames=["node-1"]) + attempts: list[int] = [] + + def flaky_discover(): + attempts.append(len(attempts) + 1) + if len(attempts) == 1: + raise RuntimeError("No console service found in services table.") + return "http://console:9999" + + async def ok(): + pass + + monkeypatch.setattr(client, "_discover_console_url", flaky_discover) + monkeypatch.setattr(client, "_fetch_ca_cert", ok) + monkeypatch.setattr(client, "_request_cert", ok) + monkeypatch.setattr(asyncio, "sleep", lambda _: ok()) + + await client.init(attempts=2) + assert attempts == [1, 2] + assert client._console_url == "http://console:9999" + + +# ── PEM runtime dir ────────────────────────────────────────────────────────── + + +def test_pem_runtime_dir_env_override(monkeypatch, tmp_path): + """TURNSTONE_TLS_PEM_DIR overrides the default location.""" + from turnstone.core.tls import tls_pem_runtime_dir + + monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(tmp_path / "custom")) + assert tls_pem_runtime_dir() == tmp_path / "custom" + + +def test_pem_runtime_dir_default(monkeypatch): + """Default lives under the system tempdir.""" + import tempfile + + from turnstone.core.tls import tls_pem_runtime_dir + + monkeypatch.delenv("TURNSTONE_TLS_PEM_DIR", raising=False) + assert tls_pem_runtime_dir() == Path(tempfile.gettempdir()) / "turnstone-tls" + + +def test_prepare_pem_runtime_dir_clears_stale(monkeypatch, tmp_path): + """Boot prep creates the dir 0700 and removes stale lacme-pem-* dirs.""" + from turnstone.core.tls import prepare_pem_runtime_dir + + root = tmp_path / "tls" + monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(root)) + stale = root / "lacme-pem-stale" + stale.mkdir(parents=True) + (stale / "key.pem").write_text("old") + (root / "unrelated").mkdir() + + result = prepare_pem_runtime_dir() + + assert result == root + assert not stale.exists() + assert (root / "unrelated").exists() # only lacme-pem-* is cleared + assert (root.stat().st_mode & 0o777) == 0o700 + + +def test_prepare_pem_runtime_dir_rejects_symlink(monkeypatch, tmp_path): + """A pre-created symlink at the root must be refused, not followed. + + On bare metal the default root sits in shared /tmp; following a + planted symlink would land key material under an attacker-chosen + path.""" + from turnstone.core.tls import prepare_pem_runtime_dir + + target = tmp_path / "elsewhere" + target.mkdir() + link = tmp_path / "tls-link" + link.symlink_to(target) + monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(link)) + + with pytest.raises(RuntimeError, match="symlink or not owned"): + prepare_pem_runtime_dir() + + +def test_refresh_runtime_pems_rotates_dir(monkeypatch, tmp_path): + """Renewal writes a fresh complete PEM dir, then drops the old one.""" + from lacme import CertificateAuthority, MemoryStore + + from turnstone.core.tls import prepare_pem_runtime_dir, refresh_runtime_pems + + monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(tmp_path / "tls")) + root = prepare_pem_runtime_dir() + + ca = CertificateAuthority(store=MemoryStore()) + ca.init() + boot_bundle = ca.issue(["node-1", "localhost"]) + renewed_bundle = ca.issue(["node-1", "localhost"]) + + boot = refresh_runtime_pems(boot_bundle, ca_pem=ca.root_cert_pem, previous=None) + boot_dir = boot.cert.parent + assert boot_dir.parent == root + + renewed = refresh_runtime_pems(renewed_bundle, ca_pem=ca.root_cert_pem, previous=boot_dir) + new_dir = renewed.cert.parent + assert new_dir.parent == root + assert not boot_dir.exists() + for name in ("fullchain.pem", "key.pem", "ca.pem"): + assert (new_dir / name).is_file() diff --git a/turnstone/core/tls.py b/turnstone/core/tls.py index 9eb16645..99045ced 100644 --- a/turnstone/core/tls.py +++ b/turnstone/core/tls.py @@ -13,6 +13,7 @@ Flow: from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -28,6 +29,81 @@ log = get_logger(__name__) _RENEW_INTERVAL_HOURS = 24 _RENEW_BEFORE_EXPIRY_DAYS = 1 +# Boot-time init retry budget: 1+2+4+8+16 s ≈ 31 s of backoff. Sized to +# absorb a whole-stack restart, where every node races the console for the +# CA cert (compose re-enforces depends_on ordering only on `up`, not +# `restart`) and the console needs a few seconds to start accepting +# connections. +TLS_INIT_RETRY_ATTEMPTS = 6 + + +def tls_pem_runtime_dir() -> Path: + """Parent directory for the boot-time PEM files. + + A fixed, well-known location (override: ``TURNSTONE_TLS_PEM_DIR``) so the + container healthcheck can present the node's own cert as an mTLS client + cert without DB access. ``write_pem_files`` creates a ``lacme-pem-*`` + subdirectory under it. + """ + import os + import tempfile + + env = os.environ.get("TURNSTONE_TLS_PEM_DIR") + return Path(env) if env else Path(tempfile.gettempdir()) / "turnstone-tls" + + +def prepare_pem_runtime_dir() -> Path: + """Create the PEM runtime dir (0700) and clear stale ``lacme-pem-*`` dirs. + + Stale subdirectories accumulate when a previous process dies before its + atexit cleanup runs (SIGKILL, OOM). Clearing them at boot — before the new + PEM dir is written — keeps exactly one live dir, so the healthcheck can't + pick up an expired cert. Assumes one node per PEM root: two processes + sharing a root would clear each other's live dirs (containers each get a + private tmpfs; on bare metal set TURNSTONE_TLS_PEM_DIR per node). + """ + import os + import shutil + import stat + + root = tls_pem_runtime_dir() + try: + st = os.lstat(root) + except FileNotFoundError: + st = None + if st is not None and (stat.S_ISLNK(st.st_mode) or st.st_uid != os.geteuid()): + # The default root lives in shared /tmp on bare metal: a hostile + # local user could pre-create it as a symlink (redirecting where the + # key material lands) or as a dir they own. Refuse both; our own + # stale dir from a prior boot passes (chmod below repairs mode). + raise RuntimeError( + f"PEM runtime dir {root} exists but is a symlink or not owned by this process" + ) + root.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(root, 0o700) + for stale in root.glob("lacme-pem-*"): + shutil.rmtree(stale, ignore_errors=True) + return root + + +def refresh_runtime_pems(bundle: Any, *, ca_pem: bytes | None, previous: Path | None) -> Any: + """Write a renewed bundle under the runtime root and drop the old dir. + + Keeps the on-disk PEMs (the healthcheck's mTLS client identity) in + lockstep with the served cert: certs live 48 hours, so the boot-time + files would expire and flip the container unhealthy two renewals in. + The new dir is written before the old one is removed, so a concurrent + probe always finds at least one complete dir. + """ + import shutil + + from lacme.mtls import write_pem_files + + new_paths = write_pem_files(bundle, ca_pem=ca_pem, directory=tls_pem_runtime_dir()) + if previous is not None and previous != new_paths.cert.parent: + shutil.rmtree(previous, ignore_errors=True) + return new_paths + def _require_lacme() -> Any: try: @@ -203,17 +279,42 @@ class TLSClient: except Exception: log.warning("tls.cert.reload_hook_failed", exc_info=True) - async def init(self) -> None: + async def init(self, *, attempts: int = 1, base_delay: float = 1.0) -> None: """Fetch CA root cert and request a service certificate. If no console_url was provided, discovers it from the services table. Performs initial cert provisioning over plain HTTP (ACME protocol provides integrity via JWS). + + With ``attempts > 1``, failures are retried with exponential backoff + (``base_delay * 2**n``). A node restarted alongside the console loses + the race for the console's listener by well under a second; without + retries that one refused connection downgrades the node to plain HTTP + for its entire lifetime, even when a valid cert sits in the store. + Discovery, CA fetch, and cert request are all idempotent, so the whole + sequence is retried as a unit. """ - if not self._console_url: - self._console_url = self._discover_console_url() - await self._fetch_ca_cert() - await self._request_cert() + import asyncio + + for attempt in range(1, attempts + 1): + try: + if not self._console_url: + self._console_url = self._discover_console_url() + await self._fetch_ca_cert() + await self._request_cert() + return + except Exception as exc: + if attempt >= attempts: + raise + delay = base_delay * 2 ** (attempt - 1) + log.warning( + "tls.init.retrying", + attempt=attempt, + max_attempts=attempts, + delay_seconds=delay, + error=f"{type(exc).__name__}: {exc}", + ) + await asyncio.sleep(delay) def _discover_console_url(self) -> str: """Look up the console URL from the services table.""" @@ -245,8 +346,11 @@ class TLSClient: resp.raise_for_status() self._ca_pem = resp.content log.info("tls.ca.fetched", url=url) - except Exception: - log.error("tls.ca.fetch_failed", url=url, exc_info=True) + except Exception as exc: + # Warning, not error: init() may retry this, and the terminal + # failure is logged by the caller. Full traceback at debug. + log.warning("tls.ca.fetch_failed", url=url, error=f"{type(exc).__name__}: {exc}") + log.debug("tls.ca.fetch_failed traceback", exc_info=True) raise async def _request_cert(self) -> None: diff --git a/turnstone/server.py b/turnstone/server.py index fb31e8d1..bedc3dd2 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1395,6 +1395,12 @@ def _build_health_dict(app_state: Any) -> dict[str, Any]: "resources": mc.resource_count, "prompts": mc.prompt_count, } + # Only present when tls.enabled: "active" (serving HTTPS) or "fallback" + # (TLS init failed, serving plain HTTP). Makes a silently-downgraded + # node observable. + tls_state = getattr(app_state, "tls_state", None) + if tls_state: + data["tls"] = tls_state return data @@ -4635,7 +4641,12 @@ def main() -> None: try: import asyncio - from turnstone.core.tls import TLSClient, build_cert_hostnames + from turnstone.core.tls import ( + TLS_INIT_RETRY_ATTEMPTS, + TLSClient, + build_cert_hostnames, + prepare_pem_runtime_dir, + ) # The advertised host (the name the collector + routing proxy dial) # is placed first so it becomes the cert's primary domain / SAN and @@ -4651,14 +4662,17 @@ def main() -> None: storage=get_storage(), hostnames=hostnames, ) - asyncio.run(tls_client.init()) + asyncio.run(tls_client.init(attempts=TLS_INIT_RETRY_ATTEMPTS)) bundle = tls_client.bundle if bundle: from lacme.mtls import write_pem_files_persistent + # Fixed parent dir (vs. a random tmpdir) so the container + # healthcheck can find the cert and probe over mTLS. pem_paths = write_pem_files_persistent( bundle, ca_pem=tls_client.ca_pem, + directory=prepare_pem_runtime_dir(), ) ssl_kwargs.update(pem_paths.as_uvicorn_kwargs()) if tls_client.ca_pem: @@ -4668,15 +4682,28 @@ def main() -> None: # Store client on app state for lifespan renewal app.state.tls_client = tls_client + pem_dir_state = {"dir": pem_paths.cert.parent} def _reload_server_cert(new_bundle: Any) -> None: """Swap a renewed cert into uvicorn's live SSL context. uvicorn loads its cert once at boot and never reloads, so without this the served cert would expire mid-process and - break every mTLS peer. + break every mTLS peer. The on-disk runtime PEMs (the + healthcheck's client identity) expire on the same clock, + so they are refreshed alongside. """ - from turnstone.core.tls import swap_context_cert + from turnstone.core.tls import refresh_runtime_pems, swap_context_cert + + try: + new_paths = refresh_runtime_pems( + new_bundle, + ca_pem=tls_client.ca_pem, + previous=pem_dir_state["dir"], + ) + pem_dir_state["dir"] = new_paths.cert.parent + except Exception: + log.warning("TLS runtime PEM refresh failed", exc_info=True) cfg = getattr(app.state, "uvicorn_config", None) live_ctx = getattr(cfg, "ssl", None) if cfg is not None else None @@ -4691,10 +4718,15 @@ def main() -> None: app.state.advertise_url = _advertise_url.replace("http://", "https://", 1) else: app.state.advertise_url = _advertise_url + app.state.tls_state = "active" log.info("TLS enabled — serving HTTPS") else: + app.state.tls_state = "fallback" log.warning("TLS enabled but no cert available") except Exception as exc: + # Surfaced as tls:"fallback" in /health — a node serving plain + # HTTP while TLS is configured should be visible, not silent. + app.state.tls_state = "fallback" log.warning( "TLS initialization failed — serving plain HTTP: %s: %s", type(exc).__name__,