fix(tls): mTLS-aware container healthcheck + boot-time init retry

A whole-stack restart races every node against the console for the CA
fetch (compose re-enforces depends_on ordering only on `up`): losers
logged one warning and served plain HTTP for their lifetime, while
winners served mTLS that the plain-HTTP container healthcheck could
never probe — leaving "healthy" plaintext nodes and "unhealthy"
working ones.

- TLSClient.init() grows attempts/base_delay retry (server passes 6
  attempts, ~31 s backoff) absorbing the boot race; per-attempt CA-fetch
  failures log warning + debug traceback instead of error tracebacks.
- healthcheck.py falls back to HTTPS when the plain probe fails,
  presenting the node's own cert as the client cert with the cluster CA
  pinned; dials localhost because the internal CA issues DNS SANs only.
  Default plain-HTTP deployments are unchanged.
- The server writes boot PEMs under a fixed root (TURNSTONE_TLS_PEM_DIR,
  default <tmpdir>/turnstone-tls) so the probe can find them; boot
  clears stale dirs and refuses a symlinked/foreign-owned root; renewal
  rewrites the PEM dir so the probe's client cert never outlives the
  served cert.
- /health reports tls: "active"|"fallback" (absent when TLS is
  disabled) so a silently downgraded node is observable.

docker/healthcheck.py

@@ -2,13 +2,69 @@
 """Health check for turnstone containers.
 
 Usage: healthcheck.py <url>
-Exit 0 if the endpoint returns {"status": "ok"}, exit 1 otherwise.
-Uses only stdlib — no pip dependencies required.
+Exit 0 if the endpoint returns {"status": "ok"} or {"status": "degraded"},
+exit 1 otherwise. Uses only stdlib — no pip dependencies required.
+
+When the node serves mTLS (tls.enabled), a plain-HTTP probe is rejected at
+the socket, so on failure this script retries over HTTPS, presenting the
+node's own certificate as the client cert and pinning the cluster CA. The
+PEM files are the ones the server writes at boot under
+$TURNSTONE_TLS_PEM_DIR (default: <tmpdir>/turnstone-tls). The host is
+rewritten to "localhost" for the TLS attempt because the internal CA issues
+DNS SANs only — certificate verification rejects a literal-IP dial.
+
+When mTLS is disabled (the default), the plain probe succeeds and nothing
+here changes: the PEM directory is never consulted.
 """
 
 import json
+import os
+import ssl
 import sys
+import tempfile
 import urllib.request
+from pathlib import Path
+from urllib.parse import urlsplit, urlunsplit
+
+
+def _check(url: str, context: ssl.SSLContext | None = None) -> None:
+    """Probe one URL; raise if unreachable or the payload is unhealthy."""
+    req = urllib.request.Request(url, method="GET")
+    with urllib.request.urlopen(req, timeout=5, context=context) as resp:
+        data = json.loads(resp.read().decode())
+    if data.get("status") not in ("ok", "degraded"):
+        raise RuntimeError(f"unhealthy payload: {data}")
+
+
+def _pem_root() -> Path:
+    """PEM runtime root.
+
+    Must mirror turnstone.core.tls.tls_pem_runtime_dir — this script is
+    standalone stdlib and cannot import turnstone; a drift-guard test in
+    tests/test_docker_healthcheck.py pins the two together.
+    """
+    root_env = os.environ.get("TURNSTONE_TLS_PEM_DIR")
+    return Path(root_env) if root_env else Path(tempfile.gettempdir()) / "turnstone-tls"
+
+
+def _find_pem_dir() -> Path | None:
+    """Locate the newest complete PEM dir written by the server at boot."""
+    root = _pem_root()
+    candidates = [
+        d
+        for d in root.glob("lacme-pem-*")
+        if all((d / name).is_file() for name in ("fullchain.pem", "key.pem", "ca.pem"))
+    ]
+    if not candidates:
+        return None
+    return max(candidates, key=lambda d: d.stat().st_mtime)
+
+
+def _tls_url(url: str) -> str:
+    """Rewrite scheme to https and host to localhost, keeping port and path."""
+    parts = urlsplit(url)
+    netloc = f"localhost:{parts.port}" if parts.port else "localhost"
+    return urlunsplit(("https", netloc, parts.path, parts.query, parts.fragment))
 
 
 def main() -> None:
@@ -18,15 +74,23 @@ def main() -> None:
 
     url = sys.argv[1]
     try:
-        req = urllib.request.Request(url, method="GET")
-        with urllib.request.urlopen(req, timeout=5) as resp:
-            data = json.loads(resp.read().decode())
-            if data.get("status") in ("ok", "degraded"):
+        _check(url)
         sys.exit(0)
-            print(f"Unhealthy: {data}", file=sys.stderr)
+    except Exception as plain_exc:
+        pem_dir = _find_pem_dir()
+        if pem_dir is None:
+            print(f"Health check failed: {plain_exc}", file=sys.stderr)
             sys.exit(1)
-    except Exception as exc:
-        print(f"Health check failed: {exc}", file=sys.stderr)
+        try:
+            context = ssl.create_default_context(cafile=str(pem_dir / "ca.pem"))
+            context.load_cert_chain(str(pem_dir / "fullchain.pem"), str(pem_dir / "key.pem"))
+            _check(_tls_url(url), context=context)
+            sys.exit(0)
+        except Exception as tls_exc:
+            print(
+                f"Health check failed: plain: {plain_exc}; mtls: {tls_exc}",
+                file=sys.stderr,
+            )
             sys.exit(1)
 
 

docs/tls.md

@@ -88,6 +88,32 @@ Console (CA + ACME Server)
 - **Frontend cert** (HTTPS): From an external ACME CA (e.g. Let's Encrypt)
   if `tls.acme_directory` is set, otherwise self-issued from the internal CA.
 
+### Boot, retry, and fallback
+
+With `tls.enabled`, a node fetches the CA cert and requests its own cert
+during startup, retrying with exponential backoff (6 attempts, ~31 s total)
+— enough to absorb a whole-stack restart where every node races the console
+for its listener. If all attempts fail, the node **falls back to plain
+HTTP** (availability over confidentiality) and reports `"tls": "fallback"`
+in `GET /health`; a node serving HTTPS reports `"tls": "active"`, and the
+key is absent when TLS is disabled. Fallback persists until the next
+restart — it is not upgraded in place.
+
+### Container healthcheck under mTLS
+
+An mTLS listener rejects plain-HTTP probes at the socket, so
+`docker/healthcheck.py` falls back to HTTPS when the plain probe fails:
+it presents the node's own cert as the client cert and pins the cluster
+CA, using the PEM files the server writes at boot under
+`$TURNSTONE_TLS_PEM_DIR` (default `<tmpdir>/turnstone-tls`). The probe
+dials `localhost` for the TLS attempt — the internal CA issues DNS SANs
+only, so a literal-IP URL would fail verification. Cert renewal rewrites
+the PEM dir alongside the live listener swap, so the probe's client cert
+never outlives the served cert. With TLS disabled the plain probe succeeds
+and the PEM directory is never consulted. On bare metal with multiple
+nodes per host, set `TURNSTONE_TLS_PEM_DIR` per node (each boot clears
+stale `lacme-pem-*` dirs under its root).
+
 ---
 
 ## Configuration

tests/test_docker_healthcheck.py

@@ -0,0 +1,213 @@
+"""Tests for docker/healthcheck.py — the container health probe.
+
+Drives the real script via subprocess against real local listeners (plain
+HTTP and mTLS with lacme-minted certs, the same CA path production uses),
+mirroring how Docker invokes it.
+"""
+
+from __future__ import annotations
+
+import http.server
+import json
+import os
+import ssl
+import subprocess
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+lacme = pytest.importorskip("lacme")
+
+SCRIPT = Path(__file__).parent.parent / "docker" / "healthcheck.py"
+
+
+def run_healthcheck(url: str, pem_root: Path | None = None) -> subprocess.CompletedProcess:
+    env = dict(os.environ)
+    # Point the script at the test's PEM root — or at an empty dir to model
+    # a plain-HTTP node with no TLS material on disk.
+    env["TURNSTONE_TLS_PEM_DIR"] = str(pem_root) if pem_root else "/nonexistent"
+    return subprocess.run(
+        [sys.executable, str(SCRIPT), url],
+        capture_output=True,
+        text=True,
+        timeout=30,
+        env=env,
+    )
+
+
+class _Handler(http.server.BaseHTTPRequestHandler):
+    payload = {"status": "ok"}
+
+    def do_GET(self):
+        body = json.dumps(self.payload).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):
+        pass
+
+
+def _serve(handler_cls, ssl_context: ssl.SSLContext | None = None) -> int:
+    """Start a daemon-thread HTTP(S) server on an ephemeral port."""
+    httpd = http.server.HTTPServer(("127.0.0.1", 0), handler_cls)
+    if ssl_context is not None:
+        httpd.socket = ssl_context.wrap_socket(httpd.socket, server_side=True)
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+    return httpd.server_address[1]
+
+
+@pytest.fixture
+def mtls_setup(tmp_path):
+    """Mint a CA + node cert exactly as the server does, write PEM files
+    under a runtime root, and build an mTLS server context requiring
+    client certs (mirrors uvicorn's ssl_cert_reqs=CERT_REQUIRED)."""
+    from lacme import CertificateAuthority, MemoryStore
+    from lacme.mtls import write_pem_files
+
+    from turnstone.core.tls import build_cert_hostnames
+
+    ca = CertificateAuthority(store=MemoryStore())
+    ca.init()
+    bundle = ca.issue(build_cert_hostnames("http://node-1:8080", bind_host="0.0.0.0"))
+
+    pem_root = tmp_path / "turnstone-tls"
+    pem_root.mkdir()
+    paths = write_pem_files(bundle, ca_pem=ca.root_cert_pem, directory=pem_root)
+
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.load_cert_chain(str(paths.cert), str(paths.key))
+    server_ctx.load_verify_locations(str(paths.ca))
+    server_ctx.verify_mode = ssl.CERT_REQUIRED
+
+    return pem_root, server_ctx
+
+
+# ── Plain HTTP (mTLS disabled — the default deployment) ─────────────────────
+
+
+def test_plain_http_ok():
+    """Default path: plain probe succeeds, PEM dir never consulted."""
+    port = _serve(_Handler)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health")
+    assert result.returncode == 0, result.stderr
+
+
+def test_plain_http_degraded_is_healthy():
+    """'degraded' (backend down, server up) still counts as container-healthy."""
+
+    class Degraded(_Handler):
+        payload = {"status": "degraded"}
+
+    port = _serve(Degraded)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health")
+    assert result.returncode == 0, result.stderr
+
+
+def test_plain_http_bad_status_fails():
+    class Bad(_Handler):
+        payload = {"status": "error"}
+
+    port = _serve(Bad)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health")
+    assert result.returncode == 1
+    assert "unhealthy payload" in result.stderr
+
+
+def test_server_down_fails():
+    """Nothing listening: fail, with or without PEM material around."""
+    result = run_healthcheck("http://127.0.0.1:9/health")
+    assert result.returncode == 1
+    assert "Health check failed" in result.stderr
+
+
+# ── mTLS (tls.enabled) ───────────────────────────────────────────────────────
+
+
+def test_mtls_probe_with_pem_dir(mtls_setup):
+    """The regression case: mTLS node + plain-HTTP probe URL.
+
+    The plain attempt is rejected at the socket; the script must fall back
+    to HTTPS with the node cert as client cert and report healthy."""
+    pem_root, server_ctx = mtls_setup
+    port = _serve(_Handler, ssl_context=server_ctx)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=pem_root)
+    assert result.returncode == 0, result.stderr
+
+
+def test_mtls_probe_without_pems_fails(mtls_setup):
+    """mTLS node but no PEM material on disk: the probe must fail."""
+    _, server_ctx = mtls_setup
+    port = _serve(_Handler, ssl_context=server_ctx)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=None)
+    assert result.returncode == 1
+    assert "Health check failed" in result.stderr
+
+
+def test_mtls_unhealthy_payload_fails(mtls_setup):
+    """A reachable mTLS server with a bad payload is still unhealthy."""
+    pem_root, server_ctx = mtls_setup
+
+    class Bad(_Handler):
+        payload = {"status": "error"}
+
+    port = _serve(Bad, ssl_context=server_ctx)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=pem_root)
+    assert result.returncode == 1
+    assert "unhealthy payload" in result.stderr
+
+
+def test_mtls_incomplete_pem_dir_fails(mtls_setup, tmp_path):
+    """A PEM dir missing the key is skipped, not half-used."""
+    _, server_ctx = mtls_setup
+    incomplete = tmp_path / "incomplete-root"
+    d = incomplete / "lacme-pem-x"
+    d.mkdir(parents=True)
+    (d / "fullchain.pem").write_text("not a cert")
+    (d / "ca.pem").write_text("not a cert")
+
+    port = _serve(_Handler, ssl_context=server_ctx)
+    result = run_healthcheck(f"http://127.0.0.1:{port}/health", pem_root=incomplete)
+    assert result.returncode == 1
+
+
+# ── Drift guards (script re-encodes contracts it cannot import) ──────────────
+
+
+def _load_script_module():
+    """Load healthcheck.py as a module — docker/ is not a package."""
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location("healthcheck_script", SCRIPT)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_default_pem_root_matches_server(monkeypatch):
+    """Drift guard: the script's default PEM root equals the server's.
+
+    The script cannot import turnstone (standalone stdlib), so the default
+    path literal is re-encoded; a rename on either side must fail here, not
+    silently break mTLS probing in production."""
+    from turnstone.core.tls import tls_pem_runtime_dir
+
+    monkeypatch.delenv("TURNSTONE_TLS_PEM_DIR", raising=False)
+    assert _load_script_module()._pem_root() == tls_pem_runtime_dir()
+
+
+def test_find_pem_dir_accepts_real_pem_layout(monkeypatch, mtls_setup):
+    """Drift guard: lacme's on-disk layout is accepted by _find_pem_dir.
+
+    Pins the lacme-pem-* dir prefix and the fullchain/key/ca filename
+    triplet against real write_pem_files output."""
+    pem_root, _ = mtls_setup
+    monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(pem_root))
+    found = _load_script_module()._find_pem_dir()
+    assert found is not None
+    assert found.parent == pem_root

tests/test_health_tls_state.py

@@ -0,0 +1,65 @@
+"""/health surfaces the node's TLS state when tls.enabled is configured.
+
+A node that falls back to plain HTTP after a failed TLS init must be
+observable (tls: "fallback"), and default plain-HTTP deployments must keep
+an unchanged payload shape (no "tls" key).
+"""
+
+from __future__ import annotations
+
+import queue
+import threading
+from unittest.mock import MagicMock
+
+import pytest
+
+
+@pytest.fixture()
+def make_client():
+    from starlette.testclient import TestClient
+
+    from turnstone.server import create_app
+
+    clients = []
+
+    def _make(tls_state: str | None = None):
+        mock_mgr = MagicMock()
+        mock_mgr.list_all.return_value = []
+        mock_mgr.max_active = 10
+        app = create_app(
+            workstreams=mock_mgr,
+            global_queue=queue.Queue(),
+            global_listeners=[],
+            global_listeners_lock=threading.Lock(),
+            skip_permissions=False,
+            jwt_secret="test-jwt-secret-minimum-32-chars!",
+        )
+        if tls_state is not None:
+            app.state.tls_state = tls_state
+        client = TestClient(app, raise_server_exceptions=False)
+        clients.append(client)
+        return client
+
+    yield _make
+    for c in clients:
+        c.close()
+
+
+def test_health_no_tls_key_by_default(make_client):
+    """mTLS disabled (default): payload shape unchanged — no tls key."""
+    resp = make_client().get("/health")
+    assert resp.status_code == 200
+    assert "tls" not in resp.json()
+
+
+def test_health_tls_active(make_client):
+    resp = make_client(tls_state="active").get("/health")
+    assert resp.status_code == 200
+    assert resp.json()["tls"] == "active"
+
+
+def test_health_tls_fallback_visible(make_client):
+    """The silent-downgrade case must be observable in /health."""
+    resp = make_client(tls_state="fallback").get("/health")
+    assert resp.status_code == 200
+    assert resp.json()["tls"] == "fallback"

tests/test_tls_client.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
@@ -86,3 +87,183 @@ def test_collector_tls_defaults():
     collector = ClusterCollector(storage=storage_mock)
     # Should store TLS settings for async client creation
     assert collector._tls_verify is True
+
+
+# ── init() retry ─────────────────────────────────────────────────────────────
+
+
+def _make_flaky_client(monkeypatch, failures: int):
+    """TLSClient whose CA fetch fails ``failures`` times, then succeeds.
+
+    Returns (client, calls, sleeps) — mutable lists recording each CA-fetch
+    attempt and each backoff delay (asyncio.sleep is stubbed out).
+    """
+    import asyncio
+
+    from turnstone.core.tls import TLSClient
+
+    client = TLSClient(
+        storage=get_storage(),
+        console_url="http://console:9999",
+        hostnames=["node-1"],
+    )
+    calls: list[int] = []
+    sleeps: list[float] = []
+
+    async def flaky_fetch():
+        calls.append(len(calls) + 1)
+        if len(calls) <= failures:
+            raise ConnectionError("console not accepting connections yet")
+
+    async def ok_request():
+        pass
+
+    async def fake_sleep(delay):
+        sleeps.append(delay)
+
+    monkeypatch.setattr(client, "_fetch_ca_cert", flaky_fetch)
+    monkeypatch.setattr(client, "_request_cert", ok_request)
+    monkeypatch.setattr(asyncio, "sleep", fake_sleep)
+    return client, calls, sleeps
+
+
+@pytest.mark.anyio
+async def test_init_default_single_attempt(monkeypatch):
+    """Default init() keeps the old behavior: one attempt, no sleep."""
+    client, calls, sleeps = _make_flaky_client(monkeypatch, failures=1)
+    with pytest.raises(ConnectionError):
+        await client.init()
+    assert calls == [1]
+    assert sleeps == []
+
+
+@pytest.mark.anyio
+async def test_init_retries_transient_failure(monkeypatch):
+    """A transient console outage is absorbed by retries with backoff."""
+    client, calls, sleeps = _make_flaky_client(monkeypatch, failures=2)
+    await client.init(attempts=6)
+    assert calls == [1, 2, 3]
+    assert sleeps == [1.0, 2.0]
+
+
+@pytest.mark.anyio
+async def test_init_retries_exhausted_raises(monkeypatch):
+    """When every attempt fails, the last error propagates."""
+    client, calls, sleeps = _make_flaky_client(monkeypatch, failures=99)
+    with pytest.raises(ConnectionError):
+        await client.init(attempts=3)
+    assert calls == [1, 2, 3]
+    assert sleeps == [1.0, 2.0]
+
+
+@pytest.mark.anyio
+async def test_init_retries_discovery_failure(monkeypatch):
+    """Console discovery (not-yet-registered console) is retried too."""
+    import asyncio
+
+    from turnstone.core.tls import TLSClient
+
+    client = TLSClient(storage=get_storage(), hostnames=["node-1"])
+    attempts: list[int] = []
+
+    def flaky_discover():
+        attempts.append(len(attempts) + 1)
+        if len(attempts) == 1:
+            raise RuntimeError("No console service found in services table.")
+        return "http://console:9999"
+
+    async def ok():
+        pass
+
+    monkeypatch.setattr(client, "_discover_console_url", flaky_discover)
+    monkeypatch.setattr(client, "_fetch_ca_cert", ok)
+    monkeypatch.setattr(client, "_request_cert", ok)
+    monkeypatch.setattr(asyncio, "sleep", lambda _: ok())
+
+    await client.init(attempts=2)
+    assert attempts == [1, 2]
+    assert client._console_url == "http://console:9999"
+
+
+# ── PEM runtime dir ──────────────────────────────────────────────────────────
+
+
+def test_pem_runtime_dir_env_override(monkeypatch, tmp_path):
+    """TURNSTONE_TLS_PEM_DIR overrides the default location."""
+    from turnstone.core.tls import tls_pem_runtime_dir
+
+    monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(tmp_path / "custom"))
+    assert tls_pem_runtime_dir() == tmp_path / "custom"
+
+
+def test_pem_runtime_dir_default(monkeypatch):
+    """Default lives under the system tempdir."""
+    import tempfile
+
+    from turnstone.core.tls import tls_pem_runtime_dir
+
+    monkeypatch.delenv("TURNSTONE_TLS_PEM_DIR", raising=False)
+    assert tls_pem_runtime_dir() == Path(tempfile.gettempdir()) / "turnstone-tls"
+
+
+def test_prepare_pem_runtime_dir_clears_stale(monkeypatch, tmp_path):
+    """Boot prep creates the dir 0700 and removes stale lacme-pem-* dirs."""
+    from turnstone.core.tls import prepare_pem_runtime_dir
+
+    root = tmp_path / "tls"
+    monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(root))
+    stale = root / "lacme-pem-stale"
+    stale.mkdir(parents=True)
+    (stale / "key.pem").write_text("old")
+    (root / "unrelated").mkdir()
+
+    result = prepare_pem_runtime_dir()
+
+    assert result == root
+    assert not stale.exists()
+    assert (root / "unrelated").exists()  # only lacme-pem-* is cleared
+    assert (root.stat().st_mode & 0o777) == 0o700
+
+
+def test_prepare_pem_runtime_dir_rejects_symlink(monkeypatch, tmp_path):
+    """A pre-created symlink at the root must be refused, not followed.
+
+    On bare metal the default root sits in shared /tmp; following a
+    planted symlink would land key material under an attacker-chosen
+    path."""
+    from turnstone.core.tls import prepare_pem_runtime_dir
+
+    target = tmp_path / "elsewhere"
+    target.mkdir()
+    link = tmp_path / "tls-link"
+    link.symlink_to(target)
+    monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(link))
+
+    with pytest.raises(RuntimeError, match="symlink or not owned"):
+        prepare_pem_runtime_dir()
+
+
+def test_refresh_runtime_pems_rotates_dir(monkeypatch, tmp_path):
+    """Renewal writes a fresh complete PEM dir, then drops the old one."""
+    from lacme import CertificateAuthority, MemoryStore
+
+    from turnstone.core.tls import prepare_pem_runtime_dir, refresh_runtime_pems
+
+    monkeypatch.setenv("TURNSTONE_TLS_PEM_DIR", str(tmp_path / "tls"))
+    root = prepare_pem_runtime_dir()
+
+    ca = CertificateAuthority(store=MemoryStore())
+    ca.init()
+    boot_bundle = ca.issue(["node-1", "localhost"])
+    renewed_bundle = ca.issue(["node-1", "localhost"])
+
+    boot = refresh_runtime_pems(boot_bundle, ca_pem=ca.root_cert_pem, previous=None)
+    boot_dir = boot.cert.parent
+    assert boot_dir.parent == root
+
+    renewed = refresh_runtime_pems(renewed_bundle, ca_pem=ca.root_cert_pem, previous=boot_dir)
+    new_dir = renewed.cert.parent
+    assert new_dir.parent == root
+    assert not boot_dir.exists()
+    for name in ("fullchain.pem", "key.pem", "ca.pem"):
+        assert (new_dir / name).is_file()

turnstone/core/tls.py

@@ -13,6 +13,7 @@ Flow:
 
 from __future__ import annotations
 
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -28,6 +29,81 @@ log = get_logger(__name__)
 _RENEW_INTERVAL_HOURS = 24
 _RENEW_BEFORE_EXPIRY_DAYS = 1
 
+# Boot-time init retry budget: 1+2+4+8+16 s ≈ 31 s of backoff. Sized to
+# absorb a whole-stack restart, where every node races the console for the
+# CA cert (compose re-enforces depends_on ordering only on `up`, not
+# `restart`) and the console needs a few seconds to start accepting
+# connections.
+TLS_INIT_RETRY_ATTEMPTS = 6
+
+
+def tls_pem_runtime_dir() -> Path:
+    """Parent directory for the boot-time PEM files.
+
+    A fixed, well-known location (override: ``TURNSTONE_TLS_PEM_DIR``) so the
+    container healthcheck can present the node's own cert as an mTLS client
+    cert without DB access. ``write_pem_files`` creates a ``lacme-pem-*``
+    subdirectory under it.
+    """
+    import os
+    import tempfile
+
+    env = os.environ.get("TURNSTONE_TLS_PEM_DIR")
+    return Path(env) if env else Path(tempfile.gettempdir()) / "turnstone-tls"
+
+
+def prepare_pem_runtime_dir() -> Path:
+    """Create the PEM runtime dir (0700) and clear stale ``lacme-pem-*`` dirs.
+
+    Stale subdirectories accumulate when a previous process dies before its
+    atexit cleanup runs (SIGKILL, OOM). Clearing them at boot — before the new
+    PEM dir is written — keeps exactly one live dir, so the healthcheck can't
+    pick up an expired cert. Assumes one node per PEM root: two processes
+    sharing a root would clear each other's live dirs (containers each get a
+    private tmpfs; on bare metal set TURNSTONE_TLS_PEM_DIR per node).
+    """
+    import os
+    import shutil
+    import stat
+
+    root = tls_pem_runtime_dir()
+    try:
+        st = os.lstat(root)
+    except FileNotFoundError:
+        st = None
+    if st is not None and (stat.S_ISLNK(st.st_mode) or st.st_uid != os.geteuid()):
+        # The default root lives in shared /tmp on bare metal: a hostile
+        # local user could pre-create it as a symlink (redirecting where the
+        # key material lands) or as a dir they own. Refuse both; our own
+        # stale dir from a prior boot passes (chmod below repairs mode).
+        raise RuntimeError(
+            f"PEM runtime dir {root} exists but is a symlink or not owned by this process"
+        )
+    root.mkdir(mode=0o700, parents=True, exist_ok=True)
+    os.chmod(root, 0o700)
+    for stale in root.glob("lacme-pem-*"):
+        shutil.rmtree(stale, ignore_errors=True)
+    return root
+
+
+def refresh_runtime_pems(bundle: Any, *, ca_pem: bytes | None, previous: Path | None) -> Any:
+    """Write a renewed bundle under the runtime root and drop the old dir.
+
+    Keeps the on-disk PEMs (the healthcheck's mTLS client identity) in
+    lockstep with the served cert: certs live 48 hours, so the boot-time
+    files would expire and flip the container unhealthy two renewals in.
+    The new dir is written before the old one is removed, so a concurrent
+    probe always finds at least one complete dir.
+    """
+    import shutil
+
+    from lacme.mtls import write_pem_files
+
+    new_paths = write_pem_files(bundle, ca_pem=ca_pem, directory=tls_pem_runtime_dir())
+    if previous is not None and previous != new_paths.cert.parent:
+        shutil.rmtree(previous, ignore_errors=True)
+    return new_paths
+
 
 def _require_lacme() -> Any:
     try:
@@ -203,17 +279,42 @@ class TLSClient:
             except Exception:
                 log.warning("tls.cert.reload_hook_failed", exc_info=True)
 
-    async def init(self) -> None:
+    async def init(self, *, attempts: int = 1, base_delay: float = 1.0) -> None:
         """Fetch CA root cert and request a service certificate.
 
         If no console_url was provided, discovers it from the services
         table. Performs initial cert provisioning over plain HTTP (ACME
         protocol provides integrity via JWS).
+
+        With ``attempts > 1``, failures are retried with exponential backoff
+        (``base_delay * 2**n``). A node restarted alongside the console loses
+        the race for the console's listener by well under a second; without
+        retries that one refused connection downgrades the node to plain HTTP
+        for its entire lifetime, even when a valid cert sits in the store.
+        Discovery, CA fetch, and cert request are all idempotent, so the whole
+        sequence is retried as a unit.
         """
+        import asyncio
+
+        for attempt in range(1, attempts + 1):
+            try:
                 if not self._console_url:
                     self._console_url = self._discover_console_url()
                 await self._fetch_ca_cert()
                 await self._request_cert()
+                return
+            except Exception as exc:
+                if attempt >= attempts:
+                    raise
+                delay = base_delay * 2 ** (attempt - 1)
+                log.warning(
+                    "tls.init.retrying",
+                    attempt=attempt,
+                    max_attempts=attempts,
+                    delay_seconds=delay,
+                    error=f"{type(exc).__name__}: {exc}",
+                )
+                await asyncio.sleep(delay)
 
     def _discover_console_url(self) -> str:
         """Look up the console URL from the services table."""
@@ -245,8 +346,11 @@ class TLSClient:
                 resp.raise_for_status()
                 self._ca_pem = resp.content
                 log.info("tls.ca.fetched", url=url)
-        except Exception:
-            log.error("tls.ca.fetch_failed", url=url, exc_info=True)
+        except Exception as exc:
+            # Warning, not error: init() may retry this, and the terminal
+            # failure is logged by the caller. Full traceback at debug.
+            log.warning("tls.ca.fetch_failed", url=url, error=f"{type(exc).__name__}: {exc}")
+            log.debug("tls.ca.fetch_failed traceback", exc_info=True)
             raise
 
     async def _request_cert(self) -> None:

turnstone/server.py

@@ -1395,6 +1395,12 @@ def _build_health_dict(app_state: Any) -> dict[str, Any]:
             "resources": mc.resource_count,
             "prompts": mc.prompt_count,
         }
+    # Only present when tls.enabled: "active" (serving HTTPS) or "fallback"
+    # (TLS init failed, serving plain HTTP). Makes a silently-downgraded
+    # node observable.
+    tls_state = getattr(app_state, "tls_state", None)
+    if tls_state:
+        data["tls"] = tls_state
     return data
 
 
@@ -4635,7 +4641,12 @@ def main() -> None:
         try:
             import asyncio
 
-            from turnstone.core.tls import TLSClient, build_cert_hostnames
+            from turnstone.core.tls import (
+                TLS_INIT_RETRY_ATTEMPTS,
+                TLSClient,
+                build_cert_hostnames,
+                prepare_pem_runtime_dir,
+            )
 
             # The advertised host (the name the collector + routing proxy dial)
             # is placed first so it becomes the cert's primary domain / SAN and
@@ -4651,14 +4662,17 @@ def main() -> None:
                 storage=get_storage(),
                 hostnames=hostnames,
             )
-            asyncio.run(tls_client.init())
+            asyncio.run(tls_client.init(attempts=TLS_INIT_RETRY_ATTEMPTS))
             bundle = tls_client.bundle
             if bundle:
                 from lacme.mtls import write_pem_files_persistent
 
+                # Fixed parent dir (vs. a random tmpdir) so the container
+                # healthcheck can find the cert and probe over mTLS.
                 pem_paths = write_pem_files_persistent(
                     bundle,
                     ca_pem=tls_client.ca_pem,
+                    directory=prepare_pem_runtime_dir(),
                 )
                 ssl_kwargs.update(pem_paths.as_uvicorn_kwargs())
                 if tls_client.ca_pem:
@@ -4668,15 +4682,28 @@ def main() -> None:
 
                 # Store client on app state for lifespan renewal
                 app.state.tls_client = tls_client
+                pem_dir_state = {"dir": pem_paths.cert.parent}
 
                 def _reload_server_cert(new_bundle: Any) -> None:
                     """Swap a renewed cert into uvicorn's live SSL context.
 
                     uvicorn loads its cert once at boot and never reloads, so
                     without this the served cert would expire mid-process and
-                    break every mTLS peer.
+                    break every mTLS peer. The on-disk runtime PEMs (the
+                    healthcheck's client identity) expire on the same clock,
+                    so they are refreshed alongside.
                     """
-                    from turnstone.core.tls import swap_context_cert
+                    from turnstone.core.tls import refresh_runtime_pems, swap_context_cert
+
+                    try:
+                        new_paths = refresh_runtime_pems(
+                            new_bundle,
+                            ca_pem=tls_client.ca_pem,
+                            previous=pem_dir_state["dir"],
+                        )
+                        pem_dir_state["dir"] = new_paths.cert.parent
+                    except Exception:
+                        log.warning("TLS runtime PEM refresh failed", exc_info=True)
 
                     cfg = getattr(app.state, "uvicorn_config", None)
                     live_ctx = getattr(cfg, "ssl", None) if cfg is not None else None
@@ -4691,10 +4718,15 @@ def main() -> None:
                     app.state.advertise_url = _advertise_url.replace("http://", "https://", 1)
                 else:
                     app.state.advertise_url = _advertise_url
+                app.state.tls_state = "active"
                 log.info("TLS enabled — serving HTTPS")
             else:
+                app.state.tls_state = "fallback"
                 log.warning("TLS enabled but no cert available")
         except Exception as exc:
+            # Surfaced as tls:"fallback" in /health — a node serving plain
+            # HTTP while TLS is configured should be visible, not silent.
+            app.state.tls_state = "fallback"
             log.warning(
                 "TLS initialization failed — serving plain HTTP: %s: %s",
                 type(exc).__name__,