mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
a3ff07a86d
A whole-stack restart races every node against the console for the CA fetch (compose re-enforces depends_on ordering only on `up`): losers logged one warning and served plain HTTP for their lifetime, while winners served mTLS that the plain-HTTP container healthcheck could never probe — leaving "healthy" plaintext nodes and "unhealthy" working ones. - TLSClient.init() grows attempts/base_delay retry (server passes 6 attempts, ~31 s backoff) absorbing the boot race; per-attempt CA-fetch failures log warning + debug traceback instead of error tracebacks. - healthcheck.py falls back to HTTPS when the plain probe fails, presenting the node's own cert as the client cert with the cluster CA pinned; dials localhost because the internal CA issues DNS SANs only. Default plain-HTTP deployments are unchanged. - The server writes boot PEMs under a fixed root (TURNSTONE_TLS_PEM_DIR, default <tmpdir>/turnstone-tls) so the probe can find them; boot clears stale dirs and refuses a symlinked/foreign-owned root; renewal rewrites the PEM dir so the probe's client cert never outlives the served cert. - /health reports tls: "active"|"fallback" (absent when TLS is disabled) so a silently downgraded node is observable.
99 lines
3.4 KiB
Python
99 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Health check for turnstone containers.
|
|
|
|
Usage: healthcheck.py <url>
|
|
Exit 0 if the endpoint returns {"status": "ok"} or {"status": "degraded"},
|
|
exit 1 otherwise. Uses only stdlib — no pip dependencies required.
|
|
|
|
When the node serves mTLS (tls.enabled), a plain-HTTP probe is rejected at
|
|
the socket, so on failure this script retries over HTTPS, presenting the
|
|
node's own certificate as the client cert and pinning the cluster CA. The
|
|
PEM files are the ones the server writes at boot under
|
|
$TURNSTONE_TLS_PEM_DIR (default: <tmpdir>/turnstone-tls). The host is
|
|
rewritten to "localhost" for the TLS attempt because the internal CA issues
|
|
DNS SANs only — certificate verification rejects a literal-IP dial.
|
|
|
|
When mTLS is disabled (the default), the plain probe succeeds and nothing
|
|
here changes: the PEM directory is never consulted.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import ssl
|
|
import sys
|
|
import tempfile
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
|
|
def _check(url: str, context: ssl.SSLContext | None = None) -> None:
|
|
"""Probe one URL; raise if unreachable or the payload is unhealthy."""
|
|
req = urllib.request.Request(url, method="GET")
|
|
with urllib.request.urlopen(req, timeout=5, context=context) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
if data.get("status") not in ("ok", "degraded"):
|
|
raise RuntimeError(f"unhealthy payload: {data}")
|
|
|
|
|
|
def _pem_root() -> Path:
|
|
"""PEM runtime root.
|
|
|
|
Must mirror turnstone.core.tls.tls_pem_runtime_dir — this script is
|
|
standalone stdlib and cannot import turnstone; a drift-guard test in
|
|
tests/test_docker_healthcheck.py pins the two together.
|
|
"""
|
|
root_env = os.environ.get("TURNSTONE_TLS_PEM_DIR")
|
|
return Path(root_env) if root_env else Path(tempfile.gettempdir()) / "turnstone-tls"
|
|
|
|
|
|
def _find_pem_dir() -> Path | None:
|
|
"""Locate the newest complete PEM dir written by the server at boot."""
|
|
root = _pem_root()
|
|
candidates = [
|
|
d
|
|
for d in root.glob("lacme-pem-*")
|
|
if all((d / name).is_file() for name in ("fullchain.pem", "key.pem", "ca.pem"))
|
|
]
|
|
if not candidates:
|
|
return None
|
|
return max(candidates, key=lambda d: d.stat().st_mtime)
|
|
|
|
|
|
def _tls_url(url: str) -> str:
|
|
"""Rewrite scheme to https and host to localhost, keeping port and path."""
|
|
parts = urlsplit(url)
|
|
netloc = f"localhost:{parts.port}" if parts.port else "localhost"
|
|
return urlunsplit(("https", netloc, parts.path, parts.query, parts.fragment))
|
|
|
|
|
|
def main() -> None:
|
|
if len(sys.argv) != 2:
|
|
print("Usage: healthcheck.py <url>", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
url = sys.argv[1]
|
|
try:
|
|
_check(url)
|
|
sys.exit(0)
|
|
except Exception as plain_exc:
|
|
pem_dir = _find_pem_dir()
|
|
if pem_dir is None:
|
|
print(f"Health check failed: {plain_exc}", file=sys.stderr)
|
|
sys.exit(1)
|
|
try:
|
|
context = ssl.create_default_context(cafile=str(pem_dir / "ca.pem"))
|
|
context.load_cert_chain(str(pem_dir / "fullchain.pem"), str(pem_dir / "key.pem"))
|
|
_check(_tls_url(url), context=context)
|
|
sys.exit(0)
|
|
except Exception as tls_exc:
|
|
print(
|
|
f"Health check failed: plain: {plain_exc}; mtls: {tls_exc}",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|