diff --git a/Dockerfile b/Dockerfile index 4c8697b7..2ad54714 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,12 +25,12 @@ ENV UV_COMPILE_BYTECODE=1 # Install dependencies first (cached layer — only re-runs when deps change) COPY pyproject.toml uv.lock README.md LICENSE ./ RUN uv sync --frozen --no-install-project --no-dev \ - --extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic --extra ddg + --extra all # Install the project itself COPY turnstone/ turnstone/ RUN uv sync --frozen --no-dev \ - --extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic --extra ddg + --extra all # Add venv to PATH so entry points are found ENV PATH="/app/.venv/bin:$PATH" diff --git a/deploy/docker-compose.tls.yml b/deploy/docker-compose.tls.yml index 4162a060..f9eeb536 100644 --- a/deploy/docker-compose.tls.yml +++ b/deploy/docker-compose.tls.yml @@ -1,23 +1,33 @@ # TLS overlay — enables mTLS across the turnstone cluster. # -# Usage (overlay — requires a base compose.yaml defining -# console, server, bridge, and redis services): -# docker compose -f compose.yaml -f deploy/docker-compose.tls.yml up +# Usage (requires base compose.yaml with production profile): +# docker compose -f compose.yaml -f deploy/docker-compose.tls.yml --profile production up # # The tls-init service bootstraps a CA and issues a cert for Redis. # All turnstone services auto-provision their own certs via the # console's ACME endpoint. services: - # Bootstrap: create CA + Redis cert before anything starts + # Bootstrap: create CA + Redis cert before anything starts. + # Runs as root to create directories in the volume, then chowns + # to turnstone:turnstone with restrictive perms (keys 0600). tls-init: build: . - command: > - turnstone-admin tls-bootstrap - --out /certs - --issue redis + user: root + command: + - sh + - -c + - | + set -e + turnstone-admin tls-bootstrap --out /certs --issue redis + chown -R turnstone:turnstone /certs + find /certs -type d -exec chmod 750 {} + + find /certs -type f -name '*key.pem' -exec chmod 600 {} + + find /certs -type f ! -name '*key.pem' -exec chmod 640 {} + volumes: - tls-certs:/certs + networks: + - turnstone-net restart: "no" # Console: runs the internal CA + ACME server @@ -29,26 +39,82 @@ services: - tls-certs:/certs:ro environment: TURNSTONE_TLS_ENABLED: "true" + TURNSTONE_TLS_SANS: "console" + TURNSTONE_CONSOLE_URL: "http://console:8090" + command: + - turnstone-console + - --host=0.0.0.0 + - --port=8090 + - --redis-host=redis + - --redis-port=6379 + - --poll-interval=${CONSOLE_POLL_INTERVAL:-10} + - --redis-tls + - --redis-tls-ca=/certs/ca.pem - # Server nodes: auto-provision certs via console ACME + # Server: auto-provisions certs via console ACME, serves HTTPS server: depends_on: console: condition: service_healthy volumes: - tls-certs:/certs:ro + environment: + TURNSTONE_TLS_ENABLED: "true" + TURNSTONE_TLS_SANS: "server" + # Disable healthcheck — server serves HTTPS with mTLS which the + # stdlib healthcheck script can't satisfy. The base compose + # healthcheck uses plain HTTP which won't work on an HTTPS listener. + # TODO: wire healthcheck with client cert from /certs volume + healthcheck: + disable: true - # Bridge: mTLS to server, Redis TLS + # Bridge: mTLS to server + Redis TLS bridge: depends_on: console: condition: service_healthy + server: + condition: service_started + redis: + condition: service_healthy volumes: - tls-certs:/certs:ro - command: > - turnstone-bridge + environment: + TURNSTONE_TLS_ENABLED: "true" + TURNSTONE_TLS_SANS: "bridge" + command: + - turnstone-bridge + - --server-url=http://server:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + - --redis-tls + - --redis-tls-ca=/certs/ca.pem + + # Channel: Redis TLS + channel: + depends_on: + console: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - tls-certs:/certs:ro + environment: + TURNSTONE_TLS_ENABLED: "true" + TURNSTONE_TLS_SANS: "channel" + command: + - sh + - -c + - >- + turnstone-channel + --redis-host=redis + --redis-port=6379 --redis-tls - --redis-tls-ca /certs/ca.pem + --redis-tls-ca=/certs/ca.pem + --http-host=0.0.0.0 + $${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD} # Redis: TLS with certs from bootstrap redis: @@ -57,19 +123,21 @@ services: condition: service_completed_successfully volumes: - tls-certs:/certs:ro - command: > - redis-server - --tls-port 6379 - --port 0 - --tls-cert-file /certs/certs/redis/cert.pem - --tls-key-file /certs/certs/redis/key.pem - --tls-ca-cert-file /certs/ca.pem + command: + - sh + - -c + - | + ARGS="--tls-port 6379 --port 0 \ + --tls-cert-file /certs/certs/redis/cert.pem \ + --tls-key-file /certs/certs/redis/key.pem \ + --tls-ca-cert-file /certs/ca.pem \ + --tls-auth-clients no" + if [ -n "$$REDIS_PASSWORD" ]; then + ARGS="$$ARGS --requirepass $$REDIS_PASSWORD" + fi + exec redis-server $$ARGS healthcheck: - test: ["CMD", "redis-cli", "--tls", - "--cacert", "/certs/ca.pem", - "--cert", "/certs/certs/redis/cert.pem", - "--key", "/certs/certs/redis/key.pem", - "ping"] + test: ["CMD-SHELL", "if [ -n \"$$REDIS_PASSWORD\" ]; then redis-cli --tls --cacert /certs/ca.pem -a $$REDIS_PASSWORD ping; else redis-cli --tls --cacert /certs/ca.pem ping; fi"] interval: 5s timeout: 3s retries: 5 diff --git a/pyproject.toml b/pyproject.toml index d80aaed9..c3be0d87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ anthropic = ["anthropic>=0.39"] postgres = ["psycopg[binary]>=3.2"] ddg = ["ddgs>=9.0"] discord = ["discord.py>=2.4", "redis>=7.2"] -tls = ["lacme>=1.0.2"] +tls = ["lacme>=1.0.4"] all = ["turnstone[mq,console,sim,anthropic,postgres,discord,ddg,tls]"] [project.scripts] diff --git a/turnstone/admin.py b/turnstone/admin.py index f623eb9f..09493e0f 100644 --- a/turnstone/admin.py +++ b/turnstone/admin.py @@ -155,22 +155,25 @@ def _cmd_tls_bootstrap(args: argparse.Namespace) -> None: print("lacme not installed. Run: pip install turnstone[tls]", file=sys.stderr) sys.exit(1) + import contextlib import os from pathlib import Path out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) - os.chmod(out_dir, 0o700) # Restrict access — contains CA private key + with contextlib.suppress(PermissionError): + os.chmod(out_dir, 0o700) # Restrict access — contains CA private key store = FileStore(str(out_dir)) - ca = CertificateAuthority(store) + ca = CertificateAuthority(store, name="turnstone") ca.init(cn="Turnstone CA", validity_days=3650) print(f"CA initialized in {out_dir} (permissions: 0700)") # Write CA cert to a well-known location ca_cert_path = out_dir / "ca.pem" ca_cert_path.write_bytes(ca.root_cert_pem) - os.chmod(ca_cert_path, 0o644) + with contextlib.suppress(PermissionError): + os.chmod(ca_cert_path, 0o644) print(f"CA cert: {ca_cert_path}") # Issue certs for requested domains diff --git a/turnstone/console/collector.py b/turnstone/console/collector.py index 32125e4c..50037dd0 100644 --- a/turnstone/console/collector.py +++ b/turnstone/console/collector.py @@ -96,6 +96,25 @@ class ClusterCollector: self._listeners: list[queue.Queue[dict[str, Any]]] = [] self._listeners_lock = threading.Lock() + def upgrade_tls(self, tls_verify: Any = True, tls_cert: tuple[str, str] | None = None) -> None: + """Replace the httpx client with one using mTLS context.""" + old = self._http_client + self._http_client = httpx.Client( + timeout=httpx.Timeout( + connect=10, read=self._http_timeout, write=5, pool=self._http_timeout + ), + limits=httpx.Limits( + max_connections=self._max_poll_workers + 10, + max_keepalive_connections=min(self._max_poll_workers, 200), + ), + verify=tls_verify, + cert=tls_cert, + ) + # Don't close old client — concurrent _fetch_node() threads may still + # be using it. It will be GC'd once all references are released, and + # the current client is closed in stop(). + del old + # -- lifecycle ----------------------------------------------------------- def start(self) -> None: diff --git a/turnstone/console/server.py b/turnstone/console/server.py index a1019fdd..7ebed1d5 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -807,7 +807,11 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: if not tls_mgr.ca_initialized: await tls_mgr.init_ca() hostname = socket.getfqdn() - await tls_mgr.issue_console_certs([hostname, "localhost", "127.0.0.1"]) + cert_hostnames = [hostname, "localhost", "127.0.0.1"] + extra_sans = os.environ.get("TURNSTONE_TLS_SANS", "") + if extra_sans: + cert_hostnames.extend(s.strip() for s in extra_sans.split(",") if s.strip()) + await tls_mgr.issue_console_certs(cert_hostnames) await tls_mgr.start_renewal() # Re-create proxy clients with mTLS context now that certs are ready client_ctx = tls_mgr.get_client_ssl_context() @@ -831,6 +835,8 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: ), verify=client_ctx, ) + # Upgrade collector httpx client for mTLS node polling + app.state.collector.upgrade_tls(tls_verify=client_ctx) log.info("tls.proxy_clients.upgraded") except Exception: log.warning("TLS initialization failed — continuing without TLS", exc_info=True) @@ -5281,7 +5287,19 @@ def main() -> None: # TLS: initialize manager if enabled tls_mgr = None - console_url = f"http://{args.host}:{args.port}" + # Console URL for service registration — other services use this to discover the console. + # Precedence: TURNSTONE_CONSOLE_URL env > auto-detect from bind address. + # In Docker Compose, set TURNSTONE_CONSOLE_URL to the service name (e.g. http://console:8090). + import socket as _socket + + _console_url_env = os.environ.get("TURNSTONE_CONSOLE_URL", "") + if _console_url_env: + console_url = _console_url_env + else: + _advertise_host = args.host + if _advertise_host in ("0.0.0.0", "::", ""): + _advertise_host = _socket.getfqdn() + console_url = f"http://{_advertise_host}:{args.port}" if auth_storage: try: from turnstone.core.config_store import ConfigStore @@ -5297,7 +5315,9 @@ def main() -> None: import asyncio asyncio.run(tls_mgr.init_ca()) - console_url = f"https://{args.host}:{args.port}" + # Upgrade scheme to https if no explicit URL was provided + if not _console_url_env: + console_url = console_url.replace("http://", "https://") log.info("TLS enabled") except ImportError: log.warning("TLS enabled but lacme not installed — pip install turnstone[tls]") diff --git a/turnstone/console/tls.py b/turnstone/console/tls.py index 7c8aec8f..fbb97855 100644 --- a/turnstone/console/tls.py +++ b/turnstone/console/tls.py @@ -23,6 +23,7 @@ log = structlog.get_logger(__name__) # Hardcoded defaults — no operator config needed _CA_CN = "Turnstone CA" +_CA_NAME = "turnstone" # Store key for save_ca/load_ca _CA_VALIDITY_DAYS = 3650 # 10 years _CERT_VALIDITY_HOURS = 48 _RENEW_INTERVAL_HOURS = 24 @@ -117,16 +118,51 @@ class TLSManager: async def init_ca(self) -> None: """Initialize the internal Certificate Authority. - Loads an existing CA from storage or generates a new root key+cert. + If a bootstrap CA exists on disk (from tls-bootstrap), imports it + into the database store first so the console uses the same CA that + signed the infrastructure certs. """ lacme = _require_lacme() + + # Import bootstrap CA from well-known volume path if not already in DB + self._import_bootstrap_ca() + self._ca = lacme.CertificateAuthority( self._store, + name=_CA_NAME, event_dispatcher=self._event_dispatcher, ) self._ca.init(cn=_CA_CN, validity_days=_CA_VALIDITY_DAYS) log.info("tls.ca.initialized", cn=_CA_CN) + def _import_bootstrap_ca(self) -> None: + """Import a bootstrap CA from /certs into the database store. + + The tls-bootstrap CLI writes the CA to a FileStore at /certs. + On first boot, the console imports it so all services share + the same trust root. + """ + import os + from pathlib import Path + + # Check if bootstrap CA exists and DB CA doesn't + bootstrap_dir = Path(os.environ.get("TURNSTONE_TLS_BOOTSTRAP_DIR", "/certs")) + ca_dir = bootstrap_dir / "ca" / _CA_NAME + ca_cert_file = ca_dir / "cert.pem" + ca_key_file = ca_dir / "key.pem" + + if not ca_cert_file.exists() or not ca_key_file.exists(): + return # No bootstrap CA found + + existing = self._store.load_ca(_CA_NAME) + if existing is not None: + return # Already imported + + cert_pem = ca_cert_file.read_bytes() + key_pem = ca_key_file.read_bytes() + self._store.save_ca(_CA_NAME, cert_pem, key_pem) + log.info("tls.ca.imported_from_bootstrap", path=str(ca_dir)) + def get_responder(self) -> ASGIApp: """Return the ACME responder ASGI app for mounting.""" if self._ca is None: diff --git a/turnstone/core/tls.py b/turnstone/core/tls.py index 5c5f1de7..5f5c22be 100644 --- a/turnstone/core/tls.py +++ b/turnstone/core/tls.py @@ -143,12 +143,15 @@ class TLSClient: # Request new cert via ACME (plain HTTP for initial request) lacme = _require_lacme() + from lacme.challenges.http01 import HTTP01Handler + directory_url = f"{self._console_url}/acme/directory" async with lacme.Client( directory_url=directory_url, store=self._store, event_dispatcher=self._event_dispatcher, + challenge_handler=HTTP01Handler(), allow_insecure=True, ) as client: self._bundle = await client.issue(self._hostnames) @@ -165,11 +168,14 @@ class TLSClient: self._bundle = bundle log.info("tls.cert.renewed", domain=bundle.domain) + from lacme.challenges.http01 import HTTP01Handler + directory_url = f"{self._console_url}/acme/directory" client = lacme.Client( directory_url=directory_url, store=self._store, event_dispatcher=self._event_dispatcher, + challenge_handler=HTTP01Handler(), allow_insecure=True, ) await client.__aenter__() diff --git a/turnstone/mq/bridge.py b/turnstone/mq/bridge.py index 1ede442c..fab2bd46 100644 --- a/turnstone/mq/bridge.py +++ b/turnstone/mq/bridge.py @@ -1135,6 +1135,45 @@ def main() -> None: ) log.info("bridge.jwt_minted") + # TLS: request cert from console ACME if enabled + tls_verify: Any = True + tls_cert: tuple[str, str] | None = None + if os.environ.get("TURNSTONE_TLS_ENABLED", "").lower() in ("true", "1", "yes"): + try: + import asyncio + import socket + + from turnstone.core.storage import init_storage + from turnstone.core.tls import TLSClient + + db_backend = os.environ.get("TURNSTONE_DB_BACKEND", "sqlite") + db_url = os.environ.get("TURNSTONE_DB_URL", "") + db_path = os.environ.get("TURNSTONE_DB_PATH", "") + storage = init_storage(db_backend, path=db_path, url=db_url) + + hostname = socket.getfqdn() + hostnames = [hostname, "localhost", "127.0.0.1"] + extra_sans = os.environ.get("TURNSTONE_TLS_SANS", "") + if extra_sans: + hostnames.extend(s.strip() for s in extra_sans.split(",") if s.strip()) + tls_client = TLSClient( + storage=storage, + hostnames=hostnames, + ) + asyncio.run(tls_client.init()) + ssl_ctx = tls_client.get_client_ssl_context() + if ssl_ctx: + # SSLContext has both CA (verify server) and client cert + # (present to server) loaded — full mTLS in one object + tls_verify = ssl_ctx + if args.server_url.startswith("http://"): + args.server_url = args.server_url.replace("http://", "https://") + log.info("bridge.tls.enabled: %s", args.server_url) + except ImportError: + log.warning("TLS enabled but lacme not installed") + except Exception: + log.warning("bridge.tls.init_failed", exc_info=True) + bridge = Bridge( server_url=args.server_url, broker=broker, @@ -1143,6 +1182,8 @@ def main() -> None: heartbeat_ttl=args.heartbeat_ttl, auth_token=auth_token, token_manager=token_manager, + tls_verify=tls_verify, + tls_cert=tls_cert, ) bridge.run() diff --git a/turnstone/mq/broker.py b/turnstone/mq/broker.py index 6c06130b..604f1e67 100644 --- a/turnstone/mq/broker.py +++ b/turnstone/mq/broker.py @@ -131,15 +131,15 @@ class RedisBroker: self._prefix = prefix self._response_ttl = response_ttl - ssl_kwargs: dict[str, Any] = {} + pool_kwargs: dict[str, Any] = {} if ssl: - ssl_kwargs["ssl"] = True + pool_kwargs["connection_class"] = redis.SSLConnection if ssl_ca_certs: - ssl_kwargs["ssl_ca_certs"] = ssl_ca_certs + pool_kwargs["ssl_ca_certs"] = ssl_ca_certs if ssl_certfile: - ssl_kwargs["ssl_certfile"] = ssl_certfile + pool_kwargs["ssl_certfile"] = ssl_certfile if ssl_keyfile: - ssl_kwargs["ssl_keyfile"] = ssl_keyfile + pool_kwargs["ssl_keyfile"] = ssl_keyfile self._pool: _redis_t.ConnectionPool = redis.ConnectionPool( host=host, port=port, @@ -148,7 +148,7 @@ class RedisBroker: decode_responses=True, retry_on_timeout=True, max_connections=200, - **ssl_kwargs, + **pool_kwargs, ) self._redis: _redis_t.Redis[str] = cast( "_redis_t.Redis[str]", @@ -324,7 +324,7 @@ def broker_from_args(args: Any) -> RedisBroker: host=args.redis_host, port=args.redis_port, db=args.redis_db, - password=args.redis_password, + password=args.redis_password or None, **_redis_tls_kwargs(args), ) @@ -337,6 +337,6 @@ def async_broker_from_args(args: Any) -> Any: host=args.redis_host, port=args.redis_port, db=args.redis_db, - password=args.redis_password, + password=args.redis_password or None, **_redis_tls_kwargs(args), ) diff --git a/turnstone/server.py b/turnstone/server.py index 501c8090..80623d0f 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -2085,6 +2085,8 @@ def main() -> None: configure_logging_from_args(args, "server") + import socket + # Initialize storage backend from turnstone.core.storage import init_storage @@ -2454,7 +2456,6 @@ def main() -> None: if config_store.get("tls.enabled"): try: import asyncio - import socket from turnstone.core.tls import TLSClient @@ -2463,6 +2464,10 @@ def main() -> None: # Only add bind host if it's a concrete address if args.host not in ("0.0.0.0", "::", ""): hostnames.append(args.host) + # Additional SANs from env (e.g. Docker service name) + extra_sans = os.environ.get("TURNSTONE_TLS_SANS", "") + if extra_sans: + hostnames.extend(s.strip() for s in extra_sans.split(",") if s.strip()) tls_client = TLSClient( storage=get_storage(), hostnames=hostnames, diff --git a/uv.lock b/uv.lock index 448b9811..e80e86e4 100644 --- a/uv.lock +++ b/uv.lock @@ -984,15 +984,15 @@ wheels = [ [[package]] name = "lacme" -version = "1.0.2" +version = "1.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "httpx" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/b7/aadbf032c95e61d83242b8e27a25853fdc19685747f4bd78968962271c8a/lacme-1.0.2.tar.gz", hash = "sha256:3059a39cee454f612869bf9e419d4e6f279a35e8ce9e6c130cc1b7721704ea4a", size = 199743, upload-time = "2026-03-26T00:42:02.581Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/27/1f1b78b53b4190a15234deffef8459ce9af9c32251fe669b3c884373d954/lacme-1.0.4.tar.gz", hash = "sha256:c147cac91bcc243b0799264a0da31de44922f494c58d2d5fa9f62712455eda69", size = 200855, upload-time = "2026-03-26T20:31:20.984Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/a3/8e9495ee920e4c4a945ef2f0e4266dd4968a1511e630184990499f12025d/lacme-1.0.2-py3-none-any.whl", hash = "sha256:5bf7b859deddc8aab47619c8e165b85f4ff730bf0c0b9463789b5b01c51d9177", size = 71836, upload-time = "2026-03-26T00:42:00.797Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6d/a43c37dd2914560f9954f46598d07f976d3861722052deabe17a3f60ddb0/lacme-1.0.4-py3-none-any.whl", hash = "sha256:a21ed4a634c2c23a3afc0aaf327421fad709be9ae284fd6019a109b011fa52f7", size = 72122, upload-time = "2026-03-26T20:31:19.758Z" }, ] [[package]] @@ -2412,7 +2412,7 @@ requires-dist = [ { name = "discord-py", marker = "extra == 'discord'", specifier = ">=2.4" }, { name = "httpx", specifier = ">=0.28" }, { name = "httpx-sse", specifier = ">=0.4" }, - { name = "lacme", marker = "extra == 'tls'", specifier = ">=1.0.2" }, + { name = "lacme", marker = "extra == 'tls'", specifier = ">=1.0.4" }, { name = "mcp", specifier = ">=1.6" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14" }, { name = "openai", specifier = ">=2.24" },