server: Support fsnotify reloading of certs (#6415)

Reload certs, keys and optionally the CA cert pool when they change on
disk.

The polling behaviour and flag is also still supported.

cmd/run.go

@@ -282,6 +282,7 @@ func initRuntime(ctx context.Context, params runCmdParams, args []string, addrSe
 	params.rt.CertificateFile = params.tlsCertFile
 	params.rt.CertificateKeyFile = params.tlsPrivateKeyFile
 	params.rt.CertificateRefresh = params.tlsCertRefresh
+	params.rt.CertPoolFile = params.tlsCACertFile
 
 	if params.tlsCACertFile != "" {
 		pool, err := loadCertPool(params.tlsCACertFile)

runtime/runtime.go

@@ -23,9 +23,6 @@ import (
 
 	"github.com/fsnotify/fsnotify"
 	"github.com/gorilla/mux"
-	"github.com/open-policy-agent/opa/internal/compiler"
-	"github.com/open-policy-agent/opa/internal/pathwatcher"
-	"github.com/open-policy-agent/opa/internal/ref"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/propagation"
@@ -33,10 +30,13 @@ import (
 
 	"github.com/open-policy-agent/opa/bundle"
 	opa_config "github.com/open-policy-agent/opa/config"
+	"github.com/open-policy-agent/opa/internal/compiler"
 	"github.com/open-policy-agent/opa/internal/config"
 	internal_tracing "github.com/open-policy-agent/opa/internal/distributedtracing"
 	internal_logging "github.com/open-policy-agent/opa/internal/logging"
+	"github.com/open-policy-agent/opa/internal/pathwatcher"
 	"github.com/open-policy-agent/opa/internal/prometheus"
+	"github.com/open-policy-agent/opa/internal/ref"
 	"github.com/open-policy-agent/opa/internal/report"
 	"github.com/open-policy-agent/opa/internal/runtime"
 	initload "github.com/open-policy-agent/opa/internal/runtime/init"
@@ -115,6 +115,8 @@ type Params struct {
 
 	// CertPool holds the CA certs trusted by the OPA server.
 	CertPool *x509.CertPool
+	// CertPoolFile, if set permits the reloading of the CA cert pool from disk
+	CertPoolFile string
 
 	// MinVersion contains the minimum TLS version that is acceptable.
 	// If zero, TLS 1.2 is currently taken as the minimum.
@@ -537,8 +539,8 @@ func (rt *Runtime) Serve(ctx context.Context) error {
 		WithPprofEnabled(rt.Params.PprofEnabled).
 		WithAddresses(*rt.Params.Addrs).
 		WithH2CEnabled(rt.Params.H2CEnabled).
+		// always use the initial values for the certificate and ca pool, reloading behavior is configured below
 		WithCertificate(rt.Params.Certificate).
-		WithCertificatePaths(rt.Params.CertificateFile, rt.Params.CertificateKeyFile, rt.Params.CertificateRefresh).
 		WithCertPool(rt.Params.CertPool).
 		WithAuthentication(rt.Params.Authentication).
 		WithAuthorization(rt.Params.Authorization).
@@ -562,6 +564,22 @@ func (rt *Runtime) Serve(ctx context.Context) error {
 		rt.server = rt.server.WithUnixSocketPermission(rt.Params.UnixSocketPerm)
 	}
 
+	// If a refresh period is set, then we will periodically reload the certificate and ca pool. Otherwise, we will only
+	// reload cert, key and ca pool files when they change on disk.
+	if rt.Params.CertificateRefresh > 0 {
+		rt.server = rt.server.WithCertRefresh(rt.Params.CertificateRefresh)
+	}
+
+	// if either the cert or the ca pool file is set then these fields will be set on the server and reloaded when they
+	// change on disk.
+	if rt.Params.CertificateFile != "" || rt.Params.CertPoolFile != "" {
+		rt.server = rt.server.WithTLSConfig(&server.TLSConfig{
+			CertFile:     rt.Params.CertificateFile,
+			KeyFile:      rt.Params.CertificateKeyFile,
+			CertPoolFile: rt.Params.CertPoolFile,
+		})
+	}
+
 	rt.server, err = rt.server.Init(ctx)
 	if err != nil {
 		rt.logger.WithFields(map[string]interface{}{"err": err}).Error("Unable to initialize server.")

server/certs.go

@@ -8,52 +8,175 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"crypto/tls"
+	"crypto/x509"
+	"fmt"
 	"io"
 	"os"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
+
+	"github.com/open-policy-agent/opa/internal/errors"
+	"github.com/open-policy-agent/opa/internal/pathwatcher"
 	"github.com/open-policy-agent/opa/logging"
 )
 
 func (s *Server) getCertificate(h *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	s.certMtx.RLock()
-	defer s.certMtx.RUnlock()
+	s.tlsConfigMtx.RLock()
+	defer s.tlsConfigMtx.RUnlock()
 	return s.cert, nil
 }
 
-func (s *Server) certLoop(logger logging.Logger) Loop {
+// reloadTLSConfig reloads the TLS config if the cert, key files or cert pool contents have changed.
+func (s *Server) reloadTLSConfig(logger logging.Logger) error {
+	s.tlsConfigMtx.Lock()
+	defer s.tlsConfigMtx.Unlock()
+
+	// reloading of the certificate key pair and the CA pool are independent operations,
+	// though errors from either operation are aggregated.
+	var errs error
+
+	// if the server has a cert configured, then we need to check the cert and key for changes.
+	if s.certFile != "" {
+		newCert, certFileHash, certKeyFileHash, updated, err := reloadCertificateKeyPair(
+			s.certFile,
+			s.certKeyFile,
+			s.certFileHash,
+			s.certKeyFileHash,
+			logger,
+		)
+		if err != nil {
+			errs = errors.Join(errs, err)
+		} else if updated {
+			s.cert = newCert
+			s.certFileHash = certFileHash
+			s.certKeyFileHash = certKeyFileHash
+
+			logger.Debug("Refreshed server certificate.")
+		}
+	}
+
+	// if the server has a cert pool configured, also attempt to reload this
+	if s.certPoolFile != "" {
+		pool, certPoolFileHash, updated, err := reloadCertificatePool(s.certPoolFile, s.certPoolFileHash, logger)
+		if err != nil {
+			errs = errors.Join(errs, err)
+		} else if updated {
+			s.certPool = pool
+			s.certPoolFileHash = certPoolFileHash
+			logger.Debug("Refreshed server CA certificate pool.")
+		}
+	}
+
+	return errs
+}
+
+// reloadCertificatePool loads the CA cert pool from the given file and returns a new pool if the file has changed.
+func reloadCertificatePool(certPoolFile string, certPoolFileHash []byte, logger logging.Logger) (*x509.CertPool, []byte, bool, error) {
+	certPoolHash, err := hash(certPoolFile)
+	if err != nil {
+		return nil, nil, false, fmt.Errorf("failed to hash CA cert pool file: %w", err)
+	}
+
+	if bytes.Equal(certPoolFileHash, certPoolHash) {
+		return nil, nil, false, nil
+	}
+	caCertPEM, err := os.ReadFile(certPoolFile)
+	if err != nil {
+		return nil, nil, false, fmt.Errorf("failed to read CA cert pool file %q: %w", certPoolFile, err)
+	}
+
+	pool := x509.NewCertPool()
+	if ok := pool.AppendCertsFromPEM(caCertPEM); !ok {
+		return nil, nil, false, fmt.Errorf("failed to load CA cert pool file %q", certPoolFile)
+	}
+
+	return pool, certPoolHash, true, nil
+}
+
+// reloadCertificateKeyPair loads the certificate and key from the given files and returns a new certificate if either
+// file has changed.
+func reloadCertificateKeyPair(
+	certFile, certKeyFile string,
+	certFileHash, certKeyFileHash []byte,
+	logger logging.Logger,
+) (*tls.Certificate, []byte, []byte, bool, error) {
+	certHash, err := hash(certFile)
+	if err != nil {
+		return nil, nil, nil, false, fmt.Errorf("failed to hash server certificate file: %w", err)
+	}
+
+	certKeyHash, err := hash(certKeyFile)
+	if err != nil {
+		return nil, nil, nil, false, fmt.Errorf("failed to hash server key file: %w", err)
+	}
+
+	differentCert := !bytes.Equal(certFileHash, certHash)
+	differentKey := !bytes.Equal(certKeyFileHash, certKeyHash)
+
+	if differentCert && !differentKey {
+		logger.Warn("Server certificate file changed but server key file did not change.")
+	}
+	if !differentCert && differentKey {
+		logger.Warn("Server key file changed but server certificate file did not change.")
+	}
+
+	if !differentCert && !differentKey {
+		return nil, nil, nil, false, nil
+	}
+
+	newCert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
+	if err != nil {
+		return nil, nil, nil, false, fmt.Errorf("server certificate key pair was not updated, update failed: %w", err)
+	}
+
+	return &newCert, certHash, certKeyHash, true, nil
+}
+
+func (s *Server) certLoopPolling(logger logging.Logger) Loop {
 	return func() error {
 		for range time.NewTicker(s.certRefresh).C {
-			certHash, err := hash(s.certFile)
+			err := s.reloadTLSConfig(logger)
 			if err != nil {
-				logger.Info("Failed to refresh server certificate: %s.", err.Error())
-				continue
-			}
-			certKeyHash, err := hash(s.certKeyFile)
-			if err != nil {
-				logger.Info("Failed to refresh server certificate: %s.", err.Error())
-				continue
+				logger.Error(fmt.Sprintf("Failed to reload TLS config: %s", err))
 			}
+		}
 
-			s.certMtx.Lock()
+		return nil
+	}
+}
 
-			different := !bytes.Equal(s.certFileHash, certHash) ||
-				!bytes.Equal(s.certKeyFileHash, certKeyHash)
+func (s *Server) certLoopNotify(logger logging.Logger) Loop {
+	return func() error {
 
-			if different { // load and store
-				newCert, err := tls.LoadX509KeyPair(s.certFile, s.certKeyFile)
+		var paths []string
+
+		// if a cert file is set, then we want to watch the cert and key
+		if s.certFile != "" {
+			paths = append(paths, s.certFile, s.certKeyFile)
+		}
+
+		// if a cert pool file is set, then we want to watch the cert pool. This might be set without the cert and key
+		// being set too.
+		if s.certPoolFile != "" {
+			paths = append(paths, s.certPoolFile)
+		}
+
+		watcher, err := pathwatcher.CreatePathWatcher(paths)
+		if err != nil {
+			return fmt.Errorf("failed to create tls path watcher: %w", err)
+		}
+
+		for evt := range watcher.Events {
+			removalMask := fsnotify.Remove | fsnotify.Rename
+			mask := fsnotify.Create | fsnotify.Write | removalMask
+			if (evt.Op & mask) != 0 {
+				err = s.reloadTLSConfig(s.manager.Logger())
 				if err != nil {
-					logger.Info("Failed to refresh server certificate: %s.", err.Error())
-					s.certMtx.Unlock()
-					continue
+					logger.Error("failed to reload TLS config: %s", err)
 				}
-				s.cert = &newCert
-				s.certFileHash = certHash
-				s.certKeyFileHash = certKeyHash
-				logger.Debug("Refreshed server certificate.")
+				logger.Info("TLS config reloaded")
 			}
-
-			s.certMtx.Unlock()
 		}
 
 		return nil

server/server.go

@@ -118,13 +118,15 @@ type Server struct {
 	authentication         AuthenticationScheme
 	authorization          AuthorizationScheme
 	cert                   *tls.Certificate
-	certMtx                sync.RWMutex
+	tlsConfigMtx           sync.RWMutex
 	certFile               string
 	certFileHash           []byte
 	certKeyFile            string
 	certKeyFileHash        []byte
 	certRefresh            time.Duration
 	certPool               *x509.CertPool
+	certPoolFile           string
+	certPoolFileHash       []byte
 	minTLSVersion          uint16
 	mtx                    sync.RWMutex
 	partials               map[string]rego.PartialResult
@@ -153,6 +155,23 @@ type Metrics interface {
 	InstrumentHandler(handler http.Handler, label string) http.Handler
 }
 
+// TLSConfig represents the TLS configuration for the server.
+// This configuration is used to configure file watchers to reload each file as it
+// changes on disk.
+type TLSConfig struct {
+	// CertFile is the path to the server's serving certificate file.
+	CertFile string
+
+	// KeyFile is the path to the server's key file, completing the key pair for the
+	// CertFile certificate.
+	KeyFile string
+
+	// CertPoolFile is the path to the CA cert pool file. The contents of this file will be
+	// reloaded when the file changes on disk and used in as trusted client CAs in the TLS config
+	// for new connections to the server.
+	CertPoolFile string
+}
+
 // Loop will contain all the calls from the server that we'll be listening on.
 type Loop func() error
 
@@ -274,6 +293,20 @@ func (s *Server) WithCertPool(pool *x509.CertPool) *Server {
 	return s
 }
 
+// WithTLSConfig sets the TLS configuration used by the server.
+func (s *Server) WithTLSConfig(tlsConfig *TLSConfig) *Server {
+	s.certFile = tlsConfig.CertFile
+	s.certKeyFile = tlsConfig.KeyFile
+	s.certPoolFile = tlsConfig.CertPoolFile
+	return s
+}
+
+// WithCertRefresh sets the period on which certs, keys and cert pools are reloaded from disk.
+func (s *Server) WithCertRefresh(refresh time.Duration) *Server {
+	s.certRefresh = refresh
+	return s
+}
+
 // WithStore sets the storage used by the server.
 func (s *Server) WithStore(store storage.Store) *Server {
 	s.store = store
@@ -566,11 +599,13 @@ func (s *Server) getListener(addr string, h http.Handler, t httpListenerType) ([
 			"cert-file":     s.certFile,
 			"cert-key-file": s.certKeyFile,
 		})
+
+		// if a manual cert refresh period has been set, then use the polling behavior,
+		// otherwise use the fsnotify default behavior
 		if s.certRefresh > 0 {
-			certLoop := s.certLoop(logger)
-			loops = []Loop{loop, certLoop}
-		} else {
-			loops = []Loop{loop}
+			loops = []Loop{loop, s.certLoopPolling(logger)}
+		} else if s.certFile != "" || s.certPoolFile != "" {
+			loops = []Loop{loop, s.certLoopNotify(logger)}
 		}
 	default:
 		err = fmt.Errorf("invalid url scheme %q", parsedURL.Scheme)
@@ -605,17 +640,28 @@ func (s *Server) getListenerForHTTPSServer(u *url.URL, h http.Handler, t httpLis
 		Handler: h,
 		TLSConfig: &tls.Config{
 			GetCertificate: s.getCertificate,
-			ClientCAs:      s.certPool,
-		},
-	}
-	if s.authentication == AuthenticationTLS {
-		httpsServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert
-	}
+			// GetConfigForClient is used to ensure that a fresh config is provided containing the latest cert pool.
+			// This is not required, but appears to be how connect time updates config should be done:
+			// https://github.com/golang/go/issues/16066#issuecomment-250606132
+			GetConfigForClient: func(info *tls.ClientHelloInfo) (*tls.Config, error) {
+				cfg := &tls.Config{
+					GetCertificate: s.getCertificate,
+					ClientCAs:      s.certPool,
+				}
 
-	if s.minTLSVersion != 0 {
-		httpsServer.TLSConfig.MinVersion = s.minTLSVersion
-	} else {
-		httpsServer.TLSConfig.MinVersion = defaultMinTLSVersion
+				if s.authentication == AuthenticationTLS {
+					cfg.ClientAuth = tls.RequireAndVerifyClientCert
+				}
+
+				if s.minTLSVersion != 0 {
+					cfg.MinVersion = s.minTLSVersion
+				} else {
+					cfg.MinVersion = defaultMinTLSVersion
+				}
+
+				return cfg, nil
+			},
+		},
 	}
 
 	l := newHTTPListener(&httpsServer, t)

server/server_test.go

@@ -9,14 +9,26 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
 	"log"
+	"math"
+	"math/big"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
@@ -26,12 +38,11 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/open-policy-agent/opa/internal/prometheus"
-
 	"github.com/open-policy-agent/opa/ast"
 	"github.com/open-policy-agent/opa/bundle"
 	"github.com/open-policy-agent/opa/config"
 	"github.com/open-policy-agent/opa/internal/distributedtracing"
+	"github.com/open-policy-agent/opa/internal/prometheus"
 	"github.com/open-policy-agent/opa/logging"
 	"github.com/open-policy-agent/opa/metrics"
 	"github.com/open-policy-agent/opa/plugins"
@@ -4852,6 +4863,823 @@ func TestDistributedTracingEnabled(t *testing.T) {
 	}
 }
 
+func TestCertPoolReloading(t *testing.T) {
+
+	ctx := context.Background()
+
+	tempDir := t.TempDir()
+
+	serverCertPath := filepath.Join(tempDir, "serverCert.pem")
+	serverCertKeyPath := filepath.Join(tempDir, "serverCertKey.pem")
+	clientCertPath := filepath.Join(tempDir, "clientCert.pem")
+	clientCertKeyPath := filepath.Join(tempDir, "clientCertKey.pem")
+	caCertPath := filepath.Join(tempDir, "ca.pem")
+
+	san := net.ParseIP("127.0.0.1")
+
+	// create the CA cert used in the cert pool and for signing server certs
+	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	caSerial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+	if err != nil {
+		t.Fatal(err)
+	}
+	caSubj := pkix.Name{
+		CommonName:   "CA",
+		SerialNumber: caSerial.String(),
+	}
+	caTemplate := &x509.Certificate{
+		BasicConstraintsValid: true,
+		SignatureAlgorithm:    x509.ECDSAWithSHA256,
+		PublicKeyAlgorithm:    x509.ECDSA,
+		PublicKey:             caKey.Public(),
+		SerialNumber:          caSerial,
+		Issuer:                caSubj,
+		Subject:               caSubj,
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(100 * time.Hour * 24 * 365),
+		KeyUsage:              x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		IsCA:                  true,
+		DNSNames:              nil,
+		EmailAddresses:        nil,
+		IPAddresses:           nil,
+	}
+
+	caCertData, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caKey.Public(), caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	caCertPEMEncoded := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caCertData,
+	})
+
+	// we write an empty file for now
+	err = os.WriteFile(caCertPath, []byte{}, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create a cert and key for the server to load at startup
+	var serverCert tls.Certificate
+
+	serverCertKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	serverCert.PrivateKey = serverCertKey
+
+	serverCertSerial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serverCertTemplate := &x509.Certificate{
+		BasicConstraintsValid: true,
+		SignatureAlgorithm:    x509.ECDSAWithSHA256,
+		PublicKeyAlgorithm:    x509.ECDSA,
+		PublicKey:             serverCertKey.Public(),
+		SerialNumber:          serverCertSerial,
+		Issuer:                caSubj,
+		Subject: pkix.Name{
+			CommonName:   "Server 1",
+			SerialNumber: serverCertSerial.String(),
+		},
+		NotBefore:      time.Now(),
+		NotAfter:       time.Now().Add(99 * time.Hour * 24 * 365),
+		KeyUsage:       x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:    []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IsCA:           false,
+		DNSNames:       nil,
+		EmailAddresses: nil,
+		IPAddresses:    []net.IP{san},
+	}
+
+	serverCertData, err := x509.CreateCertificate(rand.Reader, serverCertTemplate, caTemplate, serverCertKey.Public(), caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	serverCertPEMEncoded := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: serverCertData,
+	})
+
+	serverCertKeyMarshalled, _ := x509.MarshalPKCS8PrivateKey(serverCert.PrivateKey)
+	serverCertKeyPEMEncoded := pem.EncodeToMemory(&pem.Block{
+		Type:  "PRIVATE KEY",
+		Bytes: serverCertKeyMarshalled,
+	})
+
+	err = os.WriteFile(serverCertPath, serverCertPEMEncoded, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.WriteFile(serverCertKeyPath, serverCertKeyPEMEncoded, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create a cert and key for the client to test client auth
+	var clientCert tls.Certificate
+
+	clientCertKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientCert.PrivateKey = clientCertKey
+
+	clientCertSerial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientCertTemplate := &x509.Certificate{
+		BasicConstraintsValid: true,
+		SignatureAlgorithm:    x509.ECDSAWithSHA256,
+		PublicKeyAlgorithm:    x509.ECDSA,
+		PublicKey:             clientCertKey.Public(),
+		SerialNumber:          clientCertSerial,
+		Issuer:                caSubj,
+		Subject: pkix.Name{
+			CommonName:   "Client",
+			SerialNumber: clientCertSerial.String(),
+		},
+		NotBefore:   time.Now(),
+		NotAfter:    time.Now().Add(99 * time.Hour * 24 * 365),
+		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+
+	clientCertData, err := x509.CreateCertificate(rand.Reader, clientCertTemplate, caTemplate, clientCertKey.Public(), caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	clientCertPEMEncoded := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: clientCertData,
+	})
+
+	clientCertKeyMarshalled, _ := x509.MarshalPKCS8PrivateKey(clientCert.PrivateKey)
+	clientCertKeyPEMEncoded := pem.EncodeToMemory(&pem.Block{
+		Type:  "PRIVATE KEY",
+		Bytes: clientCertKeyMarshalled,
+	})
+
+	err = os.WriteFile(clientCertPath, clientCertPEMEncoded, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.WriteFile(clientCertKeyPath, clientCertKeyPEMEncoded, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// configure the server to use the certs
+	initialCertPool := x509.NewCertPool()
+	ok := initialCertPool.AppendCertsFromPEM(caCertPEMEncoded)
+	if !ok {
+		t.Fatal("failed to add CA cert to cert pool")
+	}
+
+	initialCert, err := tls.LoadX509KeyPair(serverCertPath, serverCertKeyPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	listener, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Unexpected error creating listener while finding free port: %s", err)
+	}
+
+	serverAddress := listener.Addr().String()
+	err = listener.Close()
+	if err != nil {
+		t.Fatalf("Unexpected error closing listener to free port: %s", err)
+	}
+
+	t.Log("server address:", serverAddress)
+
+	server := New().
+		WithAddresses([]string{serverAddress}).
+		WithStore(inmem.New()).
+		WithCertificate(&initialCert).
+		WithCertPool(x509.NewCertPool()). // empty cert pool
+		WithAuthentication(AuthenticationTLS).
+		WithTLSConfig(
+			&TLSConfig{
+				CertFile:     serverCertPath,
+				KeyFile:      serverCertKeyPath,
+				CertPoolFile: caCertPath, // currently empty
+			},
+		)
+
+	// start the server referencing the certs
+	m, err := plugins.New([]byte{}, "test", server.store)
+	if err != nil {
+		t.Fatal(err)
+	}
+	server = server.WithManager(m)
+	if err = m.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	server, err = server.Init(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	loops, err := server.Listeners()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, loop := range loops {
+		go func(serverLoop func() error) {
+			errc := make(chan error)
+			errc <- serverLoop()
+			err := <-errc
+			t.Errorf("Unexpected error from server loop: %s", err)
+		}(loop)
+	}
+
+	// wait for the server to start
+	retries := 10
+	for {
+		if retries == 0 {
+			t.Fatal("failed to start server before deadline")
+		}
+		_, err = tls.Dial("tcp", serverAddress, &tls.Config{RootCAs: initialCertPool})
+		if err != nil {
+			retries--
+			time.Sleep(300 * time.Millisecond)
+			continue
+		}
+		t.Log("server started")
+		break
+	}
+
+	// make the first request and check that the server is not trusting the client cert
+	clientKeyPair, err := tls.LoadX509KeyPair(clientCertPath, clientCertKeyPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs:      initialCertPool,
+				Certificates: []tls.Certificate{clientKeyPair},
+			},
+		},
+	}
+
+	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/v1/data", serverAddress), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = client.Do(req)
+	if !strings.Contains(err.Error(), "remote error: tls") {
+		t.Fatalf("expected unknown certificate authority error but got: %s", err)
+	}
+
+	// update the cert pool file to include the CA cert
+	err = os.WriteFile(caCertPath, caCertPEMEncoded, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make a second request and check that the server now trusts the client cert
+	retries = 10
+	for {
+		if retries == 0 {
+			t.Fatal("server didn't accept client cert before deadline")
+		}
+
+		req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/v1/data", serverAddress), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = client.Do(req)
+		if err != nil {
+			t.Log("server still doesn't trust client cert")
+			retries--
+			time.Sleep(300 * time.Millisecond)
+			continue
+		}
+
+		break
+	}
+
+	// update the cert pool file to a new & different CA that hasn't signed the client cert
+	caKey, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	caSerial, err = rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+	if err != nil {
+		t.Fatal(err)
+	}
+	caSubj = pkix.Name{
+		CommonName:   "CA 2",
+		SerialNumber: caSerial.String(),
+	}
+
+	caTemplate = &x509.Certificate{
+		BasicConstraintsValid: true,
+		SignatureAlgorithm:    x509.ECDSAWithSHA256,
+		PublicKeyAlgorithm:    x509.ECDSA,
+		PublicKey:             caKey.Public(),
+		SerialNumber:          caSerial,
+		Issuer:                caSubj,
+		Subject:               caSubj,
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(100 * time.Hour * 24 * 365),
+		KeyUsage:              x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		IsCA:                  true,
+		DNSNames:              nil,
+		EmailAddresses:        nil,
+		IPAddresses:           nil,
+	}
+
+	caCertData, err = x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caKey.Public(), caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	caCertPEMEncoded = pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caCertData,
+	})
+
+	err = os.WriteFile(caCertPath, caCertPEMEncoded, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make a final request and check that the server doesn't trust the client again
+	// since the loaded CA cert is different from the one that signed the client cert
+	retries = 10
+	for {
+		if retries == 0 {
+			t.Fatal("server didn't accept client cert before deadline")
+		}
+
+		req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/v1/data", serverAddress), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = client.Do(req)
+		if err == nil {
+			t.Log("server still trusts client cert")
+			retries--
+			time.Sleep(300 * time.Millisecond)
+			continue
+		}
+
+		if !strings.Contains(err.Error(), "remote error: tls") {
+			t.Fatalf("expected unknown certificate authority error but got: %s", err)
+		}
+
+		break
+	}
+
+	err = server.Shutdown(ctx)
+	if err != nil {
+		t.Fatalf("Unexpected error shutting down server: %s", err)
+	}
+
+}
+
+func TestCertReloading(t *testing.T) {
+
+	ctx := context.Background()
+
+	testCases := map[string]struct {
+		Server func(
+			addr string,
+			initialCert *tls.Certificate,
+			initialCertPool *x509.CertPool,
+			certFilePath, keyFilePath, caCertPath string,
+		) *Server
+	}{
+		"fs notified server": {
+			Server: func(
+				addr string,
+				initialCert *tls.Certificate,
+				initialCertPool *x509.CertPool,
+				certFilePath, keyFilePath, caCertPath string,
+			) *Server {
+				return New().
+					WithAddresses([]string{addr}).
+					WithStore(inmem.New()).
+					WithCertificate(initialCert).
+					WithCertPool(initialCertPool).
+					WithTLSConfig(
+						&TLSConfig{
+							CertFile:     certFilePath,
+							KeyFile:      keyFilePath,
+							CertPoolFile: caCertPath,
+						},
+					)
+			},
+		},
+		"interval reloaded server": {
+			Server: func(
+				addr string,
+				initialCert *tls.Certificate,
+				initialCertPool *x509.CertPool,
+				certFilePath, keyFilePath, caCertPath string,
+			) *Server {
+				return New().
+					WithAddresses([]string{addr}).
+					WithStore(inmem.New()).
+					WithCertificate(initialCert).
+					WithCertPool(initialCertPool).
+					WithCertificatePaths(
+						certFilePath,
+						keyFilePath,
+						1*time.Second,
+					)
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+
+			tempDir := t.TempDir()
+
+			serverCert1Path := filepath.Join(tempDir, "serverCert1.pem")
+			serverCert1KeyPath := filepath.Join(tempDir, "serverCert1Key.pem")
+			serverCert2Path := filepath.Join(tempDir, "serverCert2.pem")
+			serverCert2KeyPath := filepath.Join(tempDir, "serverCert2Key.pem")
+			caCertPath := filepath.Join(tempDir, "ca.pem")
+
+			t.Helper()
+
+			san := net.ParseIP("127.0.0.1")
+
+			// create the CA cert used in the cert pool and for signing server certs
+			caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			caSerial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+			if err != nil {
+				t.Fatal(err)
+			}
+			caSubj := pkix.Name{
+				CommonName:   "CA",
+				SerialNumber: caSerial.String(),
+			}
+			caTemplate := &x509.Certificate{
+				BasicConstraintsValid: true,
+				SignatureAlgorithm:    x509.ECDSAWithSHA256,
+				PublicKeyAlgorithm:    x509.ECDSA,
+				PublicKey:             caKey.Public(),
+				SerialNumber:          caSerial,
+				Issuer:                caSubj,
+				Subject:               caSubj,
+				NotBefore:             time.Now(),
+				NotAfter:              time.Now().Add(100 * time.Hour * 24 * 365),
+				KeyUsage:              x509.KeyUsageCertSign,
+				ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+				IsCA:                  true,
+			}
+
+			caCertData, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caKey.Public(), caKey)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			caCertPEMEncoded := pem.EncodeToMemory(&pem.Block{
+				Type:  "CERTIFICATE",
+				Bytes: caCertData,
+			})
+
+			err = os.WriteFile(caCertPath, caCertPEMEncoded, 0o600)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// create a cert and key for the server to load at startup
+			var serverCert1 tls.Certificate
+
+			serverCert1Key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+			if err != nil {
+				t.Fatal(err)
+			}
+			serverCert1.PrivateKey = serverCert1Key
+
+			serverCert1Serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+			if err != nil {
+				t.Fatal(err)
+			}
+			serverCert1Template := &x509.Certificate{
+				BasicConstraintsValid: true,
+				SignatureAlgorithm:    x509.ECDSAWithSHA256,
+				PublicKeyAlgorithm:    x509.ECDSA,
+				PublicKey:             serverCert1Key.Public(),
+				SerialNumber:          serverCert1Serial,
+				Issuer:                caSubj,
+				Subject: pkix.Name{
+					CommonName:   "Server 1",
+					SerialNumber: serverCert1Serial.String(),
+				},
+				NotBefore:   time.Now(),
+				NotAfter:    time.Now().Add(99 * time.Hour * 24 * 365),
+				ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+				IPAddresses: []net.IP{san},
+			}
+
+			serverCert1Data2, err := x509.CreateCertificate(rand.Reader, serverCert1Template, caTemplate, serverCert1Key.Public(), caKey)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			serverCert1PEMEncoded := pem.EncodeToMemory(&pem.Block{
+				Type:  "CERTIFICATE",
+				Bytes: serverCert1Data2,
+			})
+
+			serverCert1KeyMarshalled, _ := x509.MarshalPKCS8PrivateKey(serverCert1.PrivateKey)
+			serverCert1KeyPEMEncoded := pem.EncodeToMemory(&pem.Block{
+				Type:  "PRIVATE KEY",
+				Bytes: serverCert1KeyMarshalled,
+			})
+
+			err = os.WriteFile(serverCert1Path, serverCert1PEMEncoded, 0o600)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = os.WriteFile(serverCert1KeyPath, serverCert1KeyPEMEncoded, 0o600)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// create a cert to load after startup
+			var serverCert2 tls.Certificate
+
+			serverCert2Key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+			if err != nil {
+				t.Fatal(err)
+			}
+			serverCert2.PrivateKey = serverCert2Key
+
+			serverCert2Serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
+			if err != nil {
+				t.Fatal(err)
+			}
+			serverCert1Template = &x509.Certificate{
+				BasicConstraintsValid: true,
+				SignatureAlgorithm:    x509.ECDSAWithSHA256,
+				PublicKeyAlgorithm:    x509.ECDSA,
+				PublicKey:             serverCert2Key.Public(),
+				SerialNumber:          serverCert2Serial,
+				Issuer:                caSubj,
+				Subject: pkix.Name{
+					CommonName:   "Server 2",
+					SerialNumber: serverCert1Serial.String(),
+				},
+				NotBefore:   time.Now(),
+				NotAfter:    time.Now().Add(99 * time.Hour * 24 * 365),
+				ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+				IPAddresses: []net.IP{san},
+			}
+
+			serverCert2Data2, err := x509.CreateCertificate(rand.Reader, serverCert1Template, caTemplate, serverCert2Key.Public(), caKey)
+			if err != nil {
+				t.Fatal(err)
+			}
+			serverCert2.Certificate = [][]byte{serverCert2Data2, caCertData}
+
+			serverCert2PEMEncoded := pem.EncodeToMemory(&pem.Block{
+				Type:  "CERTIFICATE",
+				Bytes: serverCert2Data2,
+			})
+
+			serverCert2KeyMarshalled, _ := x509.MarshalPKCS8PrivateKey(serverCert2.PrivateKey)
+			serverCert2KeyPEMEncoded := pem.EncodeToMemory(&pem.Block{
+				Type:  "PRIVATE KEY",
+				Bytes: serverCert2KeyMarshalled,
+			})
+
+			err = os.WriteFile(serverCert2Path, serverCert2PEMEncoded, 0o600)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = os.WriteFile(serverCert2KeyPath, serverCert2KeyPEMEncoded, 0o600)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			certPool2 := x509.NewCertPool()
+			ok := certPool2.AppendCertsFromPEM(caCertPEMEncoded)
+			if !ok {
+				t.Fatal("failed to add CA cert to cert pool")
+			}
+			certPool, _, _, serverCert1Data, serverCert2Data := certPool2, &serverCert1, &serverCert2, serverCert1Data2, serverCert2Data2
+
+			initialCert, err := tls.LoadX509KeyPair(serverCert1Path, serverCert1KeyPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			listener, err := net.Listen("tcp", "localhost:0")
+			if err != nil {
+				t.Fatalf("Unexpected error creating listener while finding free port: %s", err)
+			}
+
+			serverAddress := listener.Addr().String()
+			err = listener.Close()
+			if err != nil {
+				t.Fatalf("Unexpected error closing listener to free port: %s", err)
+			}
+
+			t.Log("server address:", serverAddress)
+
+			server := tc.Server(serverAddress, &initialCert, certPool, serverCert1Path, serverCert1KeyPath, caCertPath)
+
+			// start the server referencing the certs
+			m, err := plugins.New([]byte{}, "test", server.store)
+			if err != nil {
+				t.Fatal(err)
+			}
+			server = server.WithManager(m)
+			if err = m.Start(ctx); err != nil {
+				t.Fatal(err)
+			}
+			server, err = server.Init(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			loops, err := server.Listeners()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			for _, loop := range loops {
+				go func(serverLoop func() error) {
+					errc := make(chan error)
+					errc <- serverLoop()
+					err := <-errc
+					t.Errorf("Unexpected error from server loop: %s", err)
+				}(loop)
+			}
+
+			// wait for the server to start
+			retries := 10
+			for {
+				if retries == 0 {
+					t.Fatal("failed to start server before deadline")
+				}
+				_, err = tls.Dial("tcp", serverAddress, &tls.Config{RootCAs: certPool})
+				if err != nil {
+					retries--
+					time.Sleep(300 * time.Millisecond)
+					continue
+				}
+				t.Log("server started")
+				break
+			}
+
+			// make the first connection, check that the server 1 cert is returned
+			retries = 10
+			for {
+				if retries == 0 {
+					t.Fatal("failed to get serverCert1 before deadline")
+				}
+				conn, err := tls.Dial("tcp", serverAddress, &tls.Config{RootCAs: certPool})
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = conn.Close()
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				certs := conn.ConnectionState().PeerCertificates
+				if len(certs) != 1 {
+					t.Fatalf("expected 1 cert, got %d", len(certs))
+				}
+
+				servedCert := certs[0]
+				if !bytes.Equal(servedCert.Raw, serverCert1Data) {
+					retries--
+					time.Sleep(300 * time.Millisecond)
+					t.Logf("expected serverCert1, got %s", servedCert.Subject)
+					continue
+				}
+
+				break
+			}
+
+			// update the cert and key files by moving the second cert into place instead
+			err = os.Rename(serverCert2Path, serverCert1Path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = os.Rename(serverCert2KeyPath, serverCert1KeyPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// make another connection, check that the server 2 cert is returned
+			retries = 10
+			for {
+				if retries == 0 {
+					t.Fatal("failed to get serverCert2 before deadline")
+				}
+
+				conn, err := tls.Dial("tcp", serverAddress, &tls.Config{RootCAs: certPool})
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = conn.Close()
+				if err != nil {
+					t.Fatal(err)
+				}
+				certs := conn.ConnectionState().PeerCertificates
+				if len(certs) != 1 {
+					t.Fatalf("expected 1 cert, got %d", len(certs))
+				}
+
+				servedCert := certs[0]
+				if !bytes.Equal(servedCert.Raw, serverCert2Data) {
+					retries--
+					time.Sleep(300 * time.Millisecond)
+					t.Logf("expected serverCert2, got %s", servedCert.Subject)
+					continue
+				}
+
+				break
+			}
+
+			// remove the certs on disk, and check that the server still serves the previous certs
+			err = os.Remove(serverCert1Path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = os.Remove(serverCert1KeyPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// make a third connection, and check that the server 2 cert is still returned despite the certs being removed
+			retries = 10
+			for {
+				if retries == 0 {
+					t.Fatal("failed to get serverCert2 before deadline")
+				}
+
+				conn, err := tls.Dial("tcp", serverAddress, &tls.Config{RootCAs: certPool})
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = conn.Close()
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				certs := conn.ConnectionState().PeerCertificates
+				if len(certs) != 1 {
+					t.Fatalf("expected 1 cert, got %d", len(certs))
+				}
+
+				servedCert := certs[0]
+				if !bytes.Equal(servedCert.Raw, serverCert2Data) {
+					retries--
+					time.Sleep(300 * time.Millisecond)
+					t.Logf("expected serverCert2, got %s", servedCert.Subject)
+					continue
+				}
+
+				break
+			}
+
+			err = server.Shutdown(ctx)
+			if err != nil {
+				t.Fatalf("Unexpected error shutting down server: %s", err)
+			}
+		})
+	}
+
+}
+
 type mockHTTPHandler struct{}
 
 func (*mockHTTPHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {