Skip to main content

rushls_common/
tls.rs

1//! TLS termination with certificates that rotate underneath a running origin.
2//!
3//! Two things make this more than a wrapper around [`tokio_rustls`].
4//!
5//! The first is reload. A `ServerConfig` is built once and never rebuilt;
6//! every handshake asks [`CertificateResolver`] which key to present, and that
7//! resolver is one [`ArcSwap`] deep. Rotating a certificate is therefore a
8//! single atomic store: live connections keep the key they negotiated with,
9//! the next handshake picks up the new one, and nothing is dropped or
10//! rebound. Only the certificate and key can change this way — the ALPN list
11//! and the crypto provider are fixed at bind time, because nothing rotates
12//! them on a schedule.
13//!
14//! The second is that the handshake must not happen on the accept path.
15//! [`axum::serve::Listener::accept`] is a single future the server awaits in
16//! sequence, so terminating TLS inside it would let one slow client hold up
17//! every other connection. Handshakes run as spawned tasks under a deadline
18//! and a fixed ceiling instead.
19
20use std::{
21    io,
22    net::SocketAddr,
23    path::{Path, PathBuf},
24    sync::Arc,
25    time::Duration,
26};
27
28use arc_swap::ArcSwap;
29
30use crate::accept::{AcceptErrors, AcceptHealth, Next};
31use notify::RecursiveMode;
32use notify_debouncer_full::{DebounceEventResult, Debouncer, RecommendedCache, new_debouncer};
33use rustls::{
34    ServerConfig,
35    client::ResolvesClientCert,
36    crypto::CryptoProvider,
37    pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject},
38    server::{ClientHello, ResolvesServerCert},
39    sign::CertifiedKey,
40};
41use thiserror::Error;
42use tokio::{
43    net::{TcpListener, TcpStream},
44    sync::mpsc,
45    task::JoinSet,
46    time::timeout,
47};
48use tokio_rustls::{TlsAcceptor, server::TlsStream};
49
50/// What a caller learns about certificates and handshakes.
51///
52/// A trait rather than a concrete reporter because the two applications count
53/// and log differently, and neither should have to adopt the other's
54/// observability model to reuse the reload machinery. Every method has a
55/// default, so an application that wants none of it implements nothing.
56///
57/// Implementations are called from the accept loop and from the watcher task,
58/// so they must not block.
59pub trait TlsObserver: Send + Sync + 'static {
60    /// A certificate pair was read and adopted.
61    fn certificate_loaded(&self, _certificate: &Path) {}
62    /// A rotation was seen but not adopted; the previous pair still serves.
63    fn certificate_rejected(&self, _certificate: &Path, _reason: &str) {}
64    /// Rotations will no longer be noticed, while TLS keeps working.
65    ///
66    /// Reported separately because it is otherwise invisible: nothing breaks
67    /// until the certificate expires, which may be months away.
68    fn certificate_watch_lost(&self, _reason: &str) {}
69    /// The listener could not accept connections and is retrying.
70    ///
71    /// `failures` counts attempts since the previous report, this one
72    /// included: a persistent failure is summarised every
73    /// [`REPORT_INTERVAL`](crate::accept::REPORT_INTERVAL) rather than reported
74    /// once per attempt.
75    fn accept_failed(&self, _reason: &str, _failures: u64) {}
76    /// The listening socket is unusable and the listener has stopped accepting.
77    ///
78    /// [`TlsListener::accept_tls`] also returns the error; this exists for the
79    /// `axum` listener, which has no way to.
80    fn listener_failed(&self, _reason: &str) {}
81    fn handshake_completed(&self) {}
82    fn handshake_failed(&self) {}
83}
84
85/// Discards everything, for a caller that wants no reporting.
86#[derive(Clone, Copy, Debug, Default)]
87pub struct IgnoreTlsEvents;
88
89impl TlsObserver for IgnoreTlsEvents {}
90
91/// Advertised so a browser can reach HTTP/2, which is what Apple's Low-Latency
92/// profile expects. Order is preference order: h2 first, HTTP/1.1 as the
93/// fallback for clients that cannot.
94const ALPN_PROTOCOLS: [&[u8]; 2] = [b"h2", b"http/1.1"];
95
96/// How long the watcher coalesces filesystem activity before reloading.
97///
98/// Rotation is never one atomic act — certbot writes two files, a Kubernetes
99/// secret mount swaps a symlinked directory. Reacting to the first event would
100/// reliably load a new certificate against the old key.
101const DEBOUNCE: Duration = Duration::from_secs(1);
102
103/// Supported protocol bounds for TCP TLS. QUIC always uses TLS 1.3.
104#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
105pub enum TlsVersion {
106    Tls12,
107    #[default]
108    Tls13,
109}
110
111impl std::str::FromStr for TlsVersion {
112    type Err = &'static str;
113
114    fn from_str(value: &str) -> Result<Self, Self::Err> {
115        match value {
116            "1.2" => Ok(Self::Tls12),
117            "1.3" => Ok(Self::Tls13),
118            _ => Err("TLS version must be 1.2 or 1.3"),
119        }
120    }
121}
122
123/// Where the certificate and its key live, and how patient the listener is.
124#[derive(Clone, Debug, Eq, PartialEq)]
125pub struct TlsSettings {
126    /// PEM, leaf first, followed by any intermediates.
127    pub certificate: PathBuf,
128    /// PEM, PKCS#8, PKCS#1, or SEC1.
129    pub key: PathBuf,
130    /// Lowest accepted TCP TLS version.
131    pub min_version: TlsVersion,
132    /// Highest accepted TCP TLS version. QUIC ignores these TCP bounds.
133    pub max_version: TlsVersion,
134    /// Bounds a connection that completes TCP and then stalls mid-handshake.
135    pub handshake_timeout: Duration,
136    /// Caps handshakes in flight at once, which is what stops a flood from
137    /// growing the task set without bound.
138    pub maximum_pending_handshakes: usize,
139}
140
141impl Default for TlsSettings {
142    fn default() -> Self {
143        Self {
144            certificate: PathBuf::new(),
145            key: PathBuf::new(),
146            min_version: TlsVersion::Tls13,
147            max_version: TlsVersion::Tls13,
148            handshake_timeout: Duration::from_secs(10),
149            maximum_pending_handshakes: 256,
150        }
151    }
152}
153
154#[derive(Debug, Error)]
155pub enum TlsError {
156    #[error("TLS min_version must not exceed max_version")]
157    InvalidVersionRange,
158    #[error("could not read {path}: {source}")]
159    Unreadable {
160        path: PathBuf,
161        source: rustls::pki_types::pem::Error,
162    },
163    #[error("{path} contains no certificate")]
164    Empty { path: PathBuf },
165    /// Covers a key that will not parse, a key belonging to some other
166    /// certificate, and a certificate rustls will not serve at all — an X.509
167    /// v1 certificate being the one an operator is most likely to produce by
168    /// accident, since `openssl req -x509` emits one unless asked otherwise.
169    #[error("the certificate and key are not usable: {0}")]
170    Unusable(rustls::Error),
171    #[error("TLS could not be configured: {0}")]
172    Configuration(rustls::Error),
173    #[error("could not watch {path} for certificate changes: {source}")]
174    Unwatchable {
175        path: PathBuf,
176        source: notify::Error,
177    },
178    /// Rotation runs as a Tokio task, so loading certificates needs a runtime.
179    /// An error rather than a panic: configuration is often resolved before
180    /// one exists, and that is a caller mistake worth a clear message.
181    #[error("certificate rotation needs a Tokio runtime; load {path} from inside one")]
182    NoRuntime { path: PathBuf },
183}
184
185impl From<TlsError> for io::Error {
186    fn from(error: TlsError) -> Self {
187        Self::other(error.to_string())
188    }
189}
190
191/// Answers every handshake from a slot that can be replaced underneath it.
192///
193/// Shared by both directions. A client certificate expires exactly as a server
194/// one does, so an outbound caller needs the same rotation story as a listener;
195/// giving each its own would mean one of them silently not reloading.
196#[derive(Debug)]
197pub struct CertificateResolver(ArcSwap<CertifiedKey>);
198
199impl ResolvesServerCert for CertificateResolver {
200    fn resolve(&self, _hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
201        Some(self.0.load_full())
202    }
203}
204
205impl ResolvesClientCert for CertificateResolver {
206    fn resolve(
207        &self,
208        _root_hint_subjects: &[&[u8]],
209        _sigschemes: &[rustls::SignatureScheme],
210    ) -> Option<Arc<CertifiedKey>> {
211        Some(self.0.load_full())
212    }
213
214    fn has_certs(&self) -> bool {
215        true
216    }
217}
218
219/// Reads a certificate and key, and proves they belong together.
220///
221/// [`CertifiedKey::from_der`] parses the key with the provider and compares
222/// its `SubjectPublicKeyInfo` against the leaf certificate's, so a rotation
223/// caught half-written is rejected here rather than at the next handshake.
224fn load(settings: &TlsSettings, provider: &CryptoProvider) -> Result<CertifiedKey, TlsError> {
225    let chain = load_certificates(&settings.certificate)?;
226    let key = load_private_key(&settings.key)?;
227    CertifiedKey::from_der(chain, key, provider).map_err(TlsError::Unusable)
228}
229
230/// A client certificate and key that reload underneath a running caller.
231///
232/// The outbound mirror of [`TlsListener`], and deliberately built from the same
233/// parts. A client certificate expires on the same schedule a server one does,
234/// so a node that could not renew without a restart would drop publishers on
235/// rotation day — the failure the listener side already avoids.
236///
237/// Resolution happens per handshake through an [`ArcSwap`], so a rotation is one
238/// atomic store: connections already established keep the identity they
239/// negotiated with, and the next handshake picks up the new one.
240pub struct ClientIdentity {
241    resolver: Arc<CertificateResolver>,
242    /// Held because dropping it stops reloads.
243    _watcher: CertificateWatch,
244}
245
246impl ClientIdentity {
247    /// Loads the pair and starts watching for rotations.
248    ///
249    /// Failing at startup is fatal, unlike failing to reload later: a node
250    /// configured to present an identity it cannot read should not come up
251    /// pretending it will be admitted.
252    pub fn new<O: TlsObserver>(
253        certificate: PathBuf,
254        key: PathBuf,
255        observer: Arc<O>,
256    ) -> Result<Self, TlsError> {
257        // The watch is expressed in terms of a settings pair, and the two
258        // timeout fields it also carries are meaningless here. Passing the
259        // defaults keeps one loader rather than two that could disagree about
260        // what a usable pair is.
261        let settings = TlsSettings {
262            certificate,
263            key,
264            ..TlsSettings::default()
265        };
266        let provider = Arc::new(rustls::crypto::ring::default_provider());
267        let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
268            &settings, &provider,
269        )?)));
270        observer.certificate_loaded(&settings.certificate);
271
272        let watcher = CertificateWatch::start(settings, Arc::clone(&resolver), provider, observer)?;
273        Ok(Self {
274            resolver,
275            _watcher: watcher,
276        })
277    }
278
279    /// The resolver a `rustls` client configuration should ask per handshake.
280    ///
281    /// Handed out rather than a built configuration so the caller keeps
282    /// ownership of its own root store and protocol versions, which are not
283    /// this type's business.
284    pub fn resolver(&self) -> Arc<dyn ResolvesClientCert> {
285        Arc::clone(&self.resolver) as Arc<dyn ResolvesClientCert>
286    }
287}
288
289/// Reads PEM certificates to trust, for a caller pinning its own authority.
290///
291/// Not watched. A trust root is the thing a rotation is validated *against*,
292/// so it turns over on a different and far slower schedule than the leaf it
293/// signs; treating it as hot-reloadable would suggest otherwise.
294pub fn load_roots(path: &Path) -> Result<Vec<CertificateDer<'static>>, TlsError> {
295    load_certificates(path)
296}
297
298/// Reads a nonempty PEM chain, preserving leaf-first certificate order.
299pub fn load_certificates(path: &Path) -> Result<Vec<CertificateDer<'static>>, TlsError> {
300    let roots = CertificateDer::pem_file_iter(path)
301        .and_then(std::iter::Iterator::collect::<Result<Vec<_>, _>>)
302        .map_err(|source| TlsError::Unreadable {
303            path: path.to_path_buf(),
304            source,
305        })?;
306    if roots.is_empty() {
307        return Err(TlsError::Empty {
308            path: path.to_path_buf(),
309        });
310    }
311    Ok(roots)
312}
313
314/// Reads a PKCS#8, PKCS#1, or SEC1 PEM private key.
315pub fn load_private_key(path: &Path) -> Result<PrivateKeyDer<'static>, TlsError> {
316    PrivateKeyDer::from_pem_file(path).map_err(|source| TlsError::Unreadable {
317        path: path.to_path_buf(),
318        source,
319    })
320}
321
322/// A bound TLS listener plus the watch that keeps its certificate current.
323///
324/// The watcher is owned here because dropping it silently stops reloads; tying
325/// its lifetime to the listener's means the two cannot get separated.
326pub struct TlsListener<O: TlsObserver> {
327    tcp: TcpListener,
328    acceptor: TlsAcceptor,
329    handshakes: JoinSet<Option<(TlsStream<TcpStream>, SocketAddr)>>,
330    settings: TlsSettings,
331    observer: Arc<O>,
332    errors: AcceptErrors,
333    _watcher: CertificateWatch,
334}
335
336impl<O: TlsObserver> TlsListener<O> {
337    /// Terminates TLS on an already-bound listener, loading the initial
338    /// certificate and starting the watch for rotations.
339    ///
340    /// Failing to load at startup is fatal, unlike failing to reload later: a
341    /// process that comes up without a usable certificate has nothing to serve.
342    pub fn new(
343        tcp: TcpListener,
344        settings: TlsSettings,
345        observer: Arc<O>,
346    ) -> Result<Self, TlsError> {
347        let (config, watcher) =
348            rotating_tcp_server_config(settings.clone(), &ALPN_PROTOCOLS, Arc::clone(&observer))?;
349
350        Ok(Self {
351            tcp,
352            acceptor: TlsAcceptor::from(config),
353            handshakes: JoinSet::new(),
354            settings,
355            observer,
356            errors: AcceptErrors::default(),
357            _watcher: watcher,
358        })
359    }
360
361    /// Feeds this listener's accept failures into a readiness probe.
362    #[must_use]
363    pub fn with_health(mut self, health: AcceptHealth) -> Self {
364        self.errors = AcceptErrors::with_health(health);
365        self
366    }
367
368    /// Mirrors the trait method so callers need not import the trait to ask.
369    pub fn local_addr(&self) -> io::Result<SocketAddr> {
370        self.tcp.local_addr()
371    }
372
373    /// Accepts the next connection whose TLS handshake has completed.
374    ///
375    /// Handshakes run as spawned tasks under a deadline and a fixed ceiling
376    /// rather than inline, so one slow client cannot hold up every other
377    /// connection waiting to be accepted.
378    ///
379    /// Temporary accept failures are retried with a backoff and reported
380    /// through the observer. An error is returned only when the listening
381    /// socket itself is unusable, so the caller can stop instead of retrying
382    /// forever.
383    pub async fn accept_tls(&mut self) -> io::Result<(TlsStream<TcpStream>, SocketAddr)> {
384        loop {
385            tokio::select! {
386                // Accepting stops at capacity rather than queueing: refusing to
387                // take the next connection is backpressure the kernel already
388                // knows how to apply, and it needs no semaphore to express.
389                accepted = self.tcp.accept(),
390                    if self.handshakes.len() < self.settings.maximum_pending_handshakes =>
391                {
392                    let (stream, peer) = match accepted {
393                        Ok(accepted) => {
394                            self.errors.succeeded();
395                            accepted
396                        }
397                        Err(error) => match self.errors.failed(&error) {
398                            Next::Stop => {
399                                self.observer.listener_failed(&error.to_string());
400                                return Err(error);
401                            }
402                            // Retrying at once would spin a core while
403                            // descriptors stay exhausted.
404                            Next::Retry { pause, report } => {
405                                if let Some(failures) = report {
406                                    self.observer.accept_failed(&error.to_string(), failures);
407                                }
408                                tokio::time::sleep(pause).await;
409                                continue;
410                            }
411                        },
412                    };
413                    let acceptor = self.acceptor.clone();
414                    let deadline = self.settings.handshake_timeout;
415                    let observer = Arc::clone(&self.observer);
416                    self.handshakes.spawn(async move {
417                        if let Ok(Ok(stream)) = timeout(deadline, acceptor.accept(stream)).await {
418                            observer.handshake_completed();
419                            Some((stream, peer))
420                        } else {
421                            // Neither outcome is reported as an event: the rate
422                            // is whatever a remote peer chooses to make it.
423                            observer.handshake_failed();
424                            None
425                        }
426                    });
427                }
428
429                Some(joined) = self.handshakes.join_next(), if !self.handshakes.is_empty() => {
430                    if let Ok(Some(ready)) = joined {
431                        return Ok(ready);
432                    }
433                }
434            }
435        }
436    }
437}
438
439/// Lets an `axum` server accept from this listener.
440///
441/// Behind a feature because the reload machinery has nothing to do with axum -
442/// non-HTTP callers use [`TlsListener::accept_tls`] directly without
443/// compiling a web framework to get certificate rotation.
444#[cfg(feature = "axum")]
445impl<O: TlsObserver> axum::serve::Listener for TlsListener<O> {
446    type Io = TlsStream<TcpStream>;
447    type Addr = SocketAddr;
448
449    /// `axum` cannot be told a listener has failed, so a fatal error parks
450    /// this listener after [`TlsObserver::listener_failed`]. A caller that must
451    /// stop on it should accept with [`TlsListener::accept_tls`] instead.
452    async fn accept(&mut self) -> (Self::Io, Self::Addr) {
453        match self.accept_tls().await {
454            Ok(accepted) => accepted,
455            Err(_) => std::future::pending().await,
456        }
457    }
458
459    fn local_addr(&self) -> io::Result<Self::Addr> {
460        self.tcp.local_addr()
461    }
462}
463
464/// Keeps the debouncer and its pump alive for as long as its owner lives.
465///
466/// Public so an outbound client can hold one too: dropping it silently stops
467/// reloads, so its lifetime has to be tied to whatever depends on the material
468/// staying current.
469pub struct CertificateWatch {
470    _debouncer: Debouncer<notify::RecommendedWatcher, RecommendedCache>,
471    pump: tokio::task::JoinHandle<()>,
472}
473
474impl Drop for CertificateWatch {
475    fn drop(&mut self) {
476        self.pump.abort();
477    }
478}
479
480impl CertificateWatch {
481    pub fn start<O: TlsObserver>(
482        settings: TlsSettings,
483        resolver: Arc<CertificateResolver>,
484        provider: Arc<CryptoProvider>,
485        observer: Arc<O>,
486    ) -> Result<Self, TlsError> {
487        // Checked before watching anything, so a refusal leaves nothing behind.
488        let runtime = tokio::runtime::Handle::try_current().map_err(|_| TlsError::NoRuntime {
489            path: settings.certificate.clone(),
490        })?;
491        let (sender, receiver) = mpsc::unbounded_channel();
492        let mut debouncer = new_debouncer(DEBOUNCE, None, move |result: DebounceEventResult| {
493            // The receiver going away means the listener is gone; there is
494            // nothing left to reload for.
495            let _ = sender.send(result);
496        })
497        .map_err(|source| TlsError::Unwatchable {
498            path: settings.certificate.clone(),
499            source,
500        })?;
501
502        // Watching the containing directories rather than the files. Rotation
503        // replaces rather than rewrites — certbot renames into place, a
504        // Kubernetes secret mount swaps a `..data` symlink — and a watch on
505        // the file itself is left pointing at an inode nobody will touch
506        // again.
507        for directory in watched_directories(&settings) {
508            debouncer
509                .watch(&directory, RecursiveMode::NonRecursive)
510                .map_err(|source| TlsError::Unwatchable {
511                    path: directory.clone(),
512                    source,
513                })?;
514        }
515
516        let pump = runtime.spawn(reload_on_change(
517            settings, resolver, provider, observer, receiver,
518        ));
519        Ok(Self {
520            _debouncer: debouncer,
521            pump,
522        })
523    }
524}
525
526/// A rustls server configuration for TCP, with a rotating certificate.
527///
528/// For callers that run their own accept loop and handshake inside it, such
529/// as an ingest listener that must reserve capacity and establish the client
530/// address before spending a TLS handshake on it. [`TlsListener`] is built on
531/// the same configuration. An empty `alpn` negotiates no application protocol,
532/// which is what protocols predating ALPN expect.
533///
534/// The returned [`CertificateWatch`] must be held for as long as the config is
535/// used; dropping it silently stops reloads.
536pub fn rotating_tcp_server_config<O: TlsObserver>(
537    settings: TlsSettings,
538    alpn: &[&[u8]],
539    observer: Arc<O>,
540) -> Result<(Arc<ServerConfig>, CertificateWatch), TlsError> {
541    let versions: &[&rustls::SupportedProtocolVersion] =
542        match (settings.min_version, settings.max_version) {
543            (TlsVersion::Tls12, TlsVersion::Tls12) => &[&rustls::version::TLS12],
544            (TlsVersion::Tls12, TlsVersion::Tls13) => {
545                &[&rustls::version::TLS13, &rustls::version::TLS12]
546            }
547            (TlsVersion::Tls13, TlsVersion::Tls13) => &[&rustls::version::TLS13],
548            (TlsVersion::Tls13, TlsVersion::Tls12) => {
549                return Err(TlsError::InvalidVersionRange);
550            }
551        };
552    // Carried explicitly rather than through `CryptoProvider::install_default`
553    // so nothing here depends on a process-global having been set first.
554    let provider = Arc::new(rustls::crypto::ring::default_provider());
555    let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
556        &settings, &provider,
557    )?)));
558    observer.certificate_loaded(&settings.certificate);
559
560    let mut config = ServerConfig::builder_with_provider(Arc::clone(&provider))
561        .with_protocol_versions(versions)
562        .map_err(TlsError::Configuration)?
563        .with_no_client_auth()
564        .with_cert_resolver(Arc::clone(&resolver) as Arc<dyn ResolvesServerCert>);
565    config.alpn_protocols = alpn.iter().map(|name| name.to_vec()).collect();
566
567    let watcher = CertificateWatch::start(settings, resolver, provider, observer)?;
568    Ok((Arc::new(config), watcher))
569}
570
571/// A rustls server configuration for QUIC, with the same rotating certificate
572/// as [`TlsListener`].
573///
574/// QUIC cannot reuse an HTTPS [`ServerConfig`]: HTTP/3 requires TLS 1.3 only,
575/// a different ALPN, and `max_early_data_size = u32::MAX` so Quinn will accept
576/// the config at all. The certificate resolver is the same [`ArcSwap`] the TCP
577/// listener uses, so a rotation is still one atomic store and this config is
578/// never rebuilt.
579///
580/// The returned [`CertificateWatch`] must be held for as long as the QUIC
581/// endpoint lives; dropping it silently stops reloads.
582pub fn rotating_quic_server_config<O: TlsObserver>(
583    settings: TlsSettings,
584    alpn: &[&[u8]],
585    observer: Arc<O>,
586) -> Result<(Arc<ServerConfig>, CertificateWatch), TlsError> {
587    let provider = Arc::new(rustls::crypto::ring::default_provider());
588    let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
589        &settings, &provider,
590    )?)));
591    observer.certificate_loaded(&settings.certificate);
592
593    let mut config = ServerConfig::builder_with_provider(Arc::clone(&provider))
594        .with_protocol_versions(&[&rustls::version::TLS13])
595        .map_err(TlsError::Configuration)?
596        .with_no_client_auth()
597        .with_cert_resolver(Arc::clone(&resolver) as Arc<dyn ResolvesServerCert>);
598    config.alpn_protocols = alpn.iter().map(|name| name.to_vec()).collect();
599    // Quinn's `QuicServerConfig::try_from` refuses a rustls config that does
600    // not advertise 0-RTT. The advertised size is the QUIC maximum; whether a
601    // session actually accepts early data is a transport decision, not a TLS one.
602    config.max_early_data_size = u32::MAX;
603
604    let watcher = CertificateWatch::start(settings, resolver, provider, observer)?;
605    Ok((Arc::new(config), watcher))
606}
607
608/// The directories holding the pair, deduplicated when both files share one.
609fn watched_directories(settings: &TlsSettings) -> Vec<PathBuf> {
610    let mut directories = Vec::with_capacity(2);
611    for path in [&settings.certificate, &settings.key] {
612        let directory = path.parent().unwrap_or(Path::new(".")).to_path_buf();
613        if !directories.contains(&directory) {
614            directories.push(directory);
615        }
616    }
617    directories
618}
619
620async fn reload_on_change<O: TlsObserver>(
621    settings: TlsSettings,
622    resolver: Arc<CertificateResolver>,
623    provider: Arc<CryptoProvider>,
624    observer: Arc<O>,
625    mut changes: mpsc::UnboundedReceiver<DebounceEventResult>,
626) {
627    while let Some(result) = changes.recv().await {
628        // Any activity in the directory is a reason to re-read, without
629        // inspecting which paths the events name.
630        //
631        // Filtering on the configured paths is the obvious design and it is
632        // wrong. A watcher reports the path the platform resolved, and a
633        // Kubernetes secret mount never resolves to the name we were given:
634        // `tls.crt` is a symlink into `..data`, itself a symlink to a
635        // timestamped directory, and rotation swaps the *symlink* — so every
636        // event names `..data` or a directory we have never heard of, and a
637        // path filter discards all of them. The certificate would then never
638        // reload, in exactly the deployment that needs it most, and the
639        // symptom would be nothing happening at all.
640        //
641        // Re-reading unconditionally costs two file reads on a dedicated
642        // secret mount, already coalesced by the debounce, and the comparison
643        // below keeps a spurious wake from being mistaken for a rotation.
644        if let Err(errors) = result {
645            let reason = errors
646                .iter()
647                .map(ToString::to_string)
648                .collect::<Vec<_>>()
649                .join("; ");
650            observer.certificate_watch_lost(&reason);
651            continue;
652        }
653
654        // Blocking reads, moved off the runtime because a rotation on a slow
655        // or network-backed volume would otherwise stall a worker thread.
656        let settings_for_load = settings.clone();
657        let provider_for_load = Arc::clone(&provider);
658        let loaded =
659            tokio::task::spawn_blocking(move || load(&settings_for_load, &provider_for_load)).await;
660
661        match loaded {
662            Ok(Ok(certified)) => {
663                // Unchanged material is the common case now that any activity
664                // wakes this loop. Swapping anyway would be harmless, but
665                // reporting it would turn the event stream — the thing an
666                // operator reads to answer "did the rotation take?" — into
667                // noise.
668                if certified.cert == resolver.0.load().cert {
669                    continue;
670                }
671                resolver.0.store(Arc::new(certified));
672                observer.certificate_loaded(&settings.certificate);
673            }
674            // The previously loaded pair stays in place. A rotation that
675            // produced an unusable file is a reason to keep serving, not to
676            // start failing every handshake.
677            Ok(Err(error)) => {
678                observer.certificate_rejected(&settings.certificate, &error.to_string())
679            }
680            Err(error) => observer.certificate_watch_lost(&error.to_string()),
681        }
682    }
683
684    // The channel closes when the debouncer is dropped, which happens with the
685    // listener. Reaching here during a shutdown is ordinary; reaching it while
686    // still serving means rotations are no longer observed.
687    observer.certificate_watch_lost("the certificate watch ended");
688}
689
690#[cfg(test)]
691mod tests {
692    use super::*;
693    use std::sync::atomic::{AtomicU64, Ordering};
694
695    /// A private directory for one test, removed if a previous run left it behind.
696    fn scratch(name: &str) -> PathBuf {
697        static NEXT: AtomicU64 = AtomicU64::new(0);
698        let unique = NEXT.fetch_add(1, Ordering::Relaxed);
699        let directory =
700            std::env::temp_dir().join(format!("rushls-tls-{name}-{}-{unique}", std::process::id()));
701        let _ = std::fs::remove_dir_all(&directory);
702        std::fs::create_dir_all(&directory).expect("the scratch directory is created");
703        directory
704    }
705
706    /// A self-signed pair written into `directory`, plus the certificate's DER.
707    fn write_pair(directory: &Path, name: &str) -> (TlsSettings, Vec<u8>) {
708        let certified = rcgen::generate_simple_self_signed([name.to_owned()])
709            .expect("a self-signed certificate is generated");
710        let settings = TlsSettings {
711            certificate: directory.join("certificate.pem"),
712            key: directory.join("key.pem"),
713            ..TlsSettings::default()
714        };
715        write_atomically(&settings.certificate, certified.cert.pem().as_bytes());
716        write_atomically(
717            &settings.key,
718            certified.signing_key.serialize_pem().as_bytes(),
719        );
720        (settings, certified.cert.der().to_vec())
721    }
722
723    /// Writes through a rename, the way real rotation tooling does.
724    ///
725    /// Writing in place would let a watcher observe a half-written file, which
726    /// is the very race the debounce exists to absorb.
727    fn write_atomically(path: &Path, contents: &[u8]) {
728        let staged = path.with_extension("staged");
729        std::fs::write(&staged, contents).expect("the staged file is written");
730        std::fs::rename(&staged, path).expect("the staged file is renamed into place");
731    }
732
733    fn provider() -> CryptoProvider {
734        rustls::crypto::ring::default_provider()
735    }
736
737    #[test]
738    fn a_matching_pair_loads() {
739        let directory = scratch("matching");
740        let (settings, _) = write_pair(&directory, "origin.test");
741
742        load(&settings, &provider()).expect("a self-signed pair is usable");
743    }
744
745    #[test]
746    fn a_key_from_a_different_certificate_is_refused() {
747        let directory = scratch("mismatched");
748        let (settings, _) = write_pair(&directory, "origin.test");
749        let other = scratch("mismatched-other");
750        let (foreign, _) = write_pair(&other, "other.test");
751        std::fs::copy(&foreign.key, &settings.key).expect("the foreign key replaces the real one");
752
753        assert!(matches!(
754            load(&settings, &provider()),
755            Err(TlsError::Unusable(_))
756        ));
757    }
758
759    #[test]
760    fn a_truncated_certificate_is_refused_rather_than_loaded_empty() {
761        let directory = scratch("truncated");
762        let (settings, _) = write_pair(&directory, "origin.test");
763        std::fs::write(&settings.certificate, b"").expect("the certificate is truncated");
764
765        assert!(matches!(
766            load(&settings, &provider()),
767            Err(TlsError::Empty { .. })
768        ));
769    }
770
771    #[test]
772    fn a_pair_sharing_one_directory_is_watched_once() {
773        let settings = TlsSettings {
774            certificate: PathBuf::from("/etc/tls/fullchain.pem"),
775            key: PathBuf::from("/etc/tls/privkey.pem"),
776            ..TlsSettings::default()
777        };
778
779        assert_eq!(watched_directories(&settings), [PathBuf::from("/etc/tls")]);
780    }
781
782    #[tokio::test]
783    async fn tls_version_bounds_control_negotiation() -> Result<(), Box<dyn std::error::Error>> {
784        let directory = scratch("protocol-versions");
785        let (mut settings, der) = write_pair(&directory, "origin.test");
786        let mut roots = rustls::RootCertStore::empty();
787        roots.add(CertificateDer::from(der))?;
788        for (min_version, max_version, versions, expected) in [
789            (
790                TlsVersion::Tls13,
791                TlsVersion::Tls13,
792                vec![&rustls::version::TLS12],
793                None,
794            ),
795            (
796                TlsVersion::Tls13,
797                TlsVersion::Tls13,
798                vec![&rustls::version::TLS13],
799                Some(rustls::ProtocolVersion::TLSv1_3),
800            ),
801            (
802                TlsVersion::Tls12,
803                TlsVersion::Tls13,
804                vec![&rustls::version::TLS12],
805                Some(rustls::ProtocolVersion::TLSv1_2),
806            ),
807            (
808                TlsVersion::Tls12,
809                TlsVersion::Tls13,
810                vec![&rustls::version::TLS12, &rustls::version::TLS13],
811                Some(rustls::ProtocolVersion::TLSv1_3),
812            ),
813            (
814                TlsVersion::Tls12,
815                TlsVersion::Tls12,
816                vec![&rustls::version::TLS13],
817                None,
818            ),
819            (
820                TlsVersion::Tls12,
821                TlsVersion::Tls12,
822                vec![&rustls::version::TLS12, &rustls::version::TLS13],
823                Some(rustls::ProtocolVersion::TLSv1_2),
824            ),
825        ] {
826            settings.min_version = min_version;
827            settings.max_version = max_version;
828            let listener = TlsListener::new(
829                TcpListener::bind("127.0.0.1:0").await?,
830                settings.clone(),
831                Arc::new(IgnoreTlsEvents),
832            )?;
833            let client = rustls::ClientConfig::builder_with_provider(Arc::new(provider()))
834                .with_protocol_versions(&versions)?
835                .with_root_certificates(roots.clone())
836                .with_no_client_auth();
837            let connector = tokio_rustls::TlsConnector::from(Arc::new(client));
838            let (client_io, server_io) = tokio::io::duplex(16 * 1024);
839            let (client_result, server_result) =
840                tokio::time::timeout(Duration::from_secs(5), async {
841                    tokio::join!(
842                        connector.connect("origin.test".try_into().unwrap(), client_io),
843                        listener.acceptor.accept(server_io),
844                    )
845                })
846                .await?;
847            if let Some(expected) = expected {
848                assert_eq!(
849                    client_result?.get_ref().1.protocol_version(),
850                    Some(expected)
851                );
852                assert_eq!(
853                    server_result?.get_ref().1.protocol_version(),
854                    Some(expected)
855                );
856            } else {
857                assert!(client_result.is_err());
858                assert!(server_result.is_err());
859            }
860        }
861        settings.min_version = TlsVersion::Tls13;
862        settings.max_version = TlsVersion::Tls12;
863        assert!(matches!(
864            TlsListener::new(
865                TcpListener::bind("127.0.0.1:0").await?,
866                settings,
867                Arc::new(IgnoreTlsEvents)
868            ),
869            Err(TlsError::InvalidVersionRange)
870        ));
871        std::fs::remove_dir_all(directory)?;
872        Ok(())
873    }
874
875    #[tokio::test]
876    async fn a_quic_config_is_tls13_with_early_data_and_custom_alpn() {
877        let directory = scratch("quic");
878        let (settings, _) = write_pair(&directory, "origin.test");
879
880        let (config, _watch) =
881            rotating_quic_server_config(settings, &[b"h3"], Arc::new(IgnoreTlsEvents))
882                .expect("a self-signed pair is usable for QUIC");
883
884        assert_eq!(config.alpn_protocols, vec![b"h3".to_vec()]);
885        assert_eq!(config.max_early_data_size, u32::MAX);
886    }
887
888    #[tokio::test]
889    async fn a_client_identity_presents_its_certificate_and_reloads_on_rotation() {
890        // The outbound mirror of the listener's rotation test. A client
891        // certificate expires the same way a server one does, so a node that
892        // could not renew without a restart would drop publishers on rotation
893        // day.
894        let directory = scratch("client-identity");
895        let (settings, first) = write_pair(&directory, "origin.internal");
896
897        let identity = ClientIdentity::new(
898            settings.certificate.clone(),
899            settings.key.clone(),
900            Arc::new(IgnoreTlsEvents),
901        )
902        .expect("the initial pair loads");
903
904        let presented = |identity: &ClientIdentity| {
905            ResolvesClientCert::resolve(&*identity.resolver, &[], &[])
906                .expect("an identity always resolves")
907                .cert[0]
908                .to_vec()
909        };
910        assert_eq!(presented(&identity), first);
911
912        // Rotate underneath the running identity.
913        let (_, second) = write_pair(&directory, "origin.internal");
914        assert_ne!(first, second, "the fixture generated a distinct pair");
915
916        // The watch debounces, so give it the debounce plus room to re-read.
917        for _ in 0..40 {
918            if presented(&identity) == second {
919                break;
920            }
921            tokio::time::sleep(Duration::from_millis(100)).await;
922        }
923
924        assert_eq!(
925            presented(&identity),
926            second,
927            "a rotated client certificate is picked up without a restart"
928        );
929    }
930
931    #[test]
932    fn a_client_identity_refuses_an_unreadable_pair_at_startup() {
933        // Fatal at startup, unlike a failed reload: a node configured to
934        // present an identity it cannot read should not come up pretending it
935        // will be admitted.
936        let directory = scratch("client-identity-missing");
937        assert!(matches!(
938            ClientIdentity::new(
939                directory.join("absent.pem"),
940                directory.join("absent.key"),
941                Arc::new(IgnoreTlsEvents),
942            ),
943            Err(TlsError::Unreadable { .. })
944        ));
945    }
946
947    #[test]
948    fn a_client_identity_outside_a_runtime_is_an_error_not_a_panic() {
949        // Configuration is often resolved before a runtime starts; spawning
950        // the rotation task there used to panic with "no reactor running".
951        let directory = scratch("client-identity-no-runtime");
952        let (settings, _) = write_pair(&directory, "client.internal");
953        assert!(matches!(
954            ClientIdentity::new(
955                settings.certificate,
956                settings.key,
957                Arc::new(IgnoreTlsEvents),
958            ),
959            Err(TlsError::NoRuntime { .. })
960        ));
961    }
962
963    #[test]
964    fn trust_roots_load_from_pem_and_an_empty_file_is_refused() {
965        let directory = scratch("roots");
966        let (settings, der) = write_pair(&directory, "authority.internal");
967
968        let roots = load_roots(&settings.certificate).expect("the authority loads");
969        assert_eq!(roots.len(), 1);
970        assert_eq!(roots[0].to_vec(), der);
971
972        let empty = directory.join("empty.pem");
973        write_atomically(&empty, b"");
974        assert!(
975            matches!(load_roots(&empty), Err(TlsError::Empty { .. })),
976            "a file that parses but names no authority is refused rather than \
977             leaving an empty trust store that refuses every endpoint"
978        );
979    }
980}