Skip to main content

rushls_common/
tls.rs

1//! TLS termination with certificates that rotate underneath a running origin.
2//!
3//! Two things make this more than a wrapper around [`tokio_rustls`].
4//!
5//! The first is reload. A `ServerConfig` is built once and never rebuilt;
6//! every handshake asks [`CertificateResolver`] which key to present, and that
7//! resolver is one [`ArcSwap`] deep. Rotating a certificate is therefore a
8//! single atomic store: live connections keep the key they negotiated with,
9//! the next handshake picks up the new one, and nothing is dropped or
10//! rebound. Only the certificate and key can change this way — the ALPN list
11//! and the crypto provider are fixed at bind time, because nothing rotates
12//! them on a schedule.
13//!
14//! The second is that the handshake must not happen on the accept path.
15//! [`axum::serve::Listener::accept`] is a single future the server awaits in
16//! sequence, so terminating TLS inside it would let one slow client hold up
17//! every other connection. Handshakes run as spawned tasks under a deadline
18//! and a fixed ceiling instead.
19
20use std::{
21    io,
22    net::SocketAddr,
23    path::{Path, PathBuf},
24    sync::Arc,
25    time::Duration,
26};
27
28use arc_swap::ArcSwap;
29
30use crate::accept::{AcceptErrors, AcceptHealth, Next};
31use notify::RecursiveMode;
32use notify_debouncer_full::{DebounceEventResult, Debouncer, RecommendedCache, new_debouncer};
33use rustls::{
34    ServerConfig,
35    client::ResolvesClientCert,
36    crypto::CryptoProvider,
37    pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject},
38    server::{ClientHello, ResolvesServerCert},
39    sign::CertifiedKey,
40};
41use thiserror::Error;
42use tokio::{
43    net::{TcpListener, TcpStream},
44    sync::mpsc,
45    task::JoinSet,
46    time::timeout,
47};
48use tokio_rustls::{TlsAcceptor, server::TlsStream};
49
50/// What a caller learns about certificates and handshakes.
51///
52/// A trait rather than a concrete reporter because the two applications count
53/// and log differently, and neither should have to adopt the other's
54/// observability model to reuse the reload machinery. Every method has a
55/// default, so an application that wants none of it implements nothing.
56///
57/// Implementations are called from the accept loop and from the watcher task,
58/// so they must not block.
59pub trait TlsObserver: Send + Sync + 'static {
60    /// A certificate pair was read and adopted.
61    fn certificate_loaded(&self, _certificate: &Path) {}
62    /// A rotation was seen but not adopted; the previous pair still serves.
63    fn certificate_rejected(&self, _certificate: &Path, _reason: &str) {}
64    /// Rotations will no longer be noticed, while TLS keeps working.
65    ///
66    /// Reported separately because it is otherwise invisible: nothing breaks
67    /// until the certificate expires, which may be months away.
68    fn certificate_watch_lost(&self, _reason: &str) {}
69    /// The listener could not accept connections and is retrying.
70    ///
71    /// `failures` counts attempts since the previous report, this one
72    /// included: a persistent failure is summarised every
73    /// [`REPORT_INTERVAL`](crate::accept::REPORT_INTERVAL) rather than reported
74    /// once per attempt.
75    fn accept_failed(&self, _reason: &str, _failures: u64) {}
76    /// The listening socket is unusable and the listener has stopped accepting.
77    ///
78    /// [`TlsListener::accept_tls`] also returns the error; this exists for the
79    /// `axum` listener, which has no way to.
80    fn listener_failed(&self, _reason: &str) {}
81    fn handshake_completed(&self) {}
82    fn handshake_failed(&self) {}
83}
84
85/// Discards everything, for a caller that wants no reporting.
86#[derive(Clone, Copy, Debug, Default)]
87pub struct IgnoreTlsEvents;
88
89impl TlsObserver for IgnoreTlsEvents {}
90
91/// Advertised so a browser can reach HTTP/2, which is what Apple's Low-Latency
92/// profile expects. Order is preference order: h2 first, HTTP/1.1 as the
93/// fallback for clients that cannot.
94const ALPN_PROTOCOLS: [&[u8]; 2] = [b"h2", b"http/1.1"];
95
96/// How long the watcher coalesces filesystem activity before reloading.
97///
98/// Rotation is never one atomic act — certbot writes two files, a Kubernetes
99/// secret mount swaps a symlinked directory. Reacting to the first event would
100/// reliably load a new certificate against the old key.
101const DEBOUNCE: Duration = Duration::from_secs(1);
102
103/// Supported protocol bounds for TCP TLS. QUIC always uses TLS 1.3.
104#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
105pub enum TlsVersion {
106    Tls12,
107    #[default]
108    Tls13,
109}
110
111impl std::str::FromStr for TlsVersion {
112    type Err = &'static str;
113
114    fn from_str(value: &str) -> Result<Self, Self::Err> {
115        match value {
116            "1.2" => Ok(Self::Tls12),
117            "1.3" => Ok(Self::Tls13),
118            _ => Err("TLS version must be 1.2 or 1.3"),
119        }
120    }
121}
122
123/// Where the certificate and its key live, and how patient the listener is.
124#[derive(Clone, Debug, Eq, PartialEq)]
125pub struct TlsSettings {
126    /// PEM, leaf first, followed by any intermediates.
127    pub certificate: PathBuf,
128    /// PEM, PKCS#8, PKCS#1, or SEC1.
129    pub key: PathBuf,
130    /// Lowest accepted TCP TLS version.
131    pub min_version: TlsVersion,
132    /// Highest accepted TCP TLS version. QUIC ignores these TCP bounds.
133    pub max_version: TlsVersion,
134    /// Bounds a connection that completes TCP and then stalls mid-handshake.
135    pub handshake_timeout: Duration,
136    /// Caps handshakes in flight at once, which is what stops a flood from
137    /// growing the task set without bound.
138    pub maximum_pending_handshakes: usize,
139}
140
141impl Default for TlsSettings {
142    fn default() -> Self {
143        Self {
144            certificate: PathBuf::new(),
145            key: PathBuf::new(),
146            min_version: TlsVersion::Tls13,
147            max_version: TlsVersion::Tls13,
148            handshake_timeout: Duration::from_secs(10),
149            maximum_pending_handshakes: 256,
150        }
151    }
152}
153
154#[derive(Debug, Error)]
155pub enum TlsError {
156    #[error("TLS min_version must not exceed max_version")]
157    InvalidVersionRange,
158    #[error("could not read {path}: {source}")]
159    Unreadable {
160        path: PathBuf,
161        source: rustls::pki_types::pem::Error,
162    },
163    #[error("{path} contains no certificate")]
164    Empty { path: PathBuf },
165    /// Covers a key that will not parse, a key belonging to some other
166    /// certificate, and a certificate rustls will not serve at all — an X.509
167    /// v1 certificate being the one an operator is most likely to produce by
168    /// accident, since `openssl req -x509` emits one unless asked otherwise.
169    #[error("the certificate and key are not usable: {0}")]
170    Unusable(rustls::Error),
171    #[error("TLS could not be configured: {0}")]
172    Configuration(rustls::Error),
173    #[error("could not watch {path} for certificate changes: {source}")]
174    Unwatchable {
175        path: PathBuf,
176        source: notify::Error,
177    },
178    /// Rotation runs as a Tokio task, so loading certificates needs a runtime.
179    /// An error rather than a panic: configuration is often resolved before
180    /// one exists, and that is a caller mistake worth a clear message.
181    #[error("certificate rotation needs a Tokio runtime; load {path} from inside one")]
182    NoRuntime { path: PathBuf },
183}
184
185impl From<TlsError> for io::Error {
186    fn from(error: TlsError) -> Self {
187        Self::other(error.to_string())
188    }
189}
190
191/// Answers every handshake from a slot that can be replaced underneath it.
192///
193/// Shared by both directions. A client certificate expires exactly as a server
194/// one does, so an outbound caller needs the same rotation story as a listener;
195/// giving each its own would mean one of them silently not reloading.
196#[derive(Debug)]
197pub struct CertificateResolver(ArcSwap<CertifiedKey>);
198
199impl CertificateResolver {
200    /// The leaf selected for new handshakes, including only accepted rotations.
201    pub fn leaf(&self) -> Option<CertificateDer<'static>> {
202        self.0.load().cert.first().cloned()
203    }
204}
205
206impl ResolvesServerCert for CertificateResolver {
207    fn resolve(&self, _hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
208        Some(self.0.load_full())
209    }
210}
211
212impl ResolvesClientCert for CertificateResolver {
213    fn resolve(
214        &self,
215        _root_hint_subjects: &[&[u8]],
216        _sigschemes: &[rustls::SignatureScheme],
217    ) -> Option<Arc<CertifiedKey>> {
218        Some(self.0.load_full())
219    }
220
221    fn has_certs(&self) -> bool {
222        true
223    }
224}
225
226/// Reads a certificate and key, and proves they belong together.
227///
228/// [`CertifiedKey::from_der`] parses the key with the provider and compares
229/// its `SubjectPublicKeyInfo` against the leaf certificate's, so a rotation
230/// caught half-written is rejected here rather than at the next handshake.
231fn load(settings: &TlsSettings, provider: &CryptoProvider) -> Result<CertifiedKey, TlsError> {
232    let chain = load_certificates(&settings.certificate)?;
233    let key = load_private_key(&settings.key)?;
234    CertifiedKey::from_der(chain, key, provider).map_err(TlsError::Unusable)
235}
236
237/// A client certificate and key that reload underneath a running caller.
238///
239/// The outbound mirror of [`TlsListener`], and deliberately built from the same
240/// parts. A client certificate expires on the same schedule a server one does,
241/// so a node that could not renew without a restart would drop publishers on
242/// rotation day — the failure the listener side already avoids.
243///
244/// Resolution happens per handshake through an [`ArcSwap`], so a rotation is one
245/// atomic store: connections already established keep the identity they
246/// negotiated with, and the next handshake picks up the new one.
247pub struct ClientIdentity {
248    resolver: Arc<CertificateResolver>,
249    /// Held because dropping it stops reloads.
250    _watcher: CertificateWatch,
251}
252
253impl ClientIdentity {
254    /// Loads the pair and starts watching for rotations.
255    ///
256    /// Failing at startup is fatal, unlike failing to reload later: a node
257    /// configured to present an identity it cannot read should not come up
258    /// pretending it will be admitted.
259    pub fn new<O: TlsObserver>(
260        certificate: PathBuf,
261        key: PathBuf,
262        observer: Arc<O>,
263    ) -> Result<Self, TlsError> {
264        // The watch is expressed in terms of a settings pair, and the two
265        // timeout fields it also carries are meaningless here. Passing the
266        // defaults keeps one loader rather than two that could disagree about
267        // what a usable pair is.
268        let settings = TlsSettings {
269            certificate,
270            key,
271            ..TlsSettings::default()
272        };
273        let provider = Arc::new(rustls::crypto::ring::default_provider());
274        let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
275            &settings, &provider,
276        )?)));
277        observer.certificate_loaded(&settings.certificate);
278
279        let watcher = CertificateWatch::start(settings, Arc::clone(&resolver), provider, observer)?;
280        Ok(Self {
281            resolver,
282            _watcher: watcher,
283        })
284    }
285
286    /// The resolver a `rustls` client configuration should ask per handshake.
287    ///
288    /// Handed out rather than a built configuration so the caller keeps
289    /// ownership of its own root store and protocol versions, which are not
290    /// this type's business.
291    pub fn resolver(&self) -> Arc<dyn ResolvesClientCert> {
292        Arc::clone(&self.resolver) as Arc<dyn ResolvesClientCert>
293    }
294}
295
296/// Reads PEM certificates to trust, for a caller pinning its own authority.
297///
298/// Not watched. A trust root is the thing a rotation is validated *against*,
299/// so it turns over on a different and far slower schedule than the leaf it
300/// signs; treating it as hot-reloadable would suggest otherwise.
301pub fn load_roots(path: &Path) -> Result<Vec<CertificateDer<'static>>, TlsError> {
302    load_certificates(path)
303}
304
305/// Reads a nonempty PEM chain, preserving leaf-first certificate order.
306pub fn load_certificates(path: &Path) -> Result<Vec<CertificateDer<'static>>, TlsError> {
307    let roots = CertificateDer::pem_file_iter(path)
308        .and_then(std::iter::Iterator::collect::<Result<Vec<_>, _>>)
309        .map_err(|source| TlsError::Unreadable {
310            path: path.to_path_buf(),
311            source,
312        })?;
313    if roots.is_empty() {
314        return Err(TlsError::Empty {
315            path: path.to_path_buf(),
316        });
317    }
318    Ok(roots)
319}
320
321/// Reads a PKCS#8, PKCS#1, or SEC1 PEM private key.
322pub fn load_private_key(path: &Path) -> Result<PrivateKeyDer<'static>, TlsError> {
323    PrivateKeyDer::from_pem_file(path).map_err(|source| TlsError::Unreadable {
324        path: path.to_path_buf(),
325        source,
326    })
327}
328
329/// A bound TLS listener plus the watch that keeps its certificate current.
330///
331/// The watcher is owned here because dropping it silently stops reloads; tying
332/// its lifetime to the listener's means the two cannot get separated.
333pub struct TlsListener<O: TlsObserver> {
334    tcp: TcpListener,
335    acceptor: TlsAcceptor,
336    handshakes: JoinSet<Option<(TlsStream<TcpStream>, SocketAddr)>>,
337    settings: TlsSettings,
338    observer: Arc<O>,
339    errors: AcceptErrors,
340    _watcher: CertificateWatch,
341}
342
343impl<O: TlsObserver> TlsListener<O> {
344    /// Terminates TLS on an already-bound listener, loading the initial
345    /// certificate and starting the watch for rotations.
346    ///
347    /// Failing to load at startup is fatal, unlike failing to reload later: a
348    /// process that comes up without a usable certificate has nothing to serve.
349    pub fn new(
350        tcp: TcpListener,
351        settings: TlsSettings,
352        observer: Arc<O>,
353    ) -> Result<Self, TlsError> {
354        let (config, watcher) =
355            rotating_tcp_server_config(settings.clone(), &ALPN_PROTOCOLS, Arc::clone(&observer))?;
356
357        Ok(Self {
358            tcp,
359            acceptor: TlsAcceptor::from(config),
360            handshakes: JoinSet::new(),
361            settings,
362            observer,
363            errors: AcceptErrors::default(),
364            _watcher: watcher,
365        })
366    }
367
368    /// Feeds this listener's accept failures into a readiness probe.
369    #[must_use]
370    pub fn with_health(mut self, health: AcceptHealth) -> Self {
371        self.errors = AcceptErrors::with_health(health);
372        self
373    }
374
375    /// Mirrors the trait method so callers need not import the trait to ask.
376    pub fn local_addr(&self) -> io::Result<SocketAddr> {
377        self.tcp.local_addr()
378    }
379
380    /// Accepts the next connection whose TLS handshake has completed.
381    ///
382    /// Handshakes run as spawned tasks under a deadline and a fixed ceiling
383    /// rather than inline, so one slow client cannot hold up every other
384    /// connection waiting to be accepted.
385    ///
386    /// Temporary accept failures are retried with a backoff and reported
387    /// through the observer. An error is returned only when the listening
388    /// socket itself is unusable, so the caller can stop instead of retrying
389    /// forever.
390    pub async fn accept_tls(&mut self) -> io::Result<(TlsStream<TcpStream>, SocketAddr)> {
391        loop {
392            tokio::select! {
393                // Accepting stops at capacity rather than queueing: refusing to
394                // take the next connection is backpressure the kernel already
395                // knows how to apply, and it needs no semaphore to express.
396                accepted = self.tcp.accept(),
397                    if self.handshakes.len() < self.settings.maximum_pending_handshakes =>
398                {
399                    let (stream, peer) = match accepted {
400                        Ok(accepted) => {
401                            self.errors.succeeded();
402                            accepted
403                        }
404                        Err(error) => match self.errors.failed(&error) {
405                            Next::Stop => {
406                                self.observer.listener_failed(&error.to_string());
407                                return Err(error);
408                            }
409                            // Retrying at once would spin a core while
410                            // descriptors stay exhausted.
411                            Next::Retry { pause, report } => {
412                                if let Some(failures) = report {
413                                    self.observer.accept_failed(&error.to_string(), failures);
414                                }
415                                tokio::time::sleep(pause).await;
416                                continue;
417                            }
418                        },
419                    };
420                    let acceptor = self.acceptor.clone();
421                    let deadline = self.settings.handshake_timeout;
422                    let observer = Arc::clone(&self.observer);
423                    self.handshakes.spawn(async move {
424                        if let Ok(Ok(stream)) = timeout(deadline, acceptor.accept(stream)).await {
425                            observer.handshake_completed();
426                            Some((stream, peer))
427                        } else {
428                            // Neither outcome is reported as an event: the rate
429                            // is whatever a remote peer chooses to make it.
430                            observer.handshake_failed();
431                            None
432                        }
433                    });
434                }
435
436                Some(joined) = self.handshakes.join_next(), if !self.handshakes.is_empty() => {
437                    if let Ok(Some(ready)) = joined {
438                        return Ok(ready);
439                    }
440                }
441            }
442        }
443    }
444}
445
446/// Lets an `axum` server accept from this listener.
447///
448/// Behind a feature because the reload machinery has nothing to do with axum -
449/// non-HTTP callers use [`TlsListener::accept_tls`] directly without
450/// compiling a web framework to get certificate rotation.
451#[cfg(feature = "axum")]
452impl<O: TlsObserver> axum::serve::Listener for TlsListener<O> {
453    type Io = TlsStream<TcpStream>;
454    type Addr = SocketAddr;
455
456    /// `axum` cannot be told a listener has failed, so a fatal error parks
457    /// this listener after [`TlsObserver::listener_failed`]. A caller that must
458    /// stop on it should accept with [`TlsListener::accept_tls`] instead.
459    async fn accept(&mut self) -> (Self::Io, Self::Addr) {
460        match self.accept_tls().await {
461            Ok(accepted) => accepted,
462            Err(_) => std::future::pending().await,
463        }
464    }
465
466    fn local_addr(&self) -> io::Result<Self::Addr> {
467        self.tcp.local_addr()
468    }
469}
470
471/// Keeps the debouncer and its pump alive for as long as its owner lives.
472///
473/// Public so an outbound client can hold one too: dropping it silently stops
474/// reloads, so its lifetime has to be tied to whatever depends on the material
475/// staying current.
476pub struct CertificateWatch {
477    resolver: Arc<CertificateResolver>,
478    _debouncer: Debouncer<notify::RecommendedWatcher, RecommendedCache>,
479    pump: tokio::task::JoinHandle<()>,
480}
481
482impl Drop for CertificateWatch {
483    fn drop(&mut self) {
484        self.pump.abort();
485    }
486}
487
488impl CertificateWatch {
489    /// Shares the listener's active certificate without reopening its files.
490    pub fn resolver(&self) -> Arc<CertificateResolver> {
491        Arc::clone(&self.resolver)
492    }
493
494    pub fn start<O: TlsObserver>(
495        settings: TlsSettings,
496        resolver: Arc<CertificateResolver>,
497        provider: Arc<CryptoProvider>,
498        observer: Arc<O>,
499    ) -> Result<Self, TlsError> {
500        // Checked before watching anything, so a refusal leaves nothing behind.
501        let runtime = tokio::runtime::Handle::try_current().map_err(|_| TlsError::NoRuntime {
502            path: settings.certificate.clone(),
503        })?;
504        let (sender, receiver) = mpsc::unbounded_channel();
505        let mut debouncer = new_debouncer(DEBOUNCE, None, move |result: DebounceEventResult| {
506            // The receiver going away means the listener is gone; there is
507            // nothing left to reload for.
508            let _ = sender.send(result);
509        })
510        .map_err(|source| TlsError::Unwatchable {
511            path: settings.certificate.clone(),
512            source,
513        })?;
514
515        // Watching the containing directories rather than the files. Rotation
516        // replaces rather than rewrites — certbot renames into place, a
517        // Kubernetes secret mount swaps a `..data` symlink — and a watch on
518        // the file itself is left pointing at an inode nobody will touch
519        // again.
520        for directory in watched_directories(&settings) {
521            debouncer
522                .watch(&directory, RecursiveMode::NonRecursive)
523                .map_err(|source| TlsError::Unwatchable {
524                    path: directory.clone(),
525                    source,
526                })?;
527        }
528
529        let pump = runtime.spawn(reload_on_change(
530            settings,
531            Arc::clone(&resolver),
532            provider,
533            observer,
534            receiver,
535        ));
536        Ok(Self {
537            resolver,
538            _debouncer: debouncer,
539            pump,
540        })
541    }
542}
543
544/// A rustls server configuration for TCP, with a rotating certificate.
545///
546/// For callers that run their own accept loop and handshake inside it, such
547/// as an ingest listener that must reserve capacity and establish the client
548/// address before spending a TLS handshake on it. [`TlsListener`] is built on
549/// the same configuration. An empty `alpn` negotiates no application protocol,
550/// which is what protocols predating ALPN expect.
551///
552/// The returned [`CertificateWatch`] must be held for as long as the config is
553/// used; dropping it silently stops reloads.
554pub fn rotating_tcp_server_config<O: TlsObserver>(
555    settings: TlsSettings,
556    alpn: &[&[u8]],
557    observer: Arc<O>,
558) -> Result<(Arc<ServerConfig>, CertificateWatch), TlsError> {
559    let versions: &[&rustls::SupportedProtocolVersion] =
560        match (settings.min_version, settings.max_version) {
561            (TlsVersion::Tls12, TlsVersion::Tls12) => &[&rustls::version::TLS12],
562            (TlsVersion::Tls12, TlsVersion::Tls13) => {
563                &[&rustls::version::TLS13, &rustls::version::TLS12]
564            }
565            (TlsVersion::Tls13, TlsVersion::Tls13) => &[&rustls::version::TLS13],
566            (TlsVersion::Tls13, TlsVersion::Tls12) => {
567                return Err(TlsError::InvalidVersionRange);
568            }
569        };
570    // Carried explicitly rather than through `CryptoProvider::install_default`
571    // so nothing here depends on a process-global having been set first.
572    let provider = Arc::new(rustls::crypto::ring::default_provider());
573    let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
574        &settings, &provider,
575    )?)));
576    observer.certificate_loaded(&settings.certificate);
577
578    let mut config = ServerConfig::builder_with_provider(Arc::clone(&provider))
579        .with_protocol_versions(versions)
580        .map_err(TlsError::Configuration)?
581        .with_no_client_auth()
582        .with_cert_resolver(Arc::clone(&resolver) as Arc<dyn ResolvesServerCert>);
583    config.alpn_protocols = alpn.iter().map(|name| name.to_vec()).collect();
584
585    let watcher = CertificateWatch::start(settings, resolver, provider, observer)?;
586    Ok((Arc::new(config), watcher))
587}
588
589/// A rustls server configuration for QUIC, with the same rotating certificate
590/// as [`TlsListener`].
591///
592/// QUIC cannot reuse an HTTPS [`ServerConfig`]: HTTP/3 requires TLS 1.3 only,
593/// a different ALPN, and `max_early_data_size = u32::MAX` so Quinn will accept
594/// the config at all. The certificate resolver is the same [`ArcSwap`] the TCP
595/// listener uses, so a rotation is still one atomic store and this config is
596/// never rebuilt.
597///
598/// The returned [`CertificateWatch`] must be held for as long as the QUIC
599/// endpoint lives; dropping it silently stops reloads.
600pub fn rotating_quic_server_config<O: TlsObserver>(
601    settings: TlsSettings,
602    alpn: &[&[u8]],
603    observer: Arc<O>,
604) -> Result<(Arc<ServerConfig>, CertificateWatch), TlsError> {
605    let provider = Arc::new(rustls::crypto::ring::default_provider());
606    let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
607        &settings, &provider,
608    )?)));
609    observer.certificate_loaded(&settings.certificate);
610
611    let mut config = ServerConfig::builder_with_provider(Arc::clone(&provider))
612        .with_protocol_versions(&[&rustls::version::TLS13])
613        .map_err(TlsError::Configuration)?
614        .with_no_client_auth()
615        .with_cert_resolver(Arc::clone(&resolver) as Arc<dyn ResolvesServerCert>);
616    config.alpn_protocols = alpn.iter().map(|name| name.to_vec()).collect();
617    // Quinn's `QuicServerConfig::try_from` refuses a rustls config that does
618    // not advertise 0-RTT. The advertised size is the QUIC maximum; whether a
619    // session actually accepts early data is a transport decision, not a TLS one.
620    config.max_early_data_size = u32::MAX;
621
622    let watcher = CertificateWatch::start(settings, resolver, provider, observer)?;
623    Ok((Arc::new(config), watcher))
624}
625
626/// The directories holding the pair, deduplicated when both files share one.
627fn watched_directories(settings: &TlsSettings) -> Vec<PathBuf> {
628    let mut directories = Vec::with_capacity(2);
629    for path in [&settings.certificate, &settings.key] {
630        let directory = path.parent().unwrap_or(Path::new(".")).to_path_buf();
631        if !directories.contains(&directory) {
632            directories.push(directory);
633        }
634    }
635    directories
636}
637
638async fn reload_on_change<O: TlsObserver>(
639    settings: TlsSettings,
640    resolver: Arc<CertificateResolver>,
641    provider: Arc<CryptoProvider>,
642    observer: Arc<O>,
643    mut changes: mpsc::UnboundedReceiver<DebounceEventResult>,
644) {
645    while let Some(result) = changes.recv().await {
646        // Any activity in the directory is a reason to re-read, without
647        // inspecting which paths the events name.
648        //
649        // Filtering on the configured paths is the obvious design and it is
650        // wrong. A watcher reports the path the platform resolved, and a
651        // Kubernetes secret mount never resolves to the name we were given:
652        // `tls.crt` is a symlink into `..data`, itself a symlink to a
653        // timestamped directory, and rotation swaps the *symlink* — so every
654        // event names `..data` or a directory we have never heard of, and a
655        // path filter discards all of them. The certificate would then never
656        // reload, in exactly the deployment that needs it most, and the
657        // symptom would be nothing happening at all.
658        //
659        // Re-reading unconditionally costs two file reads on a dedicated
660        // secret mount, already coalesced by the debounce, and the comparison
661        // below keeps a spurious wake from being mistaken for a rotation.
662        if let Err(errors) = result {
663            let reason = errors
664                .iter()
665                .map(ToString::to_string)
666                .collect::<Vec<_>>()
667                .join("; ");
668            observer.certificate_watch_lost(&reason);
669            continue;
670        }
671
672        // Blocking reads, moved off the runtime because a rotation on a slow
673        // or network-backed volume would otherwise stall a worker thread.
674        let settings_for_load = settings.clone();
675        let provider_for_load = Arc::clone(&provider);
676        let loaded =
677            tokio::task::spawn_blocking(move || load(&settings_for_load, &provider_for_load)).await;
678
679        match loaded {
680            Ok(Ok(certified)) => {
681                // Unchanged material is the common case now that any activity
682                // wakes this loop. Swapping anyway would be harmless, but
683                // reporting it would turn the event stream — the thing an
684                // operator reads to answer "did the rotation take?" — into
685                // noise.
686                if certified.cert == resolver.0.load().cert {
687                    continue;
688                }
689                resolver.0.store(Arc::new(certified));
690                observer.certificate_loaded(&settings.certificate);
691            }
692            // The previously loaded pair stays in place. A rotation that
693            // produced an unusable file is a reason to keep serving, not to
694            // start failing every handshake.
695            Ok(Err(error)) => {
696                observer.certificate_rejected(&settings.certificate, &error.to_string())
697            }
698            Err(error) => observer.certificate_watch_lost(&error.to_string()),
699        }
700    }
701
702    // The channel closes when the debouncer is dropped, which happens with the
703    // listener. Reaching here during a shutdown is ordinary; reaching it while
704    // still serving means rotations are no longer observed.
705    observer.certificate_watch_lost("the certificate watch ended");
706}
707
708#[cfg(test)]
709mod tests {
710    use super::*;
711    use std::sync::atomic::{AtomicU64, Ordering};
712
713    /// A private directory for one test, removed if a previous run left it behind.
714    fn scratch(name: &str) -> PathBuf {
715        static NEXT: AtomicU64 = AtomicU64::new(0);
716        let unique = NEXT.fetch_add(1, Ordering::Relaxed);
717        let directory =
718            std::env::temp_dir().join(format!("rushls-tls-{name}-{}-{unique}", std::process::id()));
719        let _ = std::fs::remove_dir_all(&directory);
720        std::fs::create_dir_all(&directory).expect("the scratch directory is created");
721        directory
722    }
723
724    /// A self-signed pair written into `directory`, plus the certificate's DER.
725    fn write_pair(directory: &Path, name: &str) -> (TlsSettings, Vec<u8>) {
726        let certified = rcgen::generate_simple_self_signed([name.to_owned()])
727            .expect("a self-signed certificate is generated");
728        let settings = TlsSettings {
729            certificate: directory.join("certificate.pem"),
730            key: directory.join("key.pem"),
731            ..TlsSettings::default()
732        };
733        write_atomically(&settings.certificate, certified.cert.pem().as_bytes());
734        write_atomically(
735            &settings.key,
736            certified.signing_key.serialize_pem().as_bytes(),
737        );
738        (settings, certified.cert.der().to_vec())
739    }
740
741    /// Writes through a rename, the way real rotation tooling does.
742    ///
743    /// Writing in place would let a watcher observe a half-written file, which
744    /// is the very race the debounce exists to absorb.
745    fn write_atomically(path: &Path, contents: &[u8]) {
746        let staged = path.with_extension("staged");
747        std::fs::write(&staged, contents).expect("the staged file is written");
748        std::fs::rename(&staged, path).expect("the staged file is renamed into place");
749    }
750
751    fn provider() -> CryptoProvider {
752        rustls::crypto::ring::default_provider()
753    }
754
755    #[test]
756    fn a_matching_pair_loads() {
757        let directory = scratch("matching");
758        let (settings, _) = write_pair(&directory, "origin.test");
759
760        load(&settings, &provider()).expect("a self-signed pair is usable");
761    }
762
763    #[test]
764    fn a_key_from_a_different_certificate_is_refused() {
765        let directory = scratch("mismatched");
766        let (settings, _) = write_pair(&directory, "origin.test");
767        let other = scratch("mismatched-other");
768        let (foreign, _) = write_pair(&other, "other.test");
769        std::fs::copy(&foreign.key, &settings.key).expect("the foreign key replaces the real one");
770
771        assert!(matches!(
772            load(&settings, &provider()),
773            Err(TlsError::Unusable(_))
774        ));
775    }
776
777    #[test]
778    fn a_truncated_certificate_is_refused_rather_than_loaded_empty() {
779        let directory = scratch("truncated");
780        let (settings, _) = write_pair(&directory, "origin.test");
781        std::fs::write(&settings.certificate, b"").expect("the certificate is truncated");
782
783        assert!(matches!(
784            load(&settings, &provider()),
785            Err(TlsError::Empty { .. })
786        ));
787    }
788
789    #[test]
790    fn a_pair_sharing_one_directory_is_watched_once() {
791        let settings = TlsSettings {
792            certificate: PathBuf::from("/etc/tls/fullchain.pem"),
793            key: PathBuf::from("/etc/tls/privkey.pem"),
794            ..TlsSettings::default()
795        };
796
797        assert_eq!(watched_directories(&settings), [PathBuf::from("/etc/tls")]);
798    }
799
800    #[tokio::test]
801    async fn tls_version_bounds_control_negotiation() -> Result<(), Box<dyn std::error::Error>> {
802        let directory = scratch("protocol-versions");
803        let (mut settings, der) = write_pair(&directory, "origin.test");
804        let mut roots = rustls::RootCertStore::empty();
805        roots.add(CertificateDer::from(der))?;
806        for (min_version, max_version, versions, expected) in [
807            (
808                TlsVersion::Tls13,
809                TlsVersion::Tls13,
810                vec![&rustls::version::TLS12],
811                None,
812            ),
813            (
814                TlsVersion::Tls13,
815                TlsVersion::Tls13,
816                vec![&rustls::version::TLS13],
817                Some(rustls::ProtocolVersion::TLSv1_3),
818            ),
819            (
820                TlsVersion::Tls12,
821                TlsVersion::Tls13,
822                vec![&rustls::version::TLS12],
823                Some(rustls::ProtocolVersion::TLSv1_2),
824            ),
825            (
826                TlsVersion::Tls12,
827                TlsVersion::Tls13,
828                vec![&rustls::version::TLS12, &rustls::version::TLS13],
829                Some(rustls::ProtocolVersion::TLSv1_3),
830            ),
831            (
832                TlsVersion::Tls12,
833                TlsVersion::Tls12,
834                vec![&rustls::version::TLS13],
835                None,
836            ),
837            (
838                TlsVersion::Tls12,
839                TlsVersion::Tls12,
840                vec![&rustls::version::TLS12, &rustls::version::TLS13],
841                Some(rustls::ProtocolVersion::TLSv1_2),
842            ),
843        ] {
844            settings.min_version = min_version;
845            settings.max_version = max_version;
846            let listener = TlsListener::new(
847                TcpListener::bind("127.0.0.1:0").await?,
848                settings.clone(),
849                Arc::new(IgnoreTlsEvents),
850            )?;
851            let client = rustls::ClientConfig::builder_with_provider(Arc::new(provider()))
852                .with_protocol_versions(&versions)?
853                .with_root_certificates(roots.clone())
854                .with_no_client_auth();
855            let connector = tokio_rustls::TlsConnector::from(Arc::new(client));
856            let (client_io, server_io) = tokio::io::duplex(16 * 1024);
857            let (client_result, server_result) =
858                tokio::time::timeout(Duration::from_secs(5), async {
859                    tokio::join!(
860                        connector.connect("origin.test".try_into().unwrap(), client_io),
861                        listener.acceptor.accept(server_io),
862                    )
863                })
864                .await?;
865            if let Some(expected) = expected {
866                assert_eq!(
867                    client_result?.get_ref().1.protocol_version(),
868                    Some(expected)
869                );
870                assert_eq!(
871                    server_result?.get_ref().1.protocol_version(),
872                    Some(expected)
873                );
874            } else {
875                assert!(client_result.is_err());
876                assert!(server_result.is_err());
877            }
878        }
879        settings.min_version = TlsVersion::Tls13;
880        settings.max_version = TlsVersion::Tls12;
881        assert!(matches!(
882            TlsListener::new(
883                TcpListener::bind("127.0.0.1:0").await?,
884                settings,
885                Arc::new(IgnoreTlsEvents)
886            ),
887            Err(TlsError::InvalidVersionRange)
888        ));
889        std::fs::remove_dir_all(directory)?;
890        Ok(())
891    }
892
893    #[tokio::test]
894    async fn a_quic_config_is_tls13_with_early_data_and_custom_alpn() {
895        let directory = scratch("quic");
896        let (settings, _) = write_pair(&directory, "origin.test");
897
898        let (config, _watch) =
899            rotating_quic_server_config(settings, &[b"h3"], Arc::new(IgnoreTlsEvents))
900                .expect("a self-signed pair is usable for QUIC");
901
902        assert_eq!(config.alpn_protocols, vec![b"h3".to_vec()]);
903        assert_eq!(config.max_early_data_size, u32::MAX);
904    }
905
906    #[tokio::test]
907    async fn a_client_identity_presents_its_certificate_and_reloads_on_rotation() {
908        // The outbound mirror of the listener's rotation test. A client
909        // certificate expires the same way a server one does, so a node that
910        // could not renew without a restart would drop publishers on rotation
911        // day.
912        let directory = scratch("client-identity");
913        let (settings, first) = write_pair(&directory, "origin.internal");
914
915        let identity = ClientIdentity::new(
916            settings.certificate.clone(),
917            settings.key.clone(),
918            Arc::new(IgnoreTlsEvents),
919        )
920        .expect("the initial pair loads");
921
922        let presented = |identity: &ClientIdentity| {
923            ResolvesClientCert::resolve(&*identity.resolver, &[], &[])
924                .expect("an identity always resolves")
925                .cert[0]
926                .to_vec()
927        };
928        assert_eq!(presented(&identity), first);
929
930        // Rotate underneath the running identity.
931        let (_, second) = write_pair(&directory, "origin.internal");
932        assert_ne!(first, second, "the fixture generated a distinct pair");
933
934        // The watch debounces, so give it the debounce plus room to re-read.
935        for _ in 0..40 {
936            if presented(&identity) == second {
937                break;
938            }
939            tokio::time::sleep(Duration::from_millis(100)).await;
940        }
941
942        assert_eq!(
943            presented(&identity),
944            second,
945            "a rotated client certificate is picked up without a restart"
946        );
947    }
948
949    #[test]
950    fn a_client_identity_refuses_an_unreadable_pair_at_startup() {
951        // Fatal at startup, unlike a failed reload: a node configured to
952        // present an identity it cannot read should not come up pretending it
953        // will be admitted.
954        let directory = scratch("client-identity-missing");
955        assert!(matches!(
956            ClientIdentity::new(
957                directory.join("absent.pem"),
958                directory.join("absent.key"),
959                Arc::new(IgnoreTlsEvents),
960            ),
961            Err(TlsError::Unreadable { .. })
962        ));
963    }
964
965    #[test]
966    fn a_client_identity_outside_a_runtime_is_an_error_not_a_panic() {
967        // Configuration is often resolved before a runtime starts; spawning
968        // the rotation task there used to panic with "no reactor running".
969        let directory = scratch("client-identity-no-runtime");
970        let (settings, _) = write_pair(&directory, "client.internal");
971        assert!(matches!(
972            ClientIdentity::new(
973                settings.certificate,
974                settings.key,
975                Arc::new(IgnoreTlsEvents),
976            ),
977            Err(TlsError::NoRuntime { .. })
978        ));
979    }
980
981    #[test]
982    fn trust_roots_load_from_pem_and_an_empty_file_is_refused() {
983        let directory = scratch("roots");
984        let (settings, der) = write_pair(&directory, "authority.internal");
985
986        let roots = load_roots(&settings.certificate).expect("the authority loads");
987        assert_eq!(roots.len(), 1);
988        assert_eq!(roots[0].to_vec(), der);
989
990        let empty = directory.join("empty.pem");
991        write_atomically(&empty, b"");
992        assert!(
993            matches!(load_roots(&empty), Err(TlsError::Empty { .. })),
994            "a file that parses but names no authority is refused rather than \
995             leaving an empty trust store that refuses every endpoint"
996        );
997    }
998}