sozu_lib/

tls.rs

//! A unified certificate resolver for rustls.
//!
//! Persists certificates in the Rustls
//! [`CertifiedKey` format](https://docs.rs/rustls/latest/rustls/sign/struct.CertifiedKey.html),
//! exposes them to the HTTPS listener for TLS handshakes.
#[cfg(test)]
use std::collections::HashSet;
use std::{
    collections::HashMap,
    fmt,
    str::FromStr,
    sync::{Arc, LazyLock, Mutex},
};

use rustls::{
    pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject},
    server::{ClientHello, ResolvesServerCert},
    sign::CertifiedKey,
};

use crate::crypto::any_supported_type;
use sha2::{Digest, Sha256};
use sozu_command::{
    certificate::{
        CertificateError, Fingerprint, get_cn_and_san_attributes, parse_pem, parse_x509,
        split_certificate_chain,
    },
    logging::ansi_palette,
    proto::command::{AddCertificate, CertificateAndKey, ReplaceCertificate, SocketAddress},
};

use crate::metrics::names;
use crate::router::MAX_HOSTNAME_LENGTH;
use crate::router::pattern_trie::{InsertResult, Key, KeyValue, TrieNode};

/// Module-level prefix used on every log line emitted from this module.
/// Produces a bold bright-white `TLS-RESOLVER` label (uniform across every
/// protocol) when the logger is in colored mode. The certificate resolver
/// runs at listener scope -- it has no per-session state -- so this is the
/// only macro the module needs. `RUSTLS` covers the protocol-side logs in
/// `lib/src/protocol/rustls.rs`; `TLS-RESOLVER` is intentionally distinct so
/// operators can tell handshake failures (RUSTLS) apart from cert-store
/// management noise (TLS-RESOLVER).
macro_rules! log_module_context {
    () => {{
        let (open, reset, _, _, _) = ansi_palette();
        format!(
            "{open}TLS-RESOLVER{reset}\t >>>",
            open = open,
            reset = reset
        )
    }};
}

// -----------------------------------------------------------------------------
// Default ParsedCertificateAndKey

static DEFAULT_CERTIFICATE: LazyLock<Option<Arc<CertifiedKey>>> = LazyLock::new(|| {
    let add = AddCertificate {
        certificate: CertificateAndKey {
            certificate: include_str!("../assets/certificate.pem").to_string(),
            certificate_chain: vec![include_str!("../assets/certificate_chain.pem").to_string()],
            key: include_str!("../assets/key.pem").to_string(),
            versions: vec![],
            names: vec![],
        },
        address: SocketAddress::new_v4(0, 0, 0, 0, 8080), // not used anyway
        expired_at: None,
    };
    CertifiedKeyWrapper::try_from(&add).ok().map(|c| c.inner)
});

#[derive(thiserror::Error, Debug)]
pub enum CertificateResolverError {
    #[error("failed to get common name and subject alternate names from pem, {0}")]
    InvalidCommonNameAndSubjectAlternateNames(CertificateError),
    #[error("invalid private key: {0}")]
    InvalidPrivateKey(String),
    #[error("empty key")]
    EmptyKeys,
    #[error("error parsing x509 cert from bytes: {0}")]
    ParseX509(CertificateError),
    #[error("error parsing pem formated certificate from bytes: {0}")]
    ParsePem(CertificateError),
    #[error("error parsing overriding names in new certificate: {0}")]
    ParseOverridingNames(CertificateError),
    #[error("the SNI route table cannot host a certificate name, name_bytes={}", .0.len())]
    InvalidName(String),
}

/// A wrapper around the Rustls
/// [`CertifiedKey` type](https://docs.rs/rustls/latest/rustls/sign/struct.CertifiedKey.html),
/// stored and returned by the certificate resolver.
#[derive(Clone)]
pub struct CertifiedKeyWrapper {
    inner: Arc<CertifiedKey>,
    /// domain names, override what can be found in the cert
    names: Vec<String>,
    expiration: i64,
    fingerprint: Fingerprint,
}

impl fmt::Debug for CertifiedKeyWrapper {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let names_len = self
            .names
            .iter()
            .map(String::len)
            .fold(0usize, usize::saturating_add);

        f.debug_struct("CertifiedKeyWrapper")
            .field("certificate", &"[redacted]")
            .field("certificate_chain_count", &self.inner.cert.len())
            .field("private_key", &"[redacted]")
            .field("names_count", &self.names.len())
            .field("names_len", &names_len)
            .field("expiration", &self.expiration)
            .field("fingerprint_bytes", &self.fingerprint.0.len())
            .finish()
    }
}

/// Convert an AddCertificate request into the Rustls format.
/// Support RSA and ECDSA certificates.
impl TryFrom<&AddCertificate> for CertifiedKeyWrapper {
    type Error = CertificateResolverError;

    fn try_from(add: &AddCertificate) -> Result<Self, Self::Error> {
        let cert = add.certificate.clone();

        let pem =
            parse_pem(cert.certificate.as_bytes()).map_err(CertificateResolverError::ParsePem)?;

        let x509 = parse_x509(&pem.contents).map_err(CertificateResolverError::ParseX509)?;

        // ASCII-lowercase every name at the single point where the set
        // is built, so the SNI trie key, the name index, the removal
        // walk and the SAN snapshot all agree by construction.
        //
        // The lookup side is already lowercase and there is nothing we
        // can do about it: rustls lowercases the SNI inside
        // `process_client_hello` (`DnsName::to_lowercase_owned`) before
        // `ResolvesServerCert::resolve` ever sees it, so a name stored
        // with uppercase — an X.509 SAN carrying it (RFC 5280 does not
        // forbid it) or an operator `names` override — was reachable by
        // no SNI at all and every handshake for it fell back to
        // `DEFAULT_CERTIFICATE`. DNS names compare case-insensitively
        // (RFC 4343), so normalising loses nothing.
        let overriding_names = if add.certificate.names.is_empty() {
            get_cn_and_san_attributes(&x509)
        } else {
            add.certificate.names.clone()
        };
        let overriding_names: Vec<String> = overriding_names
            .into_iter()
            .map(|mut name| {
                name.make_ascii_lowercase();
                name
            })
            .collect();

        let expiration = add
            .expired_at
            .unwrap_or(x509.validity().not_after.timestamp());

        let fingerprint = Fingerprint(Sha256::digest(&pem.contents).iter().cloned().collect());

        // The leaf is at index 0; chain entries follow. ACME clients
        // emitting `fullchain.pem` (Certbot default, lego, acme.sh)
        // place the leaf at the start, which would store
        // `[leaf, leaf, intermediate, root]` and fail strict
        // validators (Node.js `UNABLE_TO_VERIFY_LEAF_SIGNATURE`).
        // Each entry is split through `split_certificate_chain` so a
        // single multi-PEM string fans out (`parse_pem` would
        // otherwise stop at the first block); each split entry is
        // dedup'd against the leaf's DER bytes.
        let leaf_der = pem.contents;
        let mut chain = vec![CertificateDer::from(leaf_der.to_owned())];
        let mut dropped_duplicates = 0usize;
        for cert in &cert.certificate_chain {
            for split_pem in split_certificate_chain(cert.to_owned()) {
                let chain_link = parse_pem(split_pem.as_bytes())
                    .map_err(CertificateResolverError::ParsePem)?
                    .contents;

                if chain_link == leaf_der {
                    dropped_duplicates += 1;
                    continue;
                }
                chain.push(CertificateDer::from(chain_link));
            }
        }
        if dropped_duplicates > 0 {
            debug!(
                "{} dropped {} duplicate leaf certificate(s) from the supplied chain",
                log_module_context!(),
                dropped_duplicates
            );
        }

        // Parse the PEM-encoded private key into a `PrivateKeyDer` via
        // `rustls-pki-types`'s `PemObject` trait. `from_pem_slice` accepts
        // PKCS1 / PKCS8 / SEC1 key formats the same way the old
        // `rustls-pemfile::read_one` + per-variant `From::from` chain did,
        // and folds the empty-input / no-PEM-object / unsupported-format
        // cases into a single `Err` we surface as `EmptyKeys` (the
        // existing variant covers any failure to extract a key from the
        // supplied PEM blob).
        let private_key = PrivateKeyDer::from_pem_slice(cert.key.as_bytes())
            .map_err(|_| CertificateResolverError::EmptyKeys)?;

        // Postconditions of chain assembly: the leaf was pushed first, so the
        // chain is never empty and its head is exactly the parsed leaf DER.
        // Dedup only ever *drops* entries it recognises as the leaf, so the
        // assembled length can never exceed leaf + supplied links.
        debug_assert!(
            !chain.is_empty(),
            "assembled certificate chain must contain at least the leaf"
        );
        debug_assert_eq!(
            chain[0].as_ref(),
            leaf_der.as_slice(),
            "the leaf must remain at index 0 of the chain"
        );
        // SHA-256 fingerprint is exactly 32 bytes — anything else means the
        // digest pipeline changed underneath us.
        debug_assert_eq!(
            fingerprint.0.len(),
            32,
            "a SHA-256 fingerprint must be 32 bytes"
        );

        match any_supported_type(&private_key) {
            Ok(signing_key) => {
                let stored_certificate = CertifiedKeyWrapper {
                    inner: Arc::new(CertifiedKey::new(chain, signing_key)),
                    names: overriding_names,
                    expiration,
                    fingerprint,
                };
                Ok(stored_certificate)
            }
            Err(sign_error) => Err(CertificateResolverError::InvalidPrivateKey(
                sign_error.to_string(),
            )),
        }
    }
}

/// Parses and stores TLS certificates, makes them available to Rustls for TLS handshakes
///
/// the `domains` TrieNode is an addressing system to resolve a certificate
/// for a given domain name.
/// Certificates are stored in a hashmap that may contain unreachable certificates if
/// no domain name points to it.
/// `Clone` lets an HTTPS listener added next to a sibling on the same address
/// take over its certificates (`HttpsProxy::add_listener`, `lib/src/https.rs`).
#[derive(Default, Clone)]
pub struct CertificateResolver {
    /// routing one domain name to one certificate for fast resolving
    pub domains: TrieNode<Fingerprint>,
    /// a storage map: fingerprint -> stored_certificate
    certificates: HashMap<Fingerprint, CertifiedKeyWrapper>,
    /// maps each domain name to several compatible certificates, sorted by expiration date
    /// map of domain_name -> all fingerprints (and expiration) linked to this domain name
    //  the vector of (fingerprint, expiration) is sorted by expiration
    name_fingerprint_idx: HashMap<String, Vec<(Fingerprint, i64)>>,
}

impl fmt::Debug for CertificateResolver {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let indexed_certificate_links = self
            .name_fingerprint_idx
            .values()
            .map(Vec::len)
            .fold(0usize, usize::saturating_add);

        f.debug_struct("CertificateResolver")
            .field("domains", &"[redacted]")
            .field("certificates_count", &self.certificates.len())
            .field("indexed_names_count", &self.name_fingerprint_idx.len())
            .field("indexed_certificate_links", &indexed_certificate_links)
            .finish()
    }
}

impl CertificateResolver {
    /// return the certificate in the Rustls-usable form
    pub fn get_certificate(&self, fingerprint: &Fingerprint) -> Option<CertifiedKeyWrapper> {
        self.certificates.get(fingerprint).map(ToOwned::to_owned)
    }

    /// Recompute the aggregate `tls.cert.min_expires_at_seconds` gauge across
    /// every certificate currently loaded. Per-SNI granularity would explode
    /// statsd key cardinality (the resolver can easily hold tens of thousands
    /// of names on a public endpoint) and the existing `gauge!` macro has no
    /// label support, so we expose a single absolute unix-seconds reading of
    /// the soonest-expiring cert. Dashboards alert on this as the "next cert
    /// to rotate" deadline; operators query per-cert detail through the
    /// command API. `x509` timestamps are signed but `set_gauge` takes a
    /// `usize`, so we clamp already-expired certs to 0 (which is still a
    /// monotonic "panic now" signal to any alerting rule).
    ///
    /// Called from `add_certificate` / `remove_certificate` — i.e. only when
    /// the cert set actually changes, never on the hot TLS handshake path.
    fn publish_min_expiration_gauge(&self) {
        let Some(min_expiration) = self.certificates.values().map(|c| c.expiration).min() else {
            // SECURITY: an empty resolver is not "every cert just
            // expired"; it is "no cert
            // has been loaded yet" — typical at process boot before the
            // first AddCertificate request lands. Writing 0 here pages
            // SOC tooling on every restart with the same alert as a real
            // expired-cert event. Skip the emit so the gauge reflects the
            // last known good state instead of being clobbered to 0.
            return;
        };
        let clamped = min_expiration.max(0) as usize;
        gauge!(names::tls::CERT_MIN_EXPIRES_AT_SECONDS, clamped);
    }

    /// persist a certificate, after ensuring validity, and checking if it can replace another certificate.
    /// return the certificate fingerprint regardless of having inserted it or not
    pub fn add_certificate(
        &mut self,
        add: &AddCertificate,
    ) -> Result<Fingerprint, CertificateResolverError> {
        let cert_to_add = CertifiedKeyWrapper::try_from(add)?;

        trace!(
            "{} adding certificate {:?}",
            log_module_context!(),
            cert_to_add
        );

        if self.certificates.contains_key(&cert_to_add.fingerprint) {
            return Ok(cert_to_add.fingerprint);
        }

        // Reject a certificate whose name the SNI route table cannot host
        // BEFORE mutating any state, by dry-running every name against a
        // scratch trie (the grammar reasons are the same as in
        // `router::Router::add_tree_rule`). `TrieNode::insert` reports
        // such a name as a graceful `InsertResult::Failed`; discarding
        // that result below would register the certificate while
        // `self.domains` never learns the name -- a dead SNI route
        // failing every handshake with no diagnostic. The length bound
        // mirrors the router's: the trie recurses once per label.
        //
        // A name carrying `/` is refused on its BYTES, not on whether it
        // inserts. `self.domains` is the router's
        // `pattern_trie::TrieNode`, which compiles a label wrapped in
        // `/.../` into a REGEX segment, so such a name INSERTS happily
        // and then binds one certificate to an open-ended class of SNI
        // values: `/te.*/.example.com` was served for both
        // `test.example.com` and `tenant.example.com` (#1378). The
        // dry-run alone can never see it. `/` cannot occur in a DNS
        // name, so nothing legitimate is lost, and this is the rule
        // `config::validate_sni_pattern` already applies to every path
        // into the TCP SNI route table -- the certificate trie was the
        // one SNI table still open. Removal is unaffected: it re-inserts
        // the names a certificate already carries and never re-validates
        // them, so no certificate accepted by an earlier build becomes
        // un-removable.
        {
            let mut scratch: TrieNode<()> = TrieNode::root();
            for name in &cert_to_add.names {
                if name.len() > MAX_HOSTNAME_LENGTH
                    || name.contains('/')
                    || scratch.domain_insert(name.to_owned().into_bytes(), ())
                        == InsertResult::Failed
                {
                    return Err(CertificateResolverError::InvalidName(name.to_owned()));
                }
            }
        }

        // Past the duplicate guard the fingerprint is genuinely new, so the
        // store will grow by exactly one. Snapshot the count to assert that
        // delta below (ungated `let` — read only inside the debug_assert, so
        // the optimizer drops it in release while it still compiles).
        let certificates_before = self.certificates.len();
        let new_fingerprint = cert_to_add.fingerprint.clone();
        debug_assert!(
            !self.certificates.contains_key(&new_fingerprint),
            "add_certificate past the dedup guard must be inserting a new fingerprint"
        );

        for new_name in &cert_to_add.names {
            let fingerprints_for_this_name = self
                .name_fingerprint_idx
                .entry(new_name.to_owned())
                .or_default();

            fingerprints_for_this_name
                .push((cert_to_add.fingerprint.clone(), cert_to_add.expiration));

            // sort expiration ascending (longest-lived to the right)
            fingerprints_for_this_name.sort_by_key(|t| t.1);

            let longest_lived_cert = match fingerprints_for_this_name.last() {
                Some(cert) => cert,
                None => {
                    error!(
                        "{} no fingerprint for this name, this should not happen",
                        log_module_context!()
                    );
                    continue;
                }
            };

            // update the longest lived certificate in the TriNode
            self.domains.remove(&new_name.to_owned().into_bytes());
            let insert_result = self.domains.insert(
                new_name.to_owned().into_bytes(),
                longest_lived_cert.0.to_owned(),
            );
            // Canary in the same shape as `tcp.rs::insert_sni_route`: the
            // dry-run above already validated every name, so a `Failed`
            // here is an internal bug, not a control-plane input error.
            debug_assert_ne!(
                insert_result,
                InsertResult::Failed,
                "add_certificate's names must already be validated by its dry-run"
            );
            if insert_result == InsertResult::Failed {
                error!(
                    "{} the SNI trie rejected a certificate name despite passing \
                     the dry-run validation, name_bytes={}",
                    log_module_context!(),
                    new_name.len(),
                );
            }
        }

        self.certificates
            .insert(cert_to_add.fingerprint.to_owned(), cert_to_add.clone());
        self.publish_min_expiration_gauge();

        // Postconditions: the new fingerprint is now stored, the store grew by
        // exactly one (the dedup guard above ruled out an overwrite), and
        // every name the cert advertises now resolves to it through the index.
        debug_assert!(
            self.certificates.contains_key(&new_fingerprint),
            "add_certificate must store the new certificate"
        );
        debug_assert_eq!(
            self.certificates.len(),
            certificates_before + 1,
            "add_certificate must grow the store by exactly one"
        );
        debug_assert!(
            cert_to_add.names.iter().all(|name| {
                self.name_fingerprint_idx
                    .get(name)
                    .is_some_and(|fps| fps.iter().any(|(fp, _)| *fp == new_fingerprint))
            }),
            "every name of the added cert must be indexed to its fingerprint"
        );

        trace!("{} {:#?}", log_module_context!(), self);

        Ok(cert_to_add.fingerprint)
    }

    /// Delete a certificate from the resolver. May fail if there is no alternative for
    // a domain name
    pub fn remove_certificate(
        &mut self,
        fingerprint: &Fingerprint,
    ) -> Result<(), CertificateResolverError> {
        // Snapshot the store size so the postcondition can assert that a
        // present cert drops the count by exactly one and an absent one is a
        // no-op. Ungated `let`: read only inside the debug_asserts below, so
        // it compiles in release and the optimizer drops it.
        let certificates_before = self.certificates.len();
        let was_present = self.certificates.contains_key(fingerprint);

        if let Some(certificate_to_remove) = self.get_certificate(fingerprint) {
            // Names snapshot used only by the index-cleanup postcondition.
            // Gate BOTH the `let` and its assert with `#[cfg(debug_assertions)]`
            // so the clone never runs (and never warns) in release.
            #[cfg(debug_assertions)]
            let removed_names = certificate_to_remove.names.clone();
            for name in certificate_to_remove.names {
                self.domains.domain_remove(&name.as_bytes().to_vec());

                if let std::collections::hash_map::Entry::Occupied(mut entry) =
                    self.name_fingerprint_idx.entry(name.to_owned())
                {
                    // remove fingerprints from the index for this name
                    entry.get_mut().retain(|t| &t.0 != fingerprint);

                    // reinsert the longest lived certificate in the TrieNode
                    if let Some(longest_lived_cert) = entry.get().last() {
                        let insert_result = self
                            .domains
                            .insert(name.as_bytes().to_vec(), longest_lived_cert.0.to_owned());
                        // Same canary as in `add_certificate`: this name
                        // was dry-run validated when its certificate was
                        // added, so `Failed` cannot be an input error.
                        debug_assert_ne!(
                            insert_result,
                            InsertResult::Failed,
                            "remove_certificate re-inserts names that were validated on add"
                        );
                        if insert_result == InsertResult::Failed {
                            error!(
                                "{} the SNI trie rejected a re-inserted certificate name \
                                 that was validated on add, name_bytes={}",
                                log_module_context!(),
                                name.len(),
                            );
                        }
                    }

                    // clean up empty index entries to avoid memory leaks
                    if entry.get().is_empty() {
                        entry.remove();
                    }
                }
            }

            self.certificates.remove(fingerprint);
            self.publish_min_expiration_gauge();

            // Postconditions on the present-cert path: the fingerprint is
            // truly gone, the store shrank by exactly one, and no name still
            // points at the removed fingerprint in the index (a leftover would
            // let `resolve` hand back a fingerprint with no backing cert).
            debug_assert!(
                !self.certificates.contains_key(fingerprint),
                "remove_certificate must evict the fingerprint"
            );
            debug_assert_eq!(
                self.certificates.len(),
                certificates_before - 1,
                "removing a present cert must shrink the store by exactly one"
            );
            #[cfg(debug_assertions)]
            debug_assert!(
                removed_names.iter().all(|name| {
                    self.name_fingerprint_idx
                        .get(name)
                        .is_none_or(|fps| fps.iter().all(|(fp, _)| fp != fingerprint))
                }),
                "no name may still index the removed fingerprint"
            );
        } else {
            // Absent-cert path is a pure no-op: the store size is unchanged.
            debug_assert!(
                !was_present,
                "the absent-cert branch must only run when the fingerprint was not stored"
            );
            debug_assert_eq!(
                self.certificates.len(),
                certificates_before,
                "removing an absent cert must not change the store"
            );
        }
        trace!("{} {:#?}", log_module_context!(), self);

        Ok(())
    }

    /// Add the new certificate first, then remove the old one.
    /// This ordering ensures that the old certificate remains available
    /// if adding the new one fails.
    pub fn replace_certificate(
        &mut self,
        replace: &ReplaceCertificate,
    ) -> Result<Fingerprint, CertificateResolverError> {
        let add = AddCertificate {
            address: replace.address.to_owned(),
            certificate: replace.new_certificate.to_owned(),
            expired_at: replace.new_expired_at.to_owned(),
        };

        // ── Idempotent-replace short-circuit ──
        //
        // Compute the new fingerprint *before* mutating the resolver so we
        // can compare it with the old one. When `add_certificate` is
        // called with a fingerprint that already exists it early-returns
        // (lib/src/tls.rs add_certificate path) without inserting; if we
        // then unconditionally called `remove_certificate(old)` and old
        // equalled new, we would delete the entry the caller intended to
        // *retain*. An idempotent renewal — typical for retry loops, dead
        // ACME polls, or operator-driven `ReplaceCertificate` requests
        // that resubmit the same PEM — must therefore short-circuit here.
        // Any failure to materialise the wrapper (parse, sign-key check)
        // surfaces as `CertificateResolverError`, identical to the path
        // through `add_certificate`.
        let new_cert = CertifiedKeyWrapper::try_from(&add)?;
        let new_fingerprint = new_cert.fingerprint.to_owned();

        if let Ok(old_fingerprint) = Fingerprint::from_str(&replace.old_fingerprint)
            && old_fingerprint == new_fingerprint
        {
            // Idempotent replace: the new cert is byte-identical to the
            // one already serving this name. Removing `old == new` would
            // delete the entry the caller meant to keep, so we must NOT
            // touch the store — assert it is untouched on this path.
            let stored_before = self.certificates.contains_key(&new_fingerprint);
            // Re-publish the expiration gauge so dashboards observe the
            // replace request even when the certificate set is unchanged.
            self.publish_min_expiration_gauge();
            debug_assert_eq!(
                self.certificates.contains_key(&new_fingerprint),
                stored_before,
                "idempotent replace must not change whether the cert is stored"
            );
            return Ok(new_fingerprint);
        }

        let new_fingerprint = self.add_certificate(&add)?;

        // After a non-idempotent add the new certificate is in the store,
        // ready to serve handshakes before the old one is torn down (the
        // add-before-remove ordering that keeps the name continuously
        // resolvable).
        debug_assert!(
            self.certificates.contains_key(&new_fingerprint),
            "the replacement certificate must be stored before the old one is removed"
        );

        match Fingerprint::from_str(&replace.old_fingerprint) {
            Ok(old_fingerprint) => self.remove_certificate(&old_fingerprint)?,
            Err(err) => {
                // The new certificate was already added above. If we can't parse the old
                // fingerprint, the old certificate remains in the resolver (leaked).
                // We return Ok to indicate the new certificate is available, but warn
                // that cleanup of the old one failed.
                warn!(
                    "{} new certificate added but could not remove old one: \
                     failed to parse old fingerprint, {}",
                    log_module_context!(),
                    err
                );
            }
        }

        Ok(new_fingerprint)
    }

    /// return all fingerprints that are available for these domain names,
    /// provided at least one name is given
    #[cfg(test)]
    fn find_certificates_by_names(
        &self,
        names: &HashSet<String>,
    ) -> Result<HashSet<Fingerprint>, CertificateResolverError> {
        let mut fingerprints = HashSet::new();
        for name in names {
            if let Some(fprints) = self.name_fingerprint_idx.get(name) {
                fprints.iter().for_each(|fingerprint| {
                    fingerprints.insert(fingerprint.to_owned().0);
                });
            }
        }

        Ok(fingerprints)
    }

    /// return the hashset of subjects that the certificate is able to handle.
    /// the certificate must be already persisted for this check
    #[cfg(test)]
    fn certificate_names(
        &self,
        fingerprint: &Fingerprint,
    ) -> Result<HashSet<String>, CertificateResolverError> {
        if let Some(cert) = self.certificates.get(fingerprint) {
            return Ok(cert.names.iter().cloned().collect());
        }
        Ok(HashSet::new())
    }

    pub fn domain_lookup(
        &self,
        domain: &[u8],
        accept_wildcard: bool,
    ) -> Option<&KeyValue<Key, Fingerprint>> {
        self.domains.domain_lookup(domain, accept_wildcard)
    }

    /// Resolve the SAN set Sōzu would serve for `domain` (the same trie
    /// lookup rustls uses, wildcard-aware via `domain_lookup`). Returns the
    /// certificate's `names` exactly as stored — wildcards retain their
    /// leading `*.` so the caller can apply RFC 6125 §6.4.3 matching. `None`
    /// when no cert covers `domain` (rustls would fall back to
    /// `DEFAULT_CERTIFICATE`).
    ///
    /// Mirrors `MutexCertificateResolver::resolve` minus the rustls glue, so
    /// the SAN snapshot taken at handshake matches the certificate the peer
    /// actually validated (RFC 7540 §9.1.1 / RFC 9113 §9.1.1 connection
    /// reuse).
    pub fn names_for_sni(&self, domain: &[u8]) -> Option<Vec<String>> {
        let (_, fingerprint) = self.domain_lookup(domain, true)?;
        self.certificates
            .get(fingerprint)
            .map(|cert| cert.names.clone())
    }
}

// -----------------------------------------------------------------------------
// MutexWrappedCertificateResolver struct

#[derive(Default)]
pub struct MutexCertificateResolver(pub Mutex<CertificateResolver>);

impl ResolvesServerCert for MutexCertificateResolver {
    fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
        let server_name = client_hello.server_name();
        let sigschemes = client_hello.signature_schemes();

        let Some(name) = server_name else {
            error!(
                "{} cannot look up certificate: no SNI from session",
                log_module_context!()
            );
            return None;
        };
        trace!(
            "{} trying to resolve certificate with name_bytes={} signature_schemes_count={}",
            log_module_context!(),
            name.len(),
            sigschemes.len()
        );
        // Every other site uses blocking `lock()`, and silently falling
        // back to `DEFAULT_CERTIFICATE` on lock
        // contention is an attacker-detectable mismatch (different
        // chain → different fingerprint) and a footgun the moment
        // multi-threading enters the worker. Block here. Lock-poisoning
        // (panic-while-holding) is mapped to the same default-cert
        // fallback the previous `try_lock` Err arm produced — preserves
        // the existing observable behaviour for that one corner case
        // without inventing a new failure mode.
        let resolver = match self.0.lock() {
            Ok(guard) => guard,
            Err(_poisoned) => {
                error!(
                    "{} cert resolver mutex poisoned, returning default cert",
                    log_module_context!()
                );
                return DEFAULT_CERTIFICATE.clone();
            }
        };
        if let Some((_, fingerprint)) = resolver.domains.domain_lookup(name.as_bytes(), true) {
            trace!(
                "{} looking for certificate with name_bytes={} fingerprint_bytes={}",
                log_module_context!(),
                name.len(),
                fingerprint.0.len()
            );

            // Strict-binding invariant: when the SNI trie resolves a name to a
            // fingerprint, the served cert is exactly the one stored under
            // that fingerprint — never a substitute. This guards cert
            // SELECTION (our own store consistency), not the peer-supplied SNI
            // itself, so a `debug_assert` is correct here (a violation is a
            // resolver bug, not hostile traffic). The cert may legitimately be
            // `None` if the trie still indexes a fingerprint whose cert was
            // concurrently removed; we only assert the positive case.
            let cert = resolver
                .certificates
                .get(fingerprint)
                .map(|cert| cert.inner.clone());
            debug_assert!(
                cert.is_none()
                    || resolver
                        .certificates
                        .get(fingerprint)
                        .is_some_and(|stored| Arc::ptr_eq(&stored.inner, cert.as_ref().unwrap())),
                "resolved certificate must be the one stored under the looked-up fingerprint"
            );

            trace!(
                "{} certificate lookup fingerprint_bytes={} found={}",
                log_module_context!(),
                fingerprint.0.len(),
                cert.is_some()
            );
            return cert;
        }
        drop(resolver);

        // error!("could not look up a certificate for server name '{}'", name);
        // This certificate is used for TLS tunneling with another TLS termination endpoint
        // Note that this is unsafe and you should provide a valid certificate
        debug!(
            "{} default certificate is used for name_bytes={}",
            log_module_context!(),
            name.len()
        );
        incr!(names::tls::DEFAULT_CERT_USED);
        DEFAULT_CERTIFICATE.clone()
    }
}

impl MutexCertificateResolver {
    /// Snapshot of the SAN set Sōzu would serve for `domain`. Acquires the
    /// resolver lock once. Returns `None` when the underlying mutex is
    /// poisoned — the caller is expected to treat poison the same as
    /// "default cert served" (legacy fallback), mirroring `resolve`'s
    /// own poison handling at the rustls hot path.
    pub fn names_for_sni(&self, domain: &[u8]) -> Option<Vec<String>> {
        match self.0.lock() {
            Ok(guard) => guard.names_for_sni(domain),
            Err(_poisoned) => {
                error!(
                    "{} cert resolver mutex poisoned, treating as no SAN match",
                    log_module_context!()
                );
                None
            }
        }
    }
}

impl fmt::Debug for MutexCertificateResolver {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str("MutexWrappedCertificateResolver")
    }
}

// -----------------------------------------------------------------------------
// Unit tests

#[cfg(test)]
mod tests {
    use std::{
        collections::HashSet,
        error::Error,
        io::Cursor,
        sync::Arc,
        time::{Duration, SystemTime},
    };

    // use rand::{seq::SliceRandom, thread_rng};
    use sozu_command::proto::command::{
        AddCertificate, CertificateAndKey, ReplaceCertificate, SocketAddress,
    };

    use super::{
        CertificateResolver, CertificateResolverError, CertifiedKeyWrapper,
        MutexCertificateResolver,
    };

    fn drive_client_hello(resolver: Arc<MutexCertificateResolver>, server_name: String) {
        let provider = Arc::new(crate::crypto::default_provider());
        let server_config = rustls::ServerConfig::builder_with_provider(provider.clone())
            .with_protocol_versions(&[&rustls::version::TLS13])
            .expect("test server provider must support TLS 1.3")
            .with_no_client_auth()
            .with_cert_resolver(resolver);
        let client_config = rustls::ClientConfig::builder_with_provider(provider)
            .with_protocol_versions(&[&rustls::version::TLS13])
            .expect("test client provider must support TLS 1.3")
            .with_root_certificates(rustls::RootCertStore::empty())
            .with_no_client_auth();
        let server_name = rustls::pki_types::ServerName::try_from(server_name)
            .expect("test SNI must be a valid DNS name");
        let mut client = rustls::ClientConnection::new(Arc::new(client_config), server_name)
            .expect("test client connection must initialize");
        let mut server = rustls::ServerConnection::new(Arc::new(server_config))
            .expect("test server connection must initialize");

        let mut client_hello = Vec::new();
        client
            .write_tls(&mut client_hello)
            .expect("test client hello must serialize");
        server
            .read_tls(&mut Cursor::new(client_hello))
            .expect("test server must read the client hello");
        server
            .process_new_packets()
            .expect("test server must process the client hello");
    }

    /// A certificate carrying a name the SNI route table cannot host must
    /// be rejected as a whole, BEFORE any resolver state is touched. On
    /// `main` such a name panicked inside `TrieNode::insert`; once the
    /// insert reports `InsertResult::Failed` gracefully, silently
    /// discarding the result would register the certificate while
    /// `domains` never learns the name -- a dead SNI route failing every
    /// handshake with no diagnostic on either side.
    #[test]
    fn add_certificate_rejects_a_name_the_route_table_cannot_host() {
        for name in ["sni-secret.example/", ".sni-secret.example"] {
            let mut resolver = CertificateResolver::default();
            let result = resolver.add_certificate(&AddCertificate {
                address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
                certificate: CertificateAndKey {
                    certificate: include_str!("../assets/certificate.pem").to_owned(),
                    key: include_str!("../assets/key.pem").to_owned(),
                    names: vec![(*name).to_owned()],
                    ..Default::default()
                },
                expired_at: None,
            });
            assert!(result.is_err(), "{name:?} must be rejected, not stored");
            assert!(
                resolver.certificates.is_empty(),
                "{name:?} was rejected but the certificate store was mutated",
            );
            assert!(
                resolver.name_fingerprint_idx.is_empty(),
                "{name:?} was rejected but the name index was mutated",
            );
            assert!(
                resolver.domains.is_empty(),
                "{name:?} was rejected but the SNI trie was mutated",
            );
        }
    }

    // Asserts on `trace!` output: gate on the same condition that compiles it in.
    #[cfg(any(debug_assertions, feature = "logs-trace"))]
    #[test]
    fn certificate_resolver_logs_redact_matching_sni_and_fingerprint() {
        const SNI_SECRET: &str = "resolver-sni-secret.example";

        let certificate = CertificateAndKey {
            certificate: include_str!("../assets/certificate.pem").to_owned(),
            key: include_str!("../assets/key.pem").to_owned(),
            names: vec![SNI_SECRET.to_owned()],
            ..Default::default()
        };
        let fingerprint = certificate
            .fingerprint()
            .expect("test certificate fingerprint must be computable")
            .to_string();
        let output = crate::capture_test_logs_at_level("trace", move || {
            let resolver = Arc::new(MutexCertificateResolver::default());
            resolver
                .0
                .lock()
                .expect("test resolver lock must be available")
                .add_certificate(&AddCertificate {
                    address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
                    certificate,
                    expired_at: None,
                })
                .expect("test certificate must load into the resolver");
            drive_client_hello(resolver, SNI_SECRET.to_owned());
        });

        assert!(
            !output.contains(SNI_SECRET),
            "TLS resolver logs leaked the matching SNI: {output}"
        );
        assert!(
            !output.contains(&fingerprint),
            "TLS resolver logs leaked the matching certificate fingerprint: {output}"
        );
        for metadata in [
            format!("name_bytes={}", SNI_SECRET.len()),
            "fingerprint_bytes=32".to_owned(),
        ] {
            assert!(
                output.contains(&metadata),
                "TLS resolver logs omitted bounded metadata {metadata}: {output}"
            );
        }
        assert!(
            output.len() <= 4096,
            "TLS resolver log capture is not bounded: {} bytes",
            output.len()
        );
    }

    // Asserts on `trace!` output: gate on the same condition that compiles it in.
    #[cfg(any(debug_assertions, feature = "logs-trace"))]
    #[test]
    fn certificate_resolver_logs_redact_fallback_sni() {
        const SNI_SECRET: &str = "fallback-sni-secret.example";

        let output = crate::capture_test_logs_at_level("trace", move || {
            drive_client_hello(
                Arc::new(MutexCertificateResolver::default()),
                SNI_SECRET.to_owned(),
            );
        });

        assert!(
            !output.contains(SNI_SECRET),
            "TLS resolver logs leaked the fallback SNI: {output}"
        );
        assert!(
            output.contains(&format!("name_bytes={}", SNI_SECRET.len())),
            "TLS resolver fallback log omitted bounded SNI metadata: {output}"
        );
        assert!(
            output.len() <= 2048,
            "TLS resolver fallback log capture is not bounded: {} bytes",
            output.len()
        );
    }

    #[test]
    fn certificate_resolver_poison_logs_are_bounded() {
        const SNI_SECRET: &str = "poison-sni-secret.example";
        const QUERY_SECRET: &str = "POISON_QUERY_SECRET_SENTINEL";

        let query = format!("{QUERY_SECRET}{}", "x".repeat(4096));
        let output = crate::capture_test_logs_at_level("trace", move || {
            let resolver = Arc::new(MutexCertificateResolver::default());
            let poison_target = resolver.clone();
            assert!(
                std::thread::spawn(move || {
                    let _guard = poison_target
                        .0
                        .lock()
                        .expect("test resolver lock must initially be available");
                    panic!("intentionally poison the test resolver lock");
                })
                .join()
                .is_err(),
                "test poison thread must panic"
            );

            drive_client_hello(resolver.clone(), SNI_SECRET.to_owned());
            assert!(resolver.names_for_sni(query.as_bytes()).is_none());
        });

        assert!(
            !output.contains(SNI_SECRET),
            "TLS resolver poison log leaked the handshake SNI: {output}"
        );
        assert!(
            !output.contains(QUERY_SECRET),
            "TLS resolver poison log leaked the 4 KiB direct query: {output}"
        );
        assert!(
            output.len() <= 2048,
            "TLS resolver poison log capture is not bounded: {} bytes",
            output.len()
        );
    }

    #[test]
    fn certified_key_wrapper_debug_redacts_runtime_certificate_material() {
        const NAME_SECRET: &str = "CERTIFIED_KEY_NAME_SECRET_SENTINEL";

        let add = AddCertificate {
            address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
            certificate: CertificateAndKey {
                certificate: include_str!("../assets/certificate.pem").to_owned(),
                key: include_str!("../assets/key.pem").to_owned(),
                names: vec![format!("{NAME_SECRET}{}", "x".repeat(4096))],
                ..Default::default()
            },
            expired_at: None,
        };
        let certified_key =
            CertifiedKeyWrapper::try_from(&add).expect("test certificate must parse");
        let fingerprint = certified_key.fingerprint.to_string();
        let output = format!("{certified_key:?}");

        assert!(
            !output.contains(NAME_SECRET),
            "CertifiedKeyWrapper Debug leaked an overridden certificate name: {output}"
        );
        assert!(
            !output.contains(&fingerprint),
            "CertifiedKeyWrapper Debug leaked the certificate fingerprint: {output}"
        );
        assert!(
            output.contains("fingerprint_bytes: 32"),
            "CertifiedKeyWrapper Debug omitted bounded fingerprint metadata: {output}"
        );
        assert!(
            output.contains("[redacted]"),
            "CertifiedKeyWrapper Debug must make certificate redaction explicit: {output}"
        );
        assert!(
            output.len() <= 1024,
            "CertifiedKeyWrapper Debug output is not bounded: {} bytes",
            output.len()
        );
    }

    #[test]
    fn certificate_resolver_debug_redacts_indexed_names() {
        const NAME_SECRET: &str = "CERTIFICATE_RESOLVER_NAME_SECRET_SENTINEL";

        let mut resolver = CertificateResolver::default();
        resolver
            .add_certificate(&AddCertificate {
                address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
                certificate: CertificateAndKey {
                    certificate: include_str!("../assets/certificate.pem").to_owned(),
                    key: include_str!("../assets/key.pem").to_owned(),
                    // Below `MAX_HOSTNAME_LENGTH` so the name passes the
                    // add-time validation and reaches the Debug formatting
                    // under test; the redaction property is
                    // length-independent.
                    names: vec![format!("{NAME_SECRET}{}", "x".repeat(2048))],
                    ..Default::default()
                },
                expired_at: None,
            })
            .expect("test certificate must load into the resolver");
        let output = format!("{resolver:?}");

        assert!(
            !output.contains(NAME_SECRET),
            "CertificateResolver Debug leaked an indexed certificate name: {output}"
        );
        assert!(
            output.len() <= 1024,
            "CertificateResolver Debug output is not bounded: {} bytes",
            output.len()
        );
    }

    #[test]
    fn lifecycle() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();
        let certificate_and_key = CertificateAndKey {
            certificate: String::from(include_str!("../assets/certificate.pem")),
            key: String::from(include_str!("../assets/key.pem")),
            ..Default::default()
        };

        let fingerprint = resolver
            .add_certificate(&AddCertificate {
                address,
                certificate: certificate_and_key,
                expired_at: None,
            })
            .expect("could not add certificate");

        if resolver.get_certificate(&fingerprint).is_none() {
            return Err("failed to retrieve certificate".into());
        }

        // get the names to try and retrieve the certificate AFTER it is supposed to be removed
        let names = resolver.certificate_names(&fingerprint)?;

        if let Err(err) = resolver.remove_certificate(&fingerprint) {
            return Err(format!("the certificate was not removed, {err}").into());
        }

        if resolver.get_certificate(&fingerprint).is_some() {
            return Err("We have retrieved the certificate that should be deleted".into());
        }

        if !resolver.find_certificates_by_names(&names)?.is_empty() {
            return Err(
                "The certificate should be deleted but one of its names is in the index".into(),
            );
        }

        Ok(())
    }

    #[test]
    fn name_override() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();
        let certificate_and_key = CertificateAndKey {
            certificate: String::from(include_str!("../assets/certificate.pem")),
            key: String::from(include_str!("../assets/key.pem")),
            names: vec!["localhost".into(), "lolcatho.st".into()],
            ..Default::default()
        };

        let fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: certificate_and_key,
            expired_at: None,
        })?;

        if resolver.get_certificate(&fingerprint).is_none() {
            return Err("failed to retrieve certificate".into());
        }

        let mut lolcat = HashSet::new();
        lolcat.insert(String::from("lolcatho.st"));
        if resolver.find_certificates_by_names(&lolcat)?.is_empty()
            || resolver.get_certificate(&fingerprint).is_none()
        {
            return Err("failed to retrieve certificate with custom names".into());
        }

        if let Err(err) = resolver.remove_certificate(&fingerprint) {
            return Err(format!("the certificate could not be removed, {err}").into());
        }

        let names = resolver.certificate_names(&fingerprint)?;
        if !resolver.find_certificates_by_names(&names)?.is_empty()
            && resolver.get_certificate(&fingerprint).is_some()
        {
            return Err("We have retrieved the certificate that should be deleted".into());
        }

        Ok(())
    }

    #[test]
    fn keep_resolving_with_wildcard() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        // ---------------------------------------------------------------------
        // load the wildcard certificate,  expiring in 3 years
        let wildcard_example_org = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-3.pem")),
            key: String::from(include_str!("../assets/tests/key.pem")),
            ..Default::default()
        };

        let wildcard_example_org_fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: wildcard_example_org,
            expired_at: Some(
                (SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?
                    + Duration::from_secs(365 * 24 * 3600))
                .as_secs() as i64,
            ),
        })?;

        if resolver
            .get_certificate(&wildcard_example_org_fingerprint)
            .is_none()
        {
            return Err("could not load the 2-year-valid certificate".into());
        }

        // ---------------------------------------------------------------------
        // try loading the ordinary certificate, expiring in 2 years
        // this one has two names: example.org and www.example.org
        let www_example_org = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-2.pem")),
            key: String::from(include_str!("../assets/tests/key.pem")),
            ..Default::default()
        };

        let www_example_org_fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: www_example_org,
            expired_at: Some(
                (SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?
                    + Duration::from_secs(2 * 365 * 24 * 3600))
                .as_secs() as i64,
            ),
        })?;

        let www_example_org = resolver
            .domain_lookup("www.example.org".as_bytes(), true)
            .expect("there should be a www.example.org cert");
        assert_eq!(www_example_org.1, www_example_org_fingerprint);

        let test_example_org = resolver
            .domain_lookup("test.example.org".as_bytes(), true)
            .expect("there should be a test.example.org cert");
        assert_eq!(test_example_org.1, wildcard_example_org_fingerprint);

        let example_org = resolver
            .domain_lookup("example.org".as_bytes(), true)
            .expect("there should be a example.org cert");
        assert_eq!(example_org.1, www_example_org_fingerprint);

        // check that when removing the www.example.org certificate
        // the resolver falls back on the wildcard
        resolver
            .remove_certificate(&www_example_org_fingerprint)
            .expect("should be able to remove the 2-year certificate");

        let should_be_wildcard_fingerprint = resolver
            .domain_lookup("www.example.org".as_bytes(), true)
            .expect("there should be a www.example.org cert");
        assert_eq!(
            should_be_wildcard_fingerprint.1,
            wildcard_example_org_fingerprint
        );

        assert!(
            resolver
                .domain_lookup("example.org".as_bytes(), true)
                .is_none()
        );

        Ok(())
    }

    #[test]
    fn resolve_the_longer_lived_cert() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        // ---------------------------------------------------------------------
        // load the 2-year valid certificate
        let certificate_and_key_2y = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-2y.pem")),
            key: String::from(include_str!("../assets/tests/key-2y.pem")),
            ..Default::default()
        };

        let fingerprint_2y = resolver.add_certificate(&AddCertificate {
            address,
            certificate: certificate_and_key_2y,
            expired_at: None,
        })?;

        if resolver.get_certificate(&fingerprint_2y).is_none() {
            return Err("could not load the 2-year-valid certificate".into());
        }

        // ---------------------------------------------------------------------
        // try loading the 1-year valid certificate
        let certificate_and_key_1y = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-1y.pem")),
            key: String::from(include_str!("../assets/tests/key-1y.pem")),
            ..Default::default()
        };

        let fingerprint_1y = resolver.add_certificate(&AddCertificate {
            address,
            certificate: certificate_and_key_1y,
            ..Default::default()
        })?;

        let localhost_cert = resolver
            .domain_lookup("localhost".as_bytes(), true)
            .expect("there should be a localhost cert");

        assert_eq!(localhost_cert.1, fingerprint_2y);

        // check that when removing the longer-lived certificate,
        // the resolver falls back on the shorter-lived one

        resolver
            .remove_certificate(&fingerprint_2y)
            .expect("should be able to remove the 2-year certificate");

        let localhost_cert = resolver
            .domain_lookup("localhost".as_bytes(), true)
            .expect("there should be a localhost cert");

        assert_eq!(localhost_cert.1, fingerprint_1y);

        Ok(())
    }

    #[test]
    fn expiration_override() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        // ---------------------------------------------------------------------
        // load first certificate
        let certificate_and_key_1y = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-1y.pem")),
            key: String::from(include_str!("../assets/tests/key-1y.pem")),
            ..Default::default()
        };

        let fingerprint_1y_overriden = resolver.add_certificate(&AddCertificate {
            address,
            certificate: certificate_and_key_1y,
            expired_at: Some(
                (SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?
                    + Duration::from_secs(3 * 365 * 24 * 3600))
                .as_secs() as i64,
            ),
        })?;

        if resolver
            .get_certificate(&fingerprint_1y_overriden)
            .is_none()
        {
            return Err("failed to retrieve certificate".into());
        }

        // ---------------------------------------------------------------------
        // load second certificate
        let certificate_and_key_2y = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-2y.pem")),
            key: String::from(include_str!("../assets/tests/key-2y.pem")),
            ..Default::default()
        };

        let fingerprint_2y = resolver.add_certificate(&AddCertificate {
            address,
            certificate: certificate_and_key_2y,
            expired_at: None,
        })?;

        let localhost_cert = resolver
            .domain_lookup("localhost".as_bytes(), true)
            .expect("there should be a localhost cert");

        assert_eq!(localhost_cert.1, fingerprint_1y_overriden);

        // check that when removing the overriden certificate,
        // the resolver falls back on the other one

        resolver
            .remove_certificate(&fingerprint_1y_overriden)
            .expect("should be able to remove the 1-year (3-year-overriden) certificate");

        let localhost_cert = resolver
            .domain_lookup("localhost".as_bytes(), true)
            .expect("there should be a localhost cert");

        assert_eq!(localhost_cert.1, fingerprint_2y);

        Ok(())
    }

    /// Verify that `replace_certificate` adds the new cert before removing
    /// the old one, so lookup always returns a valid certificate.
    #[test]
    fn replace_certificate_add_before_remove() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        // add the initial (1-year) certificate
        let cert_1y = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-1y.pem")),
            key: String::from(include_str!("../assets/tests/key-1y.pem")),
            ..Default::default()
        };

        let fingerprint_1y = resolver.add_certificate(&AddCertificate {
            address,
            certificate: cert_1y,
            expired_at: None,
        })?;

        // sanity: the 1y cert is resolvable
        assert!(
            resolver
                .domain_lookup("localhost".as_bytes(), true)
                .is_some(),
            "initial certificate should be resolvable"
        );

        // replace with the 2-year certificate
        let cert_2y = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-2y.pem")),
            key: String::from(include_str!("../assets/tests/key-2y.pem")),
            ..Default::default()
        };

        let new_fingerprint = resolver.replace_certificate(&ReplaceCertificate {
            address,
            new_certificate: cert_2y,
            old_fingerprint: fingerprint_1y.to_string(),
            new_expired_at: None,
        })?;

        // the old certificate should be gone
        assert!(
            resolver.get_certificate(&fingerprint_1y).is_none(),
            "old certificate should have been removed"
        );

        // the new certificate should be present and resolvable
        assert!(
            resolver.get_certificate(&new_fingerprint).is_some(),
            "new certificate should be present"
        );
        let resolved = resolver
            .domain_lookup("localhost".as_bytes(), true)
            .expect("a certificate should resolve for localhost");
        assert_eq!(
            resolved.1, new_fingerprint,
            "resolved certificate should be the replacement"
        );

        Ok(())
    }

    /// When `ReplaceCertificate` carries the same certificate / fingerprint
    /// as the existing entry, the previous implementation called
    /// `add_certificate` (which early-returns on duplicate fingerprint
    /// without inserting) and then unconditionally removed the old
    /// fingerprint — i.e. the *current* entry — leaving the resolver
    /// without a certificate for that name. The fix short-circuits the
    /// idempotent case and keeps the existing entry in place. An
    /// idempotent ACME / operator retry must therefore still resolve.
    #[test]
    fn replace_certificate_with_same_fingerprint_is_noop()
    -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        let cert = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-1y.pem")),
            key: String::from(include_str!("../assets/tests/key-1y.pem")),
            ..Default::default()
        };

        let initial_fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: cert.clone(),
            expired_at: None,
        })?;

        // Replace with the SAME PEM/key — identical fingerprint expected.
        let returned_fingerprint = resolver.replace_certificate(&ReplaceCertificate {
            address,
            new_certificate: cert,
            old_fingerprint: initial_fingerprint.to_string(),
            new_expired_at: None,
        })?;

        assert_eq!(
            returned_fingerprint, initial_fingerprint,
            "idempotent replace should return the existing fingerprint"
        );

        assert!(
            resolver.get_certificate(&initial_fingerprint).is_some(),
            "idempotent replace must NOT delete the existing certificate"
        );

        let resolved = resolver
            .domain_lookup("localhost".as_bytes(), true)
            .expect("certificate should still resolve after idempotent replace");
        assert_eq!(
            resolved.1, initial_fingerprint,
            "resolver should still hand back the original fingerprint"
        );

        Ok(())
    }

    /// Verify that removing the last certificate for a domain cleans up
    /// the empty entry in `name_fingerprint_idx` (no memory leak).
    #[test]
    fn removal_cleans_up_empty_index_entries() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        let cert = CertificateAndKey {
            certificate: String::from(include_str!("../assets/tests/certificate-1y.pem")),
            key: String::from(include_str!("../assets/tests/key-1y.pem")),
            ..Default::default()
        };

        let fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: cert,
            expired_at: None,
        })?;

        // record the names associated with this cert
        let names = resolver.certificate_names(&fingerprint)?;
        assert!(
            !names.is_empty(),
            "certificate should have at least one name"
        );

        // verify index is populated
        for name in &names {
            assert!(
                resolver.name_fingerprint_idx.contains_key(name),
                "name_fingerprint_idx should contain '{name}' before removal"
            );
        }

        resolver.remove_certificate(&fingerprint)?;

        // after removal, all index entries for these names should be gone
        for name in &names {
            assert!(
                !resolver.name_fingerprint_idx.contains_key(name),
                "name_fingerprint_idx should not contain empty entry for '{name}' after removal"
            );
        }

        Ok(())
    }

    /// Many ACME clients (Certbot's `fullchain.pem`, lego, acme.sh) emit
    /// the leaf certificate at the START of the chain file. Without
    /// dedup, the resolver previously stored `[leaf, leaf, ...]` and
    /// the on-wire TLS handshake replayed the leaf twice — accepted by
    /// browsers but rejected by stricter validators (Node.js,
    /// `UNABLE_TO_VERIFY_LEAF_SIGNATURE`). The fix in
    /// `TryFrom<&AddCertificate> for CertifiedKeyWrapper` drops any
    /// chain entry whose DER bytes match the leaf. Closes #1135 / #1148.
    ///
    /// This test passes the SAME leaf PEM as both `certificate` and the
    /// sole `certificate_chain` entry (the `fullchain.pem` shape). The
    /// stored chain length must be `1` (leaf only), not `2`
    /// (`[leaf, leaf]`).
    #[test]
    fn certificate_chain_dedup_drops_duplicate_leaf() -> Result<(), Box<dyn Error + Send + Sync>> {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        let leaf_pem = String::from(include_str!("../assets/certificate.pem"));

        let cert_with_duplicated_leaf = CertificateAndKey {
            certificate: leaf_pem.clone(),
            certificate_chain: vec![leaf_pem],
            key: String::from(include_str!("../assets/key.pem")),
            ..Default::default()
        };

        let fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: cert_with_duplicated_leaf,
            expired_at: None,
        })?;

        let stored = resolver
            .get_certificate(&fingerprint)
            .ok_or("resolver lost the certificate after add")?;

        assert_eq!(
            stored.inner.cert.len(),
            1,
            "expected dedup to drop the duplicate leaf, got chain of {} cert(s)",
            stored.inner.cert.len()
        );

        Ok(())
    }

    /// When an operator passes the entire `fullchain.pem` content as a
    /// SINGLE chain entry (one string containing multiple
    /// `-----BEGIN CERTIFICATE-----` blocks back-to-back), the previous
    /// code called `parse_pem` once on the multi-PEM string, which only
    /// consumes the first PEM block and silently drops the rest. The
    /// fix splits each chain entry through
    /// `split_certificate_chain` so multi-PEM strings fan out into one
    /// entry per CA before parsing.
    ///
    /// This test concatenates two PEM blocks into a single chain entry
    /// (the leaf duplicated, so dedup also kicks in). The stored chain
    /// must end up with length 1 — the leaf — proving (a) the multi-PEM
    /// entry was split correctly, (b) the duplicate leaf was dropped.
    #[test]
    fn certificate_chain_handles_multi_pem_single_entry() -> Result<(), Box<dyn Error + Send + Sync>>
    {
        let address = SocketAddress::new_v4(127, 0, 0, 1, 8080);
        let mut resolver = CertificateResolver::default();

        let leaf_pem = String::from(include_str!("../assets/certificate.pem"));
        // Two PEM blocks concatenated into one string; both are the
        // leaf so dedup brings the result back to length 1.
        let multi_pem_chain_entry = format!("{leaf_pem}\n{leaf_pem}");

        let cert = CertificateAndKey {
            certificate: leaf_pem,
            certificate_chain: vec![multi_pem_chain_entry],
            key: String::from(include_str!("../assets/key.pem")),
            ..Default::default()
        };

        let fingerprint = resolver.add_certificate(&AddCertificate {
            address,
            certificate: cert,
            expired_at: None,
        })?;

        let stored = resolver
            .get_certificate(&fingerprint)
            .ok_or("resolver lost the certificate after add")?;

        // Without the split, `parse_pem` would only consume the first
        // PEM block and drop the second; with the split + dedup, both
        // get fanned out, both get recognised as the leaf, both get
        // dropped — leaving the original leaf at index 0 only.
        assert_eq!(
            stored.inner.cert.len(),
            1,
            "expected split + dedup to leave only the leaf, got chain of {} cert(s)",
            stored.inner.cert.len()
        );

        Ok(())
    }

    /// The SNI half of the same add/lookup case asymmetry, with the two
    /// sides swapped.
    ///
    /// RFC 4343 makes DNS names compare case-insensitively, and rustls
    /// hands `ResolvesServerCert::resolve` an SNI it has ALREADY
    /// lowercased: `rustls::server::hs::process_client_hello` stores
    /// `dns_name.to_lowercase_owned()` (rustls 0.23,
    /// `rustls-pki-types`' `DnsName::to_lowercase_owned` is an
    /// `to_ascii_lowercase`), and `ClientHello::server_name()` surfaces
    /// that field. The ADD side had no such normalisation: a SAN read off
    /// the X.509 (`command/src/certificate.rs`'s
    /// `get_cn_and_san_attributes`) or an operator-supplied `names`
    /// override went into the SNI trie and the name index verbatim.
    ///
    /// So a certificate carrying `MiXeD.Example.COM` was reachable by no
    /// SNI at all — every handshake for that name fell through to the
    /// `DEFAULT_CERTIFICATE` arm of `MutexCertificateResolver::resolve`,
    /// which also increments `names::tls::DEFAULT_CERT_USED`, while the
    /// certificate stayed recorded and queryable.
    ///
    /// To SEE THIS RED: in `CertifiedKeyWrapper::try_from`, delete the
    /// `let overriding_names: Vec<String> = overriding_names …` rebind
    /// that maps `|mut name| { name.make_ascii_lowercase(); name }`, so
    /// the names reach the trie as written. The FIRST assertion below
    /// fails — `domain_lookup(b"mixed.example.com", true)` is `None`, so
    /// the test panics there and the later ones are never reached.
    #[test]
    fn a_mixed_case_certificate_name_resolves_for_the_lowercased_sni() {
        const MIXED: &str = "MiXeD.Example.COM";
        const LOWER: &str = "mixed.example.com";

        let mut resolver = CertificateResolver::default();
        let fingerprint = resolver
            .add_certificate(&AddCertificate {
                address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
                certificate: CertificateAndKey {
                    certificate: include_str!("../assets/certificate.pem").to_owned(),
                    key: include_str!("../assets/key.pem").to_owned(),
                    names: vec![MIXED.to_owned()],
                    ..Default::default()
                },
                expired_at: None,
            })
            .expect("the mixed-case certificate must load");

        assert!(
            resolver.domain_lookup(LOWER.as_bytes(), true).is_some(),
            "rustls only ever asks for the lowercased SNI, so that is the \
             only key the trie can usefully hold",
        );
        assert_eq!(
            resolver.names_for_sni(LOWER.as_bytes()),
            Some(vec![LOWER.to_owned()]),
            "the SAN snapshot feeding the H2 `:authority` binding must \
             carry the normalised name",
        );

        // Removal walks the certificate's stored names, so it stays
        // symmetric with the add path by construction: normalise once at
        // the single point where `names` is built and both verbs agree.
        resolver
            .remove_certificate(&fingerprint)
            .expect("the mixed-case certificate must be removable");
        assert!(
            resolver.domain_lookup(LOWER.as_bytes(), true).is_none(),
            "removing the certificate must retire its normalised trie key",
        );
    }

    /// A certificate name enters the SNI `TrieNode` verbatim, and that
    /// trie is the same `lib/src/router/pattern_trie.rs` the HTTP router
    /// uses: a dot-separated label wrapped in `/.../` is compiled as a
    /// REGEX segment (`insert_recursive` -> `anchored_segment` ->
    /// `Regex::new`). So an operator `names` entry such as
    /// `/te.*/.example.com` bound ONE certificate to an open-ended class
    /// of SNI values -- measured on `c7ac070e`, that name was served for
    /// both `test.example.com` and `tenant.example.com`
    /// (sozu-proxy/sozu#1378). A name like `/.*/.example.com` is one
    /// character away from a plausible typo, and the certificate
    /// inventory an operator reads shows a SAN-shaped string while the
    /// trie holds a pattern.
    ///
    /// The validation that already existed could not catch it: it
    /// dry-runs every name into a scratch trie and asks only whether the
    /// name INSERTS, and a regex segment inserts perfectly well. The
    /// check therefore has to be on the name's BYTES, and it has to run
    /// in the same pre-mutation block as the `MAX_HOSTNAME_LENGTH` bound
    /// so nothing is half-registered. `/` cannot occur in a DNS name, so
    /// nothing legitimate is refused -- and the ordinary `*.` wildcard
    /// name, which the trie handles as a wildcard and not as a regex,
    /// still loads; the second half of this test pins that the guard was
    /// not widened into "refuse anything that is not a literal".
    ///
    /// This mirrors `validate_sni_pattern` (`command/src/config.rs`),
    /// which already rejects `/` on every path into the TCP SNI route
    /// table; the certificate trie was the one SNI table left open.
    ///
    /// To SEE THIS RED: in `add_certificate`'s pre-mutation dry-run
    /// block, delete the `|| name.contains('/')` disjunct from the
    /// rejection condition. The certificate then loads and `expect_err`
    /// panics on the returned `Ok(fingerprint)`, so the test fails at its
    /// very first statement.
    #[test]
    fn a_certificate_name_carrying_a_regex_segment_is_refused() {
        const REGEX_NAME: &str = "/te.*/.example.com";

        let mut resolver = CertificateResolver::default();
        let error = resolver
            .add_certificate(&AddCertificate {
                address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
                certificate: CertificateAndKey {
                    certificate: include_str!("../assets/certificate.pem").to_owned(),
                    key: include_str!("../assets/key.pem").to_owned(),
                    names: vec![REGEX_NAME.to_owned()],
                    ..Default::default()
                },
                expired_at: None,
            })
            .expect_err("a certificate name carrying a regex segment must be refused");

        assert!(
            matches!(&error, CertificateResolverError::InvalidName(name) if name == REGEX_NAME),
            "the refusal must be InvalidName, got {error:?}",
        );

        // Refused BEFORE any mutation, exactly like the
        // `MAX_HOSTNAME_LENGTH` bound it sits beside: no certificate
        // stored, no name indexed, and -- the point of the report -- no
        // open-ended class of SNI values bound in the trie.
        assert!(
            resolver.certificates.is_empty(),
            "a refused certificate must not be stored",
        );
        assert!(
            resolver.name_fingerprint_idx.is_empty(),
            "a refused certificate must not index any name",
        );
        for sni in ["test.example.com", "tenant.example.com", REGEX_NAME] {
            assert!(
                resolver.domain_lookup(sni.as_bytes(), true).is_none(),
                "a refused certificate name must bind no SNI, yet {sni} resolved",
            );
        }

        // The guard is on `/` alone. An ordinary wildcard SAN is not a
        // regex segment, still loads, and still serves its subtree.
        resolver
            .add_certificate(&AddCertificate {
                address: SocketAddress::new_v4(127, 0, 0, 1, 8443),
                certificate: CertificateAndKey {
                    certificate: include_str!("../assets/certificate.pem").to_owned(),
                    key: include_str!("../assets/key.pem").to_owned(),
                    names: vec!["*.example.com".to_owned()],
                    ..Default::default()
                },
                expired_at: None,
            })
            .expect("a wildcard certificate name must still load");
        assert!(
            resolver
                .domain_lookup(b"tenant.example.com", true)
                .is_some(),
            "the wildcard certificate must still serve a subdomain",
        );
    }
}