use std::{
io,
net::SocketAddr,
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use arc_swap::ArcSwap;
use crate::accept::{AcceptErrors, AcceptHealth, Next};
use notify::RecursiveMode;
use notify_debouncer_full::{DebounceEventResult, Debouncer, RecommendedCache, new_debouncer};
use rustls::{
ServerConfig,
client::ResolvesClientCert,
crypto::CryptoProvider,
pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject},
server::{ClientHello, ResolvesServerCert},
sign::CertifiedKey,
};
use thiserror::Error;
use tokio::{
net::{TcpListener, TcpStream},
sync::mpsc,
task::JoinSet,
time::timeout,
};
use tokio_rustls::{TlsAcceptor, server::TlsStream};
pub trait TlsObserver: Send + Sync + 'static {
fn certificate_loaded(&self, _certificate: &Path) {}
fn certificate_rejected(&self, _certificate: &Path, _reason: &str) {}
fn certificate_watch_lost(&self, _reason: &str) {}
fn accept_failed(&self, _reason: &str, _failures: u64) {}
fn listener_failed(&self, _reason: &str) {}
fn handshake_completed(&self) {}
fn handshake_failed(&self) {}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct IgnoreTlsEvents;
impl TlsObserver for IgnoreTlsEvents {}
const ALPN_PROTOCOLS: [&[u8]; 2] = [b"h2", b"http/1.1"];
const DEBOUNCE: Duration = Duration::from_secs(1);
#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
pub enum TlsVersion {
Tls12,
#[default]
Tls13,
}
impl std::str::FromStr for TlsVersion {
type Err = &'static str;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"1.2" => Ok(Self::Tls12),
"1.3" => Ok(Self::Tls13),
_ => Err("TLS version must be 1.2 or 1.3"),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TlsSettings {
pub certificate: PathBuf,
pub key: PathBuf,
pub min_version: TlsVersion,
pub max_version: TlsVersion,
pub handshake_timeout: Duration,
pub maximum_pending_handshakes: usize,
}
impl Default for TlsSettings {
fn default() -> Self {
Self {
certificate: PathBuf::new(),
key: PathBuf::new(),
min_version: TlsVersion::Tls13,
max_version: TlsVersion::Tls13,
handshake_timeout: Duration::from_secs(10),
maximum_pending_handshakes: 256,
}
}
}
#[derive(Debug, Error)]
pub enum TlsError {
#[error("TLS min_version must not exceed max_version")]
InvalidVersionRange,
#[error("could not read {path}: {source}")]
Unreadable {
path: PathBuf,
source: rustls::pki_types::pem::Error,
},
#[error("{path} contains no certificate")]
Empty { path: PathBuf },
#[error("the certificate and key are not usable: {0}")]
Unusable(rustls::Error),
#[error("TLS could not be configured: {0}")]
Configuration(rustls::Error),
#[error("could not watch {path} for certificate changes: {source}")]
Unwatchable {
path: PathBuf,
source: notify::Error,
},
#[error("certificate rotation needs a Tokio runtime; load {path} from inside one")]
NoRuntime { path: PathBuf },
}
impl From<TlsError> for io::Error {
fn from(error: TlsError) -> Self {
Self::other(error.to_string())
}
}
#[derive(Debug)]
pub struct CertificateResolver(ArcSwap<CertifiedKey>);
impl ResolvesServerCert for CertificateResolver {
fn resolve(&self, _hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
Some(self.0.load_full())
}
}
impl ResolvesClientCert for CertificateResolver {
fn resolve(
&self,
_root_hint_subjects: &[&[u8]],
_sigschemes: &[rustls::SignatureScheme],
) -> Option<Arc<CertifiedKey>> {
Some(self.0.load_full())
}
fn has_certs(&self) -> bool {
true
}
}
fn load(settings: &TlsSettings, provider: &CryptoProvider) -> Result<CertifiedKey, TlsError> {
let chain = load_certificates(&settings.certificate)?;
let key = load_private_key(&settings.key)?;
CertifiedKey::from_der(chain, key, provider).map_err(TlsError::Unusable)
}
pub struct ClientIdentity {
resolver: Arc<CertificateResolver>,
_watcher: CertificateWatch,
}
impl ClientIdentity {
pub fn new<O: TlsObserver>(
certificate: PathBuf,
key: PathBuf,
observer: Arc<O>,
) -> Result<Self, TlsError> {
let settings = TlsSettings {
certificate,
key,
..TlsSettings::default()
};
let provider = Arc::new(rustls::crypto::ring::default_provider());
let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
&settings, &provider,
)?)));
observer.certificate_loaded(&settings.certificate);
let watcher = CertificateWatch::start(settings, Arc::clone(&resolver), provider, observer)?;
Ok(Self {
resolver,
_watcher: watcher,
})
}
pub fn resolver(&self) -> Arc<dyn ResolvesClientCert> {
Arc::clone(&self.resolver) as Arc<dyn ResolvesClientCert>
}
}
pub fn load_roots(path: &Path) -> Result<Vec<CertificateDer<'static>>, TlsError> {
load_certificates(path)
}
pub fn load_certificates(path: &Path) -> Result<Vec<CertificateDer<'static>>, TlsError> {
let roots = CertificateDer::pem_file_iter(path)
.and_then(std::iter::Iterator::collect::<Result<Vec<_>, _>>)
.map_err(|source| TlsError::Unreadable {
path: path.to_path_buf(),
source,
})?;
if roots.is_empty() {
return Err(TlsError::Empty {
path: path.to_path_buf(),
});
}
Ok(roots)
}
pub fn load_private_key(path: &Path) -> Result<PrivateKeyDer<'static>, TlsError> {
PrivateKeyDer::from_pem_file(path).map_err(|source| TlsError::Unreadable {
path: path.to_path_buf(),
source,
})
}
pub struct TlsListener<O: TlsObserver> {
tcp: TcpListener,
acceptor: TlsAcceptor,
handshakes: JoinSet<Option<(TlsStream<TcpStream>, SocketAddr)>>,
settings: TlsSettings,
observer: Arc<O>,
errors: AcceptErrors,
_watcher: CertificateWatch,
}
impl<O: TlsObserver> TlsListener<O> {
pub fn new(
tcp: TcpListener,
settings: TlsSettings,
observer: Arc<O>,
) -> Result<Self, TlsError> {
let (config, watcher) =
rotating_tcp_server_config(settings.clone(), &ALPN_PROTOCOLS, Arc::clone(&observer))?;
Ok(Self {
tcp,
acceptor: TlsAcceptor::from(config),
handshakes: JoinSet::new(),
settings,
observer,
errors: AcceptErrors::default(),
_watcher: watcher,
})
}
#[must_use]
pub fn with_health(mut self, health: AcceptHealth) -> Self {
self.errors = AcceptErrors::with_health(health);
self
}
pub fn local_addr(&self) -> io::Result<SocketAddr> {
self.tcp.local_addr()
}
pub async fn accept_tls(&mut self) -> io::Result<(TlsStream<TcpStream>, SocketAddr)> {
loop {
tokio::select! {
accepted = self.tcp.accept(),
if self.handshakes.len() < self.settings.maximum_pending_handshakes =>
{
let (stream, peer) = match accepted {
Ok(accepted) => {
self.errors.succeeded();
accepted
}
Err(error) => match self.errors.failed(&error) {
Next::Stop => {
self.observer.listener_failed(&error.to_string());
return Err(error);
}
Next::Retry { pause, report } => {
if let Some(failures) = report {
self.observer.accept_failed(&error.to_string(), failures);
}
tokio::time::sleep(pause).await;
continue;
}
},
};
let acceptor = self.acceptor.clone();
let deadline = self.settings.handshake_timeout;
let observer = Arc::clone(&self.observer);
self.handshakes.spawn(async move {
if let Ok(Ok(stream)) = timeout(deadline, acceptor.accept(stream)).await {
observer.handshake_completed();
Some((stream, peer))
} else {
observer.handshake_failed();
None
}
});
}
Some(joined) = self.handshakes.join_next(), if !self.handshakes.is_empty() => {
if let Ok(Some(ready)) = joined {
return Ok(ready);
}
}
}
}
}
}
#[cfg(feature = "axum")]
impl<O: TlsObserver> axum::serve::Listener for TlsListener<O> {
type Io = TlsStream<TcpStream>;
type Addr = SocketAddr;
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
match self.accept_tls().await {
Ok(accepted) => accepted,
Err(_) => std::future::pending().await,
}
}
fn local_addr(&self) -> io::Result<Self::Addr> {
self.tcp.local_addr()
}
}
pub struct CertificateWatch {
_debouncer: Debouncer<notify::RecommendedWatcher, RecommendedCache>,
pump: tokio::task::JoinHandle<()>,
}
impl Drop for CertificateWatch {
fn drop(&mut self) {
self.pump.abort();
}
}
impl CertificateWatch {
pub fn start<O: TlsObserver>(
settings: TlsSettings,
resolver: Arc<CertificateResolver>,
provider: Arc<CryptoProvider>,
observer: Arc<O>,
) -> Result<Self, TlsError> {
let runtime = tokio::runtime::Handle::try_current().map_err(|_| TlsError::NoRuntime {
path: settings.certificate.clone(),
})?;
let (sender, receiver) = mpsc::unbounded_channel();
let mut debouncer = new_debouncer(DEBOUNCE, None, move |result: DebounceEventResult| {
let _ = sender.send(result);
})
.map_err(|source| TlsError::Unwatchable {
path: settings.certificate.clone(),
source,
})?;
for directory in watched_directories(&settings) {
debouncer
.watch(&directory, RecursiveMode::NonRecursive)
.map_err(|source| TlsError::Unwatchable {
path: directory.clone(),
source,
})?;
}
let pump = runtime.spawn(reload_on_change(
settings, resolver, provider, observer, receiver,
));
Ok(Self {
_debouncer: debouncer,
pump,
})
}
}
pub fn rotating_tcp_server_config<O: TlsObserver>(
settings: TlsSettings,
alpn: &[&[u8]],
observer: Arc<O>,
) -> Result<(Arc<ServerConfig>, CertificateWatch), TlsError> {
let versions: &[&rustls::SupportedProtocolVersion] =
match (settings.min_version, settings.max_version) {
(TlsVersion::Tls12, TlsVersion::Tls12) => &[&rustls::version::TLS12],
(TlsVersion::Tls12, TlsVersion::Tls13) => {
&[&rustls::version::TLS13, &rustls::version::TLS12]
}
(TlsVersion::Tls13, TlsVersion::Tls13) => &[&rustls::version::TLS13],
(TlsVersion::Tls13, TlsVersion::Tls12) => {
return Err(TlsError::InvalidVersionRange);
}
};
let provider = Arc::new(rustls::crypto::ring::default_provider());
let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
&settings, &provider,
)?)));
observer.certificate_loaded(&settings.certificate);
let mut config = ServerConfig::builder_with_provider(Arc::clone(&provider))
.with_protocol_versions(versions)
.map_err(TlsError::Configuration)?
.with_no_client_auth()
.with_cert_resolver(Arc::clone(&resolver) as Arc<dyn ResolvesServerCert>);
config.alpn_protocols = alpn.iter().map(|name| name.to_vec()).collect();
let watcher = CertificateWatch::start(settings, resolver, provider, observer)?;
Ok((Arc::new(config), watcher))
}
pub fn rotating_quic_server_config<O: TlsObserver>(
settings: TlsSettings,
alpn: &[&[u8]],
observer: Arc<O>,
) -> Result<(Arc<ServerConfig>, CertificateWatch), TlsError> {
let provider = Arc::new(rustls::crypto::ring::default_provider());
let resolver = Arc::new(CertificateResolver(ArcSwap::from_pointee(load(
&settings, &provider,
)?)));
observer.certificate_loaded(&settings.certificate);
let mut config = ServerConfig::builder_with_provider(Arc::clone(&provider))
.with_protocol_versions(&[&rustls::version::TLS13])
.map_err(TlsError::Configuration)?
.with_no_client_auth()
.with_cert_resolver(Arc::clone(&resolver) as Arc<dyn ResolvesServerCert>);
config.alpn_protocols = alpn.iter().map(|name| name.to_vec()).collect();
config.max_early_data_size = u32::MAX;
let watcher = CertificateWatch::start(settings, resolver, provider, observer)?;
Ok((Arc::new(config), watcher))
}
fn watched_directories(settings: &TlsSettings) -> Vec<PathBuf> {
let mut directories = Vec::with_capacity(2);
for path in [&settings.certificate, &settings.key] {
let directory = path.parent().unwrap_or(Path::new(".")).to_path_buf();
if !directories.contains(&directory) {
directories.push(directory);
}
}
directories
}
async fn reload_on_change<O: TlsObserver>(
settings: TlsSettings,
resolver: Arc<CertificateResolver>,
provider: Arc<CryptoProvider>,
observer: Arc<O>,
mut changes: mpsc::UnboundedReceiver<DebounceEventResult>,
) {
while let Some(result) = changes.recv().await {
if let Err(errors) = result {
let reason = errors
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("; ");
observer.certificate_watch_lost(&reason);
continue;
}
let settings_for_load = settings.clone();
let provider_for_load = Arc::clone(&provider);
let loaded =
tokio::task::spawn_blocking(move || load(&settings_for_load, &provider_for_load)).await;
match loaded {
Ok(Ok(certified)) => {
if certified.cert == resolver.0.load().cert {
continue;
}
resolver.0.store(Arc::new(certified));
observer.certificate_loaded(&settings.certificate);
}
Ok(Err(error)) => {
observer.certificate_rejected(&settings.certificate, &error.to_string())
}
Err(error) => observer.certificate_watch_lost(&error.to_string()),
}
}
observer.certificate_watch_lost("the certificate watch ended");
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU64, Ordering};
fn scratch(name: &str) -> PathBuf {
static NEXT: AtomicU64 = AtomicU64::new(0);
let unique = NEXT.fetch_add(1, Ordering::Relaxed);
let directory =
std::env::temp_dir().join(format!("rushls-tls-{name}-{}-{unique}", std::process::id()));
let _ = std::fs::remove_dir_all(&directory);
std::fs::create_dir_all(&directory).expect("the scratch directory is created");
directory
}
fn write_pair(directory: &Path, name: &str) -> (TlsSettings, Vec<u8>) {
let certified = rcgen::generate_simple_self_signed([name.to_owned()])
.expect("a self-signed certificate is generated");
let settings = TlsSettings {
certificate: directory.join("certificate.pem"),
key: directory.join("key.pem"),
..TlsSettings::default()
};
write_atomically(&settings.certificate, certified.cert.pem().as_bytes());
write_atomically(
&settings.key,
certified.signing_key.serialize_pem().as_bytes(),
);
(settings, certified.cert.der().to_vec())
}
fn write_atomically(path: &Path, contents: &[u8]) {
let staged = path.with_extension("staged");
std::fs::write(&staged, contents).expect("the staged file is written");
std::fs::rename(&staged, path).expect("the staged file is renamed into place");
}
fn provider() -> CryptoProvider {
rustls::crypto::ring::default_provider()
}
#[test]
fn a_matching_pair_loads() {
let directory = scratch("matching");
let (settings, _) = write_pair(&directory, "origin.test");
load(&settings, &provider()).expect("a self-signed pair is usable");
}
#[test]
fn a_key_from_a_different_certificate_is_refused() {
let directory = scratch("mismatched");
let (settings, _) = write_pair(&directory, "origin.test");
let other = scratch("mismatched-other");
let (foreign, _) = write_pair(&other, "other.test");
std::fs::copy(&foreign.key, &settings.key).expect("the foreign key replaces the real one");
assert!(matches!(
load(&settings, &provider()),
Err(TlsError::Unusable(_))
));
}
#[test]
fn a_truncated_certificate_is_refused_rather_than_loaded_empty() {
let directory = scratch("truncated");
let (settings, _) = write_pair(&directory, "origin.test");
std::fs::write(&settings.certificate, b"").expect("the certificate is truncated");
assert!(matches!(
load(&settings, &provider()),
Err(TlsError::Empty { .. })
));
}
#[test]
fn a_pair_sharing_one_directory_is_watched_once() {
let settings = TlsSettings {
certificate: PathBuf::from("/etc/tls/fullchain.pem"),
key: PathBuf::from("/etc/tls/privkey.pem"),
..TlsSettings::default()
};
assert_eq!(watched_directories(&settings), [PathBuf::from("/etc/tls")]);
}
#[tokio::test]
async fn tls_version_bounds_control_negotiation() -> Result<(), Box<dyn std::error::Error>> {
let directory = scratch("protocol-versions");
let (mut settings, der) = write_pair(&directory, "origin.test");
let mut roots = rustls::RootCertStore::empty();
roots.add(CertificateDer::from(der))?;
for (min_version, max_version, versions, expected) in [
(
TlsVersion::Tls13,
TlsVersion::Tls13,
vec![&rustls::version::TLS12],
None,
),
(
TlsVersion::Tls13,
TlsVersion::Tls13,
vec![&rustls::version::TLS13],
Some(rustls::ProtocolVersion::TLSv1_3),
),
(
TlsVersion::Tls12,
TlsVersion::Tls13,
vec![&rustls::version::TLS12],
Some(rustls::ProtocolVersion::TLSv1_2),
),
(
TlsVersion::Tls12,
TlsVersion::Tls13,
vec![&rustls::version::TLS12, &rustls::version::TLS13],
Some(rustls::ProtocolVersion::TLSv1_3),
),
(
TlsVersion::Tls12,
TlsVersion::Tls12,
vec![&rustls::version::TLS13],
None,
),
(
TlsVersion::Tls12,
TlsVersion::Tls12,
vec![&rustls::version::TLS12, &rustls::version::TLS13],
Some(rustls::ProtocolVersion::TLSv1_2),
),
] {
settings.min_version = min_version;
settings.max_version = max_version;
let listener = TlsListener::new(
TcpListener::bind("127.0.0.1:0").await?,
settings.clone(),
Arc::new(IgnoreTlsEvents),
)?;
let client = rustls::ClientConfig::builder_with_provider(Arc::new(provider()))
.with_protocol_versions(&versions)?
.with_root_certificates(roots.clone())
.with_no_client_auth();
let connector = tokio_rustls::TlsConnector::from(Arc::new(client));
let (client_io, server_io) = tokio::io::duplex(16 * 1024);
let (client_result, server_result) =
tokio::time::timeout(Duration::from_secs(5), async {
tokio::join!(
connector.connect("origin.test".try_into().unwrap(), client_io),
listener.acceptor.accept(server_io),
)
})
.await?;
if let Some(expected) = expected {
assert_eq!(
client_result?.get_ref().1.protocol_version(),
Some(expected)
);
assert_eq!(
server_result?.get_ref().1.protocol_version(),
Some(expected)
);
} else {
assert!(client_result.is_err());
assert!(server_result.is_err());
}
}
settings.min_version = TlsVersion::Tls13;
settings.max_version = TlsVersion::Tls12;
assert!(matches!(
TlsListener::new(
TcpListener::bind("127.0.0.1:0").await?,
settings,
Arc::new(IgnoreTlsEvents)
),
Err(TlsError::InvalidVersionRange)
));
std::fs::remove_dir_all(directory)?;
Ok(())
}
#[tokio::test]
async fn a_quic_config_is_tls13_with_early_data_and_custom_alpn() {
let directory = scratch("quic");
let (settings, _) = write_pair(&directory, "origin.test");
let (config, _watch) =
rotating_quic_server_config(settings, &[b"h3"], Arc::new(IgnoreTlsEvents))
.expect("a self-signed pair is usable for QUIC");
assert_eq!(config.alpn_protocols, vec![b"h3".to_vec()]);
assert_eq!(config.max_early_data_size, u32::MAX);
}
#[tokio::test]
async fn a_client_identity_presents_its_certificate_and_reloads_on_rotation() {
let directory = scratch("client-identity");
let (settings, first) = write_pair(&directory, "origin.internal");
let identity = ClientIdentity::new(
settings.certificate.clone(),
settings.key.clone(),
Arc::new(IgnoreTlsEvents),
)
.expect("the initial pair loads");
let presented = |identity: &ClientIdentity| {
ResolvesClientCert::resolve(&*identity.resolver, &[], &[])
.expect("an identity always resolves")
.cert[0]
.to_vec()
};
assert_eq!(presented(&identity), first);
let (_, second) = write_pair(&directory, "origin.internal");
assert_ne!(first, second, "the fixture generated a distinct pair");
for _ in 0..40 {
if presented(&identity) == second {
break;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
assert_eq!(
presented(&identity),
second,
"a rotated client certificate is picked up without a restart"
);
}
#[test]
fn a_client_identity_refuses_an_unreadable_pair_at_startup() {
let directory = scratch("client-identity-missing");
assert!(matches!(
ClientIdentity::new(
directory.join("absent.pem"),
directory.join("absent.key"),
Arc::new(IgnoreTlsEvents),
),
Err(TlsError::Unreadable { .. })
));
}
#[test]
fn a_client_identity_outside_a_runtime_is_an_error_not_a_panic() {
let directory = scratch("client-identity-no-runtime");
let (settings, _) = write_pair(&directory, "client.internal");
assert!(matches!(
ClientIdentity::new(
settings.certificate,
settings.key,
Arc::new(IgnoreTlsEvents),
),
Err(TlsError::NoRuntime { .. })
));
}
#[test]
fn trust_roots_load_from_pem_and_an_empty_file_is_refused() {
let directory = scratch("roots");
let (settings, der) = write_pair(&directory, "authority.internal");
let roots = load_roots(&settings.certificate).expect("the authority loads");
assert_eq!(roots.len(), 1);
assert_eq!(roots[0].to_vec(), der);
let empty = directory.join("empty.pem");
write_atomically(&empty, b"");
assert!(
matches!(load_roots(&empty), Err(TlsError::Empty { .. })),
"a file that parses but names no authority is refused rather than \
leaving an empty trust store that refuses every endpoint"
);
}
}