use std::{collections::HashMap, sync::{Arc, OnceLock, atomic::{AtomicU8, Ordering}}, time::{Duration, Instant}};
use arc_swap::ArcSwapOption;
use rustls::sign::CertifiedKey;
use time::OffsetDateTime;
use tokio::sync::{Mutex, RwLock};
use crate::{AlpnHandlerConfig, CertificateProvenance, DnsName, Error, EventReceiver, Result, StoredCertificate, StoredLocalCa, acme::{AcmeIssuer, ChallengeRegistry, PebbleSupervisor}, event::{EventBus, EventKind, EventLevel}, issuer::{CertIssuer, IssueReason, IssueRequest, LocalCaIssuer}, store::CertStore};
#[cfg(feature = "folder-store")]
use crate::store::FolderCertStore;
#[cfg(feature = "managed-pebble")]
use crate::acme::start_managed_pebble;
pub(crate) static GLOBAL: OnceLock<Arc<Manager>> = OnceLock::new();
pub(crate) static GLOBAL_STATE: AtomicU8 = AtomicU8::new(0);
pub(crate) struct ActiveCertificate { pub record: Arc<StoredCertificate>, pub key: Arc<CertifiedKey> }
pub(crate) struct DomainSlots { pub acme: ArcSwapOption<ActiveCertificate>, pub local: ArcSwapOption<ActiveCertificate> }
impl DomainSlots { fn new() -> Self { Self { acme: ArcSwapOption::empty(), local: ArcSwapOption::empty() } } }
pub(crate) struct Manager {
pub config: AlpnHandlerConfig,
pub store: Arc<dyn CertStore>,
pub events: Arc<EventBus>,
slots: RwLock<HashMap<DnsName, Arc<DomainSlots>>>,
issue_locks: Mutex<HashMap<(DnsName, CertificateProvenance), Arc<Mutex<()>>>>,
retry: Mutex<HashMap<(DnsName, CertificateProvenance), (u32, i64)>>,
pub local_issuer: Arc<LocalCaIssuer>,
pub acme_issuer: Arc<AcmeIssuer>,
pub challenges: Arc<ChallengeRegistry>,
_pebble: Option<PebbleSupervisor>,
}
impl Manager {
pub(crate) async fn new(mut config: AlpnHandlerConfig, challenge_port: u16) -> Result<Arc<Self>> {
config.validate()?;
let events = Arc::new(EventBus::new(config.event_channel_capacity, config.event_history_capacity, config.events, config.event_handler.clone()));
events.emit(EventLevel::Info, EventKind::ManagerInitializing);
if !config.verify_dns_before_issuance { events.emit(EventLevel::Warn, EventKind::DnsVerificationDisabled); }
let store: Arc<dyn CertStore> = if let Some(store) = config.cert_store.take() {
store
} else {
#[cfg(feature = "folder-store")]
{ Arc::new(FolderCertStore::open(config.cert_store_directory.clone()).await?) }
#[cfg(not(feature = "folder-store"))]
{ return Err(Error::InvalidConfig("cert_store is required when folder-store is disabled".into())); }
};
let pebble: Option<PebbleSupervisor> = if let crate::AcmeEndpoint::Pebble(pebble_config) = &config.acme_endpoint {
#[cfg(feature = "managed-pebble")]
{ Some(start_managed_pebble(pebble_config, challenge_port, config.pebble_startup_timeout_secs, &events).await?) }
#[cfg(not(feature = "managed-pebble"))]
{ let _ = (pebble_config, challenge_port); return Err(Error::InvalidConfig("AcmeEndpoint::Pebble requires the managed-pebble feature".into())); }
} else { None };
let pebble_root = pebble.as_ref().map(|value| value.https_root.clone());
let local_issuer = Arc::new(LocalCaIssuer::load_or_create(Arc::clone(&store), config.local_leaf_validity_days, Arc::clone(&events)).await?);
let challenges = Arc::new(ChallengeRegistry::default());
let acme_issuer = Arc::new(AcmeIssuer::new(Arc::clone(&store), config.clone(), Arc::clone(&challenges), Arc::clone(&events), pebble_root)?);
let manager = Arc::new(Self { config, store, events, slots: RwLock::new(HashMap::new()), issue_locks: Mutex::new(HashMap::new()), retry: Mutex::new(HashMap::new()), local_issuer, acme_issuer, challenges, _pebble: pebble });
manager.load_store().await?;
manager.events.emit(EventLevel::Info, EventKind::ManagerReady);
manager.spawn_renewal();
Ok(manager)
}
fn allowed(&self, name: &DnsName) -> bool { self.config.allowed_hosts.iter().any(|p| p.matches(name)) }
async fn domain_slots(&self, name: &DnsName) -> Arc<DomainSlots> {
if let Some(slot) = self.slots.read().await.get(name).cloned() { return slot; }
self.slots.write().await.entry(name.clone()).or_insert_with(|| Arc::new(DomainSlots::new())).clone()
}
async fn load_store(&self) -> Result<()> {
let now = OffsetDateTime::now_utc().unix_timestamp();
for cert in self.store.acme().list_certificates().await? { self.load_record(cert, now).await?; }
let ca = self.local_issuer.ca().await;
for cert in self.store.local().list_certificates().await? {
if cert.local_ca_fingerprint_sha256.as_deref() != Some(&ca.fingerprint_sha256) || crate::cert::validate_local_signature(&cert, &ca).is_err() {
self.store.local().quarantine_certificate(&cert, "local CA fingerprint changed").await?;
self.events.emit(EventLevel::Warn, EventKind::LocalCertificateQuarantined { domain: cert.primary_name, ca_fingerprint: cert.local_ca_fingerprint_sha256.unwrap_or_default() });
} else { self.load_record(cert, now).await?; }
}
Ok(())
}
async fn load_record(&self, cert: StoredCertificate, now: i64) -> Result<()> {
if !cert.is_valid_at(now) { return Ok(()); }
let key = cert.certified_key()?;
let domain = cert.primary_name.clone();
let provenance = cert.provenance;
let active = Arc::new(ActiveCertificate { record: Arc::new(cert), key });
let slots = self.domain_slots(&domain).await;
match provenance { CertificateProvenance::Acme => slots.acme.store(Some(active)), CertificateProvenance::Local => slots.local.store(Some(active)) }
self.events.emit(EventLevel::Info, EventKind::CertificateLoaded { domain, issuer: match provenance { CertificateProvenance::Acme => "acme", CertificateProvenance::Local => "local" }.into() });
Ok(())
}
pub(crate) async fn resolve(&self, name: &DnsName) -> Option<Arc<ActiveCertificate>> {
let slot = self.slots.read().await.get(name).cloned()?;
let now = OffsetDateTime::now_utc().unix_timestamp();
if let Some(value) = slot.acme.load_full() {
if value.record.is_valid_at(now) {
self.events.emit(EventLevel::Debug, EventKind::CertificateSelected { domain: name.clone(), issuer: value.record.issuer_id.clone() });
return Some(value);
}
slot.acme.store(None);
let _ = self.store.acme().remove_certificate(name, &value.record.issuer_id).await;
self.events.emit(EventLevel::Warn, EventKind::CertificateExpiredRemoved { domain: name.clone(), issuer: value.record.issuer_id.clone() });
}
if let Some(value) = slot.local.load_full() {
if value.record.is_valid_at(now) {
self.events.emit(EventLevel::Debug, EventKind::CertificateSelected { domain: name.clone(), issuer: value.record.issuer_id.clone() });
return Some(value);
}
slot.local.store(None);
let _ = self.store.local().remove_certificate(name).await;
self.events.emit(EventLevel::Warn, EventKind::CertificateExpiredRemoved { domain: name.clone(), issuer: value.record.issuer_id.clone() });
}
None
}
pub(crate) async fn ensure(self: &Arc<Self>, name: DnsName) -> Result<Arc<ActiveCertificate>> {
if !self.allowed(&name) { self.events.emit(EventLevel::Warn, EventKind::UnknownDomainRejected { domain: Some(name.clone()) }); return Err(Error::HostNotAllowed(name.to_string())); }
if let Some(cert) = self.resolve(&name).await { return Ok(cert); }
if matches!(self.config.acme_endpoint, crate::AcmeEndpoint::LocalOnly) {
if !self.config.local_fallback_enabled { return Err(Error::InvalidConfig("AcmeEndpoint::LocalOnly requires local_fallback_enabled".into())); }
self.issue_local(name.clone(), IssueReason::Initial, false).await?;
return self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()));
}
if !self.config.local_fallback_enabled { self.issue_acme(name.clone(), IssueReason::Initial, false).await?; return self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string())); }
let acme_manager = Arc::clone(self);
let acme_name = name.clone();
let mut acme_task = tokio::spawn(async move { acme_manager.issue_acme(acme_name, IssueReason::Initial, false).await });
let local = self.issue_local(name.clone(), IssueReason::Initial, false);
tokio::pin!(local);
tokio::select! {
local_result = &mut local => {
local_result?;
self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()))
}
acme_result = &mut acme_task => {
if acme_result.is_ok_and(|result| result.is_ok()) {
if let Some(cert) = self.resolve(&name).await { return Ok(cert); }
}
local.await?;
self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()))
}
}
}
async fn issue_local(&self, name: DnsName, reason: IssueReason, force: bool) -> Result<()> {
let lock = { self.issue_locks.lock().await.entry((name.clone(), CertificateProvenance::Local)).or_insert_with(|| Arc::new(Mutex::new(()))).clone() };
let _guard = lock.lock().await;
if !force && self.domain_slots(&name).await.local.load_full().is_some_and(|value| value.record.is_valid_at(OffsetDateTime::now_utc().unix_timestamp())) { return Ok(()); }
self.events.emit(EventLevel::Info, EventKind::IssuanceStarted { domain: name.clone(), issuer: "AcmeLocalCA".into(), reason: format!("{reason:?}").to_ascii_lowercase() });
let started = Instant::now();
let deadline = tokio::time::Instant::now() + Duration::from_secs(self.config.issuance_attempt_timeout_secs);
let operation = async {
let cert = self.local_issuer.issue(IssueRequest { domain: name.clone(), reason, deadline }).await?;
self.store.local().commit_certificate(&cert).await?;
let not_after = cert.not_after;
self.load_record(cert, OffsetDateTime::now_utc().unix_timestamp()).await?;
Result::<i64>::Ok(not_after)
};
match tokio::time::timeout_at(deadline, operation).await {
Ok(Ok(not_after)) => {
self.events.emit(EventLevel::Info, EventKind::IssuanceSucceeded { domain: name, issuer: "AcmeLocalCA".into(), not_after, elapsed: started.elapsed() });
Ok(())
}
Ok(Err(error)) => {
self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "AcmeLocalCA".into(), error: error.to_string() });
Err(error)
}
Err(_) => {
let error = Error::IssuanceTimeout(name.to_string());
self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "AcmeLocalCA".into(), error: error.to_string() });
Err(error)
}
}
}
async fn issue_acme(self: &Arc<Self>, name: DnsName, reason: IssueReason, force: bool) -> Result<()> {
let lock = { self.issue_locks.lock().await.entry((name.clone(), CertificateProvenance::Acme)).or_insert_with(|| Arc::new(Mutex::new(()))).clone() };
let _guard = lock.lock().await;
if !force && self.domain_slots(&name).await.acme.load_full().is_some_and(|v| v.record.is_valid_at(OffsetDateTime::now_utc().unix_timestamp())) { return Ok(()); }
self.events.emit(EventLevel::Info, EventKind::IssuanceStarted { domain: name.clone(), issuer: "acme".into(), reason: format!("{reason:?}").to_ascii_lowercase() });
let started = Instant::now();
let deadline = tokio::time::Instant::now() + Duration::from_secs(self.config.issuance_attempt_timeout_secs);
let operation = async {
let cert = self.acme_issuer.issue(IssueRequest { domain: name.clone(), reason, deadline }).await?;
let issuer = cert.issuer_id.clone();
let not_after = cert.not_after;
self.store.acme().commit_certificate(&cert).await?;
self.load_record(cert, OffsetDateTime::now_utc().unix_timestamp()).await?;
Result::<(String, i64)>::Ok((issuer, not_after))
};
match tokio::time::timeout_at(deadline, operation).await {
Ok(Ok((issuer, not_after))) => {
self.events.emit(EventLevel::Info, EventKind::IssuanceSucceeded { domain: name, issuer, not_after, elapsed: started.elapsed() });
Ok(())
}
Ok(Err(error)) => { self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "acme".into(), error: error.to_string() }); Err(error) }
Err(_) => {
let error = Error::IssuanceTimeout(name.to_string());
self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "acme".into(), error: error.to_string() });
Err(error)
}
}
}
pub(crate) async fn install_local_ca(&self, cert: Vec<u8>, key: Vec<u8>) -> Result<()> {
let replacement = StoredLocalCa::from_pem(String::from_utf8(cert).map_err(|e| Error::LocalCa(e.to_string()))?, String::from_utf8(key).map_err(|e| Error::LocalCa(e.to_string()))?)?;
self.local_issuer.install(replacement).await?;
self.invalidate_local().await
}
pub(crate) async fn reload_local_ca(&self) -> Result<()> { if self.local_issuer.reload().await?.is_some() { self.invalidate_local().await?; } Ok(()) }
async fn invalidate_local(&self) -> Result<()> {
let ca = self.local_issuer.ca().await;
for cert in self.store.local().list_certificates().await? {
if cert.local_ca_fingerprint_sha256.as_deref() != Some(&ca.fingerprint_sha256) {
if let Some(slot) = self.slots.read().await.get(&cert.primary_name).cloned() { slot.local.store(None); }
self.store.local().quarantine_certificate(&cert, "local CA replaced").await?;
self.events.emit(EventLevel::Warn, EventKind::LocalCertificateQuarantined { domain: cert.primary_name, ca_fingerprint: cert.local_ca_fingerprint_sha256.unwrap_or_default() });
}
}
Ok(())
}
fn spawn_renewal(self: &Arc<Self>) {
let manager = Arc::clone(self);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(manager.config.renew_check_interval_secs));
interval.tick().await;
loop { interval.tick().await; if let Err(error) = manager.renew_due().await { manager.events.emit(EventLevel::Error, EventKind::RenewalFailed { domain: DnsName::parse("invalid.local").expect("literal"), issuer: "scheduler".into(), error: error.to_string(), retry_at: OffsetDateTime::now_utc().unix_timestamp() + i64::try_from(manager.config.renew_check_interval_secs).unwrap_or(i64::MAX) }); } }
});
}
async fn renew_due(self: &Arc<Self>) -> Result<()> {
let values: Vec<_> = self.slots.read().await.iter().flat_map(|(name, slots)| {
[(CertificateProvenance::Acme, slots.acme.load_full()), (CertificateProvenance::Local, slots.local.load_full())]
.into_iter().filter_map(|(provenance, cert)| cert.map(|cert| (name.clone(), provenance, cert))).collect::<Vec<_>>()
}).collect();
self.events.emit(EventLevel::Info, EventKind::RenewalScanStarted { known_variants: values.len() });
let threshold = i64::from(self.config.renew_days_before_expiry) * 86_400;
let now = OffsetDateTime::now_utc().unix_timestamp();
let mut tasks = Vec::new();
for (name, provenance, cert) in values {
let remaining = cert.record.not_after - now;
self.events.emit(EventLevel::Info, EventKind::CertificateChecked { domain: name.clone(), issuer: cert.record.issuer_id.clone(), not_after: cert.record.not_after, days_remaining: remaining / 86_400 });
let retry_ready = self.retry.lock().await.get(&(name.clone(), provenance)).is_none_or(|(_, retry_at)| *retry_at <= now);
if remaining <= threshold && retry_ready {
self.events.emit(EventLevel::Info, EventKind::RenewalDue { domain: name.clone(), issuer: cert.record.issuer_id.clone(), not_after: cert.record.not_after });
let manager = Arc::clone(self);
tasks.push(tokio::spawn(async move { manager.renew_variant(name, provenance).await }));
}
}
for task in tasks { let _ = task.await; }
Ok(())
}
async fn renew_variant(self: &Arc<Self>, name: DnsName, provenance: CertificateProvenance) -> Result<()> {
let attempt = self.retry.lock().await.get(&(name.clone(), provenance)).map_or(1, |(failures, _)| failures.saturating_add(1));
self.events.emit(EventLevel::Info, EventKind::RenewalStarted { domain: name.clone(), issuer: format!("{provenance:?}"), attempt });
let started = Instant::now();
let result = match provenance {
CertificateProvenance::Acme => self.issue_acme(name.clone(), IssueReason::Renewal, true).await,
CertificateProvenance::Local => self.renew_local(name.clone()).await,
};
match result {
Ok(()) => {
self.retry.lock().await.remove(&(name.clone(), provenance));
let not_after = self.resolve(&name).await.map_or(0, |value| value.record.not_after);
self.events.emit(EventLevel::Info, EventKind::RenewalSucceeded { domain: name, issuer: format!("{provenance:?}"), not_after, elapsed: started.elapsed() });
Ok(())
}
Err(error) => {
let retry_at = self.schedule_retry(name.clone(), provenance).await;
self.events.emit(EventLevel::Error, EventKind::RenewalFailed { domain: name, issuer: format!("{provenance:?}"), error: error.to_string(), retry_at });
Err(error)
}
}
}
async fn schedule_retry(&self, name: DnsName, provenance: CertificateProvenance) -> i64 {
let mut retry = self.retry.lock().await;
let failures = retry.get(&(name.clone(), provenance)).map_or(1, |(count, _)| count.saturating_add(1));
let factor = 1_u64.checked_shl(failures.saturating_sub(1).min(20)).unwrap_or(u64::MAX);
let delay = self.config.renew_check_interval_secs.saturating_mul(factor).min(self.config.renew_check_max_interval_secs);
let retry_at = OffsetDateTime::now_utc().unix_timestamp().saturating_add(i64::try_from(delay).unwrap_or(i64::MAX));
retry.insert((name, provenance), (failures, retry_at));
retry_at
}
async fn renew_local(&self, name: DnsName) -> Result<()> {
self.issue_local(name, IssueReason::Renewal, true).await
}
}
pub(crate) fn global() -> Result<&'static Arc<Manager>> {
match GLOBAL_STATE.load(Ordering::Acquire) { 0 => Err(Error::AlpnHandlerNotRegistered), 1 => Err(Error::AlpnHandlerNotReady), _ => GLOBAL.get().ok_or(Error::AlpnHandlerNotReady) }
}
pub fn subscribe_events() -> Result<EventReceiver> { Ok(global()?.events.subscribe()) }
pub async fn get_certificate(name: &str) -> Result<Arc<StoredCertificate>> { let name = DnsName::parse(name)?; global()?.resolve(&name).await.map(|v| v.record.clone()).ok_or_else(|| Error::NoCertificate(name.to_string())) }
pub async fn list_certificates() -> Result<Vec<Arc<StoredCertificate>>> { let manager = global()?; let slots: Vec<_> = manager.slots.read().await.values().cloned().collect(); let now = OffsetDateTime::now_utc().unix_timestamp(); Ok(slots.into_iter().flat_map(|slot| [slot.acme.load_full(), slot.local.load_full()]).flatten().filter(|v| v.record.is_valid_at(now)).map(|v| v.record.clone()).collect()) }
pub async fn install_local_ca(cert_pem: Vec<u8>, key_pem: Vec<u8>) -> Result<()> { global()?.install_local_ca(cert_pem, key_pem).await }
pub async fn reload_local_ca() -> Result<()> { global()?.reload_local_ca().await }
#[cfg(test)]
mod tests {
use super::*;
use crate::{AcmeEndpoint, EventOutput, HostPattern, IssueReason, event::EventKind};
use url::Url;
async fn manager(local_validity_days: u32, renew_days: u32) -> (tempfile::TempDir, Arc<Manager>, DnsName) {
let directory = tempfile::tempdir().unwrap();
let domain = DnsName::parse("lifecycle.test").unwrap();
let mut config = AlpnHandlerConfig::default();
config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
config.verify_dns_before_issuance = false;
config.cert_store_directory = directory.path().to_owned();
config.events = EventOutput::Subscribe;
config.local_leaf_validity_days = local_validity_days;
config.renew_days_before_expiry = renew_days;
config.renew_check_interval_secs = 1;
config.renew_check_max_interval_secs = 4;
config.issuance_attempt_timeout_secs = 2;
config.acme_endpoint = AcmeEndpoint::Custom {
directory_url: Url::parse("http://127.0.0.1:9/directory").unwrap(),
issuer_id: "unreachable-test-acme".into(),
tls_trust_roots: Vec::new(),
};
let manager = Manager::new(config, 0).await.unwrap();
(directory, manager, domain)
}
fn drain(receiver: &mut EventReceiver) -> Vec<Arc<crate::Event>> {
let mut events = Vec::new();
while let Some(event) = receiver.try_recv().unwrap() { events.push(event); }
events
}
#[tokio::test]
async fn unreachable_acme_falls_back_to_local_and_emits_selection_events() {
let (_directory, manager, domain) = manager(30, 15).await;
let mut events = manager.events.subscribe();
let certificate = manager.ensure(domain.clone()).await.unwrap();
assert_eq!(certificate.record.provenance, CertificateProvenance::Local);
assert!(manager.store.local().load_certificate(&domain).await.unwrap().is_some());
let _ = manager.resolve(&domain).await.unwrap();
let events = drain(&mut events);
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceStarted { domain: value, issuer, .. } if value == &domain && issuer == "AcmeLocalCA")));
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceSucceeded { domain: value, issuer, .. } if value == &domain && issuer == "AcmeLocalCA")));
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::CertificateSelected { domain: value, issuer } if value == &domain && issuer == "AcmeLocalCA")));
}
#[tokio::test]
async fn local_only_never_starts_acme_issuance() {
let directory = tempfile::tempdir().unwrap();
let domain = DnsName::parse("private.internal").unwrap();
let mut config = AlpnHandlerConfig::default();
config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
config.verify_dns_before_issuance = false;
config.acme_endpoint = AcmeEndpoint::LocalOnly;
config.cert_store_directory = directory.path().to_owned();
config.events = EventOutput::Subscribe;
let manager = Manager::new(config, 0).await.unwrap();
let mut events = manager.events.subscribe();
assert_eq!(manager.ensure(domain.clone()).await.unwrap().record.provenance, CertificateProvenance::Local);
let events = drain(&mut events);
assert!(!events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceStarted { issuer, .. } if issuer == "acme")));
assert!(manager.store.acme().list_certificates().await.unwrap().is_empty());
}
#[tokio::test]
async fn expired_certificate_is_removed_from_memory_and_store() {
let (_directory, manager, domain) = manager(30, 15).await;
manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
let active = manager.resolve(&domain).await.unwrap();
let mut expired = (*active.record).clone();
expired.not_after = OffsetDateTime::now_utc().unix_timestamp() - 1;
manager.store.local().commit_certificate(&expired).await.unwrap();
let slots = manager.domain_slots(&domain).await;
slots.local.store(Some(Arc::new(ActiveCertificate { record: Arc::new(expired), key: active.key.clone() })));
let mut events = manager.events.subscribe();
assert!(manager.resolve(&domain).await.is_none());
assert!(manager.store.local().load_certificate(&domain).await.unwrap().is_none());
assert!(drain(&mut events).iter().any(|event| matches!(&event.kind, EventKind::CertificateExpiredRemoved { domain: value, .. } if value == &domain)));
}
#[tokio::test]
async fn aggressive_expiry_policy_renews_due_local_certificate() {
let (_directory, manager, domain) = manager(1, 2).await;
manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
let before = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
let mut events = manager.events.subscribe();
manager.renew_due().await.unwrap();
let after = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
assert_ne!(before, after);
let events = drain(&mut events);
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalDue { domain: value, .. } if value == &domain)));
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalStarted { domain: value, .. } if value == &domain)));
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalSucceeded { domain: value, .. } if value == &domain)));
}
#[tokio::test]
async fn retry_backoff_doubles_and_caps_per_domain_and_issuer() {
let (_directory, manager, domain) = manager(30, 15).await;
let now = OffsetDateTime::now_utc().unix_timestamp();
let first = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
let second = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
let third = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
let fourth = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
assert!((1..=2).contains(&(first - now)));
assert!((2..=3).contains(&(second - now)));
assert!((4..=5).contains(&(third - now)));
assert!((4..=5).contains(&(fourth - now)), "backoff must remain capped");
assert_eq!(manager.retry.lock().await.get(&(domain, CertificateProvenance::Acme)).unwrap().0, 4);
}
#[tokio::test]
async fn failed_acme_renewal_emits_failure_and_schedules_retry() {
let (_directory, manager, domain) = manager(30, 15).await;
let mut events = manager.events.subscribe();
assert!(manager.renew_variant(domain.clone(), CertificateProvenance::Acme).await.is_err());
assert!(manager.retry.lock().await.contains_key(&(domain.clone(), CertificateProvenance::Acme)));
let events = drain(&mut events);
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalStarted { domain: value, attempt: 1, .. } if value == &domain)));
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceFailed { domain: value, .. } if value == &domain)));
assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalFailed { domain: value, retry_at, .. } if value == &domain && *retry_at > OffsetDateTime::now_utc().unix_timestamp())));
}
#[tokio::test]
async fn retry_at_suppresses_due_scan_and_success_resets_backoff() {
let (_directory, manager, domain) = manager(1, 2).await;
manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
let before = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
manager.retry.lock().await.insert(
(domain.clone(), CertificateProvenance::Local),
(2, OffsetDateTime::now_utc().unix_timestamp() + 3_600),
);
manager.renew_due().await.unwrap();
assert_eq!(manager.resolve(&domain).await.unwrap().record.fingerprint_sha256, before);
manager.retry.lock().await.get_mut(&(domain.clone(), CertificateProvenance::Local)).unwrap().1 =
OffsetDateTime::now_utc().unix_timestamp() - 1;
manager.renew_due().await.unwrap();
assert_ne!(manager.resolve(&domain).await.unwrap().record.fingerprint_sha256, before);
assert!(!manager.retry.lock().await.contains_key(&(domain, CertificateProvenance::Local)));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn hanging_acme_renewal_obeys_absolute_attempt_deadline() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let hanging_server = tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
tokio::spawn(async move {
let _stream = stream;
tokio::time::sleep(Duration::from_secs(10)).await;
});
}
});
let directory = tempfile::tempdir().unwrap();
let domain = DnsName::parse("deadline.test").unwrap();
let mut config = AlpnHandlerConfig::default();
config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
config.verify_dns_before_issuance = false;
config.cert_store_directory = directory.path().to_owned();
config.events = EventOutput::Subscribe;
config.issuance_attempt_timeout_secs = 1;
config.acme_endpoint = AcmeEndpoint::Custom {
directory_url: Url::parse(&format!("https://{address}/directory")).unwrap(),
issuer_id: "hanging-acme".into(),
tls_trust_roots: Vec::new(),
};
let manager = Manager::new(config, 0).await.unwrap();
let started = tokio::time::Instant::now();
let error = manager.renew_variant(domain.clone(), CertificateProvenance::Acme).await.unwrap_err();
let elapsed = started.elapsed();
assert!(matches!(error, Error::IssuanceTimeout(_)), "unexpected error: {error}");
assert!(elapsed >= Duration::from_millis(900));
assert!(elapsed < Duration::from_secs(2), "attempt exceeded configured deadline: {elapsed:?}");
assert!(manager.retry.lock().await.contains_key(&(domain, CertificateProvenance::Acme)));
hanging_server.abort();
}
}