Skip to main content

rustls_letsencrypt/
manager.rs

1use std::{collections::HashMap, sync::{Arc, OnceLock, atomic::{AtomicU8, Ordering}}, time::{Duration, Instant}};
2
3use arc_swap::ArcSwapOption;
4use rustls::sign::CertifiedKey;
5use time::OffsetDateTime;
6use tokio::sync::{Mutex, RwLock};
7
8use crate::{AlpnHandlerConfig, CertificateProvenance, DnsName, Error, EventReceiver, Result, StoredCertificate, StoredLocalCa, acme::{AcmeIssuer, ChallengeRegistry, PebbleSupervisor}, event::{EventBus, EventKind, EventLevel}, issuer::{CertIssuer, IssueReason, IssueRequest, LocalCaIssuer}, store::CertStore};
9#[cfg(feature = "folder-store")]
10use crate::store::FolderCertStore;
11#[cfg(feature = "managed-pebble")]
12use crate::acme::start_managed_pebble;
13
14pub(crate) static GLOBAL: OnceLock<Arc<Manager>> = OnceLock::new();
15pub(crate) static GLOBAL_STATE: AtomicU8 = AtomicU8::new(0); // 0 absent, 1 initializing, 2 ready
16
17pub(crate) struct ActiveCertificate { pub record: Arc<StoredCertificate>, pub key: Arc<CertifiedKey> }
18pub(crate) struct DomainSlots { pub acme: ArcSwapOption<ActiveCertificate>, pub local: ArcSwapOption<ActiveCertificate> }
19impl DomainSlots { fn new() -> Self { Self { acme: ArcSwapOption::empty(), local: ArcSwapOption::empty() } } }
20
21pub(crate) struct Manager {
22    pub config: AlpnHandlerConfig,
23    pub store: Arc<dyn CertStore>,
24    pub events: Arc<EventBus>,
25    slots: RwLock<HashMap<DnsName, Arc<DomainSlots>>>,
26    issue_locks: Mutex<HashMap<(DnsName, CertificateProvenance), Arc<Mutex<()>>>>,
27    retry: Mutex<HashMap<(DnsName, CertificateProvenance), (u32, i64)>>,
28    pub local_issuer: Arc<LocalCaIssuer>,
29    pub acme_issuer: Arc<AcmeIssuer>,
30    pub challenges: Arc<ChallengeRegistry>,
31    _pebble: Option<PebbleSupervisor>,
32}
33
34impl Manager {
35    pub(crate) async fn new(mut config: AlpnHandlerConfig, challenge_port: u16) -> Result<Arc<Self>> {
36        config.validate()?;
37        let events = Arc::new(EventBus::new(config.event_channel_capacity, config.event_history_capacity, config.events, config.event_handler.clone()));
38        events.emit(EventLevel::Info, EventKind::ManagerInitializing);
39        if !config.verify_dns_before_issuance { events.emit(EventLevel::Warn, EventKind::DnsVerificationDisabled); }
40        let store: Arc<dyn CertStore> = if let Some(store) = config.cert_store.take() {
41            store
42        } else {
43            #[cfg(feature = "folder-store")]
44            { Arc::new(FolderCertStore::open(config.cert_store_directory.clone()).await?) }
45            #[cfg(not(feature = "folder-store"))]
46            { return Err(Error::InvalidConfig("cert_store is required when folder-store is disabled".into())); }
47        };
48        let pebble: Option<PebbleSupervisor> = if let crate::AcmeEndpoint::Pebble(pebble_config) = &config.acme_endpoint {
49            #[cfg(feature = "managed-pebble")]
50            { Some(start_managed_pebble(pebble_config, challenge_port, config.pebble_startup_timeout_secs, &events).await?) }
51            #[cfg(not(feature = "managed-pebble"))]
52            { let _ = (pebble_config, challenge_port); return Err(Error::InvalidConfig("AcmeEndpoint::Pebble requires the managed-pebble feature".into())); }
53        } else { None };
54        let pebble_root = pebble.as_ref().map(|value| value.https_root.clone());
55        let local_issuer = Arc::new(LocalCaIssuer::load_or_create(Arc::clone(&store), config.local_leaf_validity_days, Arc::clone(&events)).await?);
56        let challenges = Arc::new(ChallengeRegistry::default());
57        let acme_issuer = Arc::new(AcmeIssuer::new(Arc::clone(&store), config.clone(), Arc::clone(&challenges), Arc::clone(&events), pebble_root)?);
58        let manager = Arc::new(Self { config, store, events, slots: RwLock::new(HashMap::new()), issue_locks: Mutex::new(HashMap::new()), retry: Mutex::new(HashMap::new()), local_issuer, acme_issuer, challenges, _pebble: pebble });
59        manager.load_store().await?;
60        manager.events.emit(EventLevel::Info, EventKind::ManagerReady);
61        manager.spawn_renewal();
62        Ok(manager)
63    }
64
65    fn allowed(&self, name: &DnsName) -> bool { self.config.allowed_hosts.iter().any(|p| p.matches(name)) }
66
67    async fn domain_slots(&self, name: &DnsName) -> Arc<DomainSlots> {
68        if let Some(slot) = self.slots.read().await.get(name).cloned() { return slot; }
69        self.slots.write().await.entry(name.clone()).or_insert_with(|| Arc::new(DomainSlots::new())).clone()
70    }
71
72    async fn load_store(&self) -> Result<()> {
73        let now = OffsetDateTime::now_utc().unix_timestamp();
74        for cert in self.store.acme().list_certificates().await? { self.load_record(cert, now).await?; }
75        let ca = self.local_issuer.ca().await;
76        for cert in self.store.local().list_certificates().await? {
77            if cert.local_ca_fingerprint_sha256.as_deref() != Some(&ca.fingerprint_sha256) || crate::cert::validate_local_signature(&cert, &ca).is_err() {
78                self.store.local().quarantine_certificate(&cert, "local CA fingerprint changed").await?;
79                self.events.emit(EventLevel::Warn, EventKind::LocalCertificateQuarantined { domain: cert.primary_name, ca_fingerprint: cert.local_ca_fingerprint_sha256.unwrap_or_default() });
80            } else { self.load_record(cert, now).await?; }
81        }
82        Ok(())
83    }
84
85    async fn load_record(&self, cert: StoredCertificate, now: i64) -> Result<()> {
86        if !cert.is_valid_at(now) { return Ok(()); }
87        let key = cert.certified_key()?;
88        let domain = cert.primary_name.clone();
89        let provenance = cert.provenance;
90        let active = Arc::new(ActiveCertificate { record: Arc::new(cert), key });
91        let slots = self.domain_slots(&domain).await;
92        match provenance { CertificateProvenance::Acme => slots.acme.store(Some(active)), CertificateProvenance::Local => slots.local.store(Some(active)) }
93        self.events.emit(EventLevel::Info, EventKind::CertificateLoaded { domain, issuer: match provenance { CertificateProvenance::Acme => "acme", CertificateProvenance::Local => "local" }.into() });
94        Ok(())
95    }
96
97    pub(crate) async fn resolve(&self, name: &DnsName) -> Option<Arc<ActiveCertificate>> {
98        let slot = self.slots.read().await.get(name).cloned()?;
99        let now = OffsetDateTime::now_utc().unix_timestamp();
100        if let Some(value) = slot.acme.load_full() {
101            if value.record.is_valid_at(now) {
102                self.events.emit(EventLevel::Debug, EventKind::CertificateSelected { domain: name.clone(), issuer: value.record.issuer_id.clone() });
103                return Some(value);
104            }
105            slot.acme.store(None);
106            let _ = self.store.acme().remove_certificate(name, &value.record.issuer_id).await;
107            self.events.emit(EventLevel::Warn, EventKind::CertificateExpiredRemoved { domain: name.clone(), issuer: value.record.issuer_id.clone() });
108        }
109        if let Some(value) = slot.local.load_full() {
110            if value.record.is_valid_at(now) {
111                self.events.emit(EventLevel::Debug, EventKind::CertificateSelected { domain: name.clone(), issuer: value.record.issuer_id.clone() });
112                return Some(value);
113            }
114            slot.local.store(None);
115            let _ = self.store.local().remove_certificate(name).await;
116            self.events.emit(EventLevel::Warn, EventKind::CertificateExpiredRemoved { domain: name.clone(), issuer: value.record.issuer_id.clone() });
117        }
118        None
119    }
120
121    pub(crate) async fn ensure(self: &Arc<Self>, name: DnsName) -> Result<Arc<ActiveCertificate>> {
122        if !self.allowed(&name) { self.events.emit(EventLevel::Warn, EventKind::UnknownDomainRejected { domain: Some(name.clone()) }); return Err(Error::HostNotAllowed(name.to_string())); }
123        if let Some(cert) = self.resolve(&name).await { return Ok(cert); }
124        if matches!(self.config.acme_endpoint, crate::AcmeEndpoint::LocalOnly) {
125            if !self.config.local_fallback_enabled { return Err(Error::InvalidConfig("AcmeEndpoint::LocalOnly requires local_fallback_enabled".into())); }
126            self.issue_local(name.clone(), IssueReason::Initial, false).await?;
127            return self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()));
128        }
129        if !self.config.local_fallback_enabled { self.issue_acme(name.clone(), IssueReason::Initial, false).await?; return self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string())); }
130        let acme_manager = Arc::clone(self);
131        let acme_name = name.clone();
132        let mut acme_task = tokio::spawn(async move { acme_manager.issue_acme(acme_name, IssueReason::Initial, false).await });
133        let local = self.issue_local(name.clone(), IssueReason::Initial, false);
134        tokio::pin!(local);
135        tokio::select! {
136            local_result = &mut local => {
137                local_result?;
138                self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()))
139            }
140            acme_result = &mut acme_task => {
141                if acme_result.is_ok_and(|result| result.is_ok()) {
142                    if let Some(cert) = self.resolve(&name).await { return Ok(cert); }
143                }
144                local.await?;
145                self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()))
146            }
147        }
148    }
149
150    async fn issue_local(&self, name: DnsName, reason: IssueReason, force: bool) -> Result<()> {
151        let lock = { self.issue_locks.lock().await.entry((name.clone(), CertificateProvenance::Local)).or_insert_with(|| Arc::new(Mutex::new(()))).clone() };
152        let _guard = lock.lock().await;
153        if !force && self.domain_slots(&name).await.local.load_full().is_some_and(|value| value.record.is_valid_at(OffsetDateTime::now_utc().unix_timestamp())) { return Ok(()); }
154        self.events.emit(EventLevel::Info, EventKind::IssuanceStarted { domain: name.clone(), issuer: "AcmeLocalCA".into(), reason: format!("{reason:?}").to_ascii_lowercase() });
155        let started = Instant::now();
156        let deadline = tokio::time::Instant::now() + Duration::from_secs(self.config.issuance_attempt_timeout_secs);
157        let operation = async {
158            let cert = self.local_issuer.issue(IssueRequest { domain: name.clone(), reason, deadline }).await?;
159            self.store.local().commit_certificate(&cert).await?;
160            let not_after = cert.not_after;
161            self.load_record(cert, OffsetDateTime::now_utc().unix_timestamp()).await?;
162            Result::<i64>::Ok(not_after)
163        };
164        match tokio::time::timeout_at(deadline, operation).await {
165            Ok(Ok(not_after)) => {
166                self.events.emit(EventLevel::Info, EventKind::IssuanceSucceeded { domain: name, issuer: "AcmeLocalCA".into(), not_after, elapsed: started.elapsed() });
167                Ok(())
168            }
169            Ok(Err(error)) => {
170                self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "AcmeLocalCA".into(), error: error.to_string() });
171                Err(error)
172            }
173            Err(_) => {
174                let error = Error::IssuanceTimeout(name.to_string());
175                self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "AcmeLocalCA".into(), error: error.to_string() });
176                Err(error)
177            }
178        }
179    }
180
181    async fn issue_acme(self: &Arc<Self>, name: DnsName, reason: IssueReason, force: bool) -> Result<()> {
182        let lock = { self.issue_locks.lock().await.entry((name.clone(), CertificateProvenance::Acme)).or_insert_with(|| Arc::new(Mutex::new(()))).clone() };
183        let _guard = lock.lock().await;
184        if !force && self.domain_slots(&name).await.acme.load_full().is_some_and(|v| v.record.is_valid_at(OffsetDateTime::now_utc().unix_timestamp())) { return Ok(()); }
185        self.events.emit(EventLevel::Info, EventKind::IssuanceStarted { domain: name.clone(), issuer: "acme".into(), reason: format!("{reason:?}").to_ascii_lowercase() });
186        let started = Instant::now();
187        let deadline = tokio::time::Instant::now() + Duration::from_secs(self.config.issuance_attempt_timeout_secs);
188        let operation = async {
189            let cert = self.acme_issuer.issue(IssueRequest { domain: name.clone(), reason, deadline }).await?;
190            let issuer = cert.issuer_id.clone();
191            let not_after = cert.not_after;
192            self.store.acme().commit_certificate(&cert).await?;
193            self.load_record(cert, OffsetDateTime::now_utc().unix_timestamp()).await?;
194            Result::<(String, i64)>::Ok((issuer, not_after))
195        };
196        match tokio::time::timeout_at(deadline, operation).await {
197            Ok(Ok((issuer, not_after))) => {
198                self.events.emit(EventLevel::Info, EventKind::IssuanceSucceeded { domain: name, issuer, not_after, elapsed: started.elapsed() });
199                Ok(())
200            }
201            Ok(Err(error)) => { self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "acme".into(), error: error.to_string() }); Err(error) }
202            Err(_) => {
203                let error = Error::IssuanceTimeout(name.to_string());
204                self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "acme".into(), error: error.to_string() });
205                Err(error)
206            }
207        }
208    }
209
210    pub(crate) async fn install_local_ca(&self, cert: Vec<u8>, key: Vec<u8>) -> Result<()> {
211        let replacement = StoredLocalCa::from_pem(String::from_utf8(cert).map_err(|e| Error::LocalCa(e.to_string()))?, String::from_utf8(key).map_err(|e| Error::LocalCa(e.to_string()))?)?;
212        self.local_issuer.install(replacement).await?;
213        self.invalidate_local().await
214    }
215    pub(crate) async fn reload_local_ca(&self) -> Result<()> { if self.local_issuer.reload().await?.is_some() { self.invalidate_local().await?; } Ok(()) }
216    async fn invalidate_local(&self) -> Result<()> {
217        let ca = self.local_issuer.ca().await;
218        for cert in self.store.local().list_certificates().await? {
219            if cert.local_ca_fingerprint_sha256.as_deref() != Some(&ca.fingerprint_sha256) {
220                if let Some(slot) = self.slots.read().await.get(&cert.primary_name).cloned() { slot.local.store(None); }
221                self.store.local().quarantine_certificate(&cert, "local CA replaced").await?;
222                self.events.emit(EventLevel::Warn, EventKind::LocalCertificateQuarantined { domain: cert.primary_name, ca_fingerprint: cert.local_ca_fingerprint_sha256.unwrap_or_default() });
223            }
224        }
225        Ok(())
226    }
227
228    fn spawn_renewal(self: &Arc<Self>) {
229        let manager = Arc::clone(self);
230        tokio::spawn(async move {
231            let mut interval = tokio::time::interval(Duration::from_secs(manager.config.renew_check_interval_secs));
232            interval.tick().await;
233            loop { interval.tick().await; if let Err(error) = manager.renew_due().await { manager.events.emit(EventLevel::Error, EventKind::RenewalFailed { domain: DnsName::parse("invalid.local").expect("literal"), issuer: "scheduler".into(), error: error.to_string(), retry_at: OffsetDateTime::now_utc().unix_timestamp() + i64::try_from(manager.config.renew_check_interval_secs).unwrap_or(i64::MAX) }); } }
234        });
235    }
236    async fn renew_due(self: &Arc<Self>) -> Result<()> {
237        let values: Vec<_> = self.slots.read().await.iter().flat_map(|(name, slots)| {
238            [(CertificateProvenance::Acme, slots.acme.load_full()), (CertificateProvenance::Local, slots.local.load_full())]
239                .into_iter().filter_map(|(provenance, cert)| cert.map(|cert| (name.clone(), provenance, cert))).collect::<Vec<_>>()
240        }).collect();
241        self.events.emit(EventLevel::Info, EventKind::RenewalScanStarted { known_variants: values.len() });
242        let threshold = i64::from(self.config.renew_days_before_expiry) * 86_400;
243        let now = OffsetDateTime::now_utc().unix_timestamp();
244        let mut tasks = Vec::new();
245        for (name, provenance, cert) in values {
246            let remaining = cert.record.not_after - now;
247            self.events.emit(EventLevel::Info, EventKind::CertificateChecked { domain: name.clone(), issuer: cert.record.issuer_id.clone(), not_after: cert.record.not_after, days_remaining: remaining / 86_400 });
248            let retry_ready = self.retry.lock().await.get(&(name.clone(), provenance)).is_none_or(|(_, retry_at)| *retry_at <= now);
249            if remaining <= threshold && retry_ready {
250                self.events.emit(EventLevel::Info, EventKind::RenewalDue { domain: name.clone(), issuer: cert.record.issuer_id.clone(), not_after: cert.record.not_after });
251                let manager = Arc::clone(self);
252                tasks.push(tokio::spawn(async move { manager.renew_variant(name, provenance).await }));
253            }
254        }
255        for task in tasks { let _ = task.await; }
256        Ok(())
257    }
258
259    async fn renew_variant(self: &Arc<Self>, name: DnsName, provenance: CertificateProvenance) -> Result<()> {
260        let attempt = self.retry.lock().await.get(&(name.clone(), provenance)).map_or(1, |(failures, _)| failures.saturating_add(1));
261        self.events.emit(EventLevel::Info, EventKind::RenewalStarted { domain: name.clone(), issuer: format!("{provenance:?}"), attempt });
262        let started = Instant::now();
263        let result = match provenance {
264            CertificateProvenance::Acme => self.issue_acme(name.clone(), IssueReason::Renewal, true).await,
265            CertificateProvenance::Local => self.renew_local(name.clone()).await,
266        };
267        match result {
268            Ok(()) => {
269                self.retry.lock().await.remove(&(name.clone(), provenance));
270                let not_after = self.resolve(&name).await.map_or(0, |value| value.record.not_after);
271                self.events.emit(EventLevel::Info, EventKind::RenewalSucceeded { domain: name, issuer: format!("{provenance:?}"), not_after, elapsed: started.elapsed() });
272                Ok(())
273            }
274            Err(error) => {
275                let retry_at = self.schedule_retry(name.clone(), provenance).await;
276                self.events.emit(EventLevel::Error, EventKind::RenewalFailed { domain: name, issuer: format!("{provenance:?}"), error: error.to_string(), retry_at });
277                Err(error)
278            }
279        }
280    }
281
282    async fn schedule_retry(&self, name: DnsName, provenance: CertificateProvenance) -> i64 {
283        let mut retry = self.retry.lock().await;
284        let failures = retry.get(&(name.clone(), provenance)).map_or(1, |(count, _)| count.saturating_add(1));
285        let factor = 1_u64.checked_shl(failures.saturating_sub(1).min(20)).unwrap_or(u64::MAX);
286        let delay = self.config.renew_check_interval_secs.saturating_mul(factor).min(self.config.renew_check_max_interval_secs);
287        let retry_at = OffsetDateTime::now_utc().unix_timestamp().saturating_add(i64::try_from(delay).unwrap_or(i64::MAX));
288        retry.insert((name, provenance), (failures, retry_at));
289        retry_at
290    }
291    async fn renew_local(&self, name: DnsName) -> Result<()> {
292        self.issue_local(name, IssueReason::Renewal, true).await
293    }
294}
295
296pub(crate) fn global() -> Result<&'static Arc<Manager>> {
297    match GLOBAL_STATE.load(Ordering::Acquire) { 0 => Err(Error::AlpnHandlerNotRegistered), 1 => Err(Error::AlpnHandlerNotReady), _ => GLOBAL.get().ok_or(Error::AlpnHandlerNotReady) }
298}
299
300pub fn subscribe_events() -> Result<EventReceiver> { Ok(global()?.events.subscribe()) }
301pub async fn get_certificate(name: &str) -> Result<Arc<StoredCertificate>> { let name = DnsName::parse(name)?; global()?.resolve(&name).await.map(|v| v.record.clone()).ok_or_else(|| Error::NoCertificate(name.to_string())) }
302/// Return a valid certificate, issuing one on demand when the cache is empty.
303pub async fn get_or_issue_certificate(name: &str) -> Result<Arc<StoredCertificate>> { let name = DnsName::parse(name)?; Ok(global()?.ensure(name).await?.record.clone()) }
304pub async fn list_certificates() -> Result<Vec<Arc<StoredCertificate>>> { let manager = global()?; let slots: Vec<_> = manager.slots.read().await.values().cloned().collect(); let now = OffsetDateTime::now_utc().unix_timestamp(); Ok(slots.into_iter().flat_map(|slot| [slot.acme.load_full(), slot.local.load_full()]).flatten().filter(|v| v.record.is_valid_at(now)).map(|v| v.record.clone()).collect()) }
305pub async fn install_local_ca(cert_pem: Vec<u8>, key_pem: Vec<u8>) -> Result<()> { global()?.install_local_ca(cert_pem, key_pem).await }
306pub async fn reload_local_ca() -> Result<()> { global()?.reload_local_ca().await }
307
308#[cfg(test)]
309mod tests {
310    use super::*;
311    use crate::{AcmeEndpoint, EventOutput, HostPattern, IssueReason, event::EventKind};
312    use url::Url;
313
314    async fn manager(local_validity_days: u32, renew_days: u32) -> (tempfile::TempDir, Arc<Manager>, DnsName) {
315        let directory = tempfile::tempdir().unwrap();
316        let domain = DnsName::parse("lifecycle.test").unwrap();
317        let mut config = AlpnHandlerConfig::default();
318        config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
319        config.verify_dns_before_issuance = false;
320        config.cert_store_directory = directory.path().to_owned();
321        config.events = EventOutput::Subscribe;
322        config.local_leaf_validity_days = local_validity_days;
323        config.renew_days_before_expiry = renew_days;
324        config.renew_check_interval_secs = 1;
325        config.renew_check_max_interval_secs = 4;
326        config.issuance_attempt_timeout_secs = 2;
327        config.acme_endpoint = AcmeEndpoint::Custom {
328            directory_url: Url::parse("http://127.0.0.1:9/directory").unwrap(),
329            issuer_id: "unreachable-test-acme".into(),
330            tls_trust_roots: Vec::new(),
331        };
332        let manager = Manager::new(config, 0).await.unwrap();
333        (directory, manager, domain)
334    }
335
336    fn drain(receiver: &mut EventReceiver) -> Vec<Arc<crate::Event>> {
337        let mut events = Vec::new();
338        while let Some(event) = receiver.try_recv().unwrap() { events.push(event); }
339        events
340    }
341
342    #[tokio::test]
343    async fn unreachable_acme_falls_back_to_local_and_emits_selection_events() {
344        let (_directory, manager, domain) = manager(30, 15).await;
345        let mut events = manager.events.subscribe();
346        let certificate = manager.ensure(domain.clone()).await.unwrap();
347        assert_eq!(certificate.record.provenance, CertificateProvenance::Local);
348        assert!(manager.store.local().load_certificate(&domain).await.unwrap().is_some());
349        let _ = manager.resolve(&domain).await.unwrap();
350        let events = drain(&mut events);
351        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceStarted { domain: value, issuer, .. } if value == &domain && issuer == "AcmeLocalCA")));
352        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceSucceeded { domain: value, issuer, .. } if value == &domain && issuer == "AcmeLocalCA")));
353        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::CertificateSelected { domain: value, issuer } if value == &domain && issuer == "AcmeLocalCA")));
354    }
355
356    #[tokio::test]
357    async fn local_only_never_starts_acme_issuance() {
358        let directory = tempfile::tempdir().unwrap();
359        let domain = DnsName::parse("private.internal").unwrap();
360        let mut config = AlpnHandlerConfig::default();
361        config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
362        config.verify_dns_before_issuance = false;
363        config.acme_endpoint = AcmeEndpoint::LocalOnly;
364        config.cert_store_directory = directory.path().to_owned();
365        config.events = EventOutput::Subscribe;
366        let manager = Manager::new(config, 0).await.unwrap();
367        let mut events = manager.events.subscribe();
368        assert_eq!(manager.ensure(domain.clone()).await.unwrap().record.provenance, CertificateProvenance::Local);
369        let events = drain(&mut events);
370        assert!(!events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceStarted { issuer, .. } if issuer == "acme")));
371        assert!(manager.store.acme().list_certificates().await.unwrap().is_empty());
372    }
373
374    #[tokio::test]
375    async fn expired_certificate_is_removed_from_memory_and_store() {
376        let (_directory, manager, domain) = manager(30, 15).await;
377        manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
378        let active = manager.resolve(&domain).await.unwrap();
379        let mut expired = (*active.record).clone();
380        expired.not_after = OffsetDateTime::now_utc().unix_timestamp() - 1;
381        manager.store.local().commit_certificate(&expired).await.unwrap();
382        let slots = manager.domain_slots(&domain).await;
383        slots.local.store(Some(Arc::new(ActiveCertificate { record: Arc::new(expired), key: active.key.clone() })));
384        let mut events = manager.events.subscribe();
385        assert!(manager.resolve(&domain).await.is_none());
386        assert!(manager.store.local().load_certificate(&domain).await.unwrap().is_none());
387        assert!(drain(&mut events).iter().any(|event| matches!(&event.kind, EventKind::CertificateExpiredRemoved { domain: value, .. } if value == &domain)));
388    }
389
390    #[tokio::test]
391    async fn aggressive_expiry_policy_renews_due_local_certificate() {
392        let (_directory, manager, domain) = manager(1, 2).await;
393        manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
394        let before = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
395        let mut events = manager.events.subscribe();
396        manager.renew_due().await.unwrap();
397        let after = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
398        assert_ne!(before, after);
399        let events = drain(&mut events);
400        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalDue { domain: value, .. } if value == &domain)));
401        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalStarted { domain: value, .. } if value == &domain)));
402        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalSucceeded { domain: value, .. } if value == &domain)));
403    }
404
405    #[tokio::test]
406    async fn retry_backoff_doubles_and_caps_per_domain_and_issuer() {
407        let (_directory, manager, domain) = manager(30, 15).await;
408        let now = OffsetDateTime::now_utc().unix_timestamp();
409        let first = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
410        let second = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
411        let third = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
412        let fourth = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
413        assert!((1..=2).contains(&(first - now)));
414        assert!((2..=3).contains(&(second - now)));
415        assert!((4..=5).contains(&(third - now)));
416        assert!((4..=5).contains(&(fourth - now)), "backoff must remain capped");
417        assert_eq!(manager.retry.lock().await.get(&(domain, CertificateProvenance::Acme)).unwrap().0, 4);
418    }
419
420    #[tokio::test]
421    async fn failed_acme_renewal_emits_failure_and_schedules_retry() {
422        let (_directory, manager, domain) = manager(30, 15).await;
423        let mut events = manager.events.subscribe();
424        assert!(manager.renew_variant(domain.clone(), CertificateProvenance::Acme).await.is_err());
425        assert!(manager.retry.lock().await.contains_key(&(domain.clone(), CertificateProvenance::Acme)));
426        let events = drain(&mut events);
427        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalStarted { domain: value, attempt: 1, .. } if value == &domain)));
428        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceFailed { domain: value, .. } if value == &domain)));
429        assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalFailed { domain: value, retry_at, .. } if value == &domain && *retry_at > OffsetDateTime::now_utc().unix_timestamp())));
430    }
431
432    #[tokio::test]
433    async fn retry_at_suppresses_due_scan_and_success_resets_backoff() {
434        let (_directory, manager, domain) = manager(1, 2).await;
435        manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
436        let before = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
437        manager.retry.lock().await.insert(
438            (domain.clone(), CertificateProvenance::Local),
439            (2, OffsetDateTime::now_utc().unix_timestamp() + 3_600),
440        );
441        manager.renew_due().await.unwrap();
442        assert_eq!(manager.resolve(&domain).await.unwrap().record.fingerprint_sha256, before);
443
444        manager.retry.lock().await.get_mut(&(domain.clone(), CertificateProvenance::Local)).unwrap().1 =
445            OffsetDateTime::now_utc().unix_timestamp() - 1;
446        manager.renew_due().await.unwrap();
447        assert_ne!(manager.resolve(&domain).await.unwrap().record.fingerprint_sha256, before);
448        assert!(!manager.retry.lock().await.contains_key(&(domain, CertificateProvenance::Local)));
449    }
450
451    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
452    async fn hanging_acme_renewal_obeys_absolute_attempt_deadline() {
453        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
454        let address = listener.local_addr().unwrap();
455        let hanging_server = tokio::spawn(async move {
456            while let Ok((stream, _)) = listener.accept().await {
457                tokio::spawn(async move {
458                    let _stream = stream;
459                    tokio::time::sleep(Duration::from_secs(10)).await;
460                });
461            }
462        });
463
464        let directory = tempfile::tempdir().unwrap();
465        let domain = DnsName::parse("deadline.test").unwrap();
466        let mut config = AlpnHandlerConfig::default();
467        config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
468        config.verify_dns_before_issuance = false;
469        config.cert_store_directory = directory.path().to_owned();
470        config.events = EventOutput::Subscribe;
471        config.issuance_attempt_timeout_secs = 1;
472        config.acme_endpoint = AcmeEndpoint::Custom {
473            directory_url: Url::parse(&format!("https://{address}/directory")).unwrap(),
474            issuer_id: "hanging-acme".into(),
475            tls_trust_roots: Vec::new(),
476        };
477        let manager = Manager::new(config, 0).await.unwrap();
478        let started = tokio::time::Instant::now();
479        let error = manager.renew_variant(domain.clone(), CertificateProvenance::Acme).await.unwrap_err();
480        let elapsed = started.elapsed();
481        assert!(matches!(error, Error::IssuanceTimeout(_)), "unexpected error: {error}");
482        assert!(elapsed >= Duration::from_millis(900));
483        assert!(elapsed < Duration::from_secs(2), "attempt exceeded configured deadline: {elapsed:?}");
484        assert!(manager.retry.lock().await.contains_key(&(domain, CertificateProvenance::Acme)));
485        hanging_server.abort();
486    }
487}