1use std::{collections::HashMap, sync::{Arc, OnceLock, atomic::{AtomicU8, Ordering}}, time::{Duration, Instant}};
2
3use arc_swap::ArcSwapOption;
4use rustls::sign::CertifiedKey;
5use time::OffsetDateTime;
6use tokio::sync::{Mutex, RwLock};
7
8use crate::{AlpnHandlerConfig, CertificateProvenance, DnsName, Error, EventReceiver, Result, StoredCertificate, StoredLocalCa, acme::{AcmeIssuer, ChallengeRegistry, PebbleSupervisor}, event::{EventBus, EventKind, EventLevel}, issuer::{CertIssuer, IssueReason, IssueRequest, LocalCaIssuer}, store::CertStore};
9#[cfg(feature = "folder-store")]
10use crate::store::FolderCertStore;
11#[cfg(feature = "managed-pebble")]
12use crate::acme::start_managed_pebble;
13
14pub(crate) static GLOBAL: OnceLock<Arc<Manager>> = OnceLock::new();
15pub(crate) static GLOBAL_STATE: AtomicU8 = AtomicU8::new(0); pub(crate) struct ActiveCertificate { pub record: Arc<StoredCertificate>, pub key: Arc<CertifiedKey> }
18pub(crate) struct DomainSlots { pub acme: ArcSwapOption<ActiveCertificate>, pub local: ArcSwapOption<ActiveCertificate> }
19impl DomainSlots { fn new() -> Self { Self { acme: ArcSwapOption::empty(), local: ArcSwapOption::empty() } } }
20
21pub(crate) struct Manager {
22 pub config: AlpnHandlerConfig,
23 pub store: Arc<dyn CertStore>,
24 pub events: Arc<EventBus>,
25 slots: RwLock<HashMap<DnsName, Arc<DomainSlots>>>,
26 issue_locks: Mutex<HashMap<(DnsName, CertificateProvenance), Arc<Mutex<()>>>>,
27 retry: Mutex<HashMap<(DnsName, CertificateProvenance), (u32, i64)>>,
28 pub local_issuer: Arc<LocalCaIssuer>,
29 pub acme_issuer: Arc<AcmeIssuer>,
30 pub challenges: Arc<ChallengeRegistry>,
31 _pebble: Option<PebbleSupervisor>,
32}
33
34impl Manager {
35 pub(crate) async fn new(mut config: AlpnHandlerConfig, challenge_port: u16) -> Result<Arc<Self>> {
36 config.validate()?;
37 let events = Arc::new(EventBus::new(config.event_channel_capacity, config.event_history_capacity, config.events, config.event_handler.clone()));
38 events.emit(EventLevel::Info, EventKind::ManagerInitializing);
39 if !config.verify_dns_before_issuance { events.emit(EventLevel::Warn, EventKind::DnsVerificationDisabled); }
40 let store: Arc<dyn CertStore> = if let Some(store) = config.cert_store.take() {
41 store
42 } else {
43 #[cfg(feature = "folder-store")]
44 { Arc::new(FolderCertStore::open(config.cert_store_directory.clone()).await?) }
45 #[cfg(not(feature = "folder-store"))]
46 { return Err(Error::InvalidConfig("cert_store is required when folder-store is disabled".into())); }
47 };
48 let pebble: Option<PebbleSupervisor> = if let crate::AcmeEndpoint::Pebble(pebble_config) = &config.acme_endpoint {
49 #[cfg(feature = "managed-pebble")]
50 { Some(start_managed_pebble(pebble_config, challenge_port, config.pebble_startup_timeout_secs, &events).await?) }
51 #[cfg(not(feature = "managed-pebble"))]
52 { let _ = (pebble_config, challenge_port); return Err(Error::InvalidConfig("AcmeEndpoint::Pebble requires the managed-pebble feature".into())); }
53 } else { None };
54 let pebble_root = pebble.as_ref().map(|value| value.https_root.clone());
55 let local_issuer = Arc::new(LocalCaIssuer::load_or_create(Arc::clone(&store), config.local_leaf_validity_days, Arc::clone(&events)).await?);
56 let challenges = Arc::new(ChallengeRegistry::default());
57 let acme_issuer = Arc::new(AcmeIssuer::new(Arc::clone(&store), config.clone(), Arc::clone(&challenges), Arc::clone(&events), pebble_root)?);
58 let manager = Arc::new(Self { config, store, events, slots: RwLock::new(HashMap::new()), issue_locks: Mutex::new(HashMap::new()), retry: Mutex::new(HashMap::new()), local_issuer, acme_issuer, challenges, _pebble: pebble });
59 manager.load_store().await?;
60 manager.events.emit(EventLevel::Info, EventKind::ManagerReady);
61 manager.spawn_renewal();
62 Ok(manager)
63 }
64
65 fn allowed(&self, name: &DnsName) -> bool { self.config.allowed_hosts.iter().any(|p| p.matches(name)) }
66
67 async fn domain_slots(&self, name: &DnsName) -> Arc<DomainSlots> {
68 if let Some(slot) = self.slots.read().await.get(name).cloned() { return slot; }
69 self.slots.write().await.entry(name.clone()).or_insert_with(|| Arc::new(DomainSlots::new())).clone()
70 }
71
72 async fn load_store(&self) -> Result<()> {
73 let now = OffsetDateTime::now_utc().unix_timestamp();
74 for cert in self.store.acme().list_certificates().await? { self.load_record(cert, now).await?; }
75 let ca = self.local_issuer.ca().await;
76 for cert in self.store.local().list_certificates().await? {
77 if cert.local_ca_fingerprint_sha256.as_deref() != Some(&ca.fingerprint_sha256) || crate::cert::validate_local_signature(&cert, &ca).is_err() {
78 self.store.local().quarantine_certificate(&cert, "local CA fingerprint changed").await?;
79 self.events.emit(EventLevel::Warn, EventKind::LocalCertificateQuarantined { domain: cert.primary_name, ca_fingerprint: cert.local_ca_fingerprint_sha256.unwrap_or_default() });
80 } else { self.load_record(cert, now).await?; }
81 }
82 Ok(())
83 }
84
85 async fn load_record(&self, cert: StoredCertificate, now: i64) -> Result<()> {
86 if !cert.is_valid_at(now) { return Ok(()); }
87 let key = cert.certified_key()?;
88 let domain = cert.primary_name.clone();
89 let provenance = cert.provenance;
90 let active = Arc::new(ActiveCertificate { record: Arc::new(cert), key });
91 let slots = self.domain_slots(&domain).await;
92 match provenance { CertificateProvenance::Acme => slots.acme.store(Some(active)), CertificateProvenance::Local => slots.local.store(Some(active)) }
93 self.events.emit(EventLevel::Info, EventKind::CertificateLoaded { domain, issuer: match provenance { CertificateProvenance::Acme => "acme", CertificateProvenance::Local => "local" }.into() });
94 Ok(())
95 }
96
97 pub(crate) async fn resolve(&self, name: &DnsName) -> Option<Arc<ActiveCertificate>> {
98 let slot = self.slots.read().await.get(name).cloned()?;
99 let now = OffsetDateTime::now_utc().unix_timestamp();
100 if let Some(value) = slot.acme.load_full() {
101 if value.record.is_valid_at(now) {
102 self.events.emit(EventLevel::Debug, EventKind::CertificateSelected { domain: name.clone(), issuer: value.record.issuer_id.clone() });
103 return Some(value);
104 }
105 slot.acme.store(None);
106 let _ = self.store.acme().remove_certificate(name, &value.record.issuer_id).await;
107 self.events.emit(EventLevel::Warn, EventKind::CertificateExpiredRemoved { domain: name.clone(), issuer: value.record.issuer_id.clone() });
108 }
109 if let Some(value) = slot.local.load_full() {
110 if value.record.is_valid_at(now) {
111 self.events.emit(EventLevel::Debug, EventKind::CertificateSelected { domain: name.clone(), issuer: value.record.issuer_id.clone() });
112 return Some(value);
113 }
114 slot.local.store(None);
115 let _ = self.store.local().remove_certificate(name).await;
116 self.events.emit(EventLevel::Warn, EventKind::CertificateExpiredRemoved { domain: name.clone(), issuer: value.record.issuer_id.clone() });
117 }
118 None
119 }
120
121 pub(crate) async fn ensure(self: &Arc<Self>, name: DnsName) -> Result<Arc<ActiveCertificate>> {
122 if !self.allowed(&name) { self.events.emit(EventLevel::Warn, EventKind::UnknownDomainRejected { domain: Some(name.clone()) }); return Err(Error::HostNotAllowed(name.to_string())); }
123 if let Some(cert) = self.resolve(&name).await { return Ok(cert); }
124 if matches!(self.config.acme_endpoint, crate::AcmeEndpoint::LocalOnly) {
125 if !self.config.local_fallback_enabled { return Err(Error::InvalidConfig("AcmeEndpoint::LocalOnly requires local_fallback_enabled".into())); }
126 self.issue_local(name.clone(), IssueReason::Initial, false).await?;
127 return self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()));
128 }
129 if !self.config.local_fallback_enabled { self.issue_acme(name.clone(), IssueReason::Initial, false).await?; return self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string())); }
130 let acme_manager = Arc::clone(self);
131 let acme_name = name.clone();
132 let mut acme_task = tokio::spawn(async move { acme_manager.issue_acme(acme_name, IssueReason::Initial, false).await });
133 let local = self.issue_local(name.clone(), IssueReason::Initial, false);
134 tokio::pin!(local);
135 tokio::select! {
136 local_result = &mut local => {
137 local_result?;
138 self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()))
139 }
140 acme_result = &mut acme_task => {
141 if acme_result.is_ok_and(|result| result.is_ok()) {
142 if let Some(cert) = self.resolve(&name).await { return Ok(cert); }
143 }
144 local.await?;
145 self.resolve(&name).await.ok_or_else(|| Error::NoCertificate(name.to_string()))
146 }
147 }
148 }
149
150 async fn issue_local(&self, name: DnsName, reason: IssueReason, force: bool) -> Result<()> {
151 let lock = { self.issue_locks.lock().await.entry((name.clone(), CertificateProvenance::Local)).or_insert_with(|| Arc::new(Mutex::new(()))).clone() };
152 let _guard = lock.lock().await;
153 if !force && self.domain_slots(&name).await.local.load_full().is_some_and(|value| value.record.is_valid_at(OffsetDateTime::now_utc().unix_timestamp())) { return Ok(()); }
154 self.events.emit(EventLevel::Info, EventKind::IssuanceStarted { domain: name.clone(), issuer: "AcmeLocalCA".into(), reason: format!("{reason:?}").to_ascii_lowercase() });
155 let started = Instant::now();
156 let deadline = tokio::time::Instant::now() + Duration::from_secs(self.config.issuance_attempt_timeout_secs);
157 let operation = async {
158 let cert = self.local_issuer.issue(IssueRequest { domain: name.clone(), reason, deadline }).await?;
159 self.store.local().commit_certificate(&cert).await?;
160 let not_after = cert.not_after;
161 self.load_record(cert, OffsetDateTime::now_utc().unix_timestamp()).await?;
162 Result::<i64>::Ok(not_after)
163 };
164 match tokio::time::timeout_at(deadline, operation).await {
165 Ok(Ok(not_after)) => {
166 self.events.emit(EventLevel::Info, EventKind::IssuanceSucceeded { domain: name, issuer: "AcmeLocalCA".into(), not_after, elapsed: started.elapsed() });
167 Ok(())
168 }
169 Ok(Err(error)) => {
170 self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "AcmeLocalCA".into(), error: error.to_string() });
171 Err(error)
172 }
173 Err(_) => {
174 let error = Error::IssuanceTimeout(name.to_string());
175 self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "AcmeLocalCA".into(), error: error.to_string() });
176 Err(error)
177 }
178 }
179 }
180
181 async fn issue_acme(self: &Arc<Self>, name: DnsName, reason: IssueReason, force: bool) -> Result<()> {
182 let lock = { self.issue_locks.lock().await.entry((name.clone(), CertificateProvenance::Acme)).or_insert_with(|| Arc::new(Mutex::new(()))).clone() };
183 let _guard = lock.lock().await;
184 if !force && self.domain_slots(&name).await.acme.load_full().is_some_and(|v| v.record.is_valid_at(OffsetDateTime::now_utc().unix_timestamp())) { return Ok(()); }
185 self.events.emit(EventLevel::Info, EventKind::IssuanceStarted { domain: name.clone(), issuer: "acme".into(), reason: format!("{reason:?}").to_ascii_lowercase() });
186 let started = Instant::now();
187 let deadline = tokio::time::Instant::now() + Duration::from_secs(self.config.issuance_attempt_timeout_secs);
188 let operation = async {
189 let cert = self.acme_issuer.issue(IssueRequest { domain: name.clone(), reason, deadline }).await?;
190 let issuer = cert.issuer_id.clone();
191 let not_after = cert.not_after;
192 self.store.acme().commit_certificate(&cert).await?;
193 self.load_record(cert, OffsetDateTime::now_utc().unix_timestamp()).await?;
194 Result::<(String, i64)>::Ok((issuer, not_after))
195 };
196 match tokio::time::timeout_at(deadline, operation).await {
197 Ok(Ok((issuer, not_after))) => {
198 self.events.emit(EventLevel::Info, EventKind::IssuanceSucceeded { domain: name, issuer, not_after, elapsed: started.elapsed() });
199 Ok(())
200 }
201 Ok(Err(error)) => { self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "acme".into(), error: error.to_string() }); Err(error) }
202 Err(_) => {
203 let error = Error::IssuanceTimeout(name.to_string());
204 self.events.emit(EventLevel::Error, EventKind::IssuanceFailed { domain: name, issuer: "acme".into(), error: error.to_string() });
205 Err(error)
206 }
207 }
208 }
209
210 pub(crate) async fn install_local_ca(&self, cert: Vec<u8>, key: Vec<u8>) -> Result<()> {
211 let replacement = StoredLocalCa::from_pem(String::from_utf8(cert).map_err(|e| Error::LocalCa(e.to_string()))?, String::from_utf8(key).map_err(|e| Error::LocalCa(e.to_string()))?)?;
212 self.local_issuer.install(replacement).await?;
213 self.invalidate_local().await
214 }
215 pub(crate) async fn reload_local_ca(&self) -> Result<()> { if self.local_issuer.reload().await?.is_some() { self.invalidate_local().await?; } Ok(()) }
216 async fn invalidate_local(&self) -> Result<()> {
217 let ca = self.local_issuer.ca().await;
218 for cert in self.store.local().list_certificates().await? {
219 if cert.local_ca_fingerprint_sha256.as_deref() != Some(&ca.fingerprint_sha256) {
220 if let Some(slot) = self.slots.read().await.get(&cert.primary_name).cloned() { slot.local.store(None); }
221 self.store.local().quarantine_certificate(&cert, "local CA replaced").await?;
222 self.events.emit(EventLevel::Warn, EventKind::LocalCertificateQuarantined { domain: cert.primary_name, ca_fingerprint: cert.local_ca_fingerprint_sha256.unwrap_or_default() });
223 }
224 }
225 Ok(())
226 }
227
228 fn spawn_renewal(self: &Arc<Self>) {
229 let manager = Arc::clone(self);
230 tokio::spawn(async move {
231 let mut interval = tokio::time::interval(Duration::from_secs(manager.config.renew_check_interval_secs));
232 interval.tick().await;
233 loop { interval.tick().await; if let Err(error) = manager.renew_due().await { manager.events.emit(EventLevel::Error, EventKind::RenewalFailed { domain: DnsName::parse("invalid.local").expect("literal"), issuer: "scheduler".into(), error: error.to_string(), retry_at: OffsetDateTime::now_utc().unix_timestamp() + i64::try_from(manager.config.renew_check_interval_secs).unwrap_or(i64::MAX) }); } }
234 });
235 }
236 async fn renew_due(self: &Arc<Self>) -> Result<()> {
237 let values: Vec<_> = self.slots.read().await.iter().flat_map(|(name, slots)| {
238 [(CertificateProvenance::Acme, slots.acme.load_full()), (CertificateProvenance::Local, slots.local.load_full())]
239 .into_iter().filter_map(|(provenance, cert)| cert.map(|cert| (name.clone(), provenance, cert))).collect::<Vec<_>>()
240 }).collect();
241 self.events.emit(EventLevel::Info, EventKind::RenewalScanStarted { known_variants: values.len() });
242 let threshold = i64::from(self.config.renew_days_before_expiry) * 86_400;
243 let now = OffsetDateTime::now_utc().unix_timestamp();
244 let mut tasks = Vec::new();
245 for (name, provenance, cert) in values {
246 let remaining = cert.record.not_after - now;
247 self.events.emit(EventLevel::Info, EventKind::CertificateChecked { domain: name.clone(), issuer: cert.record.issuer_id.clone(), not_after: cert.record.not_after, days_remaining: remaining / 86_400 });
248 let retry_ready = self.retry.lock().await.get(&(name.clone(), provenance)).is_none_or(|(_, retry_at)| *retry_at <= now);
249 if remaining <= threshold && retry_ready {
250 self.events.emit(EventLevel::Info, EventKind::RenewalDue { domain: name.clone(), issuer: cert.record.issuer_id.clone(), not_after: cert.record.not_after });
251 let manager = Arc::clone(self);
252 tasks.push(tokio::spawn(async move { manager.renew_variant(name, provenance).await }));
253 }
254 }
255 for task in tasks { let _ = task.await; }
256 Ok(())
257 }
258
259 async fn renew_variant(self: &Arc<Self>, name: DnsName, provenance: CertificateProvenance) -> Result<()> {
260 let attempt = self.retry.lock().await.get(&(name.clone(), provenance)).map_or(1, |(failures, _)| failures.saturating_add(1));
261 self.events.emit(EventLevel::Info, EventKind::RenewalStarted { domain: name.clone(), issuer: format!("{provenance:?}"), attempt });
262 let started = Instant::now();
263 let result = match provenance {
264 CertificateProvenance::Acme => self.issue_acme(name.clone(), IssueReason::Renewal, true).await,
265 CertificateProvenance::Local => self.renew_local(name.clone()).await,
266 };
267 match result {
268 Ok(()) => {
269 self.retry.lock().await.remove(&(name.clone(), provenance));
270 let not_after = self.resolve(&name).await.map_or(0, |value| value.record.not_after);
271 self.events.emit(EventLevel::Info, EventKind::RenewalSucceeded { domain: name, issuer: format!("{provenance:?}"), not_after, elapsed: started.elapsed() });
272 Ok(())
273 }
274 Err(error) => {
275 let retry_at = self.schedule_retry(name.clone(), provenance).await;
276 self.events.emit(EventLevel::Error, EventKind::RenewalFailed { domain: name, issuer: format!("{provenance:?}"), error: error.to_string(), retry_at });
277 Err(error)
278 }
279 }
280 }
281
282 async fn schedule_retry(&self, name: DnsName, provenance: CertificateProvenance) -> i64 {
283 let mut retry = self.retry.lock().await;
284 let failures = retry.get(&(name.clone(), provenance)).map_or(1, |(count, _)| count.saturating_add(1));
285 let factor = 1_u64.checked_shl(failures.saturating_sub(1).min(20)).unwrap_or(u64::MAX);
286 let delay = self.config.renew_check_interval_secs.saturating_mul(factor).min(self.config.renew_check_max_interval_secs);
287 let retry_at = OffsetDateTime::now_utc().unix_timestamp().saturating_add(i64::try_from(delay).unwrap_or(i64::MAX));
288 retry.insert((name, provenance), (failures, retry_at));
289 retry_at
290 }
291 async fn renew_local(&self, name: DnsName) -> Result<()> {
292 self.issue_local(name, IssueReason::Renewal, true).await
293 }
294}
295
296pub(crate) fn global() -> Result<&'static Arc<Manager>> {
297 match GLOBAL_STATE.load(Ordering::Acquire) { 0 => Err(Error::AlpnHandlerNotRegistered), 1 => Err(Error::AlpnHandlerNotReady), _ => GLOBAL.get().ok_or(Error::AlpnHandlerNotReady) }
298}
299
300pub fn subscribe_events() -> Result<EventReceiver> { Ok(global()?.events.subscribe()) }
301pub async fn get_certificate(name: &str) -> Result<Arc<StoredCertificate>> { let name = DnsName::parse(name)?; global()?.resolve(&name).await.map(|v| v.record.clone()).ok_or_else(|| Error::NoCertificate(name.to_string())) }
302pub async fn get_or_issue_certificate(name: &str) -> Result<Arc<StoredCertificate>> { let name = DnsName::parse(name)?; Ok(global()?.ensure(name).await?.record.clone()) }
304pub async fn list_certificates() -> Result<Vec<Arc<StoredCertificate>>> { let manager = global()?; let slots: Vec<_> = manager.slots.read().await.values().cloned().collect(); let now = OffsetDateTime::now_utc().unix_timestamp(); Ok(slots.into_iter().flat_map(|slot| [slot.acme.load_full(), slot.local.load_full()]).flatten().filter(|v| v.record.is_valid_at(now)).map(|v| v.record.clone()).collect()) }
305pub async fn install_local_ca(cert_pem: Vec<u8>, key_pem: Vec<u8>) -> Result<()> { global()?.install_local_ca(cert_pem, key_pem).await }
306pub async fn reload_local_ca() -> Result<()> { global()?.reload_local_ca().await }
307
308#[cfg(test)]
309mod tests {
310 use super::*;
311 use crate::{AcmeEndpoint, EventOutput, HostPattern, IssueReason, event::EventKind};
312 use url::Url;
313
314 async fn manager(local_validity_days: u32, renew_days: u32) -> (tempfile::TempDir, Arc<Manager>, DnsName) {
315 let directory = tempfile::tempdir().unwrap();
316 let domain = DnsName::parse("lifecycle.test").unwrap();
317 let mut config = AlpnHandlerConfig::default();
318 config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
319 config.verify_dns_before_issuance = false;
320 config.cert_store_directory = directory.path().to_owned();
321 config.events = EventOutput::Subscribe;
322 config.local_leaf_validity_days = local_validity_days;
323 config.renew_days_before_expiry = renew_days;
324 config.renew_check_interval_secs = 1;
325 config.renew_check_max_interval_secs = 4;
326 config.issuance_attempt_timeout_secs = 2;
327 config.acme_endpoint = AcmeEndpoint::Custom {
328 directory_url: Url::parse("http://127.0.0.1:9/directory").unwrap(),
329 issuer_id: "unreachable-test-acme".into(),
330 tls_trust_roots: Vec::new(),
331 };
332 let manager = Manager::new(config, 0).await.unwrap();
333 (directory, manager, domain)
334 }
335
336 fn drain(receiver: &mut EventReceiver) -> Vec<Arc<crate::Event>> {
337 let mut events = Vec::new();
338 while let Some(event) = receiver.try_recv().unwrap() { events.push(event); }
339 events
340 }
341
342 #[tokio::test]
343 async fn unreachable_acme_falls_back_to_local_and_emits_selection_events() {
344 let (_directory, manager, domain) = manager(30, 15).await;
345 let mut events = manager.events.subscribe();
346 let certificate = manager.ensure(domain.clone()).await.unwrap();
347 assert_eq!(certificate.record.provenance, CertificateProvenance::Local);
348 assert!(manager.store.local().load_certificate(&domain).await.unwrap().is_some());
349 let _ = manager.resolve(&domain).await.unwrap();
350 let events = drain(&mut events);
351 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceStarted { domain: value, issuer, .. } if value == &domain && issuer == "AcmeLocalCA")));
352 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceSucceeded { domain: value, issuer, .. } if value == &domain && issuer == "AcmeLocalCA")));
353 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::CertificateSelected { domain: value, issuer } if value == &domain && issuer == "AcmeLocalCA")));
354 }
355
356 #[tokio::test]
357 async fn local_only_never_starts_acme_issuance() {
358 let directory = tempfile::tempdir().unwrap();
359 let domain = DnsName::parse("private.internal").unwrap();
360 let mut config = AlpnHandlerConfig::default();
361 config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
362 config.verify_dns_before_issuance = false;
363 config.acme_endpoint = AcmeEndpoint::LocalOnly;
364 config.cert_store_directory = directory.path().to_owned();
365 config.events = EventOutput::Subscribe;
366 let manager = Manager::new(config, 0).await.unwrap();
367 let mut events = manager.events.subscribe();
368 assert_eq!(manager.ensure(domain.clone()).await.unwrap().record.provenance, CertificateProvenance::Local);
369 let events = drain(&mut events);
370 assert!(!events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceStarted { issuer, .. } if issuer == "acme")));
371 assert!(manager.store.acme().list_certificates().await.unwrap().is_empty());
372 }
373
374 #[tokio::test]
375 async fn expired_certificate_is_removed_from_memory_and_store() {
376 let (_directory, manager, domain) = manager(30, 15).await;
377 manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
378 let active = manager.resolve(&domain).await.unwrap();
379 let mut expired = (*active.record).clone();
380 expired.not_after = OffsetDateTime::now_utc().unix_timestamp() - 1;
381 manager.store.local().commit_certificate(&expired).await.unwrap();
382 let slots = manager.domain_slots(&domain).await;
383 slots.local.store(Some(Arc::new(ActiveCertificate { record: Arc::new(expired), key: active.key.clone() })));
384 let mut events = manager.events.subscribe();
385 assert!(manager.resolve(&domain).await.is_none());
386 assert!(manager.store.local().load_certificate(&domain).await.unwrap().is_none());
387 assert!(drain(&mut events).iter().any(|event| matches!(&event.kind, EventKind::CertificateExpiredRemoved { domain: value, .. } if value == &domain)));
388 }
389
390 #[tokio::test]
391 async fn aggressive_expiry_policy_renews_due_local_certificate() {
392 let (_directory, manager, domain) = manager(1, 2).await;
393 manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
394 let before = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
395 let mut events = manager.events.subscribe();
396 manager.renew_due().await.unwrap();
397 let after = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
398 assert_ne!(before, after);
399 let events = drain(&mut events);
400 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalDue { domain: value, .. } if value == &domain)));
401 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalStarted { domain: value, .. } if value == &domain)));
402 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalSucceeded { domain: value, .. } if value == &domain)));
403 }
404
405 #[tokio::test]
406 async fn retry_backoff_doubles_and_caps_per_domain_and_issuer() {
407 let (_directory, manager, domain) = manager(30, 15).await;
408 let now = OffsetDateTime::now_utc().unix_timestamp();
409 let first = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
410 let second = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
411 let third = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
412 let fourth = manager.schedule_retry(domain.clone(), CertificateProvenance::Acme).await;
413 assert!((1..=2).contains(&(first - now)));
414 assert!((2..=3).contains(&(second - now)));
415 assert!((4..=5).contains(&(third - now)));
416 assert!((4..=5).contains(&(fourth - now)), "backoff must remain capped");
417 assert_eq!(manager.retry.lock().await.get(&(domain, CertificateProvenance::Acme)).unwrap().0, 4);
418 }
419
420 #[tokio::test]
421 async fn failed_acme_renewal_emits_failure_and_schedules_retry() {
422 let (_directory, manager, domain) = manager(30, 15).await;
423 let mut events = manager.events.subscribe();
424 assert!(manager.renew_variant(domain.clone(), CertificateProvenance::Acme).await.is_err());
425 assert!(manager.retry.lock().await.contains_key(&(domain.clone(), CertificateProvenance::Acme)));
426 let events = drain(&mut events);
427 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalStarted { domain: value, attempt: 1, .. } if value == &domain)));
428 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::IssuanceFailed { domain: value, .. } if value == &domain)));
429 assert!(events.iter().any(|event| matches!(&event.kind, EventKind::RenewalFailed { domain: value, retry_at, .. } if value == &domain && *retry_at > OffsetDateTime::now_utc().unix_timestamp())));
430 }
431
432 #[tokio::test]
433 async fn retry_at_suppresses_due_scan_and_success_resets_backoff() {
434 let (_directory, manager, domain) = manager(1, 2).await;
435 manager.issue_local(domain.clone(), IssueReason::Initial, false).await.unwrap();
436 let before = manager.resolve(&domain).await.unwrap().record.fingerprint_sha256.clone();
437 manager.retry.lock().await.insert(
438 (domain.clone(), CertificateProvenance::Local),
439 (2, OffsetDateTime::now_utc().unix_timestamp() + 3_600),
440 );
441 manager.renew_due().await.unwrap();
442 assert_eq!(manager.resolve(&domain).await.unwrap().record.fingerprint_sha256, before);
443
444 manager.retry.lock().await.get_mut(&(domain.clone(), CertificateProvenance::Local)).unwrap().1 =
445 OffsetDateTime::now_utc().unix_timestamp() - 1;
446 manager.renew_due().await.unwrap();
447 assert_ne!(manager.resolve(&domain).await.unwrap().record.fingerprint_sha256, before);
448 assert!(!manager.retry.lock().await.contains_key(&(domain, CertificateProvenance::Local)));
449 }
450
451 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
452 async fn hanging_acme_renewal_obeys_absolute_attempt_deadline() {
453 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
454 let address = listener.local_addr().unwrap();
455 let hanging_server = tokio::spawn(async move {
456 while let Ok((stream, _)) = listener.accept().await {
457 tokio::spawn(async move {
458 let _stream = stream;
459 tokio::time::sleep(Duration::from_secs(10)).await;
460 });
461 }
462 });
463
464 let directory = tempfile::tempdir().unwrap();
465 let domain = DnsName::parse("deadline.test").unwrap();
466 let mut config = AlpnHandlerConfig::default();
467 config.allowed_hosts = vec![HostPattern::exact(domain.as_str()).unwrap()];
468 config.verify_dns_before_issuance = false;
469 config.cert_store_directory = directory.path().to_owned();
470 config.events = EventOutput::Subscribe;
471 config.issuance_attempt_timeout_secs = 1;
472 config.acme_endpoint = AcmeEndpoint::Custom {
473 directory_url: Url::parse(&format!("https://{address}/directory")).unwrap(),
474 issuer_id: "hanging-acme".into(),
475 tls_trust_roots: Vec::new(),
476 };
477 let manager = Manager::new(config, 0).await.unwrap();
478 let started = tokio::time::Instant::now();
479 let error = manager.renew_variant(domain.clone(), CertificateProvenance::Acme).await.unwrap_err();
480 let elapsed = started.elapsed();
481 assert!(matches!(error, Error::IssuanceTimeout(_)), "unexpected error: {error}");
482 assert!(elapsed >= Duration::from_millis(900));
483 assert!(elapsed < Duration::from_secs(2), "attempt exceeded configured deadline: {elapsed:?}");
484 assert!(manager.retry.lock().await.contains_key(&(domain, CertificateProvenance::Acme)));
485 hanging_server.abort();
486 }
487}