1use std::{
28 io,
29 sync::{
30 Arc,
31 atomic::{AtomicU64, Ordering},
32 },
33 time::{Duration, Instant},
34};
35
36pub const INITIAL_BACKOFF: Duration = Duration::from_millis(50);
39pub const MAXIMUM_BACKOFF: Duration = Duration::from_secs(1);
42pub const REPORT_INTERVAL: Duration = Duration::from_secs(10);
44
45pub const PERSISTENT_FAILURE: Duration = Duration::from_secs(30);
51
52pub const QUIET_AFTER_FAILURE: Duration = Duration::from_secs(10);
60
61pub fn is_fatal(error: &io::Error) -> bool {
63 #[cfg(unix)]
64 {
65 matches!(
66 error.raw_os_error(),
67 Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK | libc::EFAULT)
68 )
69 }
70 #[cfg(not(unix))]
71 {
72 error.kind() == io::ErrorKind::InvalidInput
75 }
76}
77
78#[derive(Debug, Eq, PartialEq)]
80pub enum Next {
81 Retry {
83 pause: Duration,
84 report: Option<u64>,
87 },
88 Stop,
90}
91
92#[derive(Debug)]
94pub struct AcceptErrors {
95 backoff: Duration,
96 unreported: u64,
98 last_reported: Option<Instant>,
99 health: Option<AcceptHealth>,
101}
102
103impl Default for AcceptErrors {
104 fn default() -> Self {
105 Self {
106 backoff: INITIAL_BACKOFF,
107 unreported: 0,
108 last_reported: None,
109 health: None,
110 }
111 }
112}
113
114impl AcceptErrors {
115 pub fn with_health(health: AcceptHealth) -> Self {
117 Self {
118 health: Some(health),
119 ..Self::default()
120 }
121 }
122
123 pub fn failed(&mut self, error: &io::Error) -> Next {
125 self.failed_at(error, Instant::now())
126 }
127
128 fn failed_at(&mut self, error: &io::Error, now: Instant) -> Next {
129 if is_fatal(error) {
130 return Next::Stop;
131 }
132 if let Some(health) = &self.health {
133 health.failed_at(now);
134 }
135 self.unreported += 1;
136 let due = self
137 .last_reported
138 .is_none_or(|last| now.duration_since(last) >= REPORT_INTERVAL);
139 let report = due.then(|| {
140 self.last_reported = Some(now);
141 std::mem::take(&mut self.unreported)
142 });
143 let pause = self.backoff;
144 self.backoff = (self.backoff * 2).min(MAXIMUM_BACKOFF);
145 Next::Retry { pause, report }
146 }
147
148 pub fn succeeded(&mut self) -> bool {
153 let recovered = self.last_reported.is_some();
154 if let Some(health) = &self.health {
155 health.succeeded();
156 }
157 self.backoff = INITIAL_BACKOFF;
158 self.unreported = 0;
159 self.last_reported = None;
160 recovered
161 }
162}
163
164#[derive(Clone, Debug)]
170pub struct AcceptHealth(Arc<HealthState>);
171
172#[derive(Debug)]
173struct HealthState {
174 origin: Instant,
176 failing_since: AtomicU64,
179 failed_at: AtomicU64,
181}
182
183impl Default for AcceptHealth {
184 fn default() -> Self {
185 Self(Arc::new(HealthState {
186 origin: Instant::now(),
187 failing_since: AtomicU64::new(0),
188 failed_at: AtomicU64::new(0),
189 }))
190 }
191}
192
193impl AcceptHealth {
194 pub fn is_failing(&self) -> bool {
197 self.is_failing_at(Instant::now())
198 }
199
200 fn is_failing_at(&self, now: Instant) -> bool {
201 let since = self.0.failing_since.load(Ordering::Relaxed);
202 let last = self.0.failed_at.load(Ordering::Relaxed);
203 if since == 0 || last == 0 {
204 return false;
205 }
206 self.millis(now).saturating_sub(last) <= millis(QUIET_AFTER_FAILURE)
207 && last.saturating_sub(since) >= millis(PERSISTENT_FAILURE)
208 }
209
210 fn failed_at(&self, now: Instant) {
211 let now = self.millis(now);
212 let last = self.0.failed_at.swap(now, Ordering::Relaxed);
213 if last == 0 || now.saturating_sub(last) > millis(QUIET_AFTER_FAILURE) {
216 self.0.failing_since.store(now, Ordering::Relaxed);
217 }
218 }
219
220 fn succeeded(&self) {
221 self.0.failing_since.store(0, Ordering::Relaxed);
222 self.0.failed_at.store(0, Ordering::Relaxed);
223 }
224
225 fn millis(&self, now: Instant) -> u64 {
227 millis(now.saturating_duration_since(self.0.origin)) + 1
228 }
229}
230
231fn millis(duration: Duration) -> u64 {
232 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
233}
234
235#[cfg(all(test, unix))]
236mod tests {
237 use super::*;
238
239 fn os(code: i32) -> io::Error {
240 io::Error::from_raw_os_error(code)
241 }
242
243 #[test]
244 fn only_errors_about_the_listener_itself_are_fatal() {
245 for code in [libc::EBADF, libc::EINVAL, libc::ENOTSOCK, libc::EFAULT] {
246 assert!(is_fatal(&os(code)), "{code}");
247 }
248 for code in [
249 libc::EMFILE,
250 libc::ENFILE,
251 libc::ENOBUFS,
252 libc::ENOMEM,
253 libc::ECONNABORTED,
254 libc::EINTR,
255 libc::EPROTO,
256 libc::EPERM,
257 libc::ENETDOWN,
258 libc::EHOSTUNREACH,
259 libc::EOPNOTSUPP,
260 ] {
261 assert!(!is_fatal(&os(code)), "{code}");
262 }
263 }
264
265 #[test]
266 fn backoff_doubles_to_a_ceiling_and_resets_on_success() {
267 let mut errors = AcceptErrors::default();
268 let now = Instant::now();
269 let pauses: Vec<_> = (0..8)
270 .map(|_| match errors.failed_at(&os(libc::EMFILE), now) {
271 Next::Retry { pause, .. } => pause,
272 Next::Stop => panic!("EMFILE is temporary"),
273 })
274 .collect();
275 assert_eq!(pauses[0], INITIAL_BACKOFF);
276 assert_eq!(pauses[1], INITIAL_BACKOFF * 2);
277 assert_eq!(pauses[7], MAXIMUM_BACKOFF);
278
279 assert!(errors.succeeded(), "the first failure was reported");
280 assert!(matches!(
281 errors.failed_at(&os(libc::EMFILE), now),
282 Next::Retry {
283 pause: INITIAL_BACKOFF,
284 report: Some(1)
285 }
286 ));
287 }
288
289 #[test]
290 fn readiness_fails_only_after_persistent_failure_and_recovers() {
291 let health = AcceptHealth::default();
292 let mut errors = AcceptErrors::with_health(health.clone());
293 let start = Instant::now();
294 let at = |seconds| start + Duration::from_secs(seconds);
295
296 for second in 0..30 {
298 errors.failed_at(&os(libc::EMFILE), at(second));
299 }
300 assert!(
301 !health.is_failing_at(at(29)),
302 "29 seconds of failure is still a burst"
303 );
304 errors.failed_at(&os(libc::EMFILE), at(30));
305 assert!(health.is_failing_at(at(30)), "30 seconds is persistent");
306
307 assert!(
308 !health.is_failing_at(at(41)),
309 "ten seconds without a failure ends the run"
310 );
311 errors.failed_at(&os(libc::EMFILE), at(42));
312 assert!(
313 !health.is_failing_at(at(42)),
314 "a failure after a quiet spell starts a new run"
315 );
316
317 for second in 43..=72 {
318 errors.failed_at(&os(libc::EMFILE), at(second));
319 }
320 assert!(health.is_failing_at(at(72)));
321 errors.succeeded();
322 assert!(!health.is_failing_at(at(72)), "a success ends the run");
323 }
324
325 #[test]
326 fn a_fatal_error_does_not_count_as_a_failing_run() {
327 let health = AcceptHealth::default();
328 let mut errors = AcceptErrors::with_health(health.clone());
329 assert_eq!(errors.failed(&os(libc::EBADF)), Next::Stop);
330 assert!(!health.is_failing());
331 }
332
333 #[test]
334 fn a_fatal_error_stops_the_loop() {
335 assert_eq!(AcceptErrors::default().failed(&os(libc::EBADF)), Next::Stop);
336 }
337
338 #[test]
339 fn a_persistent_failure_is_reported_as_a_summary() {
340 let mut errors = AcceptErrors::default();
341 let start = Instant::now();
342 let report = |next| match next {
343 Next::Retry { report, .. } => report,
344 Next::Stop => panic!("EMFILE is temporary"),
345 };
346 assert_eq!(
347 report(errors.failed_at(&os(libc::EMFILE), start)),
348 Some(1),
349 "the first failure is reported at once"
350 );
351 for _ in 0..5 {
352 let later = start + Duration::from_secs(1);
353 assert_eq!(report(errors.failed_at(&os(libc::EMFILE), later)), None);
354 }
355 assert_eq!(
356 report(errors.failed_at(&os(libc::EMFILE), start + REPORT_INTERVAL)),
357 Some(6),
358 "the summary counts the repeats inside the interval"
359 );
360 }
361}