Skip to main content

rushls_common/
accept.rs

1//! What an accept loop does when `accept` fails.
2//!
3//! Almost every failure is temporary, and nothing about the listener is
4//! wrong: the process is out of descriptors or buffers (`EMFILE`, `ENFILE`,
5//! `ENOBUFS`, `ENOMEM`), or a peer went away between the kernel completing its
6//! handshake and the call returning it (`ECONNABORTED`, and on Linux the
7//! network errors `accept(2)` says to treat like `EAGAIN`). Stopping on those
8//! would turn a connection burst into dropping every live session, so the loop
9//! retries, backing off so an exhausted descriptor table does not spin a core.
10//!
11//! A few errors mean the listening socket itself is unusable (`EBADF`,
12//! `EINVAL`, `ENOTSOCK`, `EFAULT`). Retrying those would loop forever without
13//! ever accepting again, so they are fatal: the caller stops accepting, and the
14//! orchestrator restarts a working process.
15//!
16//! Nothing here logs. A persistent failure would otherwise produce one line per
17//! attempt, so [`AcceptErrors`] says when a summary is due and the application
18//! reports it through its own events or logs.
19//!
20//! A temporary failure that never clears, such as a descriptor table that
21//! stays full, would otherwise leave a process that refuses every connection
22//! while its probes report it healthy. [`AcceptHealth`] turns that into a
23//! readiness failure, so a load balancer stops sending new connections. It
24//! never fails liveness: a restart would cut every live session, and the usual
25//! cure is those sessions ending and freeing descriptors.
26
27use std::{
28    io,
29    sync::{
30        Arc,
31        atomic::{AtomicU64, Ordering},
32    },
33    time::{Duration, Instant},
34};
35
36/// First pause after a failure. Short, because the usual cause is one aborted
37/// connection and the next `accept` succeeds.
38pub const INITIAL_BACKOFF: Duration = Duration::from_millis(50);
39/// Longest pause. Bounds how long a recovered listener can sit idle, and
40/// keeps a persistent failure at one wakeup per second.
41pub const MAXIMUM_BACKOFF: Duration = Duration::from_secs(1);
42/// How often a persistent failure is reported after its first occurrence.
43pub const REPORT_INTERVAL: Duration = Duration::from_secs(10);
44
45/// How long `accept` must keep failing before [`AcceptHealth`] reports it.
46///
47/// Long enough that a burst of aborted connections or a brief descriptor spike
48/// never takes a replica out of rotation; short enough that one which cannot
49/// accept anyone stops being sent connections.
50pub const PERSISTENT_FAILURE: Duration = Duration::from_secs(30);
51
52/// Without a failure for this long, a run of failures has ended.
53///
54/// Needed because recovery is not always observable as a success: once
55/// readiness fails, the load balancer stops sending connections, so no
56/// `accept` may succeed for a long time. A quiet listener is a recovered one.
57/// The backoff tops out at [`MAXIMUM_BACKOFF`], so a continuing failure is
58/// never this quiet.
59pub const QUIET_AFTER_FAILURE: Duration = Duration::from_secs(10);
60
61/// Whether an `accept` error leaves the listener unusable.
62pub fn is_fatal(error: &io::Error) -> bool {
63    #[cfg(unix)]
64    {
65        matches!(
66            error.raw_os_error(),
67            Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK | libc::EFAULT)
68        )
69    }
70    #[cfg(not(unix))]
71    {
72        // Without errno, only the portable spelling of "this is not a usable
73        // listener" is recognised; everything else is retried.
74        error.kind() == io::ErrorKind::InvalidInput
75    }
76}
77
78/// What to do after one failed `accept`.
79#[derive(Debug, Eq, PartialEq)]
80pub enum Next {
81    /// Accept again after `pause`.
82    Retry {
83        pause: Duration,
84        /// Failures to report now, this one included, or `None` while the
85        /// previous report is recent. Reported failures are not counted twice.
86        report: Option<u64>,
87    },
88    /// The listener cannot recover; stop accepting.
89    Stop,
90}
91
92/// Failure bookkeeping for one listener's accept loop.
93#[derive(Debug)]
94pub struct AcceptErrors {
95    backoff: Duration,
96    /// Failures since the last report, included in the next one.
97    unreported: u64,
98    last_reported: Option<Instant>,
99    /// Shared with a readiness probe, when the listener has one.
100    health: Option<AcceptHealth>,
101}
102
103impl Default for AcceptErrors {
104    fn default() -> Self {
105        Self {
106            backoff: INITIAL_BACKOFF,
107            unreported: 0,
108            last_reported: None,
109            health: None,
110        }
111    }
112}
113
114impl AcceptErrors {
115    /// Also records failures and successes in `health`, for readiness.
116    pub fn with_health(health: AcceptHealth) -> Self {
117        Self {
118            health: Some(health),
119            ..Self::default()
120        }
121    }
122
123    /// Records a failure and says whether, and when, to accept again.
124    pub fn failed(&mut self, error: &io::Error) -> Next {
125        self.failed_at(error, Instant::now())
126    }
127
128    fn failed_at(&mut self, error: &io::Error, now: Instant) -> Next {
129        if is_fatal(error) {
130            return Next::Stop;
131        }
132        if let Some(health) = &self.health {
133            health.failed_at(now);
134        }
135        self.unreported += 1;
136        let due = self
137            .last_reported
138            .is_none_or(|last| now.duration_since(last) >= REPORT_INTERVAL);
139        let report = due.then(|| {
140            self.last_reported = Some(now);
141            std::mem::take(&mut self.unreported)
142        });
143        let pause = self.backoff;
144        self.backoff = (self.backoff * 2).min(MAXIMUM_BACKOFF);
145        Next::Retry { pause, report }
146    }
147
148    /// Records a successful `accept`, so the next failure starts afresh.
149    ///
150    /// Returns whether failures had been reported since the last success, for
151    /// a caller that announces recovery.
152    pub fn succeeded(&mut self) -> bool {
153        let recovered = self.last_reported.is_some();
154        if let Some(health) = &self.health {
155            health.succeeded();
156        }
157        self.backoff = INITIAL_BACKOFF;
158        self.unreported = 0;
159        self.last_reported = None;
160        recovered
161    }
162}
163
164/// Whether one listener has been failing to accept for long enough that its
165/// process should stop receiving new connections.
166///
167/// Cheap to clone and lock-free, because a readiness probe reads it while the
168/// accept loop writes it. Feed it through [`AcceptErrors::with_health`].
169#[derive(Clone, Debug)]
170pub struct AcceptHealth(Arc<HealthState>);
171
172#[derive(Debug)]
173struct HealthState {
174    /// The clock the two instants below are measured on.
175    origin: Instant,
176    /// Milliseconds since `origin`, plus one, when the current run of
177    /// failures began; zero when there is none.
178    failing_since: AtomicU64,
179    /// Milliseconds since `origin`, plus one, of the latest failure.
180    failed_at: AtomicU64,
181}
182
183impl Default for AcceptHealth {
184    fn default() -> Self {
185        Self(Arc::new(HealthState {
186            origin: Instant::now(),
187            failing_since: AtomicU64::new(0),
188            failed_at: AtomicU64::new(0),
189        }))
190    }
191}
192
193impl AcceptHealth {
194    /// Whether `accept` has failed continuously for [`PERSISTENT_FAILURE`],
195    /// most recently within [`QUIET_AFTER_FAILURE`].
196    pub fn is_failing(&self) -> bool {
197        self.is_failing_at(Instant::now())
198    }
199
200    fn is_failing_at(&self, now: Instant) -> bool {
201        let since = self.0.failing_since.load(Ordering::Relaxed);
202        let last = self.0.failed_at.load(Ordering::Relaxed);
203        if since == 0 || last == 0 {
204            return false;
205        }
206        self.millis(now).saturating_sub(last) <= millis(QUIET_AFTER_FAILURE)
207            && last.saturating_sub(since) >= millis(PERSISTENT_FAILURE)
208    }
209
210    fn failed_at(&self, now: Instant) {
211        let now = self.millis(now);
212        let last = self.0.failed_at.swap(now, Ordering::Relaxed);
213        // A failure after a quiet spell starts a new run rather than
214        // extending one that has already ended.
215        if last == 0 || now.saturating_sub(last) > millis(QUIET_AFTER_FAILURE) {
216            self.0.failing_since.store(now, Ordering::Relaxed);
217        }
218    }
219
220    fn succeeded(&self) {
221        self.0.failing_since.store(0, Ordering::Relaxed);
222        self.0.failed_at.store(0, Ordering::Relaxed);
223    }
224
225    /// Plus one, so zero can mean "never".
226    fn millis(&self, now: Instant) -> u64 {
227        millis(now.saturating_duration_since(self.0.origin)) + 1
228    }
229}
230
231fn millis(duration: Duration) -> u64 {
232    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
233}
234
235#[cfg(all(test, unix))]
236mod tests {
237    use super::*;
238
239    fn os(code: i32) -> io::Error {
240        io::Error::from_raw_os_error(code)
241    }
242
243    #[test]
244    fn only_errors_about_the_listener_itself_are_fatal() {
245        for code in [libc::EBADF, libc::EINVAL, libc::ENOTSOCK, libc::EFAULT] {
246            assert!(is_fatal(&os(code)), "{code}");
247        }
248        for code in [
249            libc::EMFILE,
250            libc::ENFILE,
251            libc::ENOBUFS,
252            libc::ENOMEM,
253            libc::ECONNABORTED,
254            libc::EINTR,
255            libc::EPROTO,
256            libc::EPERM,
257            libc::ENETDOWN,
258            libc::EHOSTUNREACH,
259            libc::EOPNOTSUPP,
260        ] {
261            assert!(!is_fatal(&os(code)), "{code}");
262        }
263    }
264
265    #[test]
266    fn backoff_doubles_to_a_ceiling_and_resets_on_success() {
267        let mut errors = AcceptErrors::default();
268        let now = Instant::now();
269        let pauses: Vec<_> = (0..8)
270            .map(|_| match errors.failed_at(&os(libc::EMFILE), now) {
271                Next::Retry { pause, .. } => pause,
272                Next::Stop => panic!("EMFILE is temporary"),
273            })
274            .collect();
275        assert_eq!(pauses[0], INITIAL_BACKOFF);
276        assert_eq!(pauses[1], INITIAL_BACKOFF * 2);
277        assert_eq!(pauses[7], MAXIMUM_BACKOFF);
278
279        assert!(errors.succeeded(), "the first failure was reported");
280        assert!(matches!(
281            errors.failed_at(&os(libc::EMFILE), now),
282            Next::Retry {
283                pause: INITIAL_BACKOFF,
284                report: Some(1)
285            }
286        ));
287    }
288
289    #[test]
290    fn readiness_fails_only_after_persistent_failure_and_recovers() {
291        let health = AcceptHealth::default();
292        let mut errors = AcceptErrors::with_health(health.clone());
293        let start = Instant::now();
294        let at = |seconds| start + Duration::from_secs(seconds);
295
296        // One failure a second, as the capped backoff produces.
297        for second in 0..30 {
298            errors.failed_at(&os(libc::EMFILE), at(second));
299        }
300        assert!(
301            !health.is_failing_at(at(29)),
302            "29 seconds of failure is still a burst"
303        );
304        errors.failed_at(&os(libc::EMFILE), at(30));
305        assert!(health.is_failing_at(at(30)), "30 seconds is persistent");
306
307        assert!(
308            !health.is_failing_at(at(41)),
309            "ten seconds without a failure ends the run"
310        );
311        errors.failed_at(&os(libc::EMFILE), at(42));
312        assert!(
313            !health.is_failing_at(at(42)),
314            "a failure after a quiet spell starts a new run"
315        );
316
317        for second in 43..=72 {
318            errors.failed_at(&os(libc::EMFILE), at(second));
319        }
320        assert!(health.is_failing_at(at(72)));
321        errors.succeeded();
322        assert!(!health.is_failing_at(at(72)), "a success ends the run");
323    }
324
325    #[test]
326    fn a_fatal_error_does_not_count_as_a_failing_run() {
327        let health = AcceptHealth::default();
328        let mut errors = AcceptErrors::with_health(health.clone());
329        assert_eq!(errors.failed(&os(libc::EBADF)), Next::Stop);
330        assert!(!health.is_failing());
331    }
332
333    #[test]
334    fn a_fatal_error_stops_the_loop() {
335        assert_eq!(AcceptErrors::default().failed(&os(libc::EBADF)), Next::Stop);
336    }
337
338    #[test]
339    fn a_persistent_failure_is_reported_as_a_summary() {
340        let mut errors = AcceptErrors::default();
341        let start = Instant::now();
342        let report = |next| match next {
343            Next::Retry { report, .. } => report,
344            Next::Stop => panic!("EMFILE is temporary"),
345        };
346        assert_eq!(
347            report(errors.failed_at(&os(libc::EMFILE), start)),
348            Some(1),
349            "the first failure is reported at once"
350        );
351        for _ in 0..5 {
352            let later = start + Duration::from_secs(1);
353            assert_eq!(report(errors.failed_at(&os(libc::EMFILE), later)), None);
354        }
355        assert_eq!(
356            report(errors.failed_at(&os(libc::EMFILE), start + REPORT_INTERVAL)),
357            Some(6),
358            "the summary counts the repeats inside the interval"
359        );
360    }
361}