fallow_process/
spawn_retry.rs

1//! Bounded retry for a spawn whose executable is momentarily busy.
2//!
3//! Unix refuses to `exec` a file while any process holds it open for writing,
4//! and the writer is not always the process that opened it: a fork from any
5//! other thread inherits the descriptor and releases it only when that child
6//! reaches its own `exec`. A process that writes an executable and runs it, or
7//! that starts a binary a package manager is still writing, meets
8//! `ExecutableFileBusy` until the window closes. Nothing about the command is
9//! wrong and the condition clears without help, so a bounded retry is the whole
10//! remedy. The blocking and Tokio spawns share the schedule below so the two
11//! cannot drift apart.
12
13use std::io;
14use std::time::{Duration, Instant};
15
16/// Total time a spawn keeps retrying an `ExecutableFileBusy` failure.
17const EXECUTABLE_BUSY_BUDGET: Duration = Duration::from_secs(1);
18
19/// First pause between `ExecutableFileBusy` retries.
20const EXECUTABLE_BUSY_FIRST_BACKOFF: Duration = Duration::from_micros(200);
21
22/// Upper bound for the doubling `ExecutableFileBusy` backoff.
23const EXECUTABLE_BUSY_MAX_BACKOFF: Duration = Duration::from_millis(20);
24
25/// Pause schedule for a spawn that keeps meeting a busy executable.
26struct ExecutableBusyRetry {
27    deadline: Instant,
28    backoff: Duration,
29}
30
31impl ExecutableBusyRetry {
32    fn start(budget: Duration) -> Self {
33        Self {
34            deadline: Instant::now() + budget,
35            backoff: EXECUTABLE_BUSY_FIRST_BACKOFF,
36        }
37    }
38
39    /// How long to wait before spawning again, or `None` when `error` is not a
40    /// busy executable or the budget is spent.
41    fn pause_after(&mut self, error: &io::Error) -> Option<Duration> {
42        if error.kind() != io::ErrorKind::ExecutableFileBusy || Instant::now() >= self.deadline {
43            return None;
44        }
45        let pause = self.backoff;
46        self.backoff = (self.backoff * 2).min(EXECUTABLE_BUSY_MAX_BACKOFF);
47        Some(pause)
48    }
49}
50
51/// Spawn `command`, waiting out an executable that is momentarily busy.
52///
53/// Every other spawn failure is returned from the first attempt, and a spawn
54/// that succeeds immediately pays nothing.
55pub fn spawn_retrying_busy_executable(
56    command: &mut std::process::Command,
57) -> io::Result<std::process::Child> {
58    retry_while_executable_busy(EXECUTABLE_BUSY_BUDGET, || command.spawn())
59}
60
61/// Spawn `command` on the Tokio runtime, waiting out an executable that is
62/// momentarily busy.
63///
64/// The pause yields to the runtime rather than blocking the worker thread.
65#[cfg(feature = "tokio")]
66pub async fn spawn_tokio_retrying_busy_executable(
67    command: &mut tokio::process::Command,
68) -> io::Result<tokio::process::Child> {
69    retry_while_executable_busy_async(EXECUTABLE_BUSY_BUDGET, || command.spawn()).await
70}
71
72/// Twin of [`retry_while_executable_busy`] whose pause yields to the runtime
73/// instead of blocking the worker thread. Both drive the same schedule, so a
74/// change to one cannot leave the other behind.
75#[cfg(feature = "tokio")]
76async fn retry_while_executable_busy_async<T>(
77    budget: Duration,
78    mut attempt: impl FnMut() -> io::Result<T>,
79) -> io::Result<T> {
80    let mut retry = ExecutableBusyRetry::start(budget);
81    loop {
82        let error = match attempt() {
83            Ok(value) => return Ok(value),
84            Err(error) => error,
85        };
86        let Some(pause) = retry.pause_after(&error) else {
87            return Err(error);
88        };
89        tokio::time::sleep(pause).await;
90    }
91}
92
93/// Repeat `attempt` while it reports `ExecutableFileBusy` and `budget` has not
94/// elapsed. Every other outcome is returned from the first attempt.
95fn retry_while_executable_busy<T>(
96    budget: Duration,
97    mut attempt: impl FnMut() -> io::Result<T>,
98) -> io::Result<T> {
99    let mut retry = ExecutableBusyRetry::start(budget);
100    loop {
101        let error = match attempt() {
102            Ok(value) => return Ok(value),
103            Err(error) => error,
104        };
105        let Some(pause) = retry.pause_after(&error) else {
106            return Err(error);
107        };
108        std::thread::sleep(pause);
109    }
110}
111
112#[cfg(test)]
113#[expect(
114    clippy::expect_used,
115    reason = "test setup failures should fail at the exact setup operation"
116)]
117mod tests {
118    use super::*;
119
120    #[test]
121    fn a_busy_executable_is_retried_until_it_is_free() {
122        let mut attempts = 0;
123        let result = retry_while_executable_busy(Duration::from_secs(30), || {
124            attempts += 1;
125            if attempts < 3 {
126                return Err(io::Error::from(io::ErrorKind::ExecutableFileBusy));
127            }
128            Ok(())
129        });
130
131        assert!(result.is_ok(), "a target that frees itself should spawn");
132        assert_eq!(attempts, 3);
133    }
134
135    #[test]
136    fn a_permanently_busy_executable_fails_after_the_budget() {
137        let mut attempts = 0;
138        let started = Instant::now();
139        let error = retry_while_executable_busy(Duration::from_millis(20), || {
140            attempts += 1;
141            Err::<(), io::Error>(io::Error::from(io::ErrorKind::ExecutableFileBusy))
142        })
143        .expect_err("a target that stays busy keeps failing");
144
145        assert_eq!(error.kind(), io::ErrorKind::ExecutableFileBusy);
146        assert!(
147            attempts > 1,
148            "the budget should cover more than one attempt"
149        );
150        assert!(started.elapsed() >= Duration::from_millis(20));
151    }
152
153    #[test]
154    fn other_spawn_failures_are_reported_without_a_retry() {
155        let mut attempts = 0;
156        let error = retry_while_executable_busy(Duration::from_secs(30), || {
157            attempts += 1;
158            Err::<(), io::Error>(io::Error::from(io::ErrorKind::NotFound))
159        })
160        .expect_err("a missing executable stays missing");
161
162        assert_eq!(error.kind(), io::ErrorKind::NotFound);
163        assert_eq!(attempts, 1);
164    }
165
166    #[test]
167    fn the_pause_doubles_up_to_the_ceiling() {
168        let busy = io::Error::from(io::ErrorKind::ExecutableFileBusy);
169        let mut retry = ExecutableBusyRetry::start(Duration::from_secs(30));
170
171        let mut pauses = Vec::new();
172        for _ in 0..12 {
173            pauses.push(retry.pause_after(&busy).expect("the budget is not spent"));
174        }
175
176        assert_eq!(pauses[0], EXECUTABLE_BUSY_FIRST_BACKOFF);
177        assert_eq!(pauses[1], EXECUTABLE_BUSY_FIRST_BACKOFF * 2);
178        assert!(pauses.windows(2).all(|pair| pair[0] <= pair[1]));
179        assert_eq!(
180            *pauses.last().expect("pauses were recorded"),
181            EXECUTABLE_BUSY_MAX_BACKOFF
182        );
183    }
184
185    #[cfg(feature = "tokio")]
186    #[tokio::test]
187    async fn an_awaited_busy_executable_is_retried_until_it_is_free() {
188        let mut attempts = 0;
189        let result = retry_while_executable_busy_async(Duration::from_secs(30), || {
190            attempts += 1;
191            if attempts < 3 {
192                return Err(io::Error::from(io::ErrorKind::ExecutableFileBusy));
193            }
194            Ok(())
195        })
196        .await;
197
198        assert!(result.is_ok(), "a target that frees itself should spawn");
199        assert_eq!(attempts, 3);
200    }
201
202    #[cfg(feature = "tokio")]
203    #[tokio::test]
204    async fn an_awaited_permanently_busy_executable_fails_after_the_budget() {
205        let mut attempts = 0;
206        let started = Instant::now();
207        let error = retry_while_executable_busy_async(Duration::from_millis(20), || {
208            attempts += 1;
209            Err::<(), io::Error>(io::Error::from(io::ErrorKind::ExecutableFileBusy))
210        })
211        .await
212        .expect_err("a target that stays busy keeps failing");
213
214        assert_eq!(error.kind(), io::ErrorKind::ExecutableFileBusy);
215        assert!(
216            attempts > 1,
217            "the budget should cover more than one attempt"
218        );
219        assert!(started.elapsed() >= Duration::from_millis(20));
220    }
221
222    #[cfg(feature = "tokio")]
223    #[tokio::test]
224    async fn a_tokio_spawn_reports_a_missing_executable_without_a_retry() {
225        let mut command = tokio::process::Command::new("fallow-executable-that-does-not-exist");
226        let started = Instant::now();
227
228        let error = spawn_tokio_retrying_busy_executable(&mut command)
229            .await
230            .expect_err("a missing executable stays missing");
231
232        assert_eq!(error.kind(), io::ErrorKind::NotFound);
233        assert!(started.elapsed() < EXECUTABLE_BUSY_BUDGET);
234    }
235}
fallow_process/spawn_retry.rs

fallow_process/
spawn_retry.rs
