Skip to main content

shep_client/
spawn.rs

1//! Autostart: [`connect_or_spawn`], [`connect_or_spawn_with`], [`SpawnError`]
2//!
3//! A bound-but-not-accepting unix socket still completes `connect(2)` into
4//! the kernel backlog, so a bare connect can't tell "nothing is listening"
5//! from "something is listening and not yet answering". Every probe here,
6//! including the first, is a full handshake via
7//! [`Client::connect_with_timeout`]. Only [`ConnectError::Connect`] (the OS
8//! refusing the connect outright) reads as "nothing to disturb, launch a
9//! daemon"; [`ConnectError::HandshakeTimeout`] on that first probe means a
10//! daemon is already bound and mid-boot, so nothing launches and probing continues.
11
12use core::fmt;
13use std::path::Path;
14use std::process::ExitStatus;
15use std::time::{Duration, Instant};
16
17use crate::client::Client;
18use crate::connection::ConnectError;
19
20/// How long the spawn-and-wait path will keep probing before giving up.
21pub const SPAWN_DEADLINE: Duration = Duration::from_secs(30);
22
23/// First retry gap; grows ×1.5 up to [`BACKOFF_CAP`] (spec §6).
24pub const BACKOFF_START: Duration = Duration::from_millis(100);
25
26/// Ceiling for the retry gap (spec §6).
27pub const BACKOFF_CAP: Duration = Duration::from_secs(5);
28
29/// The per-attempt handshake budget, defined in the (private) `connection`
30/// module and surfaced here so the whole spawn budget reads from one place.
31pub use crate::connection::HANDSHAKE_TIMEOUT;
32
33/// Exit status a `shep daemon` child uses when another daemon already holds
34/// this `$SHEP_HOME` (`shep-cli`'s `ExitCode::DaemonAlreadyRunning`).
35///
36/// Couples this client to the CLI's exit-code taxonomy: an exit status is
37/// the only channel a dead child leaves behind. Changing either side
38/// without the other reintroduces the race in [`SpawnError::DaemonExited`].
39pub const DAEMON_ALREADY_RUNNING: i32 = 10;
40
41/// Every timing [`connect_or_spawn`] obeys, injectable so tests do not spend
42/// 30 wall-clock seconds proving a probe is bounded.
43#[derive(Debug, Clone)]
44pub struct SpawnOptions {
45    /// Total wall-clock budget for the whole spawn-and-wait path, from the
46    /// first probe failure to giving up.
47    pub deadline: Duration,
48    /// The retry gap before the first post-launch probe.
49    pub backoff_start: Duration,
50    /// Ceiling the retry gap grows toward (×1.5 per attempt) but never past.
51    pub backoff_cap: Duration,
52    /// Budget for one connect-plus-handshake attempt, passed to
53    /// [`Client::connect_with_timeout`] on every probe.
54    pub handshake_timeout: Duration,
55}
56
57impl Default for SpawnOptions {
58    fn default() -> Self {
59        Self {
60            deadline: SPAWN_DEADLINE,
61            backoff_start: BACKOFF_START,
62            backoff_cap: BACKOFF_CAP,
63            handshake_timeout: HANDSHAKE_TIMEOUT,
64        }
65    }
66}
67
68/// What [`connect_or_spawn`] (or [`connect_or_spawn_with`]) had to do to hand
69/// back a connected [`Client`].
70///
71/// No `#[non_exhaustive]`: the algorithm has exactly two steps (probe,
72/// then launch and retry), and each variant is one step's success exit.
73/// Contrast [`SpawnError`], which does carry the attribute, since a
74/// failure can originate from either step, the launcher, or the deadline.
75#[derive(Debug)]
76pub enum SpawnOutcome {
77    /// The very first probe completed a handshake: a daemon was already up,
78    /// and nothing was launched.
79    Connected(Client),
80    /// The first probe found nothing (or an unfinished handshake), a daemon
81    /// was launched (or was already mid-boot), and a later probe succeeded.
82    Spawned(Client),
83}
84
85/// Why [`connect_or_spawn`] (or [`connect_or_spawn_with`]) failed to hand
86/// back a connected [`Client`].
87///
88/// Non-exhaustive: expect more variants.
89#[derive(Debug)]
90#[non_exhaustive]
91pub enum SpawnError {
92    /// The first probe failed for a reason other than "nothing is
93    /// listening" or "listening but not yet answering": a protocol
94    /// mismatch, above all. A daemon that refuses on version skew is
95    /// still a daemon; launching a second one would fail identically.
96    Connect(ConnectError),
97    /// The caller-supplied launcher itself returned an `io::Error`:
98    /// `Command::spawn()` failed before any daemon process existed to probe.
99    Launch(std::io::Error),
100    /// The launched child exited with a status other than
101    /// [`DAEMON_ALREADY_RUNNING`] before any later probe succeeded. Any
102    /// other non-zero (or signalled) exit is treated as fatal rather than
103    /// burning the rest of the deadline probing a corpse; an exit carrying
104    /// [`DAEMON_ALREADY_RUNNING`] is the losing side of a cold-start race
105    /// (another process's `flock(2)` won) and is not reported this way.
106    DaemonExited {
107        /// The child's own exit status.
108        status: ExitStatus,
109    },
110    /// No probe succeeded before `opts.deadline` elapsed.
111    DeadlineExpired {
112        /// The budget that was exceeded: `opts.deadline`, not the
113        /// wall-clock time actually spent.
114        after: Duration,
115        /// The most recent probe failure, if any probe was attempted: says
116        /// whether nothing was listening, or something never answered.
117        last: Option<ConnectError>,
118    },
119}
120
121impl SpawnError {
122    /// The [`shep_core::exit`] code a process stopping on this error uses.
123    ///
124    /// A connect keeps its own code, so a refusal stays a protocol mismatch
125    /// through autostart. Every other way autostart ends, nothing answers.
126    #[must_use]
127    pub const fn exit_code(&self) -> u8 {
128        match self {
129            Self::Connect(inner) => inner.exit_code(),
130            Self::Launch(_) | Self::DaemonExited { .. } | Self::DeadlineExpired { .. } => {
131                shep_core::exit::DAEMON_UNREACHABLE
132            }
133        }
134    }
135}
136
137impl fmt::Display for SpawnError {
138    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139        match self {
140            Self::Connect(err) => write!(f, "{err}"),
141            Self::Launch(err) => write!(f, "failed to launch the daemon: {err}"),
142            Self::DaemonExited { status } => {
143                write!(
144                    f,
145                    "the daemon process exited before it started answering: {status}"
146                )
147            }
148            Self::DeadlineExpired {
149                after,
150                last: Some(last),
151            } => write!(f, "no daemon answered within {after:?}: {last}"),
152            Self::DeadlineExpired { after, last: None } => {
153                write!(f, "no daemon answered within {after:?}")
154            }
155        }
156    }
157}
158
159impl core::error::Error for SpawnError {
160    fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
161        match self {
162            Self::Connect(err) => Some(err),
163            Self::Launch(err) => Some(err),
164            Self::DaemonExited { .. } => None,
165            Self::DeadlineExpired { last, .. } => last
166                .as_ref()
167                .map(|err| err as &(dyn core::error::Error + 'static)),
168        }
169    }
170}
171
172/// Connects to `socket`, launching a daemon via `launch` if nothing answers
173/// the first probe.
174///
175/// Shorthand for [`connect_or_spawn_with`]`(socket, launch,
176/// SpawnOptions::default())`. Production code calls this and never names a
177/// timing; [`connect_or_spawn_with`] exists so tests can inject a much
178/// shorter deadline.
179///
180/// # Errors
181///
182/// See [`connect_or_spawn_with`].
183pub async fn connect_or_spawn<L>(socket: &Path, launch: L) -> Result<SpawnOutcome, SpawnError>
184where
185    L: FnOnce() -> std::io::Result<std::process::Child> + Send + 'static,
186{
187    connect_or_spawn_with(socket, launch, SpawnOptions::default()).await
188}
189
190/// As [`connect_or_spawn`], with caller-supplied timing.
191///
192/// The first probe is a full handshake. [`ConnectError::Connect`] launches
193/// `launch` on a blocking thread, so its filesystem calls cannot stall the
194/// runtime. [`ConnectError::HandshakeTimeout`] means a daemon is mid-boot.
195///
196/// # Errors
197///
198/// - [`SpawnError::Connect`]: a probe failed with anything but
199///   [`ConnectError::Connect`] or [`ConnectError::HandshakeTimeout`].
200/// - [`SpawnError::Launch`]: `launch` itself returned an `io::Error`.
201/// - [`SpawnError::DaemonExited`]: the child exited before a probe succeeded.
202/// - [`SpawnError::DeadlineExpired`]: no probe succeeded before `opts.deadline`.
203pub async fn connect_or_spawn_with<L>(
204    socket: &Path,
205    launch: L,
206    opts: SpawnOptions,
207) -> Result<SpawnOutcome, SpawnError>
208where
209    L: FnOnce() -> std::io::Result<std::process::Child> + Send + 'static,
210{
211    match Client::connect_with_timeout(socket, opts.handshake_timeout).await {
212        Ok(client) => return Ok(SpawnOutcome::Connected(client)),
213        Err(ConnectError::Connect { .. }) => {}
214        Err(err @ ConnectError::HandshakeTimeout { .. }) => {
215            return probe_until_ready(socket, None, opts, Some(err)).await;
216        }
217        Err(other) => return Err(SpawnError::Connect(other)),
218    }
219
220    let launched = tokio::task::spawn_blocking(launch).await;
221    let child = match launched {
222        Ok(result) => result.map_err(SpawnError::Launch)?,
223        // A panic inside the launcher must stay a panic. Three tests in this
224        // task launch with `unreachable!("...")` as their whole assertion; if
225        // a `JoinError` swallowed that, all three would silently stop
226        // testing anything.
227        Err(join) if join.is_panic() => std::panic::resume_unwind(join.into_panic()),
228        Err(join) => return Err(SpawnError::Launch(std::io::Error::other(join))),
229    };
230
231    probe_until_ready(socket, Some(child), opts, None).await
232}
233
234/// The retry loop shared by both entry points. It keeps probing `socket`
235/// until one attempt completes a handshake or `opts.deadline` elapses.
236///
237/// `child` is `None` only for the no-launch branch. Otherwise it is the
238/// process just launched, dropped on every exit path. `seed` names the
239/// `ConnectError` that entered that branch.
240async fn probe_until_ready(
241    socket: &Path,
242    mut child: Option<std::process::Child>,
243    opts: SpawnOptions,
244    seed: Option<ConnectError>,
245) -> Result<SpawnOutcome, SpawnError> {
246    let start = Instant::now();
247    let mut backoff = opts.backoff_start;
248    let mut last = seed;
249    // flips once try_wait() has reaped the child, so later iterations
250    // stop calling try_wait() on an already-reaped child, which errors
251    let mut child_reaped = false;
252
253    loop {
254        if start.elapsed() >= opts.deadline {
255            return Err(SpawnError::DeadlineExpired {
256                after: opts.deadline,
257                last,
258            });
259        }
260
261        if !child_reaped
262            && let Some(proc) = child.as_mut()
263            && let Ok(Some(status)) = proc.try_wait()
264        {
265            child_reaped = true;
266            // a clean exit (status 0) is fatal too: `child` is the daemon
267            // process itself, so any exit at all means it isn't serving
268            if status.code() != Some(DAEMON_ALREADY_RUNNING) {
269                return Err(SpawnError::DaemonExited { status });
270            }
271            // Otherwise: another process won the cold-start race. The
272            // daemon it brought up is still worth probing for.
273        }
274
275        tokio::time::sleep(backoff).await;
276        backoff = backoff.mul_f64(1.5).min(opts.backoff_cap);
277
278        match Client::connect_with_timeout(socket, opts.handshake_timeout).await {
279            Ok(client) => return Ok(SpawnOutcome::Spawned(client)),
280            // still a daemon: propagate immediately rather than burn the
281            // rest of the deadline on a condition already answered
282            Err(err @ ConnectError::ProtocolMismatch { .. }) => {
283                return Err(SpawnError::Connect(err));
284            }
285            Err(err) => last = Some(err),
286        }
287    }
288}