shep_client/spawn.rs
1//! Autostart: [`connect_or_spawn`], [`connect_or_spawn_with`], [`SpawnError`]
2//!
3//! A bound-but-not-accepting unix socket still completes `connect(2)` into
4//! the kernel backlog, so a bare connect can't tell "nothing is listening"
5//! from "something is listening and not yet answering". Every probe here,
6//! including the first, is a full handshake via
7//! [`Client::connect_with_timeout`]. Only [`ConnectError::Connect`] (the OS
8//! refusing the connect outright) reads as "nothing to disturb, launch a
9//! daemon"; [`ConnectError::HandshakeTimeout`] on that first probe means a
10//! daemon is already bound and mid-boot, so nothing launches and probing continues.
11
12use core::fmt;
13use std::path::Path;
14use std::process::ExitStatus;
15use std::time::{Duration, Instant};
16
17use crate::client::Client;
18use crate::connection::ConnectError;
19
20/// How long the spawn-and-wait path will keep probing before giving up.
21pub const SPAWN_DEADLINE: Duration = Duration::from_secs(30);
22
23/// First retry gap; grows ×1.5 up to [`BACKOFF_CAP`] (spec §6).
24pub const BACKOFF_START: Duration = Duration::from_millis(100);
25
26/// Ceiling for the retry gap (spec §6).
27pub const BACKOFF_CAP: Duration = Duration::from_secs(5);
28
29/// The per-attempt handshake budget, defined in the (private) `connection`
30/// module and surfaced here so the whole spawn budget reads from one place.
31pub use crate::connection::HANDSHAKE_TIMEOUT;
32
33/// Exit status a `shep daemon` child uses when another daemon already holds
34/// this `$SHEP_HOME` (`shep-cli`'s `ExitCode::DaemonAlreadyRunning`).
35///
36/// Couples this client to the CLI's exit-code taxonomy: an exit status is
37/// the only channel a dead child leaves behind. Changing either side
38/// without the other reintroduces the race in [`SpawnError::DaemonExited`].
39pub const DAEMON_ALREADY_RUNNING: i32 = 10;
40
41/// Every timing [`connect_or_spawn`] obeys, injectable so tests do not spend
42/// 30 wall-clock seconds proving a probe is bounded.
43#[derive(Debug, Clone)]
44pub struct SpawnOptions {
45 /// Total wall-clock budget for the whole spawn-and-wait path, from the
46 /// first probe failure to giving up.
47 pub deadline: Duration,
48 /// The retry gap before the first post-launch probe.
49 pub backoff_start: Duration,
50 /// Ceiling the retry gap grows toward (×1.5 per attempt) but never past.
51 pub backoff_cap: Duration,
52 /// Budget for one connect-plus-handshake attempt, passed to
53 /// [`Client::connect_with_timeout`] on every probe.
54 pub handshake_timeout: Duration,
55}
56
57impl Default for SpawnOptions {
58 fn default() -> Self {
59 Self {
60 deadline: SPAWN_DEADLINE,
61 backoff_start: BACKOFF_START,
62 backoff_cap: BACKOFF_CAP,
63 handshake_timeout: HANDSHAKE_TIMEOUT,
64 }
65 }
66}
67
68/// What [`connect_or_spawn`] (or [`connect_or_spawn_with`]) had to do to hand
69/// back a connected [`Client`].
70///
71/// No `#[non_exhaustive]`: the algorithm has exactly two steps (probe,
72/// then launch and retry), and each variant is one step's success exit.
73/// Contrast [`SpawnError`], which does carry the attribute, since a
74/// failure can originate from either step, the launcher, or the deadline.
75#[derive(Debug)]
76pub enum SpawnOutcome {
77 /// The very first probe completed a handshake: a daemon was already up,
78 /// and nothing was launched.
79 Connected(Client),
80 /// The first probe found nothing (or an unfinished handshake), a daemon
81 /// was launched (or was already mid-boot), and a later probe succeeded.
82 Spawned(Client),
83}
84
85/// Why [`connect_or_spawn`] (or [`connect_or_spawn_with`]) failed to hand
86/// back a connected [`Client`].
87///
88/// Non-exhaustive: expect more variants.
89#[derive(Debug)]
90#[non_exhaustive]
91pub enum SpawnError {
92 /// The first probe failed for a reason other than "nothing is
93 /// listening" or "listening but not yet answering": a protocol
94 /// mismatch, above all. A daemon that refuses on version skew is
95 /// still a daemon; launching a second one would fail identically.
96 Connect(ConnectError),
97 /// The caller-supplied launcher itself returned an `io::Error`:
98 /// `Command::spawn()` failed before any daemon process existed to probe.
99 Launch(std::io::Error),
100 /// The launched child exited with a status other than
101 /// [`DAEMON_ALREADY_RUNNING`] before any later probe succeeded. Any
102 /// other non-zero (or signalled) exit is treated as fatal rather than
103 /// burning the rest of the deadline probing a corpse; an exit carrying
104 /// [`DAEMON_ALREADY_RUNNING`] is the losing side of a cold-start race
105 /// (another process's `flock(2)` won) and is not reported this way.
106 DaemonExited {
107 /// The child's own exit status.
108 status: ExitStatus,
109 },
110 /// No probe succeeded before `opts.deadline` elapsed.
111 DeadlineExpired {
112 /// The budget that was exceeded: `opts.deadline`, not the
113 /// wall-clock time actually spent.
114 after: Duration,
115 /// The most recent probe failure, if any probe was attempted: says
116 /// whether nothing was listening, or something never answered.
117 last: Option<ConnectError>,
118 },
119}
120
121impl SpawnError {
122 /// The [`shep_core::exit`] code a process stopping on this error uses.
123 ///
124 /// A connect keeps its own code, so a refusal stays a protocol mismatch
125 /// through autostart. Every other way autostart ends, nothing answers.
126 #[must_use]
127 pub const fn exit_code(&self) -> u8 {
128 match self {
129 Self::Connect(inner) => inner.exit_code(),
130 Self::Launch(_) | Self::DaemonExited { .. } | Self::DeadlineExpired { .. } => {
131 shep_core::exit::DAEMON_UNREACHABLE
132 }
133 }
134 }
135}
136
137impl fmt::Display for SpawnError {
138 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139 match self {
140 Self::Connect(err) => write!(f, "{err}"),
141 Self::Launch(err) => write!(f, "failed to launch the daemon: {err}"),
142 Self::DaemonExited { status } => {
143 write!(
144 f,
145 "the daemon process exited before it started answering: {status}"
146 )
147 }
148 Self::DeadlineExpired {
149 after,
150 last: Some(last),
151 } => write!(f, "no daemon answered within {after:?}: {last}"),
152 Self::DeadlineExpired { after, last: None } => {
153 write!(f, "no daemon answered within {after:?}")
154 }
155 }
156 }
157}
158
159impl core::error::Error for SpawnError {
160 fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
161 match self {
162 Self::Connect(err) => Some(err),
163 Self::Launch(err) => Some(err),
164 Self::DaemonExited { .. } => None,
165 Self::DeadlineExpired { last, .. } => last
166 .as_ref()
167 .map(|err| err as &(dyn core::error::Error + 'static)),
168 }
169 }
170}
171
172/// Connects to `socket`, launching a daemon via `launch` if nothing answers
173/// the first probe.
174///
175/// Shorthand for [`connect_or_spawn_with`]`(socket, launch,
176/// SpawnOptions::default())`. Production code calls this and never names a
177/// timing; [`connect_or_spawn_with`] exists so tests can inject a much
178/// shorter deadline.
179///
180/// # Errors
181///
182/// See [`connect_or_spawn_with`].
183pub async fn connect_or_spawn<L>(socket: &Path, launch: L) -> Result<SpawnOutcome, SpawnError>
184where
185 L: FnOnce() -> std::io::Result<std::process::Child> + Send + 'static,
186{
187 connect_or_spawn_with(socket, launch, SpawnOptions::default()).await
188}
189
190/// As [`connect_or_spawn`], with caller-supplied timing.
191///
192/// The first probe is a full handshake. [`ConnectError::Connect`] launches
193/// `launch` on a blocking thread, so its filesystem calls cannot stall the
194/// runtime. [`ConnectError::HandshakeTimeout`] means a daemon is mid-boot.
195///
196/// # Errors
197///
198/// - [`SpawnError::Connect`]: a probe failed with anything but
199/// [`ConnectError::Connect`] or [`ConnectError::HandshakeTimeout`].
200/// - [`SpawnError::Launch`]: `launch` itself returned an `io::Error`.
201/// - [`SpawnError::DaemonExited`]: the child exited before a probe succeeded.
202/// - [`SpawnError::DeadlineExpired`]: no probe succeeded before `opts.deadline`.
203pub async fn connect_or_spawn_with<L>(
204 socket: &Path,
205 launch: L,
206 opts: SpawnOptions,
207) -> Result<SpawnOutcome, SpawnError>
208where
209 L: FnOnce() -> std::io::Result<std::process::Child> + Send + 'static,
210{
211 match Client::connect_with_timeout(socket, opts.handshake_timeout).await {
212 Ok(client) => return Ok(SpawnOutcome::Connected(client)),
213 Err(ConnectError::Connect { .. }) => {}
214 Err(err @ ConnectError::HandshakeTimeout { .. }) => {
215 return probe_until_ready(socket, None, opts, Some(err)).await;
216 }
217 Err(other) => return Err(SpawnError::Connect(other)),
218 }
219
220 let launched = tokio::task::spawn_blocking(launch).await;
221 let child = match launched {
222 Ok(result) => result.map_err(SpawnError::Launch)?,
223 // A panic inside the launcher must stay a panic. Three tests in this
224 // task launch with `unreachable!("...")` as their whole assertion; if
225 // a `JoinError` swallowed that, all three would silently stop
226 // testing anything.
227 Err(join) if join.is_panic() => std::panic::resume_unwind(join.into_panic()),
228 Err(join) => return Err(SpawnError::Launch(std::io::Error::other(join))),
229 };
230
231 probe_until_ready(socket, Some(child), opts, None).await
232}
233
234/// The retry loop shared by both entry points. It keeps probing `socket`
235/// until one attempt completes a handshake or `opts.deadline` elapses.
236///
237/// `child` is `None` only for the no-launch branch. Otherwise it is the
238/// process just launched, dropped on every exit path. `seed` names the
239/// `ConnectError` that entered that branch.
240async fn probe_until_ready(
241 socket: &Path,
242 mut child: Option<std::process::Child>,
243 opts: SpawnOptions,
244 seed: Option<ConnectError>,
245) -> Result<SpawnOutcome, SpawnError> {
246 let start = Instant::now();
247 let mut backoff = opts.backoff_start;
248 let mut last = seed;
249 // flips once try_wait() has reaped the child, so later iterations
250 // stop calling try_wait() on an already-reaped child, which errors
251 let mut child_reaped = false;
252
253 loop {
254 if start.elapsed() >= opts.deadline {
255 return Err(SpawnError::DeadlineExpired {
256 after: opts.deadline,
257 last,
258 });
259 }
260
261 if !child_reaped
262 && let Some(proc) = child.as_mut()
263 && let Ok(Some(status)) = proc.try_wait()
264 {
265 child_reaped = true;
266 // a clean exit (status 0) is fatal too: `child` is the daemon
267 // process itself, so any exit at all means it isn't serving
268 if status.code() != Some(DAEMON_ALREADY_RUNNING) {
269 return Err(SpawnError::DaemonExited { status });
270 }
271 // Otherwise: another process won the cold-start race. The
272 // daemon it brought up is still worth probing for.
273 }
274
275 tokio::time::sleep(backoff).await;
276 backoff = backoff.mul_f64(1.5).min(opts.backoff_cap);
277
278 match Client::connect_with_timeout(socket, opts.handshake_timeout).await {
279 Ok(client) => return Ok(SpawnOutcome::Spawned(client)),
280 // still a daemon: propagate immediately rather than burn the
281 // rest of the deadline on a condition already answered
282 Err(err @ ConnectError::ProtocolMismatch { .. }) => {
283 return Err(SpawnError::Connect(err));
284 }
285 Err(err) => last = Some(err),
286 }
287 }
288}