shep-daemon 0.22.0

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
#[cfg(unix)]
use super::AdoptedReaper;
use super::dog_link::DogLink;
use super::log_protocol::{ExitOutcome, LogCtl, LogLine};
use crate::channel::{ChildMessage, ShepherdMessage};
use crate::privilege::Credentials;
use core::fmt;
use shep_core::signals::OperatorSignal;
use std::collections::BTreeMap;
use std::path::PathBuf;
use tokio::sync::{mpsc, oneshot};

/// Typed stop signal
///
/// [`StopSignal::as_raw`] gives the unix number so fake and real runners
/// record identical [`ExitOutcome`]s.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StopSignal {
    /// `SIGTERM`: graceful stop request
    Term,
    /// `SIGINT`: interrupt
    Int,
    /// `SIGQUIT`: quit, core-dumping by default
    Quit,
    /// `SIGUSR2`: user-defined signal 2
    Usr2,
    /// `SIGKILL`: unblockable, immediate
    Kill,
}

impl StopSignal {
    /// The raw unix signal number
    #[must_use]
    pub fn as_raw(self) -> i32 {
        match self {
            Self::Term => 15,
            Self::Int => 2,
            Self::Quit => 3,
            Self::Usr2 => 12,
            Self::Kill => 9,
        }
    }
}

/// IO endpoints handed back by spawn; the runner pumps internally.
///
/// The sheep task owns this and must drain every receiver: an undrained
/// `from_child` back-pressures a metric-emitting child until it stalls on
/// its own fd-3 write.
#[derive(Debug)]
pub struct ProcIo {
    /// stdout+stderr lines
    pub logs: mpsc::Receiver<LogLine>,
    /// Parsed child→daemon shepherd-channel messages
    pub from_child: mpsc::Receiver<ChildMessage>,
    /// daemon→child shepherd-channel sender
    pub to_child: mpsc::Sender<ShepherdMessage>,
    /// Control channel into this sheep's log pump
    ///
    /// The pump is the only reader of the child's stdout and stderr, and it
    /// ends when the last of these senders drops, so hold this while the
    /// child is alive: ending the pump drops the read ends of both pipes and
    /// the child's next write gets `EPIPE`/`SIGPIPE`. A send that fails
    /// means the pump is already gone, which makes a reopen a no-op.
    pub log_ctl: mpsc::Sender<LogCtl>,
    /// The shepherd's writing end of this sheep's stdin.
    ///
    /// Always present, and closed rather than absent when the app asked for
    /// no pipe: the runner drops the receiving end, so `is_closed()` is the
    /// one question a caller asks, the same shape [`Self::to_child`] uses.
    ///
    /// Hold it only for as long as the child is alive.
    pub to_stdin: mpsc::Sender<StdinWrite>,
}

/// One line to write to a sheep's stdin, and where the answer goes.
///
/// The acknowledgement is the point, as on [`LogCtl`]: an `mpsc::send` only
/// proves the message was queued, and the line may still be sitting behind a
/// pipe the app has stopped reading. The `oneshot` fires after the bytes are
/// written and flushed.
#[derive(Debug)]
pub struct StdinWrite {
    /// The line, without its terminator: the writer appends one `\n`.
    pub line: String,
    /// Fires once the line has landed, or with why it could not.
    ///
    /// A dropped sender means the writer task ended before serving this
    /// request, which happens when the child's stdin closed; the caller reads
    /// that as the pipe being gone.
    pub done: oneshot::Sender<Result<(), RunnerError>>,
}

/// One inherited sheep, and everything a runner needs to supervise it again.
///
/// Produced by the successor's `handover::adopt` and consumed by
/// [`ProcessRunner::adopt`]. The handles are owned, since adopting is taking
/// ownership of them; `None` on a pair means the predecessor had no handle
/// to carry. `Debug` is derived: no env value reaches this type.
#[cfg(unix)]
#[derive(Debug)]
pub struct AdoptSpec {
    /// The sheep's name, which a dog's connection is attributed to.
    pub name: String,
    /// The pid the sheep has been running under all along, unchanged by the
    /// handover.
    pub pid: u32,
    /// Where its stdout is logged, kept so a later rotation can reopen it.
    pub out_file: PathBuf,
    /// Where its stderr is logged, kept for the same reason.
    pub err_file: PathBuf,
    /// The read end of its stdout pipe, still the one the child writes into.
    pub out_pipe: Option<tokio::net::unix::pipe::Receiver>,
    /// The read end of its stderr pipe, likewise.
    pub err_pipe: Option<tokio::net::unix::pipe::Receiver>,
    /// The appending handle on its stdout log, written through rather than
    /// reopened so `O_APPEND` survives.
    pub out_log: Option<tokio::fs::File>,
    /// The appending handle on its stderr log, likewise.
    pub err_log: Option<tokio::fs::File>,
    /// Whether each line written through `out_log` and `err_log` starts
    /// with a [`shep_core::logstamp`] stamp, as [`SpawnSpec::log_timestamps`].
    pub log_timestamps: bool,
    /// Whether its stop signal goes to the leader alone, as
    /// [`SpawnSpec::leader_only_stop`].
    pub leader_only_stop: bool,
    /// The write end of its stdin pipe, still the one the child reads from,
    /// for a sheep whose app asked for one.
    ///
    /// The one handle here the daemon writes to. `None` is the commoner
    /// sheep, which has `/dev/null` on fd 0.
    pub stdin_pipe: Option<tokio::net::unix::pipe::Sender>,
    /// The daemon's end of its shepherd-channel socketpair, still the one
    /// whose other end is the child's fd 3.
    ///
    /// Goes both ways, so an adoption puts both pumps back on it. `None` for
    /// a sheep whose app asked for no channel, and for one whose child has
    /// closed fd 3.
    pub channel: Option<tokio::net::UnixStream>,
    /// The daemon's end of the dog's own connection, still the one whose other
    /// end is the child's fd 4. `None` for a sheep that is not a dog, and for
    /// a dog whose connection had closed, which falls back to the control
    /// socket.
    pub dog_link: Option<tokio::net::UnixStream>,
    /// What `dog_link` had subscribed to, or `None` if its dog had not yet
    /// sent its `Hello`. Read only with `dog_link`.
    pub dog_topics: Option<Vec<String>>,
    /// The one reaper this successor waits every adopted pid through.
    ///
    /// Shared rather than owned per sheep: a status can be collected once,
    /// so two reapers racing on one pid would leave one meeting `ECHILD`.
    pub reaper: std::sync::Arc<AdoptedReaper>,
}

/// What a [`ProcessRunner`] can tell about a [`SpawnSpec`] before anything is
/// spawned
///
/// The line that matters runs between [`Self::Impossible`] and
/// [`Self::Doubtful`], and it separates two kinds of claim. A path with a
/// `/` is a claim about the filesystem, which the daemon can check. A bare
/// command is a claim about the daemon's own environment, not the operator's shell.
// `#[non_exhaustive]`: an out-of-tree consumer can match this exhaustively,
// and a fourth verdict would break them with no version bump.
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Preflight {
    /// Nothing is knowable in advance.
    ///
    /// Not "this will work": every form an implementation declines to decide
    /// arrives here alongside every form it decided is fine. A caller may
    /// only act on the other two variants.
    Unknown,
    /// The spawn cannot succeed, as a certainty. Carries one reason, no
    /// trailing punctuation, ready to be printed after a sheep's name.
    ///
    /// A caller registering a batch should refuse the whole batch and
    /// register none of it.
    Impossible(String),
    /// The spawn looks like it will fail, and a caller must not refuse a
    /// batch over it. Carries a reason on the same terms as
    /// [`Self::Impossible`]. Report it and carry on: the spawn then fails for
    /// that one sheep as it would have anyway.
    Doubtful(String),
}

/// Everything a spawn needs, pre-assembled by the assembler (a later task)
#[derive(Clone, Default, PartialEq, Eq)]
pub struct SpawnSpec {
    /// Sheep name (for logging/tracing, not passed to the child)
    pub name: String,
    /// Executable path or name (resolved via `PATH` if bare)
    pub program: String,
    /// Argument vector, `argv[1..]`
    pub args: Vec<String>,
    /// Working directory; `None` inherits the daemon's
    pub cwd: Option<PathBuf>,
    /// Environment variables, fully resolved (no daemon-env leakage beyond this map)
    pub env: BTreeMap<String, String>,
    /// File stdout is appended to
    pub out_file: PathBuf,
    /// File stderr is appended to
    pub err_file: PathBuf,
    /// Start each line appended to `out_file` and `err_file` with a
    /// [`shep_core::logstamp`] stamp. `false` writes the child's lines as
    /// they came.
    pub log_timestamps: bool,
    /// Open the shepherd channel (fd 3 socketpair)
    pub channel: bool,
    /// Pipe the child's stdin, so `shep whisper` can write to it. `false`
    /// gives the child `/dev/null` on fd 0, which is what every sheep gets
    /// unless its config sets `stdin = true`.
    pub stdin: bool,
    /// Unix uid/gid to drop to before exec (`None` inherits the daemon's own
    /// identity). Resolved once per `Start` by `crate::privilege::resolve`;
    /// see that module for how `user`/`group` config names become this.
    pub credentials: Option<Credentials>,
    /// Hand the child a connection to the shepherd: on unix a dog's fd 4
    /// socketpair, open from spawn and named in
    /// [`shep_core::dogs::DOG_LINK_FD_VAR`], and on Windows a named pipe the
    /// dog opens, named in [`shep_core::dogs::DOG_LINK_PIPE_VAR`].
    ///
    /// Set by the supervisor for a dog and never by `assemble`, so a sheep
    /// cannot ask for one.
    pub dog_link: bool,
    /// Send the stop signal to the leader alone, not its process group.
    ///
    /// Set with `sandbox`: `crabpot` passes the signal on to its command,
    /// so a group signal would reach that command twice on macOS.
    /// [`RunningProcess::kill_tree`] stays group-wide, and `max_memory`
    /// still counts the whole tree.
    pub leader_only_stop: bool,
    /// The sockets the shepherd bound for this sheep's `listen` table, passed
    /// to the child from fd 3 upward in name order. `None`, and an empty set,
    /// give the child neither the descriptors nor any `LISTEN_*` variable.
    ///
    /// Shared by every instance of the sheep and held by the supervisor, so
    /// the sockets outlive any one child. Unix only: `normalize` refuses
    /// `listen` on Windows.
    pub listen: Option<std::sync::Arc<crate::listen::ListenSockets>>,
}

/// Redacted: `env` and `args` both carry whatever the operator configured,
/// a resolved `{{secret:...}}` included, and this type is the one handed to
/// `Command::envs` at exec.
impl fmt::Debug for SpawnSpec {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("SpawnSpec")
            .field("name", &self.name)
            .field("program", &self.program)
            .field("args", &format_args!("<{} args>", self.args.len()))
            .field("cwd", &self.cwd)
            .field("env", &format_args!("<{} vars>", self.env.len()))
            .finish_non_exhaustive()
    }
}

/// Error type returned from spawn and process control
///
/// `#[non_exhaustive]`: a future process-control primitive, a cgroup freeze
/// or a Windows job-object failure, would need its own variant rather than
/// stretching one of these, and an out-of-tree matcher should not break.
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RunnerError {
    /// The OS refused the spawn (exec failure, permissions, missing binary)
    SpawnFailed(String),
    /// Signal delivery failed (already reaped, `EPERM`)
    SignalFailed(String),
    /// A write to a child's stdin failed (carries the OS message, or the
    /// shepherd's own bound when the app was not reading).
    WriteFailed(String),
    /// A sheep inherited across a handover could not be taken back under
    /// supervision: this runner does not adopt at all, or the carried
    /// handles could not be wired to a pump.
    AdoptFailed(String),
}

impl fmt::Display for RunnerError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::SpawnFailed(msg) => write!(f, "process spawn failed: {msg}"),
            Self::SignalFailed(msg) => write!(f, "signal delivery failed: {msg}"),
            Self::WriteFailed(msg) => write!(f, "stdin write failed: {msg}"),
            Self::AdoptFailed(msg) => write!(f, "process adoption failed: {msg}"),
        }
    }
}

impl core::error::Error for RunnerError {}

/// A live child.
pub trait RunningProcess: Send + 'static {
    /// The OS process id
    fn pid(&self) -> u32;

    /// Resolves exactly once with the exit outcome
    ///
    /// # Cancellation safety
    ///
    /// Dropping the returned future and calling `wait` again neither
    /// restarts the wait nor loses progress toward it, as
    /// [`tokio::process::Child::wait`] guarantees.
    fn wait(&mut self) -> impl core::future::Future<Output = ExitOutcome> + Send;

    /// Sends a signal to the sheep's whole process group
    ///
    /// Group-wide: a `thing & wait` wrapper's forked child stays in its own
    /// group, and a leader-only signal would leave it running and untracked.
    /// Implementors must spawn each child as the leader of a fresh group; this
    /// and [`Self::kill_tree`] address it by [`Self::pid`], so a `setsid`
    /// escapee is beyond both. A sandboxed sheep is the exception: see
    /// [`SpawnSpec::leader_only_stop`].
    ///
    /// # Errors
    ///
    /// - [`RunnerError::SignalFailed`] if delivery failed (already reaped,
    ///   `EPERM`).
    fn signal(&mut self, sig: StopSignal) -> Result<(), RunnerError>;

    /// Sends `sig` to this sheep's own process, never its process group.
    ///
    /// Not group-wide, unlike [`Self::signal`]: this exists for a
    /// conversation between an operator and one application, and a `SIGHUP`
    /// broadcast to the group would reach whatever `sh` and runtime children
    /// are in it. The default refuses rather than widening to the group.
    ///
    /// # Errors
    ///
    /// - [`RunnerError::SignalFailed`] if delivery failed (`ESRCH`, `EPERM`)
    ///   or this implementation has no per-process delivery at all.
    fn signal_process(&mut self, sig: OperatorSignal) -> Result<(), RunnerError> {
        let _ = sig;
        Err(RunnerError::SignalFailed(
            "this runner cannot signal a single process".to_string(),
        ))
    }

    /// SIGKILLs the whole process group/tree
    ///
    /// The escalation rung above [`Self::signal`]: same group, same
    /// process-group assumption, but a signal nothing can catch or ignore.
    ///
    /// # Errors
    ///
    /// - [`RunnerError::SignalFailed`] if delivery failed (already reaped,
    ///   `EPERM`).
    fn kill_tree(&mut self) -> Result<(), RunnerError>;
}

/// Spawn seam between engine and OS
pub trait ProcessRunner: Send + Sync + 'static {
    /// The live-child type this runner produces
    type Proc: RunningProcess;

    /// Spawns per the spec, returning the proc + its IO bundle
    ///
    /// Must be called from within a Tokio runtime context: both
    /// implementations spawn background tasks internally to pump IO.
    ///
    /// # Errors
    ///
    /// - [`RunnerError::SpawnFailed`] on an exec failure, bad permissions or
    ///   a missing binary.
    fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError>;

    /// What is knowable about `spec` before anything is spawned
    ///
    /// Lets a caller refuse a whole batch before registering any of it,
    /// rather than registering the apps ahead of the one that cannot spawn.
    /// See [`Preflight`] for which verdicts a caller may refuse a batch over.
    /// The default answers [`Preflight::Unknown`], which is also the honest
    /// answer for a runner that never touches the filesystem.
    #[must_use]
    fn preflight(&self, spec: &SpawnSpec) -> Preflight {
        let _ = spec;
        Preflight::Unknown
    }

    /// Takes the stream of dog connections this runner opens at spawn
    ///
    /// One [`DogLink`] arrives per spawn whose [`SpawnSpec::dog_link`] was
    /// set, once the child has a pid. The first call returns the receiver and
    /// every later one `None`, so a runner has one server. The default is
    /// `None`: a runner that opens no connections has nothing to serve.
    fn dog_links(&self) -> Option<mpsc::UnboundedReceiver<DogLink>> {
        None
    }

    /// Rebuilds a proc around a sheep this process inherited, not started
    ///
    /// The successor half of a handover, unix only as the handover is. Every
    /// handle in `spec` crossed an `execve` with its `FD_CLOEXEC` cleared, so
    /// the sheep never noticed: same pid, same pipes, same open file
    /// description on each log. Nothing here spawns or signals.
    ///
    /// # Errors
    ///
    /// - [`RunnerError::AdoptFailed`] if this runner cannot take a process it
    ///   did not spawn (what the default answers), or if the carried handles
    ///   could not be wired to a pump.
    #[cfg(unix)]
    fn adopt(&self, spec: AdoptSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
        let _ = spec;
        Err(RunnerError::AdoptFailed(
            "this runner cannot adopt a process it did not spawn".to_string(),
        ))
    }
}

// Unix-gated because the module this case was split out of is; nothing it
// touches needs the gate.
#[cfg(all(test, unix))]
mod tests {

    use super::*;

    /// This type sits on the exec boundary, so a `tracing` call that
    /// formatted it would put every configured secret in the daemon's log.
    /// `args` counts as much as `env` now that a `{{secret:...}}` resolves
    /// into it: `--token=<value>` on an argv is a value the same way
    /// `TOKEN=<value>` on an environment is. Exact string pinned so a
    /// `derive(Debug)` refactor fails here (IR-41).
    #[test]
    fn debug_redacts_env_values_and_args() {
        let mut spec = SpawnSpec {
            name: "web".to_string(),
            program: "./srv".to_string(),
            args: vec!["--token=sk-live-abc".to_string(), "--port=8080".to_string()],
            ..Default::default()
        };
        spec.env.insert(
            "DATABASE_URL".to_string(),
            "postgres://user:hunter2@db".to_string(),
        );
        spec.env
            .insert("API_KEY".to_string(), "sk-live-abc".to_string());

        assert_eq!(
            format!("{spec:?}"),
            "SpawnSpec { name: \"web\", program: \"./srv\", args: <2 args>, cwd: None, \
                 env: <2 vars>, .. }"
        );
    }
}