shep-daemon 0.8.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
use super::super::{Handover, SheepFd, fds};
use super::adopt_descriptors::{
    adopt_channel, adopt_fd, adopt_listener, adopt_log, adopt_pipe, adopt_stdin,
};
use crate::sys;
use std::io;
use std::os::fd::RawFd;

/// Refuses a blob naming one descriptor number more than once.
///
/// Before the first adoption, never during. `sys::adopt_handover_fd`'s
/// safety argument rests on each adoption being its number's only owner: a
/// second owner closes it again on drop, reaching whatever this process
/// opened in between. A blob the daemon wrote cannot repeat a number; this
/// covers one that was edited, or left by a handover that never completed.
///
/// # Errors
///
/// Names the repeated number, which is all anything here can know.
pub(super) fn refuse_repeated_fds(blob: &Handover) -> io::Result<()> {
    let mut seen = std::collections::BTreeSet::new();
    for fd in blob.named_fds() {
        if !seen.insert(fd) {
            return Err(io::Error::new(
                io::ErrorKind::InvalidInput,
                format!(
                    "the handover blob names descriptor {fd} more than once, so adopting it \
                     would build two owners of one number"
                ),
            ));
        }
    }
    Ok(())
}

/// Run everything [`adopt`](crate::handover::adopt::adopt) will run, in the predecessor, while there is
/// still an image to refuse back to. No check is re-stated: each number
/// reaches the successor's own adoption, on a duplicate, from a reparsed blob.
///
/// # Errors
///
/// The blob does not parse as a successor would, or names a number that is
/// repeated, reserved, closed, or the wrong kind for its slot.
///
/// # Panics
///
/// Panics if called outside a tokio runtime with IO enabled.
#[track_caller]
pub fn dry_run(blob: &Handover) -> io::Result<()> {
    let value = serde_json::to_value(blob).map_err(io::Error::other)?;
    let blob = Handover::load_value(value).map_err(|error| {
        io::Error::new(
            io::ErrorKind::InvalidData,
            format!("a successor could not have read this blob back: {error}"),
        )
    })?;

    // `adopt`'s own order, so a rehearsal that stops early stops where the
    // successor would.
    refuse_repeated_fds(&blob)?;
    rehearse(blob.listener_fd, "the control listener", adopt_listener)?;
    for carried in &blob.sheep {
        let name = &carried.name;
        for (fd, slot) in carried.fds.all_kinded() {
            let Some(fd) = fd else { continue };
            rehearse(
                fd,
                &format!("sheep '{name}' {}", slot.describe()),
                |dup| match slot {
                    SheepFd::OutPipe => adopt_pipe(Some(dup), name, "stdout").map(drop),
                    SheepFd::ErrPipe => adopt_pipe(Some(dup), name, "stderr").map(drop),
                    SheepFd::OutLog => adopt_log(Some(dup), name, "stdout").map(drop),
                    SheepFd::ErrLog => adopt_log(Some(dup), name, "stderr").map(drop),
                    SheepFd::Stdin => adopt_stdin(Some(dup), name).map(drop),
                    SheepFd::Channel => adopt_channel(Some(dup), name).map(drop),
                },
            )?;
        }
    }
    rehearse(blob.pidfile_fd, "the pidfile lock", |dup| {
        adopt_fd(dup, "the pidfile lock").map(drop)
    })?;
    Ok(())
}

/// Hand `adopt_one` a duplicate of `fd`, so an adoption that takes ownership
/// can be run against a descriptor this process must keep.
///
/// # Errors
///
/// `fd` is reserved or not open, it could not be duplicated, or the adoption
/// refused the duplicate.
fn rehearse<T>(
    fd: RawFd,
    what: &str,
    adopt_one: impl FnOnce(RawFd) -> io::Result<T>,
) -> io::Result<()> {
    // The blob's number, never the duplicate's: a duplicate is always open
    // and above the floor, so a rehearsal that only saw duplicates would wave
    // through exactly the two blobs the successor is certain to refuse.
    // Labelled, because `sys::adoptable_fd` names only the number.
    sys::adoptable_fd(fd)
        .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, format!("{what}: {error}")))?;
    let duplicate = fds::duplicate_raw(fd)?;
    // `adopt_one` owns `duplicate` on both arms, so nothing leaks: the one
    // arm that returns without consuming is `sys::adopt_handover_fd`
    // refusing, which the check above rules out for a number just created
    // above the floor.
    adopt_one(duplicate).map(drop)
}

/// Remove the blob at `path`, now that its descriptors are adopted.
///
/// Called only after [`adopt`](crate::handover::adopt::adopt) has succeeded: a blob left after a refusal is
/// evidence an operator can read, while one left after a success would be
/// adopted again by the next boot. A failure to remove it is logged rather
/// than returned, the flock being rehydrated already.
pub fn discard_blob(path: &std::path::Path) {
    if let Err(error) = std::fs::remove_file(path) {
        tracing::warn!(
            path = %path.display(),
            %error,
            "the handover blob could not be removed after it was adopted"
        );
    }
}

#[cfg(test)]
mod tests {
    use super::super::adopt_descriptors::adopt;

    use std::os::fd::RawFd;

    use super::super::super::{CarriedFds, SheepFd, fds};

    use std::os::fd::IntoRawFd as _;

    use crate::handover::VERSION;
    use tokio::io::AsyncWriteExt as _;

    use super::super::testing::*;
    use super::*;

    /// `merge_logs` points every instance at one path, and
    /// [`refuse_repeated_fds`] refuses a blob naming any number twice. Each
    /// instance's pump runs its own `open_append`, so one inode is reached
    /// through two descriptions with two numbers; the interleaved file proves
    /// they were independent rather than merely distinct.
    #[tokio::test]
    async fn two_instances_sharing_one_log_file_are_both_adopted() {
        let dir = tempfile::tempdir().unwrap();
        let socket = dir.path().join("shep.sock");
        // One path for both slots: `assemble` drops the `-<instance>`
        // suffix under `merge_logs`.
        let merged = dir.path().join("web-out.log");
        let zero = std::fs::OpenOptions::new()
            .create(true)
            .append(true)
            .open(&merged)
            .unwrap();
        let one = std::fs::OpenOptions::new()
            .create(true)
            .append(true)
            .open(&merged)
            .unwrap();
        let (zero_fd, one_fd) = (zero.into_raw_fd(), one.into_raw_fd());
        assert_ne!(
            zero_fd, one_fd,
            "two `open`s on one path must yield two numbers, or the premise \
                 of this whole case is wrong"
        );
        let blob = blob_with(
            &socket,
            vec![
                carried_slot(
                    0,
                    CarriedFds {
                        out_pipe: None,
                        err_pipe: None,
                        out_log: Some(zero_fd),
                        err_log: None,
                        stdin: None,
                        channel: None,
                    },
                ),
                carried_slot(
                    1,
                    CarriedFds {
                        out_pipe: None,
                        err_pipe: None,
                        out_log: Some(one_fd),
                        err_log: None,
                        stdin: None,
                        channel: None,
                    },
                ),
            ],
        );

        let mut adopted = adopt(&blob).expect("a merged-log clustered app must be adoptable");

        assert_eq!(adopted.sheep.len(), 2, "one adopted sheep per instance");
        let zero = adopted.sheep[0].out_log.take().expect("slot 0's log");
        let one = adopted.sheep[1].out_log.take().expect("slot 1's log");
        // Alternated, so a second handle that had become an alias of the
        // first shows up as lost text rather than two clean halves. Flushed
        // per line because a `tokio::fs::File` hands the real `write(2)` to
        // the blocking pool, which finishes in its own order.
        let mut handles = [zero, one];
        for (line, slot) in [
            ("zero-1\n", 0),
            ("one-1\n", 1),
            ("zero-2\n", 0),
            ("one-2\n", 1),
        ] {
            let handle = &mut handles[slot];
            handle.write_all(line.as_bytes()).await.unwrap();
            handle.flush().await.unwrap();
        }
        assert_eq!(
            std::fs::read_to_string(&merged).unwrap(),
            "zero-1\none-1\nzero-2\none-2\n",
            "both instances append into the one file, in the order written"
        );
    }

    /// Without this the suite would pass with a `dry_run` that refused
    /// everything, turning every handover into a stop-and-start.
    #[tokio::test]
    async fn a_blob_a_successor_could_adopt_passes_the_rehearsal() {
        let predecessor = Predecessor::new();

        dry_run(&predecessor.blob()).expect("every descriptor here is the kind its slot wants");
    }

    /// Nothing about a carried reload is a descriptor, so the rehearsal
    /// covers a swap in flight through the parse alone,
    /// [`Handover::load_value`](super::super::Handover::load_value) running before any descriptor is touched.
    #[tokio::test]
    async fn a_blob_carrying_a_swap_in_flight_passes_the_rehearsal() {
        use crate::entry::ReloadState;
        use crate::supervisor::{CarriedReload, ReloadMode, ReloadPhase, ReloadSwap};

        let predecessor = Predecessor::new();
        let mut blob = predecessor.blob();
        blob.sheep[0].reload = Some(ReloadState::Drainee { new_id: Some(9) });
        blob.reloads = Some(vec![CarriedReload {
            app: "web".to_owned(),
            queue: vec![4, 5],
            mode: ReloadMode::Overlap,
            swap: ReloadSwap {
                old_id: 1,
                new_id: Some(9),
                phase: ReloadPhase::DrainOld,
            },
        }]);

        dry_run(&blob).expect("a flock mid-reload is one a successor can adopt");
    }

    /// A field added to [`CarriedSheep`](super::super::CarriedSheep) rides the reparse the rehearsal
    /// already runs, and one it could not parse would be found after the
    /// predecessor was gone.
    #[tokio::test]
    async fn a_blob_carrying_a_failed_readiness_verdict_passes_the_rehearsal() {
        let predecessor = Predecessor::new();
        let mut blob = predecessor.blob();
        blob.sheep[0].ready_failed = Some(true);

        dry_run(&blob).expect("an instance a reload gave up on is one a successor can adopt");
    }

    /// The predecessor is still supervising this flock, so everything it
    /// holds has to work afterwards. Each of the four takes ownership a
    /// different way and would close the fixture's handle without the
    /// duplicate.
    ///
    /// The connection is queued before the rehearsal, since `O_NONBLOCK`
    /// reaches the original through the duplicate and this fixture's listener
    /// is a plain `std` one that really does change. Queueing first leaves
    /// the accept below something waiting either way.
    #[tokio::test]
    async fn a_rehearsal_leaves_every_descriptor_it_checked_working() {
        use std::io::{Read as _, Write as _};

        let mut predecessor = Predecessor::new();
        predecessor.out_write.write_all(b"a bleat").unwrap();
        let socket = predecessor.socket();
        let connecting = tokio::task::spawn_blocking(move || {
            std::os::unix::net::UnixStream::connect(&socket).unwrap()
        });
        let client = connecting.await.unwrap();

        dry_run(&predecessor.blob()).expect("the fixture is adoptable");

        // The listener still listens.
        predecessor
            .listener
            .accept()
            .expect("the checked listener still accepts");
        drop(client);

        // The stdout pipe still carries what was written before the check.
        let mut buf = [0_u8; 7];
        predecessor
            .out_read
            .read_exact(&mut buf)
            .expect("the checked read end still reads");
        assert_eq!(&buf, b"a bleat");

        // The stdin pipe still carries a line the other way.
        predecessor.stdin_write.write_all(b"whisper").unwrap();
        let mut buf = [0_u8; 7];
        predecessor
            .stdin_read
            .read_exact(&mut buf)
            .expect("the checked write end still writes");
        assert_eq!(&buf, b"whisper");

        // The shepherd channel still has its child on the far end.
        predecessor.channel.write_all(b"ping").unwrap();
        let mut buf = [0_u8; 4];
        predecessor
            .child_channel
            .read_exact(&mut buf)
            .expect("the checked channel still reaches the child");
        assert_eq!(&buf, b"ping");
    }

    /// A duplicate taken and never handed to an adoption leaks one
    /// descriptor per named number, on a path that runs on every reload.
    ///
    /// Counted over a hundred passes, because the count is the whole
    /// process's and other cases run in other threads. Six hundred leaked
    /// descriptors sit well clear of a few dozen of concurrent noise.
    #[tokio::test]
    async fn a_rehearsal_leaks_no_descriptors() {
        let predecessor = Predecessor::new();
        let blob = predecessor.blob();
        let before = open_fd_count();

        for _ in 0..100 {
            dry_run(&blob).expect("the fixture is adoptable");
        }

        let after = open_fd_count();
        assert!(
            after < before + 100,
            "a hundred rehearsals of a blob naming six descriptors must not grow this \
                 process's descriptor table: {before} -> {after}"
        );
    }

    /// A closed number is already refused before the exec, since clearing
    /// `FD_CLOEXEC` on it meets `EBADF`. An open one of the wrong kind sails
    /// through that and would reach a successor with no predecessor left.
    #[tokio::test]
    async fn a_descriptor_that_is_open_but_not_a_pipe_is_refused_before_the_exec() {
        use std::os::fd::AsRawFd as _;

        let predecessor = Predecessor::new();
        let not_a_pipe = std::fs::File::open("/dev/null").unwrap();
        let mut blob = predecessor.blob();
        blob.sheep[0].fds.out_pipe = Some(not_a_pipe.as_raw_fd());

        // The premise: this number is open, so nothing before the exec
        // would have stopped it.
        crate::sys::adoptable_fd(not_a_pipe.as_raw_fd())
            .expect("the number must be open, or this proves nothing");
        fds::keep_raw_across_exec(not_a_pipe.as_raw_fd())
            .expect("the `FD_CLOEXEC` sweep must not refuse it either");

        let err = dry_run(&blob).expect_err("/dev/null is not a readable pipe");

        assert!(
            err.to_string().contains("web") && err.to_string().contains("stdout"),
            "the refusal must name the sheep and the stream: {err}"
        );
    }

    /// The rehearsal and the adoption must agree, slot by slot.
    ///
    /// A rehearsal that passes a blob the successor refuses still reaches
    /// the `execve`. Every slot is walked because four mechanisms refuse
    /// them: a readable-pipe check, a writable-pipe check, a `getpeername`,
    /// and for the two log slots no kind check at all.
    #[tokio::test]
    async fn the_rehearsal_and_the_adoption_agree_on_every_slot() {
        use std::os::fd::AsRawFd as _;

        for slot in [
            SheepFd::OutPipe,
            SheepFd::ErrPipe,
            SheepFd::OutLog,
            SheepFd::ErrLog,
            SheepFd::Stdin,
            SheepFd::Channel,
        ] {
            let predecessor = Predecessor::new();
            let wrong = std::fs::File::open("/dev/null").unwrap();
            let wrong = Some(wrong.as_raw_fd());
            let mut blob = predecessor.blob();
            let fds = &mut blob.sheep[0].fds;
            match slot {
                SheepFd::OutPipe => fds.out_pipe = wrong,
                SheepFd::ErrPipe => fds.err_pipe = wrong,
                SheepFd::OutLog => fds.out_log = wrong,
                SheepFd::ErrLog => fds.err_log = wrong,
                SheepFd::Stdin => fds.stdin = wrong,
                SheepFd::Channel => fds.channel = wrong,
            }

            assert_eq!(
                dry_run(&blob).is_err(),
                adopt_a_copy(&blob).is_err(),
                "the rehearsal and the adoption disagree about {slot:?}, so one of them is \
                     checking something the other is not"
            );
        }
    }

    /// A repeated number is refused by the same function the successor
    /// refuses it with, rather than by a second copy of the rule.
    ///
    /// The sweep before the exec cannot catch it: clearing `FD_CLOEXEC`
    /// twice on one number succeeds, so a repeat reaches the successor
    /// untouched.
    #[tokio::test]
    async fn a_blob_naming_one_descriptor_twice_is_refused_before_the_exec() {
        let predecessor = Predecessor::new();
        let mut blob = predecessor.blob();
        blob.sheep[0].fds.err_log = Some(blob.pidfile_fd);

        let err = dry_run(&blob).expect_err("one number cannot have two owners");

        assert!(
            err.to_string().contains("more than once"),
            "the refusal must be `refuse_repeated_fds`'s own: {err}"
        );
    }

    /// A number below the stdio floor is refused here, not after the exec.
    ///
    /// The check runs against the blob's number, not the duplicate's: a
    /// duplicate is always open and above the floor, so a rehearsal that only
    /// inspected duplicates would wave through exactly the two blobs
    /// `sys::adopt_handover_fd` is certain to refuse.
    #[tokio::test]
    async fn a_reserved_or_closed_number_is_refused_before_the_exec() {
        let predecessor = Predecessor::new();

        let mut reserved = predecessor.blob();
        reserved.sheep[0].fds.out_log = Some(0);
        let err = dry_run(&reserved).expect_err("stdio is owned elsewhere");
        assert!(
            err.to_string().contains("reserved for stdio"),
            "the refusal must be the successor's own wording: {err}"
        );

        // `RawFd::MAX` rather than a number this case opened and closed: a
        // closed number can be handed straight back to another thread of this
        // same process. A number above any descriptor limit is `EBADF` with
        // nothing to race against.
        let mut gone = predecessor.blob();
        gone.sheep[0].fds.err_log = Some(RawFd::MAX);
        let err = dry_run(&gone).expect_err("a number this high names nothing");
        assert!(
            err.to_string().contains("not an open descriptor"),
            "the refusal must be the successor's own wording: {err}"
        );
    }

    /// The successor reads the blob back off disk rather than being handed
    /// this struct, so the parse is one of its checks too.
    ///
    /// A successor is a different build, so one that has moved `VERSION`
    /// refuses the blob at `load_value`, past the exec and with the
    /// predecessor gone.
    #[tokio::test]
    async fn a_blob_a_successor_could_not_read_back_is_refused_before_the_exec() {
        let predecessor = Predecessor::new();
        let mut blob = predecessor.blob();
        blob.version = VERSION + 1;

        let err = dry_run(&blob).expect_err("a version this image cannot read");

        assert!(
            err.to_string()
                .contains("could not have read this blob back"),
            "the refusal must say the parse failed, not the descriptors: {err}"
        );
    }
}