shep-daemon 0.25.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
//! Dogs across a handover: which ones the blob carries, and what a successor
//! does with the ones its own `shep.toml` does not enable.
//!
//! The predecessor never reads `shep.toml`, so a dog's mark is its absence
//! from the enabled list the caller handed it. That list is the successor's to
//! read, which is why a parked dog is left behind rather than carried.

use super::super::adopt_fixtures::{
    AdoptingRunner, adoptable_child, carried, counters, daemon_fds, flock_until, without_handles,
};
use super::*;
use std::collections::BTreeMap;

#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_successor_reports_a_carried_dogs_spawn_refusal() {
    tokio::time::timeout(Duration::from_secs(5), async {
        let dir = tempfile::tempdir().unwrap();
        let (events, _rx) = crate::bus::test_bus(64);
        let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
            .spawn_adopted(
                vec![without_handles(carried("kelpie", 7, None, |entry| {
                    entry.dog = Some(DogSource::BuiltIn {
                        grants: None,
                        events: Vec::new(),
                    });
                    entry.status = ProcStatus::Errored;
                    entry.spawn_failure = Some("kelpie: spawn refused".to_string());
                }))],
                counters(8),
                Vec::new(),
            )
            .unwrap();

        let rows = sup.list().await;
        assert_eq!(rows[0].status, ProcStatus::Errored);
        assert_eq!(
            rows[0].spawn_failure.as_deref(),
            Some("kelpie: spawn refused")
        );
        sup.shutdown().await;
    })
    .await
    .expect("a successor must report a carried spawn refusal");
}

/// One dog as the boot hands it over: built-in, and named as `shep.toml`
/// would name it.
fn dog(name: &str) -> crate::dogs::DogSpec {
    crate::dogs::DogSpec {
        name: name.to_string(),
        source: DogSource::BuiltIn {
            grants: None,
            events: Vec::new(),
        },
        env: BTreeMap::new(),
    }
}

/// The app [`dog`] names, assembled the way the boot assembles it.
fn dog_app(name: &str, paths: &shep_core::paths::ShepPaths) -> shep_core::config::ResolvedApp {
    crate::dogs::dog_app(&dog(name), paths).expect("a built-in dog assembles")
}

fn names(wanted: &[&str]) -> BTreeSet<String> {
    wanted.iter().map(|name| (*name).to_string()).collect()
}

/// The blob describes the flock this daemon supervises, minus the dogs its
/// caller's config leaves out. A dog an operator stopped with `shep stop`
/// carries, since the config still names it: only a mark with no process on it
/// leaves a dog behind.
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_snapshot_carries_every_dog_that_is_not_parked_and_leaves_one_that_is() {
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    let (events, _rx) = crate::bus::test_bus(64);
    let (fds, _held) = daemon_fds(&dir);
    let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 3]);
    let sup = spawn_supervisor(runner, paths.clone(), events);
    sup.start(vec![normalize(AppConfig::minimal("api", "./srv")).unwrap()])
        .await
        .unwrap();
    // Running, and enabled: the ordinary carried dog.
    sup.start_dog(
        dog_app("metrics", &paths),
        DogSource::BuiltIn {
            grants: None,
            events: Vec::new(),
        },
    )
    .await
    .unwrap();
    // `Stopped` with no mark, which is the shape `shep stop` leaves.
    sup.register_dogs_at_rest(
        vec![(
            dog_app("bark", &paths),
            DogSource::BuiltIn {
                grants: None,
                events: Vec::new(),
            },
        )],
        BTreeSet::new(),
    )
    .await
    .unwrap();
    // Disabled: absent from the enabled list, which is a dog's whole mark.
    sup.register_dogs_at_rest(
        vec![(
            dog_app("log-rotate", &paths),
            DogSource::Adopted {
                grants: None,
                path: "/opt/bin/shep-log-rotate".to_string(),
                channel: false,
                events: Vec::new(),
            },
        )],
        names(&["log-rotate"]),
    )
    .await
    .unwrap();

    let (_candidates, blob, _parked) = sup.handover_snapshot(fds).await.unwrap();

    let carried: Vec<(&str, Option<DogSource>)> = blob
        .sheep()
        .iter()
        .map(|sheep| (sheep.name(), sheep.dog().cloned()))
        .collect();
    assert_eq!(
        carried,
        [
            ("api", None),
            (
                "metrics",
                Some(DogSource::BuiltIn {
                    grants: None,
                    events: Vec::new()
                })
            ),
            (
                "bark",
                Some(DogSource::BuiltIn {
                    grants: None,
                    events: Vec::new()
                })
            ),
        ],
        "a parked dog belongs to the successor's file, not to this flock's blob"
    );
}

/// The shape `shep disable` holds from the moment it sets the mark until its
/// stop finishes: marked, and still holding a process. It carries, because the
/// snapshot is the only way a successor learns a pid.
///
/// The mark arrives through [`SupervisorHandle::register_at_rest`], the actor's
/// door for a mark with no stop attached, and gets there without a kill ladder
/// a paused clock has to hold mid-flight.
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_snapshot_carries_a_disabled_dog_that_still_holds_a_process() {
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    let (events, _rx) = crate::bus::test_bus(64);
    let (fds, _held) = daemon_fds(&dir);
    let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
    let sup = spawn_supervisor(runner, paths.clone(), events);
    sup.start(vec![normalize(AppConfig::minimal("api", "./srv")).unwrap()])
        .await
        .unwrap();
    sup.start_dog(
        dog_app("metrics", &paths),
        DogSource::BuiltIn {
            grants: None,
            events: Vec::new(),
        },
    )
    .await
    .unwrap();
    sup.register_at_rest(vec![dog_app("metrics", &paths)], names(&["metrics"]))
        .await
        .unwrap();

    let flock = sup.list().await;
    let metrics = flock.iter().find(|row| row.name == "metrics").unwrap();
    assert_eq!(metrics.status, ProcStatus::Online, "{metrics:?}");
    assert_eq!(metrics.disabled, Some(true), "{metrics:?}");

    let (_candidates, blob, _parked) = sup.handover_snapshot(fds).await.unwrap();
    let carried: Vec<(&str, Option<DogSource>)> = blob
        .sheep()
        .iter()
        .map(|sheep| (sheep.name(), sheep.dog().cloned()))
        .collect();
    assert_eq!(
        carried,
        [
            ("api", None),
            (
                "metrics",
                Some(DogSource::BuiltIn {
                    grants: None,
                    events: Vec::new()
                })
            )
        ],
        "a marked dog still holding a process must cross the exec described"
    );
}

/// The successor's half. `metrics` crossed the `execve` running, because the
/// predecessor booted while the file named it, and the file this successor
/// boots from no longer does: it goes down through the same door `shep disable`
/// uses, so it ends stopped, marked, and with `shep enable <name>` as the way
/// back. `bark` was never carried, and lands at rest under the id this
/// successor issues.
#[cfg(unix)]
#[tokio::test]
async fn a_successor_stops_a_carried_dog_its_config_no_longer_enables() {
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    let (events, _rx) = crate::bus::test_bus(64);
    let pid = adoptable_child("sleep 30");
    let sup = SupervisorBuilder::new(TokioRunner::new(), paths.clone(), events)
        .spawn_adopted(
            vec![
                without_handles(carried("api", 0, None, |entry| {
                    entry.status = ProcStatus::Stopped;
                })),
                without_handles(carried("metrics", 1, Some(pid), |entry| {
                    entry.dog = Some(DogSource::BuiltIn {
                        grants: None,
                        events: Vec::new(),
                    });
                })),
            ],
            counters(2),
            Vec::new(),
        )
        .expect("a carried flock installs");

    crate::dogs::register_dogs_at_rest(&[dog("metrics"), dog("bark")], &[], &paths, &sup).await;

    let rows = flock_until(
        &sup,
        |rows| {
            rows.iter()
                .any(|row| row.name == "metrics" && row.status == ProcStatus::Stopped)
        },
        "a dog the successor's config does not enable must stop, not just be listed",
    )
    .await;

    let metrics = rows.iter().find(|row| row.name == "metrics").unwrap();
    assert_eq!(metrics.id, 1, "the carried row is the successor's to stop");
    assert_eq!(metrics.pid, None, "{metrics:?}");
    assert_eq!(metrics.disabled, Some(true), "{metrics:?}");
    assert_eq!(
        metrics.last_exit,
        Some(ExitInfo {
            code: None,
            signal: Some(15),
        }),
        "the process that crossed the exec must have been signalled, not dropped from the flock"
    );

    let bark = rows.iter().find(|row| row.name == "bark").unwrap();
    assert_eq!(bark.status, ProcStatus::Stopped, "{bark:?}");
    assert_eq!(bark.disabled, Some(true), "{bark:?}");
    assert_eq!(
        bark.id, 2,
        "a dog left behind gets the id this successor issues"
    );
    assert_eq!(
        rows.iter().find(|row| row.name == "api").unwrap().id,
        0,
        "the flock the successor inherited keeps its rows"
    );
}

/// The other half of the same file read: a dog the successor's config still
/// enables keeps its carried row, its pid and its process. Stopping it here
/// would turn every reload into a restart of the monitoring.
#[cfg(unix)]
#[tokio::test]
async fn a_successor_leaves_a_carried_dog_its_config_still_enables_running() {
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    let (events, _rx) = crate::bus::test_bus(64);
    let pid = adoptable_child("sleep 30");
    let sup = SupervisorBuilder::new(TokioRunner::new(), paths.clone(), events)
        .spawn_adopted(
            vec![without_handles(carried("metrics", 1, Some(pid), |entry| {
                entry.dog = Some(DogSource::BuiltIn {
                    grants: None,
                    events: Vec::new(),
                });
            }))],
            counters(2),
            Vec::new(),
        )
        .expect("a carried flock installs");

    crate::dogs::register_dogs_at_rest(&[dog("metrics")], &[dog("metrics")], &paths, &sup).await;

    let rows = sup.list().await;
    assert_eq!(rows.len(), 1, "{rows:?}");
    let metrics = &rows[0];
    assert_eq!(metrics.status, ProcStatus::Online, "{metrics:?}");
    assert_eq!(
        metrics.pid,
        Some(pid),
        "a dog the config enables must keep the process it carried"
    );
    assert_eq!(metrics.disabled, None, "{metrics:?}");
}

/// The other way a name leaves the file: the dog was already down, by `shep
/// stop` rather than `shep disable`, so it carried with its row and no mark.
/// The successor's config does not enable it either, so the same call marks
/// it, and the row stops reading as one `shep restart` could start.
#[cfg(unix)]
#[tokio::test]
async fn a_successor_marks_a_carried_dog_it_stopped_and_the_config_dropped() {
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    let (events, _rx) = crate::bus::test_bus(64);
    let sup = SupervisorBuilder::new(TokioRunner::new(), paths.clone(), events)
        .spawn_adopted(
            vec![without_handles(carried("metrics", 1, None, |entry| {
                entry.dog = Some(DogSource::BuiltIn {
                    grants: None,
                    events: Vec::new(),
                });
                entry.status = ProcStatus::Stopped;
            }))],
            counters(2),
            Vec::new(),
        )
        .expect("a carried flock installs");

    crate::dogs::register_dogs_at_rest(&[dog("metrics")], &[], &paths, &sup).await;

    let rows = sup.list().await;
    assert_eq!(rows.len(), 1, "{rows:?}");
    let metrics = &rows[0];
    assert_eq!(
        metrics.id, 1,
        "a stopped row the predecessor left is the successor's to mark, not to reissue"
    );
    assert_eq!(metrics.status, ProcStatus::Stopped, "{metrics:?}");
    assert_eq!(metrics.disabled, Some(true), "{metrics:?}");
}

/// The other way a carried dog escapes what this boot registered: its own
/// config would not assemble, so it never reached the actor at all, while the
/// process it carried is in the flock regardless. `stop_unenabled_dogs` reads
/// the flock, so that dog goes down too.
///
/// The assembly fails on `adopted_dogs = { metrics = "" }`: an adopted source
/// with no binary, which `normalize` refuses as `MissingScript`.
#[cfg(unix)]
#[tokio::test]
async fn a_successor_stops_a_carried_dog_whole_config_could_not_assemble() {
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    let (events, _rx) = crate::bus::test_bus(64);
    let pid = adoptable_child("sleep 30");
    let sup = SupervisorBuilder::new(TokioRunner::new(), paths.clone(), events)
        .spawn_adopted(
            vec![without_handles(carried("metrics", 1, Some(pid), |entry| {
                entry.dog = Some(DogSource::BuiltIn {
                    grants: None,
                    events: Vec::new(),
                });
            }))],
            counters(2),
            Vec::new(),
        )
        .expect("a carried flock installs");

    let unassemblable = crate::dogs::DogSpec {
        name: "metrics".to_string(),
        source: DogSource::Adopted {
            grants: None,
            path: String::new(),
            channel: false,
            events: Vec::new(),
        },
        env: BTreeMap::new(),
    };
    crate::dogs::register_dogs_at_rest(&[unassemblable], &[], &paths, &sup).await;

    let rows = flock_until(
        &sup,
        |rows| {
            rows.iter()
                .any(|row| row.name == "metrics" && row.status == ProcStatus::Stopped)
        },
        "a carried dog this boot could not assemble must still stop",
    )
    .await;

    let metrics = rows.iter().find(|row| row.name == "metrics").unwrap();
    assert_eq!(metrics.id, 1, "the carried row is the successor's to stop");
    assert_eq!(metrics.pid, None, "{metrics:?}");
    assert_eq!(metrics.disabled, Some(true), "{metrics:?}");
    assert_eq!(
        metrics.last_exit,
        Some(ExitInfo {
            code: None,
            signal: Some(15),
        }),
        "the dog must have gone down the ordinary ladder"
    );
    assert!(
        nix::sys::signal::kill(
            nix::unistd::Pid::from_raw(i32::try_from(pid).unwrap()),
            None
        )
        .is_err(),
        "the process must have been reaped, not merely marked"
    );
}