shep-daemon 0.15.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
//! Tests for triggering an action and matching its reply.
//!
//! An app answers on its own schedule, so a reply has to find the wait that
//! asked for it. These cover stamped and unstamped replies, timeouts, debts
//! owed by an action that already gave up, and a second reply to an answered
//! one.

use super::*;

/// Fails if a reply stops reaching the wait that asked for it: the whole
/// path, from `SupervisorHandle::trigger` through the actor, the writer's
/// end of the channel, `run_sheep`'s relay of a `ChildMessage` and back.
/// `params` are asserted on the wire; nothing in the daemon reads them.
#[tokio::test(start_paused = true)]
async fn a_triggered_action_answers_with_the_apps_reply() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("web", "./srv");
    app.channel = true;
    let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
    let mut io = runner.io_handles(0);

    // Spawned rather than awaited: the reply below is what ends this wait.
    let triggered = tokio::spawn(async move {
        handle
            .trigger(
                ProcessSelector::Name("web".to_string()),
                "gc".to_string(),
                Some("--full".to_string()),
            )
            .await
    });

    assert_eq!(
        sent_action(&mut io.to_child_rx).await,
        ShepherdMessage::Action {
            name: "gc".to_string(),
            params: Some("--full".to_string()),
            // `next_action_stamp` is read before it is incremented, so a
            // freshly-built actor's first dispatch is always stamp 0.
            id: 0,
        },
        "the action reaches the child's end of the channel as it was asked for"
    );

    io.from_child_tx
        .send(ChildMessage::ActionReply {
            action: "gc".to_string(),
            body: "swept 3".to_string(),
            // Echoing the dispatch's own stamp makes this a real round trip.
            id: Some(0),
        })
        .await
        .unwrap();

    assert_eq!(
        triggered.await.unwrap(),
        Ok(vec![ActionReply {
            id: 0,
            name: "web".to_string(),
            outcome: ActionOutcome::Replied {
                body: "swept 3".to_string()
            },
        }]),
        "the app's reply body is what the caller is answered with"
    );
}

/// fails if a `ready` on fd 3 reaches only the readiness machinery and
/// never the bus. Forwarding is a second thing the arm does.
#[tokio::test(start_paused = true)]
async fn a_ready_on_the_channel_reaches_both_the_bus_and_the_readiness_wait() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("web", "./srv");
    app.channel = true;
    app.wait_ready = true;
    // `started` subscribes the bus receiver before the start.
    let (handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
    let io = runner.io_handles(0);

    io.from_child_tx.send(ChildMessage::Ready).await.unwrap();

    // Bounded: a bus that never receives would park this case.
    let seen = tokio::time::timeout(ACTION_WINDOW, async {
        loop {
            if let BusEvent::Channel { id, message } = events.recv().await.unwrap().to_event() {
                break (id, message);
            }
        }
    })
    .await
    .expect("no channel event within the window");

    assert_eq!(seen, (0, ChildMessage::Ready));

    // The readiness half still works: the sheep goes Online off this message.
    let listed = handle.list().await;
    assert_eq!(listed[0].status, ProcStatus::Online);
}

/// fails if a metric is still only a `tracing::debug!`, which no
/// subscriber can read.
#[tokio::test(start_paused = true)]
async fn a_metric_on_the_channel_reaches_the_bus_with_its_name_and_value() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("web", "./srv");
    app.channel = true;
    let (_handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
    let io = runner.io_handles(0);

    io.from_child_tx
        .send(ChildMessage::Metric {
            name: "rps".to_string(),
            value: 42.0,
            metric_type: None,
            labels: std::collections::BTreeMap::new(),
            help: None,
        })
        .await
        .unwrap();

    let seen = tokio::time::timeout(ACTION_WINDOW, async {
        loop {
            if let BusEvent::Channel { id, message } = events.recv().await.unwrap().to_event() {
                break (id, message);
            }
        }
    })
    .await
    .expect("no channel event within the window");

    assert_eq!(
        seen,
        (
            0,
            ChildMessage::Metric {
                name: "rps".to_string(),
                value: 42.0,
                metric_type: None,
                labels: std::collections::BTreeMap::new(),
                help: None,
            }
        )
    );
}

/// fails if a note is acted on or dropped instead of reaching the bus.
#[tokio::test(start_paused = true)]
async fn a_note_on_the_channel_reaches_the_bus_as_it_was_sent() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("web", "./srv");
    app.channel = true;
    let (_handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
    let io = runner.io_handles(0);
    let note = ChildMessage::Note {
        name: "lease.web.want.1760500000".to_string(),
        value: 1.0,
    };

    io.from_child_tx.send(note.clone()).await.unwrap();

    let seen = tokio::time::timeout(ACTION_WINDOW, async {
        loop {
            let event = events.recv().await.unwrap().to_event();
            if let BusEvent::Channel { id, message } = &event {
                break (*id, message.clone(), event.topic().into_owned());
            }
        }
    })
    .await
    .expect("no channel event within the window");

    assert_eq!(seen, (0, note, "channel.note".to_string()));
}

/// fails if an `action-reply` nobody is waiting for is dropped before the
/// bus sees it. `handle_action_reply` finds no wait and discards it.
#[tokio::test(start_paused = true)]
async fn an_action_reply_no_trigger_is_waiting_for_still_reaches_the_bus() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("web", "./srv");
    app.channel = true;
    let (_handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
    let io = runner.io_handles(0);

    io.from_child_tx
        .send(ChildMessage::ActionReply {
            action: "gc".to_string(),
            body: "unprompted".to_string(),
            id: None,
        })
        .await
        .unwrap();

    let seen = tokio::time::timeout(ACTION_WINDOW, async {
        loop {
            if let BusEvent::Channel { message, .. } = events.recv().await.unwrap().to_event() {
                break message;
            }
        }
    })
    .await
    .expect("no channel event within the window");

    let ChildMessage::ActionReply { body, .. } = seen else {
        panic!("expected an action reply, got {seen:?}");
    };
    assert_eq!(body, "unprompted");
}

/// Fails if a wait for an app that never answers does not end on its own.
///
/// The action is read off the channel after the answer: a build that never
/// sent it would time out too, and look identical without that read.
#[tokio::test(start_paused = true)]
async fn a_triggered_action_times_out_when_the_app_never_answers() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("api", "./api");
    app.channel = true;
    let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
    let mut io = runner.io_handles(0);

    assert_eq!(
        handle
            .trigger(
                ProcessSelector::Name("api".to_string()),
                "stats".to_string(),
                None,
            )
            .await,
        Ok(vec![ActionReply {
            id: 0,
            name: "api".to_string(),
            outcome: ActionOutcome::TimedOut,
        }]),
        "an app that says nothing is reported as saying nothing, not waited on forever"
    );
    assert_eq!(
        sent_action(&mut io.to_child_rx).await,
        ShepherdMessage::Action {
            name: "stats".to_string(),
            params: None,
            // The actor's first and only dispatch carries stamp 0.
            id: 0,
        },
        "the timeout is an app that did not answer, not an action that was never sent"
    );
}

/// T1 times out and leaves a `gc` debt, T2 is triggered and is live, and
/// the app's next `gc` reply, carrying T2's stamp, must reach T2.
#[test]
fn a_stamped_reply_wakes_its_own_wait_even_with_a_debt_outstanding() {
    let mut waits = ActionWaits::default();

    // T1: armed, then resolved without its reply: the timeout path.
    let (t1_reply, _t1_out) = oneshot::channel();
    let (t1_waiter, _t1_body) = oneshot::channel();
    waits.arm(PendingAction {
        stamp: 1,
        action: "gc".to_string(),
        waiter: Some(t1_waiter),
        reply: t1_reply,
    });
    assert!(waits.resolve(1).is_some(), "T1 must have been live");

    // T2: armed and still live.
    let (t2_reply, _t2_out) = oneshot::channel();
    let (t2_waiter, t2_body) = oneshot::channel();
    waits.arm(PendingAction {
        stamp: 2,
        action: "gc".to_string(),
        waiter: Some(t2_waiter),
        reply: t2_reply,
    });

    let woken = waits
        .answer("gc", Some(2))
        .expect("a reply stamped with the live wait's own stamp must reach it");
    woken.send("collected".to_string()).unwrap();
    assert_eq!(t2_body.blocking_recv().unwrap(), "collected");
}

/// An app that does not echo the stamp must see the debt paid first and the
/// live wait left alone.
#[test]
fn an_unstamped_reply_still_settles_the_oldest_debt_first() {
    let mut waits = ActionWaits::default();

    let (t1_reply, _t1_out) = oneshot::channel();
    let (t1_waiter, _t1_body) = oneshot::channel();
    waits.arm(PendingAction {
        stamp: 1,
        action: "gc".to_string(),
        waiter: Some(t1_waiter),
        reply: t1_reply,
    });
    waits.resolve(1);

    let (t2_reply, _t2_out) = oneshot::channel();
    let (t2_waiter, _t2_body) = oneshot::channel();
    waits.arm(PendingAction {
        stamp: 2,
        action: "gc".to_string(),
        waiter: Some(t2_waiter),
        reply: t2_reply,
    });

    assert!(
        waits.answer("gc", None).is_none(),
        "an unstamped reply pays the debt, exactly as it did before stamping"
    );
    assert!(
        waits.answer("gc", None).is_some(),
        "and the next one reaches the live wait, exactly as it did before"
    );
}

/// The stamped path has to settle its own debt, not just skip the queue.
#[test]
fn a_stamped_reply_for_a_dead_wait_does_not_reach_a_live_one() {
    let mut waits = ActionWaits::default();

    let (t1_reply, _t1_out) = oneshot::channel();
    let (t1_waiter, _t1_body) = oneshot::channel();
    waits.arm(PendingAction {
        stamp: 1,
        action: "gc".to_string(),
        waiter: Some(t1_waiter),
        reply: t1_reply,
    });
    waits.resolve(1);

    let (t2_reply, _t2_out) = oneshot::channel();
    let (t2_waiter, _t2_body) = oneshot::channel();
    waits.arm(PendingAction {
        stamp: 2,
        action: "gc".to_string(),
        waiter: Some(t2_waiter),
        reply: t2_reply,
    });

    assert!(
        waits.answer("gc", Some(1)).is_none(),
        "T1's own late reply belongs to T1's debt, not to T2"
    );
    assert!(
        waits.answer("gc", Some(2)).is_some(),
        "and T2 is still waiting for its own"
    );
}

/// Fails if a reply owed by a wait that already timed out answers a later
/// wait for the same action. The one failure that produces a wrong answer
/// rather than an error: an app's reply names the action and nothing else.
/// Delete the `abandoned` bookkeeping in `ActionWaits::resolve` and the
/// second trigger is answered `Replied` with the first trigger's body.
#[tokio::test(start_paused = true)]
async fn a_reply_owed_by_a_timed_out_action_never_answers_the_next_one() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);

    let first = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;
    assert_eq!(
        settle_action(&mut actor, &mut mailbox).await,
        ActionOutcome::TimedOut
    );
    assert_eq!(first.await.unwrap(), ActionOutcome::TimedOut);

    let second = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;
    // The app finally answers the first `gc`, having no way to say so.
    actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
    assert_eq!(
        settle_action(&mut actor, &mut mailbox).await,
        ActionOutcome::TimedOut,
        "a reply the first `gc` was owed was handed to the second one"
    );
    assert_eq!(second.await.unwrap(), ActionOutcome::TimedOut);

    // One reply per debt: two `gc` waits have given up and the reply above
    // settled the first.
    let third = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;
    actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
    actor.handle_action_reply(0, "gc", "swept 11".to_string(), None);
    assert_eq!(
        settle_action(&mut actor, &mut mailbox).await,
        ActionOutcome::Replied {
            body: "swept 11".to_string()
        },
        "the debts outlived the replies that settled them"
    );
    assert_eq!(
        third.await.unwrap(),
        ActionOutcome::Replied {
            body: "swept 11".to_string()
        }
    );
}

/// Fails if a second reply to an already-answered action is kept for
/// anything. An app is free to write two, and the second is neither an
/// error nor a debt. Proved by a wait armed after it still timing out.
#[tokio::test(start_paused = true)]
async fn a_second_reply_to_an_answered_action_is_dropped() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);

    let answered = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;
    actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
    assert_eq!(
        settle_action(&mut actor, &mut mailbox).await,
        ActionOutcome::Replied {
            body: "swept 3".to_string()
        }
    );
    assert_eq!(
        answered.await.unwrap(),
        ActionOutcome::Replied {
            body: "swept 3".to_string()
        }
    );

    actor.handle_action_reply(0, "gc", "spent".to_string(), None);

    let next = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;
    assert_eq!(
        settle_action(&mut actor, &mut mailbox).await,
        ActionOutcome::TimedOut,
        "a spare reply was kept and used to answer a wait that came after it"
    );
    assert_eq!(next.await.unwrap(), ActionOutcome::TimedOut);
}

/// Fails if two waits for the same action on one sheep are not answered in
/// the order they were asked. Neither reply says which is which; order is a
/// property of the channel rather than of anything the daemon records.
#[tokio::test(start_paused = true)]
async fn two_waits_for_one_action_are_answered_in_the_order_they_were_asked() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);

    let first = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;
    let second = trigger_action(&mut actor, "gc");
    sent_action(&mut child_rx).await;

    actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
    actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
    settle_action(&mut actor, &mut mailbox).await;
    settle_action(&mut actor, &mut mailbox).await;

    assert_eq!(
        first.await.unwrap(),
        ActionOutcome::Replied {
            body: "swept 3".to_string()
        },
        "the earlier trigger was answered with the later reply"
    );
    assert_eq!(
        second.await.unwrap(),
        ActionOutcome::Replied {
            body: "swept 7".to_string()
        },
        "the second reply was dropped and its wait left waiting"
    );
}

/// Fails if the deadline stops covering the delivery of an action and
/// covers only the reply to it. A child that has stopped reading fd 3 backs
/// its socket up, and a send onto a full channel waits for room that is not
/// coming. The channel here is built full and left unread.
#[tokio::test(start_paused = true)]
async fn an_action_that_cannot_even_be_delivered_still_ends() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);

    // Held, never read: dropping it would make the send fail outright.
    let (to_child, _wedged) = mpsc::channel(1);
    to_child.try_send(ShepherdMessage::Shutdown).unwrap();
    actor
        .sheep
        .get_mut(&0)
        .expect("the fixture registers id 0")
        .to_child = Some(to_child);

    let answer = trigger_action(&mut actor, "gc");
    assert_eq!(
        settle_action(&mut actor, &mut mailbox).await,
        ActionOutcome::TimedOut,
        "an action that never got onto the channel left its wait parked"
    );
    assert_eq!(answer.await.unwrap(), ActionOutcome::TimedOut);
}

/// The whole path a label takes: `run_sheep`'s relay, the actor, and the
/// stats state a `Describe` walks. Keyed by the spawned pid, so a label
/// filed under the sheep's id would miss.
#[tokio::test(start_paused = true)]
async fn a_lamb_label_on_the_channel_reaches_the_describe_walk() {
    let dir = tempfile::tempdir().unwrap();
    let mut app = AppConfig::minimal("web", "./srv");
    app.channel = true;
    let root = crate::fake::FIRST_SCRIPTED_PID;
    let sampler = crate::testing::ScriptedSampler::identifying(vec![vec![
        crate::testing::identity(root, None, "srv"),
        crate::testing::identity(root + 1, Some(root), "python"),
        crate::testing::identity(root + 2, Some(root), "python"),
    ]]);
    let stats = Arc::new(crate::limits::stats::StatsState::new(Arc::new(sampler)));
    let (breaches, _breaches_rx) = mpsc::channel(1);
    let (liveness, _liveness_rx) = mpsc::channel(1);
    let (events, _events_rx) = crate::bus::test_bus(256);
    let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
    let handle =
        SupervisorBuilder::new(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events)
            .extras(Extras {
                clock: Arc::new(SystemClock),
                enforcer: Arc::new(RecordingEnforcer::default()),
                lamb_sweep: idle_sweep(),
                max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
                reports: ExtrasReports { breaches, liveness },
                stats: Arc::clone(&stats),
            })
            .spawn();
    handle.start(vec![normalize(app).unwrap()]).await.unwrap();
    let io = runner.io_handles(0);

    io.from_child_tx
        .send(ChildMessage::LambLabel {
            pid: root + 2,
            label: shep_core::protocol::LambLabel::new("worker 2").unwrap(),
        })
        .await
        .unwrap();

    // Two task hops away. The paused clock advances on each idle sleep, so
    // the timeout is what ends a label that never lands.
    let lambs = tokio::time::timeout(ACTION_WINDOW, async {
        loop {
            let lambs = stats.lambs_of(&stats.lamb_index(), root);
            if lambs.iter().any(|lamb| lamb.label.is_some()) {
                break lambs;
            }
            tokio::time::sleep(Duration::from_millis(10)).await;
        }
    })
    .await
    .expect("the label never reached the walk");

    assert_eq!(
        lambs,
        vec![
            shep_core::protocol::Lamb::new(root + 1, "python"),
            shep_core::protocol::Lamb::new(root + 2, "python").with_label("worker 2"),
        ]
    );
}