shep-daemon 0.18.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
use super::*;
use crate::fake::ScriptedSweep;
use crate::testing::capture_logs;

const GRACE: Duration = Duration::from_secs(5);

fn at(raw: u64) -> ProcInstant {
    ProcInstant::from_raw(raw)
}

fn snapshot_of(pids: &[u32]) -> LambSnapshot {
    LambSnapshot::new(pids.iter().copied(), at(1_700_000_000))
}

#[tokio::test(start_paused = true)]
async fn an_empty_snapshot_reads_nothing_and_signals_nothing() {
    let sweep = ScriptedSweep::new().with_survivors(vec![vec![7]]);

    let report = sweep_lambs(&sweep, &snapshot_of(&[]), GRACE).await;

    assert_eq!(report, SweepReport::default());
    assert!(
        sweep.looks().is_empty(),
        "an empty snapshot must not be read"
    );
    assert!(sweep.signals().is_empty());
}

#[tokio::test(start_paused = true)]
async fn a_snapshot_with_no_survivors_signals_nothing_and_does_not_wait() {
    let sweep = ScriptedSweep::new();
    let start = Instant::now();

    let report = sweep_lambs(&sweep, &snapshot_of(&[7, 8]), GRACE).await;

    assert_eq!(report, SweepReport::default());
    assert!(sweep.signals().is_empty());
    assert_eq!(
        Instant::now(),
        start,
        "nothing to signal means nothing to wait for"
    );
}

#[tokio::test(start_paused = true)]
async fn lambs_that_exit_on_term_get_no_kill_and_the_sweep_returns_at_the_first_look() {
    let sweep = ScriptedSweep::new().with_survivors(vec![vec![7, 8], vec![]]);
    let start = Instant::now();

    let report = sweep_lambs(&sweep, &snapshot_of(&[7, 8]), GRACE).await;

    assert_eq!(
        sweep.signals(),
        vec![(7, LambSignal::Term), (8, LambSignal::Term)]
    );
    assert_eq!(report.termed, vec![7, 8]);
    assert!(report.killed.is_empty());
    assert_eq!(Instant::now() - start, SWEEP_POLL_INTERVAL);
}

#[tokio::test(start_paused = true)]
async fn a_lamb_that_ignores_term_is_killed_once_the_grace_runs_out() {
    // Two survive the first look; 8 exits on its TERM and 7 never does.
    let sweep = ScriptedSweep::new().with_survivors(vec![vec![7, 8], vec![7]]);
    let start = Instant::now();

    let report = sweep_lambs(&sweep, &snapshot_of(&[7, 8]), GRACE).await;

    assert_eq!(
        Instant::now() - start,
        GRACE + KILL_SETTLE_POLL,
        "KILL waits out the whole grace, then one look sees it landed"
    );
    assert_eq!(
        sweep.signals(),
        vec![
            (7, LambSignal::Term),
            (8, LambSignal::Term),
            (7, LambSignal::Kill)
        ]
    );
    assert_eq!(report.killed, vec![7]);
    let looks = sweep.looks();
    assert_eq!(
        looks.len(),
        2 + (GRACE.as_millis() / SWEEP_POLL_INTERVAL.as_millis()) as usize,
        "one look before TERM, one per poll interval, one after KILL"
    );
    assert!(looks.iter().all(|look| *look == snapshot_of(&[7, 8])));
}

#[tokio::test(start_paused = true)]
async fn a_zero_grace_looks_once_more_then_kills() {
    let sweep = ScriptedSweep::new().with_survivors(vec![vec![7]]);
    let start = Instant::now();

    let report = sweep_lambs(&sweep, &snapshot_of(&[7]), Duration::ZERO).await;

    assert_eq!(Instant::now() - start, KILL_SETTLE_POLL);
    assert_eq!(report.killed, vec![7]);
    assert_eq!(sweep.looks().len(), 2, "one look, then one after KILL");
}

// A stop reply must not race the kernel tearing a killed lamb down.
#[tokio::test(start_paused = true)]
async fn the_sweep_returns_only_once_a_killed_lamb_has_stopped_running() {
    let sweep = ScriptedSweep::new().with_survivors(vec![vec![7]]);

    sweep_lambs(&sweep, &snapshot_of(&[7]), Duration::ZERO).await;

    assert!(sweep.survivors(&snapshot_of(&[7])).is_empty());
    assert_eq!(sweep.looks().len(), 3, "a look after KILL saw it gone");
}

// The second survivor list is the re-check after the walk: 7 still runs.
#[tokio::test(start_paused = true)]
async fn a_child_a_lamb_started_after_the_snapshot_is_termed_with_it() {
    let sweep = ScriptedSweep::new()
        .with_survivors(vec![vec![7], vec![7], vec![]])
        .with_born(vec![vec![9]]);

    let report = sweep_lambs(&sweep, &snapshot_of(&[7]), GRACE).await;

    assert_eq!(
        sweep.walks().first(),
        Some(&vec![7]),
        "walked from the survivor"
    );
    assert_eq!(
        sweep.looks().get(1),
        Some(&snapshot_of(&[7])),
        "the re-check reads the roots, before the walk's finds are merged"
    );
    assert_eq!(
        sweep.signals(),
        vec![(7, LambSignal::Term), (9, LambSignal::Term)]
    );
    assert_eq!(report.termed, vec![7, 9]);
}

// 7 exits after the survivor read and its pid is reused before the walk:
// the walk's 9 is a stranger's child.
#[tokio::test(start_paused = true)]
async fn a_walk_from_a_root_that_stopped_surviving_signals_none_of_its_finds() {
    let sweep = ScriptedSweep::new()
        .with_survivors(vec![vec![7], vec![], vec![]])
        .with_born(vec![vec![9]]);

    let report = sweep_lambs(&sweep, &snapshot_of(&[7]), GRACE).await;

    assert!(
        !sweep.signals().iter().any(|&(pid, _)| pid == 9),
        "{:?}",
        sweep.signals()
    );
    assert!(
        !sweep.signals().iter().any(|&(pid, _)| pid == 7),
        "a root that failed the re-check is not signalled on that look either"
    );
    assert!(
        sweep.looks().iter().all(|look| look.seen_at(9).is_none()),
        "the dropped find is never merged into a later look"
    );
    assert_eq!(report, SweepReport::default());
}

// One root of two fails the re-check: the look keeps the other, drops the
// walk's finds, and the next look walks again.
#[tokio::test(start_paused = true)]
async fn a_failed_re_check_keeps_the_roots_that_passed_and_walks_again() {
    let sweep = ScriptedSweep::new()
        .with_survivors(vec![vec![7, 8], vec![8], vec![8], vec![8], vec![]])
        .with_born(vec![vec![9], vec![10]]);

    let report = sweep_lambs(&sweep, &snapshot_of(&[7, 8]), GRACE).await;

    assert_eq!(
        sweep.signals(),
        vec![(8, LambSignal::Term), (10, LambSignal::Term)]
    );
    assert_eq!(sweep.walks(), vec![vec![7, 8], vec![8]]);
    assert_eq!(report.termed, vec![8, 10]);
}

// The lamb exits on its TERM, orphaning a child it started mid-grace. The
// look that found the child while its parent lived is what holds it.
#[tokio::test(start_paused = true)]
async fn a_child_born_mid_grace_outlives_its_parent_and_is_still_killed() {
    // The third list is the re-check after the walk that found 9.
    let sweep = ScriptedSweep::new()
        .with_survivors(vec![vec![7], vec![7], vec![7], vec![9]])
        .with_born(vec![vec![], vec![9], vec![]]);
    let start = Instant::now();

    let report = sweep_lambs(&sweep, &snapshot_of(&[7]), GRACE).await;

    assert_eq!(
        sweep.signals(),
        vec![
            (7, LambSignal::Term),
            (9, LambSignal::Term),
            (9, LambSignal::Kill)
        ]
    );
    assert_eq!(report.termed, vec![7, 9]);
    assert_eq!(report.killed, vec![9]);
    assert_eq!(Instant::now() - start, GRACE + KILL_SETTLE_POLL);
    assert!(
        sweep
            .looks()
            .iter()
            .skip(3)
            .all(|look| look.pids().any(|pid| pid == 9)),
        "every look after the find carries the orphan"
    );
}

#[test]
fn a_lamb_that_survives_kill_is_logged_and_left_after_the_settle_bound() {
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_time()
        .start_paused(true)
        .build()
        .unwrap();
    let sweep = ScriptedSweep::new()
        .with_survivors(vec![vec![7]])
        .unkillable(7);

    let mut elapsed = Duration::ZERO;
    let logs = capture_logs(|| {
        runtime.block_on(async {
            let start = Instant::now();
            sweep_lambs(&sweep, &snapshot_of(&[7]), Duration::ZERO).await;
            elapsed = Instant::now() - start;
        });
    });

    assert_eq!(elapsed, KILL_SETTLE);
    assert!(
        logs.contains("still running after SIGKILL") && logs.contains("lambs=[7]"),
        "{logs}"
    );
}

// A sync test over its own paused runtime: `capture_logs` only sees records
// written on this thread.
#[test]
fn a_refused_signal_is_logged_and_the_sweep_still_kills_the_rest() {
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_time()
        .start_paused(true)
        .build()
        .unwrap();
    let sweep = ScriptedSweep::new()
        .with_survivors(vec![vec![7, 8], vec![8]])
        .refusing(7);

    let mut report = SweepReport::default();
    let logs = capture_logs(|| {
        report = runtime.block_on(sweep_lambs(&sweep, &snapshot_of(&[7, 8]), GRACE));
    });

    assert_eq!(
        sweep.signals(),
        vec![
            (7, LambSignal::Term),
            (8, LambSignal::Term),
            (8, LambSignal::Kill)
        ]
    );
    assert_eq!(report.killed, vec![8]);
    assert!(
        logs.contains("lamb signal delivery failed") && logs.contains("pid=7"),
        "{logs}"
    );
    assert!(
        logs.contains("INFO") && logs.contains("swept lambs that outlived their sheep"),
        "{logs}"
    );
    assert!(
        logs.contains("SIGKILL") && logs.contains("lambs=[8]"),
        "{logs}"
    );
}

// A lamb born since the last tick is seen only by the fresh walk. Dated at
// the tick's second, it would fail the start-time check and escape.
#[test]
fn a_merge_unions_the_pids_and_dates_each_by_the_latest_look_that_saw_it() {
    let tick = LambSnapshot::new([7, 8], at(100));
    let fresh = LambSnapshot::new([8, 9], at(200));

    let merged = tick.clone().merge(fresh.clone());
    assert_eq!(
        merged.pids().collect::<Vec<_>>(),
        vec![7, 8, 9],
        "every pid either look saw"
    );
    assert_eq!(merged.seen_at(7), Some(at(100)));
    assert_eq!(merged.seen_at(8), Some(at(200)));
    assert_eq!(merged.seen_at(9), Some(at(200)));
    assert_eq!(merged.seen_at(10), None);
    assert_eq!(fresh.merge(tick), merged, "the merge is symmetric");
}

#[test]
fn a_snapshots_latest_look_is_the_latest_any_pid_was_seen() {
    let merged = LambSnapshot::new([7, 9], at(100)).merge(LambSnapshot::new([8], at(200)));
    assert_eq!(merged.latest(), Some(at(200)), "not the first pid's, 100");
    assert_eq!(LambSnapshot::default().latest(), None);
}

#[test]
fn a_snapshot_keeps_each_pid_once() {
    let snapshot = LambSnapshot::new([9, 7, 9, 7], at(5));
    assert_eq!(snapshot.pids().collect::<Vec<_>>(), vec![7, 9]);
    assert_eq!(snapshot.seen_at(7), Some(at(5)));
}

#[test]
fn the_scripted_sweep_answers_with_its_canned_snapshots_behind_dyn() {
    let fresh = LambSnapshot::new([7], at(200));
    let last = LambSnapshot::new([8], at(100));
    let scripted = ScriptedSweep::new()
        .with_fresh(1, fresh.clone())
        .with_last(1, last.clone());
    let sweep: &dyn LambSweep = &scripted;

    assert_eq!(sweep.snapshot(1), fresh);
    assert_eq!(sweep.last_snapshot(1), Some(last));
    assert!(sweep.snapshot(2).is_empty());
    assert_eq!(sweep.last_snapshot(2), None);
}