shep-daemon 0.25.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
//! [`boot`](super::boot)'s own tests, split by what each one is about
//!
//! [`dogs`] covers where a dog runs relative to the muster restore, and
//! [`readiness`] covers both of the reports a boot makes. What stays here
//! belongs to neither: the cron-sleep default, and the one case that drives
//! `boot`'s own extras reporter over the whole production chain.

mod dog_names;
mod dogs;
mod dogs_at_rest;
mod readiness;
mod retired_files;

use crate::boot::*;
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript, ScriptedRunner, ScriptedSweep};
use crate::proc_table::ProcInstant;
use crate::sweep::{LambSignal, LambSnapshot, LambSweep};
use crate::testing::{capture_logs, roll_of, test_paths};
use shep_core::config::{AppConfig, ProbeConfig, ProbeKind, normalize};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{BusEvent, ProcessEventKind, ProcessInfo};
use shep_core::selector::ProcessSelector;
use shep_core::values::UpDuration;
use std::time::Duration;

/// A laid-out `$SHEP_HOME` holding a muster roll of `apps`, all recorded as running.
///
/// The seed a boot test that restores a flock needs: [`init_dirs`] first, so
/// the restore reads the layout the roll was written into, then [`roll_of`].
///
/// # Panics
/// If the layout cannot be created or the roll cannot be written.
#[track_caller]
fn seed_roll(paths: &ShepPaths, apps: Vec<AppConfig>) {
    init_dirs(paths).unwrap();
    crate::snapshot::write_atomic(&paths.snapshot, &roll_of(apps)).unwrap();
}

/// Boots a daemon to a finished listing, and hands back the flock and the log lines.
///
/// The shape every synchronous boot test needs: `capture_logs` scopes its
/// subscriber to a closure, so a current-thread runtime drives it with
/// `block_on`, and the daemon is dropped instead of run.
/// # Panics
/// If the runtime cannot be built, the boot fails, or the listing fails.
#[track_caller]
fn boot_blocking(
    roll: Vec<AppConfig>,
    scripts: Vec<ProcScript>,
    options: BootOptions,
) -> (Vec<ProcessInfo>, String) {
    let _guard = SIGNAL_TEST_LOCK.blocking_lock();
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);
    if roll.is_empty() {
        init_dirs(&paths).unwrap();
    } else {
        seed_roll(&paths, roll);
    }

    let rt = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .unwrap();
    // `capture_logs` takes a `FnOnce`, so the boot takes `paths` out of an
    // `Option` rather than cloning it: nothing after the boot reads them.
    let mut paths = Some(paths);
    let mut boot_result = None;
    let logs = capture_logs(|| {
        boot_result = Some(rt.block_on(boot_with_idle_sweep(
            ScriptedRunner::new(scripts),
            paths.take().expect("the boot takes the paths exactly once"),
            options,
        )));
    });
    let daemon = boot_result.unwrap().expect("the boot must succeed");
    let flock = rt
        .block_on(daemon.context().supervisor.list_checked())
        .unwrap();
    drop(daemon); // no run() needed; SignalTasks::drop stops the listeners
    (flock, logs)
}

// A scripted pid can be another test's real child, so a boot test must never
// end its lambs through the real sweep. Real time: binds a real socket.
#[tokio::test]
async fn a_boot_given_a_sweep_ends_a_sheeps_lambs_through_it() {
    let _guard = SIGNAL_TEST_LOCK.lock().await;
    let dir = tempfile::tempdir().unwrap();
    let sweep = Arc::new(
        ScriptedSweep::new()
            .with_fresh(
                FIRST_SCRIPTED_PID,
                LambSnapshot::new([7], ProcInstant::from_raw(1)),
            )
            .with_survivors(vec![vec![7], vec![]]),
    );

    let daemon = boot_with_sweep(
        ScriptedRunner::new(vec![ProcScript::never_exits()]),
        test_paths(&dir),
        BootOptions::default(),
        Some(Arc::clone(&sweep) as Arc<dyn LambSweep>),
    )
    .await
    .unwrap();
    let ctx = daemon.context();
    let run = tokio::spawn(daemon.run());
    ctx.supervisor
        .start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
        .await
        .unwrap();
    tokio::time::timeout(
        Duration::from_secs(20),
        ctx.supervisor.stop(ProcessSelector::Name("web".to_owned())),
    )
    .await
    .expect("the stop must reply within the bound")
    .unwrap();

    assert_eq!(sweep.signals(), vec![(7, LambSignal::Term)]);
    ctx.shutdown();
    tokio::time::timeout(Duration::from_secs(10), run)
        .await
        .unwrap()
        .unwrap()
        .unwrap();
}

// `boot` is the one place `DEFAULT_MAX_CRON_SLEEP` is applied: the CLI
// keeps the knob an `Option` all the way down, so nothing else here would
// notice a different fallback. Whole `BootOptions` values rather than bare
// `Option`s, since that is what `boot` reads.
#[test]
fn an_unset_max_cron_sleep_falls_back_to_the_daemons_own_default() {
    assert_eq!(
        max_cron_sleep(&BootOptions::default()),
        DEFAULT_MAX_CRON_SLEEP,
        "unset means the default"
    );
    assert_eq!(
        max_cron_sleep(&BootOptions {
            max_cron_sleep: Some(Duration::from_secs(300)),
            ..BootOptions::default()
        }),
        Duration::from_secs(300),
        "a configured value must reach the workers unchanged"
    );
}

// The only case driving `boot`'s own spawn of the extras reporter, over
// the whole production chain: the actor arms the liveness loop at Online,
// the loop reports over `Extras::real`'s sender, the reporter reads it,
// and `extra_restart` lets it through. Real time, and a real `OsProber`.
#[tokio::test]
async fn a_booted_daemon_restarts_a_sheep_whose_liveness_probe_fails() {
    // Real time: binds a real socket, so it takes SIGNAL_TEST_LOCK.
    let _guard = SIGNAL_TEST_LOCK.lock().await;
    let dir = tempfile::tempdir().unwrap();
    let paths = test_paths(&dir);

    // Reserve a port, then release it: nothing listens there, so every
    // probe fails with a connection refusal and there is no race.
    let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let addr = reserved.local_addr().unwrap();
    drop(reserved);

    let daemon = boot_with_idle_sweep(
        ScriptedRunner::new(vec![ProcScript::never_exits(); 4]),
        paths,
        BootOptions::default(),
    )
    .await
    .unwrap();
    let ctx = daemon.context();
    let mut events = ctx.events.subscribe();
    let run = tokio::spawn(daemon.run());

    let mut app = AppConfig::minimal("web", "./srv");
    app.liveness_probe = Some(ProbeConfig {
        kind: ProbeKind::Tcp,
        target: addr.to_string(),
        // The loop floors anything shorter at one second, so a smaller
        // number here would be a lie about what this test waits for.
        interval: UpDuration::from_millis(1_000),
        timeout: UpDuration::from_millis(500),
        failure_threshold: 1,
    });
    ctx.supervisor
        .start(vec![normalize(app).unwrap()])
        .await
        .unwrap();

    let restarted = async {
        loop {
            match events.recv().await.map(|event| event.to_event()) {
                Ok(BusEvent::Process {
                    event: ProcessEventKind::Restart,
                    info,
                    ..
                }) => return info,
                Ok(_) => continue,
                Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
                Err(err) => panic!("the event stream closed before a restart: {err}"),
            }
        }
    };
    let info = tokio::time::timeout(Duration::from_secs(20), restarted)
        .await
        .expect("a failing liveness probe must restart its sheep");
    assert_eq!(info.id, 0);
    assert_eq!(info.restarts, 1);

    ctx.shutdown();
    tokio::time::timeout(Duration::from_secs(10), run)
        .await
        .unwrap()
        .unwrap()
        .unwrap();
}