shep-daemon 0.22.0

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
//! Restart decision logic.
//!
//! A signal exit (`code: None`) never matches `stop_exit_codes`; only `Some(code)` is tested.

use core::time::Duration;

use shep_core::config::AppConfig;

use crate::backoff::restart_delay;
use crate::entry::RestartBudget;
use crate::runner::ExitOutcome;

/// The exit code crabpot uses when its sandbox refuses to start
pub const SANDBOX_REFUSED_EXIT: i32 = 125;

/// The name of the adopted dog that launches a sheep holding a table under this key.
pub(crate) const SANDBOX_DOG: &str = "sandbox";

/// Whether `app` runs sandboxed: it sets `sandbox`, or holds an `[app.dogs.sandbox]` table
pub(crate) fn is_sandboxed(app: &AppConfig) -> bool {
    app.sandbox.is_some() || app.dogs.contains_key(SANDBOX_DOG)
}

/// Whether a sheep configured as `app` that exited with `code` was refused by its sandbox
pub fn is_sandbox_refusal(app: &AppConfig, code: Option<i32>) -> bool {
    is_sandboxed(app) && code == Some(SANDBOX_REFUSED_EXIT)
}

/// Restart decision for a process exit
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Decision {
    /// Restart the process with an optional delay (None = immediate)
    Restart {
        /// Delay before restart (None = immediate restart)
        delay: Option<Duration>,
    },
    /// Stop cleanly, do not restart
    CleanStop,
    /// Budget exhausted; restart policy failed
    Errored,
    /// A sandboxed sheep exited with [`SANDBOX_REFUSED_EXIT`]; a restart would only repeat it
    SandboxRefused,
}

/// Decides whether to restart, checking in order: manual stop, a sandbox
/// refusal, a `stop_exit_codes` match, `autorestart`, then the restart budget.
pub fn decide_on_exit(
    app: &AppConfig,
    budget: &mut RestartBudget,
    uptime: Duration,
    exit: ExitOutcome,
    manual_stop: bool,
) -> Decision {
    if manual_stop {
        return Decision::CleanStop;
    }

    if is_sandbox_refusal(app, exit.code) {
        return Decision::SandboxRefused;
    }

    if let Some(code) = exit.code
        && app.stop_exit_codes.contains(&code)
    {
        return Decision::CleanStop;
    }

    if !app.autorestart {
        return Decision::CleanStop;
    }

    let _stability = budget.note_exit(uptime, app.min_uptime.as_duration());
    if budget.exhausted(app.max_restarts) {
        return Decision::Errored;
    }

    let delay = restart_delay(app, budget.unstable_count());
    Decision::Restart { delay }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn manual_stop_wins() {
        let app = AppConfig::minimal("test", "./test");
        let mut budget = RestartBudget::default();
        let exit = ExitOutcome {
            code: Some(1),
            signal: None,
        };
        let d = decide_on_exit(&app, &mut budget, Duration::from_millis(100), exit, true);
        assert!(matches!(d, Decision::CleanStop));
    }

    #[test]
    fn stop_exit_codes_match() {
        let mut app = AppConfig::minimal("test", "./test");
        app.stop_exit_codes = vec![0, 42];
        let mut budget = RestartBudget::default();
        let exit = ExitOutcome {
            code: Some(42),
            signal: None,
        };
        let d = decide_on_exit(&app, &mut budget, Duration::from_secs(1), exit, false);
        assert!(matches!(d, Decision::CleanStop));
    }

    #[test]
    fn signal_exit_never_matches_stop_exit_codes() {
        let mut app = AppConfig::minimal("test", "./test");
        app.stop_exit_codes = vec![0];
        let mut budget = RestartBudget::default();
        let exit = ExitOutcome {
            code: None,
            signal: Some(15),
        };
        let d = decide_on_exit(&app, &mut budget, Duration::from_millis(100), exit, false);
        assert!(matches!(d, Decision::Restart { .. }));
    }

    fn exit_with(code: i32) -> ExitOutcome {
        ExitOutcome {
            code: Some(code),
            signal: None,
        }
    }

    fn dog_table() -> shep_core::config::DogTable {
        serde_json::from_value(serde_json::json!({})).unwrap()
    }

    #[test]
    fn a_sandbox_dog_table_counts_as_sandboxed() {
        let mut app = AppConfig::minimal("test", "./test");
        app.dogs.insert("sandbox".to_owned(), dog_table());
        assert!(is_sandboxed(&app));
        assert!(is_sandbox_refusal(&app, Some(125)));
    }

    #[test]
    fn a_table_for_another_dog_does_not_count_as_sandboxed() {
        let mut app = AppConfig::minimal("test", "./test");
        app.dogs.insert("jobs".to_owned(), dog_table());
        assert!(!is_sandboxed(&app));
        assert!(!is_sandbox_refusal(&app, Some(125)));
    }

    #[test]
    fn sandboxed_exit_125_is_a_refusal() {
        let mut app = AppConfig::minimal("test", "./test");
        app.sandbox = Some("sandbox.json".to_owned());
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_millis(100),
            exit_with(125),
            false,
        );
        assert_eq!(d, Decision::SandboxRefused);
    }

    #[test]
    fn sandboxed_exit_125_is_a_refusal_even_with_autorestart_off() {
        let mut app = AppConfig::minimal("test", "./test");
        app.sandbox = Some("sandbox.json".to_owned());
        app.autorestart = false;
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_secs(1),
            exit_with(125),
            false,
        );
        assert_eq!(d, Decision::SandboxRefused);
    }

    #[test]
    fn sandboxed_exit_125_beats_stop_exit_codes() {
        let mut app = AppConfig::minimal("test", "./test");
        app.sandbox = Some("sandbox.json".to_owned());
        app.stop_exit_codes = vec![125];
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_secs(1),
            exit_with(125),
            false,
        );
        assert_eq!(d, Decision::SandboxRefused);
    }

    #[test]
    fn sandboxed_exit_125_leaves_the_budget_alone() {
        let mut app = AppConfig::minimal("test", "./test");
        app.sandbox = Some("sandbox.json".to_owned());
        let mut budget = RestartBudget::default();
        decide_on_exit(
            &app,
            &mut budget,
            Duration::from_millis(100),
            exit_with(125),
            false,
        );
        assert_eq!(budget.unstable_count(), 0);
    }

    #[test]
    fn sandboxed_other_exit_code_is_ordinary() {
        let mut app = AppConfig::minimal("test", "./test");
        app.sandbox = Some("sandbox.json".to_owned());
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_secs(1),
            exit_with(1),
            false,
        );
        assert!(matches!(d, Decision::Restart { .. }));
    }

    #[test]
    fn sandboxed_manual_stop_is_not_a_refusal() {
        let mut app = AppConfig::minimal("test", "./test");
        app.sandbox = Some("sandbox.json".to_owned());
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_secs(1),
            exit_with(125),
            true,
        );
        assert_eq!(d, Decision::CleanStop);
    }

    #[test]
    fn unsandboxed_exit_125_is_an_ordinary_exit() {
        let app = AppConfig::minimal("test", "./test");
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_secs(1),
            exit_with(125),
            false,
        );
        assert!(matches!(d, Decision::Restart { .. }));
    }

    #[test]
    fn unsandboxed_exit_125_in_stop_exit_codes_stops_cleanly() {
        let mut app = AppConfig::minimal("test", "./test");
        app.stop_exit_codes = vec![125];
        let mut budget = RestartBudget::default();
        let d = decide_on_exit(
            &app,
            &mut budget,
            Duration::from_secs(1),
            exit_with(125),
            false,
        );
        assert_eq!(d, Decision::CleanStop);
    }

    #[test]
    fn autorestart_false_returns_clean_stop() {
        let mut app = AppConfig::minimal("test", "./test");
        app.autorestart = false;
        let mut budget = RestartBudget::default();
        let exit = ExitOutcome {
            code: Some(1),
            signal: None,
        };
        let d = decide_on_exit(&app, &mut budget, Duration::from_secs(1), exit, false);
        assert!(matches!(d, Decision::CleanStop));
    }

    #[test]
    fn budget_exhausted_returns_errored() {
        let mut app = AppConfig::minimal("test", "./test");
        app.min_uptime = "1000".parse().unwrap(); // 1000 ms
        // max_restarts is the real default (16); exhausted uses >=.
        let mut budget = RestartBudget::default();

        for _ in 0..16 {
            let exit = ExitOutcome {
                code: Some(1),
                signal: None,
            };
            let d = decide_on_exit(&app, &mut budget, Duration::from_millis(100), exit, false);
            if budget.unstable_count() >= 16 {
                assert!(matches!(d, Decision::Errored));
                return;
            }
        }
        panic!("budget should have been exhausted");
    }

    #[test]
    fn stable_exit_returns_restart_with_no_delay() {
        let mut app = AppConfig::minimal("test", "./test");
        app.min_uptime = "500".parse().unwrap();
        let mut budget = RestartBudget::default();
        let exit = ExitOutcome {
            code: Some(1),
            signal: None,
        };
        // Uptime >= min_uptime means stable exit
        let d = decide_on_exit(&app, &mut budget, Duration::from_secs(1), exit, false);
        assert!(matches!(d, Decision::Restart { delay: None }));
    }

    proptest::proptest! {
        #[test]
        fn budget_errors_exactly_at_max_restarts(
            exits in proptest::collection::vec(0u64..500, 16..64)
        ) {
            // ms range stays under the default min_uptime (1000ms), so every exit is unstable.
            let app = AppConfig::minimal("p", "./p");
            let mut budget = RestartBudget::default();
            for (i, ms) in exits.iter().enumerate() {
                let d = decide_on_exit(&app, &mut budget,
                    Duration::from_millis(*ms),
                    ExitOutcome { code: Some(1), signal: None }, false);
                if i < 15 {
                    proptest::prop_assert!(
                        matches!(d, Decision::Restart { delay: _ }),
                        "Expected Restart at i={}, got {:?}",
                        i,
                        d
                    );
                } else {
                    proptest::prop_assert!(
                        matches!(d, Decision::Errored),
                        "Expected Errored at i={}, got {:?}",
                        i,
                        d
                    );
                    break;
                }
            }
        }
    }
}