use core::time::Duration;
use shep_core::config::AppConfig;
use crate::backoff::restart_delay;
use crate::entry::RestartBudget;
use crate::runner::ExitOutcome;
pub const SANDBOX_REFUSED_EXIT: i32 = 125;
pub(crate) const SANDBOX_DOG: &str = "sandbox";
pub(crate) fn is_sandboxed(app: &AppConfig) -> bool {
app.dogs.contains_key(SANDBOX_DOG)
}
pub fn is_sandbox_refusal(app: &AppConfig, code: Option<i32>) -> bool {
is_sandboxed(app) && code == Some(SANDBOX_REFUSED_EXIT)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Decision {
Restart {
delay: Option<Duration>,
},
CleanStop,
Errored,
SandboxRefused,
}
pub fn decide_on_exit(
app: &AppConfig,
budget: &mut RestartBudget,
uptime: Duration,
exit: ExitOutcome,
manual_stop: bool,
oom_killed: bool,
) -> Decision {
if manual_stop {
return Decision::CleanStop;
}
if is_sandbox_refusal(app, exit.code) {
return Decision::SandboxRefused;
}
if let Some(code) = exit.code
&& app.stop_exit_codes.contains(&code)
{
return Decision::CleanStop;
}
if !app.autorestart {
return Decision::CleanStop;
}
if oom_killed {
budget.note_failed_start();
} else {
let _stability = budget.note_exit(uptime, app.min_uptime.as_duration());
}
if budget.exhausted(app.max_restarts) {
return Decision::Errored;
}
let delay = restart_delay(app, budget.unstable_count());
Decision::Restart { delay }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn manual_stop_wins() {
let app = AppConfig::minimal("test", "./test");
let mut budget = RestartBudget::default();
let exit = ExitOutcome {
code: Some(1),
signal: None,
};
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_millis(100),
exit,
true,
false,
);
assert!(matches!(d, Decision::CleanStop));
}
#[test]
fn stop_exit_codes_match() {
let mut app = AppConfig::minimal("test", "./test");
app.stop_exit_codes = vec![0, 42];
let mut budget = RestartBudget::default();
let exit = ExitOutcome {
code: Some(42),
signal: None,
};
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit,
false,
false,
);
assert!(matches!(d, Decision::CleanStop));
}
#[test]
fn signal_exit_never_matches_stop_exit_codes() {
let mut app = AppConfig::minimal("test", "./test");
app.stop_exit_codes = vec![0];
let mut budget = RestartBudget::default();
let exit = ExitOutcome {
code: None,
signal: Some(15),
};
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_millis(100),
exit,
false,
false,
);
assert!(matches!(d, Decision::Restart { .. }));
}
fn exit_with(code: i32) -> ExitOutcome {
ExitOutcome {
code: Some(code),
signal: None,
}
}
fn dog_table() -> shep_core::config::DogTable {
serde_json::from_value(serde_json::json!({})).unwrap()
}
#[test]
fn a_sandbox_dog_table_counts_as_sandboxed() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
assert!(is_sandboxed(&app));
assert!(is_sandbox_refusal(&app, Some(125)));
}
#[test]
fn a_table_for_another_dog_does_not_count_as_sandboxed() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("jobs".to_owned(), dog_table());
assert!(!is_sandboxed(&app));
assert!(!is_sandbox_refusal(&app, Some(125)));
}
#[test]
fn exit_125_from_a_sheep_with_a_sandbox_table_is_a_refusal() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_millis(100),
exit_with(125),
false,
false,
);
assert_eq!(d, Decision::SandboxRefused);
}
#[test]
fn sandboxed_exit_125_is_a_refusal_even_with_autorestart_off() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
app.autorestart = false;
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit_with(125),
false,
false,
);
assert_eq!(d, Decision::SandboxRefused);
}
#[test]
fn sandboxed_exit_125_beats_stop_exit_codes() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
app.stop_exit_codes = vec![125];
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit_with(125),
false,
false,
);
assert_eq!(d, Decision::SandboxRefused);
}
#[test]
fn sandboxed_exit_125_leaves_the_budget_alone() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
let mut budget = RestartBudget::default();
decide_on_exit(
&app,
&mut budget,
Duration::from_millis(100),
exit_with(125),
false,
false,
);
assert_eq!(budget.unstable_count(), 0);
}
#[test]
fn sandboxed_other_exit_code_is_ordinary() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit_with(1),
false,
false,
);
assert!(matches!(d, Decision::Restart { .. }));
}
#[test]
fn sandboxed_manual_stop_is_not_a_refusal() {
let mut app = AppConfig::minimal("test", "./test");
app.dogs.insert("sandbox".to_owned(), dog_table());
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit_with(125),
true,
false,
);
assert_eq!(d, Decision::CleanStop);
}
#[test]
fn exit_125_from_a_sheep_with_no_sandbox_table_is_an_ordinary_exit() {
let app = AppConfig::minimal("test", "./test");
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit_with(125),
false,
false,
);
assert!(matches!(d, Decision::Restart { .. }));
}
#[test]
fn unsandboxed_exit_125_in_stop_exit_codes_stops_cleanly() {
let mut app = AppConfig::minimal("test", "./test");
app.stop_exit_codes = vec![125];
let mut budget = RestartBudget::default();
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit_with(125),
false,
false,
);
assert_eq!(d, Decision::CleanStop);
}
#[test]
fn autorestart_false_returns_clean_stop() {
let mut app = AppConfig::minimal("test", "./test");
app.autorestart = false;
let mut budget = RestartBudget::default();
let exit = ExitOutcome {
code: Some(1),
signal: None,
};
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit,
false,
false,
);
assert!(matches!(d, Decision::CleanStop));
}
#[test]
fn budget_exhausted_returns_errored() {
let mut app = AppConfig::minimal("test", "./test");
app.min_uptime = "1000".parse().unwrap(); let mut budget = RestartBudget::default();
for _ in 0..16 {
let exit = ExitOutcome {
code: Some(1),
signal: None,
};
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_millis(100),
exit,
false,
false,
);
if budget.unstable_count() >= 16 {
assert!(matches!(d, Decision::Errored));
return;
}
}
panic!("budget should have been exhausted");
}
#[test]
fn stable_exit_returns_restart_with_no_delay() {
let mut app = AppConfig::minimal("test", "./test");
app.min_uptime = "500".parse().unwrap();
let mut budget = RestartBudget::default();
let exit = ExitOutcome {
code: Some(1),
signal: None,
};
let d = decide_on_exit(
&app,
&mut budget,
Duration::from_secs(1),
exit,
false,
false,
);
assert!(matches!(d, Decision::Restart { delay: None }));
}
proptest::proptest! {
#[test]
fn budget_errors_exactly_at_max_restarts(
exits in proptest::collection::vec(0u64..500, 16..64)
) {
let app = AppConfig::minimal("p", "./p");
let mut budget = RestartBudget::default();
for (i, ms) in exits.iter().enumerate() {
let d = decide_on_exit(&app, &mut budget,
Duration::from_millis(*ms),
ExitOutcome { code: Some(1), signal: None }, false, false);
if i < 15 {
proptest::prop_assert!(
matches!(d, Decision::Restart { delay: _ }),
"Expected Restart at i={}, got {:?}",
i,
d
);
} else {
proptest::prop_assert!(
matches!(d, Decision::Errored),
"Expected Errored at i={}, got {:?}",
i,
d
);
break;
}
}
}
}
#[test]
fn an_oom_kill_counts_against_max_restarts_however_long_the_sheep_ran() {
let mut app = AppConfig::minimal("test", "./test");
app.min_uptime = "100".parse().unwrap();
app.max_restarts = 2;
let mut budget = RestartBudget::default();
let ran_a_minute = Duration::from_secs(60);
let first = decide_on_exit(&app, &mut budget, ran_a_minute, exit_with(137), false, true);
let second = decide_on_exit(&app, &mut budget, ran_a_minute, exit_with(137), false, true);
assert!(matches!(first, Decision::Restart { .. }));
assert_eq!(second, Decision::Errored);
}
#[test]
fn a_long_running_exit_without_an_oom_kill_resets_the_budget() {
let mut app = AppConfig::minimal("test", "./test");
app.min_uptime = "100".parse().unwrap();
app.max_restarts = 2;
let mut budget = RestartBudget::default();
let ran_a_minute = Duration::from_secs(60);
for _ in 0..5 {
let d = decide_on_exit(&app, &mut budget, ran_a_minute, exit_with(1), false, false);
assert!(matches!(d, Decision::Restart { .. }));
}
assert_eq!(budget.unstable_count(), 0);
}
}