start_command/
recovery_delay.rs

1//! Random delay before each launch-time kill recovery (issue #181):
2//!
3//! ```text
4//! --on-kill-resume-delay <min[-max]>   seconds; one number is a fixed delay,
5//!                                      0 (the default) resumes at once
6//! ```
7//!
8//! One host-wide OOM event can kill several detached docker executions at the
9//! same moment. Without a delay every watcher resumes its container in the
10//! same second, all of them rebuild their working sets at once, and the next
11//! OOM event follows. A uniformly random delay spreads the recoveries out.
12
13use std::time::Duration;
14
15use serde_json::Value;
16
17/// How often a pending recovery checks whether a stop was requested.
18pub const CANCEL_CHECK_INTERVAL_MS: u64 = 1000;
19
20/// A delay range in seconds.
21#[derive(Debug, Clone, Copy, PartialEq)]
22pub struct RecoveryDelayRange {
23    pub min_seconds: f64,
24    pub max_seconds: f64,
25}
26
27fn parse_seconds(text: &str) -> Option<f64> {
28    let (whole, fraction) = match text.split_once('.') {
29        Some((whole, fraction)) => (whole, Some(fraction)),
30        None => (text, None),
31    };
32    let digits = |part: &str| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit());
33    if !digits(whole) || fraction.is_some_and(|fraction| !digits(fraction)) {
34        return None;
35    }
36    text.parse::<f64>().ok().filter(|value| value.is_finite())
37}
38
39/// Parse `30-90`, `45` or `0`; `None` when malformed or `max < min`.
40pub fn parse_recovery_delay_range(value: &str) -> Option<RecoveryDelayRange> {
41    let text = value.trim();
42    let (min, max) = match text.split_once('-') {
43        Some((min, max)) => (parse_seconds(min)?, parse_seconds(max)?),
44        None => {
45            let seconds = parse_seconds(text)?;
46            (seconds, seconds)
47        }
48    };
49    (max >= min).then_some(RecoveryDelayRange {
50        min_seconds: min,
51        max_seconds: max,
52    })
53}
54
55fn format_seconds(seconds: f64) -> String {
56    // `{}` prints 30 for 30.0 and 1.5 for 1.5, like JavaScript's `${n}`.
57    format!("{}", seconds)
58}
59
60/// `30-90` for a range, `45` for a fixed delay.
61pub fn format_recovery_delay_range(range: &RecoveryDelayRange) -> String {
62    if range.min_seconds == range.max_seconds {
63        format_seconds(range.min_seconds)
64    } else {
65        format!(
66            "{}-{}",
67            format_seconds(range.min_seconds),
68            format_seconds(range.max_seconds)
69        )
70    }
71}
72
73/// Validate and normalize the `--on-kill-resume-delay` argument.
74pub fn parse_on_kill_resume_delay_value(value: &str) -> Result<String, String> {
75    parse_recovery_delay_range(value)
76        .map(|range| format_recovery_delay_range(&range))
77        .ok_or_else(|| {
78            format!(
79                "Invalid --on-kill-resume-delay value: \"{}\". Expected seconds as <min>[-<max>] with max >= min, e.g. 30-90.",
80                value
81            )
82        })
83}
84
85/// The normalized non-zero delay range, or `None` when there is no delay.
86pub fn effective_on_kill_resume_delay(value: Option<&str>) -> Option<String> {
87    let range = parse_recovery_delay_range(value?)?;
88    (range.max_seconds > 0.0).then(|| format_recovery_delay_range(&range))
89}
90
91/// The delay range stored in a record option (a string, or a number when the
92/// record codec turned `45` into one).
93pub fn record_on_kill_resume_delay(value: Option<&Value>) -> Option<String> {
94    match value? {
95        Value::String(text) => effective_on_kill_resume_delay(Some(text)),
96        Value::Number(number) => effective_on_kill_resume_delay(Some(&number.to_string())),
97        _ => None,
98    }
99}
100
101/// A uniformly random number in `[0, 1)` from the OS random source behind
102/// `uuid` v4, so no extra dependency is needed.
103pub fn system_random() -> f64 {
104    let bytes = uuid::Uuid::new_v4().into_bytes();
105    let mut word = [0u8; 8];
106    word.copy_from_slice(&bytes[..8]);
107    // 53 random bits, the precision of an f64 mantissa.
108    (u64::from_le_bytes(word) >> 11) as f64 / (1u64 << 53) as f64
109}
110
111/// Pick a uniformly random delay in whole milliseconds; 0 without a range.
112pub fn pick_recovery_delay_ms(value: Option<&str>, random: &dyn Fn() -> f64) -> u64 {
113    let Some(range) = value.and_then(parse_recovery_delay_range) else {
114        return 0;
115    };
116    let min_ms = range.min_seconds * 1000.0;
117    let max_ms = range.max_seconds * 1000.0;
118    (min_ms + (max_ms - min_ms) * random()).round() as u64
119}
120
121/// `42s`, `42.5s`.
122pub fn format_recovery_delay(delay_ms: u64) -> String {
123    format!("{}s", (delay_ms as f64 / 100.0).round() / 10.0)
124}
125
126/// Block the current thread.
127pub fn system_sleep(ms: u64) {
128    std::thread::sleep(Duration::from_millis(ms));
129}
130
131/// Wait out the delay in short steps, giving up as soon as `should_cancel`
132/// returns true (a `--stop` during the wait). Returns true when cancelled.
133pub fn wait_for_recovery_delay(
134    delay_ms: u64,
135    sleep: &dyn Fn(u64),
136    should_cancel: &dyn Fn() -> bool,
137) -> bool {
138    let mut remaining = delay_ms;
139    while remaining > 0 {
140        if should_cancel() {
141            return true;
142        }
143        let step = remaining.min(CANCEL_CHECK_INTERVAL_MS);
144        sleep(step);
145        remaining -= step;
146    }
147    should_cancel()
148}
149
150#[cfg(test)]
151mod tests {
152    use super::*;
153
154    #[test]
155    fn system_random_stays_in_the_unit_interval() {
156        for _ in 0..1000 {
157            let value = system_random();
158            assert!((0.0..1.0).contains(&value));
159        }
160    }
161}
start_command/recovery_delay.rs

start_command/
recovery_delay.rs
