runnerd_core/api/sizing/
shown.rs

1//! What the sizing advice means as every client shows it: what a set of jobs needs and why, how the advice reads at
2//! a glance, what auto sizing does next with a profile at a given time, how much history it covers, and doctor's
3//! check of it. Plain derivations of [`Sizing`]; the daemon's advice uses [`need`] for its arithmetic.
4
5use super::{
6    Advice, AutoPlan, JobSizing, PAGE_CACHE_ROOM, Peaks, ProfileSizing, Sizing, SizingChange,
7    SizingHold, Starts,
8};
9use crate::api::{Check, CheckStatus, Tone, Toned};
10use crate::config::ByteSize;
11
12/// Advice rounds memory up to a multiple of this: 256M.
13pub const GRANULE: u64 = 256 << 20;
14
15/// Seconds between two samples of a running job's container. Clients without a `Sizing` or `SampleSeries` that
16/// says so, such as a job's figures in `runnerctl`, word their sampling caveat from it.
17pub const SAMPLE_INTERVAL: u64 = 2;
18
19/// What a set of jobs whose largest sampled peak is `peak` needs, with `margin` percent: the peak plus the margin, or
20/// plus `PAGE_CACHE_ROOM` when that is more, rounded up to `GRANULE`. The largest, not a percentile: the cap kills a
21/// job that goes over it, so a cap below a peak a job reached kills it again.
22pub fn need(peak: u64, margin: u32) -> u64 {
23    let room = (peak.saturating_mul(u64::from(margin)) / 100).max(PAGE_CACHE_ROOM);
24    peak.saturating_add(room)
25        .div_ceil(GRANULE)
26        .saturating_mul(GRANULE)
27        .max(GRANULE)
28}
29
30/// The caveat on a sampled peak, as every client words it: `memory peaks are sampled every 2 s; spikes shorter than
31/// that do not show`.
32pub fn sampled_note(interval: u64) -> String {
33    format!("memory peaks are sampled every {interval} s; spikes shorter than that do not show")
34}
35
36/// What a set of jobs needs, as the advice sizes for it, from `Sizing::need`.
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct Need {
39    /// In bytes.
40    pub memory: u64,
41    /// How it comes from the largest peak: `its largest peak + 15%` or `its largest peak + 1G for page cache`.
42    pub reason: String,
43}
44
45impl Sizing {
46    /// What `peaks` need: the advice's memory, or with too few jobs for advice, what it would size for.
47    pub fn need(&self, peaks: &Peaks, advice: Advice) -> Need {
48        let memory = match advice {
49            Advice::Keep { memory }
50            | Advice::Lower { memory, .. }
51            | Advice::Raise { memory, .. }
52            | Advice::SliceTooSmall { memory } => memory,
53            Advice::TooFew { .. } => need(peaks.memory.max, self.margin),
54        };
55        Need {
56            memory,
57            reason: self.need_reason(peaks),
58        }
59    }
60
61    /// How the advice comes from `peaks`' largest: `its largest peak + 15%`, or `its largest peak + 1G for page
62    /// cache` when that is more.
63    pub(crate) fn need_reason(&self, peaks: &Peaks) -> String {
64        if peaks.memory.max * u64::from(self.margin) / 100 >= PAGE_CACHE_ROOM {
65            format!("its largest peak + {}%", self.margin)
66        } else {
67            format!(
68                "its largest peak + {} for page cache",
69                ByteSize(PAGE_CACHE_ROOM)
70            )
71        }
72    }
73
74    /// How the advice is made, as every client explains it: `peaks are sampled every 2 s, so a shorter spike goes
75    /// unseen: advice sizes for the largest + 15%, at least 1G for page cache, rounded up to 256M, once a profile ran
76    /// 10 jobs`.
77    pub fn method_line(&self) -> String {
78        format!(
79            "peaks are sampled every {} s, so a shorter spike goes unseen: advice sizes for the largest + {}%, at \
80             least {} for page cache, rounded up to {}, once a profile ran {} jobs",
81            self.sample_interval,
82            self.margin,
83            ByteSize(PAGE_CACHE_ROOM),
84            ByteSize(GRANULE),
85            self.min_jobs
86        )
87    }
88
89    /// `Sizing::method_line` in short parts for a heading, the sampling caveat first: `peaks sampled every 2 s`,
90    /// `+15%, at least 1G, in 256M steps` and `advice after 10 jobs`.
91    pub fn method_parts(&self) -> Vec<String> {
92        vec![
93            format!("peaks sampled every {} s", self.sample_interval),
94            format!(
95                "+{}%, at least {}, in {} steps",
96                self.margin,
97                ByteSize(PAGE_CACHE_ROOM),
98                ByteSize(GRANULE)
99            ),
100            format!("advice after {} jobs", self.min_jobs),
101        ]
102    }
103
104    /// The history it was computed from: `since` to when it was computed, however long ago that was.
105    pub fn covered(&self) -> u64 {
106        self.until.saturating_sub(self.since)
107    }
108
109    /// Doctor's check of it: a warning while a profile needs more memory than its cap, else the profiles whose
110    /// advice changes something, or that all fit.
111    pub fn doctor_check(&self) -> Check {
112        let changes: Vec<_> = self
113            .profiles
114            .iter()
115            .filter(|profile| {
116                profile.advice.change().is_some()
117                    || matches!(profile.advice, Advice::SliceTooSmall { .. })
118            })
119            .collect();
120        let raise = changes
121            .iter()
122            .any(|profile| profile.advice.tone() == Tone::Bad);
123        let detail = if changes.is_empty() {
124            let too_few = self
125                .profiles
126                .iter()
127                .any(|profile| matches!(profile.advice, Advice::TooFew { .. }));
128            if too_few {
129                format!(
130                    "some profiles ran fewer than {} jobs; runnerctl sizing shows their peaks",
131                    self.min_jobs
132                )
133            } else {
134                format!(
135                    "every profile's memory fits its jobs' largest peak + {}%, at least {} for page cache",
136                    self.margin,
137                    ByteSize(PAGE_CACHE_ROOM)
138                )
139            }
140        } else {
141            changes
142                .iter()
143                .map(|profile| format!("{}: {}", profile.name, self.advice_line(profile)))
144                .collect::<Vec<_>>()
145                .join("; ")
146        };
147        Check {
148            name: "sizing".to_owned(),
149            status: if raise {
150                CheckStatus::Warn
151            } else {
152                CheckStatus::Info
153            },
154            detail,
155            fix: (!changes.is_empty()).then(|| {
156                changes
157                    .iter()
158                    .map(|profile| match profile.advice {
159                        Advice::SliceTooSmall { .. } => format!(
160                            "raise MemoryMax in the slice's unit file, sudo systemctl daemon-reload, then: \
161                             runnerctl sizing apply {}",
162                            profile.name
163                        ),
164                        _ => format!("runnerctl sizing apply {}", profile.name),
165                    })
166                    .collect::<Vec<_>>()
167                    .join("\n")
168            }),
169        }
170    }
171}
172
173impl Advice {
174    /// What it means for the reader: more memory needed is bad, a lowering worth a look, a fitting cap good, too
175    /// few jobs to tell in the background.
176    pub fn tone(&self) -> Tone {
177        match self {
178            Advice::Raise { .. } | Advice::SliceTooSmall { .. } => Tone::Bad,
179            Advice::Lower { .. } => Tone::Warn,
180            Advice::Keep { .. } => Tone::Good,
181            Advice::TooFew { .. } => Tone::Quiet,
182        }
183    }
184
185    /// It in a word or two, for a table cell: `too few`, `keep`, `grow slice`, or the new `9G × 2`.
186    pub fn word(&self) -> String {
187        match *self {
188            Advice::TooFew { .. } => "too few".to_owned(),
189            Advice::Keep { .. } => "keep".to_owned(),
190            Advice::SliceTooSmall { .. } => "grow slice".to_owned(),
191            Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => {
192                format!("{} × {slots}", ByteSize(memory))
193            }
194        }
195    }
196}
197
198impl JobSizing {
199    /// What its jobs need, for a table cell, and what that means: `max 3G` with too few runs, `9G > cap` past its
200    /// profile's cap, else `6G`.
201    pub fn needs(&self) -> Toned {
202        let (text, tone) = match self.advice {
203            Advice::TooFew { .. } => (
204                format!("max {}", ByteSize(self.peaks.memory.max)),
205                Tone::Quiet,
206            ),
207            Advice::Raise { memory, .. } | Advice::SliceTooSmall { memory } => {
208                (format!("{} > cap", ByteSize(memory)), Tone::Bad)
209            }
210            Advice::Keep { memory } | Advice::Lower { memory, .. } => {
211                (ByteSize(memory).to_string(), Tone::Neutral)
212            }
213        };
214        Toned { text, tone }
215    }
216}
217
218impl Starts {
219    /// Every job counted: warm and cold starts.
220    pub fn total(&self) -> u32 {
221        self.warm + self.cold
222    }
223}
224
225/// What auto sizing does next with one profile at a given time, from `AutoPlan::profile_plan`.
226#[derive(Debug, Clone, Copy, PartialEq, Eq)]
227pub enum ProfilePlan<'plan> {
228    /// The next look makes this change, or would once `AutoPlan::blocked` is fixed.
229    Next {
230        /// The change.
231        change: &'plan SizingChange,
232        /// Whether `AutoPlan::blocked` keeps it from being made.
233        blocked: bool,
234    },
235    /// A recent change holds it from any change until `hold.until`.
236    Held {
237        /// The hold.
238        hold: &'plan SizingHold,
239    },
240    /// A recent change holds it from a lowering until `hold.lowered_after()`; a raise may come before.
241    LowerHeld {
242        /// The hold.
243        hold: &'plan SizingHold,
244    },
245    /// Nothing: the advice asks for no change, and no hold is left.
246    Nothing,
247}
248
249impl AutoPlan {
250    /// The `cpus` raise `profile`'s jobs ask for and the next look cannot make, as a line of its own beside what it
251    /// does next, which can still change its memory or slots: `No CPU raise: 3 runs of … the cap holds them back; …`.
252    pub fn refusal(&self, profile: &str) -> Option<String> {
253        self.refused
254            .iter()
255            .find(|refused| refused.profile == profile)
256            .map(|refused| format!("No CPU raise: {}", refused.reason))
257    }
258
259    /// What it does next with `profile` at `now`: a hold whose time has passed holds nothing, however old the plan.
260    pub fn profile_plan(&self, profile: &str, now: u64) -> ProfilePlan<'_> {
261        if let Some(change) = self.next.iter().find(|change| change.profile == profile) {
262            return ProfilePlan::Next {
263                change,
264                blocked: self.blocked.is_some(),
265            };
266        }
267        match self.held.iter().find(|hold| hold.profile == profile) {
268            Some(hold) if now < hold.until => ProfilePlan::Held { hold },
269            Some(hold) if now < hold.lowered_after() => ProfilePlan::LowerHeld { hold },
270            _ => ProfilePlan::Nothing,
271        }
272    }
273}
274
275impl ProfilePlan<'_> {
276    /// When what it says ends: the hold's end, for a held profile.
277    pub fn until(&self) -> Option<u64> {
278        match self {
279            ProfilePlan::Held { hold } => Some(hold.until),
280            ProfilePlan::LowerHeld { hold } => Some(hold.lowered_after()),
281            ProfilePlan::Next { .. } | ProfilePlan::Nothing => None,
282        }
283    }
284
285    /// What it does, as the lead of its line, before `until` when it has one: `Next look, within 5 minutes: memory
286    /// 9G → 6.75G, slots 2`, `Once that is fixed: …`, `No change`, `No lowering` or `Nothing to change`.
287    pub fn headline(&self, plan: &AutoPlan) -> String {
288        match self {
289            ProfilePlan::Next {
290                change,
291                blocked: true,
292            } => format!("Once that is fixed: {}", change.line()),
293            ProfilePlan::Next { change, .. } => format!(
294                "Next look, within {}: {}",
295                super::period(plan.every),
296                change.line()
297            ),
298            ProfilePlan::Held { .. } => "No change".to_owned(),
299            ProfilePlan::LowerHeld { .. } => "No lowering".to_owned(),
300            ProfilePlan::Nothing => "Nothing to change".to_owned(),
301        }
302    }
303
304    /// Why, after when the hold's change was made for a held profile: the change's reason, `the interval holds it
305    /// until then`, `a raise may come before then`, or the advice's line.
306    pub fn reason(&self, sizing: &Sizing, profile: &ProfileSizing) -> String {
307        match self {
308            ProfilePlan::Next { change, .. } => change.reason.clone(),
309            ProfilePlan::Held { .. } => "the interval holds it until then".to_owned(),
310            ProfilePlan::LowerHeld { .. } => "a raise may come before then".to_owned(),
311            ProfilePlan::Nothing => sizing.advice_line(profile),
312        }
313    }
314
315    /// When the change that holds it was made, for a held profile.
316    pub fn changed(&self) -> Option<u64> {
317        match self {
318            ProfilePlan::Held { hold } | ProfilePlan::LowerHeld { hold } => Some(hold.changed),
319            ProfilePlan::Next { .. } | ProfilePlan::Nothing => None,
320        }
321    }
322}
323
324#[cfg(test)]
325mod tests;
runnerd_core/api/sizing/shown.rs

runnerd_core/api/sizing/
shown.rs
