Skip to main content

runnerd_core/api/sizing/
shown.rs

1//! What the sizing advice means as every client shows it: what a set of jobs needs and why, how the advice reads at
2//! a glance, what auto sizing does next with a profile at a given time, how much history it covers, and doctor's
3//! check of it. Plain derivations of [`Sizing`]; the daemon's advice uses [`need`] for its arithmetic.
4
5use super::{
6    Advice, AutoPlan, JobSizing, MemoryBound, MemoryHold, PAGE_CACHE_ROOM, Peaks, ProfileSizing,
7    Sizing, SizingChange, SizingHold, SizingRefused, Starts,
8};
9use crate::api::{
10    Check, CheckStatus, JOBS_HEADING, LabelledFigure, Tone, Toned, cores_used, duration,
11};
12use crate::config::ByteSize;
13
14/// Advice rounds memory up to a multiple of this: 256M.
15pub const GRANULE: u64 = 256 << 20;
16
17/// Seconds between two samples of a running job's container. Clients without a `Sizing` or `SampleSeries` that
18/// says so, such as a job's figures in `runnerctl`, word their sampling caveat from it.
19pub const SAMPLE_INTERVAL: u64 = 2;
20
21/// What a set of jobs whose largest sampled peak is `peak` needs, with `margin` percent: the peak plus the margin, or
22/// plus `PAGE_CACHE_ROOM` when that is more, rounded up to `GRANULE`. The largest, not a percentile: the cap kills a
23/// job that goes over it, so a cap below a peak a job reached kills it again.
24pub fn need(peak: u64, margin: u32) -> u64 {
25    let room = (peak.saturating_mul(u64::from(margin)) / 100).max(PAGE_CACHE_ROOM);
26    peak.saturating_add(room)
27        .div_ceil(GRANULE)
28        .saturating_mul(GRANULE)
29        .max(GRANULE)
30}
31
32/// The caveat on a sampled peak, as every client words it: `memory peaks are sampled every 2 s; spikes shorter than
33/// that do not show`.
34pub fn sampled_note(interval: u64) -> String {
35    format!("memory peaks are sampled every {interval} s; spikes shorter than that do not show")
36}
37
38/// What a set of jobs needs, as the advice sizes for it, from `Sizing::need`.
39#[derive(Debug, Clone, PartialEq, Eq)]
40pub struct Need {
41    /// In bytes.
42    pub memory: u64,
43    /// How it comes from the largest peak: `its largest peak + 15%` or `its largest peak + 1G for page cache`.
44    pub reason: String,
45}
46
47impl Sizing {
48    /// What `profile`'s jobs need: the advice's memory, or what a bound of the config holds it away from
49    /// (`ProfileSizing::hold`), or with too few jobs for advice, what it would size for.
50    pub fn need(&self, profile: &ProfileSizing) -> Need {
51        let peaks = &profile.peaks;
52        let memory = match profile.advice {
53            _ if let Some(hold) = profile.hold => hold.need,
54            Advice::Keep { memory }
55            | Advice::Lower { memory, .. }
56            | Advice::Raise { memory, .. }
57            | Advice::SliceTooSmall { memory } => memory,
58            Advice::TooFew { .. } => need(peaks.memory.max, self.margin),
59        };
60        Need {
61            memory,
62            reason: self.need_reason(peaks),
63        }
64    }
65
66    /// How the advice comes from `peaks`' largest: `its largest peak + 15%`, or `its largest peak + 1G for page
67    /// cache` when that is more.
68    pub(crate) fn need_reason(&self, peaks: &Peaks) -> String {
69        if peaks.memory.max * u64::from(self.margin) / 100 >= PAGE_CACHE_ROOM {
70            format!("its largest peak + {}%", self.margin)
71        } else {
72            format!(
73                "its largest peak + {} for page cache",
74                ByteSize(PAGE_CACHE_ROOM)
75            )
76        }
77    }
78
79    /// How the advice is made, as every client explains it: `peaks are sampled every 2 s, so a shorter spike goes
80    /// unseen: advice sizes for the largest + 15%, at least 1G for page cache, rounded up to 256M, once a profile ran
81    /// 10 jobs`.
82    pub fn method_line(&self) -> String {
83        format!(
84            "peaks are sampled every {} s, so a shorter spike goes unseen: advice sizes for the largest + {}%, at \
85             least {} for page cache, rounded up to {}, once a profile ran {} jobs",
86            self.sample_interval,
87            self.margin,
88            ByteSize(PAGE_CACHE_ROOM),
89            ByteSize(GRANULE),
90            self.min_jobs
91        )
92    }
93
94    /// `Sizing::method_line` in short parts for a heading, the sampling caveat first: `peaks sampled every 2 s`,
95    /// `+15%, at least 1G, in 256M steps` and `advice after 10 jobs`.
96    pub fn method_parts(&self) -> Vec<String> {
97        vec![
98            format!("peaks sampled every {} s", self.sample_interval),
99            format!(
100                "+{}%, at least {}, in {} steps",
101                self.margin,
102                ByteSize(PAGE_CACHE_ROOM),
103                ByteSize(GRANULE)
104            ),
105            format!("advice after {} jobs", self.min_jobs),
106        ]
107    }
108
109    /// The history it was computed from: `since` to when it was computed, however long ago that was.
110    pub fn covered(&self) -> u64 {
111        self.until.saturating_sub(self.since)
112    }
113
114    /// Doctor's check of it: a warning while a profile needs more memory than its cap, else the profiles whose
115    /// advice changes something, or that all fit.
116    pub fn doctor_check(&self) -> Check {
117        let changes: Vec<_> = self
118            .profiles
119            .iter()
120            .filter(|profile| {
121                profile.advice.change().is_some()
122                    || matches!(profile.advice, Advice::SliceTooSmall { .. })
123            })
124            .collect();
125        let raise = changes
126            .iter()
127            .any(|profile| profile.advice.tone() == Tone::Bad);
128        let detail = if changes.is_empty() {
129            let too_few = self
130                .profiles
131                .iter()
132                .any(|profile| matches!(profile.advice, Advice::TooFew { .. }));
133            if too_few {
134                format!(
135                    "some profiles ran fewer than {} jobs; runnerctl sizing shows their peaks",
136                    self.min_jobs
137                )
138            } else {
139                format!(
140                    "every profile's memory fits its jobs' largest peak + {}%, at least {} for page cache",
141                    self.margin,
142                    ByteSize(PAGE_CACHE_ROOM)
143                )
144            }
145        } else {
146            changes
147                .iter()
148                .map(|profile| format!("{}: {}", profile.name, self.advice_line(profile)))
149                .collect::<Vec<_>>()
150                .join("; ")
151        };
152        Check {
153            name: "sizing".to_owned(),
154            status: if raise {
155                CheckStatus::Warn
156            } else {
157                CheckStatus::Info
158            },
159            detail,
160            fix: (!changes.is_empty()).then(|| {
161                changes
162                    .iter()
163                    .map(|profile| match profile.advice {
164                        Advice::SliceTooSmall { .. } => match profile.held().map(|hold| hold.by) {
165                            Some(MemoryBound::Ceiling { .. }) => format!(
166                                "raise [host] memory_ceiling if the host can grow further, then: runnerctl \
167                                 sizing apply {}",
168                                profile.name
169                            ),
170                            Some(MemoryBound::MaxMemory | MemoryBound::MinMemory) => format!(
171                                "raise max_memory in [profiles.{0}], then: runnerctl sizing apply {0}",
172                                profile.name
173                            ),
174                            None => format!(
175                                "raise MemoryMax in the slice's unit file, sudo systemctl daemon-reload, then: \
176                                 runnerctl sizing apply {}",
177                                profile.name
178                            ),
179                        },
180                        _ => format!("runnerctl sizing apply {}", profile.name),
181                    })
182                    .collect::<Vec<_>>()
183                    .join("\n")
184            }),
185        }
186    }
187}
188
189impl Advice {
190    /// What it means for the reader: more memory needed is bad, a lowering worth a look, a fitting cap good, too
191    /// few jobs to tell in the background.
192    pub fn tone(&self) -> Tone {
193        match self {
194            Advice::Raise { .. } | Advice::SliceTooSmall { .. } => Tone::Bad,
195            Advice::Lower { .. } => Tone::Warn,
196            Advice::Keep { .. } => Tone::Good,
197            Advice::TooFew { .. } => Tone::Quiet,
198        }
199    }
200
201    /// It in a word or two, for a table cell: `too few`, `keep`, `grow slice`, or the new `9G × 2`.
202    pub fn word(&self) -> String {
203        match *self {
204            Advice::TooFew { .. } => "too few".to_owned(),
205            Advice::Keep { .. } => "keep".to_owned(),
206            Advice::SliceTooSmall { .. } => "grow slice".to_owned(),
207            Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => {
208                format!("{} × {slots}", ByteSize(memory))
209            }
210        }
211    }
212}
213
214impl ProfileSizing {
215    /// What keeps its `memory` where it is although its jobs need more, `[host] memory_ceiling` or `max_memory`: no
216    /// raise is possible until that grows. `None` while a raise is possible or the slice is what has to grow.
217    pub fn held(&self) -> Option<MemoryHold> {
218        self.hold.filter(|hold| {
219            matches!(self.advice, Advice::SliceTooSmall { .. })
220                && hold.by != MemoryBound::MinMemory
221                && self.memory >= hold.bound
222        })
223    }
224
225    /// Its figures, in the order they show: its jobs, how busy its slots were, its jobs' p50 and p95 wait, its warm
226    /// starts out of all, and its jobs' CPU p99 out of its `cpus`; `–` where no job took one.
227    pub fn figures(&self) -> Vec<LabelledFigure> {
228        let starts = &self.starts;
229        let warm = if starts.warm > 0 {
230            LabelledFigure::new("Warm starts", starts.warm.to_string())
231                .of(format!("of {}", starts.total()))
232        } else {
233            LabelledFigure::new("Warm starts", "–")
234        };
235        let cpu = match self.peaks.cpu_p99() {
236            Some(p99) => {
237                LabelledFigure::new("CPU p99", cores_used(p99)).of(format!("of {}", self.cpus))
238            }
239            None => LabelledFigure::new("CPU p99", "–"),
240        };
241        vec![
242            LabelledFigure::new(JOBS_HEADING, starts.total().to_string()),
243            LabelledFigure::new("Slots busy", format!("{}%", self.busy)),
244            LabelledFigure::new(
245                "Waited p50 / p95",
246                format!("{} / {}", duration(self.wait.p50), duration(self.wait.p95)),
247            ),
248            warm,
249            cpu,
250        ]
251    }
252
253    /// Its advice in a word or two, for a table cell: `Advice::word`, or `at ceiling` or `at max_memory` while that
254    /// holds its `memory` where it is.
255    pub fn advice_word(&self) -> String {
256        match self.held().map(|hold| hold.by) {
257            Some(MemoryBound::Ceiling { .. }) => "at ceiling".to_owned(),
258            Some(MemoryBound::MaxMemory | MemoryBound::MinMemory) => "at max_memory".to_owned(),
259            None => self.advice.word(),
260        }
261    }
262}
263
264impl JobSizing {
265    /// What its jobs need, for a table cell, and what that means: `max 3G` with too few runs, `9G > cap` past its
266    /// profile's cap, else `6G`.
267    pub fn needs(&self) -> Toned {
268        let (text, tone) = match self.advice {
269            Advice::TooFew { .. } => (
270                format!("max {}", ByteSize(self.peaks.memory.max)),
271                Tone::Quiet,
272            ),
273            Advice::Raise { memory, .. } | Advice::SliceTooSmall { memory } => {
274                (format!("{} > cap", ByteSize(memory)), Tone::Bad)
275            }
276            Advice::Keep { memory } | Advice::Lower { memory, .. } => {
277                (ByteSize(memory).to_string(), Tone::Neutral)
278            }
279        };
280        Toned { text, tone }
281    }
282}
283
284impl Peaks {
285    /// The CPU p99 of the runs at the profile's `cpus` now; `None` without one, where it would read 0.
286    pub fn cpu_p99(&self) -> Option<u64> {
287        (self.cpu_runs > 0).then_some(self.cpu.p99)
288    }
289
290    /// Which runs the CPU percentiles are over, for a profile at `cpus`, as every client says it: `of the 4 at 8
291    /// CPUs`, or without one `no run at 8 CPUs yet`.
292    pub fn cpu_runs_line(&self, cpus: u32) -> String {
293        match self.cpu_runs {
294            0 => format!("no run at {cpus} CPUs yet"),
295            runs => format!("of the {runs} at {cpus} CPUs"),
296        }
297    }
298}
299
300impl Starts {
301    /// Every job counted: warm and cold starts.
302    pub fn total(&self) -> u32 {
303        self.warm + self.cold
304    }
305}
306
307/// What auto sizing does next with one profile at a given time, from `AutoPlan::profile_plan`.
308#[derive(Debug, Clone, Copy, PartialEq, Eq)]
309pub enum ProfilePlan<'plan> {
310    /// The next look makes this change, or would once `AutoPlan::blocked` is fixed.
311    Next {
312        /// The change.
313        change: &'plan SizingChange,
314        /// Whether `AutoPlan::blocked` keeps it from being made.
315        blocked: bool,
316    },
317    /// A recent change holds it from any change until `hold.until`.
318    Held {
319        /// The hold.
320        hold: &'plan SizingHold,
321    },
322    /// A recent change holds it from a lowering until `hold.lowered_after()`; a raise may come before.
323    LowerHeld {
324        /// The hold.
325        hold: &'plan SizingHold,
326    },
327    /// Nothing: the advice asks for no change, and no hold is left.
328    Nothing,
329}
330
331impl AutoPlan {
332    /// The `cpus` and `slots` raises `profile`'s jobs ask for and the next look cannot make, each a line of its own
333    /// beside what it does next, which can still change its memory or slots: `No CPU raise: 3 runs of … the cap holds
334    /// them back; …`, then `No slot raise: 12 jobs waited 300s for a slot …; it is at its max_slots, 12`.
335    pub fn refusals(&self, profile: &str) -> Vec<String> {
336        let lines = |refused: &[SizingRefused], what: &str| -> Vec<String> {
337            refused
338                .iter()
339                .filter(|refused| refused.profile == profile)
340                .map(|refused| format!("No {what} raise: {}", refused.reason))
341                .collect()
342        };
343        let mut refusals = lines(&self.refused, "CPU");
344        refusals.extend(lines(&self.refused_slots, "slot"));
345        refusals
346    }
347
348    /// What keeps every look from changing anything, as clients say it over the profiles: `No look changes anything
349    /// now: the file changed on disk since runnerd read it; …`. `None` while the next look may write its changes.
350    pub fn blocked_line(&self) -> Option<String> {
351        self.blocked
352            .as_ref()
353            .map(|blocked| format!("No look changes anything now: {blocked}"))
354    }
355
356    /// What it does next with `profile` at `now`: a hold whose time has passed holds nothing, however old the plan.
357    pub fn profile_plan(&self, profile: &str, now: u64) -> ProfilePlan<'_> {
358        if let Some(change) = self.next.iter().find(|change| change.profile == profile) {
359            return ProfilePlan::Next {
360                change,
361                blocked: self.blocked.is_some(),
362            };
363        }
364        match self.held.iter().find(|hold| hold.profile == profile) {
365            Some(hold) if now < hold.until => ProfilePlan::Held { hold },
366            Some(hold) if now < hold.lowered_after() => ProfilePlan::LowerHeld { hold },
367            _ => ProfilePlan::Nothing,
368        }
369    }
370}
371
372impl ProfilePlan<'_> {
373    /// When what it says ends: the hold's end, for a held profile.
374    pub fn until(&self) -> Option<u64> {
375        match self {
376            ProfilePlan::Held { hold } => Some(hold.until),
377            ProfilePlan::LowerHeld { hold } => Some(hold.lowered_after()),
378            ProfilePlan::Next { .. } | ProfilePlan::Nothing => None,
379        }
380    }
381
382    /// What it does, as the lead of its line, before `until` when it has one: `Next look, within 5 minutes: memory
383    /// 9G → 6.75G, slots 2`, `Once that is fixed: …`, `No change`, `No lowering` or `Nothing to change`.
384    pub fn headline(&self, plan: &AutoPlan) -> String {
385        match self {
386            ProfilePlan::Next {
387                change,
388                blocked: true,
389            } => format!("Once that is fixed: {}", change.line()),
390            ProfilePlan::Next { change, .. } => format!(
391                "Next look, within {}: {}",
392                super::period(plan.every),
393                change.line()
394            ),
395            ProfilePlan::Held { .. } => "No change".to_owned(),
396            ProfilePlan::LowerHeld { .. } => "No lowering".to_owned(),
397            ProfilePlan::Nothing => "Nothing to change".to_owned(),
398        }
399    }
400
401    /// Why, after when the hold's change was made for a held profile: the change's reason, `the interval holds it
402    /// until then`, `a raise may come before then`, or the advice's line.
403    pub fn reason(&self, sizing: &Sizing, profile: &ProfileSizing) -> String {
404        match self {
405            ProfilePlan::Next { change, .. } => change.reason.clone(),
406            ProfilePlan::Held { .. } => "the interval holds it until then".to_owned(),
407            ProfilePlan::LowerHeld { .. } => "a raise may come before then".to_owned(),
408            ProfilePlan::Nothing => sizing.advice_line(profile),
409        }
410    }
411
412    /// When the change that holds it was made, for a held profile.
413    pub fn changed(&self) -> Option<u64> {
414        match self {
415            ProfilePlan::Held { hold } | ProfilePlan::LowerHeld { hold } => Some(hold.changed),
416            ProfilePlan::Next { .. } | ProfilePlan::Nothing => None,
417        }
418    }
419}
420
421#[cfg(test)]
422mod tests;