Skip to main content

runnerd_core/api/sizing/
shown.rs

1//! What the sizing advice means as every client shows it: what a set of jobs needs and why, how the advice reads at
2//! a glance, what auto sizing does next with a profile at a given time, how much history it covers, and doctor's
3//! check of it. Plain derivations of [`Sizing`]; the daemon's advice uses [`need`] for its arithmetic.
4
5use super::{
6    Advice, AutoPlan, JobSizing, MemoryBound, MemoryHold, PAGE_CACHE_ROOM, Peaks, ProfileSizing,
7    Sizing, SizingChange, SizingHold, Starts,
8};
9use crate::api::{Check, CheckStatus, Tone, Toned};
10use crate::config::ByteSize;
11
12/// Advice rounds memory up to a multiple of this: 256M.
13pub const GRANULE: u64 = 256 << 20;
14
15/// Seconds between two samples of a running job's container. Clients without a `Sizing` or `SampleSeries` that
16/// says so, such as a job's figures in `runnerctl`, word their sampling caveat from it.
17pub const SAMPLE_INTERVAL: u64 = 2;
18
19/// What a set of jobs whose largest sampled peak is `peak` needs, with `margin` percent: the peak plus the margin, or
20/// plus `PAGE_CACHE_ROOM` when that is more, rounded up to `GRANULE`. The largest, not a percentile: the cap kills a
21/// job that goes over it, so a cap below a peak a job reached kills it again.
22pub fn need(peak: u64, margin: u32) -> u64 {
23    let room = (peak.saturating_mul(u64::from(margin)) / 100).max(PAGE_CACHE_ROOM);
24    peak.saturating_add(room)
25        .div_ceil(GRANULE)
26        .saturating_mul(GRANULE)
27        .max(GRANULE)
28}
29
30/// The caveat on a sampled peak, as every client words it: `memory peaks are sampled every 2 s; spikes shorter than
31/// that do not show`.
32pub fn sampled_note(interval: u64) -> String {
33    format!("memory peaks are sampled every {interval} s; spikes shorter than that do not show")
34}
35
36/// What a set of jobs needs, as the advice sizes for it, from `Sizing::need`.
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct Need {
39    /// In bytes.
40    pub memory: u64,
41    /// How it comes from the largest peak: `its largest peak + 15%` or `its largest peak + 1G for page cache`.
42    pub reason: String,
43}
44
45impl Sizing {
46    /// What `profile`'s jobs need: the advice's memory, or what a bound of the config holds it away from
47    /// (`ProfileSizing::hold`), or with too few jobs for advice, what it would size for.
48    pub fn need(&self, profile: &ProfileSizing) -> Need {
49        let peaks = &profile.peaks;
50        let memory = match profile.advice {
51            _ if let Some(hold) = profile.hold => hold.need,
52            Advice::Keep { memory }
53            | Advice::Lower { memory, .. }
54            | Advice::Raise { memory, .. }
55            | Advice::SliceTooSmall { memory } => memory,
56            Advice::TooFew { .. } => need(peaks.memory.max, self.margin),
57        };
58        Need {
59            memory,
60            reason: self.need_reason(peaks),
61        }
62    }
63
64    /// How the advice comes from `peaks`' largest: `its largest peak + 15%`, or `its largest peak + 1G for page
65    /// cache` when that is more.
66    pub(crate) fn need_reason(&self, peaks: &Peaks) -> String {
67        if peaks.memory.max * u64::from(self.margin) / 100 >= PAGE_CACHE_ROOM {
68            format!("its largest peak + {}%", self.margin)
69        } else {
70            format!(
71                "its largest peak + {} for page cache",
72                ByteSize(PAGE_CACHE_ROOM)
73            )
74        }
75    }
76
77    /// How the advice is made, as every client explains it: `peaks are sampled every 2 s, so a shorter spike goes
78    /// unseen: advice sizes for the largest + 15%, at least 1G for page cache, rounded up to 256M, once a profile ran
79    /// 10 jobs`.
80    pub fn method_line(&self) -> String {
81        format!(
82            "peaks are sampled every {} s, so a shorter spike goes unseen: advice sizes for the largest + {}%, at \
83             least {} for page cache, rounded up to {}, once a profile ran {} jobs",
84            self.sample_interval,
85            self.margin,
86            ByteSize(PAGE_CACHE_ROOM),
87            ByteSize(GRANULE),
88            self.min_jobs
89        )
90    }
91
92    /// `Sizing::method_line` in short parts for a heading, the sampling caveat first: `peaks sampled every 2 s`,
93    /// `+15%, at least 1G, in 256M steps` and `advice after 10 jobs`.
94    pub fn method_parts(&self) -> Vec<String> {
95        vec![
96            format!("peaks sampled every {} s", self.sample_interval),
97            format!(
98                "+{}%, at least {}, in {} steps",
99                self.margin,
100                ByteSize(PAGE_CACHE_ROOM),
101                ByteSize(GRANULE)
102            ),
103            format!("advice after {} jobs", self.min_jobs),
104        ]
105    }
106
107    /// The history it was computed from: `since` to when it was computed, however long ago that was.
108    pub fn covered(&self) -> u64 {
109        self.until.saturating_sub(self.since)
110    }
111
112    /// Doctor's check of it: a warning while a profile needs more memory than its cap, else the profiles whose
113    /// advice changes something, or that all fit.
114    pub fn doctor_check(&self) -> Check {
115        let changes: Vec<_> = self
116            .profiles
117            .iter()
118            .filter(|profile| {
119                profile.advice.change().is_some()
120                    || matches!(profile.advice, Advice::SliceTooSmall { .. })
121            })
122            .collect();
123        let raise = changes
124            .iter()
125            .any(|profile| profile.advice.tone() == Tone::Bad);
126        let detail = if changes.is_empty() {
127            let too_few = self
128                .profiles
129                .iter()
130                .any(|profile| matches!(profile.advice, Advice::TooFew { .. }));
131            if too_few {
132                format!(
133                    "some profiles ran fewer than {} jobs; runnerctl sizing shows their peaks",
134                    self.min_jobs
135                )
136            } else {
137                format!(
138                    "every profile's memory fits its jobs' largest peak + {}%, at least {} for page cache",
139                    self.margin,
140                    ByteSize(PAGE_CACHE_ROOM)
141                )
142            }
143        } else {
144            changes
145                .iter()
146                .map(|profile| format!("{}: {}", profile.name, self.advice_line(profile)))
147                .collect::<Vec<_>>()
148                .join("; ")
149        };
150        Check {
151            name: "sizing".to_owned(),
152            status: if raise {
153                CheckStatus::Warn
154            } else {
155                CheckStatus::Info
156            },
157            detail,
158            fix: (!changes.is_empty()).then(|| {
159                changes
160                    .iter()
161                    .map(|profile| match profile.advice {
162                        Advice::SliceTooSmall { .. } => match profile.held().map(|hold| hold.by) {
163                            Some(MemoryBound::Ceiling { .. }) => format!(
164                                "raise [host] memory_ceiling if the host can grow further, then: runnerctl \
165                                 sizing apply {}",
166                                profile.name
167                            ),
168                            Some(MemoryBound::MaxMemory | MemoryBound::MinMemory) => format!(
169                                "raise max_memory in [profiles.{0}], then: runnerctl sizing apply {0}",
170                                profile.name
171                            ),
172                            None => format!(
173                                "raise MemoryMax in the slice's unit file, sudo systemctl daemon-reload, then: \
174                                 runnerctl sizing apply {}",
175                                profile.name
176                            ),
177                        },
178                        _ => format!("runnerctl sizing apply {}", profile.name),
179                    })
180                    .collect::<Vec<_>>()
181                    .join("\n")
182            }),
183        }
184    }
185}
186
187impl Advice {
188    /// What it means for the reader: more memory needed is bad, a lowering worth a look, a fitting cap good, too
189    /// few jobs to tell in the background.
190    pub fn tone(&self) -> Tone {
191        match self {
192            Advice::Raise { .. } | Advice::SliceTooSmall { .. } => Tone::Bad,
193            Advice::Lower { .. } => Tone::Warn,
194            Advice::Keep { .. } => Tone::Good,
195            Advice::TooFew { .. } => Tone::Quiet,
196        }
197    }
198
199    /// It in a word or two, for a table cell: `too few`, `keep`, `grow slice`, or the new `9G × 2`.
200    pub fn word(&self) -> String {
201        match *self {
202            Advice::TooFew { .. } => "too few".to_owned(),
203            Advice::Keep { .. } => "keep".to_owned(),
204            Advice::SliceTooSmall { .. } => "grow slice".to_owned(),
205            Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => {
206                format!("{} × {slots}", ByteSize(memory))
207            }
208        }
209    }
210}
211
212impl ProfileSizing {
213    /// What keeps its `memory` where it is although its jobs need more, `[host] memory_ceiling` or `max_memory`: no
214    /// raise is possible until that grows. `None` while a raise is possible or the slice is what has to grow.
215    pub fn held(&self) -> Option<MemoryHold> {
216        self.hold.filter(|hold| {
217            matches!(self.advice, Advice::SliceTooSmall { .. })
218                && hold.by != MemoryBound::MinMemory
219                && self.memory >= hold.bound
220        })
221    }
222
223    /// Its advice in a word or two, for a table cell: `Advice::word`, or `at ceiling` or `at max_memory` while that
224    /// holds its `memory` where it is.
225    pub fn advice_word(&self) -> String {
226        match self.held().map(|hold| hold.by) {
227            Some(MemoryBound::Ceiling { .. }) => "at ceiling".to_owned(),
228            Some(MemoryBound::MaxMemory | MemoryBound::MinMemory) => "at max_memory".to_owned(),
229            None => self.advice.word(),
230        }
231    }
232}
233
234impl JobSizing {
235    /// What its jobs need, for a table cell, and what that means: `max 3G` with too few runs, `9G > cap` past its
236    /// profile's cap, else `6G`.
237    pub fn needs(&self) -> Toned {
238        let (text, tone) = match self.advice {
239            Advice::TooFew { .. } => (
240                format!("max {}", ByteSize(self.peaks.memory.max)),
241                Tone::Quiet,
242            ),
243            Advice::Raise { memory, .. } | Advice::SliceTooSmall { memory } => {
244                (format!("{} > cap", ByteSize(memory)), Tone::Bad)
245            }
246            Advice::Keep { memory } | Advice::Lower { memory, .. } => {
247                (ByteSize(memory).to_string(), Tone::Neutral)
248            }
249        };
250        Toned { text, tone }
251    }
252}
253
254impl Peaks {
255    /// The CPU p99 of the runs at the profile's `cpus` now; `None` without one, where it would read 0.
256    pub fn cpu_p99(&self) -> Option<u64> {
257        (self.cpu_runs > 0).then_some(self.cpu.p99)
258    }
259
260    /// Which runs the CPU percentiles are over, for a profile at `cpus`, as every client says it: `of the 4 at 8
261    /// CPUs`, or without one `no run at 8 CPUs yet`.
262    pub fn cpu_runs_line(&self, cpus: u32) -> String {
263        match self.cpu_runs {
264            0 => format!("no run at {cpus} CPUs yet"),
265            runs => format!("of the {runs} at {cpus} CPUs"),
266        }
267    }
268}
269
270impl Starts {
271    /// Every job counted: warm and cold starts.
272    pub fn total(&self) -> u32 {
273        self.warm + self.cold
274    }
275}
276
277/// What auto sizing does next with one profile at a given time, from `AutoPlan::profile_plan`.
278#[derive(Debug, Clone, Copy, PartialEq, Eq)]
279pub enum ProfilePlan<'plan> {
280    /// The next look makes this change, or would once `AutoPlan::blocked` is fixed.
281    Next {
282        /// The change.
283        change: &'plan SizingChange,
284        /// Whether `AutoPlan::blocked` keeps it from being made.
285        blocked: bool,
286    },
287    /// A recent change holds it from any change until `hold.until`.
288    Held {
289        /// The hold.
290        hold: &'plan SizingHold,
291    },
292    /// A recent change holds it from a lowering until `hold.lowered_after()`; a raise may come before.
293    LowerHeld {
294        /// The hold.
295        hold: &'plan SizingHold,
296    },
297    /// Nothing: the advice asks for no change, and no hold is left.
298    Nothing,
299}
300
301impl AutoPlan {
302    /// The `cpus` raise `profile`'s jobs ask for and the next look cannot make, as a line of its own beside what it
303    /// does next, which can still change its memory or slots: `No CPU raise: 3 runs of … the cap holds them back; …`.
304    pub fn refusal(&self, profile: &str) -> Option<String> {
305        self.refused
306            .iter()
307            .find(|refused| refused.profile == profile)
308            .map(|refused| format!("No CPU raise: {}", refused.reason))
309    }
310
311    /// What keeps every look from changing anything, as clients say it over the profiles: `No look changes anything
312    /// now: the file changed on disk since runnerd read it; …`. `None` while the next look may write its changes.
313    pub fn blocked_line(&self) -> Option<String> {
314        self.blocked
315            .as_ref()
316            .map(|blocked| format!("No look changes anything now: {blocked}"))
317    }
318
319    /// What it does next with `profile` at `now`: a hold whose time has passed holds nothing, however old the plan.
320    pub fn profile_plan(&self, profile: &str, now: u64) -> ProfilePlan<'_> {
321        if let Some(change) = self.next.iter().find(|change| change.profile == profile) {
322            return ProfilePlan::Next {
323                change,
324                blocked: self.blocked.is_some(),
325            };
326        }
327        match self.held.iter().find(|hold| hold.profile == profile) {
328            Some(hold) if now < hold.until => ProfilePlan::Held { hold },
329            Some(hold) if now < hold.lowered_after() => ProfilePlan::LowerHeld { hold },
330            _ => ProfilePlan::Nothing,
331        }
332    }
333}
334
335impl ProfilePlan<'_> {
336    /// When what it says ends: the hold's end, for a held profile.
337    pub fn until(&self) -> Option<u64> {
338        match self {
339            ProfilePlan::Held { hold } => Some(hold.until),
340            ProfilePlan::LowerHeld { hold } => Some(hold.lowered_after()),
341            ProfilePlan::Next { .. } | ProfilePlan::Nothing => None,
342        }
343    }
344
345    /// What it does, as the lead of its line, before `until` when it has one: `Next look, within 5 minutes: memory
346    /// 9G → 6.75G, slots 2`, `Once that is fixed: …`, `No change`, `No lowering` or `Nothing to change`.
347    pub fn headline(&self, plan: &AutoPlan) -> String {
348        match self {
349            ProfilePlan::Next {
350                change,
351                blocked: true,
352            } => format!("Once that is fixed: {}", change.line()),
353            ProfilePlan::Next { change, .. } => format!(
354                "Next look, within {}: {}",
355                super::period(plan.every),
356                change.line()
357            ),
358            ProfilePlan::Held { .. } => "No change".to_owned(),
359            ProfilePlan::LowerHeld { .. } => "No lowering".to_owned(),
360            ProfilePlan::Nothing => "Nothing to change".to_owned(),
361        }
362    }
363
364    /// Why, after when the hold's change was made for a held profile: the change's reason, `the interval holds it
365    /// until then`, `a raise may come before then`, or the advice's line.
366    pub fn reason(&self, sizing: &Sizing, profile: &ProfileSizing) -> String {
367        match self {
368            ProfilePlan::Next { change, .. } => change.reason.clone(),
369            ProfilePlan::Held { .. } => "the interval holds it until then".to_owned(),
370            ProfilePlan::LowerHeld { .. } => "a raise may come before then".to_owned(),
371            ProfilePlan::Nothing => sizing.advice_line(profile),
372        }
373    }
374
375    /// When the change that holds it was made, for a held profile.
376    pub fn changed(&self) -> Option<u64> {
377        match self {
378            ProfilePlan::Held { hold } | ProfilePlan::LowerHeld { hold } => Some(hold.changed),
379            ProfilePlan::Next { .. } | ProfilePlan::Nothing => None,
380        }
381    }
382}
383
384#[cfg(test)]
385mod tests;