Skip to main content

runnerd_core/api/
sizing.rs

1//! Right-sizing as clients read it: per profile and per job (repository, workflow and job name), the spread of the
2//! job history's sampled memory and CPU peaks, how long jobs waited for a slot, how busy the slots were, and what
3//! to set each profile's `memory` and `slots` to. Plain data; the daemon computes it from the history. Sizes are
4//! bytes, CPU millicores, times seconds since the Unix epoch.
5
6use serde::{Deserialize, Serialize};
7
8use crate::api::AdmissionPolicy;
9use crate::config::ByteSize;
10
11mod shown;
12
13pub use shown::{GRANULE, Need, ProfilePlan, SAMPLE_INTERVAL, need, sampled_note};
14
15/// `GET /v1/sizing` query parameters.
16#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
17pub struct SizingQuery {
18    /// Only jobs queued at or after this time; default 30 days ago.
19    pub since: Option<u64>,
20}
21
22/// `GET /v1/sizing`: what the job history says each configured profile needs.
23#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
24pub struct Sizing {
25    /// The start of the history read: `since`, or the first runner recorded after it.
26    pub since: u64,
27    /// When it was computed.
28    pub until: u64,
29    /// The percent added to a sampled peak, `[sizing] margin`: peaks are sampled every `sample_interval`
30    /// seconds, so a shorter spike between two samples goes unseen.
31    pub margin: u32,
32    /// Seconds between two samples of a job's container.
33    pub sample_interval: u64,
34    /// The fewest jobs a profile or job needs before its percentiles become advice, `[sizing] min_jobs`.
35    pub min_jobs: u32,
36    /// The slice's `MemoryMax`; `None` when it is unlimited.
37    pub memory_max: Option<u64>,
38    /// `[host] admission`, which the advice budgets `MemoryMax` by.
39    pub admission: AdmissionPolicy,
40    /// Every configured profile, by name.
41    pub profiles: Vec<ProfileSizing>,
42    /// Whether the daemon applies advice itself, `[sizing] auto`.
43    pub auto: bool,
44    /// The changes auto sizing made since `since`, oldest first.
45    pub changes: Vec<SizingChange>,
46    /// Under auto sizing, what it does next and when. `None` while it is off.
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub auto_plan: Option<AutoPlan>,
49}
50
51/// What auto sizing will do: its rules, the changes its next look would make as the history and config are now, and
52/// the profiles a recent change holds.
53#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54pub struct AutoPlan {
55    /// Seconds between two looks of the daemon's.
56    pub every: u64,
57    /// `[sizing] interval`: the least time between a change to one profile and one that lowers it, in seconds.
58    pub interval: u64,
59    /// `[sizing] raise_interval`: the least time between a change to one profile and one that only raises it, in
60    /// seconds.
61    pub raise_interval: u64,
62    /// `[sizing] max_step`: the most one change moves a profile's `memory`, in percent of it.
63    pub max_step: u32,
64    /// The changes the next look would make, in the order it would make them; `at` is when this was worked out.
65    /// None of them is made while `blocked` says why.
66    pub next: Vec<SizingChange>,
67    /// The profiles a change in the last `interval` holds from some change, the hold that ends first first.
68    pub held: Vec<SizingHold>,
69    /// Why no look changes anything until the owner acts, as when the file on disk differs from the config runnerd
70    /// runs with. `None` when the next look may write `next`.
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub blocked: Option<String>,
73    /// The profiles whose jobs the CPU cap holds back and whose `cpus` the next look cannot raise, and why.
74    #[serde(default, skip_serializing_if = "Vec::is_empty")]
75    pub refused: Vec<SizingRefused>,
76    /// Under `measured` admission, the profiles whose jobs waited for a slot alone and whose `slots` the next look
77    /// cannot raise, and why (ADR 0090).
78    #[serde(default, skip_serializing_if = "Vec::is_empty")]
79    pub refused_slots: Vec<SizingRefused>,
80    /// Under `measured` admission, the profiles whose jobs waited for a slot alone (`ProfileSizing::wants_slots`)
81    /// while those queued since auto sizing last changed them did not: a look reads only those, so it adds no slot
82    /// for the waits before the change.
83    #[serde(default, skip_serializing_if = "Vec::is_empty")]
84    pub waited_before_change: Vec<String>,
85}
86
87/// A raise of a profile's `cpus` or `slots` its jobs ask for that auto sizing cannot make, as when it does not fit
88/// the CPU budget or Docker's disk.
89#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
90pub struct SizingRefused {
91    /// The profile.
92    pub profile: String,
93    /// Which job the cap holds back, or how long its jobs waited for a slot, and what keeps it from a raise.
94    pub reason: String,
95}
96
97impl SizingRefused {
98    /// `profile`'s raise, refused for `reason`.
99    pub fn new(profile: &str, reason: String) -> SizingRefused {
100        SizingRefused {
101            profile: profile.to_owned(),
102            reason,
103        }
104    }
105}
106
107/// A profile auto sizing changes nothing in before `until`, and lowers nothing in before `lower_until`.
108#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
109pub struct SizingHold {
110    /// The profile.
111    pub profile: String,
112    /// When its last change was made.
113    pub changed: u64,
114    /// When it may be raised again, in seconds since the Unix epoch.
115    pub until: u64,
116    /// When it may be lowered again.
117    pub lower_until: u64,
118}
119
120impl SizingHold {
121    /// When it may be lowered again.
122    pub fn lowered_after(&self) -> u64 {
123        self.lower_until.max(self.until)
124    }
125}
126
127/// A profile's `memory` and `slots` as auto sizing changed them in the config file.
128#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
129pub struct SizingChange {
130    /// When it was written.
131    pub at: u64,
132    /// The profile.
133    pub profile: String,
134    /// What they were: writing these back undoes it.
135    pub from: Resize,
136    /// What they are now.
137    pub to: Resize,
138    /// Why, as the advice said it then.
139    pub reason: String,
140    /// Whether it may not have been written: recorded before its file was written, and the config no longer said
141    /// whether it was when runnerd looked again. It still holds the interval, and `from` is still what it replaced.
142    pub uncertain: bool,
143}
144
145/// One profile: its limits now, its jobs' peaks and waits, and the advice.
146#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
147pub struct ProfileSizing {
148    /// The profile.
149    pub name: String,
150    /// Its `memory` now: each container's hard limit.
151    pub memory: u64,
152    /// Its `cpus` now.
153    pub cpus: u32,
154    /// Its `slots` now.
155    pub slots: u32,
156    /// Its jobs' peaks.
157    pub peaks: Peaks,
158    /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
159    /// ran (ADR 0081).
160    pub wait: Spread,
161    /// Seconds of that wait with no slot of the profile free while the slice's memory, the host's and the CPU budget
162    /// had room for the job's runner: what one more slot would have saved (ADR 0090).
163    #[serde(default)]
164    pub slot_wait: Spread,
165    /// Its jobs by whether a warm runner took them, and how long each kind waited.
166    pub starts: Starts,
167    /// The percent of the time since `since` its slots held a runner for a job, from when the job was queued, over
168    /// its `slots` now.
169    pub busy: u32,
170    /// Whether more slots would help: under `measured` admission its jobs waited a minute or more for a slot alone at
171    /// p95 (`slot_wait`); under `caps` they waited that long at all (`wait`), or its slots were busy half the time.
172    pub wants_slots: bool,
173    /// What to set `memory` and `slots` to. Sized for its largest peak, so no job gets less than it used.
174    pub advice: Advice,
175    /// When the advice is to take back a lowering auto sizing made that went too far: what showed it.
176    #[serde(default, skip_serializing_if = "Option::is_none")]
177    pub revert: Option<SizingRevert>,
178    /// When a bound of the config holds the advice's `memory` away from what its jobs need: `[host]
179    /// memory_ceiling` or `max_memory` under it, `min_memory` over it (ADR 0086, 0088).
180    #[serde(default, skip_serializing_if = "Option::is_none")]
181    pub hold: Option<MemoryHold>,
182    /// Each job that ran in it, by repository, workflow and name.
183    pub jobs: Vec<JobSizing>,
184}
185
186/// A bound of the config holding a profile's `memory` away from what its jobs need: the config check refuses a
187/// profile past it, so the advice goes no further than `bound`, and auto sizing writes it as is (ADR 0086, 0088). Under
188/// a raise it says only that no raise past `bound` is possible, not that the jobs fit.
189#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
190pub struct MemoryHold {
191    /// What the jobs need, in bytes: the raise they ask for, the `memory` a revert goes back to, or the lowering's
192    /// floor.
193    pub need: u64,
194    /// The `memory` the advice goes to at most, or for a lowering at least.
195    pub bound: u64,
196    /// Which bound it is.
197    pub by: MemoryBound,
198}
199
200/// Which bound of the config holds a profile's `memory`.
201#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
202#[serde(rename_all = "kebab-case", tag = "kind")]
203pub enum MemoryBound {
204    /// `[host] memory_ceiling` less the mirror, down to a multiple of 256M so that the config file says it exactly:
205    /// the most one runner may have (ADR 0086).
206    Ceiling {
207        /// `[host] memory_ceiling`, in bytes.
208        ceiling: u64,
209        /// The mirror's memory beside it, 0 while the mirror is off.
210        mirror: u64,
211    },
212    /// The profile's `max_memory`.
213    MaxMemory,
214    /// The profile's `min_memory`.
215    MinMemory,
216}
217
218/// A lowering of a profile's `memory` by auto sizing that went too far, which the advice takes back (ADR 0070).
219#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
220pub struct SizingRevert {
221    /// The `memory` it lowered to, in bytes.
222    pub from: u64,
223    /// The `memory` it lowered from, which the advice goes back to.
224    pub to: u64,
225    /// The job that showed it, as `owner/name workflow / job`.
226    pub job: String,
227    /// How.
228    pub cause: RevertCause,
229}
230
231/// What showed a lowering went too far.
232#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
233#[serde(rename_all = "kebab-case", tag = "kind")]
234pub enum RevertCause {
235    /// A run met the lowered cap.
236    Met,
237    /// The job, which succeeded before the lowering, failed this many times in a row at it, in different workflow
238    /// runs and on more than one branch: a heap sized from the cgroup's limit fails without the kernel's limit.
239    Failed {
240        /// How many.
241        runs: u32,
242    },
243}
244
245/// A profile's jobs by whether their runner was already up when GitHub queued them (ADR 0050).
246#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
247pub struct Starts {
248    /// Jobs that started on a warm runner: one that started before the job was queued.
249    pub warm: u32,
250    /// Their median wait from queued to started, in seconds.
251    pub warm_p50: u64,
252    /// Jobs whose runner started after they were queued.
253    pub cold: u32,
254    /// Their median wait, in seconds.
255    pub cold_p50: u64,
256}
257
258/// The jobs of one name in one workflow of one repository.
259#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
260pub struct JobSizing {
261    /// `owner/name` on GitHub.
262    pub repository: String,
263    /// The workflow's name, when the history knew it.
264    pub workflow: Option<String>,
265    /// The job's name in the workflow.
266    pub name: String,
267    /// Its runs' peaks.
268    pub peaks: Peaks,
269    /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
270    /// ran (ADR 0081).
271    pub wait: Spread,
272    /// What its profile's `memory` would be for this job alone, and the `slots` that would fit in the memory the
273    /// profile's slots take now.
274    pub advice: Advice,
275}
276
277/// The peaks of a set of jobs, each the highest of its samples.
278#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
279pub struct Peaks {
280    /// How many jobs: the number of peaks the percentiles are taken over.
281    pub jobs: u64,
282    /// Of those, how many met their container's memory cap: its own limit failed reclaim (for a run recorded
283    /// before that was told apart, a kill or a limit hit in the container, or without events a peak within 5% of it).
284    /// They may need more than they show.
285    pub at_cap: u64,
286    /// How many of their jobs are on their cap: one of their runs met it or was killed for memory, and fewer than 3
287    /// successful runs since were clear of it. No lowering while any is.
288    #[serde(default, skip_serializing_if = "is_zero")]
289    pub on_cap: u64,
290    /// Memory without page cache, in bytes.
291    pub memory: Spread,
292    /// CPU, in millicores, of the runs at the profile's `cpus` now: a run at a larger cap may have peaked past them.
293    pub cpu: Spread,
294    /// How many runs `cpu` is over: those of `jobs` at the profile's `cpus` now.
295    #[serde(default)]
296    pub cpu_runs: u64,
297}
298
299#[expect(
300    clippy::trivially_copy_pass_by_ref,
301    reason = "serde's skip_serializing_if passes a reference"
302)]
303fn is_zero(count: &u64) -> bool {
304    *count == 0
305}
306
307/// Percentiles of a set of values, by nearest rank: each is one of the values, and with fewer than 100 values
308/// `p99` is the largest.
309#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
310pub struct Spread {
311    /// The median.
312    pub p50: u64,
313    /// The 95th percentile.
314    pub p95: u64,
315    /// The 99th percentile.
316    pub p99: u64,
317    /// The largest.
318    pub max: u64,
319}
320
321/// The least share of Docker's disk, in bytes, auto sizing leaves each slot when it adds slots under measured
322/// admission: 2G. Every slot's Docker store shares a quarter of Docker's disk; at a 2G share the stores measured on
323/// a busy host keep their images between jobs, and at 1G, the least a share can be, every store over 1G is emptied
324/// after each job. What a store keeps does not set it: a store fills the share it gets (ADR 0090).
325pub const SLOT_DISK: u64 = 2 << 30;
326
327/// What a cap keeps at least over the largest peak, for page cache: 1G. Peaks leave page cache out, and a container
328/// whose memory past them leaves the kernel less than about 500M of page cache to reclaim at its cap reclaims badly,
329/// which throttles its sockets and slows its downloads (ADR 0055).
330pub const PAGE_CACHE_ROOM: u64 = 1 << 30;
331
332/// What to set a profile's `memory` and `slots` to. `memory` is the largest sampled peak plus the margin, or plus
333/// `PAGE_CACHE_ROOM` when that is more, rounded up to 256M: the cap kills a job that goes over it, so it is sized for
334/// the heaviest run, not a percentile.
335#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
336#[serde(rename_all = "kebab-case", tag = "kind")]
337pub enum Advice {
338    /// Fewer than `min_jobs` jobs: too few to advise on.
339    TooFew {
340        /// How many there are.
341        jobs: u64,
342    },
343    /// The cap fits, and lowering it would save less than a tenth or a run met its cap.
344    Keep {
345        /// What the jobs need.
346        memory: u64,
347    },
348    /// The cap is a tenth or more above what the jobs need, and no run met its cap.
349    Lower {
350        /// The new `memory`.
351        memory: u64,
352        /// The new `slots`: under `caps` admission, as many as fit in the memory the slots take now when jobs waited
353        /// for one; else the same.
354        slots: u32,
355    },
356    /// The jobs need more than the cap.
357    Raise {
358        /// The new `memory`.
359        memory: u64,
360        /// The new `slots`: the same, or under `caps` admission as many as the slice's `MemoryMax` still holds.
361        slots: u32,
362    },
363    /// The jobs need more than the cap, and the slice's `MemoryMax` does not hold one slot of what they need (under
364    /// `measured` admission, one runner beside the mirror): the slice has to grow first. Also when the cap is at
365    /// what `[host] memory_ceiling` or `max_memory` holds it to (`ProfileSizing::held`): that has to grow first.
366    SliceTooSmall {
367        /// What the jobs need.
368        memory: u64,
369    },
370}
371
372impl Advice {
373    /// The `memory` and `slots` to write, when they differ from the profile's.
374    pub fn change(&self) -> Option<Resize> {
375        match *self {
376            Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => Some(Resize {
377                memory,
378                slots,
379                cpus: None,
380            }),
381            Advice::TooFew { .. } | Advice::Keep { .. } | Advice::SliceTooSmall { .. } => None,
382        }
383    }
384}
385
386/// A profile's new `memory`, `slots` and, when auto sizing changed them, `cpus`.
387#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
388pub struct Resize {
389    /// Bytes.
390    pub memory: u64,
391    /// Slots.
392    pub slots: u32,
393    /// CPUs per slot; `None` leaves them as they are, as advice does and as auto sizing's changes recorded before it
394    /// changed them.
395    #[serde(default, skip_serializing_if = "Option::is_none")]
396    pub cpus: Option<u32>,
397}
398
399impl AutoPlan {
400    /// Its rules in words, as clients say them after `runnerd`: `looks every 5 minutes, raises a profile at most
401    /// once every 1 hour and lowers it at most once every 24 hours, moving its memory and CPUs by at most 25% at a
402    /// time`.
403    pub fn rules(&self) -> String {
404        if self.raise_interval == 0 || self.raise_interval >= self.interval {
405            return format!(
406                "looks every {} and changes a profile at most once every {}, moving its memory by at most {}% at a \
407                 time",
408                period(self.every),
409                period(self.interval),
410                self.max_step
411            );
412        }
413        format!(
414            "looks every {}, raises a profile at most once every {} and lowers it at most once every {}, moving its \
415             memory and CPUs by at most {}% at a time",
416            period(self.every),
417            period(self.raise_interval),
418            period(self.interval),
419            self.max_step
420        )
421    }
422}
423
424/// A setting's length in words, as a config's periods read: `5 minutes`, `24 hours`, `90 s`.
425pub fn period(seconds: u64) -> String {
426    let plural = |count: u64, unit: &str| {
427        if count == 1 {
428            format!("{count} {unit}")
429        } else {
430            format!("{count} {unit}s")
431        }
432    };
433    match seconds {
434        0 => "0 s".to_owned(),
435        _ if seconds.is_multiple_of(3600) => plural(seconds / 3600, "hour"),
436        _ if seconds.is_multiple_of(60) => plural(seconds / 60, "minute"),
437        _ => format!("{seconds} s"),
438    }
439}
440
441impl Resize {
442    /// What going from `from` to these sizes changes, such as `memory 9G → 6.75G, slots 2`, with `cpus 12 → 15` when
443    /// both have CPUs and they differ; without `from`, these alone, such as `memory 6.75G, slots 2`.
444    pub fn line_from(&self, from: Option<&Resize>) -> String {
445        let memory = match from {
446            Some(from) if from.memory != self.memory => format!(
447                "memory {} → {}",
448                ByteSize(from.memory),
449                ByteSize(self.memory)
450            ),
451            _ => format!("memory {}", ByteSize(self.memory)),
452        };
453        let slots = match from {
454            Some(from) if from.slots != self.slots => {
455                format!("slots {} → {}", from.slots, self.slots)
456            }
457            _ => format!("slots {}", self.slots),
458        };
459        let cpus = match (from.and_then(|from| from.cpus), self.cpus) {
460            (Some(from), Some(to)) if from != to => format!(", cpus {from} → {to}"),
461            _ => String::new(),
462        };
463        format!("{memory}, {slots}{cpus}")
464    }
465}
466
467impl SizingChange {
468    /// What it changed, such as `memory 9G → 6.75G, slots 2`, and `cpus 12 → 15` when it changed them.
469    pub fn line(&self) -> String {
470        let line = self.to.line_from(Some(&self.from));
471        if self.uncertain {
472            format!("{line} (may not have been written)")
473        } else {
474            line
475        }
476    }
477}
478
479impl Sizing {
480    /// One line on what to do with a profile and why, as `runnerctl sizing`, the dashboard and `runnerd doctor`
481    /// say it, such as `lower memory to 2.5G from 7G, its largest peak + 15%; 5 slots fit in the same 14G`.
482    /// Under `measured` admission, when jobs wanted more slots, it adds that `slots` is not bought with caps there, or
483    /// with auto sizing on, that it adds them, that it cannot when the next look refuses (`AutoPlan::refusals`), or
484    /// that it waits for the jobs since its last change when they did not wait (`AutoPlan::waited_before_change`).
485    pub fn advice_line(&self, profile: &ProfileSizing) -> String {
486        let line = self.memory_line(profile);
487        if self.admission == AdmissionPolicy::Measured
488            && profile.wants_slots
489            && !matches!(profile.advice, Advice::TooFew { .. })
490        {
491            let plan = self.auto_plan.as_ref();
492            let refused = plan.is_some_and(|plan| {
493                plan.refused_slots
494                    .iter()
495                    .any(|refused| refused.profile == profile.name)
496            });
497            let before_change =
498                plan.is_some_and(|plan| plan.waited_before_change.contains(&profile.name));
499            let then = if refused {
500                "auto sizing cannot add one now"
501            } else if before_change {
502                "auto sizing changed it since, and adds more only once jobs queued after that change wait for a slot \
503                 too"
504            } else if self.auto {
505                &format!(
506                    "auto sizing adds as many as waited at once, up to max_slots and while each slot keeps {} of \
507                     Docker's disk",
508                    ByteSize(SLOT_DISK)
509                )
510            } else {
511                "measured admission reserves predicted peaks, not caps, so more slots start them sooner: raise slots \
512                 by hand"
513            };
514            format!(
515                "{line}; jobs waited {}s for a slot (p95) while memory and CPU had room: {then}",
516                profile.slot_wait.p95
517            )
518        } else {
519            line
520        }
521    }
522
523    /// `advice_line` without what it says about slots under measured admission.
524    pub(crate) fn memory_line(&self, profile: &ProfileSizing) -> String {
525        let size = |bytes: u64| ByteSize(bytes).to_string();
526        let peaks = &profile.peaks;
527        let need = self.need_reason(peaks);
528        let at_cap = format!(
529            "{} of {} jobs ran out of memory at their cap",
530            peaks.at_cap, peaks.jobs
531        );
532        match profile.advice {
533            Advice::TooFew { jobs } => format!(
534                "{jobs} jobs, too few to advise on (it takes {}); largest peak {} of a {} cap",
535                self.min_jobs,
536                size(peaks.memory.max),
537                size(profile.memory)
538            ),
539            Advice::Keep { .. } if peaks.on_cap > 0 => format!(
540                "keep {}: {} of its jobs met their cap or were killed for memory, and fewer than 3 runs since ran \
541                 clear of it",
542                size(profile.memory),
543                peaks.on_cap
544            ),
545            Advice::Keep { memory } if memory > profile.memory => format!(
546                "keep {}: its recent runs need {}, but none is known to have run out of memory at {} since it was set",
547                size(profile.memory),
548                size(memory),
549                size(profile.memory)
550            ),
551            Advice::Keep { memory } => match profile.hold {
552                Some(hold) => format!(
553                    "keep {}, {}: {need} is {}",
554                    size(profile.memory),
555                    hold.bound_line(),
556                    size(memory)
557                ),
558                None => format!("keep {}: {need} is {}", size(profile.memory), size(memory)),
559            },
560            Advice::Lower { memory, slots } => {
561                let share = profile.memory * u64::from(profile.slots);
562                // `min_memory` holds it over what the jobs need: `lower memory to 6G from 8G, its min_memory: its
563                // largest peak + 15% is 4G`.
564                let need = match profile.hold {
565                    Some(hold) => format!("{}: {need} is {}", hold.bound_line(), size(hold.need)),
566                    None => need,
567                };
568                if self.admission == AdmissionPolicy::Measured {
569                    return format!(
570                        "lower memory to {} from {}, {need}",
571                        size(memory),
572                        size(profile.memory)
573                    );
574                }
575                let then = if slots > profile.slots {
576                    format!("{slots} slots fit in the same {}", size(share))
577                } else {
578                    format!(
579                        "frees {} of the budget",
580                        size(share - memory * u64::from(slots))
581                    )
582                };
583                format!(
584                    "lower memory to {} from {}, {need}; {then}",
585                    size(memory),
586                    size(profile.memory)
587                )
588            }
589            Advice::Raise { memory, .. } if let Some(revert) = &profile.revert => {
590                let held = match profile.hold {
591                    Some(hold) if memory < revert.to => format!(", {}", hold.bound_line()),
592                    _ => String::new(),
593                };
594                format!(
595                    "raise memory back to {}{held}: {}",
596                    size(memory),
597                    revert.why()
598                )
599            }
600            Advice::Raise { memory, slots } => {
601                let why = if peaks.at_cap > 0 {
602                    at_cap
603                } else {
604                    format!("{need} is above the {} cap", size(profile.memory))
605                };
606                let slots = if slots < profile.slots {
607                    format!("; {slots} of its {} slots fit the slice", profile.slots)
608                } else {
609                    String::new()
610                };
611                match profile.hold {
612                    Some(hold) => format!(
613                        "raise memory to {}, {}, short of the {} its jobs need: {why}{slots}",
614                        size(memory),
615                        hold.bound_line(),
616                        size(hold.need)
617                    ),
618                    None => format!("raise memory to {}: {why}{slots}", size(memory)),
619                }
620            }
621            Advice::SliceTooSmall { memory } => too_small_line(profile, memory, need),
622        }
623    }
624}
625
626/// The advice line of a profile whose jobs need `memory` and get no raise, with `need` how that comes from its
627/// peaks, or the revert that asks for it: a bound of the config holds it where it is (`ProfileSizing::held`), or the
628/// slice is too small.
629fn too_small_line(profile: &ProfileSizing, memory: u64, need: String) -> String {
630    let why = profile.revert.as_ref().map_or(need, SizingRevert::why);
631    if let Some(hold) = profile.held() {
632        let then = match hold.by {
633            MemoryBound::Ceiling { .. } => {
634                "raise [host] memory_ceiling if the host can grow further"
635            }
636            MemoryBound::MaxMemory | MemoryBound::MinMemory => "raise max_memory to give them more",
637        };
638        return format!(
639            "keep {}, {}: its jobs need {} ({why}); {then}",
640            ByteSize(profile.memory),
641            hold.bound_line(),
642            ByteSize(memory)
643        );
644    }
645    let then = profile
646        .hold
647        .map(|hold| {
648            format!(
649                "; {} then holds it at {}",
650                hold.by.name(),
651                ByteSize(hold.bound)
652            )
653        })
654        .unwrap_or_default();
655    format!(
656        "needs {} ({why}), more than the slice holds for one slot: raise the slice's MemoryMax first{then}",
657        ByteSize(memory)
658    )
659}
660
661impl MemoryHold {
662    /// What holds the `memory`, as the advice line says it after the `memory` it goes to: `the most the host's 24G
663    /// memory_ceiling leaves one runner beside the mirror's 1G`, `its max_memory` or `its min_memory`.
664    pub fn bound_line(&self) -> String {
665        match self.by {
666            MemoryBound::Ceiling { ceiling, mirror: 0 } => format!(
667                "the most the host's {} memory_ceiling leaves one runner",
668                ByteSize(ceiling)
669            ),
670            MemoryBound::Ceiling { ceiling, mirror } => format!(
671                "the most the host's {} memory_ceiling leaves one runner beside the mirror's {}",
672                ByteSize(ceiling),
673                ByteSize(mirror)
674            ),
675            MemoryBound::MaxMemory => "its max_memory".to_owned(),
676            MemoryBound::MinMemory => "its min_memory".to_owned(),
677        }
678    }
679}
680
681impl MemoryBound {
682    /// The setting, as the config names it: `[host] memory_ceiling`, `max_memory` or `min_memory`.
683    pub fn name(self) -> &'static str {
684        match self {
685            MemoryBound::Ceiling { .. } => "[host] memory_ceiling",
686            MemoryBound::MaxMemory => "max_memory",
687            MemoryBound::MinMemory => "min_memory",
688        }
689    }
690}
691
692impl SizingRevert {
693    /// What showed the lowering went too far, as the advice line says it: `halyard/ledger CI / test met the 10G cap
694    /// since auto sizing lowered it from 14G`.
695    fn why(&self) -> String {
696        let how = match self.cause {
697            RevertCause::Met => format!("met the {} cap", ByteSize(self.from)),
698            RevertCause::Failed { runs } => {
699                format!("failed {runs} times in a row at {}", ByteSize(self.from))
700            }
701        };
702        format!(
703            "{} {how} since auto sizing lowered it from {}",
704            self.job,
705            ByteSize(self.to)
706        )
707    }
708}