Skip to main content

runnerd_core/api/
budget.rs

1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::units::{duration, utc_time};
5use super::{AdmissionPolicy, Budget, CountBrake, Counted, HostMemory, Status, percent};
6use crate::config::ByteSize;
7
8/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
9/// sets the figure apart.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Reading {
12    /// Such as `8G`.
13    pub figure: String,
14    /// What follows it, such as `of 24G reserved in runnerd.slice`.
15    pub text: String,
16}
17
18impl Reading {
19    /// The figure and its words as one line.
20    pub fn line(&self) -> String {
21        format!("{} {}", self.figure, self.text)
22    }
23}
24
25impl Budget {
26    /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
27    /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; while pooling takes something off, `36.8G`
28    /// `of 46G counted in ci.slice by predicted peaks: 42.8G reserved, 6G pooled`; how runners are admitted
29    /// (`Budget::admission_note`) follows the slice.
30    pub fn memory_reading(&self) -> Reading {
31        let of = self
32            .memory_max
33            .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
34        let verb = if self.pooled.is_some() {
35            "counted"
36        } else {
37            "reserved"
38        };
39        let mut text = format!("{of}{verb} in {}", self.slice);
40        if let Some(note) = self.admission_note() {
41            text = format!("{text} {note}");
42        }
43        if let Some(pooled) = &self.pooled {
44            let reserved = ByteSize(self.memory_in_use);
45            text = format!(
46                "{text}: {reserved} reserved, {} pooled",
47                ByteSize(pooled.credit)
48            );
49        }
50        if self.memory_max.is_none() {
51            text.push_str(", which has no MemoryMax");
52        }
53        if let Some(counted) = &self.counted {
54            text = format!("{text}; {}", counted.clause());
55        }
56        Reading {
57            figure: ByteSize(self.counted()).to_string(),
58            text,
59        }
60    }
61
62    /// What `MemoryMax` goes by, in bytes: `memory_in_use`, less what pooling takes off it (ADR 0099).
63    pub fn counted(&self) -> u64 {
64        self.pooled
65            .map_or(self.memory_in_use, |pooled| pooled.counted)
66    }
67
68    /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
69    /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
70    pub fn clauses(&self) -> Vec<String> {
71        let mut clauses = vec![self.memory_reading().line()];
72        clauses.extend(self.cpu.map(|cpu| cpu.line()));
73        clauses.extend(self.host_line());
74        clauses
75    }
76
77    /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
78    /// slice 40G`; on a host with swap, `host: 31G of 64G available, swap 12.1G of 15.5G used, …` (ADR 0101); when that
79    /// is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
80    /// of its 46G, so 6.5G is free`, or past `MemoryMax` (`Budget::host_share`) `…; leaves the slice 47.5G for the 44.3G
81    /// reserved there, so 3.2G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
82    /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
83    /// (`Budget::held`), nothing is free, so it says no figure of it. While the host's limit takes a credit (ADR 0108),
84    /// what it counts follows: `…; what runs counts 23G: it uses 14G and keeps 9G for growth`
85    /// (`HostBudget::growth_clause`), unless the growth count counts, which `Budget::memory_reading` says. With programs
86    /// outside the slice large enough to
87    /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
88    /// reading of the host.
89    pub fn host_line(&self) -> Option<String> {
90        let host = self.host.as_ref()?;
91        let mut line = format!("host: {}", host.memory.available_line());
92        if let Some(swap) = host.memory.swap {
93            line = format!(
94                "{line}, swap {} of {} used",
95                ByteSize(swap.used()),
96                ByteSize(swap.total)
97            );
98        }
99        if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
100            line = format!("{line}, {} stalled for memory", percent(full.avg10));
101        }
102        let free = match host.spare {
103            0 => "nothing".to_owned(),
104            spare => ByteSize(spare).to_string(),
105        };
106        let limit = ByteSize(host.limit);
107        line = if host.stalled {
108            format!("{line}; stalling, so no runner starts")
109        } else if self.closed() {
110            format!("{line}; leaves the slice {limit}")
111        } else {
112            match self.memory_max {
113                Some(max) if self.host_withheld().is_some() => format!(
114                    "{line}; leaves the slice {}, so {free} is free",
115                    self.host_share(host.limit, max)
116                ),
117                Some(_) => format!("{line}; leaves the slice {limit}"),
118                None => format!("{line}; leaves the slice {limit}, so {free} is free"),
119            }
120        };
121        // While the growth count counts, both budgets read it and the memory's reading says what it is made of.
122        let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
123        if let Some(growth) = host
124            .growth_clause(self.memory_in_use)
125            .filter(|_| !growth_counts)
126        {
127            line = format!(
128                "{line}; what runs counts {}: {growth}",
129                ByteSize(host.counts(self.memory_in_use))
130            );
131        }
132        if !host.memory.holders.is_empty() {
133            line = format!("{line}; {}", host.memory.outside_line());
134        }
135        Some(line)
136    }
137
138    /// What the host's `limit` leaves the slice of `max`, its `MemoryMax`: `25G of its 46G`. A limit past `max` holds
139    /// back room only where `max` reads a larger credit than the limit (ADR 0099, 0108). Nothing leaves a slice more
140    /// than it has, so it says what the limit goes by: `47.5G for the 44.3G reserved there`, or while the host's limit
141    /// takes a credit, `47.5G for the 40G counted there`.
142    fn host_share(&self, limit: u64, max: u64) -> String {
143        if limit > max {
144            let host = self.host.as_ref();
145            let (counts, verb) = match host.filter(|host| host.credit.is_some()) {
146                Some(host) => (host.counts(self.memory_in_use), "counted"),
147                None => (self.memory_in_use, "reserved"),
148            };
149            format!(
150                "{} for the {} {verb} there",
151                ByteSize(limit),
152                ByteSize(counts)
153            )
154        } else {
155            format!("{} of its {}", ByteSize(limit), ByteSize(max))
156        }
157    }
158
159    /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
160    /// nothing is free.
161    fn closed(&self) -> bool {
162        self.held && self.admission == AdmissionPolicy::Measured
163    }
164
165    /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
166    /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
167    /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
168    pub fn host_withheld(&self) -> Option<u64> {
169        let host = self.host.as_ref().filter(|host| !host.stalled)?;
170        let remaining = self.remaining()?;
171        (host.spare < remaining).then(|| remaining - host.spare)
172    }
173
174    /// The budget as one line, its clauses joined by `; `.
175    pub fn line(&self) -> String {
176        self.clauses().join("; ")
177    }
178
179    /// What `MemoryMax` leaves beside what the running containers count (`Budget::counted`), in bytes: what a new
180    /// runner must fit in. `None` without `MemoryMax`.
181    pub fn remaining(&self) -> Option<u64> {
182        self.memory_max
183            .map(|max| max.saturating_sub(self.counted()))
184    }
185}
186
187impl Counted {
188    /// What the growth count is made of, or why it rests, as every client says it after the budget's figure: `what runs
189    /// uses 15.7G and keeps 12.9G for its growth`; while it rests, `the growth count rests until 15:02 UTC: what ran
190    /// grew 14.1G in 10m, past the 12.9G kept`.
191    pub fn clause(&self) -> String {
192        let size = |bytes: u64| ByteSize(bytes).to_string();
193        let rests = |until: u64| format!("the growth count rests until {}", utc_time(until));
194        match (self.brake, self.kept) {
195            (None, Some(kept)) => format!(
196                "what runs uses {} and keeps {} for its growth",
197                size(self.used),
198                size(kept)
199            ),
200            (Some(CountBrake::Grew { grew, kept, until }), _) => format!(
201                "{}: what ran grew {} in {}, past the {} kept",
202                rests(until),
203                size(grew),
204                duration(super::GROWTH_WINDOW.as_secs()),
205                size(kept)
206            ),
207            (Some(CountBrake::Restarted { until }), _) => {
208                format!(
209                    "{}: too soon after a start or a reload to judge growth",
210                    rests(until)
211                )
212            }
213            (Some(CountBrake::Guard { until: Some(until) }), _) => {
214                format!("{}: the host ran under its line", rests(until))
215            }
216            (Some(CountBrake::Guard { until: None }) | None, _) => {
217                "the growth count rests while the host is under its line or a reading is missing"
218                    .to_owned()
219            }
220            (Some(CountBrake::Unknown), _) => "the growth count rests".to_owned(),
221        }
222    }
223}
224
225impl HostMemory {
226    /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
227    /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
228    /// growing to 24G`.
229    pub fn available_line(&self) -> String {
230        let line = format!(
231            "{} of {} available",
232            ByteSize(self.available),
233            ByteSize(self.total)
234        );
235        match self.growable() {
236            0 => line,
237            _ => format!("{line}, growing to {}", ByteSize(self.grown())),
238        }
239    }
240}
241
242impl Status {
243    /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
244    /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
245    /// the room no runner may start in; and the whole the blocks are scaled against.
246    pub fn budget_parts(&self) -> BudgetParts {
247        let budget = &self.budget;
248        let slots: Vec<SlotPart> = self
249            .profiles
250            .iter()
251            .enumerate()
252            .flat_map(|(profile, status)| {
253                status.slots.iter().filter_map(move |slot| {
254                    let memory = match budget.admission {
255                        AdmissionPolicy::Caps => match &slot.runner {
256                            Some(runner) if runner.reserved > 0 => runner.reserved,
257                            _ => status.memory,
258                        },
259                        AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
260                    };
261                    Some(SlotPart {
262                        profile,
263                        slot: slot.slot,
264                        memory,
265                        reserved: slot.runner.is_some(),
266                    })
267                })
268            })
269            .collect();
270        let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
271            kind: HeldKind::Mirror,
272            memory: budget.mirror_memory,
273        });
274        let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
275            kind: HeldKind::Seed {
276                profile: seed.profile.clone(),
277            },
278            memory: seed.memory,
279        });
280        let shells = budget.shell_memory.iter().map(|shell| HeldPart {
281            kind: HeldKind::Shell { job: shell.job },
282            memory: shell.memory,
283        });
284        let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
285        let drawn: u64 = slots
286            .iter()
287            .map(|part| part.memory)
288            .chain(held.iter().map(|part| part.memory))
289            .sum();
290        let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
291        let unclaimed: u64 = slots
292            .iter()
293            .filter(|part| !part.reserved)
294            .map(|part| part.memory)
295            .sum();
296        let (free, closed) = budget.room_parts(drawn, unclaimed);
297        BudgetParts {
298            slots,
299            held,
300            free,
301            closed,
302            whole,
303        }
304    }
305}
306
307impl Budget {
308    /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
309    /// of it no runner may start in.
310    fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
311        let Some(max) = self.memory_max else {
312            return (None, None);
313        };
314        let room = max.saturating_sub(drawn);
315        // While no runner starts, all the room is closed.
316        let shut = if self.closed() {
317            Some((
318                ClosedKind::Held,
319                "closed: the slice is past 90% of its MemoryMax, so no runner starts",
320            ))
321        } else if self.host.as_ref().is_some_and(|host| host.stalled) {
322            Some((
323                ClosedKind::Stalled,
324                "closed: the host's programs stall waiting for memory, so no runner starts",
325            ))
326        } else {
327            None
328        };
329        if let Some((kind, text)) = shut {
330            let closed = (room > 0).then(|| ClosedPart {
331                memory: room,
332                kind,
333                reading: Reading {
334                    figure: ByteSize(room).to_string(),
335                    text: text.to_owned(),
336                },
337            });
338            return (None, closed);
339        }
340        let withheld = self.host_withheld();
341        // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
342        // host may hold their room too.
343        let host = withheld
344            .map(|withheld| withheld.min(room))
345            .filter(|&memory| memory > 0);
346        let (host_limit, spare) = self
347            .host
348            .as_ref()
349            .map_or((0, 0), |host| (host.limit, host.spare));
350        let memory = room - host.unwrap_or(0);
351        let mut text = match (unclaimed, withheld) {
352            (0, None) => format!("of {} free", ByteSize(max)),
353            (_, None) => format!(
354                "of {} free past every slot's cap; {} is free beside what runs",
355                ByteSize(max),
356                ByteSize(self.remaining().unwrap_or(0))
357            ),
358            (0, Some(_)) => format!(
359                "free; the host leaves the slice {}",
360                self.host_share(host_limit, max)
361            ),
362            (_, Some(_)) => format!(
363                "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
364                ByteSize(spare),
365                self.host_share(host_limit, max)
366            ),
367        };
368        // The blocks stay what runners reserve, so what pooling adds is said and not drawn.
369        if let Some(pooled) = &self.pooled {
370            text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
371        }
372        let free = (memory > 0).then(|| FreePart {
373            memory,
374            reading: Reading {
375                figure: ByteSize(memory).to_string(),
376                text,
377            },
378        });
379        let host = host.map(|memory| ClosedPart {
380            memory,
381            kind: ClosedKind::Host,
382            reading: Reading {
383                figure: ByteSize(memory).to_string(),
384                // Past `MemoryMax` the plain count holds this back, not the host's programs (`Budget::host_share`).
385                text: if host_limit > max {
386                    format!(
387                        "held back by what runs reserves: the host leaves the slice {}",
388                        self.host_share(host_limit, max)
389                    )
390                } else {
391                    format!(
392                        "{} for its other programs and page cache, which leave the slice {}",
393                        ClosedKind::Host.name(),
394                        ByteSize(host_limit)
395                    )
396                },
397            },
398        });
399        (free, host)
400    }
401}
402
403/// The budget as a bar draws it, from `Status::budget_parts`.
404#[derive(Debug, Clone, PartialEq, Eq)]
405pub struct BudgetParts {
406    /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
407    pub slots: Vec<SlotPart>,
408    /// The mirror, each seed container, then each shell open in a record.
409    pub held: Vec<HeldPart>,
410    /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
411    /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
412    /// stalls.
413    pub free: Option<FreePart>,
414    /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
415    /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
416    pub closed: Option<ClosedPart>,
417    /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
418    pub whole: u64,
419}
420
421/// A slot's block of the budget.
422#[derive(Debug, Clone, Copy, PartialEq, Eq)]
423pub struct SlotPart {
424    /// Its profile's index in `Status::profiles`.
425    pub profile: usize,
426    /// The slot, 1-based.
427    pub slot: usize,
428    /// Its size in bytes: what its runner reserves, or a free slot's cap.
429    pub memory: u64,
430    /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
431    pub reserved: bool,
432}
433
434/// Memory the budget holds for something other than a slot.
435#[derive(Debug, Clone, PartialEq, Eq)]
436pub struct HeldPart {
437    /// What holds it.
438    pub kind: HeldKind,
439    /// Its cap in bytes.
440    pub memory: u64,
441}
442
443impl HeldPart {
444    /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
445    pub fn name(&self) -> String {
446        match &self.kind {
447            HeldKind::Mirror => "mirror".to_owned(),
448            HeldKind::Seed { profile } => format!("{profile} seed"),
449            HeldKind::Shell { job } => format!("job {job} shell"),
450        }
451    }
452
453    /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
454    /// in job 42's record`.
455    pub fn holder(&self) -> String {
456        match &self.kind {
457            HeldKind::Mirror => "held by the mirror".to_owned(),
458            HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
459            HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
460        }
461    }
462}
463
464/// What holds a part of the budget beside the slots.
465#[derive(Debug, Clone, PartialEq, Eq)]
466pub enum HeldKind {
467    /// The registry mirror.
468    Mirror,
469    /// A profile's seed container, while it reseeds.
470    Seed {
471        /// The profile it reseeds.
472        profile: String,
473    },
474    /// A shell open in a job's record.
475    Shell {
476        /// The job.
477        job: u64,
478    },
479}
480
481/// What the bar leaves free past every block, less what the host holds back.
482#[derive(Debug, Clone, PartialEq, Eq)]
483pub struct FreePart {
484    /// In bytes.
485    pub memory: u64,
486    /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
487    /// leaves the slice 25.27G of its 46G` (`Budget::host_share`).
488    pub reading: Reading,
489}
490
491/// Room past the bar's blocks that no runner may start in.
492#[derive(Debug, Clone, PartialEq, Eq)]
493pub struct ClosedPart {
494    /// In bytes.
495    pub memory: u64,
496    /// What closes it.
497    pub kind: ClosedKind,
498    /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
499    /// which leave the slice 25.27G`, or past `MemoryMax` `1.5G` `held back by what runs reserves: the host leaves the
500    /// slice 24.5G for the 22G reserved there`.
501    pub reading: Reading,
502}
503
504/// What closes room past the bar's blocks.
505#[derive(Debug, Clone, Copy, PartialEq, Eq)]
506pub enum ClosedKind {
507    /// The host's other programs and its page cache need it.
508    Host,
509    /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
510    Held,
511    /// The host's programs stall waiting for memory (`HostBudget::stalled`).
512    Stalled,
513}
514
515impl ClosedKind {
516    /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
517    /// stalls`.
518    pub fn name(self) -> &'static str {
519        match self {
520            Self::Host => "held by the host",
521            Self::Held => "closed past 90%",
522            Self::Stalled => "closed while the host stalls",
523        }
524    }
525}
526
527#[cfg(test)]
528mod tests;