Skip to main content

runnerd_core/api/
budget.rs

1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::units::{duration, utc_time};
5use super::{AdmissionPolicy, Budget, CountBrake, Counted, HostMemory, Status, percent};
6use crate::config::ByteSize;
7
8/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
9/// sets the figure apart.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Reading {
12    /// Such as `8G`.
13    pub figure: String,
14    /// What follows it, such as `of 24G reserved in runnerd.slice`.
15    pub text: String,
16}
17
18/// One figure of the host's reading, as [`Budget::host_figures`] gives it: `Host available` `23G of 64G`.
19#[derive(Debug, Clone, PartialEq, Eq)]
20pub struct HostFigure {
21    /// What it measures, such as `Leaves the slice`.
22    pub label: &'static str,
23    /// The figure, such as `30G of its 33G`.
24    pub value: String,
25    /// What it is made of, when that needs saying: `most java 9.3G and qemu-system-x86 5.8G`.
26    pub note: Option<String>,
27}
28
29impl HostFigure {
30    fn new(label: &'static str, value: String) -> HostFigure {
31        HostFigure {
32            label,
33            value,
34            note: None,
35        }
36    }
37}
38
39impl Reading {
40    /// The figure and its words as one line.
41    pub fn line(&self) -> String {
42        format!("{} {}", self.figure, self.text)
43    }
44}
45
46impl Budget {
47    /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
48    /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; while pooling takes something off, `36.8G`
49    /// `of 46G counted in ci.slice by predicted peaks: 42.8G reserved, 6G pooled`; how runners are admitted
50    /// (`Budget::admission_note`) follows the slice.
51    pub fn memory_reading(&self) -> Reading {
52        let of = self
53            .memory_max
54            .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
55        let verb = if self.pooled.is_some() {
56            "counted"
57        } else {
58            "reserved"
59        };
60        let mut text = format!("{of}{verb} in {}", self.slice);
61        if let Some(note) = self.admission_note() {
62            text = format!("{text} {note}");
63        }
64        if let Some(pooled) = &self.pooled {
65            let reserved = ByteSize(self.memory_in_use);
66            text = format!(
67                "{text}: {reserved} reserved, {} pooled",
68                ByteSize(pooled.credit)
69            );
70        }
71        if self.memory_max.is_none() {
72            text.push_str(", which has no MemoryMax");
73        }
74        if let Some(counted) = &self.counted {
75            text = format!("{text}; {}", counted.clause());
76        }
77        Reading {
78            figure: ByteSize(self.counted()).to_string(),
79            text,
80        }
81    }
82
83    /// What `MemoryMax` goes by, in bytes: `memory_in_use`, less what pooling takes off it (ADR 0099).
84    pub fn counted(&self) -> u64 {
85        self.pooled
86            .map_or(self.memory_in_use, |pooled| pooled.counted)
87    }
88
89    /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
90    /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
91    pub fn clauses(&self) -> Vec<String> {
92        let mut clauses = vec![self.memory_reading().line()];
93        clauses.extend(self.cpu.map(|cpu| cpu.line()));
94        clauses.extend(self.host_line());
95        clauses
96    }
97
98    /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
99    /// slice 40G`; on a host with swap, `host: 31G of 64G available, swap 12.1G of 15.5G used, …` (ADR 0101); when that
100    /// is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
101    /// of its 46G, so 6.5G is free`, or past `MemoryMax` (`Budget::host_share`) `…; leaves the slice 47.5G for the 44.3G
102    /// reserved there, so 3.2G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
103    /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
104    /// (`Budget::held`), nothing is free, so it says no figure of it. While the host's limit takes a credit (ADR 0108),
105    /// what it counts follows: `…; what runs counts 23G: it uses 14G and keeps 9G for growth`
106    /// (`HostBudget::growth_clause`), unless the growth count counts, which `Budget::memory_reading` says. With programs
107    /// outside the slice large enough to
108    /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
109    /// reading of the host.
110    pub fn host_line(&self) -> Option<String> {
111        let host = self.host.as_ref()?;
112        let mut line = format!("host: {}", host.memory.available_line());
113        if let Some(swap) = host.memory.swap {
114            line = format!(
115                "{line}, swap {} of {} used",
116                ByteSize(swap.used()),
117                ByteSize(swap.total)
118            );
119        }
120        if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
121            line = format!("{line}, {} stalled for memory", percent(full.avg10));
122        }
123        let free = match host.spare {
124            0 => "nothing".to_owned(),
125            spare => ByteSize(spare).to_string(),
126        };
127        let limit = ByteSize(host.limit);
128        line = if host.stalled {
129            format!("{line}; stalling, so no runner starts")
130        } else if self.closed() {
131            format!("{line}; leaves the slice {limit}")
132        } else {
133            match self.memory_max {
134                Some(max) if self.host_withheld().is_some() => format!(
135                    "{line}; leaves the slice {}, so {free} is free",
136                    self.host_share(host.limit, max)
137                ),
138                Some(_) => format!("{line}; leaves the slice {limit}"),
139                None => format!("{line}; leaves the slice {limit}, so {free} is free"),
140            }
141        };
142        // While the growth count counts, both budgets read it and the memory's reading says what it is made of.
143        let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
144        if let Some(growth) = host
145            .growth_clause(self.memory_in_use)
146            .filter(|_| !growth_counts)
147        {
148            line = format!(
149                "{line}; what runs counts {}: {growth}",
150                ByteSize(host.counts(self.memory_in_use))
151            );
152        }
153        if !host.memory.holders.is_empty() {
154            line = format!("{line}; {}", host.memory.outside_line());
155        }
156        Some(line)
157    }
158
159    /// The host's line ([`Budget::host_line`]) as figures, for a client that lays them out in a row rather than as a
160    /// sentence: what the host has available, the swap used, the share of time it stalled for memory, what it leaves the
161    /// slice, what the host's limit counts of what runs while a credit stands (ADR 0108), and what programs outside CI
162    /// hold. What is free is the bar's own part ([`BudgetParts::free`]), so it is not a figure here. Empty without a
163    /// recent reading of the host.
164    pub fn host_figures(&self) -> Vec<HostFigure> {
165        let Some(host) = self.host.as_ref() else {
166            return Vec::new();
167        };
168        let memory = &host.memory;
169        let size = |bytes: u64| ByteSize(bytes).to_string();
170        let mut available = format!("{} of {}", size(memory.available), size(memory.total));
171        if memory.growable() > 0 {
172            available = format!("{available}, growing to {}", size(memory.grown()));
173        }
174        let mut figures = vec![HostFigure::new("Host available", available)];
175        if let Some(swap) = memory.swap {
176            figures.push(HostFigure::new(
177                "Swap used",
178                format!("{} of {}", size(swap.used()), size(swap.total)),
179            ));
180        }
181        if let Some(full) = memory.full.filter(|full| full.avg10 > 0) {
182            figures.push(HostFigure::new("Stalled for memory", percent(full.avg10)));
183        }
184        let leaves = if host.stalled {
185            "nothing while it stalls: no runner starts".to_owned()
186        } else {
187            match self.memory_max {
188                Some(max) if !self.closed() && self.host_withheld().is_some() => {
189                    self.host_share(host.limit, max)
190                }
191                _ => size(host.limit),
192            }
193        };
194        figures.push(HostFigure::new("Leaves the slice", leaves));
195        let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
196        if let Some(growth) = host
197            .growth_clause(self.memory_in_use)
198            .filter(|_| !growth_counts)
199        {
200            figures.push(HostFigure {
201                note: Some(growth),
202                ..HostFigure::new("What runs counts", size(host.counts(self.memory_in_use)))
203            });
204        }
205        if !memory.holders.is_empty() {
206            let named: Vec<String> = memory
207                .holders
208                .iter()
209                .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
210                .collect();
211            let note = match named.as_slice() {
212                [one] => format!("most {one}"),
213                [rest @ .., last] => format!("most {} and {last}", rest.join(", ")),
214                [] => String::new(),
215            };
216            figures.push(HostFigure {
217                note: Some(note),
218                ..HostFigure::new("Outside CI", size(memory.outside()))
219            });
220        }
221        figures
222    }
223
224    /// What the host's `limit` leaves the slice of `max`, its `MemoryMax`: `25G of its 46G`. A limit past `max` holds
225    /// back room only where `max` reads a larger credit than the limit (ADR 0099, 0108). Nothing leaves a slice more
226    /// than it has, so it says what the limit goes by: `47.5G for the 44.3G reserved there`, or while the host's limit
227    /// takes a credit, `47.5G for the 40G counted there`.
228    fn host_share(&self, limit: u64, max: u64) -> String {
229        if limit > max {
230            let host = self.host.as_ref();
231            let (counts, verb) = match host.filter(|host| host.credit.is_some()) {
232                Some(host) => (host.counts(self.memory_in_use), "counted"),
233                None => (self.memory_in_use, "reserved"),
234            };
235            format!(
236                "{} for the {} {verb} there",
237                ByteSize(limit),
238                ByteSize(counts)
239            )
240        } else {
241            format!("{} of its {}", ByteSize(limit), ByteSize(max))
242        }
243    }
244
245    /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
246    /// nothing is free.
247    fn closed(&self) -> bool {
248        self.held && self.admission == AdmissionPolicy::Measured
249    }
250
251    /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
252    /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
253    /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
254    pub fn host_withheld(&self) -> Option<u64> {
255        let host = self.host.as_ref().filter(|host| !host.stalled)?;
256        let remaining = self.remaining()?;
257        (host.spare < remaining).then(|| remaining - host.spare)
258    }
259
260    /// The budget as one line, its clauses joined by `; `.
261    pub fn line(&self) -> String {
262        self.clauses().join("; ")
263    }
264
265    /// What `MemoryMax` leaves beside what the running containers count (`Budget::counted`), in bytes: what a new
266    /// runner must fit in. `None` without `MemoryMax`.
267    pub fn remaining(&self) -> Option<u64> {
268        self.memory_max
269            .map(|max| max.saturating_sub(self.counted()))
270    }
271}
272
273impl Counted {
274    /// What the growth count is made of, or why it rests, as every client says it after the budget's figure: `what runs
275    /// uses 15.7G and keeps 12.9G for its growth`; while it rests, `the growth count rests until 15:02 UTC: what ran
276    /// grew 14.1G in 10m, past the 12.9G kept`.
277    pub fn clause(&self) -> String {
278        let size = |bytes: u64| ByteSize(bytes).to_string();
279        let rests = |until: u64| format!("the growth count rests until {}", utc_time(until));
280        match (self.brake, self.kept) {
281            (None, Some(kept)) => format!(
282                "what runs uses {} and keeps {} for its growth",
283                size(self.used),
284                size(kept)
285            ),
286            (Some(CountBrake::Grew { grew, kept, until }), _) => format!(
287                "{}: what ran grew {} in {}, past the {} kept",
288                rests(until),
289                size(grew),
290                duration(super::GROWTH_WINDOW.as_secs()),
291                size(kept)
292            ),
293            (Some(CountBrake::Restarted { until }), _) => {
294                format!(
295                    "{}: too soon after a start or a reload to judge growth",
296                    rests(until)
297                )
298            }
299            (Some(CountBrake::Guard { until: Some(until) }), _) => {
300                format!("{}: the host ran under its line", rests(until))
301            }
302            (Some(CountBrake::Guard { until: None }) | None, _) => {
303                "the growth count rests while the host is under its line or a reading is missing"
304                    .to_owned()
305            }
306            (Some(CountBrake::Unknown), _) => "the growth count rests".to_owned(),
307        }
308    }
309}
310
311impl HostMemory {
312    /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
313    /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
314    /// growing to 24G`.
315    pub fn available_line(&self) -> String {
316        let line = format!(
317            "{} of {} available",
318            ByteSize(self.available),
319            ByteSize(self.total)
320        );
321        match self.growable() {
322            0 => line,
323            _ => format!("{line}, growing to {}", ByteSize(self.grown())),
324        }
325    }
326}
327
328impl Status {
329    /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
330    /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
331    /// the room no runner may start in; and the whole the blocks are scaled against.
332    pub fn budget_parts(&self) -> BudgetParts {
333        let budget = &self.budget;
334        let slots: Vec<SlotPart> = self
335            .profiles
336            .iter()
337            .enumerate()
338            .flat_map(|(profile, status)| {
339                status.slots.iter().filter_map(move |slot| {
340                    let memory = match budget.admission {
341                        AdmissionPolicy::Caps => match &slot.runner {
342                            Some(runner) if runner.reserved > 0 => runner.reserved,
343                            _ => status.memory,
344                        },
345                        AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
346                    };
347                    Some(SlotPart {
348                        profile,
349                        slot: slot.slot,
350                        memory,
351                        reserved: slot.runner.is_some(),
352                    })
353                })
354            })
355            .collect();
356        let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
357            kind: HeldKind::Mirror,
358            memory: budget.mirror_memory,
359        });
360        let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
361            kind: HeldKind::Seed {
362                profile: seed.profile.clone(),
363            },
364            memory: seed.memory,
365        });
366        let shells = budget.shell_memory.iter().map(|shell| HeldPart {
367            kind: HeldKind::Shell { job: shell.job },
368            memory: shell.memory,
369        });
370        let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
371        let drawn: u64 = slots
372            .iter()
373            .map(|part| part.memory)
374            .chain(held.iter().map(|part| part.memory))
375            .sum();
376        let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
377        let unclaimed: u64 = slots
378            .iter()
379            .filter(|part| !part.reserved)
380            .map(|part| part.memory)
381            .sum();
382        let (free, closed) = budget.room_parts(drawn, unclaimed);
383        BudgetParts {
384            slots,
385            held,
386            free,
387            closed,
388            whole,
389        }
390    }
391}
392
393impl Budget {
394    /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
395    /// of it no runner may start in.
396    fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
397        let Some(max) = self.memory_max else {
398            return (None, None);
399        };
400        let room = max.saturating_sub(drawn);
401        // While no runner starts, all the room is closed.
402        let shut = if self.closed() {
403            Some((
404                ClosedKind::Held,
405                "closed: the slice is past 90% of its MemoryMax, so no runner starts",
406            ))
407        } else if self.host.as_ref().is_some_and(|host| host.stalled) {
408            Some((
409                ClosedKind::Stalled,
410                "closed: the host's programs stall waiting for memory, so no runner starts",
411            ))
412        } else {
413            None
414        };
415        if let Some((kind, text)) = shut {
416            let closed = (room > 0).then(|| ClosedPart {
417                memory: room,
418                kind,
419                reading: Reading {
420                    figure: ByteSize(room).to_string(),
421                    text: text.to_owned(),
422                },
423            });
424            return (None, closed);
425        }
426        let withheld = self.host_withheld();
427        // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
428        // host may hold their room too.
429        let host = withheld
430            .map(|withheld| withheld.min(room))
431            .filter(|&memory| memory > 0);
432        let (host_limit, spare) = self
433            .host
434            .as_ref()
435            .map_or((0, 0), |host| (host.limit, host.spare));
436        let memory = room - host.unwrap_or(0);
437        let mut text = match (unclaimed, withheld) {
438            (0, None) => format!("of {} free", ByteSize(max)),
439            (_, None) => format!(
440                "of {} free past every slot's cap; {} is free beside what runs",
441                ByteSize(max),
442                ByteSize(self.remaining().unwrap_or(0))
443            ),
444            (0, Some(_)) => format!(
445                "free; the host leaves the slice {}",
446                self.host_share(host_limit, max)
447            ),
448            (_, Some(_)) => format!(
449                "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
450                ByteSize(spare),
451                self.host_share(host_limit, max)
452            ),
453        };
454        // The blocks stay what runners reserve, so what pooling adds is said and not drawn.
455        if let Some(pooled) = &self.pooled {
456            text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
457        }
458        let free = (memory > 0).then(|| FreePart {
459            memory,
460            reading: Reading {
461                figure: ByteSize(memory).to_string(),
462                text,
463            },
464        });
465        let host = host.map(|memory| ClosedPart {
466            memory,
467            kind: ClosedKind::Host,
468            reading: Reading {
469                figure: ByteSize(memory).to_string(),
470                // Past `MemoryMax` the plain count holds this back, not the host's programs (`Budget::host_share`).
471                text: if host_limit > max {
472                    format!(
473                        "held back by what runs reserves: the host leaves the slice {}",
474                        self.host_share(host_limit, max)
475                    )
476                } else {
477                    format!(
478                        "{} for its other programs and page cache, which leave the slice {}",
479                        ClosedKind::Host.name(),
480                        ByteSize(host_limit)
481                    )
482                },
483            },
484        });
485        (free, host)
486    }
487}
488
489/// The budget as a bar draws it, from `Status::budget_parts`.
490#[derive(Debug, Clone, PartialEq, Eq)]
491pub struct BudgetParts {
492    /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
493    pub slots: Vec<SlotPart>,
494    /// The mirror, each seed container, then each shell open in a record.
495    pub held: Vec<HeldPart>,
496    /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
497    /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
498    /// stalls.
499    pub free: Option<FreePart>,
500    /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
501    /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
502    pub closed: Option<ClosedPart>,
503    /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
504    pub whole: u64,
505}
506
507/// A slot's block of the budget.
508#[derive(Debug, Clone, Copy, PartialEq, Eq)]
509pub struct SlotPart {
510    /// Its profile's index in `Status::profiles`.
511    pub profile: usize,
512    /// The slot, 1-based.
513    pub slot: usize,
514    /// Its size in bytes: what its runner reserves, or a free slot's cap.
515    pub memory: u64,
516    /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
517    pub reserved: bool,
518}
519
520/// Memory the budget holds for something other than a slot.
521#[derive(Debug, Clone, PartialEq, Eq)]
522pub struct HeldPart {
523    /// What holds it.
524    pub kind: HeldKind,
525    /// Its cap in bytes.
526    pub memory: u64,
527}
528
529impl HeldPart {
530    /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
531    pub fn name(&self) -> String {
532        match &self.kind {
533            HeldKind::Mirror => "mirror".to_owned(),
534            HeldKind::Seed { profile } => format!("{profile} seed"),
535            HeldKind::Shell { job } => format!("job {job} shell"),
536        }
537    }
538
539    /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
540    /// in job 42's record`.
541    pub fn holder(&self) -> String {
542        match &self.kind {
543            HeldKind::Mirror => "held by the mirror".to_owned(),
544            HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
545            HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
546        }
547    }
548}
549
550/// What holds a part of the budget beside the slots.
551#[derive(Debug, Clone, PartialEq, Eq)]
552pub enum HeldKind {
553    /// The registry mirror.
554    Mirror,
555    /// A profile's seed container, while it reseeds.
556    Seed {
557        /// The profile it reseeds.
558        profile: String,
559    },
560    /// A shell open in a job's record.
561    Shell {
562        /// The job.
563        job: u64,
564    },
565}
566
567/// What the bar leaves free past every block, less what the host holds back.
568#[derive(Debug, Clone, PartialEq, Eq)]
569pub struct FreePart {
570    /// In bytes.
571    pub memory: u64,
572    /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
573    /// leaves the slice 25.27G of its 46G` (`Budget::host_share`).
574    pub reading: Reading,
575}
576
577/// Room past the bar's blocks that no runner may start in.
578#[derive(Debug, Clone, PartialEq, Eq)]
579pub struct ClosedPart {
580    /// In bytes.
581    pub memory: u64,
582    /// What closes it.
583    pub kind: ClosedKind,
584    /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
585    /// which leave the slice 25.27G`, or past `MemoryMax` `1.5G` `held back by what runs reserves: the host leaves the
586    /// slice 24.5G for the 22G reserved there`.
587    pub reading: Reading,
588}
589
590/// What closes room past the bar's blocks.
591#[derive(Debug, Clone, Copy, PartialEq, Eq)]
592pub enum ClosedKind {
593    /// The host's other programs and its page cache need it.
594    Host,
595    /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
596    Held,
597    /// The host's programs stall waiting for memory (`HostBudget::stalled`).
598    Stalled,
599}
600
601impl ClosedKind {
602    /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
603    /// stalls`.
604    pub fn name(self) -> &'static str {
605        match self {
606            Self::Host => "held by the host",
607            Self::Held => "closed past 90%",
608            Self::Stalled => "closed while the host stalls",
609        }
610    }
611}
612
613#[cfg(test)]
614mod tests;