Skip to main content

runnerd_core/api/
budget.rs

1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::{AdmissionPolicy, Budget, HostMemory, Status, percent};
5use crate::config::ByteSize;
6
7/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
8/// sets the figure apart.
9#[derive(Debug, Clone, PartialEq, Eq)]
10pub struct Reading {
11    /// Such as `8G`.
12    pub figure: String,
13    /// What follows it, such as `of 24G reserved in runnerd.slice`.
14    pub text: String,
15}
16
17impl Reading {
18    /// The figure and its words as one line.
19    pub fn line(&self) -> String {
20        format!("{} {}", self.figure, self.text)
21    }
22}
23
24impl Budget {
25    /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
26    /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; while pooling takes something off, `36.8G`
27    /// `of 46G counted in ci.slice by predicted peaks: 42.8G reserved, 6G pooled`; how runners are admitted
28    /// (`Budget::admission_note`) follows the slice.
29    pub fn memory_reading(&self) -> Reading {
30        let of = self
31            .memory_max
32            .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
33        let verb = if self.pooled.is_some() {
34            "counted"
35        } else {
36            "reserved"
37        };
38        let mut text = format!("{of}{verb} in {}", self.slice);
39        if let Some(note) = self.admission_note() {
40            text = format!("{text} {note}");
41        }
42        if let Some(pooled) = &self.pooled {
43            let reserved = ByteSize(self.memory_in_use);
44            text = format!(
45                "{text}: {reserved} reserved, {} pooled",
46                ByteSize(pooled.credit)
47            );
48        }
49        if self.memory_max.is_none() {
50            text.push_str(", which has no MemoryMax");
51        }
52        Reading {
53            figure: ByteSize(self.counted()).to_string(),
54            text,
55        }
56    }
57
58    /// What `MemoryMax` goes by, in bytes: `memory_in_use`, less what pooling takes off it (ADR 0099).
59    pub fn counted(&self) -> u64 {
60        self.pooled
61            .map_or(self.memory_in_use, |pooled| pooled.counted)
62    }
63
64    /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
65    /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
66    pub fn clauses(&self) -> Vec<String> {
67        let mut clauses = vec![self.memory_reading().line()];
68        clauses.extend(self.cpu.map(|cpu| cpu.line()));
69        clauses.extend(self.host_line());
70        clauses
71    }
72
73    /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
74    /// slice 40G`; on a host with swap, `host: 31G of 64G available, swap 12.1G of 15.5G used, …` (ADR 0101); when that
75    /// is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
76    /// of its 46G, so 6.5G is free`, or past `MemoryMax` (`Budget::host_share`) `…; leaves the slice 47.5G for the 44.3G
77    /// reserved there, so 3.2G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
78    /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
79    /// (`Budget::held`), nothing is free, so it says no figure of it. With programs outside the slice large enough to
80    /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
81    /// reading of the host.
82    pub fn host_line(&self) -> Option<String> {
83        let host = self.host.as_ref()?;
84        let mut line = format!("host: {}", host.memory.available_line());
85        if let Some(swap) = host.memory.swap {
86            line = format!(
87                "{line}, swap {} of {} used",
88                ByteSize(swap.used()),
89                ByteSize(swap.total)
90            );
91        }
92        if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
93            line = format!("{line}, {} stalled for memory", percent(full.avg10));
94        }
95        let free = match host.spare {
96            0 => "nothing".to_owned(),
97            spare => ByteSize(spare).to_string(),
98        };
99        let limit = ByteSize(host.limit);
100        line = if host.stalled {
101            format!("{line}; stalling, so no runner starts")
102        } else if self.closed() {
103            format!("{line}; leaves the slice {limit}")
104        } else {
105            match self.memory_max {
106                Some(max) if self.host_withheld().is_some() => format!(
107                    "{line}; leaves the slice {}, so {free} is free",
108                    self.host_share(host.limit, max)
109                ),
110                Some(_) => format!("{line}; leaves the slice {limit}"),
111                None => format!("{line}; leaves the slice {limit}, so {free} is free"),
112            }
113        };
114        if !host.memory.holders.is_empty() {
115            line = format!("{line}; {}", host.memory.outside_line());
116        }
117        Some(line)
118    }
119
120    /// What the host's `limit` leaves the slice of `max`, its `MemoryMax`: `25G of its 46G`. A limit past `max` holds
121    /// back room only under a credit, which `max` reads and the limit does not (ADR 0099). Nothing leaves a slice more
122    /// than it has, so it says what the limit goes by: `47.5G for the 44.3G reserved there`.
123    fn host_share(&self, limit: u64, max: u64) -> String {
124        if limit > max {
125            format!(
126                "{} for the {} reserved there",
127                ByteSize(limit),
128                ByteSize(self.memory_in_use)
129            )
130        } else {
131            format!("{} of its {}", ByteSize(limit), ByteSize(max))
132        }
133    }
134
135    /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
136    /// nothing is free.
137    fn closed(&self) -> bool {
138        self.held && self.admission == AdmissionPolicy::Measured
139    }
140
141    /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
142    /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
143    /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
144    pub fn host_withheld(&self) -> Option<u64> {
145        let host = self.host.as_ref().filter(|host| !host.stalled)?;
146        let remaining = self.remaining()?;
147        (host.spare < remaining).then(|| remaining - host.spare)
148    }
149
150    /// The budget as one line, its clauses joined by `; `.
151    pub fn line(&self) -> String {
152        self.clauses().join("; ")
153    }
154
155    /// What `MemoryMax` leaves beside what the running containers count (`Budget::counted`), in bytes: what a new
156    /// runner must fit in. `None` without `MemoryMax`.
157    pub fn remaining(&self) -> Option<u64> {
158        self.memory_max
159            .map(|max| max.saturating_sub(self.counted()))
160    }
161}
162
163impl HostMemory {
164    /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
165    /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
166    /// growing to 24G`.
167    pub fn available_line(&self) -> String {
168        let line = format!(
169            "{} of {} available",
170            ByteSize(self.available),
171            ByteSize(self.total)
172        );
173        match self.growable() {
174            0 => line,
175            _ => format!("{line}, growing to {}", ByteSize(self.grown())),
176        }
177    }
178}
179
180impl Status {
181    /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
182    /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
183    /// the room no runner may start in; and the whole the blocks are scaled against.
184    pub fn budget_parts(&self) -> BudgetParts {
185        let budget = &self.budget;
186        let slots: Vec<SlotPart> = self
187            .profiles
188            .iter()
189            .enumerate()
190            .flat_map(|(profile, status)| {
191                status.slots.iter().filter_map(move |slot| {
192                    let memory = match budget.admission {
193                        AdmissionPolicy::Caps => match &slot.runner {
194                            Some(runner) if runner.reserved > 0 => runner.reserved,
195                            _ => status.memory,
196                        },
197                        AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
198                    };
199                    Some(SlotPart {
200                        profile,
201                        slot: slot.slot,
202                        memory,
203                        reserved: slot.runner.is_some(),
204                    })
205                })
206            })
207            .collect();
208        let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
209            kind: HeldKind::Mirror,
210            memory: budget.mirror_memory,
211        });
212        let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
213            kind: HeldKind::Seed {
214                profile: seed.profile.clone(),
215            },
216            memory: seed.memory,
217        });
218        let shells = budget.shell_memory.iter().map(|shell| HeldPart {
219            kind: HeldKind::Shell { job: shell.job },
220            memory: shell.memory,
221        });
222        let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
223        let drawn: u64 = slots
224            .iter()
225            .map(|part| part.memory)
226            .chain(held.iter().map(|part| part.memory))
227            .sum();
228        let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
229        let unclaimed: u64 = slots
230            .iter()
231            .filter(|part| !part.reserved)
232            .map(|part| part.memory)
233            .sum();
234        let (free, closed) = budget.room_parts(drawn, unclaimed);
235        BudgetParts {
236            slots,
237            held,
238            free,
239            closed,
240            whole,
241        }
242    }
243}
244
245impl Budget {
246    /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
247    /// of it no runner may start in.
248    fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
249        let Some(max) = self.memory_max else {
250            return (None, None);
251        };
252        let room = max.saturating_sub(drawn);
253        // While no runner starts, all the room is closed.
254        let shut = if self.closed() {
255            Some((
256                ClosedKind::Held,
257                "closed: the slice is past 90% of its MemoryMax, so no runner starts",
258            ))
259        } else if self.host.as_ref().is_some_and(|host| host.stalled) {
260            Some((
261                ClosedKind::Stalled,
262                "closed: the host's programs stall waiting for memory, so no runner starts",
263            ))
264        } else {
265            None
266        };
267        if let Some((kind, text)) = shut {
268            let closed = (room > 0).then(|| ClosedPart {
269                memory: room,
270                kind,
271                reading: Reading {
272                    figure: ByteSize(room).to_string(),
273                    text: text.to_owned(),
274                },
275            });
276            return (None, closed);
277        }
278        let withheld = self.host_withheld();
279        // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
280        // host may hold their room too.
281        let host = withheld
282            .map(|withheld| withheld.min(room))
283            .filter(|&memory| memory > 0);
284        let (host_limit, spare) = self
285            .host
286            .as_ref()
287            .map_or((0, 0), |host| (host.limit, host.spare));
288        let memory = room - host.unwrap_or(0);
289        let mut text = match (unclaimed, withheld) {
290            (0, None) => format!("of {} free", ByteSize(max)),
291            (_, None) => format!(
292                "of {} free past every slot's cap; {} is free beside what runs",
293                ByteSize(max),
294                ByteSize(self.remaining().unwrap_or(0))
295            ),
296            (0, Some(_)) => format!(
297                "free; the host leaves the slice {}",
298                self.host_share(host_limit, max)
299            ),
300            (_, Some(_)) => format!(
301                "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
302                ByteSize(spare),
303                self.host_share(host_limit, max)
304            ),
305        };
306        // The blocks stay what runners reserve, so what pooling adds is said and not drawn.
307        if let Some(pooled) = &self.pooled {
308            text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
309        }
310        let free = (memory > 0).then(|| FreePart {
311            memory,
312            reading: Reading {
313                figure: ByteSize(memory).to_string(),
314                text,
315            },
316        });
317        let host = host.map(|memory| ClosedPart {
318            memory,
319            kind: ClosedKind::Host,
320            reading: Reading {
321                figure: ByteSize(memory).to_string(),
322                // Past `MemoryMax` the plain count holds this back, not the host's programs (`Budget::host_share`).
323                text: if host_limit > max {
324                    format!(
325                        "held back by what runs reserves: the host leaves the slice {}",
326                        self.host_share(host_limit, max)
327                    )
328                } else {
329                    format!(
330                        "{} for its other programs and page cache, which leave the slice {}",
331                        ClosedKind::Host.name(),
332                        ByteSize(host_limit)
333                    )
334                },
335            },
336        });
337        (free, host)
338    }
339}
340
341/// The budget as a bar draws it, from `Status::budget_parts`.
342#[derive(Debug, Clone, PartialEq, Eq)]
343pub struct BudgetParts {
344    /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
345    pub slots: Vec<SlotPart>,
346    /// The mirror, each seed container, then each shell open in a record.
347    pub held: Vec<HeldPart>,
348    /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
349    /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
350    /// stalls.
351    pub free: Option<FreePart>,
352    /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
353    /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
354    pub closed: Option<ClosedPart>,
355    /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
356    pub whole: u64,
357}
358
359/// A slot's block of the budget.
360#[derive(Debug, Clone, Copy, PartialEq, Eq)]
361pub struct SlotPart {
362    /// Its profile's index in `Status::profiles`.
363    pub profile: usize,
364    /// The slot, 1-based.
365    pub slot: usize,
366    /// Its size in bytes: what its runner reserves, or a free slot's cap.
367    pub memory: u64,
368    /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
369    pub reserved: bool,
370}
371
372/// Memory the budget holds for something other than a slot.
373#[derive(Debug, Clone, PartialEq, Eq)]
374pub struct HeldPart {
375    /// What holds it.
376    pub kind: HeldKind,
377    /// Its cap in bytes.
378    pub memory: u64,
379}
380
381impl HeldPart {
382    /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
383    pub fn name(&self) -> String {
384        match &self.kind {
385            HeldKind::Mirror => "mirror".to_owned(),
386            HeldKind::Seed { profile } => format!("{profile} seed"),
387            HeldKind::Shell { job } => format!("job {job} shell"),
388        }
389    }
390
391    /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
392    /// in job 42's record`.
393    pub fn holder(&self) -> String {
394        match &self.kind {
395            HeldKind::Mirror => "held by the mirror".to_owned(),
396            HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
397            HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
398        }
399    }
400}
401
402/// What holds a part of the budget beside the slots.
403#[derive(Debug, Clone, PartialEq, Eq)]
404pub enum HeldKind {
405    /// The registry mirror.
406    Mirror,
407    /// A profile's seed container, while it reseeds.
408    Seed {
409        /// The profile it reseeds.
410        profile: String,
411    },
412    /// A shell open in a job's record.
413    Shell {
414        /// The job.
415        job: u64,
416    },
417}
418
419/// What the bar leaves free past every block, less what the host holds back.
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub struct FreePart {
422    /// In bytes.
423    pub memory: u64,
424    /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
425    /// leaves the slice 25.27G of its 46G` (`Budget::host_share`).
426    pub reading: Reading,
427}
428
429/// Room past the bar's blocks that no runner may start in.
430#[derive(Debug, Clone, PartialEq, Eq)]
431pub struct ClosedPart {
432    /// In bytes.
433    pub memory: u64,
434    /// What closes it.
435    pub kind: ClosedKind,
436    /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
437    /// which leave the slice 25.27G`, or past `MemoryMax` `1.5G` `held back by what runs reserves: the host leaves the
438    /// slice 24.5G for the 22G reserved there`.
439    pub reading: Reading,
440}
441
442/// What closes room past the bar's blocks.
443#[derive(Debug, Clone, Copy, PartialEq, Eq)]
444pub enum ClosedKind {
445    /// The host's other programs and its page cache need it.
446    Host,
447    /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
448    Held,
449    /// The host's programs stall waiting for memory (`HostBudget::stalled`).
450    Stalled,
451}
452
453impl ClosedKind {
454    /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
455    /// stalls`.
456    pub fn name(self) -> &'static str {
457        match self {
458            Self::Host => "held by the host",
459            Self::Held => "closed past 90%",
460            Self::Stalled => "closed while the host stalls",
461        }
462    }
463}
464
465#[cfg(test)]
466mod tests;