Skip to main content

runnerd_core/api/
budget.rs

1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::{AdmissionPolicy, Budget, HostMemory, Status, percent};
5use crate::config::ByteSize;
6
7/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
8/// sets the figure apart.
9#[derive(Debug, Clone, PartialEq, Eq)]
10pub struct Reading {
11    /// Such as `8G`.
12    pub figure: String,
13    /// What follows it, such as `of 24G reserved in runnerd.slice`.
14    pub text: String,
15}
16
17impl Reading {
18    /// The figure and its words as one line.
19    pub fn line(&self) -> String {
20        format!("{} {}", self.figure, self.text)
21    }
22}
23
24impl Budget {
25    /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
26    /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; how runners are admitted
27    /// (`Budget::admission_note`) follows the slice.
28    pub fn memory_reading(&self) -> Reading {
29        let of = self
30            .memory_max
31            .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
32        let mut text = format!("{of}reserved in {}", self.slice);
33        if let Some(note) = self.admission_note() {
34            text = format!("{text} {note}");
35        }
36        if self.memory_max.is_none() {
37            text.push_str(", which has no MemoryMax");
38        }
39        Reading {
40            figure: ByteSize(self.memory_in_use).to_string(),
41            text,
42        }
43    }
44
45    /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
46    /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
47    pub fn clauses(&self) -> Vec<String> {
48        let mut clauses = vec![self.memory_reading().line()];
49        clauses.extend(self.cpu.map(|cpu| cpu.line()));
50        clauses.extend(self.host_line());
51        clauses
52    }
53
54    /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
55    /// slice 40G`; when that is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
56    /// of its 46G, so 6.5G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
57    /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
58    /// (`Budget::held`), nothing is free, so it says no figure of it. With programs outside the slice large enough to
59    /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
60    /// reading of the host.
61    pub fn host_line(&self) -> Option<String> {
62        let host = self.host.as_ref()?;
63        let mut line = format!("host: {}", host.memory.available_line());
64        if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
65            line = format!("{line}, {} stalled for memory", percent(full.avg10));
66        }
67        let free = match host.spare {
68            0 => "nothing".to_owned(),
69            spare => ByteSize(spare).to_string(),
70        };
71        let limit = ByteSize(host.limit);
72        line = if host.stalled {
73            format!("{line}; stalling, so no runner starts")
74        } else if self.closed() {
75            format!("{line}; leaves the slice {limit}")
76        } else {
77            match self.memory_max {
78                Some(max) if self.host_withheld().is_some() => format!(
79                    "{line}; leaves the slice {limit} of its {}, so {free} is free",
80                    ByteSize(max)
81                ),
82                Some(_) => format!("{line}; leaves the slice {limit}"),
83                None => format!("{line}; leaves the slice {limit}, so {free} is free"),
84            }
85        };
86        if !host.memory.holders.is_empty() {
87            line = format!("{line}; {}", host.memory.outside_line());
88        }
89        Some(line)
90    }
91
92    /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
93    /// nothing is free.
94    fn closed(&self) -> bool {
95        self.held && self.admission == AdmissionPolicy::Measured
96    }
97
98    /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
99    /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
100    /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
101    pub fn host_withheld(&self) -> Option<u64> {
102        let host = self.host.as_ref().filter(|host| !host.stalled)?;
103        let remaining = self.remaining()?;
104        (host.spare < remaining).then(|| remaining - host.spare)
105    }
106
107    /// The budget as one line, its clauses joined by `; `.
108    pub fn line(&self) -> String {
109        self.clauses().join("; ")
110    }
111
112    /// What `MemoryMax` leaves beside every running container's reservation, in bytes: what a new runner must fit
113    /// in. `None` without `MemoryMax`.
114    pub fn remaining(&self) -> Option<u64> {
115        self.memory_max
116            .map(|max| max.saturating_sub(self.memory_in_use))
117    }
118}
119
120impl HostMemory {
121    /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
122    /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
123    /// growing to 24G`.
124    pub fn available_line(&self) -> String {
125        let line = format!(
126            "{} of {} available",
127            ByteSize(self.available),
128            ByteSize(self.total)
129        );
130        match self.growable() {
131            0 => line,
132            _ => format!("{line}, growing to {}", ByteSize(self.grown())),
133        }
134    }
135}
136
137impl Status {
138    /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
139    /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
140    /// the room no runner may start in; and the whole the blocks are scaled against.
141    pub fn budget_parts(&self) -> BudgetParts {
142        let budget = &self.budget;
143        let slots: Vec<SlotPart> = self
144            .profiles
145            .iter()
146            .enumerate()
147            .flat_map(|(profile, status)| {
148                status.slots.iter().filter_map(move |slot| {
149                    let memory = match budget.admission {
150                        AdmissionPolicy::Caps => match &slot.runner {
151                            Some(runner) if runner.reserved > 0 => runner.reserved,
152                            _ => status.memory,
153                        },
154                        AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
155                    };
156                    Some(SlotPart {
157                        profile,
158                        slot: slot.slot,
159                        memory,
160                        reserved: slot.runner.is_some(),
161                    })
162                })
163            })
164            .collect();
165        let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
166            kind: HeldKind::Mirror,
167            memory: budget.mirror_memory,
168        });
169        let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
170            kind: HeldKind::Seed {
171                profile: seed.profile.clone(),
172            },
173            memory: seed.memory,
174        });
175        let shells = budget.shell_memory.iter().map(|shell| HeldPart {
176            kind: HeldKind::Shell { job: shell.job },
177            memory: shell.memory,
178        });
179        let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
180        let drawn: u64 = slots
181            .iter()
182            .map(|part| part.memory)
183            .chain(held.iter().map(|part| part.memory))
184            .sum();
185        let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
186        let unclaimed: u64 = slots
187            .iter()
188            .filter(|part| !part.reserved)
189            .map(|part| part.memory)
190            .sum();
191        let (free, closed) = budget.room_parts(drawn, unclaimed);
192        BudgetParts {
193            slots,
194            held,
195            free,
196            closed,
197            whole,
198        }
199    }
200}
201
202impl Budget {
203    /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
204    /// of it no runner may start in.
205    fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
206        let Some(max) = self.memory_max else {
207            return (None, None);
208        };
209        let room = max.saturating_sub(drawn);
210        // While no runner starts, all the room is closed.
211        let shut = if self.closed() {
212            Some((
213                ClosedKind::Held,
214                "closed: the slice is past 90% of its MemoryMax, so no runner starts",
215            ))
216        } else if self.host.as_ref().is_some_and(|host| host.stalled) {
217            Some((
218                ClosedKind::Stalled,
219                "closed: the host's programs stall waiting for memory, so no runner starts",
220            ))
221        } else {
222            None
223        };
224        if let Some((kind, text)) = shut {
225            let closed = (room > 0).then(|| ClosedPart {
226                memory: room,
227                kind,
228                reading: Reading {
229                    figure: ByteSize(room).to_string(),
230                    text: text.to_owned(),
231                },
232            });
233            return (None, closed);
234        }
235        let withheld = self.host_withheld();
236        // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
237        // host may hold their room too.
238        let host = withheld
239            .map(|withheld| withheld.min(room))
240            .filter(|&memory| memory > 0);
241        let (host_limit, spare) = self
242            .host
243            .as_ref()
244            .map_or((0, 0), |host| (host.limit, host.spare));
245        let memory = room - host.unwrap_or(0);
246        let text = match (unclaimed, withheld) {
247            (0, None) => format!("of {} free", ByteSize(max)),
248            (_, None) => format!(
249                "of {} free past every slot's cap; {} is free beside what runs",
250                ByteSize(max),
251                ByteSize(self.remaining().unwrap_or(0))
252            ),
253            (0, Some(_)) => format!(
254                "free; the host leaves the slice {} of its {}",
255                ByteSize(host_limit),
256                ByteSize(max)
257            ),
258            (_, Some(_)) => format!(
259                "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {} of its {}",
260                ByteSize(spare),
261                ByteSize(host_limit),
262                ByteSize(max)
263            ),
264        };
265        let free = (memory > 0).then(|| FreePart {
266            memory,
267            reading: Reading {
268                figure: ByteSize(memory).to_string(),
269                text,
270            },
271        });
272        let host = host.map(|memory| ClosedPart {
273            memory,
274            kind: ClosedKind::Host,
275            reading: Reading {
276                figure: ByteSize(memory).to_string(),
277                text: format!(
278                    "{} for its other programs and page cache, which leave the slice {}",
279                    ClosedKind::Host.name(),
280                    ByteSize(host_limit)
281                ),
282            },
283        });
284        (free, host)
285    }
286}
287
288/// The budget as a bar draws it, from `Status::budget_parts`.
289#[derive(Debug, Clone, PartialEq, Eq)]
290pub struct BudgetParts {
291    /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
292    pub slots: Vec<SlotPart>,
293    /// The mirror, each seed container, then each shell open in a record.
294    pub held: Vec<HeldPart>,
295    /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
296    /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
297    /// stalls.
298    pub free: Option<FreePart>,
299    /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
300    /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
301    pub closed: Option<ClosedPart>,
302    /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
303    pub whole: u64,
304}
305
306/// A slot's block of the budget.
307#[derive(Debug, Clone, Copy, PartialEq, Eq)]
308pub struct SlotPart {
309    /// Its profile's index in `Status::profiles`.
310    pub profile: usize,
311    /// The slot, 1-based.
312    pub slot: usize,
313    /// Its size in bytes: what its runner reserves, or a free slot's cap.
314    pub memory: u64,
315    /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
316    pub reserved: bool,
317}
318
319/// Memory the budget holds for something other than a slot.
320#[derive(Debug, Clone, PartialEq, Eq)]
321pub struct HeldPart {
322    /// What holds it.
323    pub kind: HeldKind,
324    /// Its cap in bytes.
325    pub memory: u64,
326}
327
328impl HeldPart {
329    /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
330    pub fn name(&self) -> String {
331        match &self.kind {
332            HeldKind::Mirror => "mirror".to_owned(),
333            HeldKind::Seed { profile } => format!("{profile} seed"),
334            HeldKind::Shell { job } => format!("job {job} shell"),
335        }
336    }
337
338    /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
339    /// in job 42's record`.
340    pub fn holder(&self) -> String {
341        match &self.kind {
342            HeldKind::Mirror => "held by the mirror".to_owned(),
343            HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
344            HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
345        }
346    }
347}
348
349/// What holds a part of the budget beside the slots.
350#[derive(Debug, Clone, PartialEq, Eq)]
351pub enum HeldKind {
352    /// The registry mirror.
353    Mirror,
354    /// A profile's seed container, while it reseeds.
355    Seed {
356        /// The profile it reseeds.
357        profile: String,
358    },
359    /// A shell open in a job's record.
360    Shell {
361        /// The job.
362        job: u64,
363    },
364}
365
366/// What the bar leaves free past every block, less what the host holds back.
367#[derive(Debug, Clone, PartialEq, Eq)]
368pub struct FreePart {
369    /// In bytes.
370    pub memory: u64,
371    /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
372    /// leaves the slice 25.27G of its 46G`.
373    pub reading: Reading,
374}
375
376/// Room past the bar's blocks that no runner may start in.
377#[derive(Debug, Clone, PartialEq, Eq)]
378pub struct ClosedPart {
379    /// In bytes.
380    pub memory: u64,
381    /// What closes it.
382    pub kind: ClosedKind,
383    /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
384    /// which leave the slice 25.27G`.
385    pub reading: Reading,
386}
387
388/// What closes room past the bar's blocks.
389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
390pub enum ClosedKind {
391    /// The host's other programs and its page cache need it.
392    Host,
393    /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
394    Held,
395    /// The host's programs stall waiting for memory (`HostBudget::stalled`).
396    Stalled,
397}
398
399impl ClosedKind {
400    /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
401    /// stalls`.
402    pub fn name(self) -> &'static str {
403        match self {
404            Self::Host => "held by the host",
405            Self::Held => "closed past 90%",
406            Self::Stalled => "closed while the host stalls",
407        }
408    }
409}
410
411#[cfg(test)]
412mod tests;