runnerd_core/api/budget.rs
1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::{AdmissionPolicy, Budget, HostMemory, Status, percent};
5use crate::config::ByteSize;
6
7/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
8/// sets the figure apart.
9#[derive(Debug, Clone, PartialEq, Eq)]
10pub struct Reading {
11 /// Such as `8G`.
12 pub figure: String,
13 /// What follows it, such as `of 24G reserved in runnerd.slice`.
14 pub text: String,
15}
16
17impl Reading {
18 /// The figure and its words as one line.
19 pub fn line(&self) -> String {
20 format!("{} {}", self.figure, self.text)
21 }
22}
23
24impl Budget {
25 /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
26 /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; how runners are admitted
27 /// (`Budget::admission_note`) follows the slice.
28 pub fn memory_reading(&self) -> Reading {
29 let of = self
30 .memory_max
31 .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
32 let mut text = format!("{of}reserved in {}", self.slice);
33 if let Some(note) = self.admission_note() {
34 text = format!("{text} {note}");
35 }
36 if self.memory_max.is_none() {
37 text.push_str(", which has no MemoryMax");
38 }
39 Reading {
40 figure: ByteSize(self.memory_in_use).to_string(),
41 text,
42 }
43 }
44
45 /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
46 /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
47 pub fn clauses(&self) -> Vec<String> {
48 let mut clauses = vec![self.memory_reading().line()];
49 clauses.extend(self.cpu.map(|cpu| cpu.line()));
50 clauses.extend(self.host_line());
51 clauses
52 }
53
54 /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
55 /// slice 40G`; when that is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
56 /// of its 46G, so 6.5G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
57 /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
58 /// (`Budget::held`), nothing is free, so it says no figure of it. With programs outside the slice large enough to
59 /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
60 /// reading of the host.
61 pub fn host_line(&self) -> Option<String> {
62 let host = self.host.as_ref()?;
63 let mut line = format!("host: {}", host.memory.available_line());
64 if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
65 line = format!("{line}, {} stalled for memory", percent(full.avg10));
66 }
67 let free = match host.spare {
68 0 => "nothing".to_owned(),
69 spare => ByteSize(spare).to_string(),
70 };
71 let limit = ByteSize(host.limit);
72 line = if host.stalled {
73 format!("{line}; stalling, so no runner starts")
74 } else if self.closed() {
75 format!("{line}; leaves the slice {limit}")
76 } else {
77 match self.memory_max {
78 Some(max) if self.host_withheld().is_some() => format!(
79 "{line}; leaves the slice {limit} of its {}, so {free} is free",
80 ByteSize(max)
81 ),
82 Some(_) => format!("{line}; leaves the slice {limit}"),
83 None => format!("{line}; leaves the slice {limit}, so {free} is free"),
84 }
85 };
86 if !host.memory.holders.is_empty() {
87 line = format!("{line}; {}", host.memory.outside_line());
88 }
89 Some(line)
90 }
91
92 /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
93 /// nothing is free.
94 fn closed(&self) -> bool {
95 self.held && self.admission == AdmissionPolicy::Measured
96 }
97
98 /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
99 /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
100 /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
101 pub fn host_withheld(&self) -> Option<u64> {
102 let host = self.host.as_ref().filter(|host| !host.stalled)?;
103 let remaining = self.remaining()?;
104 (host.spare < remaining).then(|| remaining - host.spare)
105 }
106
107 /// The budget as one line, its clauses joined by `; `.
108 pub fn line(&self) -> String {
109 self.clauses().join("; ")
110 }
111
112 /// What `MemoryMax` leaves beside every running container's reservation, in bytes: what a new runner must fit
113 /// in. `None` without `MemoryMax`.
114 pub fn remaining(&self) -> Option<u64> {
115 self.memory_max
116 .map(|max| max.saturating_sub(self.memory_in_use))
117 }
118}
119
120impl HostMemory {
121 /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
122 /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
123 /// growing to 24G`.
124 pub fn available_line(&self) -> String {
125 let line = format!(
126 "{} of {} available",
127 ByteSize(self.available),
128 ByteSize(self.total)
129 );
130 match self.growable() {
131 0 => line,
132 _ => format!("{line}, growing to {}", ByteSize(self.grown())),
133 }
134 }
135}
136
137impl Status {
138 /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
139 /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
140 /// the room no runner may start in; and the whole the blocks are scaled against.
141 pub fn budget_parts(&self) -> BudgetParts {
142 let budget = &self.budget;
143 let slots: Vec<SlotPart> = self
144 .profiles
145 .iter()
146 .enumerate()
147 .flat_map(|(profile, status)| {
148 status.slots.iter().filter_map(move |slot| {
149 let memory = match budget.admission {
150 AdmissionPolicy::Caps => match &slot.runner {
151 Some(runner) if runner.reserved > 0 => runner.reserved,
152 _ => status.memory,
153 },
154 AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
155 };
156 Some(SlotPart {
157 profile,
158 slot: slot.slot,
159 memory,
160 reserved: slot.runner.is_some(),
161 })
162 })
163 })
164 .collect();
165 let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
166 kind: HeldKind::Mirror,
167 memory: budget.mirror_memory,
168 });
169 let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
170 kind: HeldKind::Seed {
171 profile: seed.profile.clone(),
172 },
173 memory: seed.memory,
174 });
175 let shells = budget.shell_memory.iter().map(|shell| HeldPart {
176 kind: HeldKind::Shell { job: shell.job },
177 memory: shell.memory,
178 });
179 let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
180 let drawn: u64 = slots
181 .iter()
182 .map(|part| part.memory)
183 .chain(held.iter().map(|part| part.memory))
184 .sum();
185 let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
186 let unclaimed: u64 = slots
187 .iter()
188 .filter(|part| !part.reserved)
189 .map(|part| part.memory)
190 .sum();
191 let (free, closed) = budget.room_parts(drawn, unclaimed);
192 BudgetParts {
193 slots,
194 held,
195 free,
196 closed,
197 whole,
198 }
199 }
200}
201
202impl Budget {
203 /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
204 /// of it no runner may start in.
205 fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
206 let Some(max) = self.memory_max else {
207 return (None, None);
208 };
209 let room = max.saturating_sub(drawn);
210 // While no runner starts, all the room is closed.
211 let shut = if self.closed() {
212 Some((
213 ClosedKind::Held,
214 "closed: the slice is past 90% of its MemoryMax, so no runner starts",
215 ))
216 } else if self.host.as_ref().is_some_and(|host| host.stalled) {
217 Some((
218 ClosedKind::Stalled,
219 "closed: the host's programs stall waiting for memory, so no runner starts",
220 ))
221 } else {
222 None
223 };
224 if let Some((kind, text)) = shut {
225 let closed = (room > 0).then(|| ClosedPart {
226 memory: room,
227 kind,
228 reading: Reading {
229 figure: ByteSize(room).to_string(),
230 text: text.to_owned(),
231 },
232 });
233 return (None, closed);
234 }
235 let withheld = self.host_withheld();
236 // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
237 // host may hold their room too.
238 let host = withheld
239 .map(|withheld| withheld.min(room))
240 .filter(|&memory| memory > 0);
241 let (host_limit, spare) = self
242 .host
243 .as_ref()
244 .map_or((0, 0), |host| (host.limit, host.spare));
245 let memory = room - host.unwrap_or(0);
246 let text = match (unclaimed, withheld) {
247 (0, None) => format!("of {} free", ByteSize(max)),
248 (_, None) => format!(
249 "of {} free past every slot's cap; {} is free beside what runs",
250 ByteSize(max),
251 ByteSize(self.remaining().unwrap_or(0))
252 ),
253 (0, Some(_)) => format!(
254 "free; the host leaves the slice {} of its {}",
255 ByteSize(host_limit),
256 ByteSize(max)
257 ),
258 (_, Some(_)) => format!(
259 "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {} of its {}",
260 ByteSize(spare),
261 ByteSize(host_limit),
262 ByteSize(max)
263 ),
264 };
265 let free = (memory > 0).then(|| FreePart {
266 memory,
267 reading: Reading {
268 figure: ByteSize(memory).to_string(),
269 text,
270 },
271 });
272 let host = host.map(|memory| ClosedPart {
273 memory,
274 kind: ClosedKind::Host,
275 reading: Reading {
276 figure: ByteSize(memory).to_string(),
277 text: format!(
278 "{} for its other programs and page cache, which leave the slice {}",
279 ClosedKind::Host.name(),
280 ByteSize(host_limit)
281 ),
282 },
283 });
284 (free, host)
285 }
286}
287
288/// The budget as a bar draws it, from `Status::budget_parts`.
289#[derive(Debug, Clone, PartialEq, Eq)]
290pub struct BudgetParts {
291 /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
292 pub slots: Vec<SlotPart>,
293 /// The mirror, each seed container, then each shell open in a record.
294 pub held: Vec<HeldPart>,
295 /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
296 /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
297 /// stalls.
298 pub free: Option<FreePart>,
299 /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
300 /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
301 pub closed: Option<ClosedPart>,
302 /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
303 pub whole: u64,
304}
305
306/// A slot's block of the budget.
307#[derive(Debug, Clone, Copy, PartialEq, Eq)]
308pub struct SlotPart {
309 /// Its profile's index in `Status::profiles`.
310 pub profile: usize,
311 /// The slot, 1-based.
312 pub slot: usize,
313 /// Its size in bytes: what its runner reserves, or a free slot's cap.
314 pub memory: u64,
315 /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
316 pub reserved: bool,
317}
318
319/// Memory the budget holds for something other than a slot.
320#[derive(Debug, Clone, PartialEq, Eq)]
321pub struct HeldPart {
322 /// What holds it.
323 pub kind: HeldKind,
324 /// Its cap in bytes.
325 pub memory: u64,
326}
327
328impl HeldPart {
329 /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
330 pub fn name(&self) -> String {
331 match &self.kind {
332 HeldKind::Mirror => "mirror".to_owned(),
333 HeldKind::Seed { profile } => format!("{profile} seed"),
334 HeldKind::Shell { job } => format!("job {job} shell"),
335 }
336 }
337
338 /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
339 /// in job 42's record`.
340 pub fn holder(&self) -> String {
341 match &self.kind {
342 HeldKind::Mirror => "held by the mirror".to_owned(),
343 HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
344 HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
345 }
346 }
347}
348
349/// What holds a part of the budget beside the slots.
350#[derive(Debug, Clone, PartialEq, Eq)]
351pub enum HeldKind {
352 /// The registry mirror.
353 Mirror,
354 /// A profile's seed container, while it reseeds.
355 Seed {
356 /// The profile it reseeds.
357 profile: String,
358 },
359 /// A shell open in a job's record.
360 Shell {
361 /// The job.
362 job: u64,
363 },
364}
365
366/// What the bar leaves free past every block, less what the host holds back.
367#[derive(Debug, Clone, PartialEq, Eq)]
368pub struct FreePart {
369 /// In bytes.
370 pub memory: u64,
371 /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
372 /// leaves the slice 25.27G of its 46G`.
373 pub reading: Reading,
374}
375
376/// Room past the bar's blocks that no runner may start in.
377#[derive(Debug, Clone, PartialEq, Eq)]
378pub struct ClosedPart {
379 /// In bytes.
380 pub memory: u64,
381 /// What closes it.
382 pub kind: ClosedKind,
383 /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
384 /// which leave the slice 25.27G`.
385 pub reading: Reading,
386}
387
388/// What closes room past the bar's blocks.
389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
390pub enum ClosedKind {
391 /// The host's other programs and its page cache need it.
392 Host,
393 /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
394 Held,
395 /// The host's programs stall waiting for memory (`HostBudget::stalled`).
396 Stalled,
397}
398
399impl ClosedKind {
400 /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
401 /// stalls`.
402 pub fn name(self) -> &'static str {
403 match self {
404 Self::Host => "held by the host",
405 Self::Held => "closed past 90%",
406 Self::Stalled => "closed while the host stalls",
407 }
408 }
409}
410
411#[cfg(test)]
412mod tests;