runnerd_core/api/budget.rs
1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::{AdmissionPolicy, Budget, HostMemory, Status, percent};
5use crate::config::ByteSize;
6
7/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
8/// sets the figure apart.
9#[derive(Debug, Clone, PartialEq, Eq)]
10pub struct Reading {
11 /// Such as `8G`.
12 pub figure: String,
13 /// What follows it, such as `of 24G reserved in runnerd.slice`.
14 pub text: String,
15}
16
17impl Reading {
18 /// The figure and its words as one line.
19 pub fn line(&self) -> String {
20 format!("{} {}", self.figure, self.text)
21 }
22}
23
24impl Budget {
25 /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
26 /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; while pooling takes something off, `36.8G`
27 /// `of 46G counted in ci.slice by predicted peaks: 42.8G reserved, 6G pooled`; how runners are admitted
28 /// (`Budget::admission_note`) follows the slice.
29 pub fn memory_reading(&self) -> Reading {
30 let of = self
31 .memory_max
32 .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
33 let verb = if self.pooled.is_some() {
34 "counted"
35 } else {
36 "reserved"
37 };
38 let mut text = format!("{of}{verb} in {}", self.slice);
39 if let Some(note) = self.admission_note() {
40 text = format!("{text} {note}");
41 }
42 if let Some(pooled) = &self.pooled {
43 let reserved = ByteSize(self.memory_in_use);
44 text = format!(
45 "{text}: {reserved} reserved, {} pooled",
46 ByteSize(pooled.credit)
47 );
48 }
49 if self.memory_max.is_none() {
50 text.push_str(", which has no MemoryMax");
51 }
52 Reading {
53 figure: ByteSize(self.counted()).to_string(),
54 text,
55 }
56 }
57
58 /// What `MemoryMax` goes by, in bytes: `memory_in_use`, less what pooling takes off it (ADR 0099).
59 pub fn counted(&self) -> u64 {
60 self.pooled
61 .map_or(self.memory_in_use, |pooled| pooled.counted)
62 }
63
64 /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
65 /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
66 pub fn clauses(&self) -> Vec<String> {
67 let mut clauses = vec![self.memory_reading().line()];
68 clauses.extend(self.cpu.map(|cpu| cpu.line()));
69 clauses.extend(self.host_line());
70 clauses
71 }
72
73 /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
74 /// slice 40G`; on a host with swap, `host: 31G of 64G available, swap 12.1G of 15.5G used, …` (ADR 0101); when that
75 /// is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
76 /// of its 46G, so 6.5G is free`, or past `MemoryMax` (`Budget::host_share`) `…; leaves the slice 47.5G for the 44.3G
77 /// reserved there, so 3.2G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
78 /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
79 /// (`Budget::held`), nothing is free, so it says no figure of it. With programs outside the slice large enough to
80 /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
81 /// reading of the host.
82 pub fn host_line(&self) -> Option<String> {
83 let host = self.host.as_ref()?;
84 let mut line = format!("host: {}", host.memory.available_line());
85 if let Some(swap) = host.memory.swap {
86 line = format!(
87 "{line}, swap {} of {} used",
88 ByteSize(swap.used()),
89 ByteSize(swap.total)
90 );
91 }
92 if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
93 line = format!("{line}, {} stalled for memory", percent(full.avg10));
94 }
95 let free = match host.spare {
96 0 => "nothing".to_owned(),
97 spare => ByteSize(spare).to_string(),
98 };
99 let limit = ByteSize(host.limit);
100 line = if host.stalled {
101 format!("{line}; stalling, so no runner starts")
102 } else if self.closed() {
103 format!("{line}; leaves the slice {limit}")
104 } else {
105 match self.memory_max {
106 Some(max) if self.host_withheld().is_some() => format!(
107 "{line}; leaves the slice {}, so {free} is free",
108 self.host_share(host.limit, max)
109 ),
110 Some(_) => format!("{line}; leaves the slice {limit}"),
111 None => format!("{line}; leaves the slice {limit}, so {free} is free"),
112 }
113 };
114 if !host.memory.holders.is_empty() {
115 line = format!("{line}; {}", host.memory.outside_line());
116 }
117 Some(line)
118 }
119
120 /// What the host's `limit` leaves the slice of `max`, its `MemoryMax`: `25G of its 46G`. A limit past `max` holds
121 /// back room only under a credit, which `max` reads and the limit does not (ADR 0099). Nothing leaves a slice more
122 /// than it has, so it says what the limit goes by: `47.5G for the 44.3G reserved there`.
123 fn host_share(&self, limit: u64, max: u64) -> String {
124 if limit > max {
125 format!(
126 "{} for the {} reserved there",
127 ByteSize(limit),
128 ByteSize(self.memory_in_use)
129 )
130 } else {
131 format!("{} of its {}", ByteSize(limit), ByteSize(max))
132 }
133 }
134
135 /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
136 /// nothing is free.
137 fn closed(&self) -> bool {
138 self.held && self.admission == AdmissionPolicy::Measured
139 }
140
141 /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
142 /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
143 /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
144 pub fn host_withheld(&self) -> Option<u64> {
145 let host = self.host.as_ref().filter(|host| !host.stalled)?;
146 let remaining = self.remaining()?;
147 (host.spare < remaining).then(|| remaining - host.spare)
148 }
149
150 /// The budget as one line, its clauses joined by `; `.
151 pub fn line(&self) -> String {
152 self.clauses().join("; ")
153 }
154
155 /// What `MemoryMax` leaves beside what the running containers count (`Budget::counted`), in bytes: what a new
156 /// runner must fit in. `None` without `MemoryMax`.
157 pub fn remaining(&self) -> Option<u64> {
158 self.memory_max
159 .map(|max| max.saturating_sub(self.counted()))
160 }
161}
162
163impl HostMemory {
164 /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
165 /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
166 /// growing to 24G`.
167 pub fn available_line(&self) -> String {
168 let line = format!(
169 "{} of {} available",
170 ByteSize(self.available),
171 ByteSize(self.total)
172 );
173 match self.growable() {
174 0 => line,
175 _ => format!("{line}, growing to {}", ByteSize(self.grown())),
176 }
177 }
178}
179
180impl Status {
181 /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
182 /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
183 /// the room no runner may start in; and the whole the blocks are scaled against.
184 pub fn budget_parts(&self) -> BudgetParts {
185 let budget = &self.budget;
186 let slots: Vec<SlotPart> = self
187 .profiles
188 .iter()
189 .enumerate()
190 .flat_map(|(profile, status)| {
191 status.slots.iter().filter_map(move |slot| {
192 let memory = match budget.admission {
193 AdmissionPolicy::Caps => match &slot.runner {
194 Some(runner) if runner.reserved > 0 => runner.reserved,
195 _ => status.memory,
196 },
197 AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
198 };
199 Some(SlotPart {
200 profile,
201 slot: slot.slot,
202 memory,
203 reserved: slot.runner.is_some(),
204 })
205 })
206 })
207 .collect();
208 let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
209 kind: HeldKind::Mirror,
210 memory: budget.mirror_memory,
211 });
212 let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
213 kind: HeldKind::Seed {
214 profile: seed.profile.clone(),
215 },
216 memory: seed.memory,
217 });
218 let shells = budget.shell_memory.iter().map(|shell| HeldPart {
219 kind: HeldKind::Shell { job: shell.job },
220 memory: shell.memory,
221 });
222 let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
223 let drawn: u64 = slots
224 .iter()
225 .map(|part| part.memory)
226 .chain(held.iter().map(|part| part.memory))
227 .sum();
228 let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
229 let unclaimed: u64 = slots
230 .iter()
231 .filter(|part| !part.reserved)
232 .map(|part| part.memory)
233 .sum();
234 let (free, closed) = budget.room_parts(drawn, unclaimed);
235 BudgetParts {
236 slots,
237 held,
238 free,
239 closed,
240 whole,
241 }
242 }
243}
244
245impl Budget {
246 /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
247 /// of it no runner may start in.
248 fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
249 let Some(max) = self.memory_max else {
250 return (None, None);
251 };
252 let room = max.saturating_sub(drawn);
253 // While no runner starts, all the room is closed.
254 let shut = if self.closed() {
255 Some((
256 ClosedKind::Held,
257 "closed: the slice is past 90% of its MemoryMax, so no runner starts",
258 ))
259 } else if self.host.as_ref().is_some_and(|host| host.stalled) {
260 Some((
261 ClosedKind::Stalled,
262 "closed: the host's programs stall waiting for memory, so no runner starts",
263 ))
264 } else {
265 None
266 };
267 if let Some((kind, text)) = shut {
268 let closed = (room > 0).then(|| ClosedPart {
269 memory: room,
270 kind,
271 reading: Reading {
272 figure: ByteSize(room).to_string(),
273 text: text.to_owned(),
274 },
275 });
276 return (None, closed);
277 }
278 let withheld = self.host_withheld();
279 // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
280 // host may hold their room too.
281 let host = withheld
282 .map(|withheld| withheld.min(room))
283 .filter(|&memory| memory > 0);
284 let (host_limit, spare) = self
285 .host
286 .as_ref()
287 .map_or((0, 0), |host| (host.limit, host.spare));
288 let memory = room - host.unwrap_or(0);
289 let mut text = match (unclaimed, withheld) {
290 (0, None) => format!("of {} free", ByteSize(max)),
291 (_, None) => format!(
292 "of {} free past every slot's cap; {} is free beside what runs",
293 ByteSize(max),
294 ByteSize(self.remaining().unwrap_or(0))
295 ),
296 (0, Some(_)) => format!(
297 "free; the host leaves the slice {}",
298 self.host_share(host_limit, max)
299 ),
300 (_, Some(_)) => format!(
301 "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
302 ByteSize(spare),
303 self.host_share(host_limit, max)
304 ),
305 };
306 // The blocks stay what runners reserve, so what pooling adds is said and not drawn.
307 if let Some(pooled) = &self.pooled {
308 text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
309 }
310 let free = (memory > 0).then(|| FreePart {
311 memory,
312 reading: Reading {
313 figure: ByteSize(memory).to_string(),
314 text,
315 },
316 });
317 let host = host.map(|memory| ClosedPart {
318 memory,
319 kind: ClosedKind::Host,
320 reading: Reading {
321 figure: ByteSize(memory).to_string(),
322 // Past `MemoryMax` the plain count holds this back, not the host's programs (`Budget::host_share`).
323 text: if host_limit > max {
324 format!(
325 "held back by what runs reserves: the host leaves the slice {}",
326 self.host_share(host_limit, max)
327 )
328 } else {
329 format!(
330 "{} for its other programs and page cache, which leave the slice {}",
331 ClosedKind::Host.name(),
332 ByteSize(host_limit)
333 )
334 },
335 },
336 });
337 (free, host)
338 }
339}
340
341/// The budget as a bar draws it, from `Status::budget_parts`.
342#[derive(Debug, Clone, PartialEq, Eq)]
343pub struct BudgetParts {
344 /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
345 pub slots: Vec<SlotPart>,
346 /// The mirror, each seed container, then each shell open in a record.
347 pub held: Vec<HeldPart>,
348 /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
349 /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
350 /// stalls.
351 pub free: Option<FreePart>,
352 /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
353 /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
354 pub closed: Option<ClosedPart>,
355 /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
356 pub whole: u64,
357}
358
359/// A slot's block of the budget.
360#[derive(Debug, Clone, Copy, PartialEq, Eq)]
361pub struct SlotPart {
362 /// Its profile's index in `Status::profiles`.
363 pub profile: usize,
364 /// The slot, 1-based.
365 pub slot: usize,
366 /// Its size in bytes: what its runner reserves, or a free slot's cap.
367 pub memory: u64,
368 /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
369 pub reserved: bool,
370}
371
372/// Memory the budget holds for something other than a slot.
373#[derive(Debug, Clone, PartialEq, Eq)]
374pub struct HeldPart {
375 /// What holds it.
376 pub kind: HeldKind,
377 /// Its cap in bytes.
378 pub memory: u64,
379}
380
381impl HeldPart {
382 /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
383 pub fn name(&self) -> String {
384 match &self.kind {
385 HeldKind::Mirror => "mirror".to_owned(),
386 HeldKind::Seed { profile } => format!("{profile} seed"),
387 HeldKind::Shell { job } => format!("job {job} shell"),
388 }
389 }
390
391 /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
392 /// in job 42's record`.
393 pub fn holder(&self) -> String {
394 match &self.kind {
395 HeldKind::Mirror => "held by the mirror".to_owned(),
396 HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
397 HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
398 }
399 }
400}
401
402/// What holds a part of the budget beside the slots.
403#[derive(Debug, Clone, PartialEq, Eq)]
404pub enum HeldKind {
405 /// The registry mirror.
406 Mirror,
407 /// A profile's seed container, while it reseeds.
408 Seed {
409 /// The profile it reseeds.
410 profile: String,
411 },
412 /// A shell open in a job's record.
413 Shell {
414 /// The job.
415 job: u64,
416 },
417}
418
419/// What the bar leaves free past every block, less what the host holds back.
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub struct FreePart {
422 /// In bytes.
423 pub memory: u64,
424 /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
425 /// leaves the slice 25.27G of its 46G` (`Budget::host_share`).
426 pub reading: Reading,
427}
428
429/// Room past the bar's blocks that no runner may start in.
430#[derive(Debug, Clone, PartialEq, Eq)]
431pub struct ClosedPart {
432 /// In bytes.
433 pub memory: u64,
434 /// What closes it.
435 pub kind: ClosedKind,
436 /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
437 /// which leave the slice 25.27G`, or past `MemoryMax` `1.5G` `held back by what runs reserves: the host leaves the
438 /// slice 24.5G for the 22G reserved there`.
439 pub reading: Reading,
440}
441
442/// What closes room past the bar's blocks.
443#[derive(Debug, Clone, Copy, PartialEq, Eq)]
444pub enum ClosedKind {
445 /// The host's other programs and its page cache need it.
446 Host,
447 /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
448 Held,
449 /// The host's programs stall waiting for memory (`HostBudget::stalled`).
450 Stalled,
451}
452
453impl ClosedKind {
454 /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
455 /// stalls`.
456 pub fn name(self) -> &'static str {
457 match self {
458 Self::Host => "held by the host",
459 Self::Held => "closed past 90%",
460 Self::Stalled => "closed while the host stalls",
461 }
462 }
463}
464
465#[cfg(test)]
466mod tests;