runnerd_core/api/budget.rs
1//! What the memory budget means as every client shows it: its line, what is reserved and free of it, what of it no
2//! runner may start in, and the parts a bar draws. Plain derivations of [`Status`]; clients only lay them out.
3
4use super::units::{duration, utc_time};
5use super::{AdmissionPolicy, Budget, CountBrake, Counted, HostMemory, Status, percent};
6use crate::config::ByteSize;
7
8/// A figure and the words that follow it, such as `8G` and `of 24G reserved in runnerd.slice`, for a client that
9/// sets the figure apart.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Reading {
12 /// Such as `8G`.
13 pub figure: String,
14 /// What follows it, such as `of 24G reserved in runnerd.slice`.
15 pub text: String,
16}
17
18impl Reading {
19 /// The figure and its words as one line.
20 pub fn line(&self) -> String {
21 format!("{} {}", self.figure, self.text)
22 }
23}
24
25impl Budget {
26 /// What is reserved of what, as every client shows it: `8G` `of 24G reserved in runnerd.slice`, or without
27 /// `MemoryMax` `8G` `reserved in runnerd.slice, which has no MemoryMax`; while pooling takes something off, `36.8G`
28 /// `of 46G counted in ci.slice by predicted peaks: 42.8G reserved, 6G pooled`; how runners are admitted
29 /// (`Budget::admission_note`) follows the slice.
30 pub fn memory_reading(&self) -> Reading {
31 let of = self
32 .memory_max
33 .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
34 let verb = if self.pooled.is_some() {
35 "counted"
36 } else {
37 "reserved"
38 };
39 let mut text = format!("{of}{verb} in {}", self.slice);
40 if let Some(note) = self.admission_note() {
41 text = format!("{text} {note}");
42 }
43 if let Some(pooled) = &self.pooled {
44 let reserved = ByteSize(self.memory_in_use);
45 text = format!(
46 "{text}: {reserved} reserved, {} pooled",
47 ByteSize(pooled.credit)
48 );
49 }
50 if self.memory_max.is_none() {
51 text.push_str(", which has no MemoryMax");
52 }
53 if let Some(counted) = &self.counted {
54 text = format!("{text}; {}", counted.clause());
55 }
56 Reading {
57 figure: ByteSize(self.counted()).to_string(),
58 text,
59 }
60 }
61
62 /// What `MemoryMax` goes by, in bytes: `memory_in_use`, less what pooling takes off it (ADR 0099).
63 pub fn counted(&self) -> u64 {
64 self.pooled
65 .map_or(self.memory_in_use, |pooled| pooled.counted)
66 }
67
68 /// The budget's clauses in the order clients show them: memory (`Budget::memory_reading`), then CPU
69 /// (`CpuBudget::line`) and the host (`Budget::host_line`) when the daemon reports them.
70 pub fn clauses(&self) -> Vec<String> {
71 let mut clauses = vec![self.memory_reading().line()];
72 clauses.extend(self.cpu.map(|cpu| cpu.line()));
73 clauses.extend(self.host_line());
74 clauses
75 }
76
77 /// The host's line as clients show it, such as `host: 31G of 64G available, 1.5% stalled for memory; leaves the
78 /// slice 40G`; on a host with swap, `host: 31G of 64G available, swap 12.1G of 15.5G used, …` (ADR 0101); when that
79 /// is less room than `MemoryMax` leaves (`Budget::host_withheld`), `…; leaves the slice 25G
80 /// of its 46G, so 6.5G is free`, or past `MemoryMax` (`Budget::host_share`) `…; leaves the slice 47.5G for the 44.3G
81 /// reserved there, so 3.2G is free`, and without `MemoryMax`, where the host alone bounds it, `…; leaves the slice
82 /// 25G, so 6.5G is free`; while it stalls, `…; stalling, so no runner starts`. While the slice is held
83 /// (`Budget::held`), nothing is free, so it says no figure of it. While the host's limit takes a credit (ADR 0108),
84 /// what it counts follows: `…; what runs counts 23G: it uses 14G and keeps 9G for growth`
85 /// (`HostBudget::growth_clause`), unless the growth count counts, which `Budget::memory_reading` says. With programs
86 /// outside the slice large enough to
87 /// name, `…; programs outside CI hold 30G, most java 9.3G` (`HostMemory::outside_line`). `None` without a recent
88 /// reading of the host.
89 pub fn host_line(&self) -> Option<String> {
90 let host = self.host.as_ref()?;
91 let mut line = format!("host: {}", host.memory.available_line());
92 if let Some(swap) = host.memory.swap {
93 line = format!(
94 "{line}, swap {} of {} used",
95 ByteSize(swap.used()),
96 ByteSize(swap.total)
97 );
98 }
99 if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
100 line = format!("{line}, {} stalled for memory", percent(full.avg10));
101 }
102 let free = match host.spare {
103 0 => "nothing".to_owned(),
104 spare => ByteSize(spare).to_string(),
105 };
106 let limit = ByteSize(host.limit);
107 line = if host.stalled {
108 format!("{line}; stalling, so no runner starts")
109 } else if self.closed() {
110 format!("{line}; leaves the slice {limit}")
111 } else {
112 match self.memory_max {
113 Some(max) if self.host_withheld().is_some() => format!(
114 "{line}; leaves the slice {}, so {free} is free",
115 self.host_share(host.limit, max)
116 ),
117 Some(_) => format!("{line}; leaves the slice {limit}"),
118 None => format!("{line}; leaves the slice {limit}, so {free} is free"),
119 }
120 };
121 // While the growth count counts, both budgets read it and the memory's reading says what it is made of.
122 let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
123 if let Some(growth) = host
124 .growth_clause(self.memory_in_use)
125 .filter(|_| !growth_counts)
126 {
127 line = format!(
128 "{line}; what runs counts {}: {growth}",
129 ByteSize(host.counts(self.memory_in_use))
130 );
131 }
132 if !host.memory.holders.is_empty() {
133 line = format!("{line}; {}", host.memory.outside_line());
134 }
135 Some(line)
136 }
137
138 /// What the host's `limit` leaves the slice of `max`, its `MemoryMax`: `25G of its 46G`. A limit past `max` holds
139 /// back room only where `max` reads a larger credit than the limit (ADR 0099, 0108). Nothing leaves a slice more
140 /// than it has, so it says what the limit goes by: `47.5G for the 44.3G reserved there`, or while the host's limit
141 /// takes a credit, `47.5G for the 40G counted there`.
142 fn host_share(&self, limit: u64, max: u64) -> String {
143 if limit > max {
144 let host = self.host.as_ref();
145 let (counts, verb) = match host.filter(|host| host.credit.is_some()) {
146 Some(host) => (host.counts(self.memory_in_use), "counted"),
147 None => (self.memory_in_use, "reserved"),
148 };
149 format!(
150 "{} for the {} {verb} there",
151 ByteSize(limit),
152 ByteSize(counts)
153 )
154 } else {
155 format!("{} of its {}", ByteSize(limit), ByteSize(max))
156 }
157 }
158
159 /// Whether measured admission holds the slice at its high-water mark (`Budget::held`), so no runner starts and
160 /// nothing is free.
161 fn closed(&self) -> bool {
162 self.held && self.admission == AdmissionPolicy::Measured
163 }
164
165 /// What the host's memory holds back of the room `MemoryMax` leaves beside what runs, in bytes: that room less
166 /// what the host has to spare (`HostBudget::spare`), when the host leaves less. `None` while it leaves as much,
167 /// while it stalls (its line says no runner starts), without a recent reading of it, and without `MemoryMax`.
168 pub fn host_withheld(&self) -> Option<u64> {
169 let host = self.host.as_ref().filter(|host| !host.stalled)?;
170 let remaining = self.remaining()?;
171 (host.spare < remaining).then(|| remaining - host.spare)
172 }
173
174 /// The budget as one line, its clauses joined by `; `.
175 pub fn line(&self) -> String {
176 self.clauses().join("; ")
177 }
178
179 /// What `MemoryMax` leaves beside what the running containers count (`Budget::counted`), in bytes: what a new
180 /// runner must fit in. `None` without `MemoryMax`.
181 pub fn remaining(&self) -> Option<u64> {
182 self.memory_max
183 .map(|max| max.saturating_sub(self.counted()))
184 }
185}
186
187impl Counted {
188 /// What the growth count is made of, or why it rests, as every client says it after the budget's figure: `what runs
189 /// uses 15.7G and keeps 12.9G for its growth`; while it rests, `the growth count rests until 15:02 UTC: what ran
190 /// grew 14.1G in 10m, past the 12.9G kept`.
191 pub fn clause(&self) -> String {
192 let size = |bytes: u64| ByteSize(bytes).to_string();
193 let rests = |until: u64| format!("the growth count rests until {}", utc_time(until));
194 match (self.brake, self.kept) {
195 (None, Some(kept)) => format!(
196 "what runs uses {} and keeps {} for its growth",
197 size(self.used),
198 size(kept)
199 ),
200 (Some(CountBrake::Grew { grew, kept, until }), _) => format!(
201 "{}: what ran grew {} in {}, past the {} kept",
202 rests(until),
203 size(grew),
204 duration(super::GROWTH_WINDOW.as_secs()),
205 size(kept)
206 ),
207 (Some(CountBrake::Restarted { until }), _) => {
208 format!(
209 "{}: too soon after a start or a reload to judge growth",
210 rests(until)
211 )
212 }
213 (Some(CountBrake::Guard { until: Some(until) }), _) => {
214 format!("{}: the host ran under its line", rests(until))
215 }
216 (Some(CountBrake::Guard { until: None }) | None, _) => {
217 "the growth count rests while the host is under its line or a reading is missing"
218 .to_owned()
219 }
220 (Some(CountBrake::Unknown), _) => "the growth count rests".to_owned(),
221 }
222 }
223}
224
225impl HostMemory {
226 /// What the host has available of its memory, as the budget's host clause and doctor say it: `31G of 64G
227 /// available`, and while it can still grow to its ceiling (`HostMemory::growable`), `1.5G of 2.1G available,
228 /// growing to 24G`.
229 pub fn available_line(&self) -> String {
230 let line = format!(
231 "{} of {} available",
232 ByteSize(self.available),
233 ByteSize(self.total)
234 );
235 match self.growable() {
236 0 => line,
237 _ => format!("{line}, growing to {}", ByteSize(self.grown())),
238 }
239 }
240}
241
242impl Status {
243 /// The budget as a bar draws it: a block per slot as large as its cap (under measured admission, a block per
244 /// runner as large as what it reserves), the mirror, seed containers and shells, then what is free and what of
245 /// the room no runner may start in; and the whole the blocks are scaled against.
246 pub fn budget_parts(&self) -> BudgetParts {
247 let budget = &self.budget;
248 let slots: Vec<SlotPart> = self
249 .profiles
250 .iter()
251 .enumerate()
252 .flat_map(|(profile, status)| {
253 status.slots.iter().filter_map(move |slot| {
254 let memory = match budget.admission {
255 AdmissionPolicy::Caps => match &slot.runner {
256 Some(runner) if runner.reserved > 0 => runner.reserved,
257 _ => status.memory,
258 },
259 AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
260 };
261 Some(SlotPart {
262 profile,
263 slot: slot.slot,
264 memory,
265 reserved: slot.runner.is_some(),
266 })
267 })
268 })
269 .collect();
270 let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
271 kind: HeldKind::Mirror,
272 memory: budget.mirror_memory,
273 });
274 let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
275 kind: HeldKind::Seed {
276 profile: seed.profile.clone(),
277 },
278 memory: seed.memory,
279 });
280 let shells = budget.shell_memory.iter().map(|shell| HeldPart {
281 kind: HeldKind::Shell { job: shell.job },
282 memory: shell.memory,
283 });
284 let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
285 let drawn: u64 = slots
286 .iter()
287 .map(|part| part.memory)
288 .chain(held.iter().map(|part| part.memory))
289 .sum();
290 let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
291 let unclaimed: u64 = slots
292 .iter()
293 .filter(|part| !part.reserved)
294 .map(|part| part.memory)
295 .sum();
296 let (free, closed) = budget.room_parts(drawn, unclaimed);
297 BudgetParts {
298 slots,
299 held,
300 free,
301 closed,
302 whole,
303 }
304 }
305}
306
307impl Budget {
308 /// The bar's room past `drawn` bytes of blocks, of which `unclaimed` are idle slots' caps: what is free, and what
309 /// of it no runner may start in.
310 fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
311 let Some(max) = self.memory_max else {
312 return (None, None);
313 };
314 let room = max.saturating_sub(drawn);
315 // While no runner starts, all the room is closed.
316 let shut = if self.closed() {
317 Some((
318 ClosedKind::Held,
319 "closed: the slice is past 90% of its MemoryMax, so no runner starts",
320 ))
321 } else if self.host.as_ref().is_some_and(|host| host.stalled) {
322 Some((
323 ClosedKind::Stalled,
324 "closed: the host's programs stall waiting for memory, so no runner starts",
325 ))
326 } else {
327 None
328 };
329 if let Some((kind, text)) = shut {
330 let closed = (room > 0).then(|| ClosedPart {
331 memory: room,
332 kind,
333 reading: Reading {
334 figure: ByteSize(room).to_string(),
335 text: text.to_owned(),
336 },
337 });
338 return (None, closed);
339 }
340 let withheld = self.host_withheld();
341 // The host's part ends the bar, within the free room: under caps, idle slots' blocks stay drawn though the
342 // host may hold their room too.
343 let host = withheld
344 .map(|withheld| withheld.min(room))
345 .filter(|&memory| memory > 0);
346 let (host_limit, spare) = self
347 .host
348 .as_ref()
349 .map_or((0, 0), |host| (host.limit, host.spare));
350 let memory = room - host.unwrap_or(0);
351 let mut text = match (unclaimed, withheld) {
352 (0, None) => format!("of {} free", ByteSize(max)),
353 (_, None) => format!(
354 "of {} free past every slot's cap; {} is free beside what runs",
355 ByteSize(max),
356 ByteSize(self.remaining().unwrap_or(0))
357 ),
358 (0, Some(_)) => format!(
359 "free; the host leaves the slice {}",
360 self.host_share(host_limit, max)
361 ),
362 (_, Some(_)) => format!(
363 "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
364 ByteSize(spare),
365 self.host_share(host_limit, max)
366 ),
367 };
368 // The blocks stay what runners reserve, so what pooling adds is said and not drawn.
369 if let Some(pooled) = &self.pooled {
370 text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
371 }
372 let free = (memory > 0).then(|| FreePart {
373 memory,
374 reading: Reading {
375 figure: ByteSize(memory).to_string(),
376 text,
377 },
378 });
379 let host = host.map(|memory| ClosedPart {
380 memory,
381 kind: ClosedKind::Host,
382 reading: Reading {
383 figure: ByteSize(memory).to_string(),
384 // Past `MemoryMax` the plain count holds this back, not the host's programs (`Budget::host_share`).
385 text: if host_limit > max {
386 format!(
387 "held back by what runs reserves: the host leaves the slice {}",
388 self.host_share(host_limit, max)
389 )
390 } else {
391 format!(
392 "{} for its other programs and page cache, which leave the slice {}",
393 ClosedKind::Host.name(),
394 ByteSize(host_limit)
395 )
396 },
397 },
398 });
399 (free, host)
400 }
401}
402
403/// The budget as a bar draws it, from `Status::budget_parts`.
404#[derive(Debug, Clone, PartialEq, Eq)]
405pub struct BudgetParts {
406 /// A block per slot, in config order: every slot under caps, only those with a runner under measured admission.
407 pub slots: Vec<SlotPart>,
408 /// The mirror, each seed container, then each shell open in a record.
409 pub held: Vec<HeldPart>,
410 /// What `MemoryMax` leaves past every block, less what is closed; `None` without `MemoryMax`, without room past
411 /// the blocks and the closed part, and while no runner starts: the slice is held (`Budget::held`) or the host
412 /// stalls.
413 pub free: Option<FreePart>,
414 /// What of the room past every block no runner may start in, drawn after what is free: all of it while the slice
415 /// is held or the host stalls, else what the host's memory holds back (`Budget::host_withheld`); `None` while neither closes any.
416 pub closed: Option<ClosedPart>,
417 /// What the blocks are scaled against: `MemoryMax`, or the blocks' sum when larger or without it.
418 pub whole: u64,
419}
420
421/// A slot's block of the budget.
422#[derive(Debug, Clone, Copy, PartialEq, Eq)]
423pub struct SlotPart {
424 /// Its profile's index in `Status::profiles`.
425 pub profile: usize,
426 /// The slot, 1-based.
427 pub slot: usize,
428 /// Its size in bytes: what its runner reserves, or a free slot's cap.
429 pub memory: u64,
430 /// Whether a runner reserves it; a free slot's block under caps is room its next runner takes, still free.
431 pub reserved: bool,
432}
433
434/// Memory the budget holds for something other than a slot.
435#[derive(Debug, Clone, PartialEq, Eq)]
436pub struct HeldPart {
437 /// What holds it.
438 pub kind: HeldKind,
439 /// Its cap in bytes.
440 pub memory: u64,
441}
442
443impl HeldPart {
444 /// Its name under its block: `mirror`, `node seed` or `job 42 shell`.
445 pub fn name(&self) -> String {
446 match &self.kind {
447 HeldKind::Mirror => "mirror".to_owned(),
448 HeldKind::Seed { profile } => format!("{profile} seed"),
449 HeldKind::Shell { job } => format!("job {job} shell"),
450 }
451 }
452
453 /// What holds it, after its size: `held by the mirror`, `held by node's seed container` or `held by the shell
454 /// in job 42's record`.
455 pub fn holder(&self) -> String {
456 match &self.kind {
457 HeldKind::Mirror => "held by the mirror".to_owned(),
458 HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
459 HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
460 }
461 }
462}
463
464/// What holds a part of the budget beside the slots.
465#[derive(Debug, Clone, PartialEq, Eq)]
466pub enum HeldKind {
467 /// The registry mirror.
468 Mirror,
469 /// A profile's seed container, while it reseeds.
470 Seed {
471 /// The profile it reseeds.
472 profile: String,
473 },
474 /// A shell open in a job's record.
475 Shell {
476 /// The job.
477 job: u64,
478 },
479}
480
481/// What the bar leaves free past every block, less what the host holds back.
482#[derive(Debug, Clone, PartialEq, Eq)]
483pub struct FreePart {
484 /// In bytes.
485 pub memory: u64,
486 /// Its figure and what it means, such as `8G` `of 24G free`, or while the host binds `6.52G` `free; the host
487 /// leaves the slice 25.27G of its 46G` (`Budget::host_share`).
488 pub reading: Reading,
489}
490
491/// Room past the bar's blocks that no runner may start in.
492#[derive(Debug, Clone, PartialEq, Eq)]
493pub struct ClosedPart {
494 /// In bytes.
495 pub memory: u64,
496 /// What closes it.
497 pub kind: ClosedKind,
498 /// Its figure and what it means, such as `20.73G` `held by the host for its other programs and page cache,
499 /// which leave the slice 25.27G`, or past `MemoryMax` `1.5G` `held back by what runs reserves: the host leaves the
500 /// slice 24.5G for the 22G reserved there`.
501 pub reading: Reading,
502}
503
504/// What closes room past the bar's blocks.
505#[derive(Debug, Clone, Copy, PartialEq, Eq)]
506pub enum ClosedKind {
507 /// The host's other programs and its page cache need it.
508 Host,
509 /// The slice is past 90% of its `MemoryMax` under measured admission (`Budget::held`).
510 Held,
511 /// The host's programs stall waiting for memory (`HostBudget::stalled`).
512 Stalled,
513}
514
515impl ClosedKind {
516 /// Its name under its block or in a key: `held by the host`, `closed past 90%` or `closed while the host
517 /// stalls`.
518 pub fn name(self) -> &'static str {
519 match self {
520 Self::Host => "held by the host",
521 Self::Held => "closed past 90%",
522 Self::Stalled => "closed while the host stalls",
523 }
524 }
525}
526
527#[cfg(test)]
528mod tests;