1use super::units::{duration, utc_time};
5use super::{AdmissionPolicy, Budget, CountBrake, Counted, HostMemory, Status, percent};
6use crate::config::ByteSize;
7
8#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Reading {
12 pub figure: String,
14 pub text: String,
16}
17
18#[derive(Debug, Clone, PartialEq, Eq)]
20pub struct HostFigure {
21 pub label: &'static str,
23 pub value: String,
25 pub note: Option<String>,
27}
28
29impl HostFigure {
30 fn new(label: &'static str, value: String) -> HostFigure {
31 HostFigure {
32 label,
33 value,
34 note: None,
35 }
36 }
37}
38
39impl Reading {
40 pub fn line(&self) -> String {
42 format!("{} {}", self.figure, self.text)
43 }
44}
45
46impl Budget {
47 pub fn memory_reading(&self) -> Reading {
52 let of = self
53 .memory_max
54 .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
55 let verb = if self.pooled.is_some() {
56 "counted"
57 } else {
58 "reserved"
59 };
60 let mut text = format!("{of}{verb} in {}", self.slice);
61 if let Some(note) = self.admission_note() {
62 text = format!("{text} {note}");
63 }
64 if let Some(pooled) = &self.pooled {
65 let reserved = ByteSize(self.memory_in_use);
66 text = format!(
67 "{text}: {reserved} reserved, {} pooled",
68 ByteSize(pooled.credit)
69 );
70 }
71 if self.memory_max.is_none() {
72 text.push_str(", which has no MemoryMax");
73 }
74 if let Some(counted) = &self.counted {
75 text = format!("{text}; {}", counted.clause());
76 }
77 Reading {
78 figure: ByteSize(self.counted()).to_string(),
79 text,
80 }
81 }
82
83 pub fn counted(&self) -> u64 {
85 self.pooled
86 .map_or(self.memory_in_use, |pooled| pooled.counted)
87 }
88
89 pub fn clauses(&self) -> Vec<String> {
92 let mut clauses = vec![self.memory_reading().line()];
93 clauses.extend(self.cpu.map(|cpu| cpu.line()));
94 clauses.extend(self.host_line());
95 clauses
96 }
97
98 pub fn host_line(&self) -> Option<String> {
111 let host = self.host.as_ref()?;
112 let mut line = format!("host: {}", host.memory.available_line());
113 if let Some(swap) = host.memory.swap {
114 line = format!(
115 "{line}, swap {} of {} used",
116 ByteSize(swap.used()),
117 ByteSize(swap.total)
118 );
119 }
120 if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
121 line = format!("{line}, {} stalled for memory", percent(full.avg10));
122 }
123 let free = match host.spare {
124 0 => "nothing".to_owned(),
125 spare => ByteSize(spare).to_string(),
126 };
127 let limit = ByteSize(host.limit);
128 line = if host.stalled {
129 format!("{line}; stalling, so no runner starts")
130 } else if self.closed() {
131 format!("{line}; leaves the slice {limit}")
132 } else {
133 match self.memory_max {
134 Some(max) if self.host_withheld().is_some() => format!(
135 "{line}; leaves the slice {}, so {free} is free",
136 self.host_share(host.limit, max)
137 ),
138 Some(_) => format!("{line}; leaves the slice {limit}"),
139 None => format!("{line}; leaves the slice {limit}, so {free} is free"),
140 }
141 };
142 let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
144 if let Some(growth) = host
145 .growth_clause(self.memory_in_use)
146 .filter(|_| !growth_counts)
147 {
148 line = format!(
149 "{line}; what runs counts {}: {growth}",
150 ByteSize(host.counts(self.memory_in_use))
151 );
152 }
153 if !host.memory.holders.is_empty() {
154 line = format!("{line}; {}", host.memory.outside_line());
155 }
156 Some(line)
157 }
158
159 pub fn host_figures(&self) -> Vec<HostFigure> {
165 let Some(host) = self.host.as_ref() else {
166 return Vec::new();
167 };
168 let memory = &host.memory;
169 let size = |bytes: u64| ByteSize(bytes).to_string();
170 let mut available = format!("{} of {}", size(memory.available), size(memory.total));
171 if memory.growable() > 0 {
172 available = format!("{available}, growing to {}", size(memory.grown()));
173 }
174 let mut figures = vec![HostFigure::new("Host available", available)];
175 if let Some(swap) = memory.swap {
176 figures.push(HostFigure::new(
177 "Swap used",
178 format!("{} of {}", size(swap.used()), size(swap.total)),
179 ));
180 }
181 if let Some(full) = memory.full.filter(|full| full.avg10 > 0) {
182 figures.push(HostFigure::new("Stalled for memory", percent(full.avg10)));
183 }
184 let leaves = if host.stalled {
185 "nothing while it stalls: no runner starts".to_owned()
186 } else {
187 match self.memory_max {
188 Some(max) if !self.closed() && self.host_withheld().is_some() => {
189 self.host_share(host.limit, max)
190 }
191 _ => size(host.limit),
192 }
193 };
194 figures.push(HostFigure::new("Leaves the slice", leaves));
195 let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
196 if let Some(growth) = host
197 .growth_clause(self.memory_in_use)
198 .filter(|_| !growth_counts)
199 {
200 figures.push(HostFigure {
201 note: Some(growth),
202 ..HostFigure::new("What runs counts", size(host.counts(self.memory_in_use)))
203 });
204 }
205 if !memory.holders.is_empty() {
206 let named: Vec<String> = memory
207 .holders
208 .iter()
209 .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
210 .collect();
211 let note = match named.as_slice() {
212 [one] => format!("most {one}"),
213 [rest @ .., last] => format!("most {} and {last}", rest.join(", ")),
214 [] => String::new(),
215 };
216 figures.push(HostFigure {
217 note: Some(note),
218 ..HostFigure::new("Outside CI", size(memory.outside()))
219 });
220 }
221 figures
222 }
223
224 fn host_share(&self, limit: u64, max: u64) -> String {
229 if limit > max {
230 let host = self.host.as_ref();
231 let (counts, verb) = match host.filter(|host| host.credit.is_some()) {
232 Some(host) => (host.counts(self.memory_in_use), "counted"),
233 None => (self.memory_in_use, "reserved"),
234 };
235 format!(
236 "{} for the {} {verb} there",
237 ByteSize(limit),
238 ByteSize(counts)
239 )
240 } else {
241 format!("{} of its {}", ByteSize(limit), ByteSize(max))
242 }
243 }
244
245 fn closed(&self) -> bool {
248 self.held && self.admission == AdmissionPolicy::Measured
249 }
250
251 pub fn host_withheld(&self) -> Option<u64> {
255 let host = self.host.as_ref().filter(|host| !host.stalled)?;
256 let remaining = self.remaining()?;
257 (host.spare < remaining).then(|| remaining - host.spare)
258 }
259
260 pub fn line(&self) -> String {
262 self.clauses().join("; ")
263 }
264
265 pub fn remaining(&self) -> Option<u64> {
268 self.memory_max
269 .map(|max| max.saturating_sub(self.counted()))
270 }
271}
272
273impl Counted {
274 pub fn clause(&self) -> String {
278 let size = |bytes: u64| ByteSize(bytes).to_string();
279 let rests = |until: u64| format!("the growth count rests until {}", utc_time(until));
280 match (self.brake, self.kept) {
281 (None, Some(kept)) => format!(
282 "what runs uses {} and keeps {} for its growth",
283 size(self.used),
284 size(kept)
285 ),
286 (Some(CountBrake::Grew { grew, kept, until }), _) => format!(
287 "{}: what ran grew {} in {}, past the {} kept",
288 rests(until),
289 size(grew),
290 duration(super::GROWTH_WINDOW.as_secs()),
291 size(kept)
292 ),
293 (Some(CountBrake::Restarted { until }), _) => {
294 format!(
295 "{}: too soon after a start or a reload to judge growth",
296 rests(until)
297 )
298 }
299 (Some(CountBrake::Guard { until: Some(until) }), _) => {
300 format!("{}: the host ran under its line", rests(until))
301 }
302 (Some(CountBrake::Guard { until: None }) | None, _) => {
303 "the growth count rests while the host is under its line or a reading is missing"
304 .to_owned()
305 }
306 (Some(CountBrake::Unknown), _) => "the growth count rests".to_owned(),
307 }
308 }
309}
310
311impl HostMemory {
312 pub fn available_line(&self) -> String {
316 let line = format!(
317 "{} of {} available",
318 ByteSize(self.available),
319 ByteSize(self.total)
320 );
321 match self.growable() {
322 0 => line,
323 _ => format!("{line}, growing to {}", ByteSize(self.grown())),
324 }
325 }
326}
327
328impl Status {
329 pub fn budget_parts(&self) -> BudgetParts {
333 let budget = &self.budget;
334 let slots: Vec<SlotPart> = self
335 .profiles
336 .iter()
337 .enumerate()
338 .flat_map(|(profile, status)| {
339 status.slots.iter().filter_map(move |slot| {
340 let memory = match budget.admission {
341 AdmissionPolicy::Caps => match &slot.runner {
342 Some(runner) if runner.reserved > 0 => runner.reserved,
343 _ => status.memory,
344 },
345 AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
346 };
347 Some(SlotPart {
348 profile,
349 slot: slot.slot,
350 memory,
351 reserved: slot.runner.is_some(),
352 })
353 })
354 })
355 .collect();
356 let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
357 kind: HeldKind::Mirror,
358 memory: budget.mirror_memory,
359 });
360 let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
361 kind: HeldKind::Seed {
362 profile: seed.profile.clone(),
363 },
364 memory: seed.memory,
365 });
366 let shells = budget.shell_memory.iter().map(|shell| HeldPart {
367 kind: HeldKind::Shell { job: shell.job },
368 memory: shell.memory,
369 });
370 let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
371 let drawn: u64 = slots
372 .iter()
373 .map(|part| part.memory)
374 .chain(held.iter().map(|part| part.memory))
375 .sum();
376 let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
377 let unclaimed: u64 = slots
378 .iter()
379 .filter(|part| !part.reserved)
380 .map(|part| part.memory)
381 .sum();
382 let (free, closed) = budget.room_parts(drawn, unclaimed);
383 BudgetParts {
384 slots,
385 held,
386 free,
387 closed,
388 whole,
389 }
390 }
391}
392
393impl Budget {
394 fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
397 let Some(max) = self.memory_max else {
398 return (None, None);
399 };
400 let room = max.saturating_sub(drawn);
401 let shut = if self.closed() {
403 Some((
404 ClosedKind::Held,
405 "closed: the slice is past 90% of its MemoryMax, so no runner starts",
406 ))
407 } else if self.host.as_ref().is_some_and(|host| host.stalled) {
408 Some((
409 ClosedKind::Stalled,
410 "closed: the host's programs stall waiting for memory, so no runner starts",
411 ))
412 } else {
413 None
414 };
415 if let Some((kind, text)) = shut {
416 let closed = (room > 0).then(|| ClosedPart {
417 memory: room,
418 kind,
419 reading: Reading {
420 figure: ByteSize(room).to_string(),
421 text: text.to_owned(),
422 },
423 });
424 return (None, closed);
425 }
426 let withheld = self.host_withheld();
427 let host = withheld
430 .map(|withheld| withheld.min(room))
431 .filter(|&memory| memory > 0);
432 let (host_limit, spare) = self
433 .host
434 .as_ref()
435 .map_or((0, 0), |host| (host.limit, host.spare));
436 let memory = room - host.unwrap_or(0);
437 let mut text = match (unclaimed, withheld) {
438 (0, None) => format!("of {} free", ByteSize(max)),
439 (_, None) => format!(
440 "of {} free past every slot's cap; {} is free beside what runs",
441 ByteSize(max),
442 ByteSize(self.remaining().unwrap_or(0))
443 ),
444 (0, Some(_)) => format!(
445 "free; the host leaves the slice {}",
446 self.host_share(host_limit, max)
447 ),
448 (_, Some(_)) => format!(
449 "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
450 ByteSize(spare),
451 self.host_share(host_limit, max)
452 ),
453 };
454 if let Some(pooled) = &self.pooled {
456 text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
457 }
458 let free = (memory > 0).then(|| FreePart {
459 memory,
460 reading: Reading {
461 figure: ByteSize(memory).to_string(),
462 text,
463 },
464 });
465 let host = host.map(|memory| ClosedPart {
466 memory,
467 kind: ClosedKind::Host,
468 reading: Reading {
469 figure: ByteSize(memory).to_string(),
470 text: if host_limit > max {
472 format!(
473 "held back by what runs reserves: the host leaves the slice {}",
474 self.host_share(host_limit, max)
475 )
476 } else {
477 format!(
478 "{} for its other programs and page cache, which leave the slice {}",
479 ClosedKind::Host.name(),
480 ByteSize(host_limit)
481 )
482 },
483 },
484 });
485 (free, host)
486 }
487}
488
489#[derive(Debug, Clone, PartialEq, Eq)]
491pub struct BudgetParts {
492 pub slots: Vec<SlotPart>,
494 pub held: Vec<HeldPart>,
496 pub free: Option<FreePart>,
500 pub closed: Option<ClosedPart>,
503 pub whole: u64,
505}
506
507#[derive(Debug, Clone, Copy, PartialEq, Eq)]
509pub struct SlotPart {
510 pub profile: usize,
512 pub slot: usize,
514 pub memory: u64,
516 pub reserved: bool,
518}
519
520#[derive(Debug, Clone, PartialEq, Eq)]
522pub struct HeldPart {
523 pub kind: HeldKind,
525 pub memory: u64,
527}
528
529impl HeldPart {
530 pub fn name(&self) -> String {
532 match &self.kind {
533 HeldKind::Mirror => "mirror".to_owned(),
534 HeldKind::Seed { profile } => format!("{profile} seed"),
535 HeldKind::Shell { job } => format!("job {job} shell"),
536 }
537 }
538
539 pub fn holder(&self) -> String {
542 match &self.kind {
543 HeldKind::Mirror => "held by the mirror".to_owned(),
544 HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
545 HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
546 }
547 }
548}
549
550#[derive(Debug, Clone, PartialEq, Eq)]
552pub enum HeldKind {
553 Mirror,
555 Seed {
557 profile: String,
559 },
560 Shell {
562 job: u64,
564 },
565}
566
567#[derive(Debug, Clone, PartialEq, Eq)]
569pub struct FreePart {
570 pub memory: u64,
572 pub reading: Reading,
575}
576
577#[derive(Debug, Clone, PartialEq, Eq)]
579pub struct ClosedPart {
580 pub memory: u64,
582 pub kind: ClosedKind,
584 pub reading: Reading,
588}
589
590#[derive(Debug, Clone, Copy, PartialEq, Eq)]
592pub enum ClosedKind {
593 Host,
595 Held,
597 Stalled,
599}
600
601impl ClosedKind {
602 pub fn name(self) -> &'static str {
605 match self {
606 Self::Host => "held by the host",
607 Self::Held => "closed past 90%",
608 Self::Stalled => "closed while the host stalls",
609 }
610 }
611}
612
613#[cfg(test)]
614mod tests;