1use super::units::{duration, utc_time};
5use super::{AdmissionPolicy, Budget, CountBrake, Counted, HostMemory, Status, percent};
6use crate::config::ByteSize;
7
8#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Reading {
12 pub figure: String,
14 pub text: String,
16}
17
18#[derive(Debug, Clone, PartialEq, Eq)]
20pub struct HostFigure {
21 pub label: &'static str,
23 pub value: String,
25 pub note: Option<String>,
27}
28
29impl HostFigure {
30 fn new(label: &'static str, value: String) -> HostFigure {
31 HostFigure {
32 label,
33 value,
34 note: None,
35 }
36 }
37}
38
39impl Reading {
40 pub fn line(&self) -> String {
42 format!("{} {}", self.figure, self.text)
43 }
44}
45
46impl Budget {
47 pub fn memory_reading(&self) -> Reading {
52 let of = self
53 .memory_max
54 .map_or_else(String::new, |max| format!("of {} ", ByteSize(max)));
55 let verb = if self.pooled.is_some() {
56 "counted"
57 } else {
58 "reserved"
59 };
60 let mut text = format!("{of}{verb} in {}", self.slice);
61 if let Some(note) = self.admission_note() {
62 text = format!("{text} {note}");
63 }
64 if let Some(pooled) = &self.pooled {
65 let reserved = ByteSize(self.memory_in_use);
66 text = format!(
67 "{text}: {reserved} reserved, {} pooled",
68 ByteSize(pooled.credit)
69 );
70 }
71 if self.memory_max.is_none() {
72 text.push_str(", which has no MemoryMax");
73 }
74 if let Some(counted) = &self.counted {
75 text = format!("{text}; {}", counted.clause());
76 }
77 Reading {
78 figure: ByteSize(self.counted()).to_string(),
79 text,
80 }
81 }
82
83 pub fn counted(&self) -> u64 {
85 self.pooled
86 .map_or(self.memory_in_use, |pooled| pooled.counted)
87 }
88
89 pub fn clauses(&self) -> Vec<String> {
92 let mut clauses = vec![self.memory_reading().line()];
93 clauses.extend(self.cpu.map(|cpu| cpu.line()));
94 clauses.extend(self.host_line());
95 clauses
96 }
97
98 pub fn host_line(&self) -> Option<String> {
111 let host = self.host.as_ref()?;
112 let mut line = format!("host: {}", host.memory.available_line());
113 if let Some(swap) = host.memory.swap {
114 line = format!(
115 "{line}, swap {} of {} used",
116 ByteSize(swap.used()),
117 ByteSize(swap.total)
118 );
119 }
120 if let Some(full) = host.memory.full.filter(|full| full.avg10 > 0) {
121 line = format!("{line}, {} stalled for memory", percent(full.avg10));
122 }
123 let free = match host.spare {
124 0 => "nothing".to_owned(),
125 spare => ByteSize(spare).to_string(),
126 };
127 let limit = ByteSize(host.limit);
128 line = if host.stalled {
129 format!("{line}; stalling, so no runner starts")
130 } else if self.closed() {
131 format!("{line}; leaves the slice {limit}")
132 } else {
133 match self.memory_max {
134 Some(max) if self.host_withheld().is_some() => format!(
135 "{line}; leaves the slice {}, so {free} is free",
136 self.host_share(host.limit, max)
137 ),
138 Some(_) => format!("{line}; leaves the slice {limit}"),
139 None => format!("{line}; leaves the slice {limit}, so {free} is free"),
140 }
141 };
142 let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
144 if let Some(growth) = host
145 .growth_clause(self.memory_in_use)
146 .filter(|_| !growth_counts)
147 {
148 line = format!(
149 "{line}; what runs counts {}: {growth}",
150 ByteSize(host.counts(self.memory_in_use))
151 );
152 }
153 if !host.memory.holders.is_empty() {
154 line = format!("{line}; {}", host.memory.outside_line());
155 }
156 Some(line)
157 }
158
159 pub fn host_figures(&self) -> Vec<HostFigure> {
165 let Some(host) = self.host.as_ref() else {
166 return Vec::new();
167 };
168 let memory = &host.memory;
169 let size = |bytes: u64| ByteSize(bytes).to_string();
170 let mut available = format!("{} of {}", size(memory.available), size(memory.total));
171 if memory.growable() > 0 {
172 available = format!("{available}, growing to {}", size(memory.grown()));
173 }
174 let mut figures = vec![HostFigure::new("Host available", available)];
175 if let Some(swap) = memory.swap {
176 figures.push(HostFigure::new(
177 "Swap used",
178 format!("{} of {}", size(swap.used()), size(swap.total)),
179 ));
180 }
181 if let Some(full) = memory.full.filter(|full| full.avg10 > 0) {
182 figures.push(HostFigure::new("Stalled for memory", percent(full.avg10)));
183 }
184 let leaves = if host.stalled {
185 "nothing while it stalls: no runner starts".to_owned()
186 } else {
187 match self.memory_max {
188 Some(max) if !self.closed() && self.host_withheld().is_some() => {
189 self.host_share(host.limit, max)
190 }
191 _ => size(host.limit),
192 }
193 };
194 figures.push(HostFigure::new("Leaves the slice", leaves));
195 let growth_counts = self.counted.is_some_and(|counted| counted.kept.is_some());
196 if let Some(growth) = host
197 .growth_clause(self.memory_in_use)
198 .filter(|_| !growth_counts)
199 {
200 figures.push(HostFigure {
201 note: Some(growth),
202 ..HostFigure::new("What runs counts", size(host.counts(self.memory_in_use)))
203 });
204 }
205 if !memory.holders.is_empty() {
206 let named: Vec<String> = memory
207 .holders
208 .iter()
209 .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
210 .collect();
211 let note = match named.as_slice() {
212 [one] => format!("most {one}"),
213 [rest @ .., last] => format!("most {} and {last}", rest.join(", ")),
214 [] => String::new(),
215 };
216 figures.push(HostFigure {
217 note: Some(note),
218 ..HostFigure::new("Outside CI", size(memory.outside()))
219 });
220 }
221 figures
222 }
223
224 fn host_share(&self, limit: u64, max: u64) -> String {
229 if limit > max {
230 let host = self.host.as_ref();
231 let (counts, verb) = match host.filter(|host| host.credit.is_some()) {
232 Some(host) => (host.counts(self.memory_in_use), "counted"),
233 None => (self.memory_in_use, "reserved"),
234 };
235 format!(
236 "{} for the {} {verb} there",
237 ByteSize(limit),
238 ByteSize(counts)
239 )
240 } else {
241 format!("{} of its {}", ByteSize(limit), ByteSize(max))
242 }
243 }
244
245 fn closed(&self) -> bool {
248 self.held && self.admission == AdmissionPolicy::Measured
249 }
250
251 pub fn host_withheld(&self) -> Option<u64> {
255 let host = self.host.as_ref().filter(|host| !host.stalled)?;
256 let remaining = self.remaining()?;
257 (host.spare < remaining).then(|| remaining - host.spare)
258 }
259
260 pub fn line(&self) -> String {
262 self.clauses().join("; ")
263 }
264
265 pub fn remaining(&self) -> Option<u64> {
268 self.memory_max
269 .map(|max| max.saturating_sub(self.counted()))
270 }
271}
272
273impl Counted {
274 pub fn clause(&self) -> String {
278 let size = |bytes: u64| ByteSize(bytes).to_string();
279 let rests = |until: u64| format!("the growth count rests until {}", utc_time(until));
280 match (self.brake, self.kept) {
281 (None, Some(kept)) => format!(
282 "what runs uses {} and keeps {} for its growth",
283 size(self.used),
284 size(kept)
285 ),
286 (Some(CountBrake::Grew { grew, kept, until }), _) => format!(
287 "{}: what ran grew {} in {}, past the {} kept",
288 rests(until),
289 size(grew),
290 duration(super::GROWTH_WINDOW.as_secs()),
291 size(kept)
292 ),
293 (Some(CountBrake::Restarted { until }), _) => {
294 format!(
295 "{}: too soon after a start or a reload to judge growth",
296 rests(until)
297 )
298 }
299 (Some(CountBrake::Guard { until: Some(until) }), _) => {
300 format!("{}: the host ran under its line", rests(until))
301 }
302 (Some(CountBrake::Guard { until: None }) | None, _) => {
303 "the growth count rests while the host is under its line or a reading is missing"
304 .to_owned()
305 }
306 (Some(CountBrake::Unknown), _) => "the growth count rests".to_owned(),
307 }
308 }
309}
310
311impl HostMemory {
312 pub fn available_line(&self) -> String {
316 let line = format!(
317 "{} of {} available",
318 ByteSize(self.available),
319 ByteSize(self.total)
320 );
321 match self.growable() {
322 0 => line,
323 _ => format!("{line}, growing to {}", ByteSize(self.grown())),
324 }
325 }
326}
327
328impl Status {
329 pub fn budget_parts(&self) -> BudgetParts {
333 let budget = &self.budget;
334 let slots: Vec<SlotPart> = self
335 .profiles
336 .iter()
337 .enumerate()
338 .flat_map(|(profile, status)| {
339 status.slots.iter().filter_map(move |slot| {
340 let memory = match budget.admission {
341 AdmissionPolicy::Caps => match &slot.runner {
342 Some(runner) if runner.reserved > 0 => runner.reserved,
343 _ => status.memory,
344 },
345 AdmissionPolicy::Measured => slot.runner.as_ref()?.reserved,
346 };
347 Some(SlotPart {
348 profile,
349 slot: slot.slot,
350 memory,
351 reserved: slot.runner.is_some(),
352 })
353 })
354 })
355 .collect();
356 let mirror = (budget.mirror_memory > 0).then_some(HeldPart {
357 kind: HeldKind::Mirror,
358 memory: budget.mirror_memory,
359 });
360 let seeds = budget.seed_memory.iter().map(|seed| HeldPart {
361 kind: HeldKind::Seed {
362 profile: seed.profile.clone(),
363 },
364 memory: seed.memory,
365 });
366 let shells = budget.shell_memory.iter().map(|shell| HeldPart {
367 kind: HeldKind::Shell { job: shell.job },
368 memory: shell.memory,
369 });
370 let held: Vec<HeldPart> = mirror.into_iter().chain(seeds).chain(shells).collect();
371 let drawn: u64 = slots
372 .iter()
373 .map(|part| part.memory)
374 .chain(held.iter().map(|part| part.memory))
375 .sum();
376 let whole = budget.memory_max.unwrap_or(drawn).max(drawn);
377 let unclaimed: u64 = slots
378 .iter()
379 .filter(|part| !part.reserved)
380 .map(|part| part.memory)
381 .sum();
382 let (free, closed) = budget.room_parts(drawn, unclaimed);
383 BudgetParts {
384 slots,
385 held,
386 free,
387 closed,
388 whole,
389 }
390 }
391}
392
393impl Budget {
394 fn room_parts(&self, drawn: u64, unclaimed: u64) -> (Option<FreePart>, Option<ClosedPart>) {
397 let Some(max) = self.memory_max else {
398 return (None, None);
399 };
400 let room = max.saturating_sub(drawn);
401 let shut = if self.closed() {
403 Some((
404 ClosedKind::Held,
405 "closed: the slice is past 90% of its MemoryMax, so no runner starts",
406 ))
407 } else if self.host.as_ref().is_some_and(|host| host.stalled) {
408 Some((
409 ClosedKind::Stalled,
410 "closed: the host's programs stall waiting for memory, so no runner starts",
411 ))
412 } else {
413 None
414 };
415 if let Some((kind, text)) = shut {
416 let closed = (room > 0).then(|| ClosedPart {
417 memory: room,
418 kind,
419 reading: Reading {
420 figure: ByteSize(room).to_string(),
421 text: text.to_owned(),
422 },
423 });
424 return (None, closed);
425 }
426 let withheld = self.host_withheld();
427 let host = withheld
430 .map(|withheld| withheld.min(room))
431 .filter(|&memory| memory > 0);
432 let (host_limit, spare) = self
433 .host
434 .as_ref()
435 .map_or((0, 0), |host| (host.limit, host.spare));
436 let memory = room - host.unwrap_or(0);
437 let mut text = match (unclaimed, withheld) {
438 (0, None) => format!("of {} free", ByteSize(max)),
439 (_, None) => format!(
440 "of {} free past every slot's cap; {} is free beside what runs",
441 ByteSize(max),
442 ByteSize(self.remaining().unwrap_or(0))
443 ),
444 (0, Some(_)) => format!(
445 "free; the host leaves the slice {}",
446 self.host_share(host_limit, max)
447 ),
448 (_, Some(_)) => format!(
449 "free past every slot's cap; {} is free beside what runs, as the host leaves the slice {}",
450 ByteSize(spare),
451 self.host_share(host_limit, max)
452 ),
453 };
454 if let Some(pooled) = &self.pooled {
456 text = format!("{text}, and pooling adds {}", ByteSize(pooled.credit));
457 }
458 let free = (memory > 0).then(|| FreePart {
459 memory,
460 reading: Reading {
461 figure: ByteSize(memory).to_string(),
462 text,
463 },
464 });
465 let host = host.map(|memory| ClosedPart {
466 memory,
467 kind: ClosedKind::Host,
468 reading: Reading {
469 figure: ByteSize(memory).to_string(),
470 text: if host_limit > max {
472 format!(
473 "held back by what runs reserves: the host leaves the slice {}",
474 self.host_share(host_limit, max)
475 )
476 } else {
477 format!(
478 "{} for its other programs and page cache, which leave the slice {}",
479 ClosedKind::Host.name(),
480 ByteSize(host_limit)
481 )
482 },
483 },
484 });
485 (free, host)
486 }
487}
488
489#[derive(Debug, Clone, PartialEq, Eq)]
491pub struct BudgetParts {
492 pub slots: Vec<SlotPart>,
494 pub held: Vec<HeldPart>,
496 pub free: Option<FreePart>,
500 pub closed: Option<ClosedPart>,
503 pub whole: u64,
505}
506
507#[derive(Debug, Clone, Copy, PartialEq, Eq)]
509pub struct SlotPart {
510 pub profile: usize,
512 pub slot: usize,
514 pub memory: u64,
516 pub reserved: bool,
518}
519
520#[derive(Debug, Clone, PartialEq, Eq)]
523pub struct SlotReading {
524 used: Option<u64>,
526 block: u64,
528 kind: &'static str,
530 state: &'static str,
532}
533
534#[derive(Debug, Clone, PartialEq, Eq)]
536pub struct SlotForm {
537 pub text: String,
539 pub room: usize,
542}
543
544impl SlotReading {
545 pub fn forms(&self) -> Vec<SlotForm> {
549 let block = ByteSize(self.block);
550 let kind = self.kind;
551 let Some(used) = self.used.map(ByteSize) else {
552 let state = self.state;
553 return [
554 format!("{state} {block} {kind}"),
555 format!("{state} {}", block.short()),
556 state.to_owned(),
557 ]
558 .into_iter()
559 .map(|text| SlotForm {
560 room: text.chars().count(),
561 text,
562 })
563 .collect();
564 };
565 let long = used.to_string();
566 let short = used.short();
567 let form = |figure: &str, widest: usize, rest: String| {
568 let text = format!("{figure}{rest}");
569 SlotForm {
570 room: text.chars().count() + widest.saturating_sub(figure.chars().count()),
571 text,
572 }
573 };
574 vec![
575 form(&long, 5, format!(" of {block} {kind}")),
576 form(&short, 4, format!(" of {} {kind}", block.short())),
577 form(&long, 5, String::new()),
578 form(&short, 4, String::new()),
579 ]
580 }
581}
582
583impl Status {
584 pub fn slot_reading(&self, part: &SlotPart, used: Option<u64>) -> SlotReading {
586 let state = self.profiles.get(part.profile).and_then(|profile| {
587 profile
588 .slots
589 .iter()
590 .find(|slot| slot.slot == part.slot)
591 .map(|slot| Status::slot_state(profile, slot))
592 });
593 SlotReading {
594 used: used.filter(|_| part.reserved),
595 block: part.memory,
596 kind: match self.budget.admission {
597 AdmissionPolicy::Caps => "cap",
598 AdmissionPolicy::Measured => "reserved",
599 },
600 state: state.as_ref().map_or("idle", super::SlotState::word),
601 }
602 }
603}
604
605#[derive(Debug, Clone, PartialEq, Eq)]
607pub struct HeldPart {
608 pub kind: HeldKind,
610 pub memory: u64,
612}
613
614impl HeldPart {
615 pub fn name(&self) -> String {
617 match &self.kind {
618 HeldKind::Mirror => "mirror".to_owned(),
619 HeldKind::Seed { profile } => format!("{profile} seed"),
620 HeldKind::Shell { job } => format!("job {job} shell"),
621 }
622 }
623
624 pub fn holder(&self) -> String {
627 match &self.kind {
628 HeldKind::Mirror => "held by the mirror".to_owned(),
629 HeldKind::Seed { profile } => format!("held by {profile}'s seed container"),
630 HeldKind::Shell { job } => format!("held by the shell in job {job}'s record"),
631 }
632 }
633}
634
635#[derive(Debug, Clone, PartialEq, Eq)]
637pub enum HeldKind {
638 Mirror,
640 Seed {
642 profile: String,
644 },
645 Shell {
647 job: u64,
649 },
650}
651
652#[derive(Debug, Clone, PartialEq, Eq)]
654pub struct FreePart {
655 pub memory: u64,
657 pub reading: Reading,
660}
661
662impl FreePart {
663 pub const NAME: &'static str = "free";
665
666 pub fn name(&self) -> &'static str {
668 Self::NAME
669 }
670}
671
672#[derive(Debug, Clone, PartialEq, Eq)]
674pub struct ClosedPart {
675 pub memory: u64,
677 pub kind: ClosedKind,
679 pub reading: Reading,
683}
684
685#[derive(Debug, Clone, Copy, PartialEq, Eq)]
687pub enum ClosedKind {
688 Host,
690 Held,
692 Stalled,
694}
695
696impl ClosedKind {
697 pub fn name(self) -> &'static str {
700 match self {
701 Self::Host => "held by the host",
702 Self::Held => "closed past 90%",
703 Self::Stalled => "closed while the host stalls",
704 }
705 }
706
707 pub fn short(self) -> &'static str {
709 match self {
710 Self::Host => "host",
711 Self::Held => "closed",
712 Self::Stalled => "stalled",
713 }
714 }
715}
716
717#[cfg(test)]
718mod tests;