1use std::collections::BTreeMap;
17use std::fmt::Write as _;
18
19use crate::api::{
20 Advice, ProfileSizing, Resize, SLOT_DISK, Sizing, SizingChange, SizingRefused, ago, cores,
21 duration, job_title,
22};
23use crate::cache::docker_budget_gib;
24use crate::config::{Admission, ByteSize, Config, Profile, cpu_budget};
25
26use super::{
27 CpusSet, GRANULE, HELD, History, SlotsSet, Throttling, Window, advise, busy, first_runner,
28 peak, short_of_slots, slot_asking_wait, spread, wants_slots,
29};
30
31pub(crate) const RAISE_RUNS: u64 = 3;
34
35const BOUND: u64 = 70;
39
40const HEADROOM: u64 = 2;
44
45const MET: u64 = 95;
49
50pub const IDLE_FOR: u64 = 7 * 24 * 3600 * 1000;
53
54const IDLE_SLOTS: u32 = 2;
57
58#[derive(Debug, Clone, Copy)]
60pub struct Context<'run> {
61 pub memory_max: Option<u64>,
63 pub cpus: Option<u64>,
65 pub window: Window,
67 pub last: &'run BTreeMap<String, u64>,
69 pub cpus_set: &'run [CpusSet],
72 pub running: &'run BTreeMap<String, u32>,
74 pub disk: Option<u64>,
77}
78
79pub fn changes(config: &Config, history: &History, context: Context<'_>) -> Vec<SizingChange> {
105 next_look(config, history, context).changes
106}
107
108#[derive(Debug, Clone, Default, PartialEq, Eq)]
110pub struct Look {
111 pub changes: Vec<SizingChange>,
113 pub refused: Vec<SizingRefused>,
116 pub refused_slots: Vec<SizingRefused>,
120 pub waited_before_change: Vec<String>,
123}
124
125pub fn next_look(config: &Config, history: &History, context: Context<'_>) -> Look {
127 let rules = &config.sizing;
128 if !rules.auto {
129 return Look::default();
130 }
131 let window = context.window;
132 let sizing = advise(config, context.memory_max, history, window);
133 let now = window.now / 1000;
134 let mut budget = Budget::new(config, &sizing.profiles);
135 let mut look = Look::default();
136 for advised in &sizing.profiles {
137 let Some(profile) = config.profiles.get(&advised.name) else {
138 continue;
139 };
140 let last = context.last.get(&advised.name).copied();
141 let since = last.map_or(u64::MAX, |at| now.saturating_sub(at));
142 if advised.revert.is_some() {
143 match reverted(config, profile, advised, &sizing, &context, &budget) {
144 Revert::Write(change) => {
145 budget.take(&advised.name, profile, change.from, change.to);
146 look.changes.push(change);
147 continue;
148 }
149 Revert::Wait => continue,
150 Revert::Outside => {}
152 }
153 }
154 if since < rules.raise_after().as_secs() {
155 continue;
156 }
157 let lowering = Lowering::after(since, rules.interval.as_secs());
158 let fresh = Fresh::read(history, advised, config.host.admission, last, window);
159 let mut reasons = Vec::new();
160 let memory = memory(config, profile, advised.advice, lowering);
161 let running = context.running.get(&advised.name).copied().unwrap_or(0);
162 let (slots, slots_reason) = match config.host.admission {
163 Admission::Caps => (
164 caps_slots(profile, advised, &fresh, memory, &context, &budget, running),
165 None,
166 ),
167 Admission::Measured => {
168 measured_slots(config, profile, advised, &fresh, &context, &budget)
169 .step(&advised.name, &mut look.refused_slots, || {
170 (lowering == Lowering::Allowed)
171 .then(|| idle_slots(config, history, advised, &context, running))?
172 })
173 .map_or((advised.slots, None), |step| (step.to, Some(step.reason)))
174 }
175 };
176 let (slots, cpus) = cpus_and_slots(
177 config,
178 profile,
179 advised,
180 history,
181 fresh.start,
182 &context,
183 &budget,
184 slots,
185 lowering,
186 );
187 if memory != profile.memory.0 {
188 reasons.push(memory_reason(
189 &sizing,
190 advised,
191 memory,
192 slots,
193 config.sizing.max_step,
194 ));
195 }
196 let advised_slots = config.host.admission == Admission::Caps && memory < profile.memory.0;
198 if let Some(reason) = slots_reason {
199 reasons.push(reason);
200 } else if slots > profile.slots && !advised_slots {
201 reasons.push(fresh.waited(last.is_some()));
202 }
203 let cpus = cpus.taken(&advised.name, &mut reasons, &mut look.refused);
204 let from = Resize {
205 memory: profile.memory.0,
206 slots: profile.slots,
207 cpus: cpus.map(|_| profile.cpus),
208 };
209 let to = Resize {
210 memory,
211 slots,
212 cpus,
213 };
214 if to == from || slots == 0 {
215 continue;
216 }
217 budget.take(&advised.name, profile, from, to);
218 look.changes.push(SizingChange {
219 at: now,
220 profile: advised.name.clone(),
221 from,
222 to,
223 reason: reasons.join("; "),
224 uncertain: false,
225 });
226 }
227 look.waited_before_change = waited_before_change(config, history, &sizing, &context);
228 look
229}
230
231fn waited_before_change(
235 config: &Config,
236 history: &History,
237 sizing: &Sizing,
238 context: &Context<'_>,
239) -> Vec<String> {
240 if config.host.admission != Admission::Measured {
241 return Vec::new();
242 }
243 sizing
244 .profiles
245 .iter()
246 .filter(|advised| advised.wants_slots)
247 .filter(|advised| {
248 let Some(last) = context.last.get(&advised.name).copied() else {
249 return false;
250 };
251 !Fresh::read(
252 history,
253 advised,
254 config.host.admission,
255 Some(last),
256 context.window,
257 )
258 .wants_slots()
259 })
260 .map(|advised| advised.name.clone())
261 .collect()
262}
263
264pub const UNDONE_RAISES: u64 = 24 * 3600;
266
267pub fn undone_raises(
272 running: &Config,
273 new: &Config,
274 changes: &[SizingChange],
275 now: u64,
276) -> Vec<String> {
277 new.profiles
278 .iter()
279 .filter_map(|(name, profile)| {
280 let memory = profile.memory.0;
281 let before = running.profiles.get(name)?.memory.0;
282 let raise = changes
283 .iter()
284 .filter(|change| {
285 change.profile == *name
286 && change.to.memory > change.from.memory
287 && now.saturating_sub(change.at) < UNDONE_RAISES
288 })
289 .max_by_key(|change| change.at)?;
290 (memory < before && memory < raise.to.memory).then(|| {
291 format!(
292 "{name}'s memory goes to {}, under the {} auto sizing raised it to {} because its jobs needed \
293 more than {}: they may run out of memory again",
294 ByteSize(memory),
295 ByteSize(raise.to.memory),
296 ago(raise.at, now),
297 ByteSize(raise.from.memory)
298 )
299 })
300 })
301 .collect()
302}
303
304#[derive(Debug, Clone, Copy, PartialEq, Eq)]
306enum Lowering {
307 Allowed,
308 Held,
309}
310
311impl Lowering {
312 fn after(since: u64, interval: u64) -> Lowering {
314 if since >= interval {
315 Lowering::Allowed
316 } else {
317 Lowering::Held
318 }
319 }
320}
321
322struct Budget {
324 caps: u64,
326 cpus: u64,
328 slots: BTreeMap<String, u32>,
330 each: BTreeMap<String, u32>,
332}
333
334impl Budget {
335 fn take(&mut self, name: &str, profile: &Profile, from: Resize, to: Resize) {
337 self.caps = self
338 .caps
339 .saturating_sub(from.memory.saturating_mul(u64::from(from.slots)))
340 .saturating_add(to.memory.saturating_mul(u64::from(to.slots)));
341 let millicores = |resize: Resize| {
342 u64::from(resize.cpus.unwrap_or(profile.cpus)) * 1000 * u64::from(resize.slots)
343 };
344 self.cpus = self
345 .cpus
346 .saturating_sub(millicores(from))
347 .saturating_add(millicores(to));
348 self.slots.insert(name.to_owned(), to.slots);
349 self.each
350 .insert(name.to_owned(), to.cpus.unwrap_or(profile.cpus));
351 }
352
353 fn new(config: &Config, profiles: &[ProfileSizing]) -> Budget {
355 Budget {
356 caps: config.memory_caps().map_or(u64::MAX, |caps| caps.0),
357 cpus: config
358 .profiles
359 .values()
360 .map(|profile| u64::from(profile.cpus) * 1000 * u64::from(profile.slots))
361 .sum(),
362 slots: profiles
363 .iter()
364 .map(|advised| (advised.name.clone(), advised.slots))
365 .collect(),
366 each: config
367 .profiles
368 .iter()
369 .map(|(name, profile)| (name.clone(), profile.cpus))
370 .collect(),
371 }
372 }
373
374 fn at_caps(&self, config: &Config, name: &str, cpus: u32) -> u64 {
378 let each = |other: &str| {
379 if other == name {
380 cpus
381 } else {
382 self.each.get(other).copied().unwrap_or(0)
383 }
384 };
385 let warm: u64 = config
386 .profiles
387 .keys()
388 .map(|other| {
389 let slots = self.slots.get(other).copied().unwrap_or(0);
390 u64::from(warm_runners(config, other, slots)) * u64::from(each(other))
391 })
392 .sum();
393 let largest = config
394 .profiles
395 .keys()
396 .map(|other| each(other))
397 .max()
398 .unwrap_or(0);
399 (warm + u64::from(largest)) * 1000
400 }
401}
402
403fn warm_runners(config: &Config, profile: &str, slots: u32) -> u32 {
405 config
406 .repositories
407 .iter()
408 .filter(|repository| repository.profile == profile)
409 .map(|repository| repository.warm)
410 .sum::<u32>()
411 .min(slots)
412}
413
414struct Fresh {
417 start: u64,
419 jobs: u64,
420 wants: bool,
422 wait: u64,
424 short: u32,
426 busy: u32,
427}
428
429impl Fresh {
430 fn read(
432 history: &History,
433 advised: &ProfileSizing,
434 admission: Admission,
435 last: Option<u64>,
436 window: Window,
437 ) -> Fresh {
438 let start = last.map_or(window.since, |at| at.saturating_mul(1000).max(window.since));
439 let jobs: Vec<_> = history
440 .jobs
441 .iter()
442 .filter(|job| job.profile == advised.name && job.queued_at >= start)
443 .collect();
444 let busy = busy(
445 history,
446 &advised.name,
447 advised.slots,
448 first_runner(history, start),
449 window.now,
450 );
451 Fresh {
452 start,
453 jobs: jobs.len() as u64,
454 wants: wants_slots(admission, &jobs, history.kept_since, busy),
455 wait: slot_asking_wait(admission, &jobs, history.kept_since),
456 short: short_of_slots(&jobs),
457 busy,
458 }
459 }
460
461 fn wants_slots(&self) -> bool {
464 self.jobs >= RAISE_RUNS && self.wants
465 }
466
467 fn waited(&self, changed: bool) -> String {
469 format!(
470 "{} jobs {}waited {}s for a slot (p95) and slots were busy {}%; one more fits the slice beside the \
471 other profiles'",
472 self.jobs,
473 if changed {
474 "since its last change "
475 } else {
476 ""
477 },
478 self.wait,
479 self.busy
480 )
481 }
482}
483
484fn memory_reason(
488 sizing: &Sizing,
489 advised: &ProfileSizing,
490 memory: u64,
491 slots: u32,
492 max_step: u32,
493) -> String {
494 let made = ProfileSizing {
495 advice: match advised.advice {
496 Advice::Lower { .. } => Advice::Lower { memory, slots },
497 Advice::Raise { .. } => Advice::Raise { memory, slots },
498 other => other,
499 },
500 ..advised.clone()
501 };
502 let line = sizing.memory_line(&made);
503 match advised.advice.change() {
504 Some(advice) if advice.memory != memory => format!(
505 "{line}; a step toward {}, at most {max_step}% a change",
506 ByteSize(advice.memory)
507 ),
508 _ => line,
509 }
510}
511
512enum Revert {
514 Write(SizingChange),
516 Wait,
519 Outside,
521}
522
523fn reverted(
527 config: &Config,
528 profile: &Profile,
529 advised: &ProfileSizing,
530 sizing: &Sizing,
531 context: &Context<'_>,
532 budget: &Budget,
533) -> Revert {
534 let Advice::Raise { memory, slots } = advised.advice else {
535 return Revert::Outside;
536 };
537 let from = Resize {
538 memory: profile.memory.0,
539 slots: profile.slots,
540 cpus: None,
541 };
542 let slots = match (config.host.admission, context.memory_max) {
545 (Admission::Caps, Some(memory_max)) => {
546 let share = profile.memory.0.saturating_mul(u64::from(profile.slots));
547 let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
548 slots.min(u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX))
549 }
550 _ => slots,
551 };
552 if slots == 0 {
553 return Revert::Wait;
554 }
555 let running = context.running.get(&advised.name).copied().unwrap_or(0);
556 let to = Resize {
557 memory,
558 slots,
559 cpus: None,
560 };
561 if config.host.admission == Admission::Caps && running > slots {
562 return Revert::Wait;
563 }
564 Revert::Write(SizingChange {
565 at: sizing.until,
566 profile: advised.name.clone(),
567 from,
568 to,
569 reason: sizing.memory_line(advised),
570 uncertain: false,
571 })
572}
573
574fn memory(config: &Config, profile: &Profile, advice: Advice, lowering: Lowering) -> u64 {
579 let current = profile.memory.0;
580 let step = current.saturating_mul(u64::from(config.sizing.max_step)) / 100;
581 match advice {
582 Advice::Lower { memory, .. } if lowering == Lowering::Allowed => {
583 let floor = current
584 .saturating_sub(step)
585 .div_ceil(GRANULE)
586 .saturating_mul(GRANULE);
587 memory.max(floor).min(current)
588 }
589 Advice::Raise { memory, .. } => memory,
590 _ => current,
591 }
592}
593
594fn caps_slots(
599 profile: &Profile,
600 advised: &ProfileSizing,
601 fresh: &Fresh,
602 memory: u64,
603 context: &Context<'_>,
604 budget: &Budget,
605 running: u32,
606) -> u32 {
607 let current = profile.slots;
608 let fit = |room: u64| u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX);
609 if let Advice::Lower { slots, .. } = advised.advice
611 && memory < profile.memory.0
612 && advised.wants_slots
613 {
614 let share = profile.memory.0.saturating_mul(u64::from(current));
615 return fit(share).min(slots).max(current);
616 }
617 let Some(memory_max) = context.memory_max else {
618 return current;
619 };
620 if memory > profile.memory.0 {
621 let share = profile.memory.0.saturating_mul(u64::from(current));
622 let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
623 let slots = fit(room).min(current);
625 return if slots < running { 0 } else { slots };
626 }
627 if fresh.wants_slots()
628 && profile.max_slots.is_none_or(|max| current < max)
629 && budget.caps.saturating_add(memory) <= memory_max
630 {
631 return current + 1;
632 }
633 current
634}
635
636struct SlotStep {
638 to: u32,
639 reason: String,
640}
641
642enum Raise {
644 Unasked,
646 To(SlotStep),
648 Refused(String),
650}
651
652impl Raise {
653 fn step(
656 self,
657 profile: &str,
658 refused: &mut Vec<SizingRefused>,
659 idle: impl FnOnce() -> Option<SlotStep>,
660 ) -> Option<SlotStep> {
661 match self {
662 Raise::To(step) => Some(step),
663 Raise::Refused(reason) => {
664 refused.push(SizingRefused::new(profile, reason));
665 None
666 }
667 Raise::Unasked => idle(),
668 }
669 }
670}
671
672fn measured_slots(
680 config: &Config,
681 profile: &Profile,
682 advised: &ProfileSizing,
683 fresh: &Fresh,
684 context: &Context<'_>,
685 budget: &Budget,
686) -> Raise {
687 let current = advised.slots;
688 if !fresh.wants_slots() {
689 return Raise::Unasked;
690 }
691 let asked = format!(
692 "{} jobs {}waited {}s for a slot (p95) while memory and CPU had room, {} at once at most",
693 fresh.jobs,
694 if context.last.contains_key(&advised.name) {
695 "since its last change "
696 } else {
697 ""
698 },
699 fresh.wait,
700 fresh.short,
701 );
702 let max = profile.max_slots.unwrap_or(u32::MAX);
703 if current >= max {
704 return Raise::Refused(format!("{asked}; it is at its max_slots, {max}"));
705 }
706 let Some(disk) = context.disk else {
707 return Raise::Refused(format!(
708 "{asked}; Docker's disk is not measured yet, which the next check of the caches does"
709 ));
710 };
711 let step = (current.saturating_mul(config.sizing.max_step) / 100).max(1);
712 let wanted = current.saturating_add(fresh.short.clamp(1, step)).min(max);
713 let others: u32 = budget
714 .slots
715 .iter()
716 .filter(|(name, _)| **name != advised.name)
717 .map(|(_, slots)| *slots)
718 .sum();
719 let Some(to) = (current.saturating_add(1)..=wanted)
720 .rev()
721 .find(|slots| share(disk, others.saturating_add(*slots)) >= SLOT_DISK)
722 else {
723 let next = current.saturating_add(1);
724 return Raise::Refused(format!(
725 "{asked}; at {next} slots, beside the other profiles' {others}, each slot would keep {}, under the {} \
726 auto sizing leaves each slot",
727 kept(
728 disk,
729 others.saturating_add(current),
730 others.saturating_add(next)
731 ),
732 ByteSize(SLOT_DISK),
733 ));
734 };
735 Raise::To(SlotStep {
736 to,
737 reason: format!(
738 "{asked}; at {to} slots each slot keeps {}",
739 kept(
740 disk,
741 others.saturating_add(current),
742 others.saturating_add(to)
743 )
744 ),
745 })
746}
747
748fn share(disk: u64, slots: u32) -> u64 {
751 docker_budget_gib(disk, u64::from(slots)) << 30
752}
753
754fn kept(disk: u64, from: u32, to: u32) -> String {
757 let (before, after) = (share(disk, from), share(disk, to));
758 if after < before {
759 format!(
760 "{} of Docker's disk, down from {}",
761 ByteSize(after),
762 ByteSize(before)
763 )
764 } else {
765 format!("{} of Docker's disk", ByteSize(after))
766 }
767}
768
769fn idle_slots(
776 config: &Config,
777 history: &History,
778 advised: &ProfileSizing,
779 context: &Context<'_>,
780 running: u32,
781) -> Option<SlotStep> {
782 let current = advised.slots;
783 let now = context.window.now;
784 let start = now.checked_sub(IDLE_FOR)?;
785 let set = slots_since(&history.slots_set, &advised.name, current, now);
786 let changed = history
788 .changes
789 .iter()
790 .filter(|change| change.profile == advised.name && change.to.slots != change.from.slots)
791 .map(|change| change.at.saturating_mul(1000))
792 .max()
793 .unwrap_or(0);
794 let first = history
796 .runners
797 .iter()
798 .map(|runner| runner.started_at)
799 .min()?;
800 if set.max(changed).max(first) > start {
801 return None;
802 }
803 let jobs = history
804 .jobs
805 .iter()
806 .filter(|job| job.profile == advised.name && job.queued_at >= start)
807 .count() as u64;
808 if jobs < u64::from(config.sizing.min_jobs) {
809 return None;
810 }
811 let mut edges: Vec<(u64, i32)> = history
812 .runners
813 .iter()
814 .filter(|runner| {
815 runner.profile == advised.name && runner.removed_at.is_none_or(|at| at > start)
816 })
817 .flat_map(|runner| {
818 [
819 (runner.started_at.max(start), 1),
820 (runner.removed_at.unwrap_or(now).min(now), -1),
821 ]
822 })
823 .collect();
824 let ran = peak(&mut edges);
825 let warm = warm_runners(config, &advised.name, current);
826 if ran.saturating_add(warm).saturating_add(IDLE_SLOTS) > current {
827 return None;
828 }
829 let to = current.saturating_sub(1).max(running).max(1);
830 (to < current).then(|| SlotStep {
831 to,
832 reason: format!(
833 "at most {ran} of its {current} slots held a runner for a job at once in the last {}{}: an idle slot \
834 shrinks every slot's share of Docker's disk",
835 duration(IDLE_FOR / 1000),
836 if warm > 0 {
837 format!(", beside {warm} warm runners")
838 } else {
839 String::new()
840 }
841 ),
842 })
843}
844
845fn slots_since(set: &[SlotsSet], profile: &str, slots: u32, now: u64) -> u64 {
847 since_set(
848 set.iter()
849 .filter(|event| event.profile == profile)
850 .map(|event| (event.slots, event.at)),
851 slots,
852 now,
853 )
854}
855
856#[expect(
860 clippy::too_many_arguments,
861 reason = "each is one thing a CPU change reads; a struct for them would be read here only"
862)]
863fn cpus_and_slots(
864 config: &Config,
865 profile: &Profile,
866 advised: &ProfileSizing,
867 history: &History,
868 since: u64,
869 context: &Context<'_>,
870 budget: &Budget,
871 slots: u32,
872 lowering: Lowering,
873) -> (u32, CpuStep) {
874 let size = |slots: u32| {
875 cpus(
876 config, profile, advised, history, since, context, budget, slots, lowering,
877 )
878 };
879 let fit = |slots: u32, cpus: &CpuStep| {
881 let cpus = match cpus {
882 CpuStep::To(cpus) => cpus.to,
883 CpuStep::Keep | CpuStep::Refused(_) => profile.cpus,
884 };
885 if slots > profile.slots {
886 cpu_slots(config, profile, context, budget, cpus)
887 .map_or(slots, |fit| slots.min(fit.max(profile.slots)))
888 } else {
889 slots
890 }
891 };
892 let mut cpus = size(slots);
893 let fitted = fit(slots, &cpus);
894 if fitted < slots {
895 cpus = size(fitted);
896 }
897 (fit(fitted, &cpus), cpus)
898}
899
900fn cpu_slots(
904 config: &Config,
905 profile: &Profile,
906 context: &Context<'_>,
907 budget: &Budget,
908 cpus: u32,
909) -> Option<u32> {
910 let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
911 else {
912 return None;
913 };
914 let Some(slice) = context.cpus else {
915 return Some(profile.slots);
916 };
917 let others = budget
918 .cpus
919 .saturating_sub(u64::from(profile.cpus) * 1000 * u64::from(profile.slots));
920 let room = cpu_budget(slice, overcommit).saturating_sub(others);
921 Some(u32::try_from(room / (u64::from(cpus.max(1)) * 1000)).unwrap_or(u32::MAX))
922}
923
924enum CpuStep {
926 Keep,
927 To(Cpus),
928 Refused(String),
930}
931
932impl CpuStep {
933 fn taken(
935 self,
936 profile: &str,
937 reasons: &mut Vec<String>,
938 refused: &mut Vec<SizingRefused>,
939 ) -> Option<u32> {
940 match self {
941 CpuStep::To(cpus) => {
942 reasons.push(cpus.reason);
943 Some(cpus.to)
944 }
945 CpuStep::Refused(reason) => {
946 refused.push(SizingRefused {
947 profile: profile.to_owned(),
948 reason,
949 });
950 None
951 }
952 CpuStep::Keep => None,
953 }
954 }
955}
956
957struct Cpus {
959 to: u32,
960 reason: String,
961}
962
963type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
965
966fn cpus_since(set: &[CpusSet], profile: &str, cpus: u32, now: u64) -> u64 {
969 since_set(
970 set.iter()
971 .filter(|event| event.profile == profile)
972 .map(|event| (event.cpus, event.at)),
973 cpus,
974 now,
975 )
976}
977
978fn since_set(events: impl Iterator<Item = (u32, u64)>, value: u32, now: u64) -> u64 {
981 let mut since = None;
982 let mut seen = false;
983 for (set, at) in events {
984 seen = true;
985 if set != value {
986 since = None;
987 } else if since.is_none() {
988 since = Some(at);
989 }
990 }
991 since.unwrap_or(if seen { now } else { 0 })
992}
993
994#[derive(Default)]
996struct JobCpu {
997 averages: Vec<u64>,
999 peak: u64,
1001 throttled: u64,
1003 peaked: u64,
1005 runs: Vec<Throttling>,
1007}
1008
1009impl JobCpu {
1010 fn held(&self) -> u64 {
1012 self.throttled + self.peaked
1013 }
1014
1015 fn held_reason(&self, job: JobKey<'_>, current: u32) -> String {
1017 let how = if self.peaked == 0 {
1018 format!("throttled in {HELD}% of their run or more")
1019 } else {
1020 format!(
1021 "throttled in {HELD}% of their run or more, or peaking at the cap where that was not measured"
1022 )
1023 };
1024 format!(
1025 "{} runs of {} were held back by its {current} CPUs, {how}: under measured admission they bound no slot, \
1026 so the cap only slows them",
1027 self.held(),
1028 job_named(job)
1029 )
1030 }
1031}
1032
1033enum Hold {
1037 Throttled,
1038 Peaked,
1039 Free,
1040}
1041
1042impl Hold {
1043 fn of(job: &super::Ran, cap: u64) -> Hold {
1044 match job.throttling().held() {
1045 Some(true) => Hold::Throttled,
1046 Some(false) => Hold::Free,
1047 None => Hold::peaked(job, cap),
1048 }
1049 }
1050
1051 fn peaked(job: &super::Ran, cap: u64) -> Hold {
1052 if job.cpu_peak.saturating_mul(100) >= cap.saturating_mul(MET) {
1053 Hold::Peaked
1054 } else {
1055 Hold::Free
1056 }
1057 }
1058}
1059
1060fn runs_at<'run>(
1065 history: &'run History,
1066 profile: &str,
1067 cap: u64,
1068 since: u64,
1069) -> BTreeMap<JobKey<'run>, JobCpu> {
1070 let mut jobs: BTreeMap<JobKey<'run>, JobCpu> = BTreeMap::new();
1071 for job in &history.jobs {
1072 if job.profile == profile
1073 && job.cpu_cap == Some(cap)
1074 && job.succeeded
1075 && job.queued_at >= since
1076 {
1077 let read = jobs
1078 .entry((&job.repository, job.workflow.as_deref(), &job.name))
1079 .or_default();
1080 read.averages
1081 .extend(job.cpu_average.filter(|average| *average > 0));
1082 read.peak = read.peak.max(job.cpu_peak);
1083 match Hold::of(job, cap) {
1084 Hold::Throttled => read.throttled += 1,
1085 Hold::Peaked => read.peaked += 1,
1086 Hold::Free => {}
1087 }
1088 read.runs.push(job.throttling());
1089 }
1090 }
1091 jobs
1092}
1093
1094#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096pub struct CpuRun {
1097 pub cap: Option<u64>,
1099 pub queued_at: u64,
1101 pub throttling: Throttling,
1103}
1104
1105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1107pub struct Medians {
1108 pub now: u64,
1110 pub before: u64,
1112}
1113
1114#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1116pub struct Unhelped {
1117 pub from: u64,
1119 pub took: Option<Medians>,
1121 pub throttled: Option<Medians>,
1123}
1124
1125pub fn unhelped_raise(runs: &[CpuRun], cap: u64, now: &[Throttling]) -> Option<Unhelped> {
1133 let from = runs
1134 .iter()
1135 .filter(|run| run.cap.is_some_and(|other| other != cap))
1136 .max_by_key(|run| run.queued_at)?
1137 .cap
1138 .filter(|before| *before < cap)?;
1139 let at_before: Vec<Throttling> = runs
1140 .iter()
1141 .filter(|run| run.cap == Some(from))
1142 .map(|run| run.throttling)
1143 .collect();
1144 if (at_before.len() as u64) < RAISE_RUNS {
1145 return None;
1146 }
1147 let median = |runs: &[Throttling], figure: fn(Throttling) -> Option<u64>| {
1148 let values: Vec<u64> = runs.iter().copied().filter_map(figure).collect();
1149 (!values.is_empty()).then(|| spread(values).p50)
1150 };
1151 let medians = |figure: fn(Throttling) -> Option<u64>| {
1152 Some(Medians {
1153 now: median(now, figure)?,
1154 before: median(&at_before, figure)?,
1155 })
1156 };
1157 let took = medians(Throttling::took);
1158 let throttled = medians(Throttling::share);
1159 let helped = |medians: Option<Medians>| medians.map(|medians| medians.now < medians.before);
1160 match (helped(took), helped(throttled)) {
1161 (None, None) | (Some(true), _) | (_, Some(true)) => None,
1162 _ => Some(Unhelped {
1163 from,
1164 took,
1165 throttled,
1166 }),
1167 }
1168}
1169
1170fn unhelped(
1172 history: &History,
1173 profile: &str,
1174 job: JobKey<'_>,
1175 cap: u64,
1176 now: &JobCpu,
1177) -> Option<String> {
1178 let runs: Vec<CpuRun> = history
1179 .jobs
1180 .iter()
1181 .filter(|run| {
1182 run.profile == profile
1183 && run.succeeded
1184 && (
1185 run.repository.as_str(),
1186 run.workflow.as_deref(),
1187 run.name.as_str(),
1188 ) == job
1189 })
1190 .map(|run| CpuRun {
1191 cap: run.cpu_cap,
1192 queued_at: run.queued_at,
1193 throttling: run.throttling(),
1194 })
1195 .collect();
1196 let unhelped = unhelped_raise(&runs, cap, &now.runs)?;
1197 let mut why = format!(
1198 "the raise from {} to its {} CPUs did not help it:",
1199 unhelped.from / 1000,
1200 cap / 1000
1201 );
1202 if let Some(took) = unhelped.took {
1203 let _ = write!(
1204 why,
1205 " its runs took {} at the median, against {} before",
1206 duration(took.now / 1000),
1207 duration(took.before / 1000)
1208 );
1209 }
1210 if let Some(throttled) = unhelped.throttled {
1211 let joined = if unhelped.took.is_some() { "," } else { "" };
1212 let _ = write!(
1213 why,
1214 "{joined} its quota throttled them in {}% of their run, against {}%",
1215 throttled.now / 10,
1216 throttled.before / 10
1217 );
1218 }
1219 Some(why)
1220}
1221
1222fn held_raise(
1227 history: &History,
1228 profile: &str,
1229 jobs: &BTreeMap<JobKey<'_>, JobCpu>,
1230 cap: u64,
1231 current: u32,
1232) -> Option<Result<String, String>> {
1233 let (helped, refused): (Vec<_>, Vec<_>) = jobs
1234 .iter()
1235 .filter(|(_, read)| read.held() >= RAISE_RUNS)
1236 .map(|(job, read)| (*job, read, unhelped(history, profile, *job, cap, read)))
1237 .partition(|(_, _, why)| why.is_none());
1238 if let Some((job, read, _)) = helped.into_iter().max_by_key(|(_, read, _)| read.held()) {
1239 return Some(Ok(read.held_reason(job, current)));
1240 }
1241 let (job, read, why) = refused.into_iter().max_by_key(|(_, read, _)| read.held())?;
1242 Some(Err(format!(
1243 "{}; {}",
1244 read.held_reason(job, current),
1245 why.unwrap_or_default()
1246 )))
1247}
1248
1249fn job_named(job: JobKey<'_>) -> String {
1251 let (repository, workflow, name) = job;
1252 format!("{repository} {}", job_title(workflow, name, None))
1253}
1254
1255#[expect(
1259 clippy::too_many_arguments,
1260 reason = "each is one thing a CPU raise reads; a struct for them would be read here only"
1261)]
1262fn raised(
1263 config: &Config,
1264 name: &str,
1265 profile: &Profile,
1266 context: &Context<'_>,
1267 budget: &Budget,
1268 slots: u32,
1269 step: u32,
1270 held: String,
1271) -> CpuStep {
1272 let current = profile.cpus;
1273 let cap = u64::from(current) * 1000;
1274 let Some(slice) = context.cpus else {
1275 return CpuStep::Refused(format!("{held}; the slice's CPUs could not be read"));
1276 };
1277 let slice_cpus = u32::try_from(slice / 1000).unwrap_or(u32::MAX);
1278 let mut to = current.saturating_add(step).min(slice_cpus);
1279 let mut why = format!("it has all {slice_cpus} of the slice's CPUs");
1280 if let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
1281 {
1282 let others = budget
1283 .cpus
1284 .saturating_sub(cap.saturating_mul(u64::from(profile.slots)));
1285 let room = cpu_budget(slice, overcommit).saturating_sub(others);
1286 let fit = room / 1000 / u64::from(slots.max(1));
1287 let fit = u32::try_from(fit).unwrap_or(u32::MAX);
1288 if fit < to {
1289 to = fit;
1290 why = format!(
1291 "{} CPUs for each of its {slots} slots do not fit the CPU budget beside the other profiles'",
1292 current.saturating_add(1)
1293 );
1294 }
1295 }
1296 if let (Admission::Measured, Some(overcommit)) =
1299 (config.host.admission, config.host.cpu_overcommit)
1300 {
1301 let room = cpu_budget(slice, overcommit);
1302 while to > current && budget.at_caps(config, name, to) > room {
1303 to -= 1;
1304 why = format!(
1305 "{} CPUs do not fit the CPU budget with every profile's warm runners and one more runner at the \
1306 largest cap",
1307 current.saturating_add(1)
1308 );
1309 }
1310 }
1311 if to > current {
1312 CpuStep::To(Cpus { to, reason: held })
1313 } else {
1314 CpuStep::Refused(format!("{held}; {why}"))
1315 }
1316}
1317
1318pub(crate) fn cpu_step(config: &Config, current: u32) -> u32 {
1320 (current.saturating_mul(config.sizing.max_step) / 100).max(1)
1321}
1322
1323#[expect(
1333 clippy::too_many_arguments,
1334 reason = "each is one thing a CPU change reads; a struct for them would be read here only"
1335)]
1336fn cpus(
1337 config: &Config,
1338 profile: &Profile,
1339 advised: &ProfileSizing,
1340 history: &History,
1341 since: u64,
1342 context: &Context<'_>,
1343 budget: &Budget,
1344 slots: u32,
1345 lowering: Lowering,
1346) -> CpuStep {
1347 let current = profile.cpus;
1348 let cap = u64::from(current) * 1000;
1349 let since = since.max(cpus_since(
1350 context.cpus_set,
1351 &advised.name,
1352 current,
1353 context.window.now,
1354 ));
1355 let jobs = runs_at(history, &advised.name, cap, since);
1356 let step = cpu_step(config, current);
1357 let raise = |held| {
1358 raised(
1359 config,
1360 &advised.name,
1361 profile,
1362 context,
1363 budget,
1364 slots,
1365 step,
1366 held,
1367 )
1368 };
1369 let bound = jobs
1370 .iter()
1371 .filter(|(_, read)| read.averages.len() as u64 >= RAISE_RUNS)
1372 .map(|(job, read)| (*job, read.averages.len(), spread(read.averages.clone()).p50))
1373 .filter(|(_, _, median)| median.saturating_mul(100) >= cap.saturating_mul(BOUND))
1374 .max_by_key(|(_, _, median)| *median);
1375 if let Some((job, runs, median)) = bound {
1376 let held = format!(
1377 "{} runs of {} averaged {} of its {current} CPUs (median): the cap holds them back",
1378 runs,
1379 job_named(job),
1380 cores(median)
1381 );
1382 return raise(held);
1383 }
1384 if config.host.admission == Admission::Measured {
1386 match held_raise(history, &advised.name, &jobs, cap, current) {
1387 Some(Ok(held)) => {
1389 return match raise(held) {
1390 CpuStep::Refused(_) => CpuStep::Keep,
1391 step => step,
1392 };
1393 }
1394 Some(Err(why)) => return CpuStep::Refused(why),
1395 None => {}
1396 }
1397 }
1398 let runs: usize = jobs.values().map(|read| read.averages.len()).sum();
1399 let peak = jobs.values().map(|read| read.peak).max().unwrap_or(0);
1400 if lowering == Lowering::Held
1404 || config.host.admission == Admission::Measured
1405 || (runs as u64) < u64::from(config.sizing.min_jobs)
1406 || jobs
1407 .values()
1408 .any(|read| read.averages.is_empty() || read.held() > 0)
1409 {
1410 return CpuStep::Keep;
1411 }
1412 let Some((job, heaviest)) = jobs
1413 .iter()
1414 .map(|(job, read)| (*job, spread(read.averages.clone()).p95))
1415 .max_by_key(|(_, p95)| *p95)
1416 else {
1417 return CpuStep::Keep;
1418 };
1419 let need = heaviest.saturating_mul(HEADROOM).max(peak);
1420 let target = u32::try_from(need.div_ceil(1000))
1421 .unwrap_or(u32::MAX)
1422 .max(1);
1423 if target >= current {
1424 return CpuStep::Keep;
1425 }
1426 let to = target.max(current.saturating_sub(step)).max(1);
1427 let mut reason = format!(
1428 "{runs} runs at its {current} CPUs peaked at {} CPUs at most, none held back by the cap, and the heaviest job, {}, \
1429 averaged {} CPUs (p95); the larger of that peak and twice that average is {target}",
1430 cores(peak),
1431 job_named(job),
1432 cores(heaviest)
1433 );
1434 if to != target {
1435 let _ = write!(
1436 reason,
1437 "; a step toward {target}, at most {}% a change",
1438 config.sizing.max_step
1439 );
1440 }
1441 CpuStep::To(Cpus { to, reason })
1442}