use std::collections::BTreeMap;
use std::fmt::Write as _;
use crate::api::{
Advice, ProfileSizing, Resize, SLOT_DISK, Sizing, SizingChange, SizingRefused, ago, cores,
duration, job_title,
};
use crate::cache::docker_budget_gib;
use crate::config::{Admission, ByteSize, Config, Profile, cpu_budget};
use super::{
CpusSet, GRANULE, HELD, History, SlotsSet, Throttling, Window, advise, busy, first_runner,
peak, short_of_slots, slot_asking_wait, spread, wants_slots,
};
pub(crate) const RAISE_RUNS: u64 = 3;
const BOUND: u64 = 70;
const HEADROOM: u64 = 2;
const MET: u64 = 95;
pub const IDLE_FOR: u64 = 7 * 24 * 3600 * 1000;
const IDLE_SLOTS: u32 = 2;
#[derive(Debug, Clone, Copy)]
pub struct Context<'run> {
pub memory_max: Option<u64>,
pub cpus: Option<u64>,
pub window: Window,
pub last: &'run BTreeMap<String, u64>,
pub cpus_set: &'run [CpusSet],
pub running: &'run BTreeMap<String, u32>,
pub disk: Option<u64>,
}
pub fn changes(config: &Config, history: &History, context: Context<'_>) -> Vec<SizingChange> {
next_look(config, history, context).changes
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Look {
pub changes: Vec<SizingChange>,
pub refused: Vec<SizingRefused>,
pub refused_slots: Vec<SizingRefused>,
pub waited_before_change: Vec<String>,
}
pub fn next_look(config: &Config, history: &History, context: Context<'_>) -> Look {
let rules = &config.sizing;
if !rules.auto {
return Look::default();
}
let window = context.window;
let sizing = advise(config, context.memory_max, history, window);
let now = window.now / 1000;
let mut budget = Budget::new(config, &sizing.profiles);
let mut look = Look::default();
for advised in &sizing.profiles {
let Some(profile) = config.profiles.get(&advised.name) else {
continue;
};
let last = context.last.get(&advised.name).copied();
let since = last.map_or(u64::MAX, |at| now.saturating_sub(at));
if advised.revert.is_some() {
match reverted(config, profile, advised, &sizing, &context, &budget) {
Revert::Write(change) => {
budget.take(&advised.name, profile, change.from, change.to);
look.changes.push(change);
continue;
}
Revert::Wait => continue,
Revert::Outside => {}
}
}
if since < rules.raise_after().as_secs() {
continue;
}
let lowering = Lowering::after(since, rules.interval.as_secs());
let fresh = Fresh::read(history, advised, config.host.admission, last, window);
let mut reasons = Vec::new();
let memory = memory(config, profile, advised.advice, lowering);
let running = context.running.get(&advised.name).copied().unwrap_or(0);
let (slots, slots_reason) = match config.host.admission {
Admission::Caps => (
caps_slots(profile, advised, &fresh, memory, &context, &budget, running),
None,
),
Admission::Measured => {
measured_slots(config, profile, advised, &fresh, &context, &budget)
.step(&advised.name, &mut look.refused_slots, || {
(lowering == Lowering::Allowed)
.then(|| idle_slots(config, history, advised, &context, running))?
})
.map_or((advised.slots, None), |step| (step.to, Some(step.reason)))
}
};
let (slots, cpus) = cpus_and_slots(
config,
profile,
advised,
history,
fresh.start,
&context,
&budget,
slots,
lowering,
);
if memory != profile.memory.0 {
reasons.push(memory_reason(
&sizing,
advised,
memory,
slots,
config.sizing.max_step,
));
}
let advised_slots = config.host.admission == Admission::Caps && memory < profile.memory.0;
if let Some(reason) = slots_reason {
reasons.push(reason);
} else if slots > profile.slots && !advised_slots {
reasons.push(fresh.waited(last.is_some()));
}
let cpus = cpus.taken(&advised.name, &mut reasons, &mut look.refused);
let from = Resize {
memory: profile.memory.0,
slots: profile.slots,
cpus: cpus.map(|_| profile.cpus),
};
let to = Resize {
memory,
slots,
cpus,
};
if to == from || slots == 0 {
continue;
}
budget.take(&advised.name, profile, from, to);
look.changes.push(SizingChange {
at: now,
profile: advised.name.clone(),
from,
to,
reason: reasons.join("; "),
uncertain: false,
});
}
look.waited_before_change = waited_before_change(config, history, &sizing, &context);
look
}
fn waited_before_change(
config: &Config,
history: &History,
sizing: &Sizing,
context: &Context<'_>,
) -> Vec<String> {
if config.host.admission != Admission::Measured {
return Vec::new();
}
sizing
.profiles
.iter()
.filter(|advised| advised.wants_slots)
.filter(|advised| {
let Some(last) = context.last.get(&advised.name).copied() else {
return false;
};
!Fresh::read(
history,
advised,
config.host.admission,
Some(last),
context.window,
)
.wants_slots()
})
.map(|advised| advised.name.clone())
.collect()
}
pub const UNDONE_RAISES: u64 = 24 * 3600;
pub fn undone_raises(
running: &Config,
new: &Config,
changes: &[SizingChange],
now: u64,
) -> Vec<String> {
new.profiles
.iter()
.filter_map(|(name, profile)| {
let memory = profile.memory.0;
let before = running.profiles.get(name)?.memory.0;
let raise = changes
.iter()
.filter(|change| {
change.profile == *name
&& change.to.memory > change.from.memory
&& now.saturating_sub(change.at) < UNDONE_RAISES
})
.max_by_key(|change| change.at)?;
(memory < before && memory < raise.to.memory).then(|| {
format!(
"{name}'s memory goes to {}, under the {} auto sizing raised it to {} because its jobs needed \
more than {}: they may run out of memory again",
ByteSize(memory),
ByteSize(raise.to.memory),
ago(raise.at, now),
ByteSize(raise.from.memory)
)
})
})
.collect()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Lowering {
Allowed,
Held,
}
impl Lowering {
fn after(since: u64, interval: u64) -> Lowering {
if since >= interval {
Lowering::Allowed
} else {
Lowering::Held
}
}
}
struct Budget {
caps: u64,
cpus: u64,
slots: BTreeMap<String, u32>,
each: BTreeMap<String, u32>,
}
impl Budget {
fn take(&mut self, name: &str, profile: &Profile, from: Resize, to: Resize) {
self.caps = self
.caps
.saturating_sub(from.memory.saturating_mul(u64::from(from.slots)))
.saturating_add(to.memory.saturating_mul(u64::from(to.slots)));
let millicores = |resize: Resize| {
u64::from(resize.cpus.unwrap_or(profile.cpus)) * 1000 * u64::from(resize.slots)
};
self.cpus = self
.cpus
.saturating_sub(millicores(from))
.saturating_add(millicores(to));
self.slots.insert(name.to_owned(), to.slots);
self.each
.insert(name.to_owned(), to.cpus.unwrap_or(profile.cpus));
}
fn new(config: &Config, profiles: &[ProfileSizing]) -> Budget {
Budget {
caps: config.memory_caps().map_or(u64::MAX, |caps| caps.0),
cpus: config
.profiles
.values()
.map(|profile| u64::from(profile.cpus) * 1000 * u64::from(profile.slots))
.sum(),
slots: profiles
.iter()
.map(|advised| (advised.name.clone(), advised.slots))
.collect(),
each: config
.profiles
.iter()
.map(|(name, profile)| (name.clone(), profile.cpus))
.collect(),
}
}
fn at_caps(&self, config: &Config, name: &str, cpus: u32) -> u64 {
let each = |other: &str| {
if other == name {
cpus
} else {
self.each.get(other).copied().unwrap_or(0)
}
};
let warm: u64 = config
.profiles
.keys()
.map(|other| {
let slots = self.slots.get(other).copied().unwrap_or(0);
u64::from(warm_runners(config, other, slots)) * u64::from(each(other))
})
.sum();
let largest = config
.profiles
.keys()
.map(|other| each(other))
.max()
.unwrap_or(0);
(warm + u64::from(largest)) * 1000
}
}
fn warm_runners(config: &Config, profile: &str, slots: u32) -> u32 {
config
.repositories
.iter()
.filter(|repository| repository.profile == profile)
.map(|repository| repository.warm)
.sum::<u32>()
.min(slots)
}
struct Fresh {
start: u64,
jobs: u64,
wants: bool,
wait: u64,
short: u32,
busy: u32,
}
impl Fresh {
fn read(
history: &History,
advised: &ProfileSizing,
admission: Admission,
last: Option<u64>,
window: Window,
) -> Fresh {
let start = last.map_or(window.since, |at| at.saturating_mul(1000).max(window.since));
let jobs: Vec<_> = history
.jobs
.iter()
.filter(|job| job.profile == advised.name && job.queued_at >= start)
.collect();
let busy = busy(
history,
&advised.name,
advised.slots,
first_runner(history, start),
window.now,
);
Fresh {
start,
jobs: jobs.len() as u64,
wants: wants_slots(admission, &jobs, history.kept_since, busy),
wait: slot_asking_wait(admission, &jobs, history.kept_since),
short: short_of_slots(&jobs),
busy,
}
}
fn wants_slots(&self) -> bool {
self.jobs >= RAISE_RUNS && self.wants
}
fn waited(&self, changed: bool) -> String {
format!(
"{} jobs {}waited {}s for a slot (p95) and slots were busy {}%; one more fits the slice beside the \
other profiles'",
self.jobs,
if changed {
"since its last change "
} else {
""
},
self.wait,
self.busy
)
}
}
fn memory_reason(
sizing: &Sizing,
advised: &ProfileSizing,
memory: u64,
slots: u32,
max_step: u32,
) -> String {
let made = ProfileSizing {
advice: match advised.advice {
Advice::Lower { .. } => Advice::Lower { memory, slots },
Advice::Raise { .. } => Advice::Raise { memory, slots },
other => other,
},
..advised.clone()
};
let line = sizing.memory_line(&made);
match advised.advice.change() {
Some(advice) if advice.memory != memory => format!(
"{line}; a step toward {}, at most {max_step}% a change",
ByteSize(advice.memory)
),
_ => line,
}
}
enum Revert {
Write(SizingChange),
Wait,
Outside,
}
fn reverted(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
sizing: &Sizing,
context: &Context<'_>,
budget: &Budget,
) -> Revert {
let Advice::Raise { memory, slots } = advised.advice else {
return Revert::Outside;
};
let from = Resize {
memory: profile.memory.0,
slots: profile.slots,
cpus: None,
};
let slots = match (config.host.admission, context.memory_max) {
(Admission::Caps, Some(memory_max)) => {
let share = profile.memory.0.saturating_mul(u64::from(profile.slots));
let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
slots.min(u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX))
}
_ => slots,
};
if slots == 0 {
return Revert::Wait;
}
let running = context.running.get(&advised.name).copied().unwrap_or(0);
let to = Resize {
memory,
slots,
cpus: None,
};
if config.host.admission == Admission::Caps && running > slots {
return Revert::Wait;
}
Revert::Write(SizingChange {
at: sizing.until,
profile: advised.name.clone(),
from,
to,
reason: sizing.memory_line(advised),
uncertain: false,
})
}
fn memory(config: &Config, profile: &Profile, advice: Advice, lowering: Lowering) -> u64 {
let current = profile.memory.0;
let step = current.saturating_mul(u64::from(config.sizing.max_step)) / 100;
match advice {
Advice::Lower { memory, .. } if lowering == Lowering::Allowed => {
let floor = current
.saturating_sub(step)
.div_ceil(GRANULE)
.saturating_mul(GRANULE);
memory.max(floor).min(current)
}
Advice::Raise { memory, .. } => memory,
_ => current,
}
}
fn caps_slots(
profile: &Profile,
advised: &ProfileSizing,
fresh: &Fresh,
memory: u64,
context: &Context<'_>,
budget: &Budget,
running: u32,
) -> u32 {
let current = profile.slots;
let fit = |room: u64| u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX);
if let Advice::Lower { slots, .. } = advised.advice
&& memory < profile.memory.0
&& advised.wants_slots
{
let share = profile.memory.0.saturating_mul(u64::from(current));
return fit(share).min(slots).max(current);
}
let Some(memory_max) = context.memory_max else {
return current;
};
if memory > profile.memory.0 {
let share = profile.memory.0.saturating_mul(u64::from(current));
let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
let slots = fit(room).min(current);
return if slots < running { 0 } else { slots };
}
if fresh.wants_slots()
&& profile.max_slots.is_none_or(|max| current < max)
&& budget.caps.saturating_add(memory) <= memory_max
{
return current + 1;
}
current
}
struct SlotStep {
to: u32,
reason: String,
}
enum Raise {
Unasked,
To(SlotStep),
Refused(String),
}
impl Raise {
fn step(
self,
profile: &str,
refused: &mut Vec<SizingRefused>,
idle: impl FnOnce() -> Option<SlotStep>,
) -> Option<SlotStep> {
match self {
Raise::To(step) => Some(step),
Raise::Refused(reason) => {
refused.push(SizingRefused::new(profile, reason));
None
}
Raise::Unasked => idle(),
}
}
}
fn measured_slots(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
fresh: &Fresh,
context: &Context<'_>,
budget: &Budget,
) -> Raise {
let current = advised.slots;
if !fresh.wants_slots() {
return Raise::Unasked;
}
let asked = format!(
"{} jobs {}waited {}s for a slot (p95) while memory and CPU had room, {} at once at most",
fresh.jobs,
if context.last.contains_key(&advised.name) {
"since its last change "
} else {
""
},
fresh.wait,
fresh.short,
);
let max = profile.max_slots.unwrap_or(u32::MAX);
if current >= max {
return Raise::Refused(format!("{asked}; it is at its max_slots, {max}"));
}
let Some(disk) = context.disk else {
return Raise::Refused(format!(
"{asked}; Docker's disk is not measured yet, which the next check of the caches does"
));
};
let step = (current.saturating_mul(config.sizing.max_step) / 100).max(1);
let wanted = current.saturating_add(fresh.short.clamp(1, step)).min(max);
let others: u32 = budget
.slots
.iter()
.filter(|(name, _)| **name != advised.name)
.map(|(_, slots)| *slots)
.sum();
let Some(to) = (current.saturating_add(1)..=wanted)
.rev()
.find(|slots| share(disk, others.saturating_add(*slots)) >= SLOT_DISK)
else {
let next = current.saturating_add(1);
return Raise::Refused(format!(
"{asked}; at {next} slots, beside the other profiles' {others}, each slot would keep {}, under the {} \
auto sizing leaves each slot",
kept(
disk,
others.saturating_add(current),
others.saturating_add(next)
),
ByteSize(SLOT_DISK),
));
};
Raise::To(SlotStep {
to,
reason: format!(
"{asked}; at {to} slots each slot keeps {}",
kept(
disk,
others.saturating_add(current),
others.saturating_add(to)
)
),
})
}
fn share(disk: u64, slots: u32) -> u64 {
docker_budget_gib(disk, u64::from(slots)) << 30
}
fn kept(disk: u64, from: u32, to: u32) -> String {
let (before, after) = (share(disk, from), share(disk, to));
if after < before {
format!(
"{} of Docker's disk, down from {}",
ByteSize(after),
ByteSize(before)
)
} else {
format!("{} of Docker's disk", ByteSize(after))
}
}
fn idle_slots(
config: &Config,
history: &History,
advised: &ProfileSizing,
context: &Context<'_>,
running: u32,
) -> Option<SlotStep> {
let current = advised.slots;
let now = context.window.now;
let start = now.checked_sub(IDLE_FOR)?;
let set = slots_since(&history.slots_set, &advised.name, current, now);
let changed = history
.changes
.iter()
.filter(|change| change.profile == advised.name && change.to.slots != change.from.slots)
.map(|change| change.at.saturating_mul(1000))
.max()
.unwrap_or(0);
let first = history
.runners
.iter()
.map(|runner| runner.started_at)
.min()?;
if set.max(changed).max(first) > start {
return None;
}
let jobs = history
.jobs
.iter()
.filter(|job| job.profile == advised.name && job.queued_at >= start)
.count() as u64;
if jobs < u64::from(config.sizing.min_jobs) {
return None;
}
let mut edges: Vec<(u64, i32)> = history
.runners
.iter()
.filter(|runner| {
runner.profile == advised.name && runner.removed_at.is_none_or(|at| at > start)
})
.flat_map(|runner| {
[
(runner.started_at.max(start), 1),
(runner.removed_at.unwrap_or(now).min(now), -1),
]
})
.collect();
let ran = peak(&mut edges);
let warm = warm_runners(config, &advised.name, current);
if ran.saturating_add(warm).saturating_add(IDLE_SLOTS) > current {
return None;
}
let to = current.saturating_sub(1).max(running).max(1);
(to < current).then(|| SlotStep {
to,
reason: format!(
"at most {ran} of its {current} slots held a runner for a job at once in the last {}{}: an idle slot \
shrinks every slot's share of Docker's disk",
duration(IDLE_FOR / 1000),
if warm > 0 {
format!(", beside {warm} warm runners")
} else {
String::new()
}
),
})
}
fn slots_since(set: &[SlotsSet], profile: &str, slots: u32, now: u64) -> u64 {
since_set(
set.iter()
.filter(|event| event.profile == profile)
.map(|event| (event.slots, event.at)),
slots,
now,
)
}
#[expect(
clippy::too_many_arguments,
reason = "each is one thing a CPU change reads; a struct for them would be read here only"
)]
fn cpus_and_slots(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
history: &History,
since: u64,
context: &Context<'_>,
budget: &Budget,
slots: u32,
lowering: Lowering,
) -> (u32, CpuStep) {
let size = |slots: u32| {
cpus(
config, profile, advised, history, since, context, budget, slots, lowering,
)
};
let fit = |slots: u32, cpus: &CpuStep| {
let cpus = match cpus {
CpuStep::To(cpus) => cpus.to,
CpuStep::Keep | CpuStep::Refused(_) => profile.cpus,
};
if slots > profile.slots {
cpu_slots(config, profile, context, budget, cpus)
.map_or(slots, |fit| slots.min(fit.max(profile.slots)))
} else {
slots
}
};
let mut cpus = size(slots);
let fitted = fit(slots, &cpus);
if fitted < slots {
cpus = size(fitted);
}
(fit(fitted, &cpus), cpus)
}
fn cpu_slots(
config: &Config,
profile: &Profile,
context: &Context<'_>,
budget: &Budget,
cpus: u32,
) -> Option<u32> {
let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
else {
return None;
};
let Some(slice) = context.cpus else {
return Some(profile.slots);
};
let others = budget
.cpus
.saturating_sub(u64::from(profile.cpus) * 1000 * u64::from(profile.slots));
let room = cpu_budget(slice, overcommit).saturating_sub(others);
Some(u32::try_from(room / (u64::from(cpus.max(1)) * 1000)).unwrap_or(u32::MAX))
}
enum CpuStep {
Keep,
To(Cpus),
Refused(String),
}
impl CpuStep {
fn taken(
self,
profile: &str,
reasons: &mut Vec<String>,
refused: &mut Vec<SizingRefused>,
) -> Option<u32> {
match self {
CpuStep::To(cpus) => {
reasons.push(cpus.reason);
Some(cpus.to)
}
CpuStep::Refused(reason) => {
refused.push(SizingRefused {
profile: profile.to_owned(),
reason,
});
None
}
CpuStep::Keep => None,
}
}
}
struct Cpus {
to: u32,
reason: String,
}
type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
fn cpus_since(set: &[CpusSet], profile: &str, cpus: u32, now: u64) -> u64 {
since_set(
set.iter()
.filter(|event| event.profile == profile)
.map(|event| (event.cpus, event.at)),
cpus,
now,
)
}
fn since_set(events: impl Iterator<Item = (u32, u64)>, value: u32, now: u64) -> u64 {
let mut since = None;
let mut seen = false;
for (set, at) in events {
seen = true;
if set != value {
since = None;
} else if since.is_none() {
since = Some(at);
}
}
since.unwrap_or(if seen { now } else { 0 })
}
#[derive(Default)]
struct JobCpu {
averages: Vec<u64>,
peak: u64,
throttled: u64,
peaked: u64,
runs: Vec<Throttling>,
}
impl JobCpu {
fn held(&self) -> u64 {
self.throttled + self.peaked
}
fn held_reason(&self, job: JobKey<'_>, current: u32) -> String {
let how = if self.peaked == 0 {
format!("throttled in {HELD}% of their run or more")
} else {
format!(
"throttled in {HELD}% of their run or more, or peaking at the cap where that was not measured"
)
};
format!(
"{} runs of {} were held back by its {current} CPUs, {how}: under measured admission they bound no slot, \
so the cap only slows them",
self.held(),
job_named(job)
)
}
}
enum Hold {
Throttled,
Peaked,
Free,
}
impl Hold {
fn of(job: &super::Ran, cap: u64) -> Hold {
match job.throttling().held() {
Some(true) => Hold::Throttled,
Some(false) => Hold::Free,
None => Hold::peaked(job, cap),
}
}
fn peaked(job: &super::Ran, cap: u64) -> Hold {
if job.cpu_peak.saturating_mul(100) >= cap.saturating_mul(MET) {
Hold::Peaked
} else {
Hold::Free
}
}
}
fn runs_at<'run>(
history: &'run History,
profile: &str,
cap: u64,
since: u64,
) -> BTreeMap<JobKey<'run>, JobCpu> {
let mut jobs: BTreeMap<JobKey<'run>, JobCpu> = BTreeMap::new();
for job in &history.jobs {
if job.profile == profile
&& job.cpu_cap == Some(cap)
&& job.succeeded
&& job.queued_at >= since
{
let read = jobs
.entry((&job.repository, job.workflow.as_deref(), &job.name))
.or_default();
read.averages
.extend(job.cpu_average.filter(|average| *average > 0));
read.peak = read.peak.max(job.cpu_peak);
match Hold::of(job, cap) {
Hold::Throttled => read.throttled += 1,
Hold::Peaked => read.peaked += 1,
Hold::Free => {}
}
read.runs.push(job.throttling());
}
}
jobs
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CpuRun {
pub cap: Option<u64>,
pub queued_at: u64,
pub throttling: Throttling,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Medians {
pub now: u64,
pub before: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Unhelped {
pub from: u64,
pub took: Option<Medians>,
pub throttled: Option<Medians>,
}
pub fn unhelped_raise(runs: &[CpuRun], cap: u64, now: &[Throttling]) -> Option<Unhelped> {
let from = runs
.iter()
.filter(|run| run.cap.is_some_and(|other| other != cap))
.max_by_key(|run| run.queued_at)?
.cap
.filter(|before| *before < cap)?;
let at_before: Vec<Throttling> = runs
.iter()
.filter(|run| run.cap == Some(from))
.map(|run| run.throttling)
.collect();
if (at_before.len() as u64) < RAISE_RUNS {
return None;
}
let median = |runs: &[Throttling], figure: fn(Throttling) -> Option<u64>| {
let values: Vec<u64> = runs.iter().copied().filter_map(figure).collect();
(!values.is_empty()).then(|| spread(values).p50)
};
let medians = |figure: fn(Throttling) -> Option<u64>| {
Some(Medians {
now: median(now, figure)?,
before: median(&at_before, figure)?,
})
};
let took = medians(Throttling::took);
let throttled = medians(Throttling::share);
let helped = |medians: Option<Medians>| medians.map(|medians| medians.now < medians.before);
match (helped(took), helped(throttled)) {
(None, None) | (Some(true), _) | (_, Some(true)) => None,
_ => Some(Unhelped {
from,
took,
throttled,
}),
}
}
fn unhelped(
history: &History,
profile: &str,
job: JobKey<'_>,
cap: u64,
now: &JobCpu,
) -> Option<String> {
let runs: Vec<CpuRun> = history
.jobs
.iter()
.filter(|run| {
run.profile == profile
&& run.succeeded
&& (
run.repository.as_str(),
run.workflow.as_deref(),
run.name.as_str(),
) == job
})
.map(|run| CpuRun {
cap: run.cpu_cap,
queued_at: run.queued_at,
throttling: run.throttling(),
})
.collect();
let unhelped = unhelped_raise(&runs, cap, &now.runs)?;
let mut why = format!(
"the raise from {} to its {} CPUs did not help it:",
unhelped.from / 1000,
cap / 1000
);
if let Some(took) = unhelped.took {
let _ = write!(
why,
" its runs took {} at the median, against {} before",
duration(took.now / 1000),
duration(took.before / 1000)
);
}
if let Some(throttled) = unhelped.throttled {
let joined = if unhelped.took.is_some() { "," } else { "" };
let _ = write!(
why,
"{joined} its quota throttled them in {}% of their run, against {}%",
throttled.now / 10,
throttled.before / 10
);
}
Some(why)
}
fn held_raise(
history: &History,
profile: &str,
jobs: &BTreeMap<JobKey<'_>, JobCpu>,
cap: u64,
current: u32,
) -> Option<Result<String, String>> {
let (helped, refused): (Vec<_>, Vec<_>) = jobs
.iter()
.filter(|(_, read)| read.held() >= RAISE_RUNS)
.map(|(job, read)| (*job, read, unhelped(history, profile, *job, cap, read)))
.partition(|(_, _, why)| why.is_none());
if let Some((job, read, _)) = helped.into_iter().max_by_key(|(_, read, _)| read.held()) {
return Some(Ok(read.held_reason(job, current)));
}
let (job, read, why) = refused.into_iter().max_by_key(|(_, read, _)| read.held())?;
Some(Err(format!(
"{}; {}",
read.held_reason(job, current),
why.unwrap_or_default()
)))
}
fn job_named(job: JobKey<'_>) -> String {
let (repository, workflow, name) = job;
format!("{repository} {}", job_title(workflow, name, None))
}
#[expect(
clippy::too_many_arguments,
reason = "each is one thing a CPU raise reads; a struct for them would be read here only"
)]
fn raised(
config: &Config,
name: &str,
profile: &Profile,
context: &Context<'_>,
budget: &Budget,
slots: u32,
step: u32,
held: String,
) -> CpuStep {
let current = profile.cpus;
let cap = u64::from(current) * 1000;
let Some(slice) = context.cpus else {
return CpuStep::Refused(format!("{held}; the slice's CPUs could not be read"));
};
let slice_cpus = u32::try_from(slice / 1000).unwrap_or(u32::MAX);
let mut to = current.saturating_add(step).min(slice_cpus);
let mut why = format!("it has all {slice_cpus} of the slice's CPUs");
if let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
{
let others = budget
.cpus
.saturating_sub(cap.saturating_mul(u64::from(profile.slots)));
let room = cpu_budget(slice, overcommit).saturating_sub(others);
let fit = room / 1000 / u64::from(slots.max(1));
let fit = u32::try_from(fit).unwrap_or(u32::MAX);
if fit < to {
to = fit;
why = format!(
"{} CPUs for each of its {slots} slots do not fit the CPU budget beside the other profiles'",
current.saturating_add(1)
);
}
}
if let (Admission::Measured, Some(overcommit)) =
(config.host.admission, config.host.cpu_overcommit)
{
let room = cpu_budget(slice, overcommit);
while to > current && budget.at_caps(config, name, to) > room {
to -= 1;
why = format!(
"{} CPUs do not fit the CPU budget with every profile's warm runners and one more runner at the \
largest cap",
current.saturating_add(1)
);
}
}
if to > current {
CpuStep::To(Cpus { to, reason: held })
} else {
CpuStep::Refused(format!("{held}; {why}"))
}
}
pub(crate) fn cpu_step(config: &Config, current: u32) -> u32 {
(current.saturating_mul(config.sizing.max_step) / 100).max(1)
}
#[expect(
clippy::too_many_arguments,
reason = "each is one thing a CPU change reads; a struct for them would be read here only"
)]
fn cpus(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
history: &History,
since: u64,
context: &Context<'_>,
budget: &Budget,
slots: u32,
lowering: Lowering,
) -> CpuStep {
let current = profile.cpus;
let cap = u64::from(current) * 1000;
let since = since.max(cpus_since(
context.cpus_set,
&advised.name,
current,
context.window.now,
));
let jobs = runs_at(history, &advised.name, cap, since);
let step = cpu_step(config, current);
let raise = |held| {
raised(
config,
&advised.name,
profile,
context,
budget,
slots,
step,
held,
)
};
let bound = jobs
.iter()
.filter(|(_, read)| read.averages.len() as u64 >= RAISE_RUNS)
.map(|(job, read)| (*job, read.averages.len(), spread(read.averages.clone()).p50))
.filter(|(_, _, median)| median.saturating_mul(100) >= cap.saturating_mul(BOUND))
.max_by_key(|(_, _, median)| *median);
if let Some((job, runs, median)) = bound {
let held = format!(
"{} runs of {} averaged {} of its {current} CPUs (median): the cap holds them back",
runs,
job_named(job),
cores(median)
);
return raise(held);
}
if config.host.admission == Admission::Measured {
match held_raise(history, &advised.name, &jobs, cap, current) {
Some(Ok(held)) => {
return match raise(held) {
CpuStep::Refused(_) => CpuStep::Keep,
step => step,
};
}
Some(Err(why)) => return CpuStep::Refused(why),
None => {}
}
}
let runs: usize = jobs.values().map(|read| read.averages.len()).sum();
let peak = jobs.values().map(|read| read.peak).max().unwrap_or(0);
if lowering == Lowering::Held
|| config.host.admission == Admission::Measured
|| (runs as u64) < u64::from(config.sizing.min_jobs)
|| jobs
.values()
.any(|read| read.averages.is_empty() || read.held() > 0)
{
return CpuStep::Keep;
}
let Some((job, heaviest)) = jobs
.iter()
.map(|(job, read)| (*job, spread(read.averages.clone()).p95))
.max_by_key(|(_, p95)| *p95)
else {
return CpuStep::Keep;
};
let need = heaviest.saturating_mul(HEADROOM).max(peak);
let target = u32::try_from(need.div_ceil(1000))
.unwrap_or(u32::MAX)
.max(1);
if target >= current {
return CpuStep::Keep;
}
let to = target.max(current.saturating_sub(step)).max(1);
let mut reason = format!(
"{runs} runs at its {current} CPUs peaked at {} CPUs at most, none held back by the cap, and the heaviest job, {}, \
averaged {} CPUs (p95); the larger of that peak and twice that average is {target}",
cores(peak),
job_named(job),
cores(heaviest)
);
if to != target {
let _ = write!(
reason,
"; a step toward {target}, at most {}% a change",
config.sizing.max_step
);
}
CpuStep::To(Cpus { to, reason })
}