use std::collections::BTreeMap;
use std::fmt::Write as _;
use crate::api::{
Advice, ProfileSizing, Resize, Sizing, SizingChange, SizingRefused, ago, cores, duration,
job_title,
};
use crate::config::{Admission, ByteSize, Config, Profile, cpu_budget};
use super::{
BUSY, CpusSet, GRANULE, HELD, History, Throttling, WAITED, Window, advise, busy, first_runner,
spread, waits,
};
pub(crate) const RAISE_RUNS: u64 = 3;
const BOUND: u64 = 70;
const HEADROOM: u64 = 2;
const MET: u64 = 95;
#[derive(Debug, Clone, Copy)]
pub struct Context<'run> {
pub memory_max: Option<u64>,
pub cpus: Option<u64>,
pub window: Window,
pub last: &'run BTreeMap<String, u64>,
pub cpus_set: &'run [CpusSet],
pub running: &'run BTreeMap<String, u32>,
}
pub fn changes(config: &Config, history: &History, context: Context<'_>) -> Vec<SizingChange> {
next_look(config, history, context).changes
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Look {
pub changes: Vec<SizingChange>,
pub refused: Vec<SizingRefused>,
}
pub fn next_look(config: &Config, history: &History, context: Context<'_>) -> Look {
let rules = &config.sizing;
if !rules.auto {
return Look::default();
}
let window = context.window;
let sizing = advise(config, context.memory_max, history, window);
let now = window.now / 1000;
let mut budget = Budget::new(config, &sizing.profiles);
let mut changes = Vec::new();
let mut refused = Vec::new();
for advised in &sizing.profiles {
let Some(profile) = config.profiles.get(&advised.name) else {
continue;
};
let last = context.last.get(&advised.name).copied();
let since = last.map_or(u64::MAX, |at| now.saturating_sub(at));
if advised.revert.is_some() {
match reverted(config, profile, advised, &sizing, &context, &budget) {
Revert::Write(change) => {
budget.take(&advised.name, profile, change.from, change.to);
changes.push(change);
continue;
}
Revert::Wait => continue,
Revert::Outside => {}
}
}
if since < rules.raise_after().as_secs() {
continue;
}
let lowering = if since >= rules.interval.as_secs() {
Lowering::Allowed
} else {
Lowering::Held
};
let fresh = Fresh::read(history, advised, last, window);
let mut reasons = Vec::new();
let memory = memory(config, profile, advised.advice, lowering);
let running = context.running.get(&advised.name).copied().unwrap_or(0);
let slots = match config.host.admission {
Admission::Caps => {
caps_slots(profile, advised, &fresh, memory, &context, &budget, running)
}
Admission::Measured => {
measured_slots(config, &sizing.profiles, advised, &fresh, &context, &budget)
}
};
let (slots, cpus) = cpus_and_slots(
config,
profile,
advised,
history,
fresh.start,
&context,
&budget,
slots,
lowering,
);
if memory != profile.memory.0 {
reasons.push(memory_reason(
&sizing,
advised,
memory,
slots,
config.sizing.max_step,
));
}
let advised_slots = config.host.admission == Admission::Caps && memory < profile.memory.0;
if slots > profile.slots && !advised_slots {
reasons.push(fresh.waited(last.is_some()));
}
let cpus = cpus.taken(&advised.name, &mut reasons, &mut refused);
let from = Resize {
memory: profile.memory.0,
slots: profile.slots,
cpus: cpus.map(|_| profile.cpus),
};
let to = Resize {
memory,
slots,
cpus,
};
if to == from || slots == 0 {
continue;
}
budget.take(&advised.name, profile, from, to);
changes.push(SizingChange {
at: now,
profile: advised.name.clone(),
from,
to,
reason: reasons.join("; "),
uncertain: false,
});
}
Look { changes, refused }
}
pub const UNDONE_RAISES: u64 = 24 * 3600;
pub fn undone_raises(
running: &Config,
new: &Config,
changes: &[SizingChange],
now: u64,
) -> Vec<String> {
new.profiles
.iter()
.filter_map(|(name, profile)| {
let memory = profile.memory.0;
let before = running.profiles.get(name)?.memory.0;
let raise = changes
.iter()
.filter(|change| {
change.profile == *name
&& change.to.memory > change.from.memory
&& now.saturating_sub(change.at) < UNDONE_RAISES
})
.max_by_key(|change| change.at)?;
(memory < before && memory < raise.to.memory).then(|| {
format!(
"{name}'s memory goes to {}, under the {} auto sizing raised it to {} because its jobs needed \
more than {}: they may run out of memory again",
ByteSize(memory),
ByteSize(raise.to.memory),
ago(raise.at, now),
ByteSize(raise.from.memory)
)
})
})
.collect()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Lowering {
Allowed,
Held,
}
struct Budget {
caps: u64,
cpus: u64,
slots: BTreeMap<String, u32>,
each: BTreeMap<String, u32>,
}
impl Budget {
fn take(&mut self, name: &str, profile: &Profile, from: Resize, to: Resize) {
self.caps = self
.caps
.saturating_sub(from.memory.saturating_mul(u64::from(from.slots)))
.saturating_add(to.memory.saturating_mul(u64::from(to.slots)));
let millicores = |resize: Resize| {
u64::from(resize.cpus.unwrap_or(profile.cpus)) * 1000 * u64::from(resize.slots)
};
self.cpus = self
.cpus
.saturating_sub(millicores(from))
.saturating_add(millicores(to));
self.slots.insert(name.to_owned(), to.slots);
self.each
.insert(name.to_owned(), to.cpus.unwrap_or(profile.cpus));
}
fn new(config: &Config, profiles: &[ProfileSizing]) -> Budget {
Budget {
caps: config.memory_caps().map_or(u64::MAX, |caps| caps.0),
cpus: config
.profiles
.values()
.map(|profile| u64::from(profile.cpus) * 1000 * u64::from(profile.slots))
.sum(),
slots: profiles
.iter()
.map(|advised| (advised.name.clone(), advised.slots))
.collect(),
each: config
.profiles
.iter()
.map(|(name, profile)| (name.clone(), profile.cpus))
.collect(),
}
}
fn at_caps(&self, config: &Config, name: &str, cpus: u32) -> u64 {
let each = |other: &str| {
if other == name {
cpus
} else {
self.each.get(other).copied().unwrap_or(0)
}
};
let warm: u64 = config
.profiles
.keys()
.map(|other| {
let slots = self.slots.get(other).copied().unwrap_or(0);
u64::from(warm_runners(config, other, slots)) * u64::from(each(other))
})
.sum();
let largest = config
.profiles
.keys()
.map(|other| each(other))
.max()
.unwrap_or(0);
(warm + u64::from(largest)) * 1000
}
}
fn warm_runners(config: &Config, profile: &str, slots: u32) -> u32 {
config
.repositories
.iter()
.filter(|repository| repository.profile == profile)
.map(|repository| repository.warm)
.sum::<u32>()
.min(slots)
}
struct Fresh {
start: u64,
jobs: u64,
wait: u64,
busy: u32,
}
impl Fresh {
fn read(
history: &History,
advised: &ProfileSizing,
last: Option<u64>,
window: Window,
) -> Fresh {
let start = last.map_or(window.since, |at| at.saturating_mul(1000).max(window.since));
let jobs: Vec<_> = history
.jobs
.iter()
.filter(|job| job.profile == advised.name && job.queued_at >= start)
.collect();
Fresh {
start,
jobs: jobs.len() as u64,
wait: waits(&jobs, history.kept_since).p95,
busy: busy(
history,
&advised.name,
advised.slots,
first_runner(history, start),
window.now,
),
}
}
fn wants_slots(&self) -> bool {
self.jobs >= RAISE_RUNS && (self.wait >= WAITED || self.busy >= BUSY)
}
fn waited(&self, changed: bool) -> String {
format!(
"{} jobs {}waited {}s for a slot (p95) and slots were busy {}%; one more fits the slice beside the \
other profiles'",
self.jobs,
if changed {
"since its last change "
} else {
""
},
self.wait,
self.busy
)
}
}
fn memory_reason(
sizing: &Sizing,
advised: &ProfileSizing,
memory: u64,
slots: u32,
max_step: u32,
) -> String {
let made = ProfileSizing {
advice: match advised.advice {
Advice::Lower { .. } => Advice::Lower { memory, slots },
Advice::Raise { .. } => Advice::Raise { memory, slots },
other => other,
},
..advised.clone()
};
let line = sizing.memory_line(&made);
match advised.advice.change() {
Some(advice) if advice.memory != memory => format!(
"{line}; a step toward {}, at most {max_step}% a change",
ByteSize(advice.memory)
),
_ => line,
}
}
enum Revert {
Write(SizingChange),
Wait,
Outside,
}
fn reverted(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
sizing: &Sizing,
context: &Context<'_>,
budget: &Budget,
) -> Revert {
let Advice::Raise { memory, slots } = advised.advice else {
return Revert::Outside;
};
let from = Resize {
memory: profile.memory.0,
slots: profile.slots,
cpus: None,
};
let slots = match (config.host.admission, context.memory_max) {
(Admission::Caps, Some(memory_max)) => {
let share = profile.memory.0.saturating_mul(u64::from(profile.slots));
let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
slots.min(u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX))
}
_ => slots,
};
if slots == 0 {
return Revert::Wait;
}
let running = context.running.get(&advised.name).copied().unwrap_or(0);
let to = Resize {
memory,
slots,
cpus: None,
};
if config.host.admission == Admission::Caps && running > slots {
return Revert::Wait;
}
Revert::Write(SizingChange {
at: sizing.until,
profile: advised.name.clone(),
from,
to,
reason: sizing.memory_line(advised),
uncertain: false,
})
}
fn memory(config: &Config, profile: &Profile, advice: Advice, lowering: Lowering) -> u64 {
let current = profile.memory.0;
let step = current.saturating_mul(u64::from(config.sizing.max_step)) / 100;
match advice {
Advice::Lower { memory, .. } if lowering == Lowering::Allowed => {
let floor = current
.saturating_sub(step)
.div_ceil(GRANULE)
.saturating_mul(GRANULE);
memory.max(floor).min(current)
}
Advice::Raise { memory, .. } => memory,
_ => current,
}
}
fn caps_slots(
profile: &Profile,
advised: &ProfileSizing,
fresh: &Fresh,
memory: u64,
context: &Context<'_>,
budget: &Budget,
running: u32,
) -> u32 {
let current = profile.slots;
let fit = |room: u64| u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX);
if let Advice::Lower { slots, .. } = advised.advice
&& memory < profile.memory.0
&& advised.wants_slots
{
let share = profile.memory.0.saturating_mul(u64::from(current));
return fit(share).min(slots).max(current);
}
let Some(memory_max) = context.memory_max else {
return current;
};
if memory > profile.memory.0 {
let share = profile.memory.0.saturating_mul(u64::from(current));
let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
let slots = fit(room).min(current);
return if slots < running { 0 } else { slots };
}
if fresh.wants_slots()
&& profile.max_slots.is_none_or(|max| current < max)
&& budget.caps.saturating_add(memory) <= memory_max
{
return current + 1;
}
current
}
fn measured_slots(
config: &Config,
profiles: &[ProfileSizing],
advised: &ProfileSizing,
fresh: &Fresh,
context: &Context<'_>,
budget: &Budget,
) -> u32 {
let current = advised.slots;
let max_slots = config
.profiles
.get(&advised.name)
.and_then(|profile| profile.max_slots);
if !fresh.wants_slots() || max_slots.is_some_and(|max| max <= current) {
return current;
}
let Some(memory_max) = context.memory_max else {
return if max_slots.is_some() {
current + 1
} else {
current
};
};
let peaks: u64 = profiles
.iter()
.map(|other| {
let peak = if matches!(other.advice, Advice::TooFew { .. }) {
other.memory
} else {
other.peaks.memory.p95
};
let slots = if other.name == advised.name {
current + 1
} else {
budget
.slots
.get(&other.name)
.copied()
.unwrap_or(other.slots)
};
peak.saturating_mul(u64::from(slots))
})
.fold(config.mirror_memory(), u64::saturating_add);
if peaks <= memory_max {
current + 1
} else {
current
}
}
#[expect(
clippy::too_many_arguments,
reason = "each is one thing a CPU change reads; a struct for them would be read here only"
)]
fn cpus_and_slots(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
history: &History,
since: u64,
context: &Context<'_>,
budget: &Budget,
slots: u32,
lowering: Lowering,
) -> (u32, CpuStep) {
let size = |slots: u32| {
cpus(
config, profile, advised, history, since, context, budget, slots, lowering,
)
};
let fit = |slots: u32, cpus: &CpuStep| {
let cpus = match cpus {
CpuStep::To(cpus) => cpus.to,
CpuStep::Keep | CpuStep::Refused(_) => profile.cpus,
};
if slots > profile.slots {
cpu_slots(config, profile, context, budget, cpus)
.map_or(slots, |fit| slots.min(fit.max(profile.slots)))
} else {
slots
}
};
let mut cpus = size(slots);
let fitted = fit(slots, &cpus);
if fitted < slots {
cpus = size(fitted);
}
(fit(fitted, &cpus), cpus)
}
fn cpu_slots(
config: &Config,
profile: &Profile,
context: &Context<'_>,
budget: &Budget,
cpus: u32,
) -> Option<u32> {
let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
else {
return None;
};
let Some(slice) = context.cpus else {
return Some(profile.slots);
};
let others = budget
.cpus
.saturating_sub(u64::from(profile.cpus) * 1000 * u64::from(profile.slots));
let room = cpu_budget(slice, overcommit).saturating_sub(others);
Some(u32::try_from(room / (u64::from(cpus.max(1)) * 1000)).unwrap_or(u32::MAX))
}
enum CpuStep {
Keep,
To(Cpus),
Refused(String),
}
impl CpuStep {
fn taken(
self,
profile: &str,
reasons: &mut Vec<String>,
refused: &mut Vec<SizingRefused>,
) -> Option<u32> {
match self {
CpuStep::To(cpus) => {
reasons.push(cpus.reason);
Some(cpus.to)
}
CpuStep::Refused(reason) => {
refused.push(SizingRefused {
profile: profile.to_owned(),
reason,
});
None
}
CpuStep::Keep => None,
}
}
}
struct Cpus {
to: u32,
reason: String,
}
type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
fn cpus_since(set: &[CpusSet], profile: &str, cpus: u32, now: u64) -> u64 {
let mut since = None;
let mut seen = false;
for event in set.iter().filter(|event| event.profile == profile) {
seen = true;
if event.cpus != cpus {
since = None;
} else if since.is_none() {
since = Some(event.at);
}
}
since.unwrap_or(if seen { now } else { 0 })
}
#[derive(Default)]
struct JobCpu {
averages: Vec<u64>,
peak: u64,
throttled: u64,
peaked: u64,
runs: Vec<Throttling>,
}
impl JobCpu {
fn held(&self) -> u64 {
self.throttled + self.peaked
}
fn held_reason(&self, job: JobKey<'_>, current: u32) -> String {
let how = if self.peaked == 0 {
format!("throttled in {HELD}% of their run or more")
} else {
format!(
"throttled in {HELD}% of their run or more, or peaking at the cap where that was not measured"
)
};
format!(
"{} runs of {} were held back by its {current} CPUs, {how}: under measured admission they bound no slot, \
so the cap only slows them",
self.held(),
job_named(job)
)
}
}
enum Hold {
Throttled,
Peaked,
Free,
}
impl Hold {
fn of(job: &super::Ran, cap: u64) -> Hold {
match job.throttling().held() {
Some(true) => Hold::Throttled,
Some(false) => Hold::Free,
None => Hold::peaked(job, cap),
}
}
fn peaked(job: &super::Ran, cap: u64) -> Hold {
if job.cpu_peak.saturating_mul(100) >= cap.saturating_mul(MET) {
Hold::Peaked
} else {
Hold::Free
}
}
}
fn runs_at<'run>(
history: &'run History,
profile: &str,
cap: u64,
since: u64,
) -> BTreeMap<JobKey<'run>, JobCpu> {
let mut jobs: BTreeMap<JobKey<'run>, JobCpu> = BTreeMap::new();
for job in &history.jobs {
if job.profile == profile
&& job.cpu_cap == Some(cap)
&& job.succeeded
&& job.queued_at >= since
{
let read = jobs
.entry((&job.repository, job.workflow.as_deref(), &job.name))
.or_default();
read.averages
.extend(job.cpu_average.filter(|average| *average > 0));
read.peak = read.peak.max(job.cpu_peak);
match Hold::of(job, cap) {
Hold::Throttled => read.throttled += 1,
Hold::Peaked => read.peaked += 1,
Hold::Free => {}
}
read.runs.push(job.throttling());
}
}
jobs
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CpuRun {
pub cap: Option<u64>,
pub queued_at: u64,
pub throttling: Throttling,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Medians {
pub now: u64,
pub before: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Unhelped {
pub from: u64,
pub took: Option<Medians>,
pub throttled: Option<Medians>,
}
pub fn unhelped_raise(runs: &[CpuRun], cap: u64, now: &[Throttling]) -> Option<Unhelped> {
let from = runs
.iter()
.filter(|run| run.cap.is_some_and(|other| other != cap))
.max_by_key(|run| run.queued_at)?
.cap
.filter(|before| *before < cap)?;
let at_before: Vec<Throttling> = runs
.iter()
.filter(|run| run.cap == Some(from))
.map(|run| run.throttling)
.collect();
if (at_before.len() as u64) < RAISE_RUNS {
return None;
}
let median = |runs: &[Throttling], figure: fn(Throttling) -> Option<u64>| {
let values: Vec<u64> = runs.iter().copied().filter_map(figure).collect();
(!values.is_empty()).then(|| spread(values).p50)
};
let medians = |figure: fn(Throttling) -> Option<u64>| {
Some(Medians {
now: median(now, figure)?,
before: median(&at_before, figure)?,
})
};
let took = medians(Throttling::took);
let throttled = medians(Throttling::share);
let helped = |medians: Option<Medians>| medians.map(|medians| medians.now < medians.before);
match (helped(took), helped(throttled)) {
(None, None) | (Some(true), _) | (_, Some(true)) => None,
_ => Some(Unhelped {
from,
took,
throttled,
}),
}
}
fn unhelped(
history: &History,
profile: &str,
job: JobKey<'_>,
cap: u64,
now: &JobCpu,
) -> Option<String> {
let runs: Vec<CpuRun> = history
.jobs
.iter()
.filter(|run| {
run.profile == profile
&& run.succeeded
&& (
run.repository.as_str(),
run.workflow.as_deref(),
run.name.as_str(),
) == job
})
.map(|run| CpuRun {
cap: run.cpu_cap,
queued_at: run.queued_at,
throttling: run.throttling(),
})
.collect();
let unhelped = unhelped_raise(&runs, cap, &now.runs)?;
let mut why = format!(
"the raise from {} to its {} CPUs did not help it:",
unhelped.from / 1000,
cap / 1000
);
if let Some(took) = unhelped.took {
let _ = write!(
why,
" its runs took {} at the median, against {} before",
duration(took.now / 1000),
duration(took.before / 1000)
);
}
if let Some(throttled) = unhelped.throttled {
let joined = if unhelped.took.is_some() { "," } else { "" };
let _ = write!(
why,
"{joined} its quota throttled them in {}% of their run, against {}%",
throttled.now / 10,
throttled.before / 10
);
}
Some(why)
}
fn held_raise(
history: &History,
profile: &str,
jobs: &BTreeMap<JobKey<'_>, JobCpu>,
cap: u64,
current: u32,
) -> Option<Result<String, String>> {
let (helped, refused): (Vec<_>, Vec<_>) = jobs
.iter()
.filter(|(_, read)| read.held() >= RAISE_RUNS)
.map(|(job, read)| (*job, read, unhelped(history, profile, *job, cap, read)))
.partition(|(_, _, why)| why.is_none());
if let Some((job, read, _)) = helped.into_iter().max_by_key(|(_, read, _)| read.held()) {
return Some(Ok(read.held_reason(job, current)));
}
let (job, read, why) = refused.into_iter().max_by_key(|(_, read, _)| read.held())?;
Some(Err(format!(
"{}; {}",
read.held_reason(job, current),
why.unwrap_or_default()
)))
}
fn job_named(job: JobKey<'_>) -> String {
let (repository, workflow, name) = job;
format!("{repository} {}", job_title(workflow, name, None))
}
#[expect(
clippy::too_many_arguments,
reason = "each is one thing a CPU raise reads; a struct for them would be read here only"
)]
fn raised(
config: &Config,
name: &str,
profile: &Profile,
context: &Context<'_>,
budget: &Budget,
slots: u32,
step: u32,
held: String,
) -> CpuStep {
let current = profile.cpus;
let cap = u64::from(current) * 1000;
let Some(slice) = context.cpus else {
return CpuStep::Refused(format!("{held}; the slice's CPUs could not be read"));
};
let slice_cpus = u32::try_from(slice / 1000).unwrap_or(u32::MAX);
let mut to = current.saturating_add(step).min(slice_cpus);
let mut why = format!("it has all {slice_cpus} of the slice's CPUs");
if let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
{
let others = budget
.cpus
.saturating_sub(cap.saturating_mul(u64::from(profile.slots)));
let room = cpu_budget(slice, overcommit).saturating_sub(others);
let fit = room / 1000 / u64::from(slots.max(1));
let fit = u32::try_from(fit).unwrap_or(u32::MAX);
if fit < to {
to = fit;
why = format!(
"{} CPUs for each of its {slots} slots do not fit the CPU budget beside the other profiles'",
current.saturating_add(1)
);
}
}
if let (Admission::Measured, Some(overcommit)) =
(config.host.admission, config.host.cpu_overcommit)
{
let room = cpu_budget(slice, overcommit);
while to > current && budget.at_caps(config, name, to) > room {
to -= 1;
why = format!(
"{} CPUs do not fit the CPU budget with every profile's warm runners and one more runner at the \
largest cap",
current.saturating_add(1)
);
}
}
if to > current {
CpuStep::To(Cpus { to, reason: held })
} else {
CpuStep::Refused(format!("{held}; {why}"))
}
}
pub(crate) fn cpu_step(config: &Config, current: u32) -> u32 {
(current.saturating_mul(config.sizing.max_step) / 100).max(1)
}
#[expect(
clippy::too_many_arguments,
reason = "each is one thing a CPU change reads; a struct for them would be read here only"
)]
fn cpus(
config: &Config,
profile: &Profile,
advised: &ProfileSizing,
history: &History,
since: u64,
context: &Context<'_>,
budget: &Budget,
slots: u32,
lowering: Lowering,
) -> CpuStep {
let current = profile.cpus;
let cap = u64::from(current) * 1000;
let since = since.max(cpus_since(
context.cpus_set,
&advised.name,
current,
context.window.now,
));
let jobs = runs_at(history, &advised.name, cap, since);
let step = cpu_step(config, current);
let raise = |held| {
raised(
config,
&advised.name,
profile,
context,
budget,
slots,
step,
held,
)
};
let bound = jobs
.iter()
.filter(|(_, read)| read.averages.len() as u64 >= RAISE_RUNS)
.map(|(job, read)| (*job, read.averages.len(), spread(read.averages.clone()).p50))
.filter(|(_, _, median)| median.saturating_mul(100) >= cap.saturating_mul(BOUND))
.max_by_key(|(_, _, median)| *median);
if let Some((job, runs, median)) = bound {
let held = format!(
"{} runs of {} averaged {} of its {current} CPUs (median): the cap holds them back",
runs,
job_named(job),
cores(median)
);
return raise(held);
}
if config.host.admission == Admission::Measured {
match held_raise(history, &advised.name, &jobs, cap, current) {
Some(Ok(held)) => {
return match raise(held) {
CpuStep::Refused(_) => CpuStep::Keep,
step => step,
};
}
Some(Err(why)) => return CpuStep::Refused(why),
None => {}
}
}
let runs: usize = jobs.values().map(|read| read.averages.len()).sum();
let peak = jobs.values().map(|read| read.peak).max().unwrap_or(0);
if lowering == Lowering::Held
|| config.host.admission == Admission::Measured
|| (runs as u64) < u64::from(config.sizing.min_jobs)
|| jobs
.values()
.any(|read| read.averages.is_empty() || read.held() > 0)
{
return CpuStep::Keep;
}
let Some((job, heaviest)) = jobs
.iter()
.map(|(job, read)| (*job, spread(read.averages.clone()).p95))
.max_by_key(|(_, p95)| *p95)
else {
return CpuStep::Keep;
};
let need = heaviest.saturating_mul(HEADROOM).max(peak);
let target = u32::try_from(need.div_ceil(1000))
.unwrap_or(u32::MAX)
.max(1);
if target >= current {
return CpuStep::Keep;
}
let to = target.max(current.saturating_sub(step)).max(1);
let mut reason = format!(
"{runs} runs at its {current} CPUs peaked at {} CPUs at most, none held back by the cap, and the heaviest job, {}, \
averaged {} CPUs (p95); the larger of that peak and twice that average is {target}",
cores(peak),
job_named(job),
cores(heaviest)
);
if to != target {
let _ = write!(
reason,
"; a step toward {target}, at most {}% a change",
config.sizing.max_step
);
}
CpuStep::To(Cpus { to, reason })
}