Skip to main content

runnerd_core/sizing/
auto.rs

1//! Auto sizing: which profiles' `memory`, `slots` and `cpus` the daemon changes now, from the advice, the runs since
2//! each profile's last change and the limits `[sizing]` and each profile set. Pure: the daemon reads the history and
3//! when each profile last changed, writes each change through the config check as `PUT /v1/config` does, and records
4//! it. Never lowers a profile while a job is on its cap, never changes `memory` from fewer than `[sizing] min_jobs`
5//! jobs, never lowers within `[sizing] interval` of a profile's last change or raises within `raise_interval`, never
6//! leaves a profile outside its bounds, and sizes each profile against the budget the changes before it in the same
7//! look leave. Reads `cpus` only from runs at the profile's current `cpus` since its last change and since the daemon
8//! got them. Under `caps` admission raises them only on those runs' CPU averages, never on peaks: a parallel build
9//! peaks at whatever cap it gets, and there `cpus` take budget. Under `measured`, where a running job reserves its
10//! average, runs the cap held back raise them too, within what the CPU budget holds at caps, and nothing lowers them.
11//! Under `caps` lowers them only when the cap held none of those runs back, never under their highest peak. A run
12//! the cap held back is one its quota throttled in 15% of its run or more, or, without that figure, one whose bursts
13//! met the cap (ADRs 0075 and 0079); under `measured` a raise goes on only while the one before made the job's runs
14//! faster or less throttled.
15
16use std::collections::BTreeMap;
17use std::fmt::Write as _;
18
19use crate::api::{
20    Advice, ProfileSizing, Resize, SLOT_DISK, Sizing, SizingChange, SizingRefused, ago, cores,
21    duration, job_title,
22};
23use crate::cache::docker_budget_gib;
24use crate::config::{Admission, ByteSize, Config, Profile, cpu_budget};
25
26use super::{
27    CpusSet, GRANULE, HELD, History, SlotsSet, Throttling, Window, advise, busy, first_runner,
28    peak, short_of_slots, slot_asking_wait, spread, wants_slots,
29};
30
31/// The fewest runs a raise of `slots` or `cpus` reads: the jobs queued since the profile's last change, or one job's
32/// runs at its current `cpus`. It does not wait for `min_jobs`: a slot too few costs every queued job a wait now.
33pub(crate) const RAISE_RUNS: u64 = 3;
34
35/// A job whose runs averaged this share of their CPU cap or more, in percent, at the median, is held back by it over
36/// the whole run, not in a burst: its checkout, cache restore and upload steps barely use a CPU, so the rest of it
37/// ran at the cap. On the owner's history no job's median came past 37%, whatever its cap: their builds are bursts.
38const BOUND: u64 = 70;
39
40/// `cpus` come down toward this many times the heaviest job's p95 CPU average, and never under the highest peak of
41/// the runs read: a job back at the cap would average half of it, far under `BOUND`, so a lowering never asks for
42/// the raise back.
43const HEADROOM: u64 = 2;
44
45/// A run without a throttling figure, as one from before schema 17, whose CPU peak came this close to its cap, in
46/// percent, met it in a burst, and counts as held back (ADR 0075). Peaks are the highest 2-second interval, so one
47/// can pass the cap a little.
48const MET: u64 = 95;
49
50/// How long a profile's slots must stay `IDLE_SLOTS` above what ran at once before auto sizing takes one away, in ms:
51/// a week, so a quiet weekend does not take back what the weekdays' bursts need.
52pub const IDLE_FOR: u64 = 7 * 24 * 3600 * 1000;
53
54/// A profile keeps this many slots more than ran at once, warm runners counted, before one is taken away: with one, a
55/// burst one slot past the week's peak would add back the slot just taken.
56const IDLE_SLOTS: u32 = 2;
57
58/// What auto sizing reads besides the config and the history.
59#[derive(Debug, Clone, Copy)]
60pub struct Context<'run> {
61    /// The slice's `MemoryMax`; `None` when it is unlimited.
62    pub memory_max: Option<u64>,
63    /// The slice's CPUs, in millicores; `None` when they could not be read, and then `cpus` are never raised.
64    pub cpus: Option<u64>,
65    /// The history's window; `now` is when the changes are made.
66    pub window: Window,
67    /// When auto sizing last changed each profile, in seconds, by name, however long ago.
68    pub last: &'run BTreeMap<String, u64>,
69    /// Each profile's `cpus` as the daemon ran them, oldest first: what a `cpus` change reads runs since, as a
70    /// hand edit, a reload or an undo leaves no change of auto sizing's.
71    pub cpus_set: &'run [CpusSet],
72    /// How many runners each profile runs now, by name.
73    pub running: &'run BTreeMap<String, u32>,
74    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
75    /// `None` before the first, and then no slot is added (ADR 0090).
76    pub disk: Option<u64>,
77}
78
79/// The changes to make now, in order, one per profile at most. Empty unless `[sizing] auto` is on.
80///
81/// A change that lowers `memory` or `cpus` waits `[sizing] interval` after the profile's last change; one that only
82/// raises waits `raise_interval`. A lowering moves `memory` toward the advice, and `cpus` move toward what their runs
83/// reach, by at most `[sizing] max_step` percent; a raise of `memory` goes straight to the advice, which only a run
84/// that met the cap gives. `memory` stays within the profile's `min_memory` and `max_memory`, `cpus` within the
85/// slice's CPUs. A lowering that went too far is taken back at once (`ProfileSizing::revert`). Under `caps`
86/// admission a profile whose jobs waited for a slot, or whose slots were busy, since its last change gets one more
87/// slot when every profile's caps still fit `MemoryMax`, and a lowered `memory` adds the slots the advice gives. Under
88/// `measured` a profile whose jobs waited for a slot while memory and CPU had room for them (`Ran::slot_waited`)
89/// since its last change gets as many more as waited at once, at most `max_step` percent of its slots, at least 1,
90/// while every slot keeps `SLOT_DISK` of Docker's disk (`measured_slots`), or a refusal says why it cannot; and one
91/// whose slots stayed `IDLE_SLOTS` above what ran at once for `IDLE_FOR` since they were last set loses one. Never past `max_slots`, never under what
92/// runs. Each change is sized as if the ones before it were written: they take budget the later ones cannot.
93///
94/// `cpus` go up `max_step` percent, at least 1, when one job's runs at them since the profile's last change and since
95/// the daemon got them (`Context::cpus_set`), 3 at least, averaged `BOUND` percent of them at the median, or under
96/// `measured` admission were held back by them (throttled, or without that figure met in a burst) while the raise
97/// before helped them (`unhelped`); up to the slice's CPUs, and with a CPU budget only as far as it holds: every
98/// profile's `slots × cpus` under `caps`; under `measured`, what that admission reserves at caps, every profile's
99/// warm runners and one more runner at the largest cap (`Budget::at_caps`). Under `caps` admission they come down, `interval` after the last change and by `max_step`
100/// percent at most, toward `HEADROOM` times the heaviest job's p95 average or the highest peak of those runs,
101/// whichever is larger, rounded up to whole CPUs, at least 1, once `min_jobs` such runs say so, every job among them
102/// has a measured average, and the cap held none of them back. Under `measured` they never come down: they bound no
103/// slot there. Only successful runs with a measured average count; see [`next_look`] for the raises it cannot make.
104pub fn changes(config: &Config, history: &History, context: Context<'_>) -> Vec<SizingChange> {
105    next_look(config, history, context).changes
106}
107
108/// What a look does: the changes [`changes`] picks, and the `cpus` and `slots` raises it cannot make.
109#[derive(Debug, Clone, Default, PartialEq, Eq)]
110pub struct Look {
111    /// The changes to make now, in order, one per profile at most.
112    pub changes: Vec<SizingChange>,
113    /// The profiles whose jobs the CPU cap holds back but whose `cpus` cannot go up: the slice's CPUs are unknown or
114    /// all theirs, the CPU budget has no room, or under `measured` admission the raise before did not help them.
115    pub refused: Vec<SizingRefused>,
116    /// Under `measured` admission, the profiles whose jobs waited for a slot alone but whose `slots` cannot go up:
117    /// Docker's disk is not measured yet, they are at `max_slots`, or one more slot would leave every slot less than
118    /// `SLOT_DISK` of Docker's disk.
119    pub refused_slots: Vec<SizingRefused>,
120    /// Under `measured` admission, the profiles changed before whose jobs waited for a slot alone in the window but
121    /// whose jobs since the change do not ask for one (`Fresh::wants_slots`): no slot is added for the waits before.
122    pub waited_before_change: Vec<String>,
123}
124
125/// The changes [`changes`] picks, and the `cpus` and `slots` raises the same look cannot make.
126pub fn next_look(config: &Config, history: &History, context: Context<'_>) -> Look {
127    let rules = &config.sizing;
128    if !rules.auto {
129        return Look::default();
130    }
131    let window = context.window;
132    let sizing = advise(config, context.memory_max, history, window);
133    let now = window.now / 1000;
134    let mut budget = Budget::new(config, &sizing.profiles);
135    let mut look = Look::default();
136    for advised in &sizing.profiles {
137        let Some(profile) = config.profiles.get(&advised.name) else {
138            continue;
139        };
140        let last = context.last.get(&advised.name).copied();
141        let since = last.map_or(u64::MAX, |at| now.saturating_sub(at));
142        if advised.revert.is_some() {
143            match reverted(config, profile, advised, &sizing, &context, &budget) {
144                Revert::Write(change) => {
145                    budget.take(&advised.name, profile, change.from, change.to);
146                    look.changes.push(change);
147                    continue;
148                }
149                Revert::Wait => continue,
150                // Raised as far as its bounds allow, as any raise is.
151                Revert::Outside => {}
152            }
153        }
154        if since < rules.raise_after().as_secs() {
155            continue;
156        }
157        let lowering = Lowering::after(since, rules.interval.as_secs());
158        let fresh = Fresh::read(history, advised, config.host.admission, last, window);
159        let mut reasons = Vec::new();
160        let memory = memory(config, profile, advised.advice, lowering);
161        let running = context.running.get(&advised.name).copied().unwrap_or(0);
162        let (slots, slots_reason) = match config.host.admission {
163            Admission::Caps => (
164                caps_slots(profile, advised, &fresh, memory, &context, &budget, running),
165                None,
166            ),
167            Admission::Measured => {
168                measured_slots(config, profile, advised, &fresh, &context, &budget)
169                    .step(&advised.name, &mut look.refused_slots, || {
170                        (lowering == Lowering::Allowed)
171                            .then(|| idle_slots(config, history, advised, &context, running))?
172                    })
173                    .map_or((advised.slots, None), |step| (step.to, Some(step.reason)))
174            }
175        };
176        let (slots, cpus) = cpus_and_slots(
177            config,
178            profile,
179            advised,
180            history,
181            fresh.start,
182            &context,
183            &budget,
184            slots,
185            lowering,
186        );
187        if memory != profile.memory.0 {
188            reasons.push(memory_reason(
189                &sizing,
190                advised,
191                memory,
192                slots,
193                config.sizing.max_step,
194            ));
195        }
196        // Under caps a lowered memory's slots are its line's.
197        let advised_slots = config.host.admission == Admission::Caps && memory < profile.memory.0;
198        if let Some(reason) = slots_reason {
199            reasons.push(reason);
200        } else if slots > profile.slots && !advised_slots {
201            reasons.push(fresh.waited(last.is_some()));
202        }
203        let cpus = cpus.taken(&advised.name, &mut reasons, &mut look.refused);
204        let from = Resize {
205            memory: profile.memory.0,
206            slots: profile.slots,
207            cpus: cpus.map(|_| profile.cpus),
208        };
209        let to = Resize {
210            memory,
211            slots,
212            cpus,
213        };
214        if to == from || slots == 0 {
215            continue;
216        }
217        budget.take(&advised.name, profile, from, to);
218        look.changes.push(SizingChange {
219            at: now,
220            profile: advised.name.clone(),
221            from,
222            to,
223            reason: reasons.join("; "),
224            uncertain: false,
225        });
226    }
227    look.waited_before_change = waited_before_change(config, history, &sizing, &context);
228    look
229}
230
231/// Under `measured` admission, the profiles changed before whose jobs in the window waited for a slot alone
232/// (`ProfileSizing::wants_slots`) but whose jobs since the change do not ask for one (`Fresh::wants_slots`), which is
233/// all a look reads: the advice must not promise a slot for the waits before the change.
234fn waited_before_change(
235    config: &Config,
236    history: &History,
237    sizing: &Sizing,
238    context: &Context<'_>,
239) -> Vec<String> {
240    if config.host.admission != Admission::Measured {
241        return Vec::new();
242    }
243    sizing
244        .profiles
245        .iter()
246        .filter(|advised| advised.wants_slots)
247        .filter(|advised| {
248            let Some(last) = context.last.get(&advised.name).copied() else {
249                return false;
250            };
251            !Fresh::read(
252                history,
253                advised,
254                config.host.admission,
255                Some(last),
256                context.window,
257            )
258            .wants_slots()
259        })
260        .map(|advised| advised.name.clone())
261        .collect()
262}
263
264/// How far back a config that lowers `memory` is compared with auto sizing's raises, in seconds: a day.
265pub const UNDONE_RAISES: u64 = 24 * 3600;
266
267/// What a new config undoes of auto sizing's `memory` raises: a line for each profile whose `memory` it lowers under
268/// what one of `changes` raised it to in the `UNDONE_RAISES` before `now` (seconds), as a config saved from a text read
269/// before the raise does. A raise comes only after the jobs needed more than the cap (ADR 0070), so they may run out
270/// of memory again. The config is not refused: the owner may mean it.
271pub fn undone_raises(
272    running: &Config,
273    new: &Config,
274    changes: &[SizingChange],
275    now: u64,
276) -> Vec<String> {
277    new.profiles
278        .iter()
279        .filter_map(|(name, profile)| {
280            let memory = profile.memory.0;
281            let before = running.profiles.get(name)?.memory.0;
282            let raise = changes
283                .iter()
284                .filter(|change| {
285                    change.profile == *name
286                        && change.to.memory > change.from.memory
287                        && now.saturating_sub(change.at) < UNDONE_RAISES
288                })
289                .max_by_key(|change| change.at)?;
290            (memory < before && memory < raise.to.memory).then(|| {
291                format!(
292                    "{name}'s memory goes to {}, under the {} auto sizing raised it to {} because its jobs needed \
293                     more than {}: they may run out of memory again",
294                    ByteSize(memory),
295                    ByteSize(raise.to.memory),
296                    ago(raise.at, now),
297                    ByteSize(raise.from.memory)
298                )
299            })
300        })
301        .collect()
302}
303
304/// Whether a change may lower `memory` or `cpus`: only `interval` after the profile's last change.
305#[derive(Debug, Clone, Copy, PartialEq, Eq)]
306enum Lowering {
307    Allowed,
308    Held,
309}
310
311impl Lowering {
312    /// Whether a change `since` seconds after the profile's last one may lower it, with `interval` seconds between.
313    fn after(since: u64, interval: u64) -> Lowering {
314        if since >= interval {
315            Lowering::Allowed
316        } else {
317            Lowering::Held
318        }
319    }
320}
321
322/// The budget as the changes picked so far leave it.
323struct Budget {
324    /// `slots × memory` over all profiles, with the mirror.
325    caps: u64,
326    /// `slots × cpus` over all profiles, in millicores.
327    cpus: u64,
328    /// Each profile's slots.
329    slots: BTreeMap<String, u32>,
330    /// Each profile's `cpus`.
331    each: BTreeMap<String, u32>,
332}
333
334impl Budget {
335    /// Takes what a change of `profile` from `from` to `to` adds, and gives back what it frees.
336    fn take(&mut self, name: &str, profile: &Profile, from: Resize, to: Resize) {
337        self.caps = self
338            .caps
339            .saturating_sub(from.memory.saturating_mul(u64::from(from.slots)))
340            .saturating_add(to.memory.saturating_mul(u64::from(to.slots)));
341        let millicores = |resize: Resize| {
342            u64::from(resize.cpus.unwrap_or(profile.cpus)) * 1000 * u64::from(resize.slots)
343        };
344        self.cpus = self
345            .cpus
346            .saturating_sub(millicores(from))
347            .saturating_add(millicores(to));
348        self.slots.insert(name.to_owned(), to.slots);
349        self.each
350            .insert(name.to_owned(), to.cpus.unwrap_or(profile.cpus));
351    }
352
353    /// The budget the config takes as it is.
354    fn new(config: &Config, profiles: &[ProfileSizing]) -> Budget {
355        Budget {
356            caps: config.memory_caps().map_or(u64::MAX, |caps| caps.0),
357            cpus: config
358                .profiles
359                .values()
360                .map(|profile| u64::from(profile.cpus) * 1000 * u64::from(profile.slots))
361                .sum(),
362            slots: profiles
363                .iter()
364                .map(|advised| (advised.name.clone(), advised.slots))
365                .collect(),
366            each: config
367                .profiles
368                .iter()
369                .map(|(name, profile)| (name.clone(), profile.cpus))
370                .collect(),
371        }
372    }
373
374    /// The CPUs, millicores, that measured admission holds at caps with `name` at `cpus`: every profile's warm
375    /// runners idle at their caps (`warm_runners`), and one more runner at the largest cap, as one of a job with no
376    /// prediction reserves.
377    fn at_caps(&self, config: &Config, name: &str, cpus: u32) -> u64 {
378        let each = |other: &str| {
379            if other == name {
380                cpus
381            } else {
382                self.each.get(other).copied().unwrap_or(0)
383            }
384        };
385        let warm: u64 = config
386            .profiles
387            .keys()
388            .map(|other| {
389                let slots = self.slots.get(other).copied().unwrap_or(0);
390                u64::from(warm_runners(config, other, slots)) * u64::from(each(other))
391            })
392            .sum();
393        let largest = config
394            .profiles
395            .keys()
396            .map(|other| each(other))
397            .max()
398            .unwrap_or(0);
399        (warm + u64::from(largest)) * 1000
400    }
401}
402
403/// How many warm runners `profile`'s pools keep at most, with `slots`: their `warm` added up, at most its slots.
404fn warm_runners(config: &Config, profile: &str, slots: u32) -> u32 {
405    config
406        .repositories
407        .iter()
408        .filter(|repository| repository.profile == profile)
409        .map(|repository| repository.warm)
410        .sum::<u32>()
411        .min(slots)
412}
413
414/// What the jobs queued since a profile's last change say about its slots: a raise reads only them, so one slot
415/// that fixed a month's waits is not followed by another every `raise_interval`.
416struct Fresh {
417    /// When the evidence starts, in milliseconds: the last change, or the window's start.
418    start: u64,
419    jobs: u64,
420    /// Whether they ask for more slots (`sizing::wants_slots`), however few.
421    wants: bool,
422    /// Seconds they waited that ask for a slot under `admission` (`slot_asking_wait`), p95.
423    wait: u64,
424    /// The most of them that waited only for a slot at once.
425    short: u32,
426    busy: u32,
427}
428
429impl Fresh {
430    /// The profile's jobs queued since `last` (seconds), or in the whole window without a last change.
431    fn read(
432        history: &History,
433        advised: &ProfileSizing,
434        admission: Admission,
435        last: Option<u64>,
436        window: Window,
437    ) -> Fresh {
438        let start = last.map_or(window.since, |at| at.saturating_mul(1000).max(window.since));
439        let jobs: Vec<_> = history
440            .jobs
441            .iter()
442            .filter(|job| job.profile == advised.name && job.queued_at >= start)
443            .collect();
444        let busy = busy(
445            history,
446            &advised.name,
447            advised.slots,
448            first_runner(history, start),
449            window.now,
450        );
451        Fresh {
452            start,
453            jobs: jobs.len() as u64,
454            wants: wants_slots(admission, &jobs, history.kept_since, busy),
455            wait: slot_asking_wait(admission, &jobs, history.kept_since),
456            short: short_of_slots(&jobs),
457            busy,
458        }
459    }
460
461    /// Whether another slot would help: enough jobs, which waited a minute for a slot at p95, or under `caps` kept
462    /// the slots busy half the time.
463    fn wants_slots(&self) -> bool {
464        self.jobs >= RAISE_RUNS && self.wants
465    }
466
467    /// Why a slot is added.
468    fn waited(&self, changed: bool) -> String {
469        format!(
470            "{} jobs {}waited {}s for a slot (p95) and slots were busy {}%; one more fits the slice beside the \
471             other profiles'",
472            self.jobs,
473            if changed {
474                "since its last change "
475            } else {
476                ""
477            },
478            self.wait,
479            self.busy
480        )
481    }
482}
483
484/// Why `memory` changes, as the advice line says it of the `memory` and `slots` this change writes, not the
485/// advice's own: a step of `max_step` stops short of a lowering's memory, and the slots that fit follow. A change
486/// short of the advice says what it goes toward.
487fn memory_reason(
488    sizing: &Sizing,
489    advised: &ProfileSizing,
490    memory: u64,
491    slots: u32,
492    max_step: u32,
493) -> String {
494    let made = ProfileSizing {
495        advice: match advised.advice {
496            Advice::Lower { .. } => Advice::Lower { memory, slots },
497            Advice::Raise { .. } => Advice::Raise { memory, slots },
498            other => other,
499        },
500        ..advised.clone()
501    };
502    let line = sizing.memory_line(&made);
503    match advised.advice.change() {
504        Some(advice) if advice.memory != memory => format!(
505            "{line}; a step toward {}, at most {max_step}% a change",
506            ByteSize(advice.memory)
507        ),
508        _ => line,
509    }
510}
511
512/// What a look does with a lowering that went too far.
513enum Revert {
514    /// Takes it back with this change.
515    Write(SizingChange),
516    /// Waits while more runners run than the slots it gives back hold, or while the changes before it in this look
517    /// leave the slice no slot at its `memory`: nothing else changes the profile meanwhile.
518    Wait,
519    /// It gets no raise: the profile is sized as any other.
520    Outside,
521}
522
523/// The change that takes back a lowering that went too far (`ProfileSizing::revert`): straight to the `memory`
524/// it came from, or as far toward it as `[host] memory_ceiling` and `max_memory` allow, and, under caps, the slots it
525/// had, without waiting for `raise_interval` or keeping to `max_step`, as an undo would.
526fn reverted(
527    config: &Config,
528    profile: &Profile,
529    advised: &ProfileSizing,
530    sizing: &Sizing,
531    context: &Context<'_>,
532    budget: &Budget,
533) -> Revert {
534    let Advice::Raise { memory, slots } = advised.advice else {
535        return Revert::Outside;
536    };
537    let from = Resize {
538        memory: profile.memory.0,
539        slots: profile.slots,
540        cpus: None,
541    };
542    // The advice holds the slots the slice holds beside the config's other profiles; under caps, the changes before
543    // it in this look may have taken some of that.
544    let slots = match (config.host.admission, context.memory_max) {
545        (Admission::Caps, Some(memory_max)) => {
546            let share = profile.memory.0.saturating_mul(u64::from(profile.slots));
547            let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
548            slots.min(u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX))
549        }
550        _ => slots,
551    };
552    if slots == 0 {
553        return Revert::Wait;
554    }
555    let running = context.running.get(&advised.name).copied().unwrap_or(0);
556    let to = Resize {
557        memory,
558        slots,
559        cpus: None,
560    };
561    if config.host.admission == Admission::Caps && running > slots {
562        return Revert::Wait;
563    }
564    Revert::Write(SizingChange {
565        at: sizing.until,
566        profile: advised.name.clone(),
567        from,
568        to,
569        reason: sizing.memory_line(advised),
570        uncertain: false,
571    })
572}
573
574/// The profile's next `memory`: a lowering at most `max_step` percent away, in steps of `GRANULE`, and only while
575/// `lowering` allows it; a raise straight to the advice's, since only a run that met the cap raises it (ADR 0070),
576/// and a kill is direct evidence. The advice keeps to `min_memory` and `max_memory`, and lowers nothing while a job
577/// is on its cap.
578fn memory(config: &Config, profile: &Profile, advice: Advice, lowering: Lowering) -> u64 {
579    let current = profile.memory.0;
580    let step = current.saturating_mul(u64::from(config.sizing.max_step)) / 100;
581    match advice {
582        Advice::Lower { memory, .. } if lowering == Lowering::Allowed => {
583            let floor = current
584                .saturating_sub(step)
585                .div_ceil(GRANULE)
586                .saturating_mul(GRANULE);
587            memory.max(floor).min(current)
588        }
589        Advice::Raise { memory, .. } => memory,
590        _ => current,
591    }
592}
593
594/// Under `caps`: as many slots as the advice gives for the new memory. A lowering whose jobs waited adds as many as
595/// the memory the slots take now holds, up to the advice's slots, which keep to `max_slots`; a raise keeps as many as the slice holds beside the other
596/// profiles, and 0, no change, while more than that run. With the memory kept, one more slot when its jobs waited
597/// since the last change and its cap fits `MemoryMax` beside every other cap, up to `max_slots`. Otherwise the same.
598fn caps_slots(
599    profile: &Profile,
600    advised: &ProfileSizing,
601    fresh: &Fresh,
602    memory: u64,
603    context: &Context<'_>,
604    budget: &Budget,
605    running: u32,
606) -> u32 {
607    let current = profile.slots;
608    let fit = |room: u64| u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX);
609    // A step toward the advice's memory fits no more slots than the advice gives, which keeps to `max_slots`.
610    if let Advice::Lower { slots, .. } = advised.advice
611        && memory < profile.memory.0
612        && advised.wants_slots
613    {
614        let share = profile.memory.0.saturating_mul(u64::from(current));
615        return fit(share).min(slots).max(current);
616    }
617    let Some(memory_max) = context.memory_max else {
618        return current;
619    };
620    if memory > profile.memory.0 {
621        let share = profile.memory.0.saturating_mul(u64::from(current));
622        let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
623        // Fewer slots than run would leave the running ones over the budget: wait for some to finish.
624        let slots = fit(room).min(current);
625        return if slots < running { 0 } else { slots };
626    }
627    if fresh.wants_slots()
628        && profile.max_slots.is_none_or(|max| current < max)
629        && budget.caps.saturating_add(memory) <= memory_max
630    {
631        return current + 1;
632    }
633    current
634}
635
636/// A change of a profile's slots under `measured` admission, and why.
637struct SlotStep {
638    to: u32,
639    reason: String,
640}
641
642/// What a profile's jobs since its last change ask of its slots under `measured` admission.
643enum Raise {
644    /// No more: nothing waited for a slot alone.
645    Unasked,
646    /// This many, and why.
647    To(SlotStep),
648    /// More, which cannot be added, and why.
649    Refused(String),
650}
651
652impl Raise {
653    /// The change of `profile`'s slots to make: the raise, or with none asked, `idle`'s lowering. A refused raise
654    /// goes to `refused`, and its jobs, which waited for a slot since the last change, keep the slots they have.
655    fn step(
656        self,
657        profile: &str,
658        refused: &mut Vec<SizingRefused>,
659        idle: impl FnOnce() -> Option<SlotStep>,
660    ) -> Option<SlotStep> {
661        match self {
662            Raise::To(step) => Some(step),
663            Raise::Refused(reason) => {
664                refused.push(SizingRefused::new(profile, reason));
665                None
666            }
667            Raise::Unasked => idle(),
668        }
669    }
670}
671
672/// Under `measured`, more slots when its jobs since the last change waited for one while memory and CPU had room
673/// (`Fresh::wants_slots`): as many as waited at once, at least 1 and at most `max_step` percent of its slots, up to
674/// `max_slots`, and only as many as leave every slot, with the slots the changes before it gave, `SLOT_DISK` of
675/// Docker's disk. A slot under measured admission costs no memory or CPU: each runner reserves its job's prediction
676/// when it starts, and the budgets stop a start that does not fit. What it costs is disk: every slot's Docker store
677/// shares the same quarter of it (`cache::docker_budget_gib`). Refused when Docker's disk was not measured yet, the
678/// profile is at `max_slots`, or one more slot leaves too little disk.
679fn measured_slots(
680    config: &Config,
681    profile: &Profile,
682    advised: &ProfileSizing,
683    fresh: &Fresh,
684    context: &Context<'_>,
685    budget: &Budget,
686) -> Raise {
687    let current = advised.slots;
688    if !fresh.wants_slots() {
689        return Raise::Unasked;
690    }
691    let asked = format!(
692        "{} jobs {}waited {}s for a slot (p95) while memory and CPU had room, {} at once at most",
693        fresh.jobs,
694        if context.last.contains_key(&advised.name) {
695            "since its last change "
696        } else {
697            ""
698        },
699        fresh.wait,
700        fresh.short,
701    );
702    let max = profile.max_slots.unwrap_or(u32::MAX);
703    if current >= max {
704        return Raise::Refused(format!("{asked}; it is at its max_slots, {max}"));
705    }
706    let Some(disk) = context.disk else {
707        return Raise::Refused(format!(
708            "{asked}; Docker's disk is not measured yet, which the next check of the caches does"
709        ));
710    };
711    let step = (current.saturating_mul(config.sizing.max_step) / 100).max(1);
712    let wanted = current.saturating_add(fresh.short.clamp(1, step)).min(max);
713    let others: u32 = budget
714        .slots
715        .iter()
716        .filter(|(name, _)| **name != advised.name)
717        .map(|(_, slots)| *slots)
718        .sum();
719    let Some(to) = (current.saturating_add(1)..=wanted)
720        .rev()
721        .find(|slots| share(disk, others.saturating_add(*slots)) >= SLOT_DISK)
722    else {
723        let next = current.saturating_add(1);
724        return Raise::Refused(format!(
725            "{asked}; at {next} slots, beside the other profiles' {others}, each slot would keep {}, under the {} \
726             auto sizing leaves each slot",
727            kept(
728                disk,
729                others.saturating_add(current),
730                others.saturating_add(next)
731            ),
732            ByteSize(SLOT_DISK),
733        ));
734    };
735    Raise::To(SlotStep {
736        to,
737        reason: format!(
738            "{asked}; at {to} slots each slot keeps {}",
739            kept(
740                disk,
741                others.saturating_add(current),
742                others.saturating_add(to)
743            )
744        ),
745    })
746}
747
748/// One slot's share of the `disk` the slots' Docker stores may keep together among `slots`, in bytes, as the
749/// runners get it.
750fn share(disk: u64, slots: u32) -> u64 {
751    docker_budget_gib(disk, u64::from(slots)) << 30
752}
753
754/// What a slot keeps of Docker's disk at `to` slots in all, and what it keeps at `from` when that is more: `2G of
755/// Docker's disk, down from 3G`.
756fn kept(disk: u64, from: u32, to: u32) -> String {
757    let (before, after) = (share(disk, from), share(disk, to));
758    if after < before {
759        format!(
760            "{} of Docker's disk, down from {}",
761            ByteSize(after),
762            ByteSize(before)
763        )
764    } else {
765        format!("{} of Docker's disk", ByteSize(after))
766    }
767}
768
769/// Under `measured`, one slot fewer when, for the `IDLE_FOR` before now, its slots stayed `IDLE_SLOTS` or more above
770/// the most runners that held one for a job at once, its warm runners counted: an idle slot shrinks every slot's
771/// share of Docker's disk and keeps a volume. Only once the history holds a runner from before that time and the
772/// profile has had its slots that long, by its `profile.changed` events and auto sizing's last change of its slots,
773/// and `min_jobs` of its jobs ran in it, so a host that was off says nothing; never under what runs or 1. This takes
774/// back a raise made by hand as well (ADR 0090).
775fn idle_slots(
776    config: &Config,
777    history: &History,
778    advised: &ProfileSizing,
779    context: &Context<'_>,
780    running: u32,
781) -> Option<SlotStep> {
782    let current = advised.slots;
783    let now = context.window.now;
784    let start = now.checked_sub(IDLE_FOR)?;
785    let set = slots_since(&history.slots_set, &advised.name, current, now);
786    // A change of its memory or cpus leaves the slots as they ran.
787    let changed = history
788        .changes
789        .iter()
790        .filter(|change| change.profile == advised.name && change.to.slots != change.from.slots)
791        .map(|change| change.at.saturating_mul(1000))
792        .max()
793        .unwrap_or(0);
794    // The history must cover the whole time: a runner from before it, the profile's slots and no change of them since.
795    let first = history
796        .runners
797        .iter()
798        .map(|runner| runner.started_at)
799        .min()?;
800    if set.max(changed).max(first) > start {
801        return None;
802    }
803    let jobs = history
804        .jobs
805        .iter()
806        .filter(|job| job.profile == advised.name && job.queued_at >= start)
807        .count() as u64;
808    if jobs < u64::from(config.sizing.min_jobs) {
809        return None;
810    }
811    let mut edges: Vec<(u64, i32)> = history
812        .runners
813        .iter()
814        .filter(|runner| {
815            runner.profile == advised.name && runner.removed_at.is_none_or(|at| at > start)
816        })
817        .flat_map(|runner| {
818            [
819                (runner.started_at.max(start), 1),
820                (runner.removed_at.unwrap_or(now).min(now), -1),
821            ]
822        })
823        .collect();
824    let ran = peak(&mut edges);
825    let warm = warm_runners(config, &advised.name, current);
826    if ran.saturating_add(warm).saturating_add(IDLE_SLOTS) > current {
827        return None;
828    }
829    let to = current.saturating_sub(1).max(running).max(1);
830    (to < current).then(|| SlotStep {
831        to,
832        reason: format!(
833            "at most {ran} of its {current} slots held a runner for a job at once in the last {}{}: an idle slot \
834             shrinks every slot's share of Docker's disk",
835            duration(IDLE_FOR / 1000),
836            if warm > 0 {
837                format!(", beside {warm} warm runners")
838            } else {
839                String::new()
840            }
841        ),
842    })
843}
844
845/// When the daemon started running the profile with `slots`, in ms, by `set`, as `cpus_since` reads `cpus`.
846fn slots_since(set: &[SlotsSet], profile: &str, slots: u32, now: u64) -> u64 {
847    since_set(
848        set.iter()
849            .filter(|event| event.profile == profile)
850            .map(|event| (event.slots, event.at)),
851        slots,
852        now,
853    )
854}
855
856/// The profile's next `cpus` and `slots`, from the `slots` memory and waits chose: under `caps` with a CPU budget,
857/// slots past what the budget holds at the new `cpus` are dropped, and the `cpus` sized again at the slots that stay,
858/// so an extra slot the budget refuses does not also refuse a CPU raise that fits without it.
859#[expect(
860    clippy::too_many_arguments,
861    reason = "each is one thing a CPU change reads; a struct for them would be read here only"
862)]
863fn cpus_and_slots(
864    config: &Config,
865    profile: &Profile,
866    advised: &ProfileSizing,
867    history: &History,
868    since: u64,
869    context: &Context<'_>,
870    budget: &Budget,
871    slots: u32,
872    lowering: Lowering,
873) -> (u32, CpuStep) {
874    let size = |slots: u32| {
875        cpus(
876            config, profile, advised, history, since, context, budget, slots, lowering,
877        )
878    };
879    // Slots the CPU budget cannot hold would wait for CPU anyway, and their waits would ask for more.
880    let fit = |slots: u32, cpus: &CpuStep| {
881        let cpus = match cpus {
882            CpuStep::To(cpus) => cpus.to,
883            CpuStep::Keep | CpuStep::Refused(_) => profile.cpus,
884        };
885        if slots > profile.slots {
886            cpu_slots(config, profile, context, budget, cpus)
887                .map_or(slots, |fit| slots.min(fit.max(profile.slots)))
888        } else {
889            slots
890        }
891    };
892    let mut cpus = size(slots);
893    let fitted = fit(slots, &cpus);
894    if fitted < slots {
895        cpus = size(fitted);
896    }
897    (fit(fitted, &cpus), cpus)
898}
899
900/// Under `caps` admission with a CPU budget, the most slots of the profile at `cpus` each that fit the budget beside
901/// every other profile's `slots × cpus`, as the changes before it left them; none more than it has when the slice's
902/// CPUs are unknown. `None` when no CPU budget bounds slots.
903fn cpu_slots(
904    config: &Config,
905    profile: &Profile,
906    context: &Context<'_>,
907    budget: &Budget,
908    cpus: u32,
909) -> Option<u32> {
910    let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
911    else {
912        return None;
913    };
914    let Some(slice) = context.cpus else {
915        return Some(profile.slots);
916    };
917    let others = budget
918        .cpus
919        .saturating_sub(u64::from(profile.cpus) * 1000 * u64::from(profile.slots));
920    let room = cpu_budget(slice, overcommit).saturating_sub(others);
921    Some(u32::try_from(room / (u64::from(cpus.max(1)) * 1000)).unwrap_or(u32::MAX))
922}
923
924/// What a look does with a profile's `cpus`.
925enum CpuStep {
926    Keep,
927    To(Cpus),
928    /// Its jobs are held back by the cap, and this says why it cannot go up.
929    Refused(String),
930}
931
932impl CpuStep {
933    /// The new `cpus`, if any, its reason added to `reasons`, or a refused raise to `refused`.
934    fn taken(
935        self,
936        profile: &str,
937        reasons: &mut Vec<String>,
938        refused: &mut Vec<SizingRefused>,
939    ) -> Option<u32> {
940        match self {
941            CpuStep::To(cpus) => {
942                reasons.push(cpus.reason);
943                Some(cpus.to)
944            }
945            CpuStep::Refused(reason) => {
946                refused.push(SizingRefused {
947                    profile: profile.to_owned(),
948                    reason,
949                });
950                None
951            }
952            CpuStep::Keep => None,
953        }
954    }
955}
956
957/// A new `cpus` and why.
958struct Cpus {
959    to: u32,
960    reason: String,
961}
962
963/// A job: its repository, workflow and name.
964type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
965
966/// When the daemon started running the profile with `cpus`, in ms, by `set`: its first event since the last one
967/// with other `cpus`. `now` when its latest has other `cpus`, as before a reload announces them; 0 with no events.
968fn cpus_since(set: &[CpusSet], profile: &str, cpus: u32, now: u64) -> u64 {
969    since_set(
970        set.iter()
971            .filter(|event| event.profile == profile)
972            .map(|event| (event.cpus, event.at)),
973        cpus,
974        now,
975    )
976}
977
978/// Since when, in ms, a profile's `events` (a value and when it was set, oldest first) have had `value`: the first
979/// since the last with another. `now` when the latest has another, as before a reload announces it; 0 with none.
980fn since_set(events: impl Iterator<Item = (u32, u64)>, value: u32, now: u64) -> u64 {
981    let mut since = None;
982    let mut seen = false;
983    for (set, at) in events {
984        seen = true;
985        if set != value {
986            since = None;
987        } else if since.is_none() {
988            since = Some(at);
989        }
990    }
991    since.unwrap_or(if seen { now } else { 0 })
992}
993
994/// What one job's runs a `cpus` change reads say.
995#[derive(Default)]
996struct JobCpu {
997    /// Their measured averages, millicores.
998    averages: Vec<u64>,
999    /// The highest of their peaks, millicores.
1000    peak: u64,
1001    /// How many of them their quota throttled in `HELD` percent of their run or more.
1002    throttled: u64,
1003    /// How many of them without a throttling figure peaked within `MET` percent of the cap.
1004    peaked: u64,
1005    /// What their quota did to each of them.
1006    runs: Vec<Throttling>,
1007}
1008
1009impl JobCpu {
1010    /// How many of them the cap held back: throttled where that was measured, else met in a burst.
1011    fn held(&self) -> u64 {
1012        self.throttled + self.peaked
1013    }
1014
1015    /// Why `job`'s runs at `current` CPUs raise them under measured admission.
1016    fn held_reason(&self, job: JobKey<'_>, current: u32) -> String {
1017        let how = if self.peaked == 0 {
1018            format!("throttled in {HELD}% of their run or more")
1019        } else {
1020            format!(
1021                "throttled in {HELD}% of their run or more, or peaking at the cap where that was not measured"
1022            )
1023        };
1024        format!(
1025            "{} runs of {} were held back by its {current} CPUs, {how}: under measured admission they bound no slot, \
1026             so the cap only slows them",
1027            self.held(),
1028            job_named(job)
1029        )
1030    }
1031}
1032
1033/// Whether the cap held a run back. With a throttling figure that alone says so, whatever the run's peak: a 2-second
1034/// peak at the cap says nothing of what the cap cost. Without one, as in runs from before schema 17 or on a kernel
1035/// that keeps none, its peak says so as it did before (ADR 0075): missing is not read as never throttled.
1036enum Hold {
1037    Throttled,
1038    Peaked,
1039    Free,
1040}
1041
1042impl Hold {
1043    fn of(job: &super::Ran, cap: u64) -> Hold {
1044        match job.throttling().held() {
1045            Some(true) => Hold::Throttled,
1046            Some(false) => Hold::Free,
1047            None => Hold::peaked(job, cap),
1048        }
1049    }
1050
1051    fn peaked(job: &super::Ran, cap: u64) -> Hold {
1052        if job.cpu_peak.saturating_mul(100) >= cap.saturating_mul(MET) {
1053            Hold::Peaked
1054        } else {
1055            Hold::Free
1056        }
1057    }
1058}
1059
1060/// The runs of the profile a `cpus` change reads, by job: successful runs whose container had `cap`, queued at or
1061/// after `since` (ms), the later of the profile's last change and when it got these `cpus` (`cpus_since`). A run
1062/// without a measured average adds none; nor does an average of 0, which only a run sampled once records. Its job is
1063/// still listed, so a lowering can tell a job it knows nothing of from one it lacks.
1064fn runs_at<'run>(
1065    history: &'run History,
1066    profile: &str,
1067    cap: u64,
1068    since: u64,
1069) -> BTreeMap<JobKey<'run>, JobCpu> {
1070    let mut jobs: BTreeMap<JobKey<'run>, JobCpu> = BTreeMap::new();
1071    for job in &history.jobs {
1072        if job.profile == profile
1073            && job.cpu_cap == Some(cap)
1074            && job.succeeded
1075            && job.queued_at >= since
1076        {
1077            let read = jobs
1078                .entry((&job.repository, job.workflow.as_deref(), &job.name))
1079                .or_default();
1080            read.averages
1081                .extend(job.cpu_average.filter(|average| *average > 0));
1082            read.peak = read.peak.max(job.cpu_peak);
1083            match Hold::of(job, cap) {
1084                Hold::Throttled => read.throttled += 1,
1085                Hold::Peaked => read.peaked += 1,
1086                Hold::Free => {}
1087            }
1088            read.runs.push(job.throttling());
1089        }
1090    }
1091    jobs
1092}
1093
1094/// One successful run of a job in a profile, as [`unhelped_raise`] reads it.
1095#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096pub struct CpuRun {
1097    /// Its container's CPU cap, millicores, when recorded.
1098    pub cap: Option<u64>,
1099    /// When it was queued, ms.
1100    pub queued_at: u64,
1101    /// What its quota did to it.
1102    pub throttling: Throttling,
1103}
1104
1105/// A figure at the median of a job's runs at its current `cpus` and at the `cpus` before.
1106#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1107pub struct Medians {
1108    /// At its current `cpus`.
1109    pub now: u64,
1110    /// At the `cpus` before.
1111    pub before: u64,
1112}
1113
1114/// A raise of `cpus` that did not help a job: its runs neither took less time nor were throttled in less of their run.
1115#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1116pub struct Unhelped {
1117    /// The CPU cap before, millicores.
1118    pub from: u64,
1119    /// How long its runs took, ms, where that is known on both sides.
1120    pub took: Option<Medians>,
1121    /// The share of their run their quota throttled them in, per mille, where that is known on both sides.
1122    pub throttled: Option<Medians>,
1123}
1124
1125/// Whether the raise that brought a job to `cap` (millicores) did not help it, by `runs`, its successful runs in one
1126/// profile at any cap, and `now`, those at `cap` the reader counts; `None` when it did, or when nothing says: no
1127/// raise came before, as when its latest runs at another cap had more, or fewer than `RAISE_RUNS` runs at the cap
1128/// before. It helped when its runs at `cap` took less time at the median than at the cap before, or were throttled
1129/// in less of their run. A tool that sizes its pool from the quota (the JVM's `availableProcessors`, cargo's `-j`,
1130/// Go's `GOMAXPROCS`) starts more workers at each raise and stays throttled, running no faster: its raises stop
1131/// there, as more `cpus` would only take memory and weight (ADR 0079).
1132pub fn unhelped_raise(runs: &[CpuRun], cap: u64, now: &[Throttling]) -> Option<Unhelped> {
1133    let from = runs
1134        .iter()
1135        .filter(|run| run.cap.is_some_and(|other| other != cap))
1136        .max_by_key(|run| run.queued_at)?
1137        .cap
1138        .filter(|before| *before < cap)?;
1139    let at_before: Vec<Throttling> = runs
1140        .iter()
1141        .filter(|run| run.cap == Some(from))
1142        .map(|run| run.throttling)
1143        .collect();
1144    if (at_before.len() as u64) < RAISE_RUNS {
1145        return None;
1146    }
1147    let median = |runs: &[Throttling], figure: fn(Throttling) -> Option<u64>| {
1148        let values: Vec<u64> = runs.iter().copied().filter_map(figure).collect();
1149        (!values.is_empty()).then(|| spread(values).p50)
1150    };
1151    let medians = |figure: fn(Throttling) -> Option<u64>| {
1152        Some(Medians {
1153            now: median(now, figure)?,
1154            before: median(&at_before, figure)?,
1155        })
1156    };
1157    let took = medians(Throttling::took);
1158    let throttled = medians(Throttling::share);
1159    let helped = |medians: Option<Medians>| medians.map(|medians| medians.now < medians.before);
1160    match (helped(took), helped(throttled)) {
1161        (None, None) | (Some(true), _) | (_, Some(true)) => None,
1162        _ => Some(Unhelped {
1163            from,
1164            took,
1165            throttled,
1166        }),
1167    }
1168}
1169
1170/// Why the raise before did not help `job`, whose runs at `cap` are `now`, as [`unhelped_raise`] finds it.
1171fn unhelped(
1172    history: &History,
1173    profile: &str,
1174    job: JobKey<'_>,
1175    cap: u64,
1176    now: &JobCpu,
1177) -> Option<String> {
1178    let runs: Vec<CpuRun> = history
1179        .jobs
1180        .iter()
1181        .filter(|run| {
1182            run.profile == profile
1183                && run.succeeded
1184                && (
1185                    run.repository.as_str(),
1186                    run.workflow.as_deref(),
1187                    run.name.as_str(),
1188                ) == job
1189        })
1190        .map(|run| CpuRun {
1191            cap: run.cpu_cap,
1192            queued_at: run.queued_at,
1193            throttling: run.throttling(),
1194        })
1195        .collect();
1196    let unhelped = unhelped_raise(&runs, cap, &now.runs)?;
1197    let mut why = format!(
1198        "the raise from {} to its {} CPUs did not help it:",
1199        unhelped.from / 1000,
1200        cap / 1000
1201    );
1202    if let Some(took) = unhelped.took {
1203        let _ = write!(
1204            why,
1205            " its runs took {} at the median, against {} before",
1206            duration(took.now / 1000),
1207            duration(took.before / 1000)
1208        );
1209    }
1210    if let Some(throttled) = unhelped.throttled {
1211        let joined = if unhelped.took.is_some() { "," } else { "" };
1212        let _ = write!(
1213            why,
1214            "{joined} its quota throttled them in {}% of their run, against {}%",
1215            throttled.now / 10,
1216            throttled.before / 10
1217        );
1218    }
1219    Some(why)
1220}
1221
1222/// Under measured admission, why the runs the cap held back raise `cpus` at `cap`: the job with the most held runs,
1223/// `RAISE_RUNS` at least, among those the raise before helped (`Ok`); or, when it helped none of them, why not, for
1224/// the one with the most (`Err`). `None` when no job's runs were held back that often. A job the raise before did not
1225/// help says nothing for another raise, but another held job still may.
1226fn held_raise(
1227    history: &History,
1228    profile: &str,
1229    jobs: &BTreeMap<JobKey<'_>, JobCpu>,
1230    cap: u64,
1231    current: u32,
1232) -> Option<Result<String, String>> {
1233    let (helped, refused): (Vec<_>, Vec<_>) = jobs
1234        .iter()
1235        .filter(|(_, read)| read.held() >= RAISE_RUNS)
1236        .map(|(job, read)| (*job, read, unhelped(history, profile, *job, cap, read)))
1237        .partition(|(_, _, why)| why.is_none());
1238    if let Some((job, read, _)) = helped.into_iter().max_by_key(|(_, read, _)| read.held()) {
1239        return Some(Ok(read.held_reason(job, current)));
1240    }
1241    let (job, read, why) = refused.into_iter().max_by_key(|(_, read, _)| read.held())?;
1242    Some(Err(format!(
1243        "{}; {}",
1244        read.held_reason(job, current),
1245        why.unwrap_or_default()
1246    )))
1247}
1248
1249/// `owner/name workflow / job`, as a CPU change's reason names a job.
1250fn job_named(job: JobKey<'_>) -> String {
1251    let (repository, workflow, name) = job;
1252    format!("{repository} {}", job_title(workflow, name, None))
1253}
1254
1255/// A raise of `name`'s `cpus` held back by the cap, as `held` says: `step` more, up to the slice's CPUs, and with a
1256/// CPU budget only as many as it holds: under `caps` every profile's `slots × cpus`, under `measured` every profile's
1257/// warm runners and one more runner at the largest cap (`Budget::at_caps`). Refused, with why, when that is none.
1258#[expect(
1259    clippy::too_many_arguments,
1260    reason = "each is one thing a CPU raise reads; a struct for them would be read here only"
1261)]
1262fn raised(
1263    config: &Config,
1264    name: &str,
1265    profile: &Profile,
1266    context: &Context<'_>,
1267    budget: &Budget,
1268    slots: u32,
1269    step: u32,
1270    held: String,
1271) -> CpuStep {
1272    let current = profile.cpus;
1273    let cap = u64::from(current) * 1000;
1274    let Some(slice) = context.cpus else {
1275        return CpuStep::Refused(format!("{held}; the slice's CPUs could not be read"));
1276    };
1277    let slice_cpus = u32::try_from(slice / 1000).unwrap_or(u32::MAX);
1278    let mut to = current.saturating_add(step).min(slice_cpus);
1279    let mut why = format!("it has all {slice_cpus} of the slice's CPUs");
1280    if let (Admission::Caps, Some(overcommit)) = (config.host.admission, config.host.cpu_overcommit)
1281    {
1282        let others = budget
1283            .cpus
1284            .saturating_sub(cap.saturating_mul(u64::from(profile.slots)));
1285        let room = cpu_budget(slice, overcommit).saturating_sub(others);
1286        let fit = room / 1000 / u64::from(slots.max(1));
1287        let fit = u32::try_from(fit).unwrap_or(u32::MAX);
1288        if fit < to {
1289            to = fit;
1290            why = format!(
1291                "{} CPUs for each of its {slots} slots do not fit the CPU budget beside the other profiles'",
1292                current.saturating_add(1)
1293            );
1294        }
1295    }
1296    // Under measured admission a running job reserves its average, whatever its cap, but an idle warm runner and a
1297    // runner whose job has no prediction reserve the cap.
1298    if let (Admission::Measured, Some(overcommit)) =
1299        (config.host.admission, config.host.cpu_overcommit)
1300    {
1301        let room = cpu_budget(slice, overcommit);
1302        while to > current && budget.at_caps(config, name, to) > room {
1303            to -= 1;
1304            why = format!(
1305                "{} CPUs do not fit the CPU budget with every profile's warm runners and one more runner at the \
1306                 largest cap",
1307                current.saturating_add(1)
1308            );
1309        }
1310    }
1311    if to > current {
1312        CpuStep::To(Cpus { to, reason: held })
1313    } else {
1314        CpuStep::Refused(format!("{held}; {why}"))
1315    }
1316}
1317
1318/// How many `cpus` one change moves a profile at `current` by: `[sizing] max_step` percent of them, at least 1.
1319pub(crate) fn cpu_step(config: &Config, current: u32) -> u32 {
1320    (current.saturating_mul(config.sizing.max_step) / 100).max(1)
1321}
1322
1323/// The profile's next `cpus`, from the CPU averages of its runs at its current `cpus` since `since` (ms), the last
1324/// change, and since it got them (`runs_at`). A raise, `max_step` percent, at least 1, when one job's `RAISE_RUNS`
1325/// runs or more averaged `BOUND` percent of the cap at the median, or under `measured` admission were held back by it
1326/// (kept, not refused, when there is no room; refused when the raise before did not help them): up to the slice's
1327/// CPUs, and with a CPU budget only as many as `raised` fits in it, at the `slots` this change gives; refused, with
1328/// why, when that is none. A lowering, under `caps`
1329/// admission while `lowering` allows it, with `min_jobs` runs, none of a job without a measured average and none the
1330/// cap held back: toward `HEADROOM` times the heaviest job's p95 average or the highest peak, whichever is larger,
1331/// rounded up to whole CPUs, at least 1, by at most `max_step` percent.
1332#[expect(
1333    clippy::too_many_arguments,
1334    reason = "each is one thing a CPU change reads; a struct for them would be read here only"
1335)]
1336fn cpus(
1337    config: &Config,
1338    profile: &Profile,
1339    advised: &ProfileSizing,
1340    history: &History,
1341    since: u64,
1342    context: &Context<'_>,
1343    budget: &Budget,
1344    slots: u32,
1345    lowering: Lowering,
1346) -> CpuStep {
1347    let current = profile.cpus;
1348    let cap = u64::from(current) * 1000;
1349    let since = since.max(cpus_since(
1350        context.cpus_set,
1351        &advised.name,
1352        current,
1353        context.window.now,
1354    ));
1355    let jobs = runs_at(history, &advised.name, cap, since);
1356    let step = cpu_step(config, current);
1357    let raise = |held| {
1358        raised(
1359            config,
1360            &advised.name,
1361            profile,
1362            context,
1363            budget,
1364            slots,
1365            step,
1366            held,
1367        )
1368    };
1369    let bound = jobs
1370        .iter()
1371        .filter(|(_, read)| read.averages.len() as u64 >= RAISE_RUNS)
1372        .map(|(job, read)| (*job, read.averages.len(), spread(read.averages.clone()).p50))
1373        .filter(|(_, _, median)| median.saturating_mul(100) >= cap.saturating_mul(BOUND))
1374        .max_by_key(|(_, _, median)| *median);
1375    if let Some((job, runs, median)) = bound {
1376        let held = format!(
1377            "{} runs of {} averaged {} of its {current} CPUs (median): the cap holds them back",
1378            runs,
1379            job_named(job),
1380            cores(median)
1381        );
1382        return raise(held);
1383    }
1384    // Under measured admission `cpus` bound no slot: a cap that holds runs back only slows them, and frees nothing.
1385    if config.host.admission == Admission::Measured {
1386        match held_raise(history, &advised.name, &jobs, cap, current) {
1387            // A burst the slice's CPUs hold back is the slice's to give, not the profile's.
1388            Some(Ok(held)) => {
1389                return match raise(held) {
1390                    CpuStep::Refused(_) => CpuStep::Keep,
1391                    step => step,
1392                };
1393            }
1394            Some(Err(why)) => return CpuStep::Refused(why),
1395            None => {}
1396        }
1397    }
1398    let runs: usize = jobs.values().map(|read| read.averages.len()).sum();
1399    let peak = jobs.values().map(|read| read.peak).max().unwrap_or(0);
1400    // Under measured admission `cpus` bound no slot, so a lowering frees nothing and only slows the bursts. A job
1401    // whose runs at the cap have no measured average may be the one that needs it, and one the cap held back ran as
1402    // fast as the cap let it: its average says what the cap allowed, not what it needs.
1403    if lowering == Lowering::Held
1404        || config.host.admission == Admission::Measured
1405        || (runs as u64) < u64::from(config.sizing.min_jobs)
1406        || jobs
1407            .values()
1408            .any(|read| read.averages.is_empty() || read.held() > 0)
1409    {
1410        return CpuStep::Keep;
1411    }
1412    let Some((job, heaviest)) = jobs
1413        .iter()
1414        .map(|(job, read)| (*job, spread(read.averages.clone()).p95))
1415        .max_by_key(|(_, p95)| *p95)
1416    else {
1417        return CpuStep::Keep;
1418    };
1419    let need = heaviest.saturating_mul(HEADROOM).max(peak);
1420    let target = u32::try_from(need.div_ceil(1000))
1421        .unwrap_or(u32::MAX)
1422        .max(1);
1423    if target >= current {
1424        return CpuStep::Keep;
1425    }
1426    let to = target.max(current.saturating_sub(step)).max(1);
1427    let mut reason = format!(
1428        "{runs} runs at its {current} CPUs peaked at {} CPUs at most, none held back by the cap, and the heaviest job, {}, \
1429         averaged {} CPUs (p95); the larger of that peak and twice that average is {target}",
1430        cores(peak),
1431        job_named(job),
1432        cores(heaviest)
1433    );
1434    if to != target {
1435        let _ = write!(
1436            reason,
1437            "; a step toward {target}, at most {}% a change",
1438            config.sizing.max_step
1439        );
1440    }
1441    CpuStep::To(Cpus { to, reason })
1442}