runnerd_core/sizing/
auto.rs

1//! Auto sizing: which profiles' `memory` and `slots` the daemon changes now, from the advice and within the limits
2//! `[sizing]` and each profile set. Pure: the daemon reads the history and when each profile last changed, writes
3//! each change through the config check as `PUT /v1/config` does, and records it. Never lowers a profile a run met
4//! the cap of, never sizes a profile from fewer than `[sizing] min_jobs` jobs, never leaves a profile outside its
5//! bounds, and sizes each profile against the budget the changes before it in the same look leave.
6
7use std::collections::BTreeMap;
8
9use crate::api::{Advice, ProfileSizing, Resize, SizingChange};
10use crate::config::{Admission, Config, Profile};
11
12use super::{GRANULE, History, Window, advise};
13
14/// What auto sizing reads besides the config and the history.
15#[derive(Debug, Clone, Copy)]
16pub struct Context<'run> {
17    /// The slice's `MemoryMax`; `None` when it is unlimited.
18    pub memory_max: Option<u64>,
19    /// The history's window; `now` is when the changes are made.
20    pub window: Window,
21    /// When auto sizing last changed each profile, in seconds, by name, however long ago.
22    pub last: &'run BTreeMap<String, u64>,
23    /// How many runners each profile runs now, by name.
24    pub running: &'run BTreeMap<String, u32>,
25}
26
27/// The changes to make now, in order: one per profile whose advice is to change it, that no change in the last
28/// `[sizing] interval` touched. Empty unless `[sizing] auto` is on. Each moves `memory` toward the advice by at most
29/// `[sizing] max_step` percent, within the profile's `min_memory` and `max_memory`. Under `caps` admission `slots`
30/// follow the memory as the advice has them, at most `max_slots`, and never under what runs; under `measured`, a
31/// profile whose jobs wait gets one more slot, up to `max_slots`, when every profile's slots fit the slice at their
32/// jobs' p95 peaks. Each change is sized as if the ones before it were written: they take budget the later ones
33/// cannot.
34pub fn changes(config: &Config, history: &History, context: Context<'_>) -> Vec<SizingChange> {
35    let rules = &config.sizing;
36    if !rules.auto {
37        return Vec::new();
38    }
39    let window = context.window;
40    let sizing = advise(config, context.memory_max, history, window);
41    let now = window.now / 1000;
42    let interval = rules.interval.as_secs();
43    let mut budget = Budget {
44        caps: config.memory_caps().map_or(u64::MAX, |caps| caps.0),
45        slots: sizing
46            .profiles
47            .iter()
48            .map(|advised| (advised.name.clone(), advised.slots))
49            .collect(),
50    };
51    let mut changes = Vec::new();
52    for advised in &sizing.profiles {
53        let Some(profile) = config.profiles.get(&advised.name) else {
54            continue;
55        };
56        if context
57            .last
58            .get(&advised.name)
59            .is_some_and(|at| now < at.saturating_add(interval))
60        {
61            continue;
62        }
63        let from = Resize {
64            memory: profile.memory.0,
65            slots: profile.slots,
66        };
67        let mut reasons = Vec::new();
68        let memory = memory(
69            config,
70            profile,
71            advised.advice,
72            met_cap(history, &advised.name),
73        );
74        if memory != from.memory {
75            reasons.push(sizing.memory_line(advised));
76        }
77        let running = context.running.get(&advised.name).copied().unwrap_or(0);
78        let slots = match config.host.admission {
79            Admission::Caps => caps_slots(profile, advised, memory, &context, &budget, running),
80            Admission::Measured => {
81                let slots = measured_slots(config, &sizing.profiles, advised, &context, &budget);
82                if slots > from.slots {
83                    reasons.push(format!(
84                        "jobs waited {}s for a slot (p95) and slots were busy {}%; {slots} slots at their jobs' \
85                         p95 peaks fit the slice beside the other profiles'",
86                        advised.wait.p95, advised.busy
87                    ));
88                }
89                slots
90            }
91        };
92        let to = Resize { memory, slots };
93        if to == from || slots == 0 || !within(profile, from, to) {
94            continue;
95        }
96        budget.caps = budget
97            .caps
98            .saturating_sub(from.memory.saturating_mul(u64::from(from.slots)))
99            .saturating_add(to.memory.saturating_mul(u64::from(to.slots)));
100        budget.slots.insert(advised.name.clone(), to.slots);
101        changes.push(SizingChange {
102            at: now,
103            profile: advised.name.clone(),
104            from,
105            to,
106            reason: reasons.join("; "),
107            uncertain: false,
108        });
109    }
110    changes
111}
112
113/// The budget as the changes picked so far leave it.
114struct Budget {
115    /// `slots × memory` over all profiles, with the mirror.
116    caps: u64,
117    /// Each profile's slots.
118    slots: BTreeMap<String, u32>,
119}
120
121/// Whether `to` keeps the profile within its bounds: `memory` within `min_memory` and `max_memory`, and `slots` at
122/// most `max_slots` when they grow.
123fn within(profile: &Profile, from: Resize, to: Resize) -> bool {
124    profile.min_memory.is_none_or(|min| to.memory >= min.0)
125        && profile.max_memory.is_none_or(|max| to.memory <= max.0)
126        && (to.slots <= from.slots || profile.max_slots.is_some_and(|max| to.slots <= max))
127}
128
129/// Whether a run of the profile in the history, cancelled ones too, was killed for memory or met its cap.
130fn met_cap(history: &History, profile: &str) -> bool {
131    history
132        .jobs
133        .iter()
134        .chain(&history.cancelled)
135        .filter(|job| job.profile == profile)
136        .any(|job| job.oom_killed || job.memory_cap.is_some_and(|cap| job.met_cap(cap)))
137}
138
139/// The profile's next `memory`: the advice's, at most `max_step` percent away and within its bounds, in steps of
140/// `GRANULE` toward the advice. A lowering waits while a run met the cap.
141fn memory(config: &Config, profile: &Profile, advice: Advice, met_cap: bool) -> u64 {
142    let current = profile.memory.0;
143    let step = current.saturating_mul(u64::from(config.sizing.max_step)) / 100;
144    match advice {
145        Advice::Lower { memory, .. } if !met_cap => {
146            let floor = current
147                .saturating_sub(step)
148                .div_ceil(GRANULE)
149                .saturating_mul(GRANULE);
150            let bound = profile.min_memory.map_or(0, |min| min.0);
151            memory.max(floor).max(bound).min(current)
152        }
153        Advice::Raise { memory, .. } => {
154            let ceiling = current.saturating_add(step) / GRANULE * GRANULE;
155            let bound = profile.max_memory.map_or(u64::MAX, |max| max.0);
156            memory.min(ceiling).min(bound).max(current)
157        }
158        _ => current,
159    }
160}
161
162/// Under `caps`: as many slots as the advice gives for the new memory. A lowering whose jobs waited adds as many as
163/// the memory the slots take now holds, up to `max_slots`; a raise keeps as many as the slice holds beside the other
164/// profiles, and 0, no change, while more than that run. Otherwise the same.
165fn caps_slots(
166    profile: &Profile,
167    advised: &ProfileSizing,
168    memory: u64,
169    context: &Context<'_>,
170    budget: &Budget,
171    running: u32,
172) -> u32 {
173    let current = profile.slots;
174    let fit = |room: u64| u32::try_from(room / memory.max(1)).unwrap_or(u32::MAX);
175    if memory < profile.memory.0 && advised.wants_slots {
176        let share = profile.memory.0.saturating_mul(u64::from(current));
177        return fit(share)
178            .min(profile.max_slots.unwrap_or(current))
179            .max(current);
180    }
181    if memory > profile.memory.0 {
182        let Some(memory_max) = context.memory_max else {
183            return current;
184        };
185        let share = profile.memory.0.saturating_mul(u64::from(current));
186        let room = memory_max.saturating_sub(budget.caps.saturating_sub(share));
187        // Fewer slots than run would leave the running ones over the budget: wait for some to finish.
188        let slots = fit(room).min(current);
189        return if slots < running { 0 } else { slots };
190    }
191    current
192}
193
194/// Under `measured`: one more slot when its jobs waited and every profile's slots, this one's with the new one, fit
195/// the slice beside the mirror at their jobs' p95 peaks (at its cap, a profile with fewer than `min_jobs` jobs), up
196/// to `max_slots`, with the slots the changes before it gave. Otherwise the same.
197fn measured_slots(
198    config: &Config,
199    profiles: &[ProfileSizing],
200    advised: &ProfileSizing,
201    context: &Context<'_>,
202    budget: &Budget,
203) -> u32 {
204    let current = advised.slots;
205    let wants = advised.wants_slots && !matches!(advised.advice, Advice::TooFew { .. });
206    let Some(max_slots) = config
207        .profiles
208        .get(&advised.name)
209        .and_then(|profile| profile.max_slots)
210        .filter(|max| wants && *max > current)
211    else {
212        return current;
213    };
214    let Some(memory_max) = context.memory_max else {
215        return (current + 1).min(max_slots);
216    };
217    let peaks: u64 = profiles
218        .iter()
219        .map(|other| {
220            let peak = if matches!(other.advice, Advice::TooFew { .. }) {
221                other.memory
222            } else {
223                other.peaks.memory.p95
224            };
225            let slots = if other.name == advised.name {
226                current + 1
227            } else {
228                budget
229                    .slots
230                    .get(&other.name)
231                    .copied()
232                    .unwrap_or(other.slots)
233            };
234            peak.saturating_mul(u64::from(slots))
235        })
236        .fold(config.mirror_memory(), u64::saturating_add);
237    if peaks <= memory_max {
238        current + 1
239    } else {
240        current
241    }
242}
runnerd_core/sizing/auto.rs

runnerd_core/sizing/
auto.rs
