Skip to main content

runnerd_core/sizing/
memory.rs

1//! What a profile's runs say about its `memory` (ADR 0070): each job's sizing window (`admission::recent`), what a
2//! raise needs (`raise_need`: only runs that met the current cap since the profile took it), what a lowering must
3//! stay above (`lower_floor`: every windowed run's need), and a lowering auto sizing made that went too far, which is
4//! reverted. Pure; `advise` and auto sizing read it, never a client.
5
6use std::collections::BTreeSet;
7
8use crate::admission::Key;
9use crate::admission::recent::{self, Counted, Ending, JobRuns, RECENT_RUNS, Recent, SIZING_TIMES};
10use crate::api::{self, Peaks, Resize, RevertCause, SizingRevert, job_title};
11use crate::config::{Admission, Config};
12
13use super::{Capped, History, MemorySet, Ran, spread};
14
15/// How far back the windows read, in ms: 30 days, whatever `since` a client asked for.
16pub const HISTORY: u64 = 30 * 24 * 3600 * 1000;
17
18/// How far back auto sizing's recorded changes are read for a revert, in ms: 60 days, so a lowering up to 30 days
19/// before the run that met its cap is found while that run is in the history.
20pub const CHANGES_READ: u64 = 2 * HISTORY;
21
22/// A lowering is reverted after this many failures in a row at it, of a job that succeeded at the cap it came from.
23pub const FAILED_RUNS: usize = 3;
24
25/// Every job's sizing window from `history`'s last 30 days to `now` (ms).
26pub(super) fn windows(config: &Config, history: &History, now: u64) -> Recent {
27    let size = SIZING_TIMES.saturating_mul(RECENT_RUNS.max(u64::from(config.sizing.min_jobs)));
28    let mut recent = Recent::new(size);
29    let start = now.saturating_sub(HISTORY);
30    for (run, ending) in recent::rows(history) {
31        if run.queued_at < start {
32            continue;
33        }
34        recent.read(
35            run,
36            &run.profile,
37            ending,
38            cap_of(config, run),
39            recent::finished(run),
40        );
41    }
42    recent
43}
44
45/// The cap a run's container had, else its profile's now.
46fn cap_of(config: &Config, run: &Ran) -> u64 {
47    run.memory_cap
48        .or_else(|| config.profiles.get(&run.profile).map(|p| p.memory.0))
49        .unwrap_or(u64::MAX)
50}
51
52/// The peaks of the runs in some windows; `on_cap` how many of their jobs are on a hold at the cap. CPU peaks only of
53/// the runs at `cpus`, the profile's now, in millicores: one at a larger cap may peak past it.
54pub(super) fn peaks<'run>(
55    runs: impl Iterator<Item = &'run Counted>,
56    on_cap: u64,
57    cpus: u64,
58) -> Peaks {
59    let runs: Vec<&Counted> = runs.collect();
60    let cpu: Vec<u64> = runs
61        .iter()
62        .filter(|run| run.cpu_cap == Some(cpus))
63        .map(|run| run.cpu_peak)
64        .collect();
65    Peaks {
66        jobs: runs.len() as u64,
67        at_cap: runs.iter().filter(|run| run.capped.met()).count() as u64,
68        on_cap,
69        memory: spread(runs.iter().map(|run| run.peak).collect()),
70        cpu_runs: cpu.len() as u64,
71        cpu: spread(cpu),
72    }
73}
74
75/// What a windowed run says its job needs at least: above its cap when it met it, else its peak's need, at most
76/// its cap, since it ran clear of it.
77fn run_need(run: &Counted, margin: u32) -> u64 {
78    if run.capped.met() {
79        api::need(run.peak.max(run.cap), margin)
80    } else {
81        api::need(run.peak, margin).min(run.cap)
82    }
83}
84
85/// The largest need of a job's windowed runs.
86pub(super) fn floor(runs: &JobRuns, margin: u32) -> u64 {
87    runs.window()
88        .map(|run| run_need(run, margin))
89        .max()
90        .unwrap_or(0)
91}
92
93/// What a job's windowed runs at caps above `above` say it needs, each at most its own cap: a raise goes back at
94/// most to a cap the job ran at, and a need above that comes from a run that meets it. A run recorded without the
95/// local counter (`Capped::Assumed`) says nothing here: it never raises a cap.
96fn larger_caps(runs: &JobRuns, margin: u32, above: u64) -> u64 {
97    runs.window()
98        .filter(|run| run.cap > above && run.capped != Capped::Assumed)
99        .map(|run| run_need(run, margin).min(run.cap))
100        .max()
101        .unwrap_or(0)
102}
103
104/// Whether `held` is on its cap for sizing: a hold `recent::CLEAN_RUNS` clear runs have not ended, however old.
105pub(super) fn on_cap(runs: &JobRuns, now: u64) -> bool {
106    runs.held(now, None).is_some()
107}
108
109/// When the daemon started running `profile` with `memory`, in ms, by `set`: its first event since the last one
110/// with other `memory`. `now` when its latest has other `memory`; 0 with no events.
111pub(super) fn took(set: &[MemorySet], profile: &str, memory: u64, now: u64) -> u64 {
112    let mut since = None;
113    let mut seen = false;
114    for event in set.iter().filter(|event| event.profile == profile) {
115        seen = true;
116        if event.memory != memory {
117            since = None;
118        } else if since.is_none() {
119            since = Some(event.at);
120        }
121    }
122    since.unwrap_or(if seen { now } else { 0 })
123}
124
125/// What a raise of `profile`, whose `memory` is `cap` since `took` (ms), needs from the job `key`, `None` for every
126/// job: the largest of `need(max(peak, cap))` over its runs that met `cap` since then, and of that job's need at
127/// larger caps (`larger_caps`), since its last sample before a kill rarely shows what it needed. 0 without such a
128/// run.
129pub(super) fn raise_need(
130    history: &History,
131    recent: &Recent,
132    profile: &str,
133    cap: u64,
134    took: u64,
135    margin: u32,
136    key: Option<&Key>,
137) -> u64 {
138    history
139        .jobs
140        .iter()
141        .chain(&history.cancelled)
142        .filter(|run| {
143            run.profile == profile
144                && run.memory_cap == Some(cap)
145                && recent::finished(run) >= took
146                && run.capped(cap) == Capped::Met
147                && key.is_none_or(|key| Key::of(run) == *key)
148        })
149        .map(|run| {
150            let larger = recent
151                .job(&Key::of(run), profile)
152                .map_or(0, |runs| larger_caps(runs, margin, cap));
153            api::need(run.memory_peak.max(cap), margin).max(larger)
154        })
155        .max()
156        .unwrap_or(0)
157}
158
159/// The lowerings auto sizing made of `profile` that went too far, oldest first: since each, a run at the cap it
160/// lowered to met it, or a job that succeeded at the cap it came from failed `FAILED_RUNS` times in a row at it.
161fn too_far<'change>(
162    history: &'change History,
163    profile: &str,
164) -> impl Iterator<Item = (&'change api::SizingChange, SizingRevert)> {
165    history
166        .changes
167        .iter()
168        .filter(move |change| change.profile == profile && change.to.memory < change.from.memory)
169        .filter_map(move |change| {
170            cause(history, profile, change).map(|(job, cause)| {
171                (
172                    change,
173                    SizingRevert {
174                        from: change.to.memory,
175                        to: change.from.memory,
176                        job,
177                        cause,
178                    },
179                )
180            })
181        })
182}
183
184/// What shows that `lowering` of `profile` went too far, if anything: the job and how.
185fn cause(
186    history: &History,
187    profile: &str,
188    lowering: &api::SizingChange,
189) -> Option<(String, RevertCause)> {
190    let (lowered, from) = (lowering.to.memory, lowering.from.memory);
191    let since = lowering.at.saturating_mul(1000);
192    let after: Vec<(&Ran, Ending)> = recent::rows(history)
193        .into_iter()
194        .filter(|(run, _)| {
195            run.profile == profile
196                && run.memory_cap == Some(lowered)
197                && recent::finished(run) >= since
198        })
199        .collect();
200    let named = |run: &Ran| {
201        format!(
202            "{} {}",
203            run.repository,
204            job_title(run.workflow.as_deref(), &run.name, None)
205        )
206    };
207    if let Some((run, _)) = after
208        .iter()
209        .find(|(run, _)| run.capped(lowered) == Capped::Met)
210    {
211        return Some((named(run), RevertCause::Met));
212    }
213    let jobs: BTreeSet<Key> = after.iter().map(|(run, _)| Key::of(run)).collect();
214    for key in jobs {
215        // Before the revert only runs from before the lowering ran at `from`; after it, its runs at `from` since keep
216        // the floor while the failures are in the history.
217        let succeeded_at_from = history.jobs.iter().any(|run| {
218            run.profile == profile
219                && Key::of(run) == key
220                && run.succeeded
221                && run.memory_cap == Some(from)
222        });
223        if !succeeded_at_from {
224            continue;
225        }
226        let mut row: Vec<(u64, Option<&str>)> = Vec::new();
227        for (run, ending) in after.iter().filter(|(run, _)| Key::of(run) == key) {
228            if *ending == Ending::Cancelled {
229                continue;
230            }
231            if run.succeeded {
232                row.clear();
233            } else if run.failed && !row.iter().any(|(id, _)| *id == run.run_id) {
234                row.push((run.run_id, run.branch.as_deref()));
235            }
236            let branches: BTreeSet<Option<&str>> = row.iter().map(|(_, branch)| *branch).collect();
237            if row.len() >= FAILED_RUNS && branches.len() >= 2 {
238                return Some((
239                    named(run),
240                    RevertCause::Failed {
241                        runs: u32::try_from(row.len()).unwrap_or(u32::MAX),
242                    },
243                ));
244            }
245        }
246    }
247    None
248}
249
250/// The revert auto sizing makes now of `profile`, whose `memory` and `slots` are `current`, if any: its latest
251/// recorded change of `memory` lowered it to what it has, or raised it partway back toward where a lowering came from
252/// (each raise from where the change before it left `memory`, as when `[host] memory_ceiling` held a revert short,
253/// ADR 0086), the daemon has run each since (`set`) and no other `memory` in between, and that lowering went too far.
254/// A hand edit anywhere along the chain breaks it, even one set back to the same `memory`. Changes of only `slots` or `cpus` since do not hide it. Under caps the slots go back to what they were
255/// before the lowering, so the memory fits the slice.
256pub(super) fn revert(
257    config: &Config,
258    history: &History,
259    profile: &str,
260    current: Resize,
261) -> Option<(SizingRevert, Resize)> {
262    let mut memory_changes = history
263        .changes
264        .iter()
265        .filter(|change| change.profile == profile && change.to.memory != change.from.memory)
266        .rev();
267    // Newest first: the latest change, then the raises before it, back to the lowering.
268    let mut chain = vec![memory_changes.next()?];
269    while let Some(raise) = chain
270        .last()
271        .filter(|change| change.to.memory > change.from.memory)
272    {
273        let before = memory_changes.next()?;
274        if before.to.memory != raise.from.memory {
275            return None;
276        }
277        chain.push(before);
278    }
279    let (last, lowering) = (chain[0], chain[chain.len() - 1]);
280    if last.to.memory != current.memory || last.to.memory >= lowering.from.memory {
281        return None;
282    }
283    let events: Vec<&MemorySet> = history
284        .memory_set
285        .iter()
286        .filter(|event| event.profile == profile && event.at >= lowering.at.saturating_mul(1000))
287        .collect();
288    if !events
289        .iter()
290        .any(|event| event.at >= last.at.saturating_mul(1000))
291    {
292        return None;
293    }
294    // The daemon ran each link's memory until the next: any other, even one set back by hand since, breaks it.
295    let ran_the_chain = events.iter().all(|event| {
296        chain
297            .iter()
298            .find(|change| change.at.saturating_mul(1000) <= event.at)
299            .is_some_and(|change| change.to.memory == event.memory)
300    });
301    if !ran_the_chain {
302        return None;
303    }
304    let (_, revert) = too_far(history, profile).find(|(change, _)| *change == lowering)?;
305    let slots = match config.host.admission {
306        Admission::Caps => lowering.from.slots,
307        Admission::Measured => current.slots,
308    };
309    Some((
310        revert,
311        Resize {
312            memory: lowering.from.memory,
313            slots,
314            cpus: None,
315        },
316    ))
317}
318
319/// The floor reverts set on `profile`'s lowerings: the largest cap a lowering that went too far came from, while
320/// the run that showed it is in the history.
321pub(super) fn revert_floor(history: &History, profile: &str) -> u64 {
322    too_far(history, profile)
323        .map(|(_, revert)| revert.to)
324        .max()
325        .unwrap_or(0)
326}