1use std::collections::BTreeSet;
7
8use crate::admission::Key;
9use crate::admission::recent::{self, Counted, Ending, JobRuns, RECENT_RUNS, Recent, SIZING_TIMES};
10use crate::api::{self, Peaks, Resize, RevertCause, SizingRevert, job_title};
11use crate::config::{Admission, Config};
12
13use super::{Capped, History, MemorySet, Ran, spread};
14
15pub const HISTORY: u64 = 30 * 24 * 3600 * 1000;
17
18pub const CHANGES_READ: u64 = 2 * HISTORY;
21
22pub const FAILED_RUNS: usize = 3;
24
25pub(super) fn windows(config: &Config, history: &History, now: u64) -> Recent {
27 let size = SIZING_TIMES.saturating_mul(RECENT_RUNS.max(u64::from(config.sizing.min_jobs)));
28 let mut recent = Recent::new(size);
29 let start = now.saturating_sub(HISTORY);
30 for (run, ending) in recent::rows(history) {
31 if run.queued_at < start {
32 continue;
33 }
34 recent.read(
35 run,
36 &run.profile,
37 ending,
38 cap_of(config, run),
39 recent::finished(run),
40 );
41 }
42 recent
43}
44
45fn cap_of(config: &Config, run: &Ran) -> u64 {
47 run.memory_cap
48 .or_else(|| config.profiles.get(&run.profile).map(|p| p.memory.0))
49 .unwrap_or(u64::MAX)
50}
51
52pub(super) fn peaks<'run>(
55 runs: impl Iterator<Item = &'run Counted>,
56 on_cap: u64,
57 cpus: u64,
58) -> Peaks {
59 let runs: Vec<&Counted> = runs.collect();
60 let cpu: Vec<u64> = runs
61 .iter()
62 .filter(|run| run.cpu_cap == Some(cpus))
63 .map(|run| run.cpu_peak)
64 .collect();
65 Peaks {
66 jobs: runs.len() as u64,
67 at_cap: runs.iter().filter(|run| run.capped.met()).count() as u64,
68 on_cap,
69 memory: spread(runs.iter().map(|run| run.peak).collect()),
70 cpu_runs: cpu.len() as u64,
71 cpu: spread(cpu),
72 }
73}
74
75fn run_need(run: &Counted, margin: u32) -> u64 {
78 if run.capped.met() {
79 api::need(run.peak.max(run.cap), margin)
80 } else {
81 api::need(run.peak, margin).min(run.cap)
82 }
83}
84
85pub(super) fn floor(runs: &JobRuns, margin: u32) -> u64 {
87 runs.window()
88 .map(|run| run_need(run, margin))
89 .max()
90 .unwrap_or(0)
91}
92
93fn larger_caps(runs: &JobRuns, margin: u32, above: u64) -> u64 {
97 runs.window()
98 .filter(|run| run.cap > above && run.capped != Capped::Assumed)
99 .map(|run| run_need(run, margin).min(run.cap))
100 .max()
101 .unwrap_or(0)
102}
103
104pub(super) fn on_cap(runs: &JobRuns, now: u64) -> bool {
106 runs.held(now, None).is_some()
107}
108
109pub(super) fn took(set: &[MemorySet], profile: &str, memory: u64, now: u64) -> u64 {
112 let mut since = None;
113 let mut seen = false;
114 for event in set.iter().filter(|event| event.profile == profile) {
115 seen = true;
116 if event.memory != memory {
117 since = None;
118 } else if since.is_none() {
119 since = Some(event.at);
120 }
121 }
122 since.unwrap_or(if seen { now } else { 0 })
123}
124
125pub(super) fn raise_need(
130 history: &History,
131 recent: &Recent,
132 profile: &str,
133 cap: u64,
134 took: u64,
135 margin: u32,
136 key: Option<&Key>,
137) -> u64 {
138 history
139 .jobs
140 .iter()
141 .chain(&history.cancelled)
142 .filter(|run| {
143 run.profile == profile
144 && run.memory_cap == Some(cap)
145 && recent::finished(run) >= took
146 && run.capped(cap) == Capped::Met
147 && key.is_none_or(|key| Key::of(run) == *key)
148 })
149 .map(|run| {
150 let larger = recent
151 .job(&Key::of(run), profile)
152 .map_or(0, |runs| larger_caps(runs, margin, cap));
153 api::need(run.memory_peak.max(cap), margin).max(larger)
154 })
155 .max()
156 .unwrap_or(0)
157}
158
159fn too_far<'change>(
162 history: &'change History,
163 profile: &str,
164) -> impl Iterator<Item = (&'change api::SizingChange, SizingRevert)> {
165 history
166 .changes
167 .iter()
168 .filter(move |change| change.profile == profile && change.to.memory < change.from.memory)
169 .filter_map(move |change| {
170 cause(history, profile, change).map(|(job, cause)| {
171 (
172 change,
173 SizingRevert {
174 from: change.to.memory,
175 to: change.from.memory,
176 job,
177 cause,
178 },
179 )
180 })
181 })
182}
183
184fn cause(
186 history: &History,
187 profile: &str,
188 lowering: &api::SizingChange,
189) -> Option<(String, RevertCause)> {
190 let (lowered, from) = (lowering.to.memory, lowering.from.memory);
191 let since = lowering.at.saturating_mul(1000);
192 let after: Vec<(&Ran, Ending)> = recent::rows(history)
193 .into_iter()
194 .filter(|(run, _)| {
195 run.profile == profile
196 && run.memory_cap == Some(lowered)
197 && recent::finished(run) >= since
198 })
199 .collect();
200 let named = |run: &Ran| {
201 format!(
202 "{} {}",
203 run.repository,
204 job_title(run.workflow.as_deref(), &run.name, None)
205 )
206 };
207 if let Some((run, _)) = after
208 .iter()
209 .find(|(run, _)| run.capped(lowered) == Capped::Met)
210 {
211 return Some((named(run), RevertCause::Met));
212 }
213 let jobs: BTreeSet<Key> = after.iter().map(|(run, _)| Key::of(run)).collect();
214 for key in jobs {
215 let succeeded_at_from = history.jobs.iter().any(|run| {
218 run.profile == profile
219 && Key::of(run) == key
220 && run.succeeded
221 && run.memory_cap == Some(from)
222 });
223 if !succeeded_at_from {
224 continue;
225 }
226 let mut row: Vec<(u64, Option<&str>)> = Vec::new();
227 for (run, ending) in after.iter().filter(|(run, _)| Key::of(run) == key) {
228 if *ending == Ending::Cancelled {
229 continue;
230 }
231 if run.succeeded {
232 row.clear();
233 } else if run.failed && !row.iter().any(|(id, _)| *id == run.run_id) {
234 row.push((run.run_id, run.branch.as_deref()));
235 }
236 let branches: BTreeSet<Option<&str>> = row.iter().map(|(_, branch)| *branch).collect();
237 if row.len() >= FAILED_RUNS && branches.len() >= 2 {
238 return Some((
239 named(run),
240 RevertCause::Failed {
241 runs: u32::try_from(row.len()).unwrap_or(u32::MAX),
242 },
243 ));
244 }
245 }
246 }
247 None
248}
249
250pub(super) fn revert(
257 config: &Config,
258 history: &History,
259 profile: &str,
260 current: Resize,
261) -> Option<(SizingRevert, Resize)> {
262 let mut memory_changes = history
263 .changes
264 .iter()
265 .filter(|change| change.profile == profile && change.to.memory != change.from.memory)
266 .rev();
267 let mut chain = vec![memory_changes.next()?];
269 while let Some(raise) = chain
270 .last()
271 .filter(|change| change.to.memory > change.from.memory)
272 {
273 let before = memory_changes.next()?;
274 if before.to.memory != raise.from.memory {
275 return None;
276 }
277 chain.push(before);
278 }
279 let (last, lowering) = (chain[0], chain[chain.len() - 1]);
280 if last.to.memory != current.memory || last.to.memory >= lowering.from.memory {
281 return None;
282 }
283 let events: Vec<&MemorySet> = history
284 .memory_set
285 .iter()
286 .filter(|event| event.profile == profile && event.at >= lowering.at.saturating_mul(1000))
287 .collect();
288 if !events
289 .iter()
290 .any(|event| event.at >= last.at.saturating_mul(1000))
291 {
292 return None;
293 }
294 let ran_the_chain = events.iter().all(|event| {
296 chain
297 .iter()
298 .find(|change| change.at.saturating_mul(1000) <= event.at)
299 .is_some_and(|change| change.to.memory == event.memory)
300 });
301 if !ran_the_chain {
302 return None;
303 }
304 let (_, revert) = too_far(history, profile).find(|(change, _)| *change == lowering)?;
305 let slots = match config.host.admission {
306 Admission::Caps => lowering.from.slots,
307 Admission::Measured => current.slots,
308 };
309 Some((
310 revert,
311 Resize {
312 memory: lowering.from.memory,
313 slots,
314 cpus: None,
315 },
316 ))
317}
318
319pub(super) fn revert_floor(history: &History, profile: &str) -> u64 {
322 too_far(history, profile)
323 .map(|(_, revert)| revert.to)
324 .max()
325 .unwrap_or(0)
326}