1use std::collections::{BTreeMap, BTreeSet};
8
9use crate::api::{
10 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
11 RunLink, Severity, WaitSpread, WeekWaits,
12};
13use crate::config::{ByteSize, Config};
14use crate::sizing::spread;
15
16const SECOND: u64 = 1000;
17const DAY: u64 = 24 * 3600 * SECOND;
18const WEEK: u64 = 7 * DAY;
19
20const RECENT: usize = 5;
22const SLOWER_PERCENT: u64 = 20;
24const SLOWER_SECONDS: u64 = 30;
26
27const CAP_RUNS: usize = 10;
29const CAP_MIN_RUNS: usize = 5;
30const NEAR_CAP: u64 = 85;
32const NEAR_CAP_WARNING: usize = 2;
34const FLAKY_WARNING: u32 = 2;
36
37const WAITS_PERCENT: u64 = 50;
40const WAITS_SECONDS: u64 = 60;
41const WAITS_MIN_JOBS: u32 = 5;
42const WAITS_ONE_IN_20: u32 = 20;
44
45const THROTTLED_MIN_RUNS: u32 = 2;
47
48#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct Run {
51 pub id: u64,
53 pub repository: String,
55 pub workflow: Option<String>,
57 pub name: String,
59 pub branch: Option<String>,
61 pub url: Option<String>,
63 pub run_id: u64,
65 pub run_attempt: u32,
67 pub state: JobState,
69 pub conclusion: Option<String>,
71 pub queued_at: u64,
73 pub started_at: Option<u64>,
75 pub finished_at: Option<u64>,
77 pub runner_started_at: Option<u64>,
79 pub profile: Option<String>,
81 pub memory_peak: Option<u64>,
83 pub build_cache: Option<api::BuildCacheCounts>,
85 pub sock_throttled: Option<u64>,
87}
88
89impl Run {
90 fn finished(&self) -> u64 {
91 self.finished_at.unwrap_or(0)
92 }
93
94 fn succeeded(&self) -> bool {
95 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
96 }
97
98 fn failed(&self) -> bool {
100 self.state == JobState::Lost || self.failed_itself()
101 }
102
103 fn failed_itself(&self) -> bool {
105 self.state == JobState::Completed
106 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
107 }
108
109 fn took(&self) -> Option<u64> {
111 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
112 }
113
114 fn waited(&self, kept_since: u64) -> Option<u64> {
117 Some(
118 self.started_at?
119 .saturating_sub(self.queued_at.max(kept_since))
120 / SECOND,
121 )
122 }
123
124 fn link(&self) -> RunLink {
125 RunLink {
126 job: self.id,
127 run_id: self.run_id,
128 run_attempt: self.run_attempt,
129 branch: self.branch.clone(),
130 url: self.url.clone(),
131 at: self.finished() / SECOND,
132 }
133 }
134}
135
136#[derive(Debug, Clone, Default, PartialEq, Eq)]
138pub struct History {
139 pub runs: Vec<Run>,
141 pub kept_since: u64,
144}
145
146type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
148
149fn key_of(run: &Run) -> JobKey<'_> {
150 (&run.repository, run.workflow.as_deref(), &run.name)
151}
152
153pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
156 let runs = &history.runs;
157 let since = now.saturating_sub(u64::from(days) * DAY);
158 let window: Vec<&Run> = runs
159 .iter()
160 .filter(|run| {
161 matches!(run.state, JobState::Completed | JobState::Lost)
162 && run
163 .finished_at
164 .is_some_and(|at| (since..=now).contains(&at))
165 })
166 .collect();
167 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
168 for run in &window {
169 jobs.entry(key_of(run)).or_default().push(run);
170 }
171 let mut found = Vec::new();
172 for (key, runs) in &mut jobs {
173 runs.sort_by_key(|run| (run.finished(), run.id));
174 found.extend(slower(*key, runs));
175 found.extend(flaky(*key, runs));
176 found.extend(near_cap(*key, runs, config));
177 found.extend(cold_cache(*key, runs));
178 }
179 found.extend(throttled(&window));
180 let waits = waits(history, now);
181 found.extend(waits.iter().filter_map(waits_insight));
182 found.sort_by(|a, b| {
183 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
184 b.severity,
185 &b.repository,
186 &b.workflow,
187 &b.job,
188 &b.key,
189 ))
190 });
191 Insights {
192 since: since / SECOND,
193 until: now / SECOND,
194 insights: found,
195 waits,
196 }
197}
198
199pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
202 let runs = &history.runs;
203 let since = now.saturating_sub(DAY);
204 let finished: Vec<&Run> = runs
205 .iter()
206 .filter(|run| {
207 matches!(run.state, JobState::Completed | JobState::Lost)
208 && run.finished_at.is_some_and(|at| at > since && at <= now)
209 })
210 .collect();
211 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
212 .insights
213 .into_iter()
214 .map(|insight| insight.key)
215 .collect();
216 let new = insights(history, config, api::INSIGHT_DAYS, now)
217 .insights
218 .into_iter()
219 .filter(|insight| !before.contains(&insight.key))
220 .collect();
221 let mut started: Vec<&Run> = runs
222 .iter()
223 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
224 .collect();
225 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
226 Digest {
227 since: since / SECOND,
228 until: now / SECOND,
229 finished: finished.len() as u64,
230 failed: finished.iter().filter(|run| run.failed()).count() as u64,
231 new,
232 waits: started
233 .into_iter()
234 .take(3)
235 .map(|run| LongWait {
236 job: run.id,
237 repository: run.repository.clone(),
238 workflow: run.workflow.clone(),
239 name: run.name.clone(),
240 waited: run.waited(history.kept_since).unwrap_or(0),
241 url: run.url.clone(),
242 })
243 .collect(),
244 savings: None,
245 }
246}
247
248fn subject((repository, workflow, name): JobKey<'_>) -> String {
250 match workflow {
251 Some(workflow) => format!("{repository}: {workflow} / {name}"),
252 None => format!("{repository}: {name}"),
253 }
254}
255
256fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
257 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
258}
259
260fn insight(
261 key: JobKey<'_>,
262 kind: &str,
263 severity: Severity,
264 sentence: String,
265 found: InsightKind,
266 runs: Vec<RunLink>,
267) -> Insight {
268 Insight {
269 key: key_text(kind, key),
270 severity,
271 repository: key.0.to_owned(),
272 workflow: key.1.map(str::to_owned),
273 job: Some(key.2.to_owned()),
274 sentence,
275 kind: found,
276 runs,
277 }
278}
279
280fn median(values: &[u64]) -> u64 {
282 let mut values = values.to_vec();
283 values.sort_unstable();
284 let middle = values.len() / 2;
285 match values.len() {
286 0 => 0,
287 len if len % 2 == 1 => values[middle],
288 _ => u64::midpoint(values[middle - 1], values[middle]),
289 }
290}
291
292fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
294 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
295 runs.iter()
296 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
297 .filter_map(|run| Some((*run, run.took()?)))
298 .collect()
299 };
300 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
301 for run in runs.iter().filter(|run| run.succeeded()) {
302 if let Some(branch) = &run.branch {
303 *branches.entry(branch).or_default() += 1;
304 }
305 }
306 let busiest = branches
309 .iter()
310 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
311 .map(|(branch, _)| *branch)?;
312 let (branch, timed) = ["main", "master"]
313 .into_iter()
314 .map(|branch| (branch, timed_on(branch)))
315 .find(|(_, timed)| timed.len() >= 2 * RECENT)
316 .unwrap_or_else(|| (busiest, timed_on(busiest)));
317 if timed.len() < 2 * RECENT {
318 return None;
319 }
320 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
321 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
322 let before = median(&took(baseline));
323 let now = median(&took(recent));
324 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
325 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
328 return None;
329 }
330 let mut since = None;
332 let mut fast_in_a_row = 0;
333 for (run, took) in &timed {
334 if *took >= threshold {
335 fast_in_a_row = 0;
336 since = since.or(Some(*run));
337 } else {
338 fast_in_a_row += 1;
339 if fast_in_a_row >= 2 {
340 since = None;
341 }
342 }
343 }
344 let since = since?;
345 let sentence = format!(
346 "{} on {branch} takes {} now, up from {}.",
347 subject(key),
348 span(now),
349 span(before)
350 );
351 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
352 Some(insight(
353 key,
354 "slower",
355 Severity::Warning,
356 sentence,
357 InsightKind::Slower {
358 branch: Some(branch.to_owned()),
359 before,
360 now,
361 since: Some(since.link()),
362 },
363 links,
364 ))
365}
366
367fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
369 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
370 for run in runs {
371 by_run.entry(run.run_id).or_default().push(run);
372 }
373 let mut failed_first: Vec<&Run> = by_run
374 .values()
375 .filter_map(|attempts| {
376 attempts.iter().copied().find(|failed| {
377 failed.failed_itself()
378 && attempts
379 .iter()
380 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
381 })
382 })
383 .collect();
384 if failed_first.is_empty() {
385 return None;
386 }
387 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
388 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
389 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
390 let sentence = format!(
391 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
392 subject(key)
393 );
394 let severity = if flaky >= FLAKY_WARNING {
396 Severity::Warning
397 } else {
398 Severity::Notice
399 };
400 Some(insight(
401 key,
402 "flaky",
403 severity,
404 sentence,
405 InsightKind::Flaky { flaky, total },
406 failed_first.iter().take(5).map(|run| run.link()).collect(),
407 ))
408}
409
410fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
413 let measured = runs
414 .iter()
415 .rev()
416 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
417 let name = measured.clone().next()?.profile.as_deref()?;
418 let measured: Vec<&Run> = measured
419 .filter(|run| run.profile.as_deref() == Some(name))
420 .take(CAP_RUNS)
421 .copied()
422 .collect();
423 if measured.len() < CAP_MIN_RUNS {
424 return None;
425 }
426 let profile = config.profiles.get(name)?;
427 let cap = profile.memory.0;
428 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
429 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
430 let peak = spread(peaks.clone()).p95;
431 if cap == 0 || !near(&peak) {
432 return None;
433 }
434 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
435 let auto = if !config.sizing.auto {
436 AutoNote::Off
437 } else if profile
438 .max_memory
439 .is_some_and(|max| profile.memory.0 >= max.0)
440 {
441 AutoNote::AtMax
442 } else {
443 AutoNote::Raises
444 };
445 let what = match auto {
446 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
447 AutoNote::AtMax => {
448 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
449 }
450 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
451 };
452 let sentence = format!(
453 "{} peaks at {}, {}% of {name}'s {}. {what}",
454 subject(key),
455 ByteSize(peak),
456 peak.saturating_mul(100) / cap,
457 ByteSize(cap)
458 );
459 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
461 Severity::Notice
462 } else {
463 Severity::Warning
464 };
465 Some(insight(
466 key,
467 "near-cap",
468 severity,
469 sentence,
470 InsightKind::NearCap {
471 profile: name.to_owned(),
472 peak,
473 cap,
474 auto,
475 },
476 measured.iter().take(5).map(|run| run.link()).collect(),
477 ))
478}
479
480const COLD_RUNS: usize = 5;
482const COLD_MIN_RUNS: usize = 3;
483const COLD_MIN_LOOKUPS: u64 = 10;
484const COLD_PERCENT: u64 = 10;
487
488fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
489 let writers: Vec<&Run> = runs
490 .iter()
491 .rev()
492 .filter(|run| {
493 run.build_cache
494 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
495 })
496 .take(COLD_RUNS)
497 .copied()
498 .collect();
499 if writers.len() < COLD_MIN_RUNS {
500 return None;
501 }
502 let mut total = api::BuildCacheCounts::default();
503 for run in &writers {
504 total.add(&run.build_cache.unwrap_or_default());
505 }
506 let percent = total.hit_percent()?;
507 if percent > COLD_PERCENT {
508 return None;
509 }
510 let branch = writers.first()?.branch.clone();
511 let sentence = format!(
512 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
513 differ from run to run.",
514 subject(key),
515 branch
516 .as_deref()
517 .map(|branch| format!(" on {branch}"))
518 .unwrap_or_default(),
519 writers.len()
520 );
521 Some(insight(
522 key,
523 "cold-cache",
524 Severity::Notice,
525 sentence,
526 InsightKind::ColdCache {
527 branch,
528 hits: total.hits,
529 lookups: total.lookups(),
530 },
531 writers.iter().map(|run| run.link()).collect(),
532 ))
533}
534
535fn throttled(window: &[&Run]) -> Vec<Insight> {
537 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
538 for run in window {
539 repositories.entry(&run.repository).or_default().push(run);
540 }
541 repositories
542 .into_iter()
543 .filter_map(|(repository, mut runs)| {
544 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
545 let hit: Vec<&Run> = runs
546 .iter()
547 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
548 .copied()
549 .collect();
550 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
551 if throttled < THROTTLED_MIN_RUNS {
552 return None;
553 }
554 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
555 Some(Insight {
556 key: format!("throttled:{repository}"),
557 severity: Severity::Warning,
558 repository: repository.to_owned(),
559 workflow: None,
560 job: None,
561 sentence: format!(
562 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
563 host, or a job at its cap, was short of memory."
564 ),
565 kind: InsightKind::Throttled { throttled, total },
566 runs: hit.iter().take(5).map(|run| run.link()).collect(),
567 })
568 })
569 .collect()
570}
571
572fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
576 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
577 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
578 for run in &history.runs {
579 let Some(started) = run.started_at.filter(|at| *at <= now) else {
580 continue;
581 };
582 let week = match now - started {
583 back if back < WEEK => 0,
584 back if back < 2 * WEEK && !before_kept => 1,
585 _ => continue,
586 };
587 weeks.entry(&run.repository).or_default()[week].push(run);
588 }
589 weeks
590 .into_iter()
591 .map(|(repository, [now, before])| RepositoryWaits {
592 repository: repository.to_owned(),
593 now: week_waits(&now, history.kept_since),
594 before: week_waits(&before, history.kept_since),
595 })
596 .collect()
597}
598
599fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
600 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
601 let of = |runs: Vec<&&Run>| {
602 let waited: Vec<u64> = runs
603 .iter()
604 .filter_map(|run| run.waited(kept_since))
605 .collect();
606 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
607 let spread = (!waited.is_empty()).then(|| spread(waited));
608 WaitSpread {
609 jobs,
610 p50: spread.as_ref().map(|spread| spread.p50),
611 p95: spread.as_ref().map(|spread| spread.p95),
612 }
613 };
614 WeekWaits {
615 all: of(runs.iter().collect()),
616 warm: of(runs.iter().filter(warm).collect()),
617 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
618 }
619}
620
621fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
622 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
623 return None;
624 }
625 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
626 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
627 if now < threshold {
628 return None;
629 }
630 Some(Insight {
631 key: format!("waits:{}", waits.repository),
632 severity: Severity::Notice,
633 repository: waits.repository.clone(),
634 workflow: None,
635 job: None,
636 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
637 format!(
638 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
639 waits.repository,
640 span(now),
641 span(before)
642 )
643 } else {
644 format!(
645 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
646 waits.repository,
647 span(now),
648 span(before)
649 )
650 },
651 kind: InsightKind::Waits { before, now },
652 runs: Vec::new(),
653 })
654}
655
656fn span(seconds: u64) -> String {
658 match seconds {
659 0..60 => format!("{seconds}s"),
660 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
661 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
662 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
663 }
664}
665
666#[cfg(test)]
667mod tests;