1use std::collections::{BTreeMap, BTreeSet};
8
9use crate::api::{
10 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
11 RunLink, Severity, WaitSpread, WeekWaits,
12};
13use crate::config::{ByteSize, Config};
14use crate::sizing::spread;
15
16const SECOND: u64 = 1000;
17const DAY: u64 = 24 * 3600 * SECOND;
18const WEEK: u64 = 7 * DAY;
19
20const RECENT: usize = 5;
22const SLOWER_PERCENT: u64 = 20;
24const SLOWER_SECONDS: u64 = 30;
26
27const CAP_RUNS: usize = 10;
29const CAP_MIN_RUNS: usize = 5;
30const NEAR_CAP: u64 = 85;
32const NEAR_CAP_WARNING: usize = 2;
34const FLAKY_WARNING: u32 = 2;
36
37const WAITS_PERCENT: u64 = 50;
40const WAITS_SECONDS: u64 = 60;
41const WAITS_MIN_JOBS: u32 = 5;
42const WAITS_ONE_IN_20: u32 = 20;
44
45const THROTTLED_MIN_RUNS: u32 = 2;
47
48#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct Run {
51 pub id: u64,
53 pub repository: String,
55 pub workflow: Option<String>,
57 pub name: String,
59 pub branch: Option<String>,
61 pub url: Option<String>,
63 pub run_id: u64,
65 pub run_attempt: u32,
67 pub state: JobState,
69 pub conclusion: Option<String>,
71 pub queued_at: u64,
73 pub started_at: Option<u64>,
75 pub finished_at: Option<u64>,
77 pub runner_started_at: Option<u64>,
79 pub profile: Option<String>,
81 pub memory_peak: Option<u64>,
83 pub build_cache: Option<api::BuildCacheCounts>,
85 pub sock_throttled: Option<u64>,
87 pub unseen: u64,
89}
90
91impl Run {
92 fn finished(&self) -> u64 {
93 self.finished_at.unwrap_or(0)
94 }
95
96 fn succeeded(&self) -> bool {
97 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
98 }
99
100 fn failed(&self) -> bool {
102 self.state == JobState::Lost || self.failed_itself()
103 }
104
105 fn failed_itself(&self) -> bool {
107 self.state == JobState::Completed
108 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
109 }
110
111 fn took(&self) -> Option<u64> {
113 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
114 }
115
116 fn waited(&self, kept_since: u64) -> Option<u64> {
118 let wait = crate::sizing::Wait {
119 queued_at: self.queued_at,
120 started_at: self.started_at?,
121 unseen: self.unseen,
122 };
123 Some(wait.counted(kept_since) / SECOND)
124 }
125
126 fn link(&self) -> RunLink {
127 RunLink {
128 job: self.id,
129 run_id: self.run_id,
130 run_attempt: self.run_attempt,
131 branch: self.branch.clone(),
132 url: self.url.clone(),
133 at: self.finished() / SECOND,
134 }
135 }
136}
137
138#[derive(Debug, Clone, Default, PartialEq, Eq)]
140pub struct History {
141 pub runs: Vec<Run>,
143 pub kept_since: u64,
146}
147
148type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
150
151fn key_of(run: &Run) -> JobKey<'_> {
152 (&run.repository, run.workflow.as_deref(), &run.name)
153}
154
155pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
158 let runs = &history.runs;
159 let since = now.saturating_sub(u64::from(days) * DAY);
160 let window: Vec<&Run> = runs
161 .iter()
162 .filter(|run| {
163 matches!(run.state, JobState::Completed | JobState::Lost)
164 && run
165 .finished_at
166 .is_some_and(|at| (since..=now).contains(&at))
167 })
168 .collect();
169 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
170 for run in &window {
171 jobs.entry(key_of(run)).or_default().push(run);
172 }
173 let mut found = Vec::new();
174 for (key, runs) in &mut jobs {
175 runs.sort_by_key(|run| (run.finished(), run.id));
176 found.extend(slower(*key, runs));
177 found.extend(flaky(*key, runs));
178 found.extend(near_cap(*key, runs, config));
179 found.extend(cold_cache(*key, runs));
180 }
181 found.extend(throttled(&window));
182 let waits = waits(history, now);
183 found.extend(waits.iter().filter_map(waits_insight));
184 found.sort_by(|a, b| {
185 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
186 b.severity,
187 &b.repository,
188 &b.workflow,
189 &b.job,
190 &b.key,
191 ))
192 });
193 Insights {
194 since: since / SECOND,
195 until: now / SECOND,
196 insights: found,
197 waits,
198 }
199}
200
201pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
204 let runs = &history.runs;
205 let since = now.saturating_sub(DAY);
206 let finished: Vec<&Run> = runs
207 .iter()
208 .filter(|run| {
209 matches!(run.state, JobState::Completed | JobState::Lost)
210 && run.finished_at.is_some_and(|at| at > since && at <= now)
211 })
212 .collect();
213 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
214 .insights
215 .into_iter()
216 .map(|insight| insight.key)
217 .collect();
218 let new = insights(history, config, api::INSIGHT_DAYS, now)
219 .insights
220 .into_iter()
221 .filter(|insight| !before.contains(&insight.key))
222 .collect();
223 let mut started: Vec<&Run> = runs
224 .iter()
225 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
226 .collect();
227 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
228 Digest {
229 since: since / SECOND,
230 until: now / SECOND,
231 finished: finished.len() as u64,
232 failed: finished.iter().filter(|run| run.failed()).count() as u64,
233 new,
234 waits: started
235 .into_iter()
236 .take(3)
237 .map(|run| LongWait {
238 job: run.id,
239 repository: run.repository.clone(),
240 workflow: run.workflow.clone(),
241 name: run.name.clone(),
242 waited: run.waited(history.kept_since).unwrap_or(0),
243 url: run.url.clone(),
244 })
245 .collect(),
246 savings: None,
247 }
248}
249
250fn subject((repository, workflow, name): JobKey<'_>) -> String {
252 match workflow {
253 Some(workflow) => format!("{repository}: {workflow} / {name}"),
254 None => format!("{repository}: {name}"),
255 }
256}
257
258fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
259 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
260}
261
262fn insight(
263 key: JobKey<'_>,
264 kind: &str,
265 severity: Severity,
266 sentence: String,
267 found: InsightKind,
268 runs: Vec<RunLink>,
269) -> Insight {
270 Insight {
271 key: key_text(kind, key),
272 severity,
273 repository: key.0.to_owned(),
274 workflow: key.1.map(str::to_owned),
275 job: Some(key.2.to_owned()),
276 sentence,
277 kind: found,
278 runs,
279 }
280}
281
282fn median(values: &[u64]) -> u64 {
284 let mut values = values.to_vec();
285 values.sort_unstable();
286 let middle = values.len() / 2;
287 match values.len() {
288 0 => 0,
289 len if len % 2 == 1 => values[middle],
290 _ => u64::midpoint(values[middle - 1], values[middle]),
291 }
292}
293
294fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
296 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
297 runs.iter()
298 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
299 .filter_map(|run| Some((*run, run.took()?)))
300 .collect()
301 };
302 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
303 for run in runs.iter().filter(|run| run.succeeded()) {
304 if let Some(branch) = &run.branch {
305 *branches.entry(branch).or_default() += 1;
306 }
307 }
308 let busiest = branches
311 .iter()
312 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
313 .map(|(branch, _)| *branch)?;
314 let (branch, timed) = ["main", "master"]
315 .into_iter()
316 .map(|branch| (branch, timed_on(branch)))
317 .find(|(_, timed)| timed.len() >= 2 * RECENT)
318 .unwrap_or_else(|| (busiest, timed_on(busiest)));
319 if timed.len() < 2 * RECENT {
320 return None;
321 }
322 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
323 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
324 let before = median(&took(baseline));
325 let now = median(&took(recent));
326 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
327 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
330 return None;
331 }
332 let mut since = None;
334 let mut fast_in_a_row = 0;
335 for (run, took) in &timed {
336 if *took >= threshold {
337 fast_in_a_row = 0;
338 since = since.or(Some(*run));
339 } else {
340 fast_in_a_row += 1;
341 if fast_in_a_row >= 2 {
342 since = None;
343 }
344 }
345 }
346 let since = since?;
347 let sentence = format!(
348 "{} on {branch} takes {} now, up from {}.",
349 subject(key),
350 span(now),
351 span(before)
352 );
353 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
354 Some(insight(
355 key,
356 "slower",
357 Severity::Warning,
358 sentence,
359 InsightKind::Slower {
360 branch: Some(branch.to_owned()),
361 before,
362 now,
363 since: Some(since.link()),
364 },
365 links,
366 ))
367}
368
369fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
371 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
372 for run in runs {
373 by_run.entry(run.run_id).or_default().push(run);
374 }
375 let mut failed_first: Vec<&Run> = by_run
376 .values()
377 .filter_map(|attempts| {
378 attempts.iter().copied().find(|failed| {
379 failed.failed_itself()
380 && attempts
381 .iter()
382 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
383 })
384 })
385 .collect();
386 if failed_first.is_empty() {
387 return None;
388 }
389 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
390 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
391 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
392 let sentence = format!(
393 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
394 subject(key)
395 );
396 let severity = if flaky >= FLAKY_WARNING {
398 Severity::Warning
399 } else {
400 Severity::Notice
401 };
402 Some(insight(
403 key,
404 "flaky",
405 severity,
406 sentence,
407 InsightKind::Flaky { flaky, total },
408 failed_first.iter().take(5).map(|run| run.link()).collect(),
409 ))
410}
411
412fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
415 let measured = runs
416 .iter()
417 .rev()
418 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
419 let name = measured.clone().next()?.profile.as_deref()?;
420 let measured: Vec<&Run> = measured
421 .filter(|run| run.profile.as_deref() == Some(name))
422 .take(CAP_RUNS)
423 .copied()
424 .collect();
425 if measured.len() < CAP_MIN_RUNS {
426 return None;
427 }
428 let profile = config.profiles.get(name)?;
429 let cap = profile.memory.0;
430 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
431 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
432 let peak = spread(peaks.clone()).p95;
433 if cap == 0 || !near(&peak) {
434 return None;
435 }
436 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
437 let auto = if !config.sizing.auto {
438 AutoNote::Off
439 } else if profile
440 .max_memory
441 .is_some_and(|max| profile.memory.0 >= max.0)
442 {
443 AutoNote::AtMax
444 } else {
445 AutoNote::Raises
446 };
447 let what = match auto {
448 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
449 AutoNote::AtMax => {
450 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
451 }
452 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
453 };
454 let sentence = format!(
455 "{} peaks at {}, {}% of {name}'s {}. {what}",
456 subject(key),
457 ByteSize(peak),
458 peak.saturating_mul(100) / cap,
459 ByteSize(cap)
460 );
461 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
463 Severity::Notice
464 } else {
465 Severity::Warning
466 };
467 Some(insight(
468 key,
469 "near-cap",
470 severity,
471 sentence,
472 InsightKind::NearCap {
473 profile: name.to_owned(),
474 peak,
475 cap,
476 auto,
477 },
478 measured.iter().take(5).map(|run| run.link()).collect(),
479 ))
480}
481
482const COLD_RUNS: usize = 5;
484const COLD_MIN_RUNS: usize = 3;
485const COLD_MIN_LOOKUPS: u64 = 10;
486const COLD_PERCENT: u64 = 10;
489
490fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
491 let writers: Vec<&Run> = runs
492 .iter()
493 .rev()
494 .filter(|run| {
495 run.build_cache
496 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
497 })
498 .take(COLD_RUNS)
499 .copied()
500 .collect();
501 if writers.len() < COLD_MIN_RUNS {
502 return None;
503 }
504 let mut total = api::BuildCacheCounts::default();
505 for run in &writers {
506 total.add(&run.build_cache.unwrap_or_default());
507 }
508 let percent = total.hit_percent()?;
509 if percent > COLD_PERCENT {
510 return None;
511 }
512 let branch = writers.first()?.branch.clone();
513 let sentence = format!(
514 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
515 differ from run to run.",
516 subject(key),
517 branch
518 .as_deref()
519 .map(|branch| format!(" on {branch}"))
520 .unwrap_or_default(),
521 writers.len()
522 );
523 Some(insight(
524 key,
525 "cold-cache",
526 Severity::Notice,
527 sentence,
528 InsightKind::ColdCache {
529 branch,
530 hits: total.hits,
531 lookups: total.lookups(),
532 },
533 writers.iter().map(|run| run.link()).collect(),
534 ))
535}
536
537fn throttled(window: &[&Run]) -> Vec<Insight> {
539 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
540 for run in window {
541 repositories.entry(&run.repository).or_default().push(run);
542 }
543 repositories
544 .into_iter()
545 .filter_map(|(repository, mut runs)| {
546 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
547 let hit: Vec<&Run> = runs
548 .iter()
549 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
550 .copied()
551 .collect();
552 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
553 if throttled < THROTTLED_MIN_RUNS {
554 return None;
555 }
556 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
557 Some(Insight {
558 key: format!("throttled:{repository}"),
559 severity: Severity::Warning,
560 repository: repository.to_owned(),
561 workflow: None,
562 job: None,
563 sentence: format!(
564 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
565 host, or a job at its cap, was short of memory."
566 ),
567 kind: InsightKind::Throttled { throttled, total },
568 runs: hit.iter().take(5).map(|run| run.link()).collect(),
569 })
570 })
571 .collect()
572}
573
574fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
578 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
579 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
580 for run in &history.runs {
581 let Some(started) = run.started_at.filter(|at| *at <= now) else {
582 continue;
583 };
584 let week = match now - started {
585 back if back < WEEK => 0,
586 back if back < 2 * WEEK && !before_kept => 1,
587 _ => continue,
588 };
589 weeks.entry(&run.repository).or_default()[week].push(run);
590 }
591 weeks
592 .into_iter()
593 .map(|(repository, [now, before])| RepositoryWaits {
594 repository: repository.to_owned(),
595 now: week_waits(&now, history.kept_since),
596 before: week_waits(&before, history.kept_since),
597 })
598 .collect()
599}
600
601fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
602 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
603 let of = |runs: Vec<&&Run>| {
604 let waited: Vec<u64> = runs
605 .iter()
606 .filter_map(|run| run.waited(kept_since))
607 .collect();
608 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
609 let spread = (!waited.is_empty()).then(|| spread(waited));
610 WaitSpread {
611 jobs,
612 p50: spread.as_ref().map(|spread| spread.p50),
613 p95: spread.as_ref().map(|spread| spread.p95),
614 }
615 };
616 WeekWaits {
617 all: of(runs.iter().collect()),
618 warm: of(runs.iter().filter(warm).collect()),
619 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
620 }
621}
622
623fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
624 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
625 return None;
626 }
627 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
628 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
629 if now < threshold {
630 return None;
631 }
632 Some(Insight {
633 key: format!("waits:{}", waits.repository),
634 severity: Severity::Notice,
635 repository: waits.repository.clone(),
636 workflow: None,
637 job: None,
638 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
639 format!(
640 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
641 waits.repository,
642 span(now),
643 span(before)
644 )
645 } else {
646 format!(
647 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
648 waits.repository,
649 span(now),
650 span(before)
651 )
652 },
653 kind: InsightKind::Waits { before, now },
654 runs: Vec::new(),
655 })
656}
657
658fn span(seconds: u64) -> String {
660 match seconds {
661 0..60 => format!("{seconds}s"),
662 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
663 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
664 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
665 }
666}
667
668#[cfg(test)]
669mod tests;