1use std::collections::{BTreeMap, BTreeSet};
9
10use crate::api::{
11 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
12 RunLink, Severity, WaitSpread, WeekWaits,
13};
14use crate::config::{Admission, ByteSize, Config};
15use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
16use crate::sizing::{HELD, Throttling, spread};
17
18const SECOND: u64 = 1000;
19const DAY: u64 = 24 * 3600 * SECOND;
20const WEEK: u64 = 7 * DAY;
21
22const RECENT: usize = 5;
24const SLOWER_PERCENT: u64 = 20;
26const SLOWER_SECONDS: u64 = 30;
28
29const CAP_RUNS: usize = 10;
31const CAP_MIN_RUNS: usize = 5;
32const NEAR_CAP: u64 = 85;
34const NEAR_CAP_WARNING: usize = 2;
36const FLAKY_WARNING: u32 = 2;
38
39const WAITS_PERCENT: u64 = 50;
42const WAITS_SECONDS: u64 = 60;
43const WAITS_MIN_JOBS: u32 = 5;
44const WAITS_ONE_IN_20: u32 = 20;
46
47const THROTTLED_MIN_RUNS: u32 = 2;
49
50const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
52const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
55const HOST_HELD_FRESH: u64 = 60 * SECOND;
59
60#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct Run {
63 pub id: u64,
65 pub repository: String,
67 pub workflow: Option<String>,
69 pub name: String,
71 pub branch: Option<String>,
73 pub url: Option<String>,
75 pub run_id: u64,
77 pub run_attempt: u32,
79 pub state: JobState,
81 pub conclusion: Option<String>,
83 pub queued_at: u64,
85 pub started_at: Option<u64>,
87 pub finished_at: Option<u64>,
89 pub runner_started_at: Option<u64>,
91 pub profile: Option<String>,
93 pub memory_peak: Option<u64>,
95 pub build_cache: Option<api::BuildCacheCounts>,
97 pub sock_throttled: Option<u64>,
99 pub cpu_cap: Option<u64>,
101 pub cpu_periods: Option<u64>,
103 pub cpu_throttled: Option<u64>,
105 pub unseen: u64,
107}
108
109impl Run {
110 fn finished(&self) -> u64 {
111 self.finished_at.unwrap_or(0)
112 }
113
114 fn succeeded(&self) -> bool {
115 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
116 }
117
118 fn failed(&self) -> bool {
120 self.state == JobState::Lost || self.failed_itself()
121 }
122
123 fn failed_itself(&self) -> bool {
125 self.state == JobState::Completed
126 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
127 }
128
129 fn took(&self) -> Option<u64> {
131 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
132 }
133
134 fn waited(&self, kept_since: u64) -> Option<u64> {
136 let wait = crate::sizing::Wait {
137 queued_at: self.queued_at,
138 started_at: self.started_at?,
139 unseen: self.unseen,
140 };
141 Some(wait.counted(kept_since) / SECOND)
142 }
143
144 fn throttling(&self) -> Throttling {
145 Throttling {
146 started_at: self.started_at,
147 finished_at: self.finished_at,
148 periods: self.cpu_periods,
149 throttled: self.cpu_throttled,
150 }
151 }
152
153 fn link(&self) -> RunLink {
154 RunLink {
155 job: self.id,
156 run_id: self.run_id,
157 run_attempt: self.run_attempt,
158 branch: self.branch.clone(),
159 url: self.url.clone(),
160 at: self.finished() / SECOND,
161 }
162 }
163}
164
165#[derive(Debug, Clone, PartialEq, Eq)]
168pub struct HostHold {
169 pub memory: api::HostMemory,
171 pub seen: u64,
173 pub jobs: Vec<HeldJob>,
175}
176
177#[derive(Debug, Clone, PartialEq, Eq)]
179pub struct HeldJob {
180 pub repository: String,
182 pub workflow: Option<String>,
184 pub name: String,
186 pub waited: u64,
188}
189
190#[derive(Debug, Clone, Default, PartialEq, Eq)]
192pub struct History {
193 pub runs: Vec<Run>,
195 pub kept_since: u64,
198 pub host: Option<HostHold>,
200}
201
202type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
204
205fn key_of(run: &Run) -> JobKey<'_> {
206 (&run.repository, run.workflow.as_deref(), &run.name)
207}
208
209pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
212 let runs = &history.runs;
213 let since = now.saturating_sub(u64::from(days) * DAY);
214 let window: Vec<&Run> = runs
215 .iter()
216 .filter(|run| {
217 matches!(run.state, JobState::Completed | JobState::Lost)
218 && run
219 .finished_at
220 .is_some_and(|at| (since..=now).contains(&at))
221 })
222 .collect();
223 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
224 for run in &window {
225 jobs.entry(key_of(run)).or_default().push(run);
226 }
227 let mut found = Vec::new();
228 for (key, runs) in &mut jobs {
229 runs.sort_by_key(|run| (run.finished(), run.id));
230 found.extend(slower(*key, runs));
231 found.extend(flaky(*key, runs));
232 found.extend(near_cap(*key, runs, config));
233 found.extend(cold_cache(*key, runs));
234 found.extend(cpu_held(*key, runs, config));
235 }
236 found.extend(throttled(&window));
237 found.extend(
238 history
239 .host
240 .as_ref()
241 .and_then(|hold| host_held(hold, config, now)),
242 );
243 let waits = waits(history, now);
244 found.extend(waits.iter().filter_map(waits_insight));
245 found.sort_by(|a, b| {
246 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
247 b.severity,
248 &b.repository,
249 &b.workflow,
250 &b.job,
251 &b.key,
252 ))
253 });
254 Insights {
255 since: since / SECOND,
256 until: now / SECOND,
257 insights: found,
258 waits,
259 }
260}
261
262pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
265 let runs = &history.runs;
266 let since = now.saturating_sub(DAY);
267 let finished: Vec<&Run> = runs
268 .iter()
269 .filter(|run| {
270 matches!(run.state, JobState::Completed | JobState::Lost)
271 && run.finished_at.is_some_and(|at| at > since && at <= now)
272 })
273 .collect();
274 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
275 .insights
276 .into_iter()
277 .map(|insight| insight.key)
278 .collect();
279 let new = insights(history, config, api::INSIGHT_DAYS, now)
280 .insights
281 .into_iter()
282 .filter(|insight| !before.contains(&insight.key))
283 .collect();
284 let mut started: Vec<&Run> = runs
285 .iter()
286 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
287 .collect();
288 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
289 Digest {
290 since: since / SECOND,
291 until: now / SECOND,
292 finished: finished.len() as u64,
293 failed: finished.iter().filter(|run| run.failed()).count() as u64,
294 new,
295 waits: started
296 .into_iter()
297 .take(3)
298 .map(|run| LongWait {
299 job: run.id,
300 repository: run.repository.clone(),
301 workflow: run.workflow.clone(),
302 name: run.name.clone(),
303 waited: run.waited(history.kept_since).unwrap_or(0),
304 url: run.url.clone(),
305 })
306 .collect(),
307 savings: None,
308 }
309}
310
311fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
323 let reading = hold.memory.at.saturating_mul(SECOND);
324 let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
326 let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
327 let fresh =
328 |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
329 if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
330 return None;
331 }
332 let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
333 if longest.waited < HOST_HELD_AFTER {
334 return None;
335 }
336 let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
337 let waited = longest.waited / SECOND;
338 let sentence = Insight::host_held_sentence(
339 jobs,
340 waited,
341 &subject((
342 &longest.repository,
343 longest.workflow.as_deref(),
344 &longest.name,
345 )),
346 &hold.memory,
347 );
348 Some(Insight {
349 key: format!("host-held:{}", now / HOST_HELD_EVERY),
350 severity: Severity::Warning,
351 repository: longest.repository.clone(),
352 workflow: None,
353 job: None,
354 sentence,
355 kind: InsightKind::HostHeld { jobs, waited },
356 runs: Vec::new(),
357 })
358}
359
360fn subject((repository, workflow, name): JobKey<'_>) -> String {
362 match workflow {
363 Some(workflow) => format!("{repository}: {workflow} / {name}"),
364 None => format!("{repository}: {name}"),
365 }
366}
367
368fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
369 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
370}
371
372fn insight(
373 key: JobKey<'_>,
374 kind: &str,
375 severity: Severity,
376 sentence: String,
377 found: InsightKind,
378 runs: Vec<RunLink>,
379) -> Insight {
380 Insight {
381 key: key_text(kind, key),
382 severity,
383 repository: key.0.to_owned(),
384 workflow: key.1.map(str::to_owned),
385 job: Some(key.2.to_owned()),
386 sentence,
387 kind: found,
388 runs,
389 }
390}
391
392fn median(values: &[u64]) -> u64 {
394 let mut values = values.to_vec();
395 values.sort_unstable();
396 let middle = values.len() / 2;
397 match values.len() {
398 0 => 0,
399 len if len % 2 == 1 => values[middle],
400 _ => u64::midpoint(values[middle - 1], values[middle]),
401 }
402}
403
404fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
406 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
407 runs.iter()
408 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
409 .filter_map(|run| Some((*run, run.took()?)))
410 .collect()
411 };
412 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
413 for run in runs.iter().filter(|run| run.succeeded()) {
414 if let Some(branch) = &run.branch {
415 *branches.entry(branch).or_default() += 1;
416 }
417 }
418 let busiest = branches
421 .iter()
422 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
423 .map(|(branch, _)| *branch)?;
424 let (branch, timed) = ["main", "master"]
425 .into_iter()
426 .map(|branch| (branch, timed_on(branch)))
427 .find(|(_, timed)| timed.len() >= 2 * RECENT)
428 .unwrap_or_else(|| (busiest, timed_on(busiest)));
429 if timed.len() < 2 * RECENT {
430 return None;
431 }
432 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
433 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
434 let before = median(&took(baseline));
435 let now = median(&took(recent));
436 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
437 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
440 return None;
441 }
442 let mut since = None;
444 let mut fast_in_a_row = 0;
445 for (run, took) in &timed {
446 if *took >= threshold {
447 fast_in_a_row = 0;
448 since = since.or(Some(*run));
449 } else {
450 fast_in_a_row += 1;
451 if fast_in_a_row >= 2 {
452 since = None;
453 }
454 }
455 }
456 let since = since?;
457 let sentence = format!(
458 "{} on {branch} takes {} now, up from {}.",
459 subject(key),
460 span(now),
461 span(before)
462 );
463 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
464 Some(insight(
465 key,
466 "slower",
467 Severity::Warning,
468 sentence,
469 InsightKind::Slower {
470 branch: Some(branch.to_owned()),
471 before,
472 now,
473 since: Some(since.link()),
474 },
475 links,
476 ))
477}
478
479fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
481 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
482 for run in runs {
483 by_run.entry(run.run_id).or_default().push(run);
484 }
485 let mut failed_first: Vec<&Run> = by_run
486 .values()
487 .filter_map(|attempts| {
488 attempts.iter().copied().find(|failed| {
489 failed.failed_itself()
490 && attempts
491 .iter()
492 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
493 })
494 })
495 .collect();
496 if failed_first.is_empty() {
497 return None;
498 }
499 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
500 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
501 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
502 let sentence = format!(
503 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
504 subject(key)
505 );
506 let severity = if flaky >= FLAKY_WARNING {
508 Severity::Warning
509 } else {
510 Severity::Notice
511 };
512 Some(insight(
513 key,
514 "flaky",
515 severity,
516 sentence,
517 InsightKind::Flaky { flaky, total },
518 failed_first.iter().take(5).map(|run| run.link()).collect(),
519 ))
520}
521
522fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
525 let measured = runs
526 .iter()
527 .rev()
528 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
529 let name = measured.clone().next()?.profile.as_deref()?;
530 let measured: Vec<&Run> = measured
531 .filter(|run| run.profile.as_deref() == Some(name))
532 .take(CAP_RUNS)
533 .copied()
534 .collect();
535 if measured.len() < CAP_MIN_RUNS {
536 return None;
537 }
538 let profile = config.profiles.get(name)?;
539 let cap = profile.memory.0;
540 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
541 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
542 let peak = spread(peaks.clone()).p95;
543 if cap == 0 || !near(&peak) {
544 return None;
545 }
546 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
547 let auto = if !config.sizing.auto {
548 AutoNote::Off
549 } else if profile
550 .max_memory
551 .is_some_and(|max| profile.memory.0 >= max.0)
552 {
553 AutoNote::AtMax
554 } else {
555 AutoNote::Raises
556 };
557 let what = match auto {
558 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
559 AutoNote::AtMax => {
560 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
561 }
562 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
563 };
564 let sentence = format!(
565 "{} peaks at {}, {}% of {name}'s {}. {what}",
566 subject(key),
567 ByteSize(peak),
568 peak.saturating_mul(100) / cap,
569 ByteSize(cap)
570 );
571 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
573 Severity::Notice
574 } else {
575 Severity::Warning
576 };
577 Some(insight(
578 key,
579 "near-cap",
580 severity,
581 sentence,
582 InsightKind::NearCap {
583 profile: name.to_owned(),
584 peak,
585 cap,
586 auto,
587 },
588 measured.iter().take(5).map(|run| run.link()).collect(),
589 ))
590}
591
592const COLD_RUNS: usize = 5;
594const COLD_MIN_RUNS: usize = 3;
595const COLD_MIN_LOOKUPS: u64 = 10;
596const COLD_PERCENT: u64 = 10;
599
600fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
601 let writers: Vec<&Run> = runs
602 .iter()
603 .rev()
604 .filter(|run| {
605 run.build_cache
606 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
607 })
608 .take(COLD_RUNS)
609 .copied()
610 .collect();
611 if writers.len() < COLD_MIN_RUNS {
612 return None;
613 }
614 let mut total = api::BuildCacheCounts::default();
615 for run in &writers {
616 total.add(&run.build_cache.unwrap_or_default());
617 }
618 let percent = total.hit_percent()?;
619 if percent > COLD_PERCENT {
620 return None;
621 }
622 let branch = writers.first()?.branch.clone();
623 let sentence = format!(
624 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
625 differ from run to run.",
626 subject(key),
627 branch
628 .as_deref()
629 .map(|branch| format!(" on {branch}"))
630 .unwrap_or_default(),
631 writers.len()
632 );
633 Some(insight(
634 key,
635 "cold-cache",
636 Severity::Notice,
637 sentence,
638 InsightKind::ColdCache {
639 branch,
640 hits: total.hits,
641 lookups: total.lookups(),
642 },
643 writers.iter().map(|run| run.link()).collect(),
644 ))
645}
646
647fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
654 let measured = runs.iter().rev().filter(|run| {
655 run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
656 });
657 let last = measured.clone().next()?;
658 let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
659 let profile = config.profiles.get(name)?;
660 if u64::from(profile.cpus) * 1000 != cap {
662 return None;
663 }
664 let at_cap: Vec<&Run> = measured
665 .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
666 .take(CAP_RUNS)
667 .copied()
668 .collect();
669 let held: Vec<&Run> = at_cap
670 .iter()
671 .filter(|run| run.throttling().held() == Some(true))
672 .copied()
673 .collect();
674 if (held.len() as u64) < RAISE_RUNS {
675 return None;
676 }
677 let cpus = profile.cpus;
678 let auto = config.host.admission == Admission::Measured && config.sizing.auto;
679 let raised_from = auto
680 .then(|| {
681 let ran: Vec<CpuRun> = runs
682 .iter()
683 .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
684 .map(|run| CpuRun {
685 cap: run.cpu_cap,
686 queued_at: run.queued_at,
687 throttling: run.throttling(),
688 })
689 .collect();
690 let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
691 unhelped_raise(&ran, cap, &now)
692 })
693 .flatten()
694 .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
695 let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
696 let what = match (raised_from, set) {
697 (Some(from), _) => format!(
698 "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
699 tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
700 sizing does next."
701 ),
702 (None, Some(set)) if config.host.admission == Admission::Measured => {
703 format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
704 }
705 (None, Some(set)) => format!(
706 "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
707 cpus to {set} if runnerctl sizing shows the budget has room."
708 ),
709 (None, None) => format!(
710 "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
711 room; runnerctl sizing says when it cannot."
712 ),
713 };
714 let sentence = format!(
715 "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
716 its run or more. {what}",
717 subject(key),
718 held.len(),
719 at_cap.len()
720 );
721 let severity = if auto && raised_from.is_none() {
723 Severity::Notice
724 } else {
725 Severity::Warning
726 };
727 Some(insight(
728 key,
729 "cpu-held",
730 severity,
731 sentence,
732 InsightKind::CpuHeld {
733 profile: name.to_owned(),
734 cpus,
735 held: u32::try_from(held.len()).unwrap_or(u32::MAX),
736 total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
737 raised_from,
738 set,
739 },
740 held.iter().take(5).map(|run| run.link()).collect(),
741 ))
742}
743
744fn throttled(window: &[&Run]) -> Vec<Insight> {
746 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
747 for run in window {
748 repositories.entry(&run.repository).or_default().push(run);
749 }
750 repositories
751 .into_iter()
752 .filter_map(|(repository, mut runs)| {
753 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
754 let hit: Vec<&Run> = runs
755 .iter()
756 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
757 .copied()
758 .collect();
759 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
760 if throttled < THROTTLED_MIN_RUNS {
761 return None;
762 }
763 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
764 Some(Insight {
765 key: format!("throttled:{repository}"),
766 severity: Severity::Warning,
767 repository: repository.to_owned(),
768 workflow: None,
769 job: None,
770 sentence: format!(
771 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
772 host, or a job at its cap, was short of memory."
773 ),
774 kind: InsightKind::Throttled { throttled, total },
775 runs: hit.iter().take(5).map(|run| run.link()).collect(),
776 })
777 })
778 .collect()
779}
780
781fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
785 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
786 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
787 for run in &history.runs {
788 let Some(started) = run.started_at.filter(|at| *at <= now) else {
789 continue;
790 };
791 let week = match now - started {
792 back if back < WEEK => 0,
793 back if back < 2 * WEEK && !before_kept => 1,
794 _ => continue,
795 };
796 weeks.entry(&run.repository).or_default()[week].push(run);
797 }
798 weeks
799 .into_iter()
800 .map(|(repository, [now, before])| RepositoryWaits {
801 repository: repository.to_owned(),
802 now: week_waits(&now, history.kept_since),
803 before: week_waits(&before, history.kept_since),
804 })
805 .collect()
806}
807
808fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
809 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
810 let of = |runs: Vec<&&Run>| {
811 let waited: Vec<u64> = runs
812 .iter()
813 .filter_map(|run| run.waited(kept_since))
814 .collect();
815 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
816 let spread = (!waited.is_empty()).then(|| spread(waited));
817 WaitSpread {
818 jobs,
819 p50: spread.as_ref().map(|spread| spread.p50),
820 p95: spread.as_ref().map(|spread| spread.p95),
821 }
822 };
823 WeekWaits {
824 all: of(runs.iter().collect()),
825 warm: of(runs.iter().filter(warm).collect()),
826 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
827 }
828}
829
830fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
831 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
832 return None;
833 }
834 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
835 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
836 if now < threshold {
837 return None;
838 }
839 Some(Insight {
840 key: format!("waits:{}", waits.repository),
841 severity: Severity::Notice,
842 repository: waits.repository.clone(),
843 workflow: None,
844 job: None,
845 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
846 format!(
847 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
848 waits.repository,
849 span(now),
850 span(before)
851 )
852 } else {
853 format!(
854 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
855 waits.repository,
856 span(now),
857 span(before)
858 )
859 },
860 kind: InsightKind::Waits { before, now },
861 runs: Vec::new(),
862 })
863}
864
865fn span(seconds: u64) -> String {
867 match seconds {
868 0..60 => format!("{seconds}s"),
869 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
870 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
871 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
872 }
873}
874
875#[cfg(test)]
876mod tests;