1use std::collections::{BTreeMap, BTreeSet};
10
11use crate::api::{
12 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
13 RunLink, Severity, WaitSpread, WeekWaits,
14};
15use crate::config::{Admission, ByteSize, Config};
16use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
17use crate::sizing::{HELD, Throttling, spread};
18
19mod disk;
20mod host;
21
22const SECOND: u64 = 1000;
23const DAY: u64 = 24 * 3600 * SECOND;
24const WEEK: u64 = 7 * DAY;
25
26const RECENT: usize = 5;
28const SLOWER_PERCENT: u64 = 20;
30const SLOWER_SECONDS: u64 = 30;
32
33const CAP_RUNS: usize = 10;
35const CAP_MIN_RUNS: usize = 5;
36const NEAR_CAP: u64 = 85;
38const NEAR_CAP_WARNING: usize = 2;
40const FLAKY_WARNING: u32 = 2;
42
43const WAITS_PERCENT: u64 = 50;
46const WAITS_SECONDS: u64 = 60;
47const WAITS_MIN_JOBS: u32 = 5;
48const WAITS_ONE_IN_20: u32 = 20;
50
51const THROTTLED_MIN_RUNS: u32 = 2;
53
54const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
56const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
59const HOST_HELD_FRESH: u64 = 60 * SECOND;
63
64#[derive(Debug, Clone, PartialEq, Eq)]
66pub struct Run {
67 pub id: u64,
69 pub repository: String,
71 pub workflow: Option<String>,
73 pub name: String,
75 pub branch: Option<String>,
77 pub url: Option<String>,
79 pub run_id: u64,
81 pub run_attempt: u32,
83 pub state: JobState,
85 pub conclusion: Option<String>,
87 pub queued_at: u64,
89 pub started_at: Option<u64>,
91 pub finished_at: Option<u64>,
93 pub runner_started_at: Option<u64>,
95 pub profile: Option<String>,
97 pub memory_peak: Option<u64>,
99 pub build_cache: Option<api::BuildCacheCounts>,
101 pub sock_throttled: Option<u64>,
103 pub cpu_cap: Option<u64>,
105 pub cpu_periods: Option<u64>,
107 pub cpu_throttled: Option<u64>,
109 pub disk: Option<api::DiskWritten>,
111 pub unseen: u64,
113}
114
115impl Run {
116 fn finished(&self) -> u64 {
117 self.finished_at.unwrap_or(0)
118 }
119
120 fn succeeded(&self) -> bool {
121 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
122 }
123
124 fn failed(&self) -> bool {
126 self.state == JobState::Lost || self.failed_itself()
127 }
128
129 fn failed_itself(&self) -> bool {
131 self.state == JobState::Completed
132 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
133 }
134
135 fn took(&self) -> Option<u64> {
137 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
138 }
139
140 fn waited(&self, kept_since: u64) -> Option<u64> {
142 let wait = crate::sizing::Wait {
143 queued_at: self.queued_at,
144 started_at: self.started_at?,
145 unseen: self.unseen,
146 };
147 Some(wait.counted(kept_since) / SECOND)
148 }
149
150 fn throttling(&self) -> Throttling {
151 Throttling {
152 started_at: self.started_at,
153 finished_at: self.finished_at,
154 periods: self.cpu_periods,
155 throttled: self.cpu_throttled,
156 }
157 }
158
159 fn link(&self) -> RunLink {
160 RunLink {
161 job: self.id,
162 run_id: self.run_id,
163 run_attempt: self.run_attempt,
164 branch: self.branch.clone(),
165 url: self.url.clone(),
166 at: self.finished() / SECOND,
167 }
168 }
169}
170
171#[derive(Debug, Clone, PartialEq, Eq)]
174pub struct HostHold {
175 pub memory: api::HostMemory,
177 pub seen: u64,
179 pub jobs: Vec<HeldJob>,
181}
182
183#[derive(Debug, Clone, PartialEq, Eq)]
185pub struct HeldJob {
186 pub repository: String,
188 pub workflow: Option<String>,
190 pub name: String,
192 pub waited: u64,
194}
195
196#[derive(Debug, Clone, Copy, PartialEq, Eq)]
198pub struct HostReading {
199 pub at: u64,
201 pub total: u64,
203 pub available: u64,
205 pub swap_free: Option<u64>,
207 pub credit: u64,
209 pub running: bool,
211}
212
213#[derive(Debug, Clone, Default, PartialEq, Eq)]
215pub struct History {
216 pub runs: Vec<Run>,
218 pub kept_since: u64,
221 pub host: Option<HostHold>,
223 pub readings: Vec<HostReading>,
225 pub disk: Vec<crate::observe::disk::Window>,
227}
228
229type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
231
232fn key_of(run: &Run) -> JobKey<'_> {
233 (&run.repository, run.workflow.as_deref(), &run.name)
234}
235
236pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
239 let runs = &history.runs;
240 let since = now.saturating_sub(u64::from(days) * DAY);
241 let window: Vec<&Run> = runs
242 .iter()
243 .filter(|run| {
244 matches!(run.state, JobState::Completed | JobState::Lost)
245 && run
246 .finished_at
247 .is_some_and(|at| (since..=now).contains(&at))
248 })
249 .collect();
250 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
251 for run in &window {
252 jobs.entry(key_of(run)).or_default().push(run);
253 }
254 let mut found = Vec::new();
255 for (key, runs) in &mut jobs {
256 runs.sort_by_key(|run| (run.finished(), run.id));
257 found.extend(slower(*key, runs));
258 found.extend(flaky(*key, runs));
259 found.extend(near_cap(*key, runs, config));
260 found.extend(cold_cache(*key, runs));
261 found.extend(cpu_held(*key, runs, config));
262 }
263 found.extend(throttled(&window));
264 found.extend(
265 history
266 .host
267 .as_ref()
268 .and_then(|hold| host_held(hold, config, now)),
269 );
270 let host_days = host::days(&history.readings, since, now);
271 found.extend(host::low(&host_days));
272 let waits = waits(history, now);
273 found.extend(waits.iter().filter_map(waits_insight));
274 found.sort_by(|a, b| {
275 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
276 b.severity,
277 &b.repository,
278 &b.workflow,
279 &b.job,
280 &b.key,
281 ))
282 });
283 Insights {
284 since: since / SECOND,
285 until: now / SECOND,
286 insights: found,
287 waits,
288 host: host_days,
289 disk: disk::writes(&window, &history.disk, since, now),
290 }
291}
292
293pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
296 let runs = &history.runs;
297 let since = now.saturating_sub(DAY);
298 let finished: Vec<&Run> = runs
299 .iter()
300 .filter(|run| {
301 matches!(run.state, JobState::Completed | JobState::Lost)
302 && run.finished_at.is_some_and(|at| at > since && at <= now)
303 })
304 .collect();
305 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
306 .insights
307 .into_iter()
308 .map(|insight| insight.key)
309 .collect();
310 let new = insights(history, config, api::INSIGHT_DAYS, now)
311 .insights
312 .into_iter()
313 .filter(|insight| !before.contains(&insight.key))
314 .collect();
315 let mut started: Vec<&Run> = runs
316 .iter()
317 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
318 .collect();
319 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
320 Digest {
321 since: since / SECOND,
322 until: now / SECOND,
323 finished: finished.len() as u64,
324 failed: finished.iter().filter(|run| run.failed()).count() as u64,
325 new,
326 waits: started
327 .into_iter()
328 .take(3)
329 .map(|run| LongWait {
330 job: run.id,
331 repository: run.repository.clone(),
332 workflow: run.workflow.clone(),
333 name: run.name.clone(),
334 waited: run.waited(history.kept_since).unwrap_or(0),
335 url: run.url.clone(),
336 })
337 .collect(),
338 savings: None,
339 host: host::hazard(&history.readings, since, now),
340 }
341}
342
343fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
355 let reading = hold.memory.at.saturating_mul(SECOND);
356 let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
358 let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
359 let fresh =
360 |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
361 if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
362 return None;
363 }
364 let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
365 if longest.waited < HOST_HELD_AFTER {
366 return None;
367 }
368 let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
369 let waited = longest.waited / SECOND;
370 let sentence = Insight::host_held_sentence(
371 jobs,
372 waited,
373 &subject((
374 &longest.repository,
375 longest.workflow.as_deref(),
376 &longest.name,
377 )),
378 &hold.memory,
379 );
380 Some(Insight {
381 key: format!("host-held:{}", now / HOST_HELD_EVERY),
382 severity: Severity::Warning,
383 repository: longest.repository.clone(),
384 workflow: None,
385 job: None,
386 sentence,
387 kind: InsightKind::HostHeld { jobs, waited },
388 runs: Vec::new(),
389 })
390}
391
392fn subject((repository, workflow, name): JobKey<'_>) -> String {
394 match workflow {
395 Some(workflow) => format!("{repository}: {workflow} / {name}"),
396 None => format!("{repository}: {name}"),
397 }
398}
399
400fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
401 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
402}
403
404fn insight(
405 key: JobKey<'_>,
406 kind: &str,
407 severity: Severity,
408 sentence: String,
409 found: InsightKind,
410 runs: Vec<RunLink>,
411) -> Insight {
412 Insight {
413 key: key_text(kind, key),
414 severity,
415 repository: key.0.to_owned(),
416 workflow: key.1.map(str::to_owned),
417 job: Some(key.2.to_owned()),
418 sentence,
419 kind: found,
420 runs,
421 }
422}
423
424fn median(values: &[u64]) -> u64 {
426 let mut values = values.to_vec();
427 values.sort_unstable();
428 let middle = values.len() / 2;
429 match values.len() {
430 0 => 0,
431 len if len % 2 == 1 => values[middle],
432 _ => u64::midpoint(values[middle - 1], values[middle]),
433 }
434}
435
436fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
438 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
439 runs.iter()
440 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
441 .filter_map(|run| Some((*run, run.took()?)))
442 .collect()
443 };
444 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
445 for run in runs.iter().filter(|run| run.succeeded()) {
446 if let Some(branch) = &run.branch {
447 *branches.entry(branch).or_default() += 1;
448 }
449 }
450 let busiest = branches
453 .iter()
454 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
455 .map(|(branch, _)| *branch)?;
456 let (branch, timed) = ["main", "master"]
457 .into_iter()
458 .map(|branch| (branch, timed_on(branch)))
459 .find(|(_, timed)| timed.len() >= 2 * RECENT)
460 .unwrap_or_else(|| (busiest, timed_on(busiest)));
461 if timed.len() < 2 * RECENT {
462 return None;
463 }
464 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
465 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
466 let before = median(&took(baseline));
467 let now = median(&took(recent));
468 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
469 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
472 return None;
473 }
474 let mut since = None;
476 let mut fast_in_a_row = 0;
477 for (run, took) in &timed {
478 if *took >= threshold {
479 fast_in_a_row = 0;
480 since = since.or(Some(*run));
481 } else {
482 fast_in_a_row += 1;
483 if fast_in_a_row >= 2 {
484 since = None;
485 }
486 }
487 }
488 let since = since?;
489 let sentence = format!(
490 "{} on {branch} takes {} now, up from {}.",
491 subject(key),
492 span(now),
493 span(before)
494 );
495 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
496 Some(insight(
497 key,
498 "slower",
499 Severity::Warning,
500 sentence,
501 InsightKind::Slower {
502 branch: Some(branch.to_owned()),
503 before,
504 now,
505 since: Some(since.link()),
506 },
507 links,
508 ))
509}
510
511fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
513 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
514 for run in runs {
515 by_run.entry(run.run_id).or_default().push(run);
516 }
517 let mut failed_first: Vec<&Run> = by_run
518 .values()
519 .filter_map(|attempts| {
520 attempts.iter().copied().find(|failed| {
521 failed.failed_itself()
522 && attempts
523 .iter()
524 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
525 })
526 })
527 .collect();
528 if failed_first.is_empty() {
529 return None;
530 }
531 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
532 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
533 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
534 let sentence = format!(
535 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
536 subject(key)
537 );
538 let severity = if flaky >= FLAKY_WARNING {
540 Severity::Warning
541 } else {
542 Severity::Notice
543 };
544 Some(insight(
545 key,
546 "flaky",
547 severity,
548 sentence,
549 InsightKind::Flaky { flaky, total },
550 failed_first.iter().take(5).map(|run| run.link()).collect(),
551 ))
552}
553
554fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
557 let measured = runs
558 .iter()
559 .rev()
560 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
561 let name = measured.clone().next()?.profile.as_deref()?;
562 let measured: Vec<&Run> = measured
563 .filter(|run| run.profile.as_deref() == Some(name))
564 .take(CAP_RUNS)
565 .copied()
566 .collect();
567 if measured.len() < CAP_MIN_RUNS {
568 return None;
569 }
570 let profile = config.profiles.get(name)?;
571 let cap = profile.memory.0;
572 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
573 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
574 let peak = spread(peaks.clone()).p95;
575 if cap == 0 || !near(&peak) {
576 return None;
577 }
578 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
579 let auto = if !config.sizing.auto {
580 AutoNote::Off
581 } else if profile
582 .max_memory
583 .is_some_and(|max| profile.memory.0 >= max.0)
584 {
585 AutoNote::AtMax
586 } else {
587 AutoNote::Raises
588 };
589 let what = match auto {
590 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
591 AutoNote::AtMax => {
592 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
593 }
594 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
595 };
596 let sentence = format!(
597 "{} peaks at {}, {}% of {name}'s {}. {what}",
598 subject(key),
599 ByteSize(peak),
600 peak.saturating_mul(100) / cap,
601 ByteSize(cap)
602 );
603 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
605 Severity::Notice
606 } else {
607 Severity::Warning
608 };
609 Some(insight(
610 key,
611 "near-cap",
612 severity,
613 sentence,
614 InsightKind::NearCap {
615 profile: name.to_owned(),
616 peak,
617 cap,
618 auto,
619 },
620 measured.iter().take(5).map(|run| run.link()).collect(),
621 ))
622}
623
624const COLD_RUNS: usize = 5;
626const COLD_MIN_RUNS: usize = 3;
627const COLD_MIN_LOOKUPS: u64 = 10;
628const COLD_PERCENT: u64 = 10;
631
632fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
633 let writers: Vec<&Run> = runs
634 .iter()
635 .rev()
636 .filter(|run| {
637 run.build_cache
638 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
639 })
640 .take(COLD_RUNS)
641 .copied()
642 .collect();
643 if writers.len() < COLD_MIN_RUNS {
644 return None;
645 }
646 let mut total = api::BuildCacheCounts::default();
647 for run in &writers {
648 total.add(&run.build_cache.unwrap_or_default());
649 }
650 let percent = total.hit_percent()?;
651 if percent > COLD_PERCENT {
652 return None;
653 }
654 let branch = writers.first()?.branch.clone();
655 let sentence = format!(
656 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
657 differ from run to run.",
658 subject(key),
659 branch
660 .as_deref()
661 .map(|branch| format!(" on {branch}"))
662 .unwrap_or_default(),
663 writers.len()
664 );
665 Some(insight(
666 key,
667 "cold-cache",
668 Severity::Notice,
669 sentence,
670 InsightKind::ColdCache {
671 branch,
672 hits: total.hits,
673 lookups: total.lookups(),
674 },
675 writers.iter().map(|run| run.link()).collect(),
676 ))
677}
678
679fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
686 let measured = runs.iter().rev().filter(|run| {
687 run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
688 });
689 let last = measured.clone().next()?;
690 let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
691 let profile = config.profiles.get(name)?;
692 if u64::from(profile.cpus) * 1000 != cap {
694 return None;
695 }
696 let at_cap: Vec<&Run> = measured
697 .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
698 .take(CAP_RUNS)
699 .copied()
700 .collect();
701 let held: Vec<&Run> = at_cap
702 .iter()
703 .filter(|run| run.throttling().held() == Some(true))
704 .copied()
705 .collect();
706 if (held.len() as u64) < RAISE_RUNS {
707 return None;
708 }
709 let cpus = profile.cpus;
710 let auto = config.host.admission == Admission::Measured && config.sizing.auto;
711 let raised_from = auto
712 .then(|| {
713 let ran: Vec<CpuRun> = runs
714 .iter()
715 .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
716 .map(|run| CpuRun {
717 cap: run.cpu_cap,
718 queued_at: run.queued_at,
719 throttling: run.throttling(),
720 })
721 .collect();
722 let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
723 unhelped_raise(&ran, cap, &now)
724 })
725 .flatten()
726 .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
727 let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
728 let what = match (raised_from, set) {
729 (Some(from), _) => format!(
730 "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
731 tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
732 sizing does next."
733 ),
734 (None, Some(set)) if config.host.admission == Admission::Measured => {
735 format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
736 }
737 (None, Some(set)) => format!(
738 "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
739 cpus to {set} if runnerctl sizing shows the budget has room."
740 ),
741 (None, None) => format!(
742 "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
743 room; runnerctl sizing says when it cannot."
744 ),
745 };
746 let sentence = format!(
747 "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
748 its run or more. {what}",
749 subject(key),
750 held.len(),
751 at_cap.len()
752 );
753 let severity = if auto && raised_from.is_none() {
755 Severity::Notice
756 } else {
757 Severity::Warning
758 };
759 Some(insight(
760 key,
761 "cpu-held",
762 severity,
763 sentence,
764 InsightKind::CpuHeld {
765 profile: name.to_owned(),
766 cpus,
767 held: u32::try_from(held.len()).unwrap_or(u32::MAX),
768 total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
769 raised_from,
770 set,
771 },
772 held.iter().take(5).map(|run| run.link()).collect(),
773 ))
774}
775
776fn throttled(window: &[&Run]) -> Vec<Insight> {
778 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
779 for run in window {
780 repositories.entry(&run.repository).or_default().push(run);
781 }
782 repositories
783 .into_iter()
784 .filter_map(|(repository, mut runs)| {
785 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
786 let hit: Vec<&Run> = runs
787 .iter()
788 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
789 .copied()
790 .collect();
791 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
792 if throttled < THROTTLED_MIN_RUNS {
793 return None;
794 }
795 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
796 Some(Insight {
797 key: format!("throttled:{repository}"),
798 severity: Severity::Warning,
799 repository: repository.to_owned(),
800 workflow: None,
801 job: None,
802 sentence: format!(
803 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
804 host, or a job at its cap, was short of memory."
805 ),
806 kind: InsightKind::Throttled { throttled, total },
807 runs: hit.iter().take(5).map(|run| run.link()).collect(),
808 })
809 })
810 .collect()
811}
812
813fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
817 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
818 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
819 for run in &history.runs {
820 let Some(started) = run.started_at.filter(|at| *at <= now) else {
821 continue;
822 };
823 let week = match now - started {
824 back if back < WEEK => 0,
825 back if back < 2 * WEEK && !before_kept => 1,
826 _ => continue,
827 };
828 weeks.entry(&run.repository).or_default()[week].push(run);
829 }
830 weeks
831 .into_iter()
832 .map(|(repository, [now, before])| RepositoryWaits {
833 repository: repository.to_owned(),
834 now: week_waits(&now, history.kept_since),
835 before: week_waits(&before, history.kept_since),
836 })
837 .collect()
838}
839
840fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
841 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
842 let of = |runs: Vec<&&Run>| {
843 let waited: Vec<u64> = runs
844 .iter()
845 .filter_map(|run| run.waited(kept_since))
846 .collect();
847 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
848 let spread = (!waited.is_empty()).then(|| spread(waited));
849 WaitSpread {
850 jobs,
851 p50: spread.as_ref().map(|spread| spread.p50),
852 p95: spread.as_ref().map(|spread| spread.p95),
853 }
854 };
855 WeekWaits {
856 all: of(runs.iter().collect()),
857 warm: of(runs.iter().filter(warm).collect()),
858 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
859 }
860}
861
862fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
863 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
864 return None;
865 }
866 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
867 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
868 if now < threshold {
869 return None;
870 }
871 Some(Insight {
872 key: format!("waits:{}", waits.repository),
873 severity: Severity::Notice,
874 repository: waits.repository.clone(),
875 workflow: None,
876 job: None,
877 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
878 format!(
879 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
880 waits.repository,
881 span(now),
882 span(before)
883 )
884 } else {
885 format!(
886 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
887 waits.repository,
888 span(now),
889 span(before)
890 )
891 },
892 kind: InsightKind::Waits { before, now },
893 runs: Vec::new(),
894 })
895}
896
897fn span(seconds: u64) -> String {
899 match seconds {
900 0..60 => format!("{seconds}s"),
901 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
902 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
903 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
904 }
905}
906
907#[cfg(test)]
908mod tests;