1use std::collections::{BTreeMap, BTreeSet};
10
11use crate::api::{
12 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
13 RunLink, Severity, WaitSpread, WeekWaits,
14};
15use crate::config::{Admission, ByteSize, Config};
16use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
17use crate::sizing::{HELD, Throttling, spread};
18
19mod host;
20
21const SECOND: u64 = 1000;
22const DAY: u64 = 24 * 3600 * SECOND;
23const WEEK: u64 = 7 * DAY;
24
25const RECENT: usize = 5;
27const SLOWER_PERCENT: u64 = 20;
29const SLOWER_SECONDS: u64 = 30;
31
32const CAP_RUNS: usize = 10;
34const CAP_MIN_RUNS: usize = 5;
35const NEAR_CAP: u64 = 85;
37const NEAR_CAP_WARNING: usize = 2;
39const FLAKY_WARNING: u32 = 2;
41
42const WAITS_PERCENT: u64 = 50;
45const WAITS_SECONDS: u64 = 60;
46const WAITS_MIN_JOBS: u32 = 5;
47const WAITS_ONE_IN_20: u32 = 20;
49
50const THROTTLED_MIN_RUNS: u32 = 2;
52
53const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
55const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
58const HOST_HELD_FRESH: u64 = 60 * SECOND;
62
63#[derive(Debug, Clone, PartialEq, Eq)]
65pub struct Run {
66 pub id: u64,
68 pub repository: String,
70 pub workflow: Option<String>,
72 pub name: String,
74 pub branch: Option<String>,
76 pub url: Option<String>,
78 pub run_id: u64,
80 pub run_attempt: u32,
82 pub state: JobState,
84 pub conclusion: Option<String>,
86 pub queued_at: u64,
88 pub started_at: Option<u64>,
90 pub finished_at: Option<u64>,
92 pub runner_started_at: Option<u64>,
94 pub profile: Option<String>,
96 pub memory_peak: Option<u64>,
98 pub build_cache: Option<api::BuildCacheCounts>,
100 pub sock_throttled: Option<u64>,
102 pub cpu_cap: Option<u64>,
104 pub cpu_periods: Option<u64>,
106 pub cpu_throttled: Option<u64>,
108 pub unseen: u64,
110}
111
112impl Run {
113 fn finished(&self) -> u64 {
114 self.finished_at.unwrap_or(0)
115 }
116
117 fn succeeded(&self) -> bool {
118 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
119 }
120
121 fn failed(&self) -> bool {
123 self.state == JobState::Lost || self.failed_itself()
124 }
125
126 fn failed_itself(&self) -> bool {
128 self.state == JobState::Completed
129 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
130 }
131
132 fn took(&self) -> Option<u64> {
134 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
135 }
136
137 fn waited(&self, kept_since: u64) -> Option<u64> {
139 let wait = crate::sizing::Wait {
140 queued_at: self.queued_at,
141 started_at: self.started_at?,
142 unseen: self.unseen,
143 };
144 Some(wait.counted(kept_since) / SECOND)
145 }
146
147 fn throttling(&self) -> Throttling {
148 Throttling {
149 started_at: self.started_at,
150 finished_at: self.finished_at,
151 periods: self.cpu_periods,
152 throttled: self.cpu_throttled,
153 }
154 }
155
156 fn link(&self) -> RunLink {
157 RunLink {
158 job: self.id,
159 run_id: self.run_id,
160 run_attempt: self.run_attempt,
161 branch: self.branch.clone(),
162 url: self.url.clone(),
163 at: self.finished() / SECOND,
164 }
165 }
166}
167
168#[derive(Debug, Clone, PartialEq, Eq)]
171pub struct HostHold {
172 pub memory: api::HostMemory,
174 pub seen: u64,
176 pub jobs: Vec<HeldJob>,
178}
179
180#[derive(Debug, Clone, PartialEq, Eq)]
182pub struct HeldJob {
183 pub repository: String,
185 pub workflow: Option<String>,
187 pub name: String,
189 pub waited: u64,
191}
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
195pub struct HostReading {
196 pub at: u64,
198 pub total: u64,
200 pub available: u64,
202 pub swap_free: Option<u64>,
204 pub credit: u64,
206 pub running: bool,
208}
209
210#[derive(Debug, Clone, Default, PartialEq, Eq)]
212pub struct History {
213 pub runs: Vec<Run>,
215 pub kept_since: u64,
218 pub host: Option<HostHold>,
220 pub readings: Vec<HostReading>,
222}
223
224type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
226
227fn key_of(run: &Run) -> JobKey<'_> {
228 (&run.repository, run.workflow.as_deref(), &run.name)
229}
230
231pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
234 let runs = &history.runs;
235 let since = now.saturating_sub(u64::from(days) * DAY);
236 let window: Vec<&Run> = runs
237 .iter()
238 .filter(|run| {
239 matches!(run.state, JobState::Completed | JobState::Lost)
240 && run
241 .finished_at
242 .is_some_and(|at| (since..=now).contains(&at))
243 })
244 .collect();
245 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
246 for run in &window {
247 jobs.entry(key_of(run)).or_default().push(run);
248 }
249 let mut found = Vec::new();
250 for (key, runs) in &mut jobs {
251 runs.sort_by_key(|run| (run.finished(), run.id));
252 found.extend(slower(*key, runs));
253 found.extend(flaky(*key, runs));
254 found.extend(near_cap(*key, runs, config));
255 found.extend(cold_cache(*key, runs));
256 found.extend(cpu_held(*key, runs, config));
257 }
258 found.extend(throttled(&window));
259 found.extend(
260 history
261 .host
262 .as_ref()
263 .and_then(|hold| host_held(hold, config, now)),
264 );
265 let host_days = host::days(&history.readings, since, now);
266 found.extend(host::low(&host_days));
267 let waits = waits(history, now);
268 found.extend(waits.iter().filter_map(waits_insight));
269 found.sort_by(|a, b| {
270 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
271 b.severity,
272 &b.repository,
273 &b.workflow,
274 &b.job,
275 &b.key,
276 ))
277 });
278 Insights {
279 since: since / SECOND,
280 until: now / SECOND,
281 insights: found,
282 waits,
283 host: host_days,
284 }
285}
286
287pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
290 let runs = &history.runs;
291 let since = now.saturating_sub(DAY);
292 let finished: Vec<&Run> = runs
293 .iter()
294 .filter(|run| {
295 matches!(run.state, JobState::Completed | JobState::Lost)
296 && run.finished_at.is_some_and(|at| at > since && at <= now)
297 })
298 .collect();
299 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
300 .insights
301 .into_iter()
302 .map(|insight| insight.key)
303 .collect();
304 let new = insights(history, config, api::INSIGHT_DAYS, now)
305 .insights
306 .into_iter()
307 .filter(|insight| !before.contains(&insight.key))
308 .collect();
309 let mut started: Vec<&Run> = runs
310 .iter()
311 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
312 .collect();
313 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
314 Digest {
315 since: since / SECOND,
316 until: now / SECOND,
317 finished: finished.len() as u64,
318 failed: finished.iter().filter(|run| run.failed()).count() as u64,
319 new,
320 waits: started
321 .into_iter()
322 .take(3)
323 .map(|run| LongWait {
324 job: run.id,
325 repository: run.repository.clone(),
326 workflow: run.workflow.clone(),
327 name: run.name.clone(),
328 waited: run.waited(history.kept_since).unwrap_or(0),
329 url: run.url.clone(),
330 })
331 .collect(),
332 savings: None,
333 host: host::hazard(&history.readings, since, now),
334 }
335}
336
337fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
349 let reading = hold.memory.at.saturating_mul(SECOND);
350 let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
352 let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
353 let fresh =
354 |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
355 if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
356 return None;
357 }
358 let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
359 if longest.waited < HOST_HELD_AFTER {
360 return None;
361 }
362 let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
363 let waited = longest.waited / SECOND;
364 let sentence = Insight::host_held_sentence(
365 jobs,
366 waited,
367 &subject((
368 &longest.repository,
369 longest.workflow.as_deref(),
370 &longest.name,
371 )),
372 &hold.memory,
373 );
374 Some(Insight {
375 key: format!("host-held:{}", now / HOST_HELD_EVERY),
376 severity: Severity::Warning,
377 repository: longest.repository.clone(),
378 workflow: None,
379 job: None,
380 sentence,
381 kind: InsightKind::HostHeld { jobs, waited },
382 runs: Vec::new(),
383 })
384}
385
386fn subject((repository, workflow, name): JobKey<'_>) -> String {
388 match workflow {
389 Some(workflow) => format!("{repository}: {workflow} / {name}"),
390 None => format!("{repository}: {name}"),
391 }
392}
393
394fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
395 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
396}
397
398fn insight(
399 key: JobKey<'_>,
400 kind: &str,
401 severity: Severity,
402 sentence: String,
403 found: InsightKind,
404 runs: Vec<RunLink>,
405) -> Insight {
406 Insight {
407 key: key_text(kind, key),
408 severity,
409 repository: key.0.to_owned(),
410 workflow: key.1.map(str::to_owned),
411 job: Some(key.2.to_owned()),
412 sentence,
413 kind: found,
414 runs,
415 }
416}
417
418fn median(values: &[u64]) -> u64 {
420 let mut values = values.to_vec();
421 values.sort_unstable();
422 let middle = values.len() / 2;
423 match values.len() {
424 0 => 0,
425 len if len % 2 == 1 => values[middle],
426 _ => u64::midpoint(values[middle - 1], values[middle]),
427 }
428}
429
430fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
432 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
433 runs.iter()
434 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
435 .filter_map(|run| Some((*run, run.took()?)))
436 .collect()
437 };
438 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
439 for run in runs.iter().filter(|run| run.succeeded()) {
440 if let Some(branch) = &run.branch {
441 *branches.entry(branch).or_default() += 1;
442 }
443 }
444 let busiest = branches
447 .iter()
448 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
449 .map(|(branch, _)| *branch)?;
450 let (branch, timed) = ["main", "master"]
451 .into_iter()
452 .map(|branch| (branch, timed_on(branch)))
453 .find(|(_, timed)| timed.len() >= 2 * RECENT)
454 .unwrap_or_else(|| (busiest, timed_on(busiest)));
455 if timed.len() < 2 * RECENT {
456 return None;
457 }
458 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
459 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
460 let before = median(&took(baseline));
461 let now = median(&took(recent));
462 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
463 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
466 return None;
467 }
468 let mut since = None;
470 let mut fast_in_a_row = 0;
471 for (run, took) in &timed {
472 if *took >= threshold {
473 fast_in_a_row = 0;
474 since = since.or(Some(*run));
475 } else {
476 fast_in_a_row += 1;
477 if fast_in_a_row >= 2 {
478 since = None;
479 }
480 }
481 }
482 let since = since?;
483 let sentence = format!(
484 "{} on {branch} takes {} now, up from {}.",
485 subject(key),
486 span(now),
487 span(before)
488 );
489 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
490 Some(insight(
491 key,
492 "slower",
493 Severity::Warning,
494 sentence,
495 InsightKind::Slower {
496 branch: Some(branch.to_owned()),
497 before,
498 now,
499 since: Some(since.link()),
500 },
501 links,
502 ))
503}
504
505fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
507 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
508 for run in runs {
509 by_run.entry(run.run_id).or_default().push(run);
510 }
511 let mut failed_first: Vec<&Run> = by_run
512 .values()
513 .filter_map(|attempts| {
514 attempts.iter().copied().find(|failed| {
515 failed.failed_itself()
516 && attempts
517 .iter()
518 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
519 })
520 })
521 .collect();
522 if failed_first.is_empty() {
523 return None;
524 }
525 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
526 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
527 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
528 let sentence = format!(
529 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
530 subject(key)
531 );
532 let severity = if flaky >= FLAKY_WARNING {
534 Severity::Warning
535 } else {
536 Severity::Notice
537 };
538 Some(insight(
539 key,
540 "flaky",
541 severity,
542 sentence,
543 InsightKind::Flaky { flaky, total },
544 failed_first.iter().take(5).map(|run| run.link()).collect(),
545 ))
546}
547
548fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
551 let measured = runs
552 .iter()
553 .rev()
554 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
555 let name = measured.clone().next()?.profile.as_deref()?;
556 let measured: Vec<&Run> = measured
557 .filter(|run| run.profile.as_deref() == Some(name))
558 .take(CAP_RUNS)
559 .copied()
560 .collect();
561 if measured.len() < CAP_MIN_RUNS {
562 return None;
563 }
564 let profile = config.profiles.get(name)?;
565 let cap = profile.memory.0;
566 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
567 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
568 let peak = spread(peaks.clone()).p95;
569 if cap == 0 || !near(&peak) {
570 return None;
571 }
572 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
573 let auto = if !config.sizing.auto {
574 AutoNote::Off
575 } else if profile
576 .max_memory
577 .is_some_and(|max| profile.memory.0 >= max.0)
578 {
579 AutoNote::AtMax
580 } else {
581 AutoNote::Raises
582 };
583 let what = match auto {
584 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
585 AutoNote::AtMax => {
586 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
587 }
588 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
589 };
590 let sentence = format!(
591 "{} peaks at {}, {}% of {name}'s {}. {what}",
592 subject(key),
593 ByteSize(peak),
594 peak.saturating_mul(100) / cap,
595 ByteSize(cap)
596 );
597 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
599 Severity::Notice
600 } else {
601 Severity::Warning
602 };
603 Some(insight(
604 key,
605 "near-cap",
606 severity,
607 sentence,
608 InsightKind::NearCap {
609 profile: name.to_owned(),
610 peak,
611 cap,
612 auto,
613 },
614 measured.iter().take(5).map(|run| run.link()).collect(),
615 ))
616}
617
618const COLD_RUNS: usize = 5;
620const COLD_MIN_RUNS: usize = 3;
621const COLD_MIN_LOOKUPS: u64 = 10;
622const COLD_PERCENT: u64 = 10;
625
626fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
627 let writers: Vec<&Run> = runs
628 .iter()
629 .rev()
630 .filter(|run| {
631 run.build_cache
632 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
633 })
634 .take(COLD_RUNS)
635 .copied()
636 .collect();
637 if writers.len() < COLD_MIN_RUNS {
638 return None;
639 }
640 let mut total = api::BuildCacheCounts::default();
641 for run in &writers {
642 total.add(&run.build_cache.unwrap_or_default());
643 }
644 let percent = total.hit_percent()?;
645 if percent > COLD_PERCENT {
646 return None;
647 }
648 let branch = writers.first()?.branch.clone();
649 let sentence = format!(
650 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
651 differ from run to run.",
652 subject(key),
653 branch
654 .as_deref()
655 .map(|branch| format!(" on {branch}"))
656 .unwrap_or_default(),
657 writers.len()
658 );
659 Some(insight(
660 key,
661 "cold-cache",
662 Severity::Notice,
663 sentence,
664 InsightKind::ColdCache {
665 branch,
666 hits: total.hits,
667 lookups: total.lookups(),
668 },
669 writers.iter().map(|run| run.link()).collect(),
670 ))
671}
672
673fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
680 let measured = runs.iter().rev().filter(|run| {
681 run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
682 });
683 let last = measured.clone().next()?;
684 let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
685 let profile = config.profiles.get(name)?;
686 if u64::from(profile.cpus) * 1000 != cap {
688 return None;
689 }
690 let at_cap: Vec<&Run> = measured
691 .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
692 .take(CAP_RUNS)
693 .copied()
694 .collect();
695 let held: Vec<&Run> = at_cap
696 .iter()
697 .filter(|run| run.throttling().held() == Some(true))
698 .copied()
699 .collect();
700 if (held.len() as u64) < RAISE_RUNS {
701 return None;
702 }
703 let cpus = profile.cpus;
704 let auto = config.host.admission == Admission::Measured && config.sizing.auto;
705 let raised_from = auto
706 .then(|| {
707 let ran: Vec<CpuRun> = runs
708 .iter()
709 .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
710 .map(|run| CpuRun {
711 cap: run.cpu_cap,
712 queued_at: run.queued_at,
713 throttling: run.throttling(),
714 })
715 .collect();
716 let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
717 unhelped_raise(&ran, cap, &now)
718 })
719 .flatten()
720 .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
721 let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
722 let what = match (raised_from, set) {
723 (Some(from), _) => format!(
724 "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
725 tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
726 sizing does next."
727 ),
728 (None, Some(set)) if config.host.admission == Admission::Measured => {
729 format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
730 }
731 (None, Some(set)) => format!(
732 "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
733 cpus to {set} if runnerctl sizing shows the budget has room."
734 ),
735 (None, None) => format!(
736 "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
737 room; runnerctl sizing says when it cannot."
738 ),
739 };
740 let sentence = format!(
741 "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
742 its run or more. {what}",
743 subject(key),
744 held.len(),
745 at_cap.len()
746 );
747 let severity = if auto && raised_from.is_none() {
749 Severity::Notice
750 } else {
751 Severity::Warning
752 };
753 Some(insight(
754 key,
755 "cpu-held",
756 severity,
757 sentence,
758 InsightKind::CpuHeld {
759 profile: name.to_owned(),
760 cpus,
761 held: u32::try_from(held.len()).unwrap_or(u32::MAX),
762 total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
763 raised_from,
764 set,
765 },
766 held.iter().take(5).map(|run| run.link()).collect(),
767 ))
768}
769
770fn throttled(window: &[&Run]) -> Vec<Insight> {
772 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
773 for run in window {
774 repositories.entry(&run.repository).or_default().push(run);
775 }
776 repositories
777 .into_iter()
778 .filter_map(|(repository, mut runs)| {
779 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
780 let hit: Vec<&Run> = runs
781 .iter()
782 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
783 .copied()
784 .collect();
785 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
786 if throttled < THROTTLED_MIN_RUNS {
787 return None;
788 }
789 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
790 Some(Insight {
791 key: format!("throttled:{repository}"),
792 severity: Severity::Warning,
793 repository: repository.to_owned(),
794 workflow: None,
795 job: None,
796 sentence: format!(
797 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
798 host, or a job at its cap, was short of memory."
799 ),
800 kind: InsightKind::Throttled { throttled, total },
801 runs: hit.iter().take(5).map(|run| run.link()).collect(),
802 })
803 })
804 .collect()
805}
806
807fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
811 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
812 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
813 for run in &history.runs {
814 let Some(started) = run.started_at.filter(|at| *at <= now) else {
815 continue;
816 };
817 let week = match now - started {
818 back if back < WEEK => 0,
819 back if back < 2 * WEEK && !before_kept => 1,
820 _ => continue,
821 };
822 weeks.entry(&run.repository).or_default()[week].push(run);
823 }
824 weeks
825 .into_iter()
826 .map(|(repository, [now, before])| RepositoryWaits {
827 repository: repository.to_owned(),
828 now: week_waits(&now, history.kept_since),
829 before: week_waits(&before, history.kept_since),
830 })
831 .collect()
832}
833
834fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
835 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
836 let of = |runs: Vec<&&Run>| {
837 let waited: Vec<u64> = runs
838 .iter()
839 .filter_map(|run| run.waited(kept_since))
840 .collect();
841 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
842 let spread = (!waited.is_empty()).then(|| spread(waited));
843 WaitSpread {
844 jobs,
845 p50: spread.as_ref().map(|spread| spread.p50),
846 p95: spread.as_ref().map(|spread| spread.p95),
847 }
848 };
849 WeekWaits {
850 all: of(runs.iter().collect()),
851 warm: of(runs.iter().filter(warm).collect()),
852 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
853 }
854}
855
856fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
857 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
858 return None;
859 }
860 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
861 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
862 if now < threshold {
863 return None;
864 }
865 Some(Insight {
866 key: format!("waits:{}", waits.repository),
867 severity: Severity::Notice,
868 repository: waits.repository.clone(),
869 workflow: None,
870 job: None,
871 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
872 format!(
873 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
874 waits.repository,
875 span(now),
876 span(before)
877 )
878 } else {
879 format!(
880 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
881 waits.repository,
882 span(now),
883 span(before)
884 )
885 },
886 kind: InsightKind::Waits { before, now },
887 runs: Vec::new(),
888 })
889}
890
891fn span(seconds: u64) -> String {
893 match seconds {
894 0..60 => format!("{seconds}s"),
895 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
896 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
897 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
898 }
899}
900
901#[cfg(test)]
902mod tests;