1use std::collections::{BTreeMap, BTreeSet};
10
11use crate::api::{
12 self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
13 RunLink, Severity, WaitSpread, WeekWaits,
14};
15use crate::config::{Admission, ByteSize, Config};
16use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
17use crate::sizing::{HELD, Throttling, spread};
18
19mod disk;
20mod host;
21mod queue;
22
23const SECOND: u64 = 1000;
24const DAY: u64 = 24 * 3600 * SECOND;
25const WEEK: u64 = 7 * DAY;
26
27const RECENT: usize = 5;
29const SLOWER_PERCENT: u64 = 20;
31const SLOWER_SECONDS: u64 = 30;
33
34const CAP_RUNS: usize = 10;
36const CAP_MIN_RUNS: usize = 5;
37const NEAR_CAP: u64 = 85;
39const NEAR_CAP_WARNING: usize = 2;
41const FLAKY_WARNING: u32 = 2;
43
44const WAITS_PERCENT: u64 = 50;
47const WAITS_SECONDS: u64 = 60;
48const WAITS_MIN_JOBS: u32 = 5;
49const WAITS_ONE_IN_20: u32 = 20;
51
52const THROTTLED_MIN_RUNS: u32 = 2;
54
55const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
57const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
60const HOST_HELD_FRESH: u64 = 60 * SECOND;
64
65#[derive(Debug, Clone, PartialEq, Eq)]
67pub struct Run {
68 pub id: u64,
70 pub repository: String,
72 pub workflow: Option<String>,
74 pub name: String,
76 pub branch: Option<String>,
78 pub url: Option<String>,
80 pub run_id: u64,
82 pub run_attempt: u32,
84 pub state: JobState,
86 pub conclusion: Option<String>,
88 pub queued_at: u64,
90 pub started_at: Option<u64>,
92 pub finished_at: Option<u64>,
94 pub runner_started_at: Option<u64>,
96 pub profile: Option<String>,
98 pub memory_peak: Option<u64>,
100 pub build_cache: Option<api::BuildCacheCounts>,
102 pub sock_throttled: Option<u64>,
104 pub cpu_cap: Option<u64>,
106 pub cpu_periods: Option<u64>,
108 pub cpu_throttled: Option<u64>,
110 pub disk: Option<api::DiskWritten>,
112 pub unseen: u64,
114 pub seen_at: Option<u64>,
116 pub unseen_unlisted: u64,
118 pub causes: Option<BTreeMap<api::WaitCause, u64>>,
120}
121
122impl Run {
123 fn finished(&self) -> u64 {
124 self.finished_at.unwrap_or(0)
125 }
126
127 fn succeeded(&self) -> bool {
128 self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
129 }
130
131 fn failed(&self) -> bool {
133 self.state == JobState::Lost || self.failed_itself()
134 }
135
136 fn failed_itself(&self) -> bool {
138 self.state == JobState::Completed
139 && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
140 }
141
142 fn took(&self) -> Option<u64> {
144 Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
145 }
146
147 fn waited(&self, kept_since: u64) -> Option<u64> {
149 let wait = crate::sizing::Wait {
150 queued_at: self.queued_at,
151 started_at: self.started_at?,
152 unseen: self.unseen,
153 };
154 Some(wait.counted(kept_since) / SECOND)
155 }
156
157 fn throttling(&self) -> Throttling {
158 Throttling {
159 started_at: self.started_at,
160 finished_at: self.finished_at,
161 periods: self.cpu_periods,
162 throttled: self.cpu_throttled,
163 }
164 }
165
166 fn link(&self) -> RunLink {
167 RunLink {
168 job: self.id,
169 run_id: self.run_id,
170 run_attempt: self.run_attempt,
171 branch: self.branch.clone(),
172 url: self.url.clone(),
173 at: self.finished() / SECOND,
174 }
175 }
176}
177
178#[derive(Debug, Clone, PartialEq, Eq)]
181pub struct HostHold {
182 pub memory: api::HostMemory,
184 pub seen: u64,
186 pub jobs: Vec<HeldJob>,
188}
189
190#[derive(Debug, Clone, PartialEq, Eq)]
192pub struct HeldJob {
193 pub repository: String,
195 pub workflow: Option<String>,
197 pub name: String,
199 pub waited: u64,
201}
202
203#[derive(Debug, Clone, Copy, PartialEq, Eq)]
205pub struct HostReading {
206 pub at: u64,
208 pub total: u64,
210 pub available: u64,
212 pub swap_free: Option<u64>,
214 pub credit: u64,
216 pub running: bool,
218}
219
220#[derive(Debug, Clone, Default, PartialEq, Eq)]
222pub struct History {
223 pub runs: Vec<Run>,
225 pub kept_since: u64,
228 pub host: Option<HostHold>,
230 pub readings: Vec<HostReading>,
232 pub disk: Vec<crate::observe::disk::Window>,
234}
235
236type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
238
239fn key_of(run: &Run) -> JobKey<'_> {
240 (&run.repository, run.workflow.as_deref(), &run.name)
241}
242
243pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
246 let runs = &history.runs;
247 let since = now.saturating_sub(u64::from(days) * DAY);
248 let window: Vec<&Run> = runs
249 .iter()
250 .filter(|run| {
251 matches!(run.state, JobState::Completed | JobState::Lost)
252 && run
253 .finished_at
254 .is_some_and(|at| (since..=now).contains(&at))
255 })
256 .collect();
257 let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
258 for run in &window {
259 jobs.entry(key_of(run)).or_default().push(run);
260 }
261 let mut found = Vec::new();
262 for (key, runs) in &mut jobs {
263 runs.sort_by_key(|run| (run.finished(), run.id));
264 found.extend(slower(*key, runs));
265 found.extend(flaky(*key, runs));
266 found.extend(near_cap(*key, runs, config));
267 found.extend(cold_cache(*key, runs));
268 found.extend(cpu_held(*key, runs, config));
269 }
270 found.extend(throttled(&window));
271 found.extend(
272 history
273 .host
274 .as_ref()
275 .and_then(|hold| host_held(hold, config, now)),
276 );
277 let host_days = host::days(&history.readings, since, now);
278 found.extend(host::low(&host_days));
279 let waits = waits(history, now);
280 found.extend(waits.iter().filter_map(waits_insight));
281 found.sort_by(|a, b| {
282 (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
283 b.severity,
284 &b.repository,
285 &b.workflow,
286 &b.job,
287 &b.key,
288 ))
289 });
290 Insights {
291 since: since / SECOND,
292 until: now / SECOND,
293 insights: found,
294 waits,
295 host: host_days,
296 disk: disk::writes(&window, &history.disk, since, now),
297 queue: queue::split(&window, history.kept_since),
298 }
299}
300
301pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
304 let runs = &history.runs;
305 let since = now.saturating_sub(DAY);
306 let finished: Vec<&Run> = runs
307 .iter()
308 .filter(|run| {
309 matches!(run.state, JobState::Completed | JobState::Lost)
310 && run.finished_at.is_some_and(|at| at > since && at <= now)
311 })
312 .collect();
313 let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
314 .insights
315 .into_iter()
316 .map(|insight| insight.key)
317 .collect();
318 let new = insights(history, config, api::INSIGHT_DAYS, now)
319 .insights
320 .into_iter()
321 .filter(|insight| !before.contains(&insight.key))
322 .collect();
323 let mut started: Vec<&Run> = runs
324 .iter()
325 .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
326 .collect();
327 started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
328 Digest {
329 since: since / SECOND,
330 until: now / SECOND,
331 finished: finished.len() as u64,
332 failed: finished.iter().filter(|run| run.failed()).count() as u64,
333 new,
334 waits: started
335 .into_iter()
336 .take(3)
337 .map(|run| LongWait {
338 job: run.id,
339 repository: run.repository.clone(),
340 workflow: run.workflow.clone(),
341 name: run.name.clone(),
342 waited: run.waited(history.kept_since).unwrap_or(0),
343 url: run.url.clone(),
344 })
345 .collect(),
346 savings: None,
347 host: host::hazard(&history.readings, since, now),
348 }
349}
350
351fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
363 let reading = hold.memory.at.saturating_mul(SECOND);
364 let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
366 let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
367 let fresh =
368 |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
369 if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
370 return None;
371 }
372 let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
373 if longest.waited < HOST_HELD_AFTER {
374 return None;
375 }
376 let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
377 let waited = longest.waited / SECOND;
378 let sentence = Insight::host_held_sentence(
379 jobs,
380 waited,
381 &subject((
382 &longest.repository,
383 longest.workflow.as_deref(),
384 &longest.name,
385 )),
386 &hold.memory,
387 );
388 Some(Insight {
389 key: format!("host-held:{}", now / HOST_HELD_EVERY),
390 severity: Severity::Warning,
391 repository: longest.repository.clone(),
392 workflow: None,
393 job: None,
394 sentence,
395 kind: InsightKind::HostHeld { jobs, waited },
396 runs: Vec::new(),
397 })
398}
399
400fn subject((repository, workflow, name): JobKey<'_>) -> String {
402 match workflow {
403 Some(workflow) => format!("{repository}: {workflow} / {name}"),
404 None => format!("{repository}: {name}"),
405 }
406}
407
408fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
409 format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
410}
411
412fn insight(
413 key: JobKey<'_>,
414 kind: &str,
415 severity: Severity,
416 sentence: String,
417 found: InsightKind,
418 runs: Vec<RunLink>,
419) -> Insight {
420 Insight {
421 key: key_text(kind, key),
422 severity,
423 repository: key.0.to_owned(),
424 workflow: key.1.map(str::to_owned),
425 job: Some(key.2.to_owned()),
426 sentence,
427 kind: found,
428 runs,
429 }
430}
431
432fn median(values: &[u64]) -> u64 {
434 let mut values = values.to_vec();
435 values.sort_unstable();
436 let middle = values.len() / 2;
437 match values.len() {
438 0 => 0,
439 len if len % 2 == 1 => values[middle],
440 _ => u64::midpoint(values[middle - 1], values[middle]),
441 }
442}
443
444fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
446 let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
447 runs.iter()
448 .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
449 .filter_map(|run| Some((*run, run.took()?)))
450 .collect()
451 };
452 let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
453 for run in runs.iter().filter(|run| run.succeeded()) {
454 if let Some(branch) = &run.branch {
455 *branches.entry(branch).or_default() += 1;
456 }
457 }
458 let busiest = branches
461 .iter()
462 .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
463 .map(|(branch, _)| *branch)?;
464 let (branch, timed) = ["main", "master"]
465 .into_iter()
466 .map(|branch| (branch, timed_on(branch)))
467 .find(|(_, timed)| timed.len() >= 2 * RECENT)
468 .unwrap_or_else(|| (busiest, timed_on(busiest)));
469 if timed.len() < 2 * RECENT {
470 return None;
471 }
472 let (baseline, recent) = timed.split_at(timed.len() - RECENT);
473 let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
474 let before = median(&took(baseline));
475 let now = median(&took(recent));
476 let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
477 if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
480 return None;
481 }
482 let mut since = None;
484 let mut fast_in_a_row = 0;
485 for (run, took) in &timed {
486 if *took >= threshold {
487 fast_in_a_row = 0;
488 since = since.or(Some(*run));
489 } else {
490 fast_in_a_row += 1;
491 if fast_in_a_row >= 2 {
492 since = None;
493 }
494 }
495 }
496 let since = since?;
497 let sentence = format!(
498 "{} on {branch} takes {} now, up from {}.",
499 subject(key),
500 span(now),
501 span(before)
502 );
503 let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
504 Some(insight(
505 key,
506 "slower",
507 Severity::Warning,
508 sentence,
509 InsightKind::Slower {
510 branch: Some(branch.to_owned()),
511 before,
512 now,
513 since: Some(since.link()),
514 },
515 links,
516 ))
517}
518
519fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
521 let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
522 for run in runs {
523 by_run.entry(run.run_id).or_default().push(run);
524 }
525 let mut failed_first: Vec<&Run> = by_run
526 .values()
527 .filter_map(|attempts| {
528 attempts.iter().copied().find(|failed| {
529 failed.failed_itself()
530 && attempts
531 .iter()
532 .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
533 })
534 })
535 .collect();
536 if failed_first.is_empty() {
537 return None;
538 }
539 failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
540 let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
541 let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
542 let sentence = format!(
543 "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
544 subject(key)
545 );
546 let severity = if flaky >= FLAKY_WARNING {
548 Severity::Warning
549 } else {
550 Severity::Notice
551 };
552 Some(insight(
553 key,
554 "flaky",
555 severity,
556 sentence,
557 InsightKind::Flaky { flaky, total },
558 failed_first.iter().take(5).map(|run| run.link()).collect(),
559 ))
560}
561
562fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
565 let measured = runs
566 .iter()
567 .rev()
568 .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
569 let name = measured.clone().next()?.profile.as_deref()?;
570 let measured: Vec<&Run> = measured
571 .filter(|run| run.profile.as_deref() == Some(name))
572 .take(CAP_RUNS)
573 .copied()
574 .collect();
575 if measured.len() < CAP_MIN_RUNS {
576 return None;
577 }
578 let profile = config.profiles.get(name)?;
579 let cap = profile.memory.0;
580 let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
581 let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
582 let peak = spread(peaks.clone()).p95;
583 if cap == 0 || !near(&peak) {
584 return None;
585 }
586 let near_runs = peaks.iter().filter(|peak| near(peak)).count();
587 let auto = if !config.sizing.auto {
588 AutoNote::Off
589 } else if profile
590 .max_memory
591 .is_some_and(|max| profile.memory.0 >= max.0)
592 {
593 AutoNote::AtMax
594 } else {
595 AutoNote::Raises
596 };
597 let what = match auto {
598 AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
599 AutoNote::AtMax => {
600 format!("{name} is at its max_memory, so auto sizing cannot raise it.")
601 }
602 AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
603 };
604 let sentence = format!(
605 "{} peaks at {}, {}% of {name}'s {}. {what}",
606 subject(key),
607 ByteSize(peak),
608 peak.saturating_mul(100) / cap,
609 ByteSize(cap)
610 );
611 let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
613 Severity::Notice
614 } else {
615 Severity::Warning
616 };
617 Some(insight(
618 key,
619 "near-cap",
620 severity,
621 sentence,
622 InsightKind::NearCap {
623 profile: name.to_owned(),
624 peak,
625 cap,
626 auto,
627 },
628 measured.iter().take(5).map(|run| run.link()).collect(),
629 ))
630}
631
632const COLD_RUNS: usize = 5;
634const COLD_MIN_RUNS: usize = 3;
635const COLD_MIN_LOOKUPS: u64 = 10;
636const COLD_PERCENT: u64 = 10;
639
640fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
641 let writers: Vec<&Run> = runs
642 .iter()
643 .rev()
644 .filter(|run| {
645 run.build_cache
646 .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
647 })
648 .take(COLD_RUNS)
649 .copied()
650 .collect();
651 if writers.len() < COLD_MIN_RUNS {
652 return None;
653 }
654 let mut total = api::BuildCacheCounts::default();
655 for run in &writers {
656 total.add(&run.build_cache.unwrap_or_default());
657 }
658 let percent = total.hit_percent()?;
659 if percent > COLD_PERCENT {
660 return None;
661 }
662 let branch = writers.first()?.branch.clone();
663 let sentence = format!(
664 "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
665 differ from run to run.",
666 subject(key),
667 branch
668 .as_deref()
669 .map(|branch| format!(" on {branch}"))
670 .unwrap_or_default(),
671 writers.len()
672 );
673 Some(insight(
674 key,
675 "cold-cache",
676 Severity::Notice,
677 sentence,
678 InsightKind::ColdCache {
679 branch,
680 hits: total.hits,
681 lookups: total.lookups(),
682 },
683 writers.iter().map(|run| run.link()).collect(),
684 ))
685}
686
687fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
694 let measured = runs.iter().rev().filter(|run| {
695 run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
696 });
697 let last = measured.clone().next()?;
698 let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
699 let profile = config.profiles.get(name)?;
700 if u64::from(profile.cpus) * 1000 != cap {
702 return None;
703 }
704 let at_cap: Vec<&Run> = measured
705 .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
706 .take(CAP_RUNS)
707 .copied()
708 .collect();
709 let held: Vec<&Run> = at_cap
710 .iter()
711 .filter(|run| run.throttling().held() == Some(true))
712 .copied()
713 .collect();
714 if (held.len() as u64) < RAISE_RUNS {
715 return None;
716 }
717 let cpus = profile.cpus;
718 let auto = config.host.admission == Admission::Measured && config.sizing.auto;
719 let raised_from = auto
720 .then(|| {
721 let ran: Vec<CpuRun> = runs
722 .iter()
723 .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
724 .map(|run| CpuRun {
725 cap: run.cpu_cap,
726 queued_at: run.queued_at,
727 throttling: run.throttling(),
728 })
729 .collect();
730 let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
731 unhelped_raise(&ran, cap, &now)
732 })
733 .flatten()
734 .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
735 let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
736 let what = match (raised_from, set) {
737 (Some(from), _) => format!(
738 "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
739 tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
740 sizing does next."
741 ),
742 (None, Some(set)) if config.host.admission == Admission::Measured => {
743 format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
744 }
745 (None, Some(set)) => format!(
746 "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
747 cpus to {set} if runnerctl sizing shows the budget has room."
748 ),
749 (None, None) => format!(
750 "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
751 room; runnerctl sizing says when it cannot."
752 ),
753 };
754 let sentence = format!(
755 "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
756 its run or more. {what}",
757 subject(key),
758 held.len(),
759 at_cap.len()
760 );
761 let severity = if auto && raised_from.is_none() {
763 Severity::Notice
764 } else {
765 Severity::Warning
766 };
767 Some(insight(
768 key,
769 "cpu-held",
770 severity,
771 sentence,
772 InsightKind::CpuHeld {
773 profile: name.to_owned(),
774 cpus,
775 held: u32::try_from(held.len()).unwrap_or(u32::MAX),
776 total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
777 raised_from,
778 set,
779 },
780 held.iter().take(5).map(|run| run.link()).collect(),
781 ))
782}
783
784fn throttled(window: &[&Run]) -> Vec<Insight> {
786 let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
787 for run in window {
788 repositories.entry(&run.repository).or_default().push(run);
789 }
790 repositories
791 .into_iter()
792 .filter_map(|(repository, mut runs)| {
793 runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
794 let hit: Vec<&Run> = runs
795 .iter()
796 .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
797 .copied()
798 .collect();
799 let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
800 if throttled < THROTTLED_MIN_RUNS {
801 return None;
802 }
803 let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
804 Some(Insight {
805 key: format!("throttled:{repository}"),
806 severity: Severity::Warning,
807 repository: repository.to_owned(),
808 workflow: None,
809 job: None,
810 sentence: format!(
811 "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
812 host, or a job at its cap, was short of memory."
813 ),
814 kind: InsightKind::Throttled { throttled, total },
815 runs: hit.iter().take(5).map(|run| run.link()).collect(),
816 })
817 })
818 .collect()
819}
820
821fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
825 let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
826 let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
827 for run in &history.runs {
828 let Some(started) = run.started_at.filter(|at| *at <= now) else {
829 continue;
830 };
831 let week = match now - started {
832 back if back < WEEK => 0,
833 back if back < 2 * WEEK && !before_kept => 1,
834 _ => continue,
835 };
836 weeks.entry(&run.repository).or_default()[week].push(run);
837 }
838 weeks
839 .into_iter()
840 .map(|(repository, [now, before])| RepositoryWaits {
841 repository: repository.to_owned(),
842 now: week_waits(&now, history.kept_since),
843 before: week_waits(&before, history.kept_since),
844 })
845 .collect()
846}
847
848fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
849 let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
850 let of = |runs: Vec<&&Run>| {
851 let waited: Vec<u64> = runs
852 .iter()
853 .filter_map(|run| run.waited(kept_since))
854 .collect();
855 let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
856 let spread = (!waited.is_empty()).then(|| spread(waited));
857 WaitSpread {
858 jobs,
859 p50: spread.as_ref().map(|spread| spread.p50),
860 p95: spread.as_ref().map(|spread| spread.p95),
861 }
862 };
863 WeekWaits {
864 all: of(runs.iter().collect()),
865 warm: of(runs.iter().filter(warm).collect()),
866 cold: of(runs.iter().filter(|run| !warm(run)).collect()),
867 }
868}
869
870fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
871 if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
872 return None;
873 }
874 let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
875 let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
876 if now < threshold {
877 return None;
878 }
879 Some(Insight {
880 key: format!("waits:{}", waits.repository),
881 severity: Severity::Notice,
882 repository: waits.repository.clone(),
883 workflow: None,
884 job: None,
885 sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
886 format!(
887 "{}: the longest wait for a runner this week is {}, up from {} the week before.",
888 waits.repository,
889 span(now),
890 span(before)
891 )
892 } else {
893 format!(
894 "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
895 waits.repository,
896 span(now),
897 span(before)
898 )
899 },
900 kind: InsightKind::Waits { before, now },
901 runs: Vec::new(),
902 })
903}
904
905fn span(seconds: u64) -> String {
907 match seconds {
908 0..60 => format!("{seconds}s"),
909 60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
910 60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
911 _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
912 }
913}
914
915#[cfg(test)]
916mod tests;