Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, jobs their CPU cap held back by the rule auto sizing reads (ADR 0087), and
4//! how long each repository's jobs wait, this week against the last when the history holds all of last week; and the
5//! daily digest. Pure: the daemon reads the history, the config and the clock and passes them in. It sends nothing
6//! and changes nothing.
7
8use std::collections::{BTreeMap, BTreeSet};
9
10use crate::api::{
11    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
12    RunLink, Severity, WaitSpread, WeekWaits,
13};
14use crate::config::{Admission, ByteSize, Config};
15use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
16use crate::sizing::{HELD, Throttling, spread};
17
18const SECOND: u64 = 1000;
19const DAY: u64 = 24 * 3600 * SECOND;
20const WEEK: u64 = 7 * DAY;
21
22/// The last runs a slower job's median is taken over; the baseline needs as many before them.
23const RECENT: usize = 5;
24/// A job is slower when its recent median is this share above its baseline's, in percent…
25const SLOWER_PERCENT: u64 = 20;
26/// …and at least this many seconds above it.
27const SLOWER_SECONDS: u64 = 30;
28
29/// The last runs whose memory peaks are read, and the fewest that say anything.
30const CAP_RUNS: usize = 10;
31const CAP_MIN_RUNS: usize = 5;
32/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
33const NEAR_CAP: u64 = 85;
34/// Runs at `NEAR_CAP` or more that make it a warning.
35const NEAR_CAP_WARNING: usize = 2;
36/// Flaky runs that make it a warning.
37const FLAKY_WARNING: u32 = 2;
38
39/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
40/// above it, each week with this many jobs.
41const WAITS_PERCENT: u64 = 50;
42const WAITS_SECONDS: u64 = 60;
43const WAITS_MIN_JOBS: u32 = 5;
44/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
45const WAITS_ONE_IN_20: u32 = 20;
46
47/// A repository's runs the kernel throttled for memory pressure that make an insight.
48const THROTTLED_MIN_RUNS: u32 = 2;
49
50/// One job the history closed, as insights read it. Times in milliseconds.
51#[derive(Debug, Clone, PartialEq, Eq)]
52pub struct Run {
53    /// GitHub's job ID.
54    pub id: u64,
55    /// `owner/name`.
56    pub repository: String,
57    /// The workflow's name, when known.
58    pub workflow: Option<String>,
59    /// The job's name.
60    pub name: String,
61    /// The branch the run is for.
62    pub branch: Option<String>,
63    /// The job's page on GitHub.
64    pub url: Option<String>,
65    /// The workflow run.
66    pub run_id: u64,
67    /// The run's attempt, 1 for the first.
68    pub run_attempt: u32,
69    /// `Completed` or `Lost`.
70    pub state: JobState,
71    /// GitHub's conclusion, for a completed job.
72    pub conclusion: Option<String>,
73    /// When GitHub queued it.
74    pub queued_at: u64,
75    /// When it started.
76    pub started_at: Option<u64>,
77    /// When it finished, or was found lost.
78    pub finished_at: Option<u64>,
79    /// When its runner's container started: before `queued_at`, it found a warm runner.
80    pub runner_started_at: Option<u64>,
81    /// The profile its runner ran in.
82    pub profile: Option<String>,
83    /// Its sampled memory peak.
84    pub memory_peak: Option<u64>,
85    /// What the build cache served it, if its runner had the build cache.
86    pub build_cache: Option<api::BuildCacheCounts>,
87    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
88    pub sock_throttled: Option<u64>,
89    /// Its container's CPU cap, millicores, when recorded.
90    pub cpu_cap: Option<u64>,
91    /// The periods of its CPU quota it had work in; `None` where the kernel kept no figure, as before schema 17.
92    pub cpu_periods: Option<u64>,
93    /// Of those, the periods its quota throttled it in.
94    pub cpu_throttled: Option<u64>,
95    /// How much of its wait passed while no runnerd ran.
96    pub unseen: u64,
97}
98
99impl Run {
100    fn finished(&self) -> u64 {
101        self.finished_at.unwrap_or(0)
102    }
103
104    fn succeeded(&self) -> bool {
105        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
106    }
107
108    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
109    fn failed(&self) -> bool {
110        self.state == JobState::Lost || self.failed_itself()
111    }
112
113    /// Failed or timed out: what the job did, not what became of its runner.
114    fn failed_itself(&self) -> bool {
115        self.state == JobState::Completed
116            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
117    }
118
119    /// Seconds from start to finish.
120    fn took(&self) -> Option<u64> {
121        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
122    }
123
124    /// Its [wait](crate::sizing::Wait) in seconds, once it started.
125    fn waited(&self, kept_since: u64) -> Option<u64> {
126        let wait = crate::sizing::Wait {
127            queued_at: self.queued_at,
128            started_at: self.started_at?,
129            unseen: self.unseen,
130        };
131        Some(wait.counted(kept_since) / SECOND)
132    }
133
134    fn throttling(&self) -> Throttling {
135        Throttling {
136            started_at: self.started_at,
137            finished_at: self.finished_at,
138            periods: self.cpu_periods,
139            throttled: self.cpu_throttled,
140        }
141    }
142
143    fn link(&self) -> RunLink {
144        RunLink {
145            job: self.id,
146            run_id: self.run_id,
147            run_attempt: self.run_attempt,
148            branch: self.branch.clone(),
149            url: self.url.clone(),
150            at: self.finished() / SECOND,
151        }
152    }
153}
154
155/// What insights read from the job history.
156#[derive(Debug, Clone, Default, PartialEq, Eq)]
157pub struct History {
158    /// The jobs it closed in the days read.
159    pub runs: Vec<Run>,
160    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
161    /// start left only some, so no week before it is compared.
162    pub kept_since: u64,
163}
164
165/// A job as insights tell jobs apart: repository, workflow and name.
166type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
167
168fn key_of(run: &Run) -> JobKey<'_> {
169    (&run.repository, run.workflow.as_deref(), &run.name)
170}
171
172/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
173/// only the runs finished by `now` count.
174pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
175    let runs = &history.runs;
176    let since = now.saturating_sub(u64::from(days) * DAY);
177    let window: Vec<&Run> = runs
178        .iter()
179        .filter(|run| {
180            matches!(run.state, JobState::Completed | JobState::Lost)
181                && run
182                    .finished_at
183                    .is_some_and(|at| (since..=now).contains(&at))
184        })
185        .collect();
186    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
187    for run in &window {
188        jobs.entry(key_of(run)).or_default().push(run);
189    }
190    let mut found = Vec::new();
191    for (key, runs) in &mut jobs {
192        runs.sort_by_key(|run| (run.finished(), run.id));
193        found.extend(slower(*key, runs));
194        found.extend(flaky(*key, runs));
195        found.extend(near_cap(*key, runs, config));
196        found.extend(cold_cache(*key, runs));
197        found.extend(cpu_held(*key, runs, config));
198    }
199    found.extend(throttled(&window));
200    let waits = waits(history, now);
201    found.extend(waits.iter().filter_map(waits_insight));
202    found.sort_by(|a, b| {
203        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
204            b.severity,
205            &b.repository,
206            &b.workflow,
207            &b.job,
208            &b.key,
209        ))
210    });
211    Insights {
212        since: since / SECOND,
213        until: now / SECOND,
214        insights: found,
215        waits,
216    }
217}
218
219/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
220/// before, and the longest waits.
221pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
222    let runs = &history.runs;
223    let since = now.saturating_sub(DAY);
224    let finished: Vec<&Run> = runs
225        .iter()
226        .filter(|run| {
227            matches!(run.state, JobState::Completed | JobState::Lost)
228                && run.finished_at.is_some_and(|at| at > since && at <= now)
229        })
230        .collect();
231    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
232        .insights
233        .into_iter()
234        .map(|insight| insight.key)
235        .collect();
236    let new = insights(history, config, api::INSIGHT_DAYS, now)
237        .insights
238        .into_iter()
239        .filter(|insight| !before.contains(&insight.key))
240        .collect();
241    let mut started: Vec<&Run> = runs
242        .iter()
243        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
244        .collect();
245    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
246    Digest {
247        since: since / SECOND,
248        until: now / SECOND,
249        finished: finished.len() as u64,
250        failed: finished.iter().filter(|run| run.failed()).count() as u64,
251        new,
252        waits: started
253            .into_iter()
254            .take(3)
255            .map(|run| LongWait {
256                job: run.id,
257                repository: run.repository.clone(),
258                workflow: run.workflow.clone(),
259                name: run.name.clone(),
260                waited: run.waited(history.kept_since).unwrap_or(0),
261                url: run.url.clone(),
262            })
263            .collect(),
264        savings: None,
265    }
266}
267
268/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
269fn subject((repository, workflow, name): JobKey<'_>) -> String {
270    match workflow {
271        Some(workflow) => format!("{repository}: {workflow} / {name}"),
272        None => format!("{repository}: {name}"),
273    }
274}
275
276fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
277    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
278}
279
280fn insight(
281    key: JobKey<'_>,
282    kind: &str,
283    severity: Severity,
284    sentence: String,
285    found: InsightKind,
286    runs: Vec<RunLink>,
287) -> Insight {
288    Insight {
289        key: key_text(kind, key),
290        severity,
291        repository: key.0.to_owned(),
292        workflow: key.1.map(str::to_owned),
293        job: Some(key.2.to_owned()),
294        sentence,
295        kind: found,
296        runs,
297    }
298}
299
300/// The median: the mean of the middle two for an even count.
301fn median(values: &[u64]) -> u64 {
302    let mut values = values.to_vec();
303    values.sort_unstable();
304    let middle = values.len() / 2;
305    match values.len() {
306        0 => 0,
307        len if len % 2 == 1 => values[middle],
308        _ => u64::midpoint(values[middle - 1], values[middle]),
309    }
310}
311
312/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
313fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
314    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
315        runs.iter()
316            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
317            .filter_map(|run| Some((*run, run.took()?)))
318            .collect()
319    };
320    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
321    for run in runs.iter().filter(|run| run.succeeded()) {
322        if let Some(branch) = &run.branch {
323            *branches.entry(branch).or_default() += 1;
324        }
325    }
326    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
327    // most runs, and of equal counts the first by name.
328    let busiest = branches
329        .iter()
330        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
331        .map(|(branch, _)| *branch)?;
332    let (branch, timed) = ["main", "master"]
333        .into_iter()
334        .map(|branch| (branch, timed_on(branch)))
335        .find(|(_, timed)| timed.len() >= 2 * RECENT)
336        .unwrap_or_else(|| (busiest, timed_on(busiest)));
337    if timed.len() < 2 * RECENT {
338        return None;
339    }
340    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
341    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
342    let before = median(&took(baseline));
343    let now = median(&took(recent));
344    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
345    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
346    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
347    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
348        return None;
349    }
350    // The first slow run after which no two runs in a row were fast.
351    let mut since = None;
352    let mut fast_in_a_row = 0;
353    for (run, took) in &timed {
354        if *took >= threshold {
355            fast_in_a_row = 0;
356            since = since.or(Some(*run));
357        } else {
358            fast_in_a_row += 1;
359            if fast_in_a_row >= 2 {
360                since = None;
361            }
362        }
363    }
364    let since = since?;
365    let sentence = format!(
366        "{} on {branch} takes {} now, up from {}.",
367        subject(key),
368        span(now),
369        span(before)
370    );
371    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
372    Some(insight(
373        key,
374        "slower",
375        Severity::Warning,
376        sentence,
377        InsightKind::Slower {
378            branch: Some(branch.to_owned()),
379            before,
380            now,
381            since: Some(since.link()),
382        },
383        links,
384    ))
385}
386
387/// A job that failed and then passed on a re-run of the same run.
388fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
389    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
390    for run in runs {
391        by_run.entry(run.run_id).or_default().push(run);
392    }
393    let mut failed_first: Vec<&Run> = by_run
394        .values()
395        .filter_map(|attempts| {
396            attempts.iter().copied().find(|failed| {
397                failed.failed_itself()
398                    && attempts
399                        .iter()
400                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
401            })
402        })
403        .collect();
404    if failed_first.is_empty() {
405        return None;
406    }
407    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
408    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
409    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
410    let sentence = format!(
411        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
412        subject(key)
413    );
414    // Once in the window may be chance: a notice; twice, a warning.
415    let severity = if flaky >= FLAKY_WARNING {
416        Severity::Warning
417    } else {
418        Severity::Notice
419    };
420    Some(insight(
421        key,
422        "flaky",
423        severity,
424        sentence,
425        InsightKind::Flaky { flaky, total },
426        failed_first.iter().take(5).map(|run| run.link()).collect(),
427    ))
428}
429
430/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
431/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
432fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
433    let measured = runs
434        .iter()
435        .rev()
436        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
437    let name = measured.clone().next()?.profile.as_deref()?;
438    let measured: Vec<&Run> = measured
439        .filter(|run| run.profile.as_deref() == Some(name))
440        .take(CAP_RUNS)
441        .copied()
442        .collect();
443    if measured.len() < CAP_MIN_RUNS {
444        return None;
445    }
446    let profile = config.profiles.get(name)?;
447    let cap = profile.memory.0;
448    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
449    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
450    let peak = spread(peaks.clone()).p95;
451    if cap == 0 || !near(&peak) {
452        return None;
453    }
454    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
455    let auto = if !config.sizing.auto {
456        AutoNote::Off
457    } else if profile
458        .max_memory
459        .is_some_and(|max| profile.memory.0 >= max.0)
460    {
461        AutoNote::AtMax
462    } else {
463        AutoNote::Raises
464    };
465    let what = match auto {
466        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
467        AutoNote::AtMax => {
468            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
469        }
470        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
471    };
472    let sentence = format!(
473        "{} peaks at {}, {}% of {name}'s {}. {what}",
474        subject(key),
475        ByteSize(peak),
476        peak.saturating_mul(100) / cap,
477        ByteSize(cap)
478    );
479    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
480    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
481        Severity::Notice
482    } else {
483        Severity::Warning
484    };
485    Some(insight(
486        key,
487        "near-cap",
488        severity,
489        sentence,
490        InsightKind::NearCap {
491            profile: name.to_owned(),
492            peak,
493            cap,
494            auto,
495        },
496        measured.iter().take(5).map(|run| run.link()).collect(),
497    ))
498}
499
500/// A job's last runs that could write to the build cache, each with this many lookups at least…
501const COLD_RUNS: usize = 5;
502const COLD_MIN_RUNS: usize = 3;
503const COLD_MIN_LOOKUPS: u64 = 10;
504/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
505/// next should find most of it.
506const COLD_PERCENT: u64 = 10;
507
508fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
509    let writers: Vec<&Run> = runs
510        .iter()
511        .rev()
512        .filter(|run| {
513            run.build_cache
514                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
515        })
516        .take(COLD_RUNS)
517        .copied()
518        .collect();
519    if writers.len() < COLD_MIN_RUNS {
520        return None;
521    }
522    let mut total = api::BuildCacheCounts::default();
523    for run in &writers {
524        total.add(&run.build_cache.unwrap_or_default());
525    }
526    let percent = total.hit_percent()?;
527    if percent > COLD_PERCENT {
528        return None;
529    }
530    let branch = writers.first()?.branch.clone();
531    let sentence = format!(
532        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
533         differ from run to run.",
534        subject(key),
535        branch
536            .as_deref()
537            .map(|branch| format!(" on {branch}"))
538            .unwrap_or_default(),
539        writers.len()
540    );
541    Some(insight(
542        key,
543        "cold-cache",
544        Severity::Notice,
545        sentence,
546        InsightKind::ColdCache {
547            branch,
548            hits: total.hits,
549            lookups: total.lookups(),
550        },
551        writers.iter().map(|run| run.link()).collect(),
552    ))
553}
554
555/// A job whose CPU cap held back `RAISE_RUNS` or more of its last `CAP_RUNS` runs that measured it, as auto sizing
556/// reads a run (`Throttling::held`): successful runs, as sizing reads only those, in the profile and at the cap of its
557/// last such run, the profile's `cpus` today. A run without a throttling figure is unknown, neither held nor free.
558/// Says what the owner can do: under `measured` admission auto sizing raises `cpus` while each raise helps, under
559/// `caps` it does not for throttling, as each CPU takes budget there. A raise that did not help the job is told as
560/// this job's evidence only: auto sizing may still raise the profile for another job, or on the runs' averages.
561fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
562    let measured = runs.iter().rev().filter(|run| {
563        run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
564    });
565    let last = measured.clone().next()?;
566    let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
567    let profile = config.profiles.get(name)?;
568    // Runs at a cap the profile no longer has say nothing of the one it has: its next runs will.
569    if u64::from(profile.cpus) * 1000 != cap {
570        return None;
571    }
572    let at_cap: Vec<&Run> = measured
573        .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
574        .take(CAP_RUNS)
575        .copied()
576        .collect();
577    let held: Vec<&Run> = at_cap
578        .iter()
579        .filter(|run| run.throttling().held() == Some(true))
580        .copied()
581        .collect();
582    if (held.len() as u64) < RAISE_RUNS {
583        return None;
584    }
585    let cpus = profile.cpus;
586    let auto = config.host.admission == Admission::Measured && config.sizing.auto;
587    let raised_from = auto
588        .then(|| {
589            let ran: Vec<CpuRun> = runs
590                .iter()
591                .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
592                .map(|run| CpuRun {
593                    cap: run.cpu_cap,
594                    queued_at: run.queued_at,
595                    throttling: run.throttling(),
596                })
597                .collect();
598            let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
599            unhelped_raise(&ran, cap, &now)
600        })
601        .flatten()
602        .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
603    let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
604    let what = match (raised_from, set) {
605        (Some(from), _) => format!(
606            "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
607             tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
608             sizing does next."
609        ),
610        (None, Some(set)) if config.host.admission == Admission::Measured => {
611            format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
612        }
613        (None, Some(set)) => format!(
614            "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
615             cpus to {set} if runnerctl sizing shows the budget has room."
616        ),
617        (None, None) => format!(
618            "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
619             room; runnerctl sizing says when it cannot."
620        ),
621    };
622    let sentence = format!(
623        "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
624         its run or more. {what}",
625        subject(key),
626        held.len(),
627        at_cap.len()
628    );
629    // Auto sizing deals with it: a notice. The owner acts on the rest.
630    let severity = if auto && raised_from.is_none() {
631        Severity::Notice
632    } else {
633        Severity::Warning
634    };
635    Some(insight(
636        key,
637        "cpu-held",
638        severity,
639        sentence,
640        InsightKind::CpuHeld {
641            profile: name.to_owned(),
642            cpus,
643            held: u32::try_from(held.len()).unwrap_or(u32::MAX),
644            total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
645            raised_from,
646            set,
647        },
648        held.iter().take(5).map(|run| run.link()).collect(),
649    ))
650}
651
652/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
653fn throttled(window: &[&Run]) -> Vec<Insight> {
654    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
655    for run in window {
656        repositories.entry(&run.repository).or_default().push(run);
657    }
658    repositories
659        .into_iter()
660        .filter_map(|(repository, mut runs)| {
661            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
662            let hit: Vec<&Run> = runs
663                .iter()
664                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
665                .copied()
666                .collect();
667            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
668            if throttled < THROTTLED_MIN_RUNS {
669                return None;
670            }
671            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
672            Some(Insight {
673                key: format!("throttled:{repository}"),
674                severity: Severity::Warning,
675                repository: repository.to_owned(),
676                workflow: None,
677                job: None,
678                sentence: format!(
679                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
680                     host, or a job at its cap, was short of memory."
681                ),
682                kind: InsightKind::Throttled { throttled, total },
683                runs: hit.iter().take(5).map(|run| run.link()).collect(),
684            })
685        })
686        .collect()
687}
688
689/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
690/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
691/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
692fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
693    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
694    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
695    for run in &history.runs {
696        let Some(started) = run.started_at.filter(|at| *at <= now) else {
697            continue;
698        };
699        let week = match now - started {
700            back if back < WEEK => 0,
701            back if back < 2 * WEEK && !before_kept => 1,
702            _ => continue,
703        };
704        weeks.entry(&run.repository).or_default()[week].push(run);
705    }
706    weeks
707        .into_iter()
708        .map(|(repository, [now, before])| RepositoryWaits {
709            repository: repository.to_owned(),
710            now: week_waits(&now, history.kept_since),
711            before: week_waits(&before, history.kept_since),
712        })
713        .collect()
714}
715
716fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
717    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
718    let of = |runs: Vec<&&Run>| {
719        let waited: Vec<u64> = runs
720            .iter()
721            .filter_map(|run| run.waited(kept_since))
722            .collect();
723        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
724        let spread = (!waited.is_empty()).then(|| spread(waited));
725        WaitSpread {
726            jobs,
727            p50: spread.as_ref().map(|spread| spread.p50),
728            p95: spread.as_ref().map(|spread| spread.p95),
729        }
730    };
731    WeekWaits {
732        all: of(runs.iter().collect()),
733        warm: of(runs.iter().filter(warm).collect()),
734        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
735    }
736}
737
738fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
739    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
740        return None;
741    }
742    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
743    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
744    if now < threshold {
745        return None;
746    }
747    Some(Insight {
748        key: format!("waits:{}", waits.repository),
749        severity: Severity::Notice,
750        repository: waits.repository.clone(),
751        workflow: None,
752        job: None,
753        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
754            format!(
755                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
756                waits.repository,
757                span(now),
758                span(before)
759            )
760        } else {
761            format!(
762                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
763                waits.repository,
764                span(now),
765                span(before)
766            )
767        },
768        kind: InsightKind::Waits { before, now },
769        runs: Vec::new(),
770    })
771}
772
773/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
774fn span(seconds: u64) -> String {
775    match seconds {
776        0..60 => format!("{seconds}s"),
777        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
778        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
779        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
780    }
781}
782
783#[cfg(test)]
784mod tests;