Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, jobs their CPU cap held back by the rule auto sizing reads (ADR 0087), jobs
4//! held for the host's memory by programs outside CI (ADR 0100), and how long each repository's jobs wait, this week against the last when the history holds all of last week; and the
5//! daily digest. What the host's memory came to while CI ran, and the warning when it ran low on a host credit
6//! (ADR 0108), are `host`'s. Pure: the daemon reads the history, the config and the clock and passes them in. It sends nothing
7//! and changes nothing.
8
9use std::collections::{BTreeMap, BTreeSet};
10
11use crate::api::{
12    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
13    RunLink, Severity, WaitSpread, WeekWaits,
14};
15use crate::config::{Admission, ByteSize, Config};
16use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
17use crate::sizing::{HELD, Throttling, spread};
18
19mod host;
20
21const SECOND: u64 = 1000;
22const DAY: u64 = 24 * 3600 * SECOND;
23const WEEK: u64 = 7 * DAY;
24
25/// The last runs a slower job's median is taken over; the baseline needs as many before them.
26const RECENT: usize = 5;
27/// A job is slower when its recent median is this share above its baseline's, in percent…
28const SLOWER_PERCENT: u64 = 20;
29/// …and at least this many seconds above it.
30const SLOWER_SECONDS: u64 = 30;
31
32/// The last runs whose memory peaks are read, and the fewest that say anything.
33const CAP_RUNS: usize = 10;
34const CAP_MIN_RUNS: usize = 5;
35/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
36const NEAR_CAP: u64 = 85;
37/// Runs at `NEAR_CAP` or more that make it a warning.
38const NEAR_CAP_WARNING: usize = 2;
39/// Flaky runs that make it a warning.
40const FLAKY_WARNING: u32 = 2;
41
42/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
43/// above it, each week with this many jobs.
44const WAITS_PERCENT: u64 = 50;
45const WAITS_SECONDS: u64 = 60;
46const WAITS_MIN_JOBS: u32 = 5;
47/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
48const WAITS_ONE_IN_20: u32 = 20;
49
50/// A repository's runs the kernel throttled for memory pressure that make an insight.
51const THROTTLED_MIN_RUNS: u32 = 2;
52
53/// A queued job's wait for the host's memory that makes an insight, in milliseconds, while a job waits for it now.
54const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
55/// The insight's key changes this often, so it is told once in each such window however long the wait lasts, in
56/// milliseconds.
57const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
58/// How old the host's reading may be for the insight to hold, in milliseconds, and the least the engine's look at the
59/// waiting jobs may be, which is three `[host] poll_interval`s when that is more: a daemon that stopped looking says
60/// nothing about now.
61const HOST_HELD_FRESH: u64 = 60 * SECOND;
62
63/// One job the history closed, as insights read it. Times in milliseconds.
64#[derive(Debug, Clone, PartialEq, Eq)]
65pub struct Run {
66    /// GitHub's job ID.
67    pub id: u64,
68    /// `owner/name`.
69    pub repository: String,
70    /// The workflow's name, when known.
71    pub workflow: Option<String>,
72    /// The job's name.
73    pub name: String,
74    /// The branch the run is for.
75    pub branch: Option<String>,
76    /// The job's page on GitHub.
77    pub url: Option<String>,
78    /// The workflow run.
79    pub run_id: u64,
80    /// The run's attempt, 1 for the first.
81    pub run_attempt: u32,
82    /// `Completed` or `Lost`.
83    pub state: JobState,
84    /// GitHub's conclusion, for a completed job.
85    pub conclusion: Option<String>,
86    /// When GitHub queued it.
87    pub queued_at: u64,
88    /// When it started.
89    pub started_at: Option<u64>,
90    /// When it finished, or was found lost.
91    pub finished_at: Option<u64>,
92    /// When its runner's container started: before `queued_at`, it found a warm runner.
93    pub runner_started_at: Option<u64>,
94    /// The profile its runner ran in.
95    pub profile: Option<String>,
96    /// Its sampled memory peak.
97    pub memory_peak: Option<u64>,
98    /// What the build cache served it, if its runner had the build cache.
99    pub build_cache: Option<api::BuildCacheCounts>,
100    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
101    pub sock_throttled: Option<u64>,
102    /// Its container's CPU cap, millicores, when recorded.
103    pub cpu_cap: Option<u64>,
104    /// The periods of its CPU quota it had work in; `None` where the kernel kept no figure, as before schema 17.
105    pub cpu_periods: Option<u64>,
106    /// Of those, the periods its quota throttled it in.
107    pub cpu_throttled: Option<u64>,
108    /// How much of its wait passed while no runnerd ran.
109    pub unseen: u64,
110}
111
112impl Run {
113    fn finished(&self) -> u64 {
114        self.finished_at.unwrap_or(0)
115    }
116
117    fn succeeded(&self) -> bool {
118        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
119    }
120
121    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
122    fn failed(&self) -> bool {
123        self.state == JobState::Lost || self.failed_itself()
124    }
125
126    /// Failed or timed out: what the job did, not what became of its runner.
127    fn failed_itself(&self) -> bool {
128        self.state == JobState::Completed
129            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
130    }
131
132    /// Seconds from start to finish.
133    fn took(&self) -> Option<u64> {
134        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
135    }
136
137    /// Its [wait](crate::sizing::Wait) in seconds, once it started.
138    fn waited(&self, kept_since: u64) -> Option<u64> {
139        let wait = crate::sizing::Wait {
140            queued_at: self.queued_at,
141            started_at: self.started_at?,
142            unseen: self.unseen,
143        };
144        Some(wait.counted(kept_since) / SECOND)
145    }
146
147    fn throttling(&self) -> Throttling {
148        Throttling {
149            started_at: self.started_at,
150            finished_at: self.finished_at,
151            periods: self.cpu_periods,
152            throttled: self.cpu_throttled,
153        }
154    }
155
156    fn link(&self) -> RunLink {
157        RunLink {
158            job: self.id,
159            run_id: self.run_id,
160            run_attempt: self.run_attempt,
161            branch: self.branch.clone(),
162            url: self.url.clone(),
163            at: self.finished() / SECOND,
164        }
165    }
166}
167
168/// The jobs waiting for the host's memory at one moment, and the reading that says what holds it. What the daemon
169/// knows of now, not of the history.
170#[derive(Debug, Clone, PartialEq, Eq)]
171pub struct HostHold {
172    /// The host's memory as last read.
173    pub memory: api::HostMemory,
174    /// When the engine last saw these jobs wait for it, in ms.
175    pub seen: u64,
176    /// The jobs waiting for it then that are still queued.
177    pub jobs: Vec<HeldJob>,
178}
179
180/// A queued job waiting for the host's memory.
181#[derive(Debug, Clone, PartialEq, Eq)]
182pub struct HeldJob {
183    /// `owner/name`.
184    pub repository: String,
185    /// The workflow's name, when known.
186    pub workflow: Option<String>,
187    /// The job's name.
188    pub name: String,
189    /// How long it has waited for the host's memory, in ms (`jobs.host_waited`).
190    pub waited: u64,
191}
192
193/// A reading of the host's memory as a `budget.changed` event stored it (ADR 0108).
194#[derive(Debug, Clone, Copy, PartialEq, Eq)]
195pub struct HostReading {
196    /// When it was stored, in ms.
197    pub at: u64,
198    /// The host's `MemTotal`, in bytes.
199    pub total: u64,
200    /// Its `MemAvailable`.
201    pub available: u64,
202    /// Its `SwapFree`, on a host with swap.
203    pub swap_free: Option<u64>,
204    /// What the host's limit took off the plain count then: 0 for no credit, as before 0.0.30.
205    pub credit: u64,
206    /// Whether a runner ran: the budget counted more than the mirror.
207    pub running: bool,
208}
209
210/// What insights read from the job history.
211#[derive(Debug, Clone, Default, PartialEq, Eq)]
212pub struct History {
213    /// The jobs it closed in the days read.
214    pub runs: Vec<Run>,
215    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
216    /// start left only some, so no week before it is compared.
217    pub kept_since: u64,
218    /// The jobs held for the host's memory now; `None` when none is, or the daemon has no reading of it.
219    pub host: Option<HostHold>,
220    /// The host's readings the budget's events stored, oldest first.
221    pub readings: Vec<HostReading>,
222}
223
224/// A job as insights tell jobs apart: repository, workflow and name.
225type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
226
227fn key_of(run: &Run) -> JobKey<'_> {
228    (&run.repository, run.workflow.as_deref(), &run.name)
229}
230
231/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
232/// only the runs finished by `now` count.
233pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
234    let runs = &history.runs;
235    let since = now.saturating_sub(u64::from(days) * DAY);
236    let window: Vec<&Run> = runs
237        .iter()
238        .filter(|run| {
239            matches!(run.state, JobState::Completed | JobState::Lost)
240                && run
241                    .finished_at
242                    .is_some_and(|at| (since..=now).contains(&at))
243        })
244        .collect();
245    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
246    for run in &window {
247        jobs.entry(key_of(run)).or_default().push(run);
248    }
249    let mut found = Vec::new();
250    for (key, runs) in &mut jobs {
251        runs.sort_by_key(|run| (run.finished(), run.id));
252        found.extend(slower(*key, runs));
253        found.extend(flaky(*key, runs));
254        found.extend(near_cap(*key, runs, config));
255        found.extend(cold_cache(*key, runs));
256        found.extend(cpu_held(*key, runs, config));
257    }
258    found.extend(throttled(&window));
259    found.extend(
260        history
261            .host
262            .as_ref()
263            .and_then(|hold| host_held(hold, config, now)),
264    );
265    let host_days = host::days(&history.readings, since, now);
266    found.extend(host::low(&host_days));
267    let waits = waits(history, now);
268    found.extend(waits.iter().filter_map(waits_insight));
269    found.sort_by(|a, b| {
270        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
271            b.severity,
272            &b.repository,
273            &b.workflow,
274            &b.job,
275            &b.key,
276        ))
277    });
278    Insights {
279        since: since / SECOND,
280        until: now / SECOND,
281        insights: found,
282        waits,
283        host: host_days,
284    }
285}
286
287/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
288/// before, and the longest waits.
289pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
290    let runs = &history.runs;
291    let since = now.saturating_sub(DAY);
292    let finished: Vec<&Run> = runs
293        .iter()
294        .filter(|run| {
295            matches!(run.state, JobState::Completed | JobState::Lost)
296                && run.finished_at.is_some_and(|at| at > since && at <= now)
297        })
298        .collect();
299    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
300        .insights
301        .into_iter()
302        .map(|insight| insight.key)
303        .collect();
304    let new = insights(history, config, api::INSIGHT_DAYS, now)
305        .insights
306        .into_iter()
307        .filter(|insight| !before.contains(&insight.key))
308        .collect();
309    let mut started: Vec<&Run> = runs
310        .iter()
311        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
312        .collect();
313    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
314    Digest {
315        since: since / SECOND,
316        until: now / SECOND,
317        finished: finished.len() as u64,
318        failed: finished.iter().filter(|run| run.failed()).count() as u64,
319        new,
320        waits: started
321            .into_iter()
322            .take(3)
323            .map(|run| LongWait {
324                job: run.id,
325                repository: run.repository.clone(),
326                workflow: run.workflow.clone(),
327                name: run.name.clone(),
328                waited: run.waited(history.kept_since).unwrap_or(0),
329                url: run.url.clone(),
330            })
331            .collect(),
332        savings: None,
333        host: host::hazard(&history.readings, since, now),
334    }
335}
336
337/// A job has waited for the host's memory for 30 minutes or more, and `hold` says a job waits for it at `now` (ms):
338/// the host's memory is short, and the programs outside CI that hold it are the owner's to close. Worded here once
339/// for `runnerctl`, the dashboard, the TUI and notifications, with the longest waiter's repository. It names the
340/// programs `HostMemory::outside_line` names and says closing them frees only part of what is held, since the figure
341/// also counts the kernel and compressed swap.
342///
343/// Holds from the host's reading and the engine's look at the waiting jobs: the reading no older than a minute, the
344/// look no older than that or three `[host] poll_interval`s, neither from after `now`, the second of the look (a plan
345/// in that second is not from after it). With no reading, or a stale one, nothing is said. Its key is the same for 6
346/// hours and then changes, so a notification target tells it once in each window of them (told keys are stored, so a
347/// restart repeats none).
348fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
349    let reading = hold.memory.at.saturating_mul(SECOND);
350    // The engine plans once a `poll_interval`, so its look is as old as three of them before it is missed.
351    let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
352    let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
353    let fresh =
354        |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
355    if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
356        return None;
357    }
358    let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
359    if longest.waited < HOST_HELD_AFTER {
360        return None;
361    }
362    let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
363    let waited = longest.waited / SECOND;
364    let sentence = Insight::host_held_sentence(
365        jobs,
366        waited,
367        &subject((
368            &longest.repository,
369            longest.workflow.as_deref(),
370            &longest.name,
371        )),
372        &hold.memory,
373    );
374    Some(Insight {
375        key: format!("host-held:{}", now / HOST_HELD_EVERY),
376        severity: Severity::Warning,
377        repository: longest.repository.clone(),
378        workflow: None,
379        job: None,
380        sentence,
381        kind: InsightKind::HostHeld { jobs, waited },
382        runs: Vec::new(),
383    })
384}
385
386/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
387fn subject((repository, workflow, name): JobKey<'_>) -> String {
388    match workflow {
389        Some(workflow) => format!("{repository}: {workflow} / {name}"),
390        None => format!("{repository}: {name}"),
391    }
392}
393
394fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
395    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
396}
397
398fn insight(
399    key: JobKey<'_>,
400    kind: &str,
401    severity: Severity,
402    sentence: String,
403    found: InsightKind,
404    runs: Vec<RunLink>,
405) -> Insight {
406    Insight {
407        key: key_text(kind, key),
408        severity,
409        repository: key.0.to_owned(),
410        workflow: key.1.map(str::to_owned),
411        job: Some(key.2.to_owned()),
412        sentence,
413        kind: found,
414        runs,
415    }
416}
417
418/// The median: the mean of the middle two for an even count.
419fn median(values: &[u64]) -> u64 {
420    let mut values = values.to_vec();
421    values.sort_unstable();
422    let middle = values.len() / 2;
423    match values.len() {
424        0 => 0,
425        len if len % 2 == 1 => values[middle],
426        _ => u64::midpoint(values[middle - 1], values[middle]),
427    }
428}
429
430/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
431fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
432    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
433        runs.iter()
434            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
435            .filter_map(|run| Some((*run, run.took()?)))
436            .collect()
437    };
438    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
439    for run in runs.iter().filter(|run| run.succeeded()) {
440        if let Some(branch) = &run.branch {
441            *branches.entry(branch).or_default() += 1;
442        }
443    }
444    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
445    // most runs, and of equal counts the first by name.
446    let busiest = branches
447        .iter()
448        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
449        .map(|(branch, _)| *branch)?;
450    let (branch, timed) = ["main", "master"]
451        .into_iter()
452        .map(|branch| (branch, timed_on(branch)))
453        .find(|(_, timed)| timed.len() >= 2 * RECENT)
454        .unwrap_or_else(|| (busiest, timed_on(busiest)));
455    if timed.len() < 2 * RECENT {
456        return None;
457    }
458    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
459    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
460    let before = median(&took(baseline));
461    let now = median(&took(recent));
462    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
463    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
464    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
465    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
466        return None;
467    }
468    // The first slow run after which no two runs in a row were fast.
469    let mut since = None;
470    let mut fast_in_a_row = 0;
471    for (run, took) in &timed {
472        if *took >= threshold {
473            fast_in_a_row = 0;
474            since = since.or(Some(*run));
475        } else {
476            fast_in_a_row += 1;
477            if fast_in_a_row >= 2 {
478                since = None;
479            }
480        }
481    }
482    let since = since?;
483    let sentence = format!(
484        "{} on {branch} takes {} now, up from {}.",
485        subject(key),
486        span(now),
487        span(before)
488    );
489    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
490    Some(insight(
491        key,
492        "slower",
493        Severity::Warning,
494        sentence,
495        InsightKind::Slower {
496            branch: Some(branch.to_owned()),
497            before,
498            now,
499            since: Some(since.link()),
500        },
501        links,
502    ))
503}
504
505/// A job that failed and then passed on a re-run of the same run.
506fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
507    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
508    for run in runs {
509        by_run.entry(run.run_id).or_default().push(run);
510    }
511    let mut failed_first: Vec<&Run> = by_run
512        .values()
513        .filter_map(|attempts| {
514            attempts.iter().copied().find(|failed| {
515                failed.failed_itself()
516                    && attempts
517                        .iter()
518                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
519            })
520        })
521        .collect();
522    if failed_first.is_empty() {
523        return None;
524    }
525    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
526    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
527    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
528    let sentence = format!(
529        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
530        subject(key)
531    );
532    // Once in the window may be chance: a notice; twice, a warning.
533    let severity = if flaky >= FLAKY_WARNING {
534        Severity::Warning
535    } else {
536        Severity::Notice
537    };
538    Some(insight(
539        key,
540        "flaky",
541        severity,
542        sentence,
543        InsightKind::Flaky { flaky, total },
544        failed_first.iter().take(5).map(|run| run.link()).collect(),
545    ))
546}
547
548/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
549/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
550fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
551    let measured = runs
552        .iter()
553        .rev()
554        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
555    let name = measured.clone().next()?.profile.as_deref()?;
556    let measured: Vec<&Run> = measured
557        .filter(|run| run.profile.as_deref() == Some(name))
558        .take(CAP_RUNS)
559        .copied()
560        .collect();
561    if measured.len() < CAP_MIN_RUNS {
562        return None;
563    }
564    let profile = config.profiles.get(name)?;
565    let cap = profile.memory.0;
566    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
567    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
568    let peak = spread(peaks.clone()).p95;
569    if cap == 0 || !near(&peak) {
570        return None;
571    }
572    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
573    let auto = if !config.sizing.auto {
574        AutoNote::Off
575    } else if profile
576        .max_memory
577        .is_some_and(|max| profile.memory.0 >= max.0)
578    {
579        AutoNote::AtMax
580    } else {
581        AutoNote::Raises
582    };
583    let what = match auto {
584        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
585        AutoNote::AtMax => {
586            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
587        }
588        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
589    };
590    let sentence = format!(
591        "{} peaks at {}, {}% of {name}'s {}. {what}",
592        subject(key),
593        ByteSize(peak),
594        peak.saturating_mul(100) / cap,
595        ByteSize(cap)
596    );
597    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
598    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
599        Severity::Notice
600    } else {
601        Severity::Warning
602    };
603    Some(insight(
604        key,
605        "near-cap",
606        severity,
607        sentence,
608        InsightKind::NearCap {
609            profile: name.to_owned(),
610            peak,
611            cap,
612            auto,
613        },
614        measured.iter().take(5).map(|run| run.link()).collect(),
615    ))
616}
617
618/// A job's last runs that could write to the build cache, each with this many lookups at least…
619const COLD_RUNS: usize = 5;
620const COLD_MIN_RUNS: usize = 3;
621const COLD_MIN_LOOKUPS: u64 = 10;
622/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
623/// next should find most of it.
624const COLD_PERCENT: u64 = 10;
625
626fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
627    let writers: Vec<&Run> = runs
628        .iter()
629        .rev()
630        .filter(|run| {
631            run.build_cache
632                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
633        })
634        .take(COLD_RUNS)
635        .copied()
636        .collect();
637    if writers.len() < COLD_MIN_RUNS {
638        return None;
639    }
640    let mut total = api::BuildCacheCounts::default();
641    for run in &writers {
642        total.add(&run.build_cache.unwrap_or_default());
643    }
644    let percent = total.hit_percent()?;
645    if percent > COLD_PERCENT {
646        return None;
647    }
648    let branch = writers.first()?.branch.clone();
649    let sentence = format!(
650        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
651         differ from run to run.",
652        subject(key),
653        branch
654            .as_deref()
655            .map(|branch| format!(" on {branch}"))
656            .unwrap_or_default(),
657        writers.len()
658    );
659    Some(insight(
660        key,
661        "cold-cache",
662        Severity::Notice,
663        sentence,
664        InsightKind::ColdCache {
665            branch,
666            hits: total.hits,
667            lookups: total.lookups(),
668        },
669        writers.iter().map(|run| run.link()).collect(),
670    ))
671}
672
673/// A job whose CPU cap held back `RAISE_RUNS` or more of its last `CAP_RUNS` runs that measured it, as auto sizing
674/// reads a run (`Throttling::held`): successful runs, as sizing reads only those, in the profile and at the cap of its
675/// last such run, the profile's `cpus` today. A run without a throttling figure is unknown, neither held nor free.
676/// Says what the owner can do: under `measured` admission auto sizing raises `cpus` while each raise helps, under
677/// `caps` it does not for throttling, as each CPU takes budget there. A raise that did not help the job is told as
678/// this job's evidence only: auto sizing may still raise the profile for another job, or on the runs' averages.
679fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
680    let measured = runs.iter().rev().filter(|run| {
681        run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
682    });
683    let last = measured.clone().next()?;
684    let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
685    let profile = config.profiles.get(name)?;
686    // Runs at a cap the profile no longer has say nothing of the one it has: its next runs will.
687    if u64::from(profile.cpus) * 1000 != cap {
688        return None;
689    }
690    let at_cap: Vec<&Run> = measured
691        .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
692        .take(CAP_RUNS)
693        .copied()
694        .collect();
695    let held: Vec<&Run> = at_cap
696        .iter()
697        .filter(|run| run.throttling().held() == Some(true))
698        .copied()
699        .collect();
700    if (held.len() as u64) < RAISE_RUNS {
701        return None;
702    }
703    let cpus = profile.cpus;
704    let auto = config.host.admission == Admission::Measured && config.sizing.auto;
705    let raised_from = auto
706        .then(|| {
707            let ran: Vec<CpuRun> = runs
708                .iter()
709                .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
710                .map(|run| CpuRun {
711                    cap: run.cpu_cap,
712                    queued_at: run.queued_at,
713                    throttling: run.throttling(),
714                })
715                .collect();
716            let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
717            unhelped_raise(&ran, cap, &now)
718        })
719        .flatten()
720        .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
721    let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
722    let what = match (raised_from, set) {
723        (Some(from), _) => format!(
724            "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
725             tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
726             sizing does next."
727        ),
728        (None, Some(set)) if config.host.admission == Admission::Measured => {
729            format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
730        }
731        (None, Some(set)) => format!(
732            "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
733             cpus to {set} if runnerctl sizing shows the budget has room."
734        ),
735        (None, None) => format!(
736            "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
737             room; runnerctl sizing says when it cannot."
738        ),
739    };
740    let sentence = format!(
741        "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
742         its run or more. {what}",
743        subject(key),
744        held.len(),
745        at_cap.len()
746    );
747    // Auto sizing deals with it: a notice. The owner acts on the rest.
748    let severity = if auto && raised_from.is_none() {
749        Severity::Notice
750    } else {
751        Severity::Warning
752    };
753    Some(insight(
754        key,
755        "cpu-held",
756        severity,
757        sentence,
758        InsightKind::CpuHeld {
759            profile: name.to_owned(),
760            cpus,
761            held: u32::try_from(held.len()).unwrap_or(u32::MAX),
762            total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
763            raised_from,
764            set,
765        },
766        held.iter().take(5).map(|run| run.link()).collect(),
767    ))
768}
769
770/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
771fn throttled(window: &[&Run]) -> Vec<Insight> {
772    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
773    for run in window {
774        repositories.entry(&run.repository).or_default().push(run);
775    }
776    repositories
777        .into_iter()
778        .filter_map(|(repository, mut runs)| {
779            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
780            let hit: Vec<&Run> = runs
781                .iter()
782                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
783                .copied()
784                .collect();
785            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
786            if throttled < THROTTLED_MIN_RUNS {
787                return None;
788            }
789            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
790            Some(Insight {
791                key: format!("throttled:{repository}"),
792                severity: Severity::Warning,
793                repository: repository.to_owned(),
794                workflow: None,
795                job: None,
796                sentence: format!(
797                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
798                     host, or a job at its cap, was short of memory."
799                ),
800                kind: InsightKind::Throttled { throttled, total },
801                runs: hit.iter().take(5).map(|run| run.link()).collect(),
802            })
803        })
804        .collect()
805}
806
807/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
808/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
809/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
810fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
811    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
812    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
813    for run in &history.runs {
814        let Some(started) = run.started_at.filter(|at| *at <= now) else {
815            continue;
816        };
817        let week = match now - started {
818            back if back < WEEK => 0,
819            back if back < 2 * WEEK && !before_kept => 1,
820            _ => continue,
821        };
822        weeks.entry(&run.repository).or_default()[week].push(run);
823    }
824    weeks
825        .into_iter()
826        .map(|(repository, [now, before])| RepositoryWaits {
827            repository: repository.to_owned(),
828            now: week_waits(&now, history.kept_since),
829            before: week_waits(&before, history.kept_since),
830        })
831        .collect()
832}
833
834fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
835    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
836    let of = |runs: Vec<&&Run>| {
837        let waited: Vec<u64> = runs
838            .iter()
839            .filter_map(|run| run.waited(kept_since))
840            .collect();
841        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
842        let spread = (!waited.is_empty()).then(|| spread(waited));
843        WaitSpread {
844            jobs,
845            p50: spread.as_ref().map(|spread| spread.p50),
846            p95: spread.as_ref().map(|spread| spread.p95),
847        }
848    };
849    WeekWaits {
850        all: of(runs.iter().collect()),
851        warm: of(runs.iter().filter(warm).collect()),
852        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
853    }
854}
855
856fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
857    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
858        return None;
859    }
860    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
861    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
862    if now < threshold {
863        return None;
864    }
865    Some(Insight {
866        key: format!("waits:{}", waits.repository),
867        severity: Severity::Notice,
868        repository: waits.repository.clone(),
869        workflow: None,
870        job: None,
871        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
872            format!(
873                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
874                waits.repository,
875                span(now),
876                span(before)
877            )
878        } else {
879            format!(
880                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
881                waits.repository,
882                span(now),
883                span(before)
884            )
885        },
886        kind: InsightKind::Waits { before, now },
887        runs: Vec::new(),
888    })
889}
890
891/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
892fn span(seconds: u64) -> String {
893    match seconds {
894        0..60 => format!("{seconds}s"),
895        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
896        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
897        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
898    }
899}
900
901#[cfg(test)]
902mod tests;