Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, jobs their CPU cap held back by the rule auto sizing reads (ADR 0087), jobs
4//! held for the host's memory by programs outside CI (ADR 0100), and how long each repository's jobs wait, this week against the last when the history holds all of last week; and the
5//! daily digest. Pure: the daemon reads the history, the config and the clock and passes them in. It sends nothing
6//! and changes nothing.
7
8use std::collections::{BTreeMap, BTreeSet};
9
10use crate::api::{
11    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
12    RunLink, Severity, WaitSpread, WeekWaits,
13};
14use crate::config::{Admission, ByteSize, Config};
15use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
16use crate::sizing::{HELD, Throttling, spread};
17
18const SECOND: u64 = 1000;
19const DAY: u64 = 24 * 3600 * SECOND;
20const WEEK: u64 = 7 * DAY;
21
22/// The last runs a slower job's median is taken over; the baseline needs as many before them.
23const RECENT: usize = 5;
24/// A job is slower when its recent median is this share above its baseline's, in percent…
25const SLOWER_PERCENT: u64 = 20;
26/// …and at least this many seconds above it.
27const SLOWER_SECONDS: u64 = 30;
28
29/// The last runs whose memory peaks are read, and the fewest that say anything.
30const CAP_RUNS: usize = 10;
31const CAP_MIN_RUNS: usize = 5;
32/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
33const NEAR_CAP: u64 = 85;
34/// Runs at `NEAR_CAP` or more that make it a warning.
35const NEAR_CAP_WARNING: usize = 2;
36/// Flaky runs that make it a warning.
37const FLAKY_WARNING: u32 = 2;
38
39/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
40/// above it, each week with this many jobs.
41const WAITS_PERCENT: u64 = 50;
42const WAITS_SECONDS: u64 = 60;
43const WAITS_MIN_JOBS: u32 = 5;
44/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
45const WAITS_ONE_IN_20: u32 = 20;
46
47/// A repository's runs the kernel throttled for memory pressure that make an insight.
48const THROTTLED_MIN_RUNS: u32 = 2;
49
50/// A queued job's wait for the host's memory that makes an insight, in milliseconds, while a job waits for it now.
51const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
52/// The insight's key changes this often, so it is told once in each such window however long the wait lasts, in
53/// milliseconds.
54const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
55/// How old the host's reading may be for the insight to hold, in milliseconds, and the least the engine's look at the
56/// waiting jobs may be, which is three `[host] poll_interval`s when that is more: a daemon that stopped looking says
57/// nothing about now.
58const HOST_HELD_FRESH: u64 = 60 * SECOND;
59
60/// One job the history closed, as insights read it. Times in milliseconds.
61#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct Run {
63    /// GitHub's job ID.
64    pub id: u64,
65    /// `owner/name`.
66    pub repository: String,
67    /// The workflow's name, when known.
68    pub workflow: Option<String>,
69    /// The job's name.
70    pub name: String,
71    /// The branch the run is for.
72    pub branch: Option<String>,
73    /// The job's page on GitHub.
74    pub url: Option<String>,
75    /// The workflow run.
76    pub run_id: u64,
77    /// The run's attempt, 1 for the first.
78    pub run_attempt: u32,
79    /// `Completed` or `Lost`.
80    pub state: JobState,
81    /// GitHub's conclusion, for a completed job.
82    pub conclusion: Option<String>,
83    /// When GitHub queued it.
84    pub queued_at: u64,
85    /// When it started.
86    pub started_at: Option<u64>,
87    /// When it finished, or was found lost.
88    pub finished_at: Option<u64>,
89    /// When its runner's container started: before `queued_at`, it found a warm runner.
90    pub runner_started_at: Option<u64>,
91    /// The profile its runner ran in.
92    pub profile: Option<String>,
93    /// Its sampled memory peak.
94    pub memory_peak: Option<u64>,
95    /// What the build cache served it, if its runner had the build cache.
96    pub build_cache: Option<api::BuildCacheCounts>,
97    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
98    pub sock_throttled: Option<u64>,
99    /// Its container's CPU cap, millicores, when recorded.
100    pub cpu_cap: Option<u64>,
101    /// The periods of its CPU quota it had work in; `None` where the kernel kept no figure, as before schema 17.
102    pub cpu_periods: Option<u64>,
103    /// Of those, the periods its quota throttled it in.
104    pub cpu_throttled: Option<u64>,
105    /// How much of its wait passed while no runnerd ran.
106    pub unseen: u64,
107}
108
109impl Run {
110    fn finished(&self) -> u64 {
111        self.finished_at.unwrap_or(0)
112    }
113
114    fn succeeded(&self) -> bool {
115        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
116    }
117
118    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
119    fn failed(&self) -> bool {
120        self.state == JobState::Lost || self.failed_itself()
121    }
122
123    /// Failed or timed out: what the job did, not what became of its runner.
124    fn failed_itself(&self) -> bool {
125        self.state == JobState::Completed
126            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
127    }
128
129    /// Seconds from start to finish.
130    fn took(&self) -> Option<u64> {
131        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
132    }
133
134    /// Its [wait](crate::sizing::Wait) in seconds, once it started.
135    fn waited(&self, kept_since: u64) -> Option<u64> {
136        let wait = crate::sizing::Wait {
137            queued_at: self.queued_at,
138            started_at: self.started_at?,
139            unseen: self.unseen,
140        };
141        Some(wait.counted(kept_since) / SECOND)
142    }
143
144    fn throttling(&self) -> Throttling {
145        Throttling {
146            started_at: self.started_at,
147            finished_at: self.finished_at,
148            periods: self.cpu_periods,
149            throttled: self.cpu_throttled,
150        }
151    }
152
153    fn link(&self) -> RunLink {
154        RunLink {
155            job: self.id,
156            run_id: self.run_id,
157            run_attempt: self.run_attempt,
158            branch: self.branch.clone(),
159            url: self.url.clone(),
160            at: self.finished() / SECOND,
161        }
162    }
163}
164
165/// The jobs waiting for the host's memory at one moment, and the reading that says what holds it. What the daemon
166/// knows of now, not of the history.
167#[derive(Debug, Clone, PartialEq, Eq)]
168pub struct HostHold {
169    /// The host's memory as last read.
170    pub memory: api::HostMemory,
171    /// When the engine last saw these jobs wait for it, in ms.
172    pub seen: u64,
173    /// The jobs waiting for it then that are still queued.
174    pub jobs: Vec<HeldJob>,
175}
176
177/// A queued job waiting for the host's memory.
178#[derive(Debug, Clone, PartialEq, Eq)]
179pub struct HeldJob {
180    /// `owner/name`.
181    pub repository: String,
182    /// The workflow's name, when known.
183    pub workflow: Option<String>,
184    /// The job's name.
185    pub name: String,
186    /// How long it has waited for the host's memory, in ms (`jobs.host_waited`).
187    pub waited: u64,
188}
189
190/// What insights read from the job history.
191#[derive(Debug, Clone, Default, PartialEq, Eq)]
192pub struct History {
193    /// The jobs it closed in the days read.
194    pub runs: Vec<Run>,
195    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
196    /// start left only some, so no week before it is compared.
197    pub kept_since: u64,
198    /// The jobs held for the host's memory now; `None` when none is, or the daemon has no reading of it.
199    pub host: Option<HostHold>,
200}
201
202/// A job as insights tell jobs apart: repository, workflow and name.
203type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
204
205fn key_of(run: &Run) -> JobKey<'_> {
206    (&run.repository, run.workflow.as_deref(), &run.name)
207}
208
209/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
210/// only the runs finished by `now` count.
211pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
212    let runs = &history.runs;
213    let since = now.saturating_sub(u64::from(days) * DAY);
214    let window: Vec<&Run> = runs
215        .iter()
216        .filter(|run| {
217            matches!(run.state, JobState::Completed | JobState::Lost)
218                && run
219                    .finished_at
220                    .is_some_and(|at| (since..=now).contains(&at))
221        })
222        .collect();
223    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
224    for run in &window {
225        jobs.entry(key_of(run)).or_default().push(run);
226    }
227    let mut found = Vec::new();
228    for (key, runs) in &mut jobs {
229        runs.sort_by_key(|run| (run.finished(), run.id));
230        found.extend(slower(*key, runs));
231        found.extend(flaky(*key, runs));
232        found.extend(near_cap(*key, runs, config));
233        found.extend(cold_cache(*key, runs));
234        found.extend(cpu_held(*key, runs, config));
235    }
236    found.extend(throttled(&window));
237    found.extend(
238        history
239            .host
240            .as_ref()
241            .and_then(|hold| host_held(hold, config, now)),
242    );
243    let waits = waits(history, now);
244    found.extend(waits.iter().filter_map(waits_insight));
245    found.sort_by(|a, b| {
246        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
247            b.severity,
248            &b.repository,
249            &b.workflow,
250            &b.job,
251            &b.key,
252        ))
253    });
254    Insights {
255        since: since / SECOND,
256        until: now / SECOND,
257        insights: found,
258        waits,
259    }
260}
261
262/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
263/// before, and the longest waits.
264pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
265    let runs = &history.runs;
266    let since = now.saturating_sub(DAY);
267    let finished: Vec<&Run> = runs
268        .iter()
269        .filter(|run| {
270            matches!(run.state, JobState::Completed | JobState::Lost)
271                && run.finished_at.is_some_and(|at| at > since && at <= now)
272        })
273        .collect();
274    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
275        .insights
276        .into_iter()
277        .map(|insight| insight.key)
278        .collect();
279    let new = insights(history, config, api::INSIGHT_DAYS, now)
280        .insights
281        .into_iter()
282        .filter(|insight| !before.contains(&insight.key))
283        .collect();
284    let mut started: Vec<&Run> = runs
285        .iter()
286        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
287        .collect();
288    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
289    Digest {
290        since: since / SECOND,
291        until: now / SECOND,
292        finished: finished.len() as u64,
293        failed: finished.iter().filter(|run| run.failed()).count() as u64,
294        new,
295        waits: started
296            .into_iter()
297            .take(3)
298            .map(|run| LongWait {
299                job: run.id,
300                repository: run.repository.clone(),
301                workflow: run.workflow.clone(),
302                name: run.name.clone(),
303                waited: run.waited(history.kept_since).unwrap_or(0),
304                url: run.url.clone(),
305            })
306            .collect(),
307        savings: None,
308    }
309}
310
311/// A job has waited for the host's memory for 30 minutes or more, and `hold` says a job waits for it at `now` (ms):
312/// the host's memory is short, and the programs outside CI that hold it are the owner's to close. Worded here once
313/// for `runnerctl`, the dashboard, the TUI and notifications, with the longest waiter's repository. It names the
314/// programs `HostMemory::outside_line` names and says closing them frees only part of what is held, since the figure
315/// also counts the kernel and compressed swap.
316///
317/// Holds from the host's reading and the engine's look at the waiting jobs: the reading no older than a minute, the
318/// look no older than that or three `[host] poll_interval`s, neither from after `now`, the second of the look (a plan
319/// in that second is not from after it). With no reading, or a stale one, nothing is said. Its key is the same for 6
320/// hours and then changes, so a notification target tells it once in each window of them (told keys are stored, so a
321/// restart repeats none).
322fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
323    let reading = hold.memory.at.saturating_mul(SECOND);
324    // The engine plans once a `poll_interval`, so its look is as old as three of them before it is missed.
325    let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
326    let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
327    let fresh =
328        |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
329    if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
330        return None;
331    }
332    let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
333    if longest.waited < HOST_HELD_AFTER {
334        return None;
335    }
336    let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
337    let waited = longest.waited / SECOND;
338    let sentence = Insight::host_held_sentence(
339        jobs,
340        waited,
341        &subject((
342            &longest.repository,
343            longest.workflow.as_deref(),
344            &longest.name,
345        )),
346        &hold.memory,
347    );
348    Some(Insight {
349        key: format!("host-held:{}", now / HOST_HELD_EVERY),
350        severity: Severity::Warning,
351        repository: longest.repository.clone(),
352        workflow: None,
353        job: None,
354        sentence,
355        kind: InsightKind::HostHeld { jobs, waited },
356        runs: Vec::new(),
357    })
358}
359
360/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
361fn subject((repository, workflow, name): JobKey<'_>) -> String {
362    match workflow {
363        Some(workflow) => format!("{repository}: {workflow} / {name}"),
364        None => format!("{repository}: {name}"),
365    }
366}
367
368fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
369    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
370}
371
372fn insight(
373    key: JobKey<'_>,
374    kind: &str,
375    severity: Severity,
376    sentence: String,
377    found: InsightKind,
378    runs: Vec<RunLink>,
379) -> Insight {
380    Insight {
381        key: key_text(kind, key),
382        severity,
383        repository: key.0.to_owned(),
384        workflow: key.1.map(str::to_owned),
385        job: Some(key.2.to_owned()),
386        sentence,
387        kind: found,
388        runs,
389    }
390}
391
392/// The median: the mean of the middle two for an even count.
393fn median(values: &[u64]) -> u64 {
394    let mut values = values.to_vec();
395    values.sort_unstable();
396    let middle = values.len() / 2;
397    match values.len() {
398        0 => 0,
399        len if len % 2 == 1 => values[middle],
400        _ => u64::midpoint(values[middle - 1], values[middle]),
401    }
402}
403
404/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
405fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
406    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
407        runs.iter()
408            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
409            .filter_map(|run| Some((*run, run.took()?)))
410            .collect()
411    };
412    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
413    for run in runs.iter().filter(|run| run.succeeded()) {
414        if let Some(branch) = &run.branch {
415            *branches.entry(branch).or_default() += 1;
416        }
417    }
418    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
419    // most runs, and of equal counts the first by name.
420    let busiest = branches
421        .iter()
422        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
423        .map(|(branch, _)| *branch)?;
424    let (branch, timed) = ["main", "master"]
425        .into_iter()
426        .map(|branch| (branch, timed_on(branch)))
427        .find(|(_, timed)| timed.len() >= 2 * RECENT)
428        .unwrap_or_else(|| (busiest, timed_on(busiest)));
429    if timed.len() < 2 * RECENT {
430        return None;
431    }
432    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
433    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
434    let before = median(&took(baseline));
435    let now = median(&took(recent));
436    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
437    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
438    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
439    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
440        return None;
441    }
442    // The first slow run after which no two runs in a row were fast.
443    let mut since = None;
444    let mut fast_in_a_row = 0;
445    for (run, took) in &timed {
446        if *took >= threshold {
447            fast_in_a_row = 0;
448            since = since.or(Some(*run));
449        } else {
450            fast_in_a_row += 1;
451            if fast_in_a_row >= 2 {
452                since = None;
453            }
454        }
455    }
456    let since = since?;
457    let sentence = format!(
458        "{} on {branch} takes {} now, up from {}.",
459        subject(key),
460        span(now),
461        span(before)
462    );
463    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
464    Some(insight(
465        key,
466        "slower",
467        Severity::Warning,
468        sentence,
469        InsightKind::Slower {
470            branch: Some(branch.to_owned()),
471            before,
472            now,
473            since: Some(since.link()),
474        },
475        links,
476    ))
477}
478
479/// A job that failed and then passed on a re-run of the same run.
480fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
481    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
482    for run in runs {
483        by_run.entry(run.run_id).or_default().push(run);
484    }
485    let mut failed_first: Vec<&Run> = by_run
486        .values()
487        .filter_map(|attempts| {
488            attempts.iter().copied().find(|failed| {
489                failed.failed_itself()
490                    && attempts
491                        .iter()
492                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
493            })
494        })
495        .collect();
496    if failed_first.is_empty() {
497        return None;
498    }
499    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
500    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
501    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
502    let sentence = format!(
503        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
504        subject(key)
505    );
506    // Once in the window may be chance: a notice; twice, a warning.
507    let severity = if flaky >= FLAKY_WARNING {
508        Severity::Warning
509    } else {
510        Severity::Notice
511    };
512    Some(insight(
513        key,
514        "flaky",
515        severity,
516        sentence,
517        InsightKind::Flaky { flaky, total },
518        failed_first.iter().take(5).map(|run| run.link()).collect(),
519    ))
520}
521
522/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
523/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
524fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
525    let measured = runs
526        .iter()
527        .rev()
528        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
529    let name = measured.clone().next()?.profile.as_deref()?;
530    let measured: Vec<&Run> = measured
531        .filter(|run| run.profile.as_deref() == Some(name))
532        .take(CAP_RUNS)
533        .copied()
534        .collect();
535    if measured.len() < CAP_MIN_RUNS {
536        return None;
537    }
538    let profile = config.profiles.get(name)?;
539    let cap = profile.memory.0;
540    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
541    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
542    let peak = spread(peaks.clone()).p95;
543    if cap == 0 || !near(&peak) {
544        return None;
545    }
546    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
547    let auto = if !config.sizing.auto {
548        AutoNote::Off
549    } else if profile
550        .max_memory
551        .is_some_and(|max| profile.memory.0 >= max.0)
552    {
553        AutoNote::AtMax
554    } else {
555        AutoNote::Raises
556    };
557    let what = match auto {
558        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
559        AutoNote::AtMax => {
560            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
561        }
562        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
563    };
564    let sentence = format!(
565        "{} peaks at {}, {}% of {name}'s {}. {what}",
566        subject(key),
567        ByteSize(peak),
568        peak.saturating_mul(100) / cap,
569        ByteSize(cap)
570    );
571    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
572    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
573        Severity::Notice
574    } else {
575        Severity::Warning
576    };
577    Some(insight(
578        key,
579        "near-cap",
580        severity,
581        sentence,
582        InsightKind::NearCap {
583            profile: name.to_owned(),
584            peak,
585            cap,
586            auto,
587        },
588        measured.iter().take(5).map(|run| run.link()).collect(),
589    ))
590}
591
592/// A job's last runs that could write to the build cache, each with this many lookups at least…
593const COLD_RUNS: usize = 5;
594const COLD_MIN_RUNS: usize = 3;
595const COLD_MIN_LOOKUPS: u64 = 10;
596/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
597/// next should find most of it.
598const COLD_PERCENT: u64 = 10;
599
600fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
601    let writers: Vec<&Run> = runs
602        .iter()
603        .rev()
604        .filter(|run| {
605            run.build_cache
606                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
607        })
608        .take(COLD_RUNS)
609        .copied()
610        .collect();
611    if writers.len() < COLD_MIN_RUNS {
612        return None;
613    }
614    let mut total = api::BuildCacheCounts::default();
615    for run in &writers {
616        total.add(&run.build_cache.unwrap_or_default());
617    }
618    let percent = total.hit_percent()?;
619    if percent > COLD_PERCENT {
620        return None;
621    }
622    let branch = writers.first()?.branch.clone();
623    let sentence = format!(
624        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
625         differ from run to run.",
626        subject(key),
627        branch
628            .as_deref()
629            .map(|branch| format!(" on {branch}"))
630            .unwrap_or_default(),
631        writers.len()
632    );
633    Some(insight(
634        key,
635        "cold-cache",
636        Severity::Notice,
637        sentence,
638        InsightKind::ColdCache {
639            branch,
640            hits: total.hits,
641            lookups: total.lookups(),
642        },
643        writers.iter().map(|run| run.link()).collect(),
644    ))
645}
646
647/// A job whose CPU cap held back `RAISE_RUNS` or more of its last `CAP_RUNS` runs that measured it, as auto sizing
648/// reads a run (`Throttling::held`): successful runs, as sizing reads only those, in the profile and at the cap of its
649/// last such run, the profile's `cpus` today. A run without a throttling figure is unknown, neither held nor free.
650/// Says what the owner can do: under `measured` admission auto sizing raises `cpus` while each raise helps, under
651/// `caps` it does not for throttling, as each CPU takes budget there. A raise that did not help the job is told as
652/// this job's evidence only: auto sizing may still raise the profile for another job, or on the runs' averages.
653fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
654    let measured = runs.iter().rev().filter(|run| {
655        run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
656    });
657    let last = measured.clone().next()?;
658    let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
659    let profile = config.profiles.get(name)?;
660    // Runs at a cap the profile no longer has say nothing of the one it has: its next runs will.
661    if u64::from(profile.cpus) * 1000 != cap {
662        return None;
663    }
664    let at_cap: Vec<&Run> = measured
665        .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
666        .take(CAP_RUNS)
667        .copied()
668        .collect();
669    let held: Vec<&Run> = at_cap
670        .iter()
671        .filter(|run| run.throttling().held() == Some(true))
672        .copied()
673        .collect();
674    if (held.len() as u64) < RAISE_RUNS {
675        return None;
676    }
677    let cpus = profile.cpus;
678    let auto = config.host.admission == Admission::Measured && config.sizing.auto;
679    let raised_from = auto
680        .then(|| {
681            let ran: Vec<CpuRun> = runs
682                .iter()
683                .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
684                .map(|run| CpuRun {
685                    cap: run.cpu_cap,
686                    queued_at: run.queued_at,
687                    throttling: run.throttling(),
688                })
689                .collect();
690            let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
691            unhelped_raise(&ran, cap, &now)
692        })
693        .flatten()
694        .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
695    let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
696    let what = match (raised_from, set) {
697        (Some(from), _) => format!(
698            "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
699             tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
700             sizing does next."
701        ),
702        (None, Some(set)) if config.host.admission == Admission::Measured => {
703            format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
704        }
705        (None, Some(set)) => format!(
706            "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
707             cpus to {set} if runnerctl sizing shows the budget has room."
708        ),
709        (None, None) => format!(
710            "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
711             room; runnerctl sizing says when it cannot."
712        ),
713    };
714    let sentence = format!(
715        "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
716         its run or more. {what}",
717        subject(key),
718        held.len(),
719        at_cap.len()
720    );
721    // Auto sizing deals with it: a notice. The owner acts on the rest.
722    let severity = if auto && raised_from.is_none() {
723        Severity::Notice
724    } else {
725        Severity::Warning
726    };
727    Some(insight(
728        key,
729        "cpu-held",
730        severity,
731        sentence,
732        InsightKind::CpuHeld {
733            profile: name.to_owned(),
734            cpus,
735            held: u32::try_from(held.len()).unwrap_or(u32::MAX),
736            total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
737            raised_from,
738            set,
739        },
740        held.iter().take(5).map(|run| run.link()).collect(),
741    ))
742}
743
744/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
745fn throttled(window: &[&Run]) -> Vec<Insight> {
746    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
747    for run in window {
748        repositories.entry(&run.repository).or_default().push(run);
749    }
750    repositories
751        .into_iter()
752        .filter_map(|(repository, mut runs)| {
753            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
754            let hit: Vec<&Run> = runs
755                .iter()
756                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
757                .copied()
758                .collect();
759            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
760            if throttled < THROTTLED_MIN_RUNS {
761                return None;
762            }
763            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
764            Some(Insight {
765                key: format!("throttled:{repository}"),
766                severity: Severity::Warning,
767                repository: repository.to_owned(),
768                workflow: None,
769                job: None,
770                sentence: format!(
771                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
772                     host, or a job at its cap, was short of memory."
773                ),
774                kind: InsightKind::Throttled { throttled, total },
775                runs: hit.iter().take(5).map(|run| run.link()).collect(),
776            })
777        })
778        .collect()
779}
780
781/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
782/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
783/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
784fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
785    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
786    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
787    for run in &history.runs {
788        let Some(started) = run.started_at.filter(|at| *at <= now) else {
789            continue;
790        };
791        let week = match now - started {
792            back if back < WEEK => 0,
793            back if back < 2 * WEEK && !before_kept => 1,
794            _ => continue,
795        };
796        weeks.entry(&run.repository).or_default()[week].push(run);
797    }
798    weeks
799        .into_iter()
800        .map(|(repository, [now, before])| RepositoryWaits {
801            repository: repository.to_owned(),
802            now: week_waits(&now, history.kept_since),
803            before: week_waits(&before, history.kept_since),
804        })
805        .collect()
806}
807
808fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
809    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
810    let of = |runs: Vec<&&Run>| {
811        let waited: Vec<u64> = runs
812            .iter()
813            .filter_map(|run| run.waited(kept_since))
814            .collect();
815        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
816        let spread = (!waited.is_empty()).then(|| spread(waited));
817        WaitSpread {
818            jobs,
819            p50: spread.as_ref().map(|spread| spread.p50),
820            p95: spread.as_ref().map(|spread| spread.p95),
821        }
822    };
823    WeekWaits {
824        all: of(runs.iter().collect()),
825        warm: of(runs.iter().filter(warm).collect()),
826        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
827    }
828}
829
830fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
831    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
832        return None;
833    }
834    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
835    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
836    if now < threshold {
837        return None;
838    }
839    Some(Insight {
840        key: format!("waits:{}", waits.repository),
841        severity: Severity::Notice,
842        repository: waits.repository.clone(),
843        workflow: None,
844        job: None,
845        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
846            format!(
847                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
848                waits.repository,
849                span(now),
850                span(before)
851            )
852        } else {
853            format!(
854                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
855                waits.repository,
856                span(now),
857                span(before)
858            )
859        },
860        kind: InsightKind::Waits { before, now },
861        runs: Vec::new(),
862    })
863}
864
865/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
866fn span(seconds: u64) -> String {
867    match seconds {
868        0..60 => format!("{seconds}s"),
869        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
870        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
871        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
872    }
873}
874
875#[cfg(test)]
876mod tests;