Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, jobs their CPU cap held back by the rule auto sizing reads (ADR 0087), jobs
4//! held for the host's memory by programs outside CI (ADR 0100), and how long each repository's jobs wait, this week against the last when the history holds all of last week; and the
5//! daily digest. What the host's memory came to while CI ran, and the warning when it ran low on a host credit
6//! (ADR 0108), are `host`'s. Pure: the daemon reads the history, the config and the clock and passes them in. It sends nothing
7//! and changes nothing.
8
9use std::collections::{BTreeMap, BTreeSet};
10
11use crate::api::{
12    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
13    RunLink, Severity, WaitSpread, WeekWaits,
14};
15use crate::config::{Admission, ByteSize, Config};
16use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
17use crate::sizing::{HELD, Throttling, spread};
18
19mod disk;
20mod host;
21mod queue;
22
23const SECOND: u64 = 1000;
24const DAY: u64 = 24 * 3600 * SECOND;
25const WEEK: u64 = 7 * DAY;
26
27/// The last runs a slower job's median is taken over; the baseline needs as many before them.
28const RECENT: usize = 5;
29/// A job is slower when its recent median is this share above its baseline's, in percent…
30const SLOWER_PERCENT: u64 = 20;
31/// …and at least this many seconds above it.
32const SLOWER_SECONDS: u64 = 30;
33
34/// The last runs whose memory peaks are read, and the fewest that say anything.
35const CAP_RUNS: usize = 10;
36const CAP_MIN_RUNS: usize = 5;
37/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
38const NEAR_CAP: u64 = 85;
39/// Runs at `NEAR_CAP` or more that make it a warning.
40const NEAR_CAP_WARNING: usize = 2;
41/// Flaky runs that make it a warning.
42const FLAKY_WARNING: u32 = 2;
43
44/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
45/// above it, each week with this many jobs.
46const WAITS_PERCENT: u64 = 50;
47const WAITS_SECONDS: u64 = 60;
48const WAITS_MIN_JOBS: u32 = 5;
49/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
50const WAITS_ONE_IN_20: u32 = 20;
51
52/// A repository's runs the kernel throttled for memory pressure that make an insight.
53const THROTTLED_MIN_RUNS: u32 = 2;
54
55/// A queued job's wait for the host's memory that makes an insight, in milliseconds, while a job waits for it now.
56const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
57/// The insight's key changes this often, so it is told once in each such window however long the wait lasts, in
58/// milliseconds.
59const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
60/// How old the host's reading may be for the insight to hold, in milliseconds, and the least the engine's look at the
61/// waiting jobs may be, which is three `[host] poll_interval`s when that is more: a daemon that stopped looking says
62/// nothing about now.
63const HOST_HELD_FRESH: u64 = 60 * SECOND;
64
65/// One job the history closed, as insights read it. Times in milliseconds.
66#[derive(Debug, Clone, PartialEq, Eq)]
67pub struct Run {
68    /// GitHub's job ID.
69    pub id: u64,
70    /// `owner/name`.
71    pub repository: String,
72    /// The workflow's name, when known.
73    pub workflow: Option<String>,
74    /// The job's name.
75    pub name: String,
76    /// The branch the run is for.
77    pub branch: Option<String>,
78    /// The job's page on GitHub.
79    pub url: Option<String>,
80    /// The workflow run.
81    pub run_id: u64,
82    /// The run's attempt, 1 for the first.
83    pub run_attempt: u32,
84    /// `Completed` or `Lost`.
85    pub state: JobState,
86    /// GitHub's conclusion, for a completed job.
87    pub conclusion: Option<String>,
88    /// When GitHub queued it.
89    pub queued_at: u64,
90    /// When it started.
91    pub started_at: Option<u64>,
92    /// When it finished, or was found lost.
93    pub finished_at: Option<u64>,
94    /// When its runner's container started: before `queued_at`, it found a warm runner.
95    pub runner_started_at: Option<u64>,
96    /// The profile its runner ran in.
97    pub profile: Option<String>,
98    /// Its sampled memory peak.
99    pub memory_peak: Option<u64>,
100    /// What the build cache served it, if its runner had the build cache.
101    pub build_cache: Option<api::BuildCacheCounts>,
102    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
103    pub sock_throttled: Option<u64>,
104    /// Its container's CPU cap, millicores, when recorded.
105    pub cpu_cap: Option<u64>,
106    /// The periods of its CPU quota it had work in; `None` where the kernel kept no figure, as before schema 17.
107    pub cpu_periods: Option<u64>,
108    /// Of those, the periods its quota throttled it in.
109    pub cpu_throttled: Option<u64>,
110    /// What its runner container wrote to the disks under Docker's root (ADR 0143), when measured.
111    pub disk: Option<api::DiskWritten>,
112    /// How much of its wait passed while no runnerd ran.
113    pub unseen: u64,
114    /// When runnerd first listed it queued; `None` when no listing did.
115    pub seen_at: Option<u64>,
116    /// Of `unseen`, what passed before runnerd first listed it: the downtime its first listing found.
117    pub unseen_unlisted: u64,
118    /// How long it waited for each cause, in ms (ADR 0154); `None` for a row from before runnerd counted them.
119    pub causes: Option<BTreeMap<api::WaitCause, u64>>,
120}
121
122impl Run {
123    fn finished(&self) -> u64 {
124        self.finished_at.unwrap_or(0)
125    }
126
127    fn succeeded(&self) -> bool {
128        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
129    }
130
131    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
132    fn failed(&self) -> bool {
133        self.state == JobState::Lost || self.failed_itself()
134    }
135
136    /// Failed or timed out: what the job did, not what became of its runner.
137    fn failed_itself(&self) -> bool {
138        self.state == JobState::Completed
139            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
140    }
141
142    /// Seconds from start to finish.
143    fn took(&self) -> Option<u64> {
144        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
145    }
146
147    /// Its [wait](crate::sizing::Wait) in seconds, once it started.
148    fn waited(&self, kept_since: u64) -> Option<u64> {
149        let wait = crate::sizing::Wait {
150            queued_at: self.queued_at,
151            started_at: self.started_at?,
152            unseen: self.unseen,
153        };
154        Some(wait.counted(kept_since) / SECOND)
155    }
156
157    fn throttling(&self) -> Throttling {
158        Throttling {
159            started_at: self.started_at,
160            finished_at: self.finished_at,
161            periods: self.cpu_periods,
162            throttled: self.cpu_throttled,
163        }
164    }
165
166    fn link(&self) -> RunLink {
167        RunLink {
168            job: self.id,
169            run_id: self.run_id,
170            run_attempt: self.run_attempt,
171            branch: self.branch.clone(),
172            url: self.url.clone(),
173            at: self.finished() / SECOND,
174        }
175    }
176}
177
178/// The jobs waiting for the host's memory at one moment, and the reading that says what holds it. What the daemon
179/// knows of now, not of the history.
180#[derive(Debug, Clone, PartialEq, Eq)]
181pub struct HostHold {
182    /// The host's memory as last read.
183    pub memory: api::HostMemory,
184    /// When the engine last saw these jobs wait for it, in ms.
185    pub seen: u64,
186    /// The jobs waiting for it then that are still queued.
187    pub jobs: Vec<HeldJob>,
188}
189
190/// A queued job waiting for the host's memory.
191#[derive(Debug, Clone, PartialEq, Eq)]
192pub struct HeldJob {
193    /// `owner/name`.
194    pub repository: String,
195    /// The workflow's name, when known.
196    pub workflow: Option<String>,
197    /// The job's name.
198    pub name: String,
199    /// How long it has waited for the host's memory, in ms (`jobs.host_waited`).
200    pub waited: u64,
201}
202
203/// A reading of the host's memory as a `budget.changed` event stored it (ADR 0108).
204#[derive(Debug, Clone, Copy, PartialEq, Eq)]
205pub struct HostReading {
206    /// When it was stored, in ms.
207    pub at: u64,
208    /// The host's `MemTotal`, in bytes.
209    pub total: u64,
210    /// Its `MemAvailable`.
211    pub available: u64,
212    /// Its `SwapFree`, on a host with swap.
213    pub swap_free: Option<u64>,
214    /// What the host's limit took off the plain count then: 0 for no credit, as before 0.0.30.
215    pub credit: u64,
216    /// Whether a runner ran: the budget counted more than the mirror.
217    pub running: bool,
218}
219
220/// What insights read from the job history.
221#[derive(Debug, Clone, Default, PartialEq, Eq)]
222pub struct History {
223    /// The jobs it closed in the days read.
224    pub runs: Vec<Run>,
225    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
226    /// start left only some, so no week before it is compared.
227    pub kept_since: u64,
228    /// The jobs held for the host's memory now; `None` when none is, or the daemon has no reading of it.
229    pub host: Option<HostHold>,
230    /// The host's readings the budget's events stored, oldest first.
231    pub readings: Vec<HostReading>,
232    /// The windows of what the disks under Docker's root wrote, oldest first (ADR 0143).
233    pub disk: Vec<crate::observe::disk::Window>,
234}
235
236/// A job as insights tell jobs apart: repository, workflow and name.
237type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
238
239fn key_of(run: &Run) -> JobKey<'_> {
240    (&run.repository, run.workflow.as_deref(), &run.name)
241}
242
243/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
244/// only the runs finished by `now` count.
245pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
246    let runs = &history.runs;
247    let since = now.saturating_sub(u64::from(days) * DAY);
248    let window: Vec<&Run> = runs
249        .iter()
250        .filter(|run| {
251            matches!(run.state, JobState::Completed | JobState::Lost)
252                && run
253                    .finished_at
254                    .is_some_and(|at| (since..=now).contains(&at))
255        })
256        .collect();
257    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
258    for run in &window {
259        jobs.entry(key_of(run)).or_default().push(run);
260    }
261    let mut found = Vec::new();
262    for (key, runs) in &mut jobs {
263        runs.sort_by_key(|run| (run.finished(), run.id));
264        found.extend(slower(*key, runs));
265        found.extend(flaky(*key, runs));
266        found.extend(near_cap(*key, runs, config));
267        found.extend(cold_cache(*key, runs));
268        found.extend(cpu_held(*key, runs, config));
269    }
270    found.extend(throttled(&window));
271    found.extend(
272        history
273            .host
274            .as_ref()
275            .and_then(|hold| host_held(hold, config, now)),
276    );
277    let host_days = host::days(&history.readings, since, now);
278    found.extend(host::low(&host_days));
279    let waits = waits(history, now);
280    found.extend(waits.iter().filter_map(waits_insight));
281    found.sort_by(|a, b| {
282        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
283            b.severity,
284            &b.repository,
285            &b.workflow,
286            &b.job,
287            &b.key,
288        ))
289    });
290    Insights {
291        since: since / SECOND,
292        until: now / SECOND,
293        insights: found,
294        waits,
295        host: host_days,
296        disk: disk::writes(&window, &history.disk, since, now),
297        queue: queue::split(&window, history.kept_since),
298    }
299}
300
301/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
302/// before, and the longest waits.
303pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
304    let runs = &history.runs;
305    let since = now.saturating_sub(DAY);
306    let finished: Vec<&Run> = runs
307        .iter()
308        .filter(|run| {
309            matches!(run.state, JobState::Completed | JobState::Lost)
310                && run.finished_at.is_some_and(|at| at > since && at <= now)
311        })
312        .collect();
313    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
314        .insights
315        .into_iter()
316        .map(|insight| insight.key)
317        .collect();
318    let new = insights(history, config, api::INSIGHT_DAYS, now)
319        .insights
320        .into_iter()
321        .filter(|insight| !before.contains(&insight.key))
322        .collect();
323    let mut started: Vec<&Run> = runs
324        .iter()
325        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
326        .collect();
327    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
328    Digest {
329        since: since / SECOND,
330        until: now / SECOND,
331        finished: finished.len() as u64,
332        failed: finished.iter().filter(|run| run.failed()).count() as u64,
333        new,
334        waits: started
335            .into_iter()
336            .take(3)
337            .map(|run| LongWait {
338                job: run.id,
339                repository: run.repository.clone(),
340                workflow: run.workflow.clone(),
341                name: run.name.clone(),
342                waited: run.waited(history.kept_since).unwrap_or(0),
343                url: run.url.clone(),
344            })
345            .collect(),
346        savings: None,
347        host: host::hazard(&history.readings, since, now),
348    }
349}
350
351/// A job has waited for the host's memory for 30 minutes or more, and `hold` says a job waits for it at `now` (ms):
352/// the host's memory is short, and the programs outside CI that hold it are the owner's to close. Worded here once
353/// for `runnerctl`, the dashboard, the TUI and notifications, with the longest waiter's repository. It names the
354/// programs `HostMemory::outside_line` names and says closing them frees only part of what is held, since the figure
355/// also counts the kernel and compressed swap.
356///
357/// Holds from the host's reading and the engine's look at the waiting jobs: the reading no older than a minute, the
358/// look no older than that or three `[host] poll_interval`s, neither from after `now`, the second of the look (a plan
359/// in that second is not from after it). With no reading, or a stale one, nothing is said. Its key is the same for 6
360/// hours and then changes, so a notification target tells it once in each window of them (told keys are stored, so a
361/// restart repeats none).
362fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
363    let reading = hold.memory.at.saturating_mul(SECOND);
364    // The engine plans once a `poll_interval`, so its look is as old as three of them before it is missed.
365    let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
366    let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
367    let fresh =
368        |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
369    if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
370        return None;
371    }
372    let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
373    if longest.waited < HOST_HELD_AFTER {
374        return None;
375    }
376    let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
377    let waited = longest.waited / SECOND;
378    let sentence = Insight::host_held_sentence(
379        jobs,
380        waited,
381        &subject((
382            &longest.repository,
383            longest.workflow.as_deref(),
384            &longest.name,
385        )),
386        &hold.memory,
387    );
388    Some(Insight {
389        key: format!("host-held:{}", now / HOST_HELD_EVERY),
390        severity: Severity::Warning,
391        repository: longest.repository.clone(),
392        workflow: None,
393        job: None,
394        sentence,
395        kind: InsightKind::HostHeld { jobs, waited },
396        runs: Vec::new(),
397    })
398}
399
400/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
401fn subject((repository, workflow, name): JobKey<'_>) -> String {
402    match workflow {
403        Some(workflow) => format!("{repository}: {workflow} / {name}"),
404        None => format!("{repository}: {name}"),
405    }
406}
407
408fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
409    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
410}
411
412fn insight(
413    key: JobKey<'_>,
414    kind: &str,
415    severity: Severity,
416    sentence: String,
417    found: InsightKind,
418    runs: Vec<RunLink>,
419) -> Insight {
420    Insight {
421        key: key_text(kind, key),
422        severity,
423        repository: key.0.to_owned(),
424        workflow: key.1.map(str::to_owned),
425        job: Some(key.2.to_owned()),
426        sentence,
427        kind: found,
428        runs,
429    }
430}
431
432/// The median: the mean of the middle two for an even count.
433fn median(values: &[u64]) -> u64 {
434    let mut values = values.to_vec();
435    values.sort_unstable();
436    let middle = values.len() / 2;
437    match values.len() {
438        0 => 0,
439        len if len % 2 == 1 => values[middle],
440        _ => u64::midpoint(values[middle - 1], values[middle]),
441    }
442}
443
444/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
445fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
446    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
447        runs.iter()
448            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
449            .filter_map(|run| Some((*run, run.took()?)))
450            .collect()
451    };
452    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
453    for run in runs.iter().filter(|run| run.succeeded()) {
454        if let Some(branch) = &run.branch {
455            *branches.entry(branch).or_default() += 1;
456        }
457    }
458    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
459    // most runs, and of equal counts the first by name.
460    let busiest = branches
461        .iter()
462        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
463        .map(|(branch, _)| *branch)?;
464    let (branch, timed) = ["main", "master"]
465        .into_iter()
466        .map(|branch| (branch, timed_on(branch)))
467        .find(|(_, timed)| timed.len() >= 2 * RECENT)
468        .unwrap_or_else(|| (busiest, timed_on(busiest)));
469    if timed.len() < 2 * RECENT {
470        return None;
471    }
472    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
473    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
474    let before = median(&took(baseline));
475    let now = median(&took(recent));
476    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
477    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
478    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
479    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
480        return None;
481    }
482    // The first slow run after which no two runs in a row were fast.
483    let mut since = None;
484    let mut fast_in_a_row = 0;
485    for (run, took) in &timed {
486        if *took >= threshold {
487            fast_in_a_row = 0;
488            since = since.or(Some(*run));
489        } else {
490            fast_in_a_row += 1;
491            if fast_in_a_row >= 2 {
492                since = None;
493            }
494        }
495    }
496    let since = since?;
497    let sentence = format!(
498        "{} on {branch} takes {} now, up from {}.",
499        subject(key),
500        span(now),
501        span(before)
502    );
503    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
504    Some(insight(
505        key,
506        "slower",
507        Severity::Warning,
508        sentence,
509        InsightKind::Slower {
510            branch: Some(branch.to_owned()),
511            before,
512            now,
513            since: Some(since.link()),
514        },
515        links,
516    ))
517}
518
519/// A job that failed and then passed on a re-run of the same run.
520fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
521    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
522    for run in runs {
523        by_run.entry(run.run_id).or_default().push(run);
524    }
525    let mut failed_first: Vec<&Run> = by_run
526        .values()
527        .filter_map(|attempts| {
528            attempts.iter().copied().find(|failed| {
529                failed.failed_itself()
530                    && attempts
531                        .iter()
532                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
533            })
534        })
535        .collect();
536    if failed_first.is_empty() {
537        return None;
538    }
539    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
540    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
541    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
542    let sentence = format!(
543        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
544        subject(key)
545    );
546    // Once in the window may be chance: a notice; twice, a warning.
547    let severity = if flaky >= FLAKY_WARNING {
548        Severity::Warning
549    } else {
550        Severity::Notice
551    };
552    Some(insight(
553        key,
554        "flaky",
555        severity,
556        sentence,
557        InsightKind::Flaky { flaky, total },
558        failed_first.iter().take(5).map(|run| run.link()).collect(),
559    ))
560}
561
562/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
563/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
564fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
565    let measured = runs
566        .iter()
567        .rev()
568        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
569    let name = measured.clone().next()?.profile.as_deref()?;
570    let measured: Vec<&Run> = measured
571        .filter(|run| run.profile.as_deref() == Some(name))
572        .take(CAP_RUNS)
573        .copied()
574        .collect();
575    if measured.len() < CAP_MIN_RUNS {
576        return None;
577    }
578    let profile = config.profiles.get(name)?;
579    let cap = profile.memory.0;
580    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
581    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
582    let peak = spread(peaks.clone()).p95;
583    if cap == 0 || !near(&peak) {
584        return None;
585    }
586    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
587    let auto = if !config.sizing.auto {
588        AutoNote::Off
589    } else if profile
590        .max_memory
591        .is_some_and(|max| profile.memory.0 >= max.0)
592    {
593        AutoNote::AtMax
594    } else {
595        AutoNote::Raises
596    };
597    let what = match auto {
598        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
599        AutoNote::AtMax => {
600            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
601        }
602        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
603    };
604    let sentence = format!(
605        "{} peaks at {}, {}% of {name}'s {}. {what}",
606        subject(key),
607        ByteSize(peak),
608        peak.saturating_mul(100) / cap,
609        ByteSize(cap)
610    );
611    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
612    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
613        Severity::Notice
614    } else {
615        Severity::Warning
616    };
617    Some(insight(
618        key,
619        "near-cap",
620        severity,
621        sentence,
622        InsightKind::NearCap {
623            profile: name.to_owned(),
624            peak,
625            cap,
626            auto,
627        },
628        measured.iter().take(5).map(|run| run.link()).collect(),
629    ))
630}
631
632/// A job's last runs that could write to the build cache, each with this many lookups at least…
633const COLD_RUNS: usize = 5;
634const COLD_MIN_RUNS: usize = 3;
635const COLD_MIN_LOOKUPS: u64 = 10;
636/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
637/// next should find most of it.
638const COLD_PERCENT: u64 = 10;
639
640fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
641    let writers: Vec<&Run> = runs
642        .iter()
643        .rev()
644        .filter(|run| {
645            run.build_cache
646                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
647        })
648        .take(COLD_RUNS)
649        .copied()
650        .collect();
651    if writers.len() < COLD_MIN_RUNS {
652        return None;
653    }
654    let mut total = api::BuildCacheCounts::default();
655    for run in &writers {
656        total.add(&run.build_cache.unwrap_or_default());
657    }
658    let percent = total.hit_percent()?;
659    if percent > COLD_PERCENT {
660        return None;
661    }
662    let branch = writers.first()?.branch.clone();
663    let sentence = format!(
664        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
665         differ from run to run.",
666        subject(key),
667        branch
668            .as_deref()
669            .map(|branch| format!(" on {branch}"))
670            .unwrap_or_default(),
671        writers.len()
672    );
673    Some(insight(
674        key,
675        "cold-cache",
676        Severity::Notice,
677        sentence,
678        InsightKind::ColdCache {
679            branch,
680            hits: total.hits,
681            lookups: total.lookups(),
682        },
683        writers.iter().map(|run| run.link()).collect(),
684    ))
685}
686
687/// A job whose CPU cap held back `RAISE_RUNS` or more of its last `CAP_RUNS` runs that measured it, as auto sizing
688/// reads a run (`Throttling::held`): successful runs, as sizing reads only those, in the profile and at the cap of its
689/// last such run, the profile's `cpus` today. A run without a throttling figure is unknown, neither held nor free.
690/// Says what the owner can do: under `measured` admission auto sizing raises `cpus` while each raise helps, under
691/// `caps` it does not for throttling, as each CPU takes budget there. A raise that did not help the job is told as
692/// this job's evidence only: auto sizing may still raise the profile for another job, or on the runs' averages.
693fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
694    let measured = runs.iter().rev().filter(|run| {
695        run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
696    });
697    let last = measured.clone().next()?;
698    let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
699    let profile = config.profiles.get(name)?;
700    // Runs at a cap the profile no longer has say nothing of the one it has: its next runs will.
701    if u64::from(profile.cpus) * 1000 != cap {
702        return None;
703    }
704    let at_cap: Vec<&Run> = measured
705        .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
706        .take(CAP_RUNS)
707        .copied()
708        .collect();
709    let held: Vec<&Run> = at_cap
710        .iter()
711        .filter(|run| run.throttling().held() == Some(true))
712        .copied()
713        .collect();
714    if (held.len() as u64) < RAISE_RUNS {
715        return None;
716    }
717    let cpus = profile.cpus;
718    let auto = config.host.admission == Admission::Measured && config.sizing.auto;
719    let raised_from = auto
720        .then(|| {
721            let ran: Vec<CpuRun> = runs
722                .iter()
723                .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
724                .map(|run| CpuRun {
725                    cap: run.cpu_cap,
726                    queued_at: run.queued_at,
727                    throttling: run.throttling(),
728                })
729                .collect();
730            let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
731            unhelped_raise(&ran, cap, &now)
732        })
733        .flatten()
734        .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
735    let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
736    let what = match (raised_from, set) {
737        (Some(from), _) => format!(
738            "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
739             tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
740             sizing does next."
741        ),
742        (None, Some(set)) if config.host.admission == Admission::Measured => {
743            format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
744        }
745        (None, Some(set)) => format!(
746            "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
747             cpus to {set} if runnerctl sizing shows the budget has room."
748        ),
749        (None, None) => format!(
750            "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
751             room; runnerctl sizing says when it cannot."
752        ),
753    };
754    let sentence = format!(
755        "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
756         its run or more. {what}",
757        subject(key),
758        held.len(),
759        at_cap.len()
760    );
761    // Auto sizing deals with it: a notice. The owner acts on the rest.
762    let severity = if auto && raised_from.is_none() {
763        Severity::Notice
764    } else {
765        Severity::Warning
766    };
767    Some(insight(
768        key,
769        "cpu-held",
770        severity,
771        sentence,
772        InsightKind::CpuHeld {
773            profile: name.to_owned(),
774            cpus,
775            held: u32::try_from(held.len()).unwrap_or(u32::MAX),
776            total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
777            raised_from,
778            set,
779        },
780        held.iter().take(5).map(|run| run.link()).collect(),
781    ))
782}
783
784/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
785fn throttled(window: &[&Run]) -> Vec<Insight> {
786    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
787    for run in window {
788        repositories.entry(&run.repository).or_default().push(run);
789    }
790    repositories
791        .into_iter()
792        .filter_map(|(repository, mut runs)| {
793            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
794            let hit: Vec<&Run> = runs
795                .iter()
796                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
797                .copied()
798                .collect();
799            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
800            if throttled < THROTTLED_MIN_RUNS {
801                return None;
802            }
803            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
804            Some(Insight {
805                key: format!("throttled:{repository}"),
806                severity: Severity::Warning,
807                repository: repository.to_owned(),
808                workflow: None,
809                job: None,
810                sentence: format!(
811                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
812                     host, or a job at its cap, was short of memory."
813                ),
814                kind: InsightKind::Throttled { throttled, total },
815                runs: hit.iter().take(5).map(|run| run.link()).collect(),
816            })
817        })
818        .collect()
819}
820
821/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
822/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
823/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
824fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
825    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
826    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
827    for run in &history.runs {
828        let Some(started) = run.started_at.filter(|at| *at <= now) else {
829            continue;
830        };
831        let week = match now - started {
832            back if back < WEEK => 0,
833            back if back < 2 * WEEK && !before_kept => 1,
834            _ => continue,
835        };
836        weeks.entry(&run.repository).or_default()[week].push(run);
837    }
838    weeks
839        .into_iter()
840        .map(|(repository, [now, before])| RepositoryWaits {
841            repository: repository.to_owned(),
842            now: week_waits(&now, history.kept_since),
843            before: week_waits(&before, history.kept_since),
844        })
845        .collect()
846}
847
848fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
849    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
850    let of = |runs: Vec<&&Run>| {
851        let waited: Vec<u64> = runs
852            .iter()
853            .filter_map(|run| run.waited(kept_since))
854            .collect();
855        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
856        let spread = (!waited.is_empty()).then(|| spread(waited));
857        WaitSpread {
858            jobs,
859            p50: spread.as_ref().map(|spread| spread.p50),
860            p95: spread.as_ref().map(|spread| spread.p95),
861        }
862    };
863    WeekWaits {
864        all: of(runs.iter().collect()),
865        warm: of(runs.iter().filter(warm).collect()),
866        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
867    }
868}
869
870fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
871    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
872        return None;
873    }
874    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
875    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
876    if now < threshold {
877        return None;
878    }
879    Some(Insight {
880        key: format!("waits:{}", waits.repository),
881        severity: Severity::Notice,
882        repository: waits.repository.clone(),
883        workflow: None,
884        job: None,
885        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
886            format!(
887                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
888                waits.repository,
889                span(now),
890                span(before)
891            )
892        } else {
893            format!(
894                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
895                waits.repository,
896                span(now),
897                span(before)
898            )
899        },
900        kind: InsightKind::Waits { before, now },
901        runs: Vec::new(),
902    })
903}
904
905/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
906fn span(seconds: u64) -> String {
907    match seconds {
908        0..60 => format!("{seconds}s"),
909        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
910        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
911        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
912    }
913}
914
915#[cfg(test)]
916mod tests;