Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, jobs their CPU cap held back by the rule auto sizing reads (ADR 0087), jobs
4//! held for the host's memory by programs outside CI (ADR 0100), and how long each repository's jobs wait, this week against the last when the history holds all of last week; and the
5//! daily digest. What the host's memory came to while CI ran, and the warning when it ran low on a host credit
6//! (ADR 0108), are `host`'s. Pure: the daemon reads the history, the config and the clock and passes them in. It sends nothing
7//! and changes nothing.
8
9use std::collections::{BTreeMap, BTreeSet};
10
11use crate::api::{
12    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
13    RunLink, Severity, WaitSpread, WeekWaits,
14};
15use crate::config::{Admission, ByteSize, Config};
16use crate::sizing::auto::{CpuRun, RAISE_RUNS, cpu_step, unhelped_raise};
17use crate::sizing::{HELD, Throttling, spread};
18
19mod disk;
20mod host;
21
22const SECOND: u64 = 1000;
23const DAY: u64 = 24 * 3600 * SECOND;
24const WEEK: u64 = 7 * DAY;
25
26/// The last runs a slower job's median is taken over; the baseline needs as many before them.
27const RECENT: usize = 5;
28/// A job is slower when its recent median is this share above its baseline's, in percent…
29const SLOWER_PERCENT: u64 = 20;
30/// …and at least this many seconds above it.
31const SLOWER_SECONDS: u64 = 30;
32
33/// The last runs whose memory peaks are read, and the fewest that say anything.
34const CAP_RUNS: usize = 10;
35const CAP_MIN_RUNS: usize = 5;
36/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
37const NEAR_CAP: u64 = 85;
38/// Runs at `NEAR_CAP` or more that make it a warning.
39const NEAR_CAP_WARNING: usize = 2;
40/// Flaky runs that make it a warning.
41const FLAKY_WARNING: u32 = 2;
42
43/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
44/// above it, each week with this many jobs.
45const WAITS_PERCENT: u64 = 50;
46const WAITS_SECONDS: u64 = 60;
47const WAITS_MIN_JOBS: u32 = 5;
48/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
49const WAITS_ONE_IN_20: u32 = 20;
50
51/// A repository's runs the kernel throttled for memory pressure that make an insight.
52const THROTTLED_MIN_RUNS: u32 = 2;
53
54/// A queued job's wait for the host's memory that makes an insight, in milliseconds, while a job waits for it now.
55const HOST_HELD_AFTER: u64 = 30 * 60 * SECOND;
56/// The insight's key changes this often, so it is told once in each such window however long the wait lasts, in
57/// milliseconds.
58const HOST_HELD_EVERY: u64 = 6 * 3600 * SECOND;
59/// How old the host's reading may be for the insight to hold, in milliseconds, and the least the engine's look at the
60/// waiting jobs may be, which is three `[host] poll_interval`s when that is more: a daemon that stopped looking says
61/// nothing about now.
62const HOST_HELD_FRESH: u64 = 60 * SECOND;
63
64/// One job the history closed, as insights read it. Times in milliseconds.
65#[derive(Debug, Clone, PartialEq, Eq)]
66pub struct Run {
67    /// GitHub's job ID.
68    pub id: u64,
69    /// `owner/name`.
70    pub repository: String,
71    /// The workflow's name, when known.
72    pub workflow: Option<String>,
73    /// The job's name.
74    pub name: String,
75    /// The branch the run is for.
76    pub branch: Option<String>,
77    /// The job's page on GitHub.
78    pub url: Option<String>,
79    /// The workflow run.
80    pub run_id: u64,
81    /// The run's attempt, 1 for the first.
82    pub run_attempt: u32,
83    /// `Completed` or `Lost`.
84    pub state: JobState,
85    /// GitHub's conclusion, for a completed job.
86    pub conclusion: Option<String>,
87    /// When GitHub queued it.
88    pub queued_at: u64,
89    /// When it started.
90    pub started_at: Option<u64>,
91    /// When it finished, or was found lost.
92    pub finished_at: Option<u64>,
93    /// When its runner's container started: before `queued_at`, it found a warm runner.
94    pub runner_started_at: Option<u64>,
95    /// The profile its runner ran in.
96    pub profile: Option<String>,
97    /// Its sampled memory peak.
98    pub memory_peak: Option<u64>,
99    /// What the build cache served it, if its runner had the build cache.
100    pub build_cache: Option<api::BuildCacheCounts>,
101    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
102    pub sock_throttled: Option<u64>,
103    /// Its container's CPU cap, millicores, when recorded.
104    pub cpu_cap: Option<u64>,
105    /// The periods of its CPU quota it had work in; `None` where the kernel kept no figure, as before schema 17.
106    pub cpu_periods: Option<u64>,
107    /// Of those, the periods its quota throttled it in.
108    pub cpu_throttled: Option<u64>,
109    /// What its runner container wrote to the disks under Docker's root (ADR 0143), when measured.
110    pub disk: Option<api::DiskWritten>,
111    /// How much of its wait passed while no runnerd ran.
112    pub unseen: u64,
113}
114
115impl Run {
116    fn finished(&self) -> u64 {
117        self.finished_at.unwrap_or(0)
118    }
119
120    fn succeeded(&self) -> bool {
121        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
122    }
123
124    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
125    fn failed(&self) -> bool {
126        self.state == JobState::Lost || self.failed_itself()
127    }
128
129    /// Failed or timed out: what the job did, not what became of its runner.
130    fn failed_itself(&self) -> bool {
131        self.state == JobState::Completed
132            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
133    }
134
135    /// Seconds from start to finish.
136    fn took(&self) -> Option<u64> {
137        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
138    }
139
140    /// Its [wait](crate::sizing::Wait) in seconds, once it started.
141    fn waited(&self, kept_since: u64) -> Option<u64> {
142        let wait = crate::sizing::Wait {
143            queued_at: self.queued_at,
144            started_at: self.started_at?,
145            unseen: self.unseen,
146        };
147        Some(wait.counted(kept_since) / SECOND)
148    }
149
150    fn throttling(&self) -> Throttling {
151        Throttling {
152            started_at: self.started_at,
153            finished_at: self.finished_at,
154            periods: self.cpu_periods,
155            throttled: self.cpu_throttled,
156        }
157    }
158
159    fn link(&self) -> RunLink {
160        RunLink {
161            job: self.id,
162            run_id: self.run_id,
163            run_attempt: self.run_attempt,
164            branch: self.branch.clone(),
165            url: self.url.clone(),
166            at: self.finished() / SECOND,
167        }
168    }
169}
170
171/// The jobs waiting for the host's memory at one moment, and the reading that says what holds it. What the daemon
172/// knows of now, not of the history.
173#[derive(Debug, Clone, PartialEq, Eq)]
174pub struct HostHold {
175    /// The host's memory as last read.
176    pub memory: api::HostMemory,
177    /// When the engine last saw these jobs wait for it, in ms.
178    pub seen: u64,
179    /// The jobs waiting for it then that are still queued.
180    pub jobs: Vec<HeldJob>,
181}
182
183/// A queued job waiting for the host's memory.
184#[derive(Debug, Clone, PartialEq, Eq)]
185pub struct HeldJob {
186    /// `owner/name`.
187    pub repository: String,
188    /// The workflow's name, when known.
189    pub workflow: Option<String>,
190    /// The job's name.
191    pub name: String,
192    /// How long it has waited for the host's memory, in ms (`jobs.host_waited`).
193    pub waited: u64,
194}
195
196/// A reading of the host's memory as a `budget.changed` event stored it (ADR 0108).
197#[derive(Debug, Clone, Copy, PartialEq, Eq)]
198pub struct HostReading {
199    /// When it was stored, in ms.
200    pub at: u64,
201    /// The host's `MemTotal`, in bytes.
202    pub total: u64,
203    /// Its `MemAvailable`.
204    pub available: u64,
205    /// Its `SwapFree`, on a host with swap.
206    pub swap_free: Option<u64>,
207    /// What the host's limit took off the plain count then: 0 for no credit, as before 0.0.30.
208    pub credit: u64,
209    /// Whether a runner ran: the budget counted more than the mirror.
210    pub running: bool,
211}
212
213/// What insights read from the job history.
214#[derive(Debug, Clone, Default, PartialEq, Eq)]
215pub struct History {
216    /// The jobs it closed in the days read.
217    pub runs: Vec<Run>,
218    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
219    /// start left only some, so no week before it is compared.
220    pub kept_since: u64,
221    /// The jobs held for the host's memory now; `None` when none is, or the daemon has no reading of it.
222    pub host: Option<HostHold>,
223    /// The host's readings the budget's events stored, oldest first.
224    pub readings: Vec<HostReading>,
225    /// The windows of what the disks under Docker's root wrote, oldest first (ADR 0143).
226    pub disk: Vec<crate::observe::disk::Window>,
227}
228
229/// A job as insights tell jobs apart: repository, workflow and name.
230type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
231
232fn key_of(run: &Run) -> JobKey<'_> {
233    (&run.repository, run.workflow.as_deref(), &run.name)
234}
235
236/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
237/// only the runs finished by `now` count.
238pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
239    let runs = &history.runs;
240    let since = now.saturating_sub(u64::from(days) * DAY);
241    let window: Vec<&Run> = runs
242        .iter()
243        .filter(|run| {
244            matches!(run.state, JobState::Completed | JobState::Lost)
245                && run
246                    .finished_at
247                    .is_some_and(|at| (since..=now).contains(&at))
248        })
249        .collect();
250    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
251    for run in &window {
252        jobs.entry(key_of(run)).or_default().push(run);
253    }
254    let mut found = Vec::new();
255    for (key, runs) in &mut jobs {
256        runs.sort_by_key(|run| (run.finished(), run.id));
257        found.extend(slower(*key, runs));
258        found.extend(flaky(*key, runs));
259        found.extend(near_cap(*key, runs, config));
260        found.extend(cold_cache(*key, runs));
261        found.extend(cpu_held(*key, runs, config));
262    }
263    found.extend(throttled(&window));
264    found.extend(
265        history
266            .host
267            .as_ref()
268            .and_then(|hold| host_held(hold, config, now)),
269    );
270    let host_days = host::days(&history.readings, since, now);
271    found.extend(host::low(&host_days));
272    let waits = waits(history, now);
273    found.extend(waits.iter().filter_map(waits_insight));
274    found.sort_by(|a, b| {
275        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
276            b.severity,
277            &b.repository,
278            &b.workflow,
279            &b.job,
280            &b.key,
281        ))
282    });
283    Insights {
284        since: since / SECOND,
285        until: now / SECOND,
286        insights: found,
287        waits,
288        host: host_days,
289        disk: disk::writes(&window, &history.disk, since, now),
290    }
291}
292
293/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
294/// before, and the longest waits.
295pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
296    let runs = &history.runs;
297    let since = now.saturating_sub(DAY);
298    let finished: Vec<&Run> = runs
299        .iter()
300        .filter(|run| {
301            matches!(run.state, JobState::Completed | JobState::Lost)
302                && run.finished_at.is_some_and(|at| at > since && at <= now)
303        })
304        .collect();
305    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
306        .insights
307        .into_iter()
308        .map(|insight| insight.key)
309        .collect();
310    let new = insights(history, config, api::INSIGHT_DAYS, now)
311        .insights
312        .into_iter()
313        .filter(|insight| !before.contains(&insight.key))
314        .collect();
315    let mut started: Vec<&Run> = runs
316        .iter()
317        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
318        .collect();
319    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
320    Digest {
321        since: since / SECOND,
322        until: now / SECOND,
323        finished: finished.len() as u64,
324        failed: finished.iter().filter(|run| run.failed()).count() as u64,
325        new,
326        waits: started
327            .into_iter()
328            .take(3)
329            .map(|run| LongWait {
330                job: run.id,
331                repository: run.repository.clone(),
332                workflow: run.workflow.clone(),
333                name: run.name.clone(),
334                waited: run.waited(history.kept_since).unwrap_or(0),
335                url: run.url.clone(),
336            })
337            .collect(),
338        savings: None,
339        host: host::hazard(&history.readings, since, now),
340    }
341}
342
343/// A job has waited for the host's memory for 30 minutes or more, and `hold` says a job waits for it at `now` (ms):
344/// the host's memory is short, and the programs outside CI that hold it are the owner's to close. Worded here once
345/// for `runnerctl`, the dashboard, the TUI and notifications, with the longest waiter's repository. It names the
346/// programs `HostMemory::outside_line` names and says closing them frees only part of what is held, since the figure
347/// also counts the kernel and compressed swap.
348///
349/// Holds from the host's reading and the engine's look at the waiting jobs: the reading no older than a minute, the
350/// look no older than that or three `[host] poll_interval`s, neither from after `now`, the second of the look (a plan
351/// in that second is not from after it). With no reading, or a stale one, nothing is said. Its key is the same for 6
352/// hours and then changes, so a notification target tells it once in each window of them (told keys are stored, so a
353/// restart repeats none).
354fn host_held(hold: &HostHold, config: &Config, now: u64) -> Option<Insight> {
355    let reading = hold.memory.at.saturating_mul(SECOND);
356    // The engine plans once a `poll_interval`, so its look is as old as three of them before it is missed.
357    let poll = u64::try_from(config.host.poll_interval.as_millis()).unwrap_or(u64::MAX);
358    let plan_fresh = HOST_HELD_FRESH.max(poll.saturating_mul(3));
359    let fresh =
360        |at: u64, within: u64| at / SECOND <= now / SECOND && now.saturating_sub(at) <= within;
361    if !fresh(reading, HOST_HELD_FRESH) || !fresh(hold.seen, plan_fresh) {
362        return None;
363    }
364    let longest = hold.jobs.iter().max_by_key(|job| job.waited)?;
365    if longest.waited < HOST_HELD_AFTER {
366        return None;
367    }
368    let jobs = u32::try_from(hold.jobs.len()).unwrap_or(u32::MAX);
369    let waited = longest.waited / SECOND;
370    let sentence = Insight::host_held_sentence(
371        jobs,
372        waited,
373        &subject((
374            &longest.repository,
375            longest.workflow.as_deref(),
376            &longest.name,
377        )),
378        &hold.memory,
379    );
380    Some(Insight {
381        key: format!("host-held:{}", now / HOST_HELD_EVERY),
382        severity: Severity::Warning,
383        repository: longest.repository.clone(),
384        workflow: None,
385        job: None,
386        sentence,
387        kind: InsightKind::HostHeld { jobs, waited },
388        runs: Vec::new(),
389    })
390}
391
392/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
393fn subject((repository, workflow, name): JobKey<'_>) -> String {
394    match workflow {
395        Some(workflow) => format!("{repository}: {workflow} / {name}"),
396        None => format!("{repository}: {name}"),
397    }
398}
399
400fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
401    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
402}
403
404fn insight(
405    key: JobKey<'_>,
406    kind: &str,
407    severity: Severity,
408    sentence: String,
409    found: InsightKind,
410    runs: Vec<RunLink>,
411) -> Insight {
412    Insight {
413        key: key_text(kind, key),
414        severity,
415        repository: key.0.to_owned(),
416        workflow: key.1.map(str::to_owned),
417        job: Some(key.2.to_owned()),
418        sentence,
419        kind: found,
420        runs,
421    }
422}
423
424/// The median: the mean of the middle two for an even count.
425fn median(values: &[u64]) -> u64 {
426    let mut values = values.to_vec();
427    values.sort_unstable();
428    let middle = values.len() / 2;
429    match values.len() {
430        0 => 0,
431        len if len % 2 == 1 => values[middle],
432        _ => u64::midpoint(values[middle - 1], values[middle]),
433    }
434}
435
436/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
437fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
438    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
439        runs.iter()
440            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
441            .filter_map(|run| Some((*run, run.took()?)))
442            .collect()
443    };
444    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
445    for run in runs.iter().filter(|run| run.succeeded()) {
446        if let Some(branch) = &run.branch {
447            *branches.entry(branch).or_default() += 1;
448        }
449    }
450    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
451    // most runs, and of equal counts the first by name.
452    let busiest = branches
453        .iter()
454        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
455        .map(|(branch, _)| *branch)?;
456    let (branch, timed) = ["main", "master"]
457        .into_iter()
458        .map(|branch| (branch, timed_on(branch)))
459        .find(|(_, timed)| timed.len() >= 2 * RECENT)
460        .unwrap_or_else(|| (busiest, timed_on(busiest)));
461    if timed.len() < 2 * RECENT {
462        return None;
463    }
464    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
465    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
466    let before = median(&took(baseline));
467    let now = median(&took(recent));
468    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
469    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
470    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
471    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
472        return None;
473    }
474    // The first slow run after which no two runs in a row were fast.
475    let mut since = None;
476    let mut fast_in_a_row = 0;
477    for (run, took) in &timed {
478        if *took >= threshold {
479            fast_in_a_row = 0;
480            since = since.or(Some(*run));
481        } else {
482            fast_in_a_row += 1;
483            if fast_in_a_row >= 2 {
484                since = None;
485            }
486        }
487    }
488    let since = since?;
489    let sentence = format!(
490        "{} on {branch} takes {} now, up from {}.",
491        subject(key),
492        span(now),
493        span(before)
494    );
495    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
496    Some(insight(
497        key,
498        "slower",
499        Severity::Warning,
500        sentence,
501        InsightKind::Slower {
502            branch: Some(branch.to_owned()),
503            before,
504            now,
505            since: Some(since.link()),
506        },
507        links,
508    ))
509}
510
511/// A job that failed and then passed on a re-run of the same run.
512fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
513    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
514    for run in runs {
515        by_run.entry(run.run_id).or_default().push(run);
516    }
517    let mut failed_first: Vec<&Run> = by_run
518        .values()
519        .filter_map(|attempts| {
520            attempts.iter().copied().find(|failed| {
521                failed.failed_itself()
522                    && attempts
523                        .iter()
524                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
525            })
526        })
527        .collect();
528    if failed_first.is_empty() {
529        return None;
530    }
531    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
532    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
533    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
534    let sentence = format!(
535        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
536        subject(key)
537    );
538    // Once in the window may be chance: a notice; twice, a warning.
539    let severity = if flaky >= FLAKY_WARNING {
540        Severity::Warning
541    } else {
542        Severity::Notice
543    };
544    Some(insight(
545        key,
546        "flaky",
547        severity,
548        sentence,
549        InsightKind::Flaky { flaky, total },
550        failed_first.iter().take(5).map(|run| run.link()).collect(),
551    ))
552}
553
554/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
555/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
556fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
557    let measured = runs
558        .iter()
559        .rev()
560        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
561    let name = measured.clone().next()?.profile.as_deref()?;
562    let measured: Vec<&Run> = measured
563        .filter(|run| run.profile.as_deref() == Some(name))
564        .take(CAP_RUNS)
565        .copied()
566        .collect();
567    if measured.len() < CAP_MIN_RUNS {
568        return None;
569    }
570    let profile = config.profiles.get(name)?;
571    let cap = profile.memory.0;
572    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
573    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
574    let peak = spread(peaks.clone()).p95;
575    if cap == 0 || !near(&peak) {
576        return None;
577    }
578    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
579    let auto = if !config.sizing.auto {
580        AutoNote::Off
581    } else if profile
582        .max_memory
583        .is_some_and(|max| profile.memory.0 >= max.0)
584    {
585        AutoNote::AtMax
586    } else {
587        AutoNote::Raises
588    };
589    let what = match auto {
590        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
591        AutoNote::AtMax => {
592            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
593        }
594        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
595    };
596    let sentence = format!(
597        "{} peaks at {}, {}% of {name}'s {}. {what}",
598        subject(key),
599        ByteSize(peak),
600        peak.saturating_mul(100) / cap,
601        ByteSize(cap)
602    );
603    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
604    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
605        Severity::Notice
606    } else {
607        Severity::Warning
608    };
609    Some(insight(
610        key,
611        "near-cap",
612        severity,
613        sentence,
614        InsightKind::NearCap {
615            profile: name.to_owned(),
616            peak,
617            cap,
618            auto,
619        },
620        measured.iter().take(5).map(|run| run.link()).collect(),
621    ))
622}
623
624/// A job's last runs that could write to the build cache, each with this many lookups at least…
625const COLD_RUNS: usize = 5;
626const COLD_MIN_RUNS: usize = 3;
627const COLD_MIN_LOOKUPS: u64 = 10;
628/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
629/// next should find most of it.
630const COLD_PERCENT: u64 = 10;
631
632fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
633    let writers: Vec<&Run> = runs
634        .iter()
635        .rev()
636        .filter(|run| {
637            run.build_cache
638                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
639        })
640        .take(COLD_RUNS)
641        .copied()
642        .collect();
643    if writers.len() < COLD_MIN_RUNS {
644        return None;
645    }
646    let mut total = api::BuildCacheCounts::default();
647    for run in &writers {
648        total.add(&run.build_cache.unwrap_or_default());
649    }
650    let percent = total.hit_percent()?;
651    if percent > COLD_PERCENT {
652        return None;
653    }
654    let branch = writers.first()?.branch.clone();
655    let sentence = format!(
656        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
657         differ from run to run.",
658        subject(key),
659        branch
660            .as_deref()
661            .map(|branch| format!(" on {branch}"))
662            .unwrap_or_default(),
663        writers.len()
664    );
665    Some(insight(
666        key,
667        "cold-cache",
668        Severity::Notice,
669        sentence,
670        InsightKind::ColdCache {
671            branch,
672            hits: total.hits,
673            lookups: total.lookups(),
674        },
675        writers.iter().map(|run| run.link()).collect(),
676    ))
677}
678
679/// A job whose CPU cap held back `RAISE_RUNS` or more of its last `CAP_RUNS` runs that measured it, as auto sizing
680/// reads a run (`Throttling::held`): successful runs, as sizing reads only those, in the profile and at the cap of its
681/// last such run, the profile's `cpus` today. A run without a throttling figure is unknown, neither held nor free.
682/// Says what the owner can do: under `measured` admission auto sizing raises `cpus` while each raise helps, under
683/// `caps` it does not for throttling, as each CPU takes budget there. A raise that did not help the job is told as
684/// this job's evidence only: auto sizing may still raise the profile for another job, or on the runs' averages.
685fn cpu_held(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
686    let measured = runs.iter().rev().filter(|run| {
687        run.succeeded() && run.profile.is_some() && run.throttling().held().is_some()
688    });
689    let last = measured.clone().next()?;
690    let (name, cap) = (last.profile.as_deref()?, last.cpu_cap?);
691    let profile = config.profiles.get(name)?;
692    // Runs at a cap the profile no longer has say nothing of the one it has: its next runs will.
693    if u64::from(profile.cpus) * 1000 != cap {
694        return None;
695    }
696    let at_cap: Vec<&Run> = measured
697        .filter(|run| run.profile.as_deref() == Some(name) && run.cpu_cap == Some(cap))
698        .take(CAP_RUNS)
699        .copied()
700        .collect();
701    let held: Vec<&Run> = at_cap
702        .iter()
703        .filter(|run| run.throttling().held() == Some(true))
704        .copied()
705        .collect();
706    if (held.len() as u64) < RAISE_RUNS {
707        return None;
708    }
709    let cpus = profile.cpus;
710    let auto = config.host.admission == Admission::Measured && config.sizing.auto;
711    let raised_from = auto
712        .then(|| {
713            let ran: Vec<CpuRun> = runs
714                .iter()
715                .filter(|run| run.succeeded() && run.profile.as_deref() == Some(name))
716                .map(|run| CpuRun {
717                    cap: run.cpu_cap,
718                    queued_at: run.queued_at,
719                    throttling: run.throttling(),
720                })
721                .collect();
722            let now: Vec<Throttling> = at_cap.iter().map(|run| run.throttling()).collect();
723            unhelped_raise(&ran, cap, &now)
724        })
725        .flatten()
726        .map(|unhelped| u32::try_from(unhelped.from / 1000).unwrap_or(u32::MAX));
727    let set = (!auto).then(|| cpus.saturating_add(cpu_step(config, cpus)));
728    let what = match (raised_from, set) {
729        (Some(from), _) => format!(
730            "The raise of {name} from {from} to {cpus} CPUs did not make it faster or less throttled, as when its \
731             tools start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
732             sizing does next."
733        ),
734        (None, Some(set)) if config.host.admission == Admission::Measured => {
735            format!("Auto sizing is off: set {name}'s cpus to {set}, or turn it on.")
736        }
737        (None, Some(set)) => format!(
738            "Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: set {name}'s \
739             cpus to {set} if runnerctl sizing shows the budget has room."
740        ),
741        (None, None) => format!(
742            "Auto sizing raises {name}'s cpus while each raise makes it faster or less throttled and the slice has \
743             room; runnerctl sizing says when it cannot."
744        ),
745    };
746    let sentence = format!(
747        "{} was held back by {name}'s {cpus} CPUs in {} of its last {} runs, its quota throttling it in {HELD}% of \
748         its run or more. {what}",
749        subject(key),
750        held.len(),
751        at_cap.len()
752    );
753    // Auto sizing deals with it: a notice. The owner acts on the rest.
754    let severity = if auto && raised_from.is_none() {
755        Severity::Notice
756    } else {
757        Severity::Warning
758    };
759    Some(insight(
760        key,
761        "cpu-held",
762        severity,
763        sentence,
764        InsightKind::CpuHeld {
765            profile: name.to_owned(),
766            cpus,
767            held: u32::try_from(held.len()).unwrap_or(u32::MAX),
768            total: u32::try_from(at_cap.len()).unwrap_or(u32::MAX),
769            raised_from,
770            set,
771        },
772        held.iter().take(5).map(|run| run.link()).collect(),
773    ))
774}
775
776/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
777fn throttled(window: &[&Run]) -> Vec<Insight> {
778    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
779    for run in window {
780        repositories.entry(&run.repository).or_default().push(run);
781    }
782    repositories
783        .into_iter()
784        .filter_map(|(repository, mut runs)| {
785            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
786            let hit: Vec<&Run> = runs
787                .iter()
788                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
789                .copied()
790                .collect();
791            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
792            if throttled < THROTTLED_MIN_RUNS {
793                return None;
794            }
795            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
796            Some(Insight {
797                key: format!("throttled:{repository}"),
798                severity: Severity::Warning,
799                repository: repository.to_owned(),
800                workflow: None,
801                job: None,
802                sentence: format!(
803                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
804                     host, or a job at its cap, was short of memory."
805                ),
806                kind: InsightKind::Throttled { throttled, total },
807                runs: hit.iter().take(5).map(|run| run.link()).collect(),
808            })
809        })
810        .collect()
811}
812
813/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
814/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
815/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
816fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
817    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
818    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
819    for run in &history.runs {
820        let Some(started) = run.started_at.filter(|at| *at <= now) else {
821            continue;
822        };
823        let week = match now - started {
824            back if back < WEEK => 0,
825            back if back < 2 * WEEK && !before_kept => 1,
826            _ => continue,
827        };
828        weeks.entry(&run.repository).or_default()[week].push(run);
829    }
830    weeks
831        .into_iter()
832        .map(|(repository, [now, before])| RepositoryWaits {
833            repository: repository.to_owned(),
834            now: week_waits(&now, history.kept_since),
835            before: week_waits(&before, history.kept_since),
836        })
837        .collect()
838}
839
840fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
841    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
842    let of = |runs: Vec<&&Run>| {
843        let waited: Vec<u64> = runs
844            .iter()
845            .filter_map(|run| run.waited(kept_since))
846            .collect();
847        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
848        let spread = (!waited.is_empty()).then(|| spread(waited));
849        WaitSpread {
850            jobs,
851            p50: spread.as_ref().map(|spread| spread.p50),
852            p95: spread.as_ref().map(|spread| spread.p95),
853        }
854    };
855    WeekWaits {
856        all: of(runs.iter().collect()),
857        warm: of(runs.iter().filter(warm).collect()),
858        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
859    }
860}
861
862fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
863    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
864        return None;
865    }
866    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
867    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
868    if now < threshold {
869        return None;
870    }
871    Some(Insight {
872        key: format!("waits:{}", waits.repository),
873        severity: Severity::Notice,
874        repository: waits.repository.clone(),
875        workflow: None,
876        job: None,
877        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
878            format!(
879                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
880                waits.repository,
881                span(now),
882                span(before)
883            )
884        } else {
885            format!(
886                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
887                waits.repository,
888                span(now),
889                span(before)
890            )
891        },
892        kind: InsightKind::Waits { before, now },
893        runs: Vec::new(),
894    })
895}
896
897/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
898fn span(seconds: u64) -> String {
899    match seconds {
900        0..60 => format!("{seconds}s"),
901        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
902        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
903        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
904    }
905}
906
907#[cfg(test)]
908mod tests;