Skip to main content

runnerd_core/api/insights/
shown.rs

1//! What insights and the digest mean as every client shows them: each insight's labels and tone, how many warn,
2//! the window they come from, what the page says without any, and the digest's lines. Plain derivations of
3//! [`Insights`] and [`Digest`]; the dashboard, `runnerctl`, the TUI and notifications only lay them out.
4
5use super::{
6    Digest, HostHazard, HostSpan, Insight, InsightKind, Insights, LongWait, RepositoryWaits,
7    Severity, WaitSpread,
8};
9use crate::api::units::short_date;
10use crate::api::{HostMemory, Tone, duration};
11use crate::config::ByteSize;
12use std::fmt::Write as _;
13
14impl InsightKind {
15    /// Its name as a label: `Slower`, `Flaky`, `Near its cap`, `Cold cache`, `Memory pressure`, `CPU cap`,
16    /// `Host memory`, `Host low`, `Waits`, or `Insight` for a kind from a later release.
17    pub fn label(&self) -> &'static str {
18        match self {
19            InsightKind::Slower { .. } => "Slower",
20            InsightKind::Flaky { .. } => "Flaky",
21            InsightKind::NearCap { .. } => "Near its cap",
22            InsightKind::ColdCache { .. } => "Cold cache",
23            InsightKind::Throttled { .. } => "Memory pressure",
24            InsightKind::CpuHeld { .. } => "CPU cap",
25            InsightKind::HostHeld { .. } => "Host memory",
26            InsightKind::HostLow { .. } => "Host low",
27            InsightKind::Waits { .. } => "Waits",
28            InsightKind::Unknown => "Insight",
29        }
30    }
31
32    /// Its name as one word, the API's `kind`: `slower`, `flaky`, `near-cap`, `cold-cache`, `throttled`,
33    /// `cpu-held`, `host-held`, `host-low` or `waits`; `unknown` for a kind from a later release.
34    pub fn word(&self) -> &'static str {
35        match self {
36            InsightKind::Slower { .. } => "slower",
37            InsightKind::Flaky { .. } => "flaky",
38            InsightKind::NearCap { .. } => "near-cap",
39            InsightKind::ColdCache { .. } => "cold-cache",
40            InsightKind::Throttled { .. } => "throttled",
41            InsightKind::CpuHeld { .. } => "cpu-held",
42            InsightKind::HostHeld { .. } => "host-held",
43            InsightKind::HostLow { .. } => "host-low",
44            InsightKind::Waits { .. } => "waits",
45            InsightKind::Unknown => "unknown",
46        }
47    }
48}
49
50impl Insight {
51    /// What a [`HostHeld`](InsightKind::HostHeld) insight says: `3 jobs wait for the host's memory, the longest for
52    /// 42m 10s (halyard/api: CI / build); programs outside CI hold 30G, most java 9.3G and claude 5.1G. Closing programs
53    /// frees only part of that.` `longest` names the job that waited longest; `memory` is the
54    /// reading that names who holds the memory. The part the programs named leave is the kernel, compressed swap and
55    /// the many small programs, so the sentence never says closing them frees the whole figure.
56    pub fn host_held_sentence(
57        jobs: u32,
58        waited: u64,
59        longest: &str,
60        memory: &HostMemory,
61    ) -> String {
62        let who = if jobs == 1 {
63            "1 job waits".to_owned()
64        } else {
65            format!("{jobs} jobs wait")
66        };
67        format!(
68            "{who} for the host's memory, the longest for {} ({longest}); {}. Closing programs frees only part of \
69             that.",
70            duration(waited),
71            memory.outside_line()
72        )
73    }
74}
75
76/// The number an insight is about, short, for a figure beside its sentence: `6m 10s` over `from 4m 40s`.
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct InsightFigure {
79    /// The number: `6m 10s`, `2 of 31`, `8.1G`, `4%`.
80    pub value: String,
81    /// What it is, short and lower case: `from 4m 40s`, `runs passed on a re-run`.
82    pub label: String,
83}
84
85impl Insight {
86    /// The number it is about, with what it is; `None` for a kind from a later release. Its sentence says the rest.
87    pub fn figure(&self) -> Option<InsightFigure> {
88        let figure = |value: String, label: String| Some(InsightFigure { value, label });
89        let of = |part: u32, whole: u32| format!("{part} of {whole}");
90        match &self.kind {
91            InsightKind::Slower { before, now, .. } => {
92                figure(duration(*now), format!("from {}", duration(*before)))
93            }
94            InsightKind::Flaky { flaky, total } => {
95                figure(of(*flaky, *total), "runs passed on a re-run".to_owned())
96            }
97            InsightKind::NearCap {
98                profile, peak, cap, ..
99            } => figure(
100                ByteSize(*peak).to_string(),
101                format!("p95 peak, of {profile}'s {}", ByteSize(*cap)),
102            ),
103            InsightKind::ColdCache { hits, lookups, .. } => {
104                let percent = (u128::from(*hits) * 100 + u128::from(*lookups) / 2)
105                    / u128::from((*lookups).max(1));
106                figure(
107                    format!("{}%", percent.min(100)),
108                    format!("of {lookups} lookups hit"),
109                )
110            }
111            InsightKind::Throttled { throttled, total } => {
112                figure(of(*throttled, *total), "runs throttled".to_owned())
113            }
114            InsightKind::Waits { before, now } => figure(
115                duration(*now),
116                format!("p95 wait, from {}", duration(*before)),
117            ),
118            InsightKind::CpuHeld {
119                profile,
120                cpus,
121                held,
122                total,
123                ..
124            } => figure(
125                of(*held, *total),
126                format!("runs held back by {profile}'s {cpus} CPUs"),
127            ),
128            InsightKind::HostHeld { jobs, waited } => figure(
129                duration(*waited),
130                if *jobs == 1 {
131                    "1 job waits for the host's memory".to_owned()
132                } else {
133                    format!("longest of {jobs} waits for the host's memory")
134                },
135            ),
136            InsightKind::HostLow { lowest, .. } => figure(
137                ByteSize(*lowest).to_string(),
138                "least memory available".to_owned(),
139            ),
140            InsightKind::Unknown => None,
141        }
142    }
143}
144
145impl Severity {
146    /// `warning` or `notice`.
147    pub fn label(self) -> &'static str {
148        match self {
149            Severity::Warning => "warning",
150            Severity::Notice => "notice",
151        }
152    }
153}
154
155impl Insight {
156    /// What it means for the reader, by its severity alone, whatever its kind: a warning is worth a look, a notice
157    /// a plain fact.
158    pub fn tone(&self) -> Tone {
159        match self.severity {
160            Severity::Warning => Tone::Warn,
161            Severity::Notice => Tone::Neutral,
162        }
163    }
164}
165
166impl Insights {
167    /// How many insights warn.
168    pub fn warning_count(&self) -> usize {
169        self.insights
170            .iter()
171            .filter(|insight| insight.severity == Severity::Warning)
172            .count()
173    }
174
175    /// What the insights page says over its list: how many insights it lists, how many of them warn, and the window
176    /// they come from.
177    pub fn heading(&self) -> InsightsHeading {
178        let count = self.insights.len();
179        let figure = match count {
180            0 => "no insights".to_owned(),
181            1 => "1 insight".to_owned(),
182            count => format!("{count} insights"),
183        };
184        let note = match self.warning_count() {
185            0 => format!(" {}", self.window_line()),
186            1 => format!(", 1 warning, {}", self.window_line()),
187            warnings => format!(", {warnings} warnings, {}", self.window_line()),
188        };
189        InsightsHeading { figure, note }
190    }
191
192    /// The whole days its window covers.
193    pub fn days(&self) -> u64 {
194        self.until.saturating_sub(self.since) / 86_400
195    }
196
197    /// Where its insights come from: `from the last 30 days`.
198    pub fn window_line(&self) -> String {
199        format!("from the last {} days", self.days())
200    }
201
202    /// What it says without any insight: `nothing stands out from the last 30 days`.
203    pub fn empty_line(&self) -> String {
204        format!("nothing stands out {}", self.window_line())
205    }
206}
207
208/// What the insights page says over its list, as [`Insights::heading`] words it: `5 insights, 4 warnings, from the
209/// last 14 days`, `1 insight from the last 14 days`, `no insights from the last 14 days`. A client may set the figure
210/// apart; together, as its `Display` writes them, they read as one line.
211#[derive(Debug, Clone, PartialEq, Eq)]
212pub struct InsightsHeading {
213    /// How many it lists: `5 insights`, `1 insight`, `no insights`.
214    pub figure: String,
215    /// The rest, starting with what joins it to the figure: `, 4 warnings, from the last 14 days`, or ` from the last
216    /// 14 days` when none warns.
217    pub note: String,
218}
219
220impl std::fmt::Display for InsightsHeading {
221    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
222        write!(formatter, "{}{}", self.figure, self.note)
223    }
224}
225
226impl WaitSpread {
227    /// `value`, one of its percentiles, when it has jobs to take it over; `None` to show none.
228    pub fn reading(&self, value: Option<u64>) -> Option<u64> {
229        value.filter(|_| self.jobs > 0)
230    }
231}
232
233impl Insight {
234    /// What a [`HostLow`](InsightKind::HostLow) insight says: `On Nov 6 the host had under 6.2G available for 34s while
235    /// a host credit stood, down to 4.1G. The credit lets runners start on memory the running ones may still grow
236    /// into; [host] pooled_count = false makes the budgets count every reservation again.`
237    pub fn host_low_sentence(day: u64, seconds: u64, lowest: u64, line: u64) -> String {
238        format!(
239            "On {} the host had under {} available for {} while a host credit stood, down to {}. The credit lets \
240             runners start on memory the running ones may still grow into; [host] pooled_count = false makes the \
241             budgets count every reservation again.",
242            short_date(day),
243            ByteSize(line),
244            duration(seconds),
245            ByteSize(lowest)
246        )
247    }
248}
249
250impl HostHazard {
251    /// What the host's memory while CI ran is called where it shows, as a heading or before its line.
252    pub const TITLE: &'static str = "Host memory while CI ran";
253
254    /// The window in a line: `CI ran 9h 12m, 6h 10m of it on a host credit; under 19.4G available for 1h 04m (41m on
255    /// a credit); under 6.2G available for 34s (34s on a credit); lowest 4.1G (4.5G on a credit); swap free at least
256    /// 8.2G`.
257    /// Without a host credit in it, nothing is said of one.
258    pub fn line(&self) -> String {
259        let credited = self.credited > 0;
260        let share = |seconds: u64| {
261            if credited {
262                format!(" ({} on a credit)", duration(seconds))
263            } else {
264                String::new()
265            }
266        };
267        let under = |line: u64, span: HostSpan| match span.seconds {
268            0 => format!("never under {} available", ByteSize(line)),
269            seconds => format!(
270                "under {} available for {}{}",
271                ByteSize(line),
272                duration(seconds),
273                share(span.credited)
274            ),
275        };
276        let mut line = format!("CI ran {}", duration(self.running));
277        if credited {
278            let _ = write!(line, ", {} of it on a host credit", duration(self.credited));
279        }
280        let _ = write!(
281            line,
282            "; {}; {}; lowest {}",
283            under(self.low_line, self.low),
284            under(self.critical_line, self.critical),
285            ByteSize(self.lowest)
286        );
287        if let Some(lowest) = self.lowest_credited {
288            let _ = write!(line, " ({} on a credit)", ByteSize(lowest));
289        }
290        if let Some(swap) = self.lowest_swap_free {
291            let _ = write!(line, "; swap free at least {}", ByteSize(swap));
292        }
293        line
294    }
295
296    /// The window's UTC day and its line, as `runnerctl insights` and the dashboard list the days: `Nov 6: CI ran …`.
297    pub fn dated_line(&self) -> String {
298        format!("{}: {}", short_date(self.since), self.line())
299    }
300}
301
302impl Digest {
303    /// Its day in a line: `12 jobs finished, 2 failed`, then `, 1 new insight` when some are new.
304    pub fn summary_line(&self) -> String {
305        let jobs = if self.finished == 1 { "job" } else { "jobs" };
306        let line = format!("{} {jobs} finished, {} failed", self.finished, self.failed);
307        match self.new.len() {
308            0 => line,
309            1 => format!("{line}, 1 new insight"),
310            new => format!("{line}, {new} new insights"),
311        }
312    }
313
314    /// The host's memory over its day in a line, `Host memory while CI ran: CI ran …`, or nothing when CI did not
315    /// run.
316    pub fn host_line(&self) -> Option<String> {
317        self.host
318            .as_ref()
319            .map(|host| format!("{}: {}.", HostHazard::TITLE, host.line()))
320    }
321}
322
323impl LongWait {
324    /// It in a line: `halyard/api CI / test waited 1m 30s`, or `halyard/api job 7 waited 1m 30s` until the daemon
325    /// knows the job's name.
326    pub fn line(&self) -> String {
327        format!(
328            "{} {} waited {}",
329            self.repository,
330            crate::api::job_title(self.workflow.as_deref(), &self.name, Some(self.job)),
331            duration(self.waited)
332        )
333    }
334}
335
336/// One of a repository's wait figures: its label, such as `Warm median`, and its value, `–` without jobs to take it
337/// over.
338#[derive(Debug, Clone, PartialEq, Eq)]
339pub struct WaitFigure {
340    /// `Jobs`, `Median`, `p95`, `Warm median`, `Cold median` or `p95 the week before`.
341    pub label: &'static str,
342    /// The count of jobs, or a wait such as `4m 10s`.
343    pub value: String,
344}
345
346impl RepositoryWaits {
347    /// What a repository's waits are called where they show.
348    pub const TITLE: &'static str = "Waits this week";
349
350    /// Its figures, in the order they show: the week's jobs, their median and p95 wait, the warm and the cold
351    /// median, and the week before's p95.
352    pub fn figures(&self) -> Vec<WaitFigure> {
353        let wait = |spread: &WaitSpread, value: Option<u64>| {
354            spread
355                .reading(value)
356                .map_or_else(|| "–".to_owned(), duration)
357        };
358        let now = &self.now;
359        let before = &self.before.all;
360        [
361            (crate::api::JOBS_HEADING, now.all.jobs.to_string()),
362            ("Median", wait(&now.all, now.all.p50)),
363            ("p95", wait(&now.all, now.all.p95)),
364            ("Warm median", wait(&now.warm, now.warm.p50)),
365            ("Cold median", wait(&now.cold, now.cold.p50)),
366            ("p95 the week before", wait(before, before.p95)),
367        ]
368        .into_iter()
369        .map(|(label, value)| WaitFigure { label, value })
370        .collect()
371    }
372}
373
374#[cfg(test)]
375mod tests;