Skip to main content

runnerd_core/api/insights/
shown.rs

1//! What insights and the digest mean as every client shows them: each insight's labels and tone, how many warn,
2//! the window they come from, what the page says without any, and the digest's lines. Plain derivations of
3//! [`Insights`] and [`Digest`]; the dashboard, `runnerctl`, the TUI and notifications only lay them out.
4
5use super::{
6    Digest, HostHazard, HostSpan, Insight, InsightKind, Insights, LongWait, ProfileQueue,
7    RepositoryWaits, Severity, WaitSpread,
8};
9use crate::api::LabelledFigure;
10use crate::api::units::short_date;
11use crate::api::{HostMemory, Tone, duration};
12use crate::config::ByteSize;
13use std::fmt::Write as _;
14
15impl InsightKind {
16    /// Its name as a label: `Slower`, `Flaky`, `Near its cap`, `Cold cache`, `Memory pressure`, `CPU cap`,
17    /// `Host memory`, `Host low`, `Waits`, or `Insight` for a kind from a later release.
18    pub fn label(&self) -> &'static str {
19        match self {
20            InsightKind::Slower { .. } => "Slower",
21            InsightKind::Flaky { .. } => "Flaky",
22            InsightKind::NearCap { .. } => "Near its cap",
23            InsightKind::ColdCache { .. } => "Cold cache",
24            InsightKind::Throttled { .. } => "Memory pressure",
25            InsightKind::CpuHeld { .. } => "CPU cap",
26            InsightKind::HostHeld { .. } => "Host memory",
27            InsightKind::HostLow { .. } => "Host low",
28            InsightKind::Waits { .. } => "Waits",
29            InsightKind::Unknown => "Insight",
30        }
31    }
32
33    /// Its name as one word, the API's `kind`: `slower`, `flaky`, `near-cap`, `cold-cache`, `throttled`,
34    /// `cpu-held`, `host-held`, `host-low` or `waits`; `unknown` for a kind from a later release.
35    pub fn word(&self) -> &'static str {
36        match self {
37            InsightKind::Slower { .. } => "slower",
38            InsightKind::Flaky { .. } => "flaky",
39            InsightKind::NearCap { .. } => "near-cap",
40            InsightKind::ColdCache { .. } => "cold-cache",
41            InsightKind::Throttled { .. } => "throttled",
42            InsightKind::CpuHeld { .. } => "cpu-held",
43            InsightKind::HostHeld { .. } => "host-held",
44            InsightKind::HostLow { .. } => "host-low",
45            InsightKind::Waits { .. } => "waits",
46            InsightKind::Unknown => "unknown",
47        }
48    }
49}
50
51impl Insight {
52    /// What a [`HostHeld`](InsightKind::HostHeld) insight says: `3 jobs wait for the host's memory, the longest for
53    /// 42m 10s (halyard/api: CI / build); programs outside CI hold 30G, most java 9.3G and claude 5.1G. Closing programs
54    /// frees only part of that.` `longest` names the job that waited longest; `memory` is the
55    /// reading that names who holds the memory. The part the programs named leave is the kernel, compressed swap and
56    /// the many small programs, so the sentence never says closing them frees the whole figure.
57    pub fn host_held_sentence(
58        jobs: u32,
59        waited: u64,
60        longest: &str,
61        memory: &HostMemory,
62    ) -> String {
63        let who = if jobs == 1 {
64            "1 job waits".to_owned()
65        } else {
66            format!("{jobs} jobs wait")
67        };
68        format!(
69            "{who} for the host's memory, the longest for {} ({longest}); {}. Closing programs frees only part of \
70             that.",
71            duration(waited),
72            memory.outside_line()
73        )
74    }
75}
76
77/// The number an insight is about, short, for a figure beside its sentence: `6m 10s` over `from 4m 40s`.
78#[derive(Debug, Clone, PartialEq, Eq)]
79pub struct InsightFigure {
80    /// The number: `6m 10s`, `2 of 31`, `8.1G`, `4%`.
81    pub value: String,
82    /// What it is, short and lower case: `from 4m 40s`, `runs passed on a re-run`.
83    pub label: String,
84}
85
86impl Insight {
87    /// The number it is about, with what it is; `None` for a kind from a later release. Its sentence says the rest.
88    pub fn figure(&self) -> Option<InsightFigure> {
89        let figure = |value: String, label: String| Some(InsightFigure { value, label });
90        let of = |part: u32, whole: u32| format!("{part} of {whole}");
91        match &self.kind {
92            InsightKind::Slower { before, now, .. } => {
93                figure(duration(*now), format!("from {}", duration(*before)))
94            }
95            InsightKind::Flaky { flaky, total } => {
96                figure(of(*flaky, *total), "runs passed on a re-run".to_owned())
97            }
98            InsightKind::NearCap {
99                profile, peak, cap, ..
100            } => figure(
101                ByteSize(*peak).to_string(),
102                format!("p95 peak, of {profile}'s {}", ByteSize(*cap)),
103            ),
104            InsightKind::ColdCache { hits, lookups, .. } => {
105                let percent = (u128::from(*hits) * 100 + u128::from(*lookups) / 2)
106                    / u128::from((*lookups).max(1));
107                figure(
108                    format!("{}%", percent.min(100)),
109                    format!("of {lookups} lookups hit"),
110                )
111            }
112            InsightKind::Throttled { throttled, total } => {
113                figure(of(*throttled, *total), "runs throttled".to_owned())
114            }
115            InsightKind::Waits { before, now } => figure(
116                duration(*now),
117                format!("p95 wait, from {}", duration(*before)),
118            ),
119            InsightKind::CpuHeld {
120                profile,
121                cpus,
122                held,
123                total,
124                ..
125            } => figure(
126                of(*held, *total),
127                format!("runs held back by {profile}'s {cpus} CPUs"),
128            ),
129            InsightKind::HostHeld { jobs, waited } => figure(
130                duration(*waited),
131                if *jobs == 1 {
132                    "1 job waits for the host's memory".to_owned()
133                } else {
134                    format!("longest of {jobs} waits for the host's memory")
135                },
136            ),
137            InsightKind::HostLow { lowest, .. } => figure(
138                ByteSize(*lowest).to_string(),
139                "least memory available".to_owned(),
140            ),
141            InsightKind::Unknown => None,
142        }
143    }
144}
145
146impl Severity {
147    /// `warning` or `notice`.
148    pub fn label(self) -> &'static str {
149        match self {
150            Severity::Warning => "warning",
151            Severity::Notice => "notice",
152        }
153    }
154}
155
156impl Insight {
157    /// What it means for the reader, by its severity alone, whatever its kind: a warning is worth a look, a notice
158    /// a plain fact.
159    pub fn tone(&self) -> Tone {
160        match self.severity {
161            Severity::Warning => Tone::Warn,
162            Severity::Notice => Tone::Neutral,
163        }
164    }
165}
166
167impl Insights {
168    /// How many insights warn.
169    pub fn warning_count(&self) -> usize {
170        self.insights
171            .iter()
172            .filter(|insight| insight.severity == Severity::Warning)
173            .count()
174    }
175
176    /// What the insights page says over its list: how many insights it lists, how many of them warn, and the window
177    /// they come from.
178    pub fn heading(&self) -> InsightsHeading {
179        let count = self.insights.len();
180        let figure = match count {
181            0 => "no insights".to_owned(),
182            1 => "1 insight".to_owned(),
183            count => format!("{count} insights"),
184        };
185        let note = match self.warning_count() {
186            0 => format!(" {}", self.window_line()),
187            1 => format!(", 1 warning, {}", self.window_line()),
188            warnings => format!(", {warnings} warnings, {}", self.window_line()),
189        };
190        InsightsHeading { figure, note }
191    }
192
193    /// The whole days its window covers.
194    pub fn days(&self) -> u64 {
195        self.until.saturating_sub(self.since) / 86_400
196    }
197
198    /// Where its insights come from: `from the last 30 days`.
199    pub fn window_line(&self) -> String {
200        format!("from the last {} days", self.days())
201    }
202
203    /// What it says without any insight: `nothing stands out from the last 30 days`.
204    pub fn empty_line(&self) -> String {
205        format!("nothing stands out {}", self.window_line())
206    }
207}
208
209/// What the insights page says over its list, as [`Insights::heading`] words it: `5 insights, 4 warnings, from the
210/// last 14 days`, `1 insight from the last 14 days`, `no insights from the last 14 days`. A client may set the figure
211/// apart; together, as its `Display` writes them, they read as one line.
212#[derive(Debug, Clone, PartialEq, Eq)]
213pub struct InsightsHeading {
214    /// How many it lists: `5 insights`, `1 insight`, `no insights`.
215    pub figure: String,
216    /// The rest, starting with what joins it to the figure: `, 4 warnings, from the last 14 days`, or ` from the last
217    /// 14 days` when none warns.
218    pub note: String,
219}
220
221impl std::fmt::Display for InsightsHeading {
222    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223        write!(formatter, "{}{}", self.figure, self.note)
224    }
225}
226
227impl WaitSpread {
228    /// `value`, one of its percentiles, when it has jobs to take it over; `None` to show none.
229    pub fn reading(&self, value: Option<u64>) -> Option<u64> {
230        value.filter(|_| self.jobs > 0)
231    }
232}
233
234impl Insight {
235    /// What a [`HostLow`](InsightKind::HostLow) insight says: `On Nov 6 the host had under 6.2G available for 34s while
236    /// a host credit stood, down to 4.1G. The credit lets runners start on memory the running ones may still grow
237    /// into; [host] pooled_count = false makes the budgets count every reservation again.`
238    pub fn host_low_sentence(day: u64, seconds: u64, lowest: u64, line: u64) -> String {
239        format!(
240            "On {} the host had under {} available for {} while a host credit stood, down to {}. The credit lets \
241             runners start on memory the running ones may still grow into; [host] pooled_count = false makes the \
242             budgets count every reservation again.",
243            short_date(day),
244            ByteSize(line),
245            duration(seconds),
246            ByteSize(lowest)
247        )
248    }
249}
250
251impl HostHazard {
252    /// What the host's memory while CI ran is called where it shows, as a heading or before its line.
253    pub const TITLE: &'static str = "Host memory while CI ran";
254
255    /// The window in a line: `CI ran 9h 12m, 6h 10m of it on a host credit; under 19.4G available for 1h 04m (41m on
256    /// a credit); under 6.2G available for 34s (34s on a credit); lowest 4.1G (4.5G on a credit); swap free at least
257    /// 8.2G`.
258    /// Without a host credit in it, nothing is said of one.
259    pub fn line(&self) -> String {
260        let credited = self.credited > 0;
261        let share = |seconds: u64| {
262            if credited {
263                format!(" ({} on a credit)", duration(seconds))
264            } else {
265                String::new()
266            }
267        };
268        let under = |line: u64, span: HostSpan| match span.seconds {
269            0 => format!("never under {} available", ByteSize(line)),
270            seconds => format!(
271                "under {} available for {}{}",
272                ByteSize(line),
273                duration(seconds),
274                share(span.credited)
275            ),
276        };
277        let mut line = format!("CI ran {}", duration(self.running));
278        if credited {
279            let _ = write!(line, ", {} of it on a host credit", duration(self.credited));
280        }
281        let _ = write!(
282            line,
283            "; {}; {}; lowest {}",
284            under(self.low_line, self.low),
285            under(self.critical_line, self.critical),
286            ByteSize(self.lowest)
287        );
288        if let Some(lowest) = self.lowest_credited {
289            let _ = write!(line, " ({} on a credit)", ByteSize(lowest));
290        }
291        if let Some(swap) = self.lowest_swap_free {
292            let _ = write!(line, "; swap free at least {}", ByteSize(swap));
293        }
294        line
295    }
296
297    /// The window's UTC day and its line, as `runnerctl insights` and the dashboard list the days: `Nov 6: CI ran …`.
298    pub fn dated_line(&self) -> String {
299        format!("{}: {}", short_date(self.since), self.line())
300    }
301}
302
303impl Digest {
304    /// Its day in a line: `12 jobs finished, 2 failed`, then `, 1 new insight` when some are new.
305    pub fn summary_line(&self) -> String {
306        let jobs = if self.finished == 1 { "job" } else { "jobs" };
307        let line = format!("{} {jobs} finished, {} failed", self.finished, self.failed);
308        match self.new.len() {
309            0 => line,
310            1 => format!("{line}, 1 new insight"),
311            new => format!("{line}, {new} new insights"),
312        }
313    }
314
315    /// The host's memory over its day in a line, `Host memory while CI ran: CI ran …`, or nothing when CI did not
316    /// run.
317    pub fn host_line(&self) -> Option<String> {
318        self.host
319            .as_ref()
320            .map(|host| format!("{}: {}.", HostHazard::TITLE, host.line()))
321    }
322}
323
324impl LongWait {
325    /// It in a line: `halyard/api CI / test waited 1m 30s`, or `halyard/api job 7 waited 1m 30s` until the daemon
326    /// knows the job's name.
327    pub fn line(&self) -> String {
328        format!(
329            "{} {} waited {}",
330            self.repository,
331            crate::api::job_title(self.workflow.as_deref(), &self.name, Some(self.job)),
332            duration(self.waited)
333        )
334    }
335}
336
337/// One of a repository's wait figures: its label, such as `Warm median`, and its value, `–` without jobs to take it
338/// over.
339#[derive(Debug, Clone, PartialEq, Eq)]
340pub struct WaitFigure {
341    /// `Jobs`, `Median`, `p95`, `Warm median`, `Cold median` or `p95 the week before`.
342    pub label: &'static str,
343    /// The count of jobs, or a wait such as `4m 10s`.
344    pub value: String,
345}
346
347impl RepositoryWaits {
348    /// What a repository's waits are called where they show.
349    pub const TITLE: &'static str = "Waits this week";
350
351    /// Its figures, in the order they show: the week's jobs, their median and p95 wait, the warm and the cold
352    /// median, and the week before's p95.
353    pub fn figures(&self) -> Vec<WaitFigure> {
354        let wait = |spread: &WaitSpread, value: Option<u64>| {
355            spread
356                .reading(value)
357                .map_or_else(|| "–".to_owned(), duration)
358        };
359        let now = &self.now;
360        let before = &self.before.all;
361        [
362            (crate::api::JOBS_HEADING, now.all.jobs.to_string()),
363            ("Median", wait(&now.all, now.all.p50)),
364            ("p95", wait(&now.all, now.all.p95)),
365            ("Warm median", wait(&now.warm, now.warm.p50)),
366            ("Cold median", wait(&now.cold, now.cold.p50)),
367            ("p95 the week before", wait(before, before.p95)),
368        ]
369        .into_iter()
370        .map(|(label, value)| WaitFigure { label, value })
371        .collect()
372    }
373}
374
375impl ProfileQueue {
376    /// What the split of the profiles' queue time is called where it shows.
377    pub const TITLE: &'static str = "Why jobs queued";
378    /// The label of the wait before runnerd first listed a job.
379    pub const BEFORE_LISTED: &'static str = "Before runnerd listed it";
380    /// The label of the wait no plan held a job for.
381    pub const UNCOVERED: &'static str = "No plan covered";
382
383    /// It in a line: `jvm: 42 jobs waited 12h 30m in all`, then `; 3 jobs from before runnerd counted why are left
384    /// out` when some are, and `; 2 jobs' causes are estimates, scaled into their waits` when some are;
385    /// `jvm: 3 jobs ended from before runnerd counted why jobs wait` when none is counted.
386    pub fn line(&self) -> String {
387        let jobs = |count: u32| {
388            if count == 1 {
389                "1 job".to_owned()
390            } else {
391                format!("{count} jobs")
392            }
393        };
394        if self.jobs == 0 {
395            return format!(
396                "{}: {} ended from before runnerd counted why jobs wait",
397                self.profile,
398                jobs(self.unrecorded)
399            );
400        }
401        let mut line = format!(
402            "{}: {} waited {} in all",
403            self.profile,
404            jobs(self.jobs),
405            duration(seconds(self.waited_ms))
406        );
407        if self.unrecorded > 0 {
408            let _ = write!(
409                line,
410                "; {} from before runnerd counted why {} left out",
411                jobs(self.unrecorded),
412                if self.unrecorded == 1 { "is" } else { "are" }
413            );
414        }
415        match self.estimated {
416            0 => {}
417            1 => line.push_str("; 1 job's causes are estimates, scaled into its wait"),
418            estimated => {
419                let _ = write!(
420                    line,
421                    "; {estimated} jobs' causes are estimates, scaled into their waits"
422                );
423            }
424        }
425        line
426    }
427
428    /// What held the jobs, longest first, each with its share of their waits, taken from the milliseconds:
429    /// `Host memory` `6h 10m` `49%`; then the wait before runnerd listed them and the wait no plan covered, when there
430    /// was some. Empty when no job was counted.
431    pub fn figures(&self) -> Vec<LabelledFigure> {
432        let share = |ms: u64| {
433            let percent = (u128::from(ms) * 100 + u128::from(self.waited_ms) / 2)
434                / u128::from(self.waited_ms.max(1));
435            format!("{}%", percent.min(100))
436        };
437        let figure =
438            |label: &str, ms: u64| LabelledFigure::new(label, duration(seconds(ms))).of(share(ms));
439        let mut figures: Vec<LabelledFigure> = self
440            .causes
441            .iter()
442            .map(|cause| figure(cause.cause.label(), cause.waited_ms))
443            .collect();
444        for (label, ms) in [
445            (Self::BEFORE_LISTED, self.before_listed_ms),
446            (Self::UNCOVERED, self.uncovered_ms),
447        ] {
448            if ms > 0 {
449                figures.push(figure(label, ms));
450            }
451        }
452        figures
453    }
454}
455
456/// `ms` in whole seconds, rounded, as durations show.
457fn seconds(ms: u64) -> u64 {
458    ms.saturating_add(500) / 1000
459}
460
461#[cfg(test)]
462mod tests;