Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, and how long each repository's jobs wait, this week against the last when the
4//! history holds all of last week; and the daily digest. Pure: the daemon reads the history, the
5//! config and the clock and passes them in. It sends nothing and changes nothing.
6
7use std::collections::{BTreeMap, BTreeSet};
8
9use crate::api::{
10    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
11    RunLink, Severity, WaitSpread, WeekWaits,
12};
13use crate::config::{ByteSize, Config};
14use crate::sizing::spread;
15
16const SECOND: u64 = 1000;
17const DAY: u64 = 24 * 3600 * SECOND;
18const WEEK: u64 = 7 * DAY;
19
20/// The last runs a slower job's median is taken over; the baseline needs as many before them.
21const RECENT: usize = 5;
22/// A job is slower when its recent median is this share above its baseline's, in percent…
23const SLOWER_PERCENT: u64 = 20;
24/// …and at least this many seconds above it.
25const SLOWER_SECONDS: u64 = 30;
26
27/// The last runs whose memory peaks are read, and the fewest that say anything.
28const CAP_RUNS: usize = 10;
29const CAP_MIN_RUNS: usize = 5;
30/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
31const NEAR_CAP: u64 = 85;
32/// Runs at `NEAR_CAP` or more that make it a warning.
33const NEAR_CAP_WARNING: usize = 2;
34/// Flaky runs that make it a warning.
35const FLAKY_WARNING: u32 = 2;
36
37/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
38/// above it, each week with this many jobs.
39const WAITS_PERCENT: u64 = 50;
40const WAITS_SECONDS: u64 = 60;
41const WAITS_MIN_JOBS: u32 = 5;
42/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
43const WAITS_ONE_IN_20: u32 = 20;
44
45/// A repository's runs the kernel throttled for memory pressure that make an insight.
46const THROTTLED_MIN_RUNS: u32 = 2;
47
48/// One job the history closed, as insights read it. Times in milliseconds.
49#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct Run {
51    /// GitHub's job ID.
52    pub id: u64,
53    /// `owner/name`.
54    pub repository: String,
55    /// The workflow's name, when known.
56    pub workflow: Option<String>,
57    /// The job's name.
58    pub name: String,
59    /// The branch the run is for.
60    pub branch: Option<String>,
61    /// The job's page on GitHub.
62    pub url: Option<String>,
63    /// The workflow run.
64    pub run_id: u64,
65    /// The run's attempt, 1 for the first.
66    pub run_attempt: u32,
67    /// `Completed` or `Lost`.
68    pub state: JobState,
69    /// GitHub's conclusion, for a completed job.
70    pub conclusion: Option<String>,
71    /// When GitHub queued it.
72    pub queued_at: u64,
73    /// When it started.
74    pub started_at: Option<u64>,
75    /// When it finished, or was found lost.
76    pub finished_at: Option<u64>,
77    /// When its runner's container started: before `queued_at`, it found a warm runner.
78    pub runner_started_at: Option<u64>,
79    /// The profile its runner ran in.
80    pub profile: Option<String>,
81    /// Its sampled memory peak.
82    pub memory_peak: Option<u64>,
83    /// What the build cache served it, if its runner had the build cache.
84    pub build_cache: Option<api::BuildCacheCounts>,
85    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
86    pub sock_throttled: Option<u64>,
87    /// How much of its wait passed while no runnerd ran.
88    pub unseen: u64,
89}
90
91impl Run {
92    fn finished(&self) -> u64 {
93        self.finished_at.unwrap_or(0)
94    }
95
96    fn succeeded(&self) -> bool {
97        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
98    }
99
100    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
101    fn failed(&self) -> bool {
102        self.state == JobState::Lost || self.failed_itself()
103    }
104
105    /// Failed or timed out: what the job did, not what became of its runner.
106    fn failed_itself(&self) -> bool {
107        self.state == JobState::Completed
108            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
109    }
110
111    /// Seconds from start to finish.
112    fn took(&self) -> Option<u64> {
113        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
114    }
115
116    /// Its [wait](crate::sizing::Wait) in seconds, once it started.
117    fn waited(&self, kept_since: u64) -> Option<u64> {
118        let wait = crate::sizing::Wait {
119            queued_at: self.queued_at,
120            started_at: self.started_at?,
121            unseen: self.unseen,
122        };
123        Some(wait.counted(kept_since) / SECOND)
124    }
125
126    fn link(&self) -> RunLink {
127        RunLink {
128            job: self.id,
129            run_id: self.run_id,
130            run_attempt: self.run_attempt,
131            branch: self.branch.clone(),
132            url: self.url.clone(),
133            at: self.finished() / SECOND,
134        }
135    }
136}
137
138/// What insights read from the job history.
139#[derive(Debug, Clone, Default, PartialEq, Eq)]
140pub struct History {
141    /// The jobs it closed in the days read.
142    pub runs: Vec<Run>,
143    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
144    /// start left only some, so no week before it is compared.
145    pub kept_since: u64,
146}
147
148/// A job as insights tell jobs apart: repository, workflow and name.
149type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
150
151fn key_of(run: &Run) -> JobKey<'_> {
152    (&run.repository, run.workflow.as_deref(), &run.name)
153}
154
155/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
156/// only the runs finished by `now` count.
157pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
158    let runs = &history.runs;
159    let since = now.saturating_sub(u64::from(days) * DAY);
160    let window: Vec<&Run> = runs
161        .iter()
162        .filter(|run| {
163            matches!(run.state, JobState::Completed | JobState::Lost)
164                && run
165                    .finished_at
166                    .is_some_and(|at| (since..=now).contains(&at))
167        })
168        .collect();
169    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
170    for run in &window {
171        jobs.entry(key_of(run)).or_default().push(run);
172    }
173    let mut found = Vec::new();
174    for (key, runs) in &mut jobs {
175        runs.sort_by_key(|run| (run.finished(), run.id));
176        found.extend(slower(*key, runs));
177        found.extend(flaky(*key, runs));
178        found.extend(near_cap(*key, runs, config));
179        found.extend(cold_cache(*key, runs));
180    }
181    found.extend(throttled(&window));
182    let waits = waits(history, now);
183    found.extend(waits.iter().filter_map(waits_insight));
184    found.sort_by(|a, b| {
185        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
186            b.severity,
187            &b.repository,
188            &b.workflow,
189            &b.job,
190            &b.key,
191        ))
192    });
193    Insights {
194        since: since / SECOND,
195        until: now / SECOND,
196        insights: found,
197        waits,
198    }
199}
200
201/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
202/// before, and the longest waits.
203pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
204    let runs = &history.runs;
205    let since = now.saturating_sub(DAY);
206    let finished: Vec<&Run> = runs
207        .iter()
208        .filter(|run| {
209            matches!(run.state, JobState::Completed | JobState::Lost)
210                && run.finished_at.is_some_and(|at| at > since && at <= now)
211        })
212        .collect();
213    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
214        .insights
215        .into_iter()
216        .map(|insight| insight.key)
217        .collect();
218    let new = insights(history, config, api::INSIGHT_DAYS, now)
219        .insights
220        .into_iter()
221        .filter(|insight| !before.contains(&insight.key))
222        .collect();
223    let mut started: Vec<&Run> = runs
224        .iter()
225        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
226        .collect();
227    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
228    Digest {
229        since: since / SECOND,
230        until: now / SECOND,
231        finished: finished.len() as u64,
232        failed: finished.iter().filter(|run| run.failed()).count() as u64,
233        new,
234        waits: started
235            .into_iter()
236            .take(3)
237            .map(|run| LongWait {
238                job: run.id,
239                repository: run.repository.clone(),
240                workflow: run.workflow.clone(),
241                name: run.name.clone(),
242                waited: run.waited(history.kept_since).unwrap_or(0),
243                url: run.url.clone(),
244            })
245            .collect(),
246        savings: None,
247    }
248}
249
250/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
251fn subject((repository, workflow, name): JobKey<'_>) -> String {
252    match workflow {
253        Some(workflow) => format!("{repository}: {workflow} / {name}"),
254        None => format!("{repository}: {name}"),
255    }
256}
257
258fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
259    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
260}
261
262fn insight(
263    key: JobKey<'_>,
264    kind: &str,
265    severity: Severity,
266    sentence: String,
267    found: InsightKind,
268    runs: Vec<RunLink>,
269) -> Insight {
270    Insight {
271        key: key_text(kind, key),
272        severity,
273        repository: key.0.to_owned(),
274        workflow: key.1.map(str::to_owned),
275        job: Some(key.2.to_owned()),
276        sentence,
277        kind: found,
278        runs,
279    }
280}
281
282/// The median: the mean of the middle two for an even count.
283fn median(values: &[u64]) -> u64 {
284    let mut values = values.to_vec();
285    values.sort_unstable();
286    let middle = values.len() / 2;
287    match values.len() {
288        0 => 0,
289        len if len % 2 == 1 => values[middle],
290        _ => u64::midpoint(values[middle - 1], values[middle]),
291    }
292}
293
294/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
295fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
296    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
297        runs.iter()
298            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
299            .filter_map(|run| Some((*run, run.took()?)))
300            .collect()
301    };
302    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
303    for run in runs.iter().filter(|run| run.succeeded()) {
304        if let Some(branch) = &run.branch {
305            *branches.entry(branch).or_default() += 1;
306        }
307    }
308    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
309    // most runs, and of equal counts the first by name.
310    let busiest = branches
311        .iter()
312        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
313        .map(|(branch, _)| *branch)?;
314    let (branch, timed) = ["main", "master"]
315        .into_iter()
316        .map(|branch| (branch, timed_on(branch)))
317        .find(|(_, timed)| timed.len() >= 2 * RECENT)
318        .unwrap_or_else(|| (busiest, timed_on(busiest)));
319    if timed.len() < 2 * RECENT {
320        return None;
321    }
322    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
323    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
324    let before = median(&took(baseline));
325    let now = median(&took(recent));
326    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
327    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
328    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
329    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
330        return None;
331    }
332    // The first slow run after which no two runs in a row were fast.
333    let mut since = None;
334    let mut fast_in_a_row = 0;
335    for (run, took) in &timed {
336        if *took >= threshold {
337            fast_in_a_row = 0;
338            since = since.or(Some(*run));
339        } else {
340            fast_in_a_row += 1;
341            if fast_in_a_row >= 2 {
342                since = None;
343            }
344        }
345    }
346    let since = since?;
347    let sentence = format!(
348        "{} on {branch} takes {} now, up from {}.",
349        subject(key),
350        span(now),
351        span(before)
352    );
353    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
354    Some(insight(
355        key,
356        "slower",
357        Severity::Warning,
358        sentence,
359        InsightKind::Slower {
360            branch: Some(branch.to_owned()),
361            before,
362            now,
363            since: Some(since.link()),
364        },
365        links,
366    ))
367}
368
369/// A job that failed and then passed on a re-run of the same run.
370fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
371    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
372    for run in runs {
373        by_run.entry(run.run_id).or_default().push(run);
374    }
375    let mut failed_first: Vec<&Run> = by_run
376        .values()
377        .filter_map(|attempts| {
378            attempts.iter().copied().find(|failed| {
379                failed.failed_itself()
380                    && attempts
381                        .iter()
382                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
383            })
384        })
385        .collect();
386    if failed_first.is_empty() {
387        return None;
388    }
389    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
390    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
391    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
392    let sentence = format!(
393        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
394        subject(key)
395    );
396    // Once in the window may be chance: a notice; twice, a warning.
397    let severity = if flaky >= FLAKY_WARNING {
398        Severity::Warning
399    } else {
400        Severity::Notice
401    };
402    Some(insight(
403        key,
404        "flaky",
405        severity,
406        sentence,
407        InsightKind::Flaky { flaky, total },
408        failed_first.iter().take(5).map(|run| run.link()).collect(),
409    ))
410}
411
412/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
413/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
414fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
415    let measured = runs
416        .iter()
417        .rev()
418        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
419    let name = measured.clone().next()?.profile.as_deref()?;
420    let measured: Vec<&Run> = measured
421        .filter(|run| run.profile.as_deref() == Some(name))
422        .take(CAP_RUNS)
423        .copied()
424        .collect();
425    if measured.len() < CAP_MIN_RUNS {
426        return None;
427    }
428    let profile = config.profiles.get(name)?;
429    let cap = profile.memory.0;
430    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
431    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
432    let peak = spread(peaks.clone()).p95;
433    if cap == 0 || !near(&peak) {
434        return None;
435    }
436    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
437    let auto = if !config.sizing.auto {
438        AutoNote::Off
439    } else if profile
440        .max_memory
441        .is_some_and(|max| profile.memory.0 >= max.0)
442    {
443        AutoNote::AtMax
444    } else {
445        AutoNote::Raises
446    };
447    let what = match auto {
448        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
449        AutoNote::AtMax => {
450            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
451        }
452        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
453    };
454    let sentence = format!(
455        "{} peaks at {}, {}% of {name}'s {}. {what}",
456        subject(key),
457        ByteSize(peak),
458        peak.saturating_mul(100) / cap,
459        ByteSize(cap)
460    );
461    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
462    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
463        Severity::Notice
464    } else {
465        Severity::Warning
466    };
467    Some(insight(
468        key,
469        "near-cap",
470        severity,
471        sentence,
472        InsightKind::NearCap {
473            profile: name.to_owned(),
474            peak,
475            cap,
476            auto,
477        },
478        measured.iter().take(5).map(|run| run.link()).collect(),
479    ))
480}
481
482/// A job's last runs that could write to the build cache, each with this many lookups at least…
483const COLD_RUNS: usize = 5;
484const COLD_MIN_RUNS: usize = 3;
485const COLD_MIN_LOOKUPS: u64 = 10;
486/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
487/// next should find most of it.
488const COLD_PERCENT: u64 = 10;
489
490fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
491    let writers: Vec<&Run> = runs
492        .iter()
493        .rev()
494        .filter(|run| {
495            run.build_cache
496                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
497        })
498        .take(COLD_RUNS)
499        .copied()
500        .collect();
501    if writers.len() < COLD_MIN_RUNS {
502        return None;
503    }
504    let mut total = api::BuildCacheCounts::default();
505    for run in &writers {
506        total.add(&run.build_cache.unwrap_or_default());
507    }
508    let percent = total.hit_percent()?;
509    if percent > COLD_PERCENT {
510        return None;
511    }
512    let branch = writers.first()?.branch.clone();
513    let sentence = format!(
514        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
515         differ from run to run.",
516        subject(key),
517        branch
518            .as_deref()
519            .map(|branch| format!(" on {branch}"))
520            .unwrap_or_default(),
521        writers.len()
522    );
523    Some(insight(
524        key,
525        "cold-cache",
526        Severity::Notice,
527        sentence,
528        InsightKind::ColdCache {
529            branch,
530            hits: total.hits,
531            lookups: total.lookups(),
532        },
533        writers.iter().map(|run| run.link()).collect(),
534    ))
535}
536
537/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
538fn throttled(window: &[&Run]) -> Vec<Insight> {
539    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
540    for run in window {
541        repositories.entry(&run.repository).or_default().push(run);
542    }
543    repositories
544        .into_iter()
545        .filter_map(|(repository, mut runs)| {
546            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
547            let hit: Vec<&Run> = runs
548                .iter()
549                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
550                .copied()
551                .collect();
552            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
553            if throttled < THROTTLED_MIN_RUNS {
554                return None;
555            }
556            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
557            Some(Insight {
558                key: format!("throttled:{repository}"),
559                severity: Severity::Warning,
560                repository: repository.to_owned(),
561                workflow: None,
562                job: None,
563                sentence: format!(
564                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
565                     host, or a job at its cap, was short of memory."
566                ),
567                kind: InsightKind::Throttled { throttled, total },
568                runs: hit.iter().take(5).map(|run| run.link()).collect(),
569            })
570        })
571        .collect()
572}
573
574/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
575/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
576/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
577fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
578    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
579    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
580    for run in &history.runs {
581        let Some(started) = run.started_at.filter(|at| *at <= now) else {
582            continue;
583        };
584        let week = match now - started {
585            back if back < WEEK => 0,
586            back if back < 2 * WEEK && !before_kept => 1,
587            _ => continue,
588        };
589        weeks.entry(&run.repository).or_default()[week].push(run);
590    }
591    weeks
592        .into_iter()
593        .map(|(repository, [now, before])| RepositoryWaits {
594            repository: repository.to_owned(),
595            now: week_waits(&now, history.kept_since),
596            before: week_waits(&before, history.kept_since),
597        })
598        .collect()
599}
600
601fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
602    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
603    let of = |runs: Vec<&&Run>| {
604        let waited: Vec<u64> = runs
605            .iter()
606            .filter_map(|run| run.waited(kept_since))
607            .collect();
608        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
609        let spread = (!waited.is_empty()).then(|| spread(waited));
610        WaitSpread {
611            jobs,
612            p50: spread.as_ref().map(|spread| spread.p50),
613            p95: spread.as_ref().map(|spread| spread.p95),
614        }
615    };
616    WeekWaits {
617        all: of(runs.iter().collect()),
618        warm: of(runs.iter().filter(warm).collect()),
619        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
620    }
621}
622
623fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
624    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
625        return None;
626    }
627    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
628    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
629    if now < threshold {
630        return None;
631    }
632    Some(Insight {
633        key: format!("waits:{}", waits.repository),
634        severity: Severity::Notice,
635        repository: waits.repository.clone(),
636        workflow: None,
637        job: None,
638        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
639            format!(
640                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
641                waits.repository,
642                span(now),
643                span(before)
644            )
645        } else {
646            format!(
647                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
648                waits.repository,
649                span(now),
650                span(before)
651            )
652        },
653        kind: InsightKind::Waits { before, now },
654        runs: Vec::new(),
655    })
656}
657
658/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
659fn span(seconds: u64) -> String {
660    match seconds {
661        0..60 => format!("{seconds}s"),
662        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
663        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
664        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
665    }
666}
667
668#[cfg(test)]
669mod tests;