Skip to main content

runnerd_core/
insights.rs

1//! Insights from the job history (ADR 0051): jobs that got slower and stayed so, jobs that fail and pass on a re-run,
2//! jobs whose memory nears their profile's cap, jobs that find almost nothing in the build cache, repositories whose
3//! downloads memory pressure throttled, and how long each repository's jobs wait, this week against the last when the
4//! history holds all of last week; and the daily digest. Pure: the daemon reads the history, the
5//! config and the clock and passes them in. It sends nothing and changes nothing.
6
7use std::collections::{BTreeMap, BTreeSet};
8
9use crate::api::{
10    self, AutoNote, Digest, Insight, InsightKind, Insights, JobState, LongWait, RepositoryWaits,
11    RunLink, Severity, WaitSpread, WeekWaits,
12};
13use crate::config::{ByteSize, Config};
14use crate::sizing::spread;
15
16const SECOND: u64 = 1000;
17const DAY: u64 = 24 * 3600 * SECOND;
18const WEEK: u64 = 7 * DAY;
19
20/// The last runs a slower job's median is taken over; the baseline needs as many before them.
21const RECENT: usize = 5;
22/// A job is slower when its recent median is this share above its baseline's, in percent…
23const SLOWER_PERCENT: u64 = 20;
24/// …and at least this many seconds above it.
25const SLOWER_SECONDS: u64 = 30;
26
27/// The last runs whose memory peaks are read, and the fewest that say anything.
28const CAP_RUNS: usize = 10;
29const CAP_MIN_RUNS: usize = 5;
30/// A job is near its cap when the 95th percentile of its peaks is this share of it, in percent.
31const NEAR_CAP: u64 = 85;
32/// Runs at `NEAR_CAP` or more that make it a warning.
33const NEAR_CAP_WARNING: usize = 2;
34/// Flaky runs that make it a warning.
35const FLAKY_WARNING: u32 = 2;
36
37/// A repository's waits are an insight when this week's p95 is this share above last week's, in percent, and a minute
38/// above it, each week with this many jobs.
39const WAITS_PERCENT: u64 = 50;
40const WAITS_SECONDS: u64 = 60;
41const WAITS_MIN_JOBS: u32 = 5;
42/// The jobs a week needs for its p95 to be other than its longest wait, the nearest rank `spread` takes.
43const WAITS_ONE_IN_20: u32 = 20;
44
45/// A repository's runs the kernel throttled for memory pressure that make an insight.
46const THROTTLED_MIN_RUNS: u32 = 2;
47
48/// One job the history closed, as insights read it. Times in milliseconds.
49#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct Run {
51    /// GitHub's job ID.
52    pub id: u64,
53    /// `owner/name`.
54    pub repository: String,
55    /// The workflow's name, when known.
56    pub workflow: Option<String>,
57    /// The job's name.
58    pub name: String,
59    /// The branch the run is for.
60    pub branch: Option<String>,
61    /// The job's page on GitHub.
62    pub url: Option<String>,
63    /// The workflow run.
64    pub run_id: u64,
65    /// The run's attempt, 1 for the first.
66    pub run_attempt: u32,
67    /// `Completed` or `Lost`.
68    pub state: JobState,
69    /// GitHub's conclusion, for a completed job.
70    pub conclusion: Option<String>,
71    /// When GitHub queued it.
72    pub queued_at: u64,
73    /// When it started.
74    pub started_at: Option<u64>,
75    /// When it finished, or was found lost.
76    pub finished_at: Option<u64>,
77    /// When its runner's container started: before `queued_at`, it found a warm runner.
78    pub runner_started_at: Option<u64>,
79    /// The profile its runner ran in.
80    pub profile: Option<String>,
81    /// Its sampled memory peak.
82    pub memory_peak: Option<u64>,
83    /// What the build cache served it, if its runner had the build cache.
84    pub build_cache: Option<api::BuildCacheCounts>,
85    /// Times the kernel throttled its container's sockets for memory pressure, when counted.
86    pub sock_throttled: Option<u64>,
87}
88
89impl Run {
90    fn finished(&self) -> u64 {
91        self.finished_at.unwrap_or(0)
92    }
93
94    fn succeeded(&self) -> bool {
95        self.state == JobState::Completed && self.conclusion.as_deref() == Some("success")
96    }
97
98    /// Failed as `GET /v1/stats` counts failures: failed, timed out or lost.
99    fn failed(&self) -> bool {
100        self.state == JobState::Lost || self.failed_itself()
101    }
102
103    /// Failed or timed out: what the job did, not what became of its runner.
104    fn failed_itself(&self) -> bool {
105        self.state == JobState::Completed
106            && matches!(self.conclusion.as_deref(), Some("failure" | "timed_out"))
107    }
108
109    /// Seconds from start to finish.
110    fn took(&self) -> Option<u64> {
111        Some(self.finished_at?.saturating_sub(self.started_at?) / SECOND)
112    }
113
114    /// Seconds from queued, or from `kept_since` (ms) if that came later, as for a job queued before runnerd first
115    /// ran, to started.
116    fn waited(&self, kept_since: u64) -> Option<u64> {
117        Some(
118            self.started_at?
119                .saturating_sub(self.queued_at.max(kept_since))
120                / SECOND,
121        )
122    }
123
124    fn link(&self) -> RunLink {
125        RunLink {
126            job: self.id,
127            run_id: self.run_id,
128            run_attempt: self.run_attempt,
129            branch: self.branch.clone(),
130            url: self.url.clone(),
131            at: self.finished() / SECOND,
132        }
133    }
134}
135
136/// What insights read from the job history.
137#[derive(Debug, Clone, Default, PartialEq, Eq)]
138pub struct History {
139    /// The jobs it closed in the days read.
140    pub runs: Vec<Run>,
141    /// From when it holds every job that ended, in ms (`history_kept`): before that, pruning or the daemon's first
142    /// start left only some, so no week before it is compared.
143    pub kept_since: u64,
144}
145
146/// A job as insights tell jobs apart: repository, workflow and name.
147type JobKey<'run> = (&'run str, Option<&'run str>, &'run str);
148
149fn key_of(run: &Run) -> JobKey<'_> {
150    (&run.repository, run.workflow.as_deref(), &run.name)
151}
152
153/// The insights of the `days` before `now` (ms), and the waits of the 14 days before it. `history` may hold more:
154/// only the runs finished by `now` count.
155pub fn insights(history: &History, config: &Config, days: u32, now: u64) -> Insights {
156    let runs = &history.runs;
157    let since = now.saturating_sub(u64::from(days) * DAY);
158    let window: Vec<&Run> = runs
159        .iter()
160        .filter(|run| {
161            matches!(run.state, JobState::Completed | JobState::Lost)
162                && run
163                    .finished_at
164                    .is_some_and(|at| (since..=now).contains(&at))
165        })
166        .collect();
167    let mut jobs: BTreeMap<JobKey<'_>, Vec<&Run>> = BTreeMap::new();
168    for run in &window {
169        jobs.entry(key_of(run)).or_default().push(run);
170    }
171    let mut found = Vec::new();
172    for (key, runs) in &mut jobs {
173        runs.sort_by_key(|run| (run.finished(), run.id));
174        found.extend(slower(*key, runs));
175        found.extend(flaky(*key, runs));
176        found.extend(near_cap(*key, runs, config));
177        found.extend(cold_cache(*key, runs));
178    }
179    found.extend(throttled(&window));
180    let waits = waits(history, now);
181    found.extend(waits.iter().filter_map(waits_insight));
182    found.sort_by(|a, b| {
183        (a.severity, &a.repository, &a.workflow, &a.job, &a.key).cmp(&(
184            b.severity,
185            &b.repository,
186            &b.workflow,
187            &b.job,
188            &b.key,
189        ))
190    });
191    Insights {
192        since: since / SECOND,
193        until: now / SECOND,
194        insights: found,
195        waits,
196    }
197}
198
199/// The 24 hours before `now` (ms): what finished and failed, the insights that hold at `now` and did not 24 hours
200/// before, and the longest waits.
201pub fn digest(history: &History, config: &Config, now: u64) -> Digest {
202    let runs = &history.runs;
203    let since = now.saturating_sub(DAY);
204    let finished: Vec<&Run> = runs
205        .iter()
206        .filter(|run| {
207            matches!(run.state, JobState::Completed | JobState::Lost)
208                && run.finished_at.is_some_and(|at| at > since && at <= now)
209        })
210        .collect();
211    let before: BTreeSet<String> = insights(history, config, api::INSIGHT_DAYS, since)
212        .insights
213        .into_iter()
214        .map(|insight| insight.key)
215        .collect();
216    let new = insights(history, config, api::INSIGHT_DAYS, now)
217        .insights
218        .into_iter()
219        .filter(|insight| !before.contains(&insight.key))
220        .collect();
221    let mut started: Vec<&Run> = runs
222        .iter()
223        .filter(|run| run.started_at.is_some_and(|at| at > since && at <= now))
224        .collect();
225    started.sort_by_key(|run| (std::cmp::Reverse(run.waited(history.kept_since)), run.id));
226    Digest {
227        since: since / SECOND,
228        until: now / SECOND,
229        finished: finished.len() as u64,
230        failed: finished.iter().filter(|run| run.failed()).count() as u64,
231        new,
232        waits: started
233            .into_iter()
234            .take(3)
235            .map(|run| LongWait {
236                job: run.id,
237                repository: run.repository.clone(),
238                workflow: run.workflow.clone(),
239                name: run.name.clone(),
240                waited: run.waited(history.kept_since).unwrap_or(0),
241                url: run.url.clone(),
242            })
243            .collect(),
244        savings: None,
245    }
246}
247
248/// `repository: workflow / job`, or `repository: job` when the workflow is not known.
249fn subject((repository, workflow, name): JobKey<'_>) -> String {
250    match workflow {
251        Some(workflow) => format!("{repository}: {workflow} / {name}"),
252        None => format!("{repository}: {name}"),
253    }
254}
255
256fn key_text(kind: &str, (repository, workflow, name): JobKey<'_>) -> String {
257    format!("{kind}:{repository}/{}/{name}", workflow.unwrap_or(""))
258}
259
260fn insight(
261    key: JobKey<'_>,
262    kind: &str,
263    severity: Severity,
264    sentence: String,
265    found: InsightKind,
266    runs: Vec<RunLink>,
267) -> Insight {
268    Insight {
269        key: key_text(kind, key),
270        severity,
271        repository: key.0.to_owned(),
272        workflow: key.1.map(str::to_owned),
273        job: Some(key.2.to_owned()),
274        sentence,
275        kind: found,
276        runs,
277    }
278}
279
280/// The median: the mean of the middle two for an even count.
281fn median(values: &[u64]) -> u64 {
282    let mut values = values.to_vec();
283    values.sort_unstable();
284    let middle = values.len() / 2;
285    match values.len() {
286        0 => 0,
287        len if len % 2 == 1 => values[middle],
288        _ => u64::midpoint(values[middle - 1], values[middle]),
289    }
290}
291
292/// A job whose successful runs on `main` or `master`, or else its busiest branch, got slower and stayed so.
293fn slower(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
294    let timed_on = |branch: &str| -> Vec<(&Run, u64)> {
295        runs.iter()
296            .filter(|run| run.succeeded() && run.branch.as_deref() == Some(branch))
297            .filter_map(|run| Some((*run, run.took()?)))
298            .collect()
299    };
300    let mut branches: BTreeMap<&str, usize> = BTreeMap::new();
301    for run in runs.iter().filter(|run| run.succeeded()) {
302        if let Some(branch) = &run.branch {
303            *branches.entry(branch).or_default() += 1;
304        }
305    }
306    // The default branch when it has runs enough, so a pull request that adds tests is not a regression; else the
307    // most runs, and of equal counts the first by name.
308    let busiest = branches
309        .iter()
310        .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))
311        .map(|(branch, _)| *branch)?;
312    let (branch, timed) = ["main", "master"]
313        .into_iter()
314        .map(|branch| (branch, timed_on(branch)))
315        .find(|(_, timed)| timed.len() >= 2 * RECENT)
316        .unwrap_or_else(|| (busiest, timed_on(busiest)));
317    if timed.len() < 2 * RECENT {
318        return None;
319    }
320    let (baseline, recent) = timed.split_at(timed.len() - RECENT);
321    let took = |runs: &[(&Run, u64)]| runs.iter().map(|(_, took)| *took).collect::<Vec<_>>();
322    let before = median(&took(baseline));
323    let now = median(&took(recent));
324    let threshold = (before + before * SLOWER_PERCENT / 100).max(before + SLOWER_SECONDS);
325    // Every recent run slower than the baseline's median: a job that flips between two speeds, such as build cache
326    // hits and misses, moves its median with a few more slow runs in a row without getting slower.
327    if now < threshold || recent.iter().any(|(_, took)| *took <= before) {
328        return None;
329    }
330    // The first slow run after which no two runs in a row were fast.
331    let mut since = None;
332    let mut fast_in_a_row = 0;
333    for (run, took) in &timed {
334        if *took >= threshold {
335            fast_in_a_row = 0;
336            since = since.or(Some(*run));
337        } else {
338            fast_in_a_row += 1;
339            if fast_in_a_row >= 2 {
340                since = None;
341            }
342        }
343    }
344    let since = since?;
345    let sentence = format!(
346        "{} on {branch} takes {} now, up from {}.",
347        subject(key),
348        span(now),
349        span(before)
350    );
351    let links = recent.iter().rev().map(|(run, _)| run.link()).collect();
352    Some(insight(
353        key,
354        "slower",
355        Severity::Warning,
356        sentence,
357        InsightKind::Slower {
358            branch: Some(branch.to_owned()),
359            before,
360            now,
361            since: Some(since.link()),
362        },
363        links,
364    ))
365}
366
367/// A job that failed and then passed on a re-run of the same run.
368fn flaky(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
369    let mut by_run: BTreeMap<u64, Vec<&Run>> = BTreeMap::new();
370    for run in runs {
371        by_run.entry(run.run_id).or_default().push(run);
372    }
373    let mut failed_first: Vec<&Run> = by_run
374        .values()
375        .filter_map(|attempts| {
376            attempts.iter().copied().find(|failed| {
377                failed.failed_itself()
378                    && attempts
379                        .iter()
380                        .any(|later| later.run_attempt > failed.run_attempt && later.succeeded())
381            })
382        })
383        .collect();
384    if failed_first.is_empty() {
385        return None;
386    }
387    failed_first.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
388    let flaky = u32::try_from(failed_first.len()).unwrap_or(u32::MAX);
389    let total = u32::try_from(by_run.len()).unwrap_or(u32::MAX);
390    let sentence = format!(
391        "{} failed and then passed on a re-run in {flaky} of its {total} runs.",
392        subject(key)
393    );
394    // Once in the window may be chance: a notice; twice, a warning.
395    let severity = if flaky >= FLAKY_WARNING {
396        Severity::Warning
397    } else {
398        Severity::Notice
399    };
400    Some(insight(
401        key,
402        "flaky",
403        severity,
404        sentence,
405        InsightKind::Flaky { flaky, total },
406        failed_first.iter().take(5).map(|run| run.link()).collect(),
407    ))
408}
409
410/// A job whose last runs' memory peaks near its profile's `memory`: the profile of its last measured run, whose
411/// runs alone count, as a job a repository serves in several profiles peaks apart in each.
412fn near_cap(key: JobKey<'_>, runs: &[&Run], config: &Config) -> Option<Insight> {
413    let measured = runs
414        .iter()
415        .rev()
416        .filter(|run| run.memory_peak.is_some() && run.profile.is_some());
417    let name = measured.clone().next()?.profile.as_deref()?;
418    let measured: Vec<&Run> = measured
419        .filter(|run| run.profile.as_deref() == Some(name))
420        .take(CAP_RUNS)
421        .copied()
422        .collect();
423    if measured.len() < CAP_MIN_RUNS {
424        return None;
425    }
426    let profile = config.profiles.get(name)?;
427    let cap = profile.memory.0;
428    let peaks: Vec<u64> = measured.iter().filter_map(|run| run.memory_peak).collect();
429    let near = |peak: &u64| peak.saturating_mul(100) >= cap.saturating_mul(NEAR_CAP);
430    let peak = spread(peaks.clone()).p95;
431    if cap == 0 || !near(&peak) {
432        return None;
433    }
434    let near_runs = peaks.iter().filter(|peak| near(peak)).count();
435    let auto = if !config.sizing.auto {
436        AutoNote::Off
437    } else if profile
438        .max_memory
439        .is_some_and(|max| profile.memory.0 >= max.0)
440    {
441        AutoNote::AtMax
442    } else {
443        AutoNote::Raises
444    };
445    let what = match auto {
446        AutoNote::Raises => format!("Auto sizing raises {name} when a run meets the cap."),
447        AutoNote::AtMax => {
448            format!("{name} is at its max_memory, so auto sizing cannot raise it.")
449        }
450        AutoNote::Off => format!("Auto sizing is off: raise {name}'s memory, or turn it on."),
451    };
452    let sentence = format!(
453        "{} peaks at {}, {}% of {name}'s {}. {what}",
454        subject(key),
455        ByteSize(peak),
456        peak.saturating_mul(100) / cap,
457        ByteSize(cap)
458    );
459    // Auto sizing deals with it, and one spike may be chance: a notice. The owner acts on the rest.
460    let severity = if auto == AutoNote::Raises || near_runs < NEAR_CAP_WARNING {
461        Severity::Notice
462    } else {
463        Severity::Warning
464    };
465    Some(insight(
466        key,
467        "near-cap",
468        severity,
469        sentence,
470        InsightKind::NearCap {
471            profile: name.to_owned(),
472            peak,
473            cap,
474            auto,
475        },
476        measured.iter().take(5).map(|run| run.link()).collect(),
477    ))
478}
479
480/// A job's last runs that could write to the build cache, each with this many lookups at least…
481const COLD_RUNS: usize = 5;
482const COLD_MIN_RUNS: usize = 3;
483const COLD_MIN_LOOKUPS: u64 = 10;
484/// …found at most this share of their lookups, in percent: the runs before them stored what they built, so the
485/// next should find most of it.
486const COLD_PERCENT: u64 = 10;
487
488fn cold_cache(key: JobKey<'_>, runs: &[&Run]) -> Option<Insight> {
489    let writers: Vec<&Run> = runs
490        .iter()
491        .rev()
492        .filter(|run| {
493            run.build_cache
494                .is_some_and(|cache| cache.writer && cache.lookups() >= COLD_MIN_LOOKUPS)
495        })
496        .take(COLD_RUNS)
497        .copied()
498        .collect();
499    if writers.len() < COLD_MIN_RUNS {
500        return None;
501    }
502    let mut total = api::BuildCacheCounts::default();
503    for run in &writers {
504        total.add(&run.build_cache.unwrap_or_default());
505    }
506    let percent = total.hit_percent()?;
507    if percent > COLD_PERCENT {
508        return None;
509    }
510    let branch = writers.first()?.branch.clone();
511    let sentence = format!(
512        "{} got {percent}% of its build cache lookups from the cache{} over its last {} runs: its tasks' inputs \
513         differ from run to run.",
514        subject(key),
515        branch
516            .as_deref()
517            .map(|branch| format!(" on {branch}"))
518            .unwrap_or_default(),
519        writers.len()
520    );
521    Some(insight(
522        key,
523        "cold-cache",
524        Severity::Notice,
525        sentence,
526        InsightKind::ColdCache {
527            branch,
528            hits: total.hits,
529            lookups: total.lookups(),
530        },
531        writers.iter().map(|run| run.link()).collect(),
532    ))
533}
534
535/// One insight per repository the kernel throttled `THROTTLED_MIN_RUNS` or more of the runs of in `window`.
536fn throttled(window: &[&Run]) -> Vec<Insight> {
537    let mut repositories: BTreeMap<&str, Vec<&Run>> = BTreeMap::new();
538    for run in window {
539        repositories.entry(&run.repository).or_default().push(run);
540    }
541    repositories
542        .into_iter()
543        .filter_map(|(repository, mut runs)| {
544            runs.sort_by_key(|run| std::cmp::Reverse((run.finished(), run.id)));
545            let hit: Vec<&Run> = runs
546                .iter()
547                .filter(|run| run.sock_throttled.is_some_and(|count| count > 0))
548                .copied()
549                .collect();
550            let throttled = u32::try_from(hit.len()).unwrap_or(u32::MAX);
551            if throttled < THROTTLED_MIN_RUNS {
552                return None;
553            }
554            let total = u32::try_from(runs.len()).unwrap_or(u32::MAX);
555            Some(Insight {
556                key: format!("throttled:{repository}"),
557                severity: Severity::Warning,
558                repository: repository.to_owned(),
559                workflow: None,
560                job: None,
561                sentence: format!(
562                    "{repository}: memory pressure slowed the downloads of {throttled} of its {total} jobs: the \
563                     host, or a job at its cap, was short of memory."
564                ),
565                kind: InsightKind::Throttled { throttled, total },
566                runs: hit.iter().take(5).map(|run| run.link()).collect(),
567            })
568        })
569        .collect()
570}
571
572/// Each repository's waits in the 7 days before `now` (ms) and the 7 before those, each counted from no earlier than
573/// when the history holds every job. The week before stays empty when it starts before that: it would hold a sliver
574/// of it, such as the daemon's first minutes. This week is what the history has of it either way.
575fn waits(history: &History, now: u64) -> Vec<RepositoryWaits> {
576    let before_kept = now.saturating_sub(2 * WEEK) < history.kept_since;
577    let mut weeks: BTreeMap<&str, [Vec<&Run>; 2]> = BTreeMap::new();
578    for run in &history.runs {
579        let Some(started) = run.started_at.filter(|at| *at <= now) else {
580            continue;
581        };
582        let week = match now - started {
583            back if back < WEEK => 0,
584            back if back < 2 * WEEK && !before_kept => 1,
585            _ => continue,
586        };
587        weeks.entry(&run.repository).or_default()[week].push(run);
588    }
589    weeks
590        .into_iter()
591        .map(|(repository, [now, before])| RepositoryWaits {
592            repository: repository.to_owned(),
593            now: week_waits(&now, history.kept_since),
594            before: week_waits(&before, history.kept_since),
595        })
596        .collect()
597}
598
599fn week_waits(runs: &[&Run], kept_since: u64) -> WeekWaits {
600    let warm = |run: &&&Run| run.runner_started_at.is_some_and(|at| at <= run.queued_at);
601    let of = |runs: Vec<&&Run>| {
602        let waited: Vec<u64> = runs
603            .iter()
604            .filter_map(|run| run.waited(kept_since))
605            .collect();
606        let jobs = u32::try_from(waited.len()).unwrap_or(u32::MAX);
607        let spread = (!waited.is_empty()).then(|| spread(waited));
608        WaitSpread {
609            jobs,
610            p50: spread.as_ref().map(|spread| spread.p50),
611            p95: spread.as_ref().map(|spread| spread.p95),
612        }
613    };
614    WeekWaits {
615        all: of(runs.iter().collect()),
616        warm: of(runs.iter().filter(warm).collect()),
617        cold: of(runs.iter().filter(|run| !warm(run)).collect()),
618    }
619}
620
621fn waits_insight(waits: &RepositoryWaits) -> Option<Insight> {
622    if waits.now.all.jobs < WAITS_MIN_JOBS || waits.before.all.jobs < WAITS_MIN_JOBS {
623        return None;
624    }
625    let (before, now) = (waits.before.all.p95?, waits.now.all.p95?);
626    let threshold = (before + before * WAITS_PERCENT / 100).max(before + WAITS_SECONDS);
627    if now < threshold {
628        return None;
629    }
630    Some(Insight {
631        key: format!("waits:{}", waits.repository),
632        severity: Severity::Notice,
633        repository: waits.repository.clone(),
634        workflow: None,
635        job: None,
636        sentence: if waits.now.all.jobs < WAITS_ONE_IN_20 {
637            format!(
638                "{}: the longest wait for a runner this week is {}, up from {} the week before.",
639                waits.repository,
640                span(now),
641                span(before)
642            )
643        } else {
644            format!(
645                "{}: 1 job in 20 waits {} or more for a runner this week, up from {} the week before.",
646                waits.repository,
647                span(now),
648                span(before)
649            )
650        },
651        kind: InsightKind::Waits { before, now },
652        runs: Vec::new(),
653    })
654}
655
656/// A duration as a sentence reads it: `45s`, `4m 30s`, `1h 5m`.
657fn span(seconds: u64) -> String {
658    match seconds {
659        0..60 => format!("{seconds}s"),
660        60..3600 if seconds.is_multiple_of(60) => format!("{}m", seconds / 60),
661        60..3600 => format!("{}m {}s", seconds / 60, seconds % 60),
662        _ => format!("{}h {}m", seconds / 3600, seconds % 3600 / 60),
663    }
664}
665
666#[cfg(test)]
667mod tests;