Skip to main content

runnerd_core/api/jobs/
facts.rs

1//! What every client shows of a job: its name, its outcome and its steps', how long it waited and ran, its phases
2//! and milestones, its figures as samples come in, what it used over a step, and how it compares with the same
3//! job's latest runs. Each returns meaning, a label, a figure or an [`OutcomeKind`]; the dashboard, the TUI and
4//! `runnerctl` only draw it, each colouring a kind its own way. Nothing here formats for one client.
5
6use std::borrow::Cow;
7use std::fmt;
8
9use super::{
10    FailureCause, Job, JobState, JobSummary, RunnerProbe, Sample, SampleSeries, Step, Timeline,
11};
12use crate::api::{QueuedJob, Record, Tone, Toned};
13
14/// What kind of outcome a job's or a step's is, for each client to colour as it colours that meaning.
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub enum OutcomeKind {
17    /// It succeeded.
18    Succeeded,
19    /// It failed.
20    Failed,
21    /// It failed because the runner failed it, not its code: a job read as `runner fault` (`JobSummary::runner_fault`).
22    /// Only a job's outcome has it.
23    RunnerFault,
24    /// It is under way.
25    Running,
26    /// It has not begun: waiting for a runner, or a step not reached yet.
27    Waiting,
28    /// It ended without succeeding or failing: cancelled, skipped, run elsewhere, or ended before any runner took it.
29    Ended,
30}
31
32/// A job's or a step's outcome: its word and what it means.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct Outcome {
35    /// Such as `success`, `timed out` or `running`.
36    pub label: Cow<'static, str>,
37    /// What it means.
38    pub kind: OutcomeKind,
39}
40
41impl Outcome {
42    fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
43        Outcome {
44            label: Cow::Borrowed(label),
45            kind,
46        }
47    }
48
49    /// A conclusion of GitHub's, its underscores as spaces: `timed_out` is `timed out`.
50    fn concluded(conclusion: &str) -> Outcome {
51        let kind = match conclusion {
52            "success" => OutcomeKind::Succeeded,
53            failed if failed_conclusion(failed) => OutcomeKind::Failed,
54            _ => OutcomeKind::Ended,
55        };
56        Outcome {
57            label: Cow::Owned(conclusion.replace('_', " ")),
58            kind,
59        }
60    }
61}
62
63impl FailureCause {
64    /// Its word, as a client with little room writes it: `action download`, `action resolution` or `checkout`.
65    /// `None` for a cause this build does not know.
66    pub fn label(self) -> Option<&'static str> {
67        match self {
68            FailureCause::ActionArchive => Some("action download"),
69            FailureCause::ActionResolve => Some("action resolution"),
70            FailureCause::Checkout => Some("checkout"),
71            FailureCause::Unknown => None,
72        }
73    }
74
75    /// What it means, in a line. `None` for a cause this build does not know.
76    pub fn line(self) -> Option<&'static str> {
77        match self {
78            FailureCause::ActionArchive => {
79                Some("the runner could not download or unpack an action while it set up the job")
80            }
81            FailureCause::ActionResolve => {
82                Some("the runner could not resolve an action with GitHub while it set up the job")
83            }
84            FailureCause::Checkout => Some("actions/checkout could not fetch the repository"),
85            FailureCause::Unknown => None,
86        }
87    }
88}
89
90/// The conclusions that count as a failure, as `GET /v1/stats` and the insights count them.
91fn failed_conclusion(conclusion: &str) -> bool {
92    matches!(conclusion, "failure" | "timed_out")
93}
94
95impl JobSummary {
96    /// Whether it failed, as `GET /v1/stats` counts failures: it concluded `failure` or `timed_out`, or it lost its
97    /// runner. A job another machine ran is not this daemon's to count.
98    pub fn is_failure(&self) -> bool {
99        match self.state {
100            JobState::Lost => true,
101            JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
102            JobState::Queued | JobState::Running | JobState::Elsewhere => false,
103        }
104    }
105
106    /// Its outcome: where it is while it can change, then how it ended. Its kind is [`OutcomeKind::Failed`] or
107    /// [`OutcomeKind::RunnerFault`] exactly when it [failed](JobSummary::is_failure); a job that concluded failed and
108    /// that the runner failed reads `runner fault` (`runner_fault`) and counts as failed all the same. A job that left the queue without starting, as one a newer push
109    /// cancelled, did not run elsewhere: it reads `cancelled in queue`, or as it concluded.
110    pub fn outcome(&self) -> Outcome {
111        outcome_of(
112            self.state,
113            self.conclusion.as_deref(),
114            self.timeline.started_at.is_some(),
115            self.runner_fault,
116        )
117    }
118
119    /// What its log says failed it, as every client writes it under its outcome: `the runner could not download or
120    /// unpack an action while it set up the job`. `None` for a job that did not fail, and for one whose log does not
121    /// say (`failure_cause`).
122    pub fn failure_line(&self) -> Option<&'static str> {
123        self.failure_cause
124            .filter(|_| self.is_failure())
125            .and_then(FailureCause::line)
126    }
127
128    /// That runnerd re-ran it, as every client writes it under its outcome: `runnerd re-ran the run's failed jobs as
129    /// attempt 2, since the runner failed this one`. `None` for a job runnerd did not re-run (`rerun_at`).
130    pub fn rerun_line(&self) -> Option<String> {
131        self.rerun_at.map(|_| {
132            format!(
133                "runnerd re-ran the run's failed jobs as attempt {}, since the runner failed this one",
134                self.run_attempt.saturating_add(1)
135            )
136        })
137    }
138
139    /// What the probe after its runner exited found (`probe`, ADR 0155), as every client writes it under its outcome,
140    /// with its tone: `a probe after the job found the runner's mounts failing`. For a job that failed, and for any
141    /// job whose probe found a call failing; a failed job without a verdict reads `no probe verdict recorded`, never
142    /// as clean. `None` for every other job, and for a verdict this build does not know.
143    pub fn probe_line(&self) -> Option<Toned> {
144        let found = |text: &str, tone| {
145            Some(Toned {
146                text: text.to_owned(),
147                tone,
148            })
149        };
150        match self.probe {
151            None if self.is_failure() => found("no probe verdict recorded", Tone::Quiet),
152            None | Some(RunnerProbe::Unknown) => None,
153            Some(RunnerProbe::Clean | RunnerProbe::Broken) if !self.is_failure() => None,
154            Some(RunnerProbe::Clean) => found(
155                "the probe's sampled mount and /proc calls succeeded",
156                Tone::Quiet,
157            ),
158            Some(RunnerProbe::Broken) => found("a probe after the job could not run", Tone::Quiet),
159            Some(RunnerProbe::Mount) => found(
160                "a probe after the job found the runner's mounts failing",
161                Tone::Warn,
162            ),
163            Some(RunnerProbe::Procfs) => found(
164                "a probe after the job found the runner's /proc failing to open",
165                Tone::Warn,
166            ),
167            Some(RunnerProbe::MountAndProcfs) => found(
168                "a probe after the job found the runner's mounts and /proc failing",
169                Tone::Warn,
170            ),
171            Some(RunnerProbe::TimedOut) => found(
172                "a probe after the job did not finish within 3 seconds",
173                Tone::Warn,
174            ),
175        }
176    }
177
178    /// Seconds it waited for a runner: until it started, or until `now` while it may still start. `None` for a job
179    /// that ended without starting here.
180    pub fn waited(&self, now: u64) -> Option<u64> {
181        let end = match self.timeline.started_at {
182            Some(started) => started,
183            None if !self.state.is_final() => now,
184            None => return None,
185        };
186        Some(end.saturating_sub(self.timeline.queued_at))
187    }
188
189    /// What of its wait at `now` passed while no runnerd ran, to follow it: `8h 10m of it while runnerd was not
190    /// running`, never more than the wait. `None` when none did.
191    pub fn unseen_note(&self, now: u64) -> Option<String> {
192        let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
193            waited.min(self.timeline.unseen)
194        });
195        (unseen > 0).then(|| {
196            format!(
197                "{} of it while runnerd was not running",
198                crate::api::duration(unseen)
199            )
200        })
201    }
202
203    /// What held its CPU back by `now`: `held at its CPU cap in 18% of its busy time, waited for a CPU 31% of its
204    /// run`. The cap's part is left out without a quota's figures or busy periods, the wait's without pressure stall
205    /// information or a run time; `None` when both are. A share that is not 0 never reads as 0%.
206    pub fn cpu_held(&self, now: u64) -> Option<String> {
207        let metrics = &self.metrics;
208        let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
209            0 => "under 1%".to_owned(),
210            percent => format!("{}%", percent.min(100)),
211        };
212        let cap = metrics
213            .cpu_periods
214            .zip(metrics.cpu_throttled)
215            .filter(|(periods, _)| *periods > 0)
216            .map(|(periods, throttled)| match throttled {
217                0 => "never held at its CPU cap".to_owned(),
218                throttled => format!(
219                    "held at its CPU cap in {} of its busy time",
220                    share(throttled, periods)
221                ),
222            });
223        let waited = metrics
224            .cpu_stalled
225            .zip(self.ran(now).filter(|ran| *ran > 0))
226            .map(|(stalled, ran)| match stalled {
227                0 => "never waited for a CPU".to_owned(),
228                stalled => format!(
229                    "waited for a CPU {} of its run",
230                    share(stalled, ran.saturating_mul(1000))
231                ),
232            });
233        let parts: Vec<String> = cap.into_iter().chain(waited).collect();
234        (!parts.is_empty()).then(|| parts.join(", "))
235    }
236
237    /// Seconds it ran: until it finished, or until `now` while it may still finish. `None` before it started, and
238    /// for a final job without a finish time, which no client claims ran for as long as it has been since.
239    pub fn ran(&self, now: u64) -> Option<u64> {
240        let started = self.timeline.started_at?;
241        let end = match self.timeline.finished_at {
242            Some(finished) => finished,
243            None if !self.state.is_final() => now,
244            None => return None,
245        };
246        Some(end.saturating_sub(started))
247    }
248
249    /// Its phases in order, each from when it began to when it ended, `None` while it goes on. A phase without an
250    /// end goes on while the job can change; a final job's ends when the job finished, or is left out when no time
251    /// says how long it took. Cleaning up follows the finish of a job whose runner started here, until its cleanup.
252    pub fn phases(&self) -> Vec<Phase> {
253        let timeline = &self.timeline;
254        let open = !self.state.is_final();
255        // `Some(end)` for a phase to show, `None` for one to leave out.
256        let until = |end: Option<u64>| match end {
257            Some(end) => Some(Some(end)),
258            None if open => Some(None),
259            None => timeline.finished_at.map(Some),
260        };
261        let runner = timeline.runner_from();
262        let mut phases = Vec::new();
263        if let Some(to) = until(runner.or(timeline.started_at)) {
264            phases.push(Phase {
265                kind: PhaseKind::Waiting,
266                from: timeline.queued_at,
267                to,
268            });
269        }
270        if let Some(from) = runner
271            && let Some(to) = until(timeline.started_at)
272        {
273            phases.push(Phase {
274                kind: PhaseKind::Starting,
275                from,
276                to,
277            });
278        }
279        if let Some(from) = timeline.started_at
280            && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
281        {
282            phases.push(Phase {
283                kind: PhaseKind::Running,
284                from,
285                to,
286            });
287        }
288        if let Some(from) = timeline.finished_at
289            && runner.is_some()
290        {
291            phases.push(Phase {
292                kind: PhaseKind::CleaningUp,
293                from,
294                to: timeline.cleaned_at,
295            });
296        }
297        phases
298    }
299
300    /// Its name as every client writes it: `workflow / job`.
301    pub fn title(&self) -> JobTitle<'_> {
302        job_title(self.workflow.as_deref(), &self.name, Some(self.id))
303    }
304}
305
306/// [`JobSummary::outcome`] of a job in `state`, concluded `conclusion`, that `started` here or not.
307pub(crate) fn outcome_of(
308    state: JobState,
309    conclusion: Option<&str>,
310    started: bool,
311    runner_fault: bool,
312) -> Outcome {
313    match (state, conclusion) {
314        (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
315        (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
316        (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
317        (JobState::Elsewhere, Some("cancelled")) if !started => {
318            Outcome::new("cancelled in queue", OutcomeKind::Ended)
319        }
320        (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
321            kind: OutcomeKind::Ended,
322            ..Outcome::concluded(conclusion)
323        },
324        (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
325        (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
326        (JobState::Completed, Some(conclusion))
327            if runner_fault && failed_conclusion(conclusion) =>
328        {
329            Outcome::new("runner fault", OutcomeKind::RunnerFault)
330        }
331        (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
332    }
333}
334
335/// A stretch of a job's life.
336#[derive(Debug, Clone, Copy, PartialEq, Eq)]
337pub struct Phase {
338    /// Which.
339    pub kind: PhaseKind,
340    /// When it began.
341    pub from: u64,
342    /// When it ended; `None` while it goes on.
343    pub to: Option<u64>,
344}
345
346impl Phase {
347    /// How long it took, or has taken by `now`.
348    pub fn seconds(&self, now: u64) -> u64 {
349        self.to.unwrap_or(now).saturating_sub(self.from)
350    }
351}
352
353/// The phases of a job, in the order they happen.
354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
355pub enum PhaseKind {
356    /// Queued, without a runner.
357    Waiting,
358    /// A runner took it and is starting.
359    Starting,
360    /// The runner runs it.
361    Running,
362    /// It finished; its container and registration are being removed.
363    CleaningUp,
364}
365
366impl PhaseKind {
367    /// Its name, as every client labels it.
368    pub fn label(self) -> &'static str {
369        match self {
370            PhaseKind::Waiting => "Waiting for a runner",
371            PhaseKind::Starting => "Runner starting",
372            PhaseKind::Running => "Running",
373            PhaseKind::CleaningUp => "Cleaning up",
374        }
375    }
376}
377
378/// A moment of a job's life.
379#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub struct Milestone {
381    /// Such as `Container started`.
382    pub label: &'static str,
383    /// When.
384    pub at: u64,
385}
386
387impl Timeline {
388    /// The moments it has times for, earliest first: a warm runner's container started before the job was queued.
389    /// When runnerd first listed it splits its wait in three (ADR 0092): before it, runnerd had not seen the job;
390    /// from it to its container's start, runnerd had; then the runner booted. A listing never comes before the
391    /// queueing, however GitHub's clock and this machine's differ.
392    pub fn milestones(&self) -> Vec<Milestone> {
393        let mut milestones: Vec<Milestone> = [
394            ("Queued", Some(self.queued_at)),
395            (
396                "Seen by runnerd",
397                self.seen_at.map(|seen| seen.max(self.queued_at)),
398            ),
399            ("Container started", self.container_started_at),
400            ("Job started", self.started_at),
401            ("Finished", self.finished_at),
402            ("Cleaned up", self.cleaned_at),
403        ]
404        .into_iter()
405        .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
406        .collect();
407        milestones.sort_by_key(|milestone| milestone.at);
408        milestones
409    }
410}
411
412impl Step {
413    /// Its outcome: how it concluded, else whether it runs or waits. A step that timed out failed, one that was
414    /// cancelled or skipped did not.
415    pub fn outcome(&self) -> Outcome {
416        match (self.conclusion.as_deref(), self.status.as_str()) {
417            (Some(conclusion), _) => Outcome::concluded(conclusion),
418            (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
419            (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
420            (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
421        }
422    }
423
424    /// Whether it runs now.
425    pub fn is_running(&self) -> bool {
426        self.status == "in_progress" && self.conclusion.is_none()
427    }
428
429    /// Seconds it took, until `now` while it runs; `None` before it started and for a step that ended without a
430    /// completion time.
431    pub fn took(&self, now: u64) -> Option<u64> {
432        let started = self.started_at?;
433        let end = match self.completed_at {
434            Some(completed) => completed,
435            None if self.is_running() => now,
436            None => return None,
437        };
438        Some(end.saturating_sub(started))
439    }
440}
441
442/// A job's name as every client writes it, in parts for a page that sets the workflow apart.
443#[derive(Debug, Clone, PartialEq, Eq)]
444pub struct JobTitle<'job> {
445    /// The workflow's name, when known.
446    pub workflow: Option<&'job str>,
447    /// The job's name, or `job <id>` until the daemon knows it.
448    pub name: Cow<'job, str>,
449}
450
451/// `workflow / job`, or the job's name alone; for a job whose name the daemon does not know yet, as for one its
452/// runner took before the daemon saw it queued, `job <id>`, or `a job` without an ID.
453pub fn job_title<'job>(
454    workflow: Option<&'job str>,
455    name: &'job str,
456    id: Option<u64>,
457) -> JobTitle<'job> {
458    let name = match (name, id) {
459        ("", Some(id)) => Cow::Owned(format!("job {id}")),
460        ("", None) => Cow::Borrowed("a job"),
461        (name, _) => Cow::Borrowed(name),
462    };
463    JobTitle { workflow, name }
464}
465
466impl fmt::Display for JobTitle<'_> {
467    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
468        match self.workflow {
469            Some(workflow) => write!(f, "{workflow} / {}", self.name),
470            None => f.write_str(&self.name),
471        }
472    }
473}
474
475impl QueuedJob {
476    /// Its name as [`JobSummary::title`] writes a job's.
477    pub fn title(&self) -> JobTitle<'_> {
478        job_title(self.workflow.as_deref(), &self.name, Some(self.id))
479    }
480}
481
482impl Record {
483    /// Its job's name as [`JobSummary::title`] writes a job's.
484    pub fn title(&self) -> JobTitle<'_> {
485        job_title(
486            self.workflow.as_deref(),
487            self.name.as_deref().unwrap_or_default(),
488            Some(self.job),
489        )
490    }
491}
492
493impl Job {
494    /// Takes a sample of its container as it arrives: while the job runs, its peaks rise to it, high-water marks
495    /// that deleting old samples does not lower. A final job's figures are the daemon's totals and stay.
496    pub fn observe(&mut self, sample: &Sample) {
497        let metrics = &mut self.summary.metrics;
498        if !self.summary.state.is_final() {
499            metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
500            metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
501        }
502    }
503
504    /// Takes the job as read again. While it runs, its peaks keep the highest seen, since the store's lag the
505    /// samples a client saw arrive; once it is final, the read is the daemon's totals.
506    pub fn reread(&mut self, read: Job) {
507        let seen = self.summary.metrics.clone();
508        *self = read;
509        if !self.summary.state.is_final() {
510            let metrics = &mut self.summary.metrics;
511            metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
512            metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
513        }
514    }
515
516    /// How it compares with the same job's latest `runs`, oldest first as given: this job among them as it is now,
517    /// not as the runs were read, since it may have started, finished or peaked higher since.
518    pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
519        let runs: Vec<JobSummary> = runs
520            .iter()
521            .map(|run| {
522                if run.id == self.summary.id {
523                    self.summary.clone()
524                } else {
525                    run.clone()
526                }
527            })
528            .collect();
529        // Another machine's runs are not this daemon's to compare.
530        let finished: Vec<&JobSummary> = runs
531            .iter()
532            .filter(|run| {
533                matches!(run.state, JobState::Completed | JobState::Lost)
534                    && run.timeline.started_at.is_some()
535            })
536            .collect();
537        // The lower of the middle two for an even count: a duration one of the runs took.
538        let median = |mut values: Vec<u64>| {
539            values.sort_unstable();
540            values.get(values.len().saturating_sub(1) / 2).copied()
541        };
542        RunComparison {
543            median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
544            median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
545            finished: finished.len(),
546            failed: finished.iter().filter(|run| run.is_failure()).count(),
547            largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
548            runs,
549        }
550    }
551}
552
553/// A job's latest runs and what they say together, from [`Job::compare`].
554#[derive(Debug, Clone, PartialEq, Eq)]
555pub struct RunComparison {
556    /// Oldest first, this job as it is now.
557    pub runs: Vec<JobSummary>,
558    /// The median of the finished runs' run times.
559    pub median_ran: Option<u64>,
560    /// The median of the finished runs' waits.
561    pub median_waited: Option<u64>,
562    /// The runs that started and ended here: those the medians and failures count.
563    pub finished: usize,
564    /// Of those, the ones that [failed](JobSummary::is_failure).
565    pub failed: usize,
566    /// The largest memory peak of any run.
567    pub largest_peak: Option<u64>,
568}
569
570impl SampleSeries {
571    /// What the container used from `from` to `to`, from the samples taken after `from` up to `to`: each covers the
572    /// interval before it, so steps that meet at a second share no sample. `None` when none was taken in it.
573    pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
574        usage_of(&samples_between(&self.samples, from, to))
575    }
576}
577
578/// The samples of `samples` taken after `from` up to `to`.
579pub(crate) fn samples_between(samples: &[Sample], from: u64, to: u64) -> Vec<&Sample> {
580    samples
581        .iter()
582        .filter(|sample| from < sample.at && sample.at <= to)
583        .collect()
584}
585
586/// What `inside` say together: their highest memory and their CPU averaged; `None` for no sample.
587pub(crate) fn usage_of(inside: &[&Sample]) -> Option<SampleUsage> {
588    let count = u64::try_from(inside.len())
589        .ok()
590        .filter(|count| *count > 0)?;
591    Some(SampleUsage {
592        memory_peak: inside.iter().map(|sample| sample.memory).max()?,
593        cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
594    })
595}
596
597/// What a container used over a stretch of its run, from [`SampleSeries::usage_between`].
598#[derive(Debug, Clone, Copy, PartialEq, Eq)]
599pub struct SampleUsage {
600    /// The highest memory of the samples, without page cache.
601    pub memory_peak: u64,
602    /// The samples' CPU use averaged, in millicores.
603    pub cpu_average: u64,
604}
605
606impl SampleUsage {
607    /// `peak 4.1G`.
608    pub fn memory(&self) -> String {
609        format!("peak {}", crate::config::ByteSize(self.memory_peak))
610    }
611
612    /// `average 1.5 cores`.
613    pub fn cpu(&self) -> String {
614        format!("average {}", super::super::in_cores(self.cpu_average))
615    }
616
617    /// Both, `peak 4.1G, average 1.5 cores`.
618    pub fn description(&self) -> String {
619        format!("{}, {}", self.memory(), self.cpu())
620    }
621}
622
623#[cfg(test)]
624mod tests;