Skip to main content

runnerd_core/api/jobs/
facts.rs

1//! What every client shows of a job: its name, its outcome and its steps', how long it waited and ran, its phases
2//! and milestones, its figures as samples come in, what it used over a step, and how it compares with the same
3//! job's latest runs. Each returns meaning, a label, a figure or an [`OutcomeKind`]; the dashboard, the TUI and
4//! `runnerctl` only draw it, each colouring a kind its own way. Nothing here formats for one client.
5
6use std::borrow::Cow;
7use std::fmt;
8
9use super::{Job, JobState, JobSummary, Sample, SampleSeries, Step, Timeline};
10use crate::api::{QueuedJob, Record};
11
12/// What kind of outcome a job's or a step's is, for each client to colour as it colours that meaning.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum OutcomeKind {
15    /// It succeeded.
16    Succeeded,
17    /// It failed.
18    Failed,
19    /// It failed because the runner failed it, not its code: a job read as `runner fault` (`JobSummary::runner_fault`).
20    /// Only a job's outcome has it.
21    RunnerFault,
22    /// It is under way.
23    Running,
24    /// It has not begun: waiting for a runner, or a step not reached yet.
25    Waiting,
26    /// It ended without succeeding or failing: cancelled, skipped, run elsewhere, or ended before any runner took it.
27    Ended,
28}
29
30/// A job's or a step's outcome: its word and what it means.
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct Outcome {
33    /// Such as `success`, `timed out` or `running`.
34    pub label: Cow<'static, str>,
35    /// What it means.
36    pub kind: OutcomeKind,
37}
38
39impl Outcome {
40    fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
41        Outcome {
42            label: Cow::Borrowed(label),
43            kind,
44        }
45    }
46
47    /// A conclusion of GitHub's, its underscores as spaces: `timed_out` is `timed out`.
48    fn concluded(conclusion: &str) -> Outcome {
49        let kind = match conclusion {
50            "success" => OutcomeKind::Succeeded,
51            failed if failed_conclusion(failed) => OutcomeKind::Failed,
52            _ => OutcomeKind::Ended,
53        };
54        Outcome {
55            label: Cow::Owned(conclusion.replace('_', " ")),
56            kind,
57        }
58    }
59}
60
61/// The conclusions that count as a failure, as `GET /v1/stats` and the insights count them.
62fn failed_conclusion(conclusion: &str) -> bool {
63    matches!(conclusion, "failure" | "timed_out")
64}
65
66impl JobSummary {
67    /// Whether it failed, as `GET /v1/stats` counts failures: it concluded `failure` or `timed_out`, or it lost its
68    /// runner. A job another machine ran is not this daemon's to count.
69    pub fn is_failure(&self) -> bool {
70        match self.state {
71            JobState::Lost => true,
72            JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
73            JobState::Queued | JobState::Running | JobState::Elsewhere => false,
74        }
75    }
76
77    /// Its outcome: where it is while it can change, then how it ended. Its kind is [`OutcomeKind::Failed`] or
78    /// [`OutcomeKind::RunnerFault`] exactly when it [failed](JobSummary::is_failure); a job that concluded failed and
79    /// that the runner failed reads `runner fault` (`runner_fault`) and counts as failed all the same. A job that left the queue without starting, as one a newer push
80    /// cancelled, did not run elsewhere: it reads `cancelled in queue`, or as it concluded.
81    pub fn outcome(&self) -> Outcome {
82        outcome_of(
83            self.state,
84            self.conclusion.as_deref(),
85            self.timeline.started_at.is_some(),
86            self.runner_fault,
87        )
88    }
89
90    /// Seconds it waited for a runner: until it started, or until `now` while it may still start. `None` for a job
91    /// that ended without starting here.
92    pub fn waited(&self, now: u64) -> Option<u64> {
93        let end = match self.timeline.started_at {
94            Some(started) => started,
95            None if !self.state.is_final() => now,
96            None => return None,
97        };
98        Some(end.saturating_sub(self.timeline.queued_at))
99    }
100
101    /// What of its wait at `now` passed while no runnerd ran, to follow it: `8h 10m of it while runnerd was not
102    /// running`, never more than the wait. `None` when none did.
103    pub fn unseen_note(&self, now: u64) -> Option<String> {
104        let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
105            waited.min(self.timeline.unseen)
106        });
107        (unseen > 0).then(|| {
108            format!(
109                "{} of it while runnerd was not running",
110                crate::api::duration(unseen)
111            )
112        })
113    }
114
115    /// What held its CPU back by `now`: `held at its CPU cap in 18% of its busy time, waited for a CPU 31% of its
116    /// run`. The cap's part is left out without a quota's figures or busy periods, the wait's without pressure stall
117    /// information or a run time; `None` when both are. A share that is not 0 never reads as 0%.
118    pub fn cpu_held(&self, now: u64) -> Option<String> {
119        let metrics = &self.metrics;
120        let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
121            0 => "under 1%".to_owned(),
122            percent => format!("{}%", percent.min(100)),
123        };
124        let cap = metrics
125            .cpu_periods
126            .zip(metrics.cpu_throttled)
127            .filter(|(periods, _)| *periods > 0)
128            .map(|(periods, throttled)| match throttled {
129                0 => "never held at its CPU cap".to_owned(),
130                throttled => format!(
131                    "held at its CPU cap in {} of its busy time",
132                    share(throttled, periods)
133                ),
134            });
135        let waited = metrics
136            .cpu_stalled
137            .zip(self.ran(now).filter(|ran| *ran > 0))
138            .map(|(stalled, ran)| match stalled {
139                0 => "never waited for a CPU".to_owned(),
140                stalled => format!(
141                    "waited for a CPU {} of its run",
142                    share(stalled, ran.saturating_mul(1000))
143                ),
144            });
145        let parts: Vec<String> = cap.into_iter().chain(waited).collect();
146        (!parts.is_empty()).then(|| parts.join(", "))
147    }
148
149    /// Seconds it ran: until it finished, or until `now` while it may still finish. `None` before it started, and
150    /// for a final job without a finish time, which no client claims ran for as long as it has been since.
151    pub fn ran(&self, now: u64) -> Option<u64> {
152        let started = self.timeline.started_at?;
153        let end = match self.timeline.finished_at {
154            Some(finished) => finished,
155            None if !self.state.is_final() => now,
156            None => return None,
157        };
158        Some(end.saturating_sub(started))
159    }
160
161    /// Its phases in order, each from when it began to when it ended, `None` while it goes on. A phase without an
162    /// end goes on while the job can change; a final job's ends when the job finished, or is left out when no time
163    /// says how long it took. Cleaning up follows the finish of a job whose runner started here, until its cleanup.
164    pub fn phases(&self) -> Vec<Phase> {
165        let timeline = &self.timeline;
166        let open = !self.state.is_final();
167        // `Some(end)` for a phase to show, `None` for one to leave out.
168        let until = |end: Option<u64>| match end {
169            Some(end) => Some(Some(end)),
170            None if open => Some(None),
171            None => timeline.finished_at.map(Some),
172        };
173        let runner = timeline.runner_from();
174        let mut phases = Vec::new();
175        if let Some(to) = until(runner.or(timeline.started_at)) {
176            phases.push(Phase {
177                kind: PhaseKind::Waiting,
178                from: timeline.queued_at,
179                to,
180            });
181        }
182        if let Some(from) = runner
183            && let Some(to) = until(timeline.started_at)
184        {
185            phases.push(Phase {
186                kind: PhaseKind::Starting,
187                from,
188                to,
189            });
190        }
191        if let Some(from) = timeline.started_at
192            && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
193        {
194            phases.push(Phase {
195                kind: PhaseKind::Running,
196                from,
197                to,
198            });
199        }
200        if let Some(from) = timeline.finished_at
201            && runner.is_some()
202        {
203            phases.push(Phase {
204                kind: PhaseKind::CleaningUp,
205                from,
206                to: timeline.cleaned_at,
207            });
208        }
209        phases
210    }
211
212    /// Its name as every client writes it: `workflow / job`.
213    pub fn title(&self) -> JobTitle<'_> {
214        job_title(self.workflow.as_deref(), &self.name, Some(self.id))
215    }
216}
217
218/// [`JobSummary::outcome`] of a job in `state`, concluded `conclusion`, that `started` here or not.
219pub(crate) fn outcome_of(
220    state: JobState,
221    conclusion: Option<&str>,
222    started: bool,
223    runner_fault: bool,
224) -> Outcome {
225    match (state, conclusion) {
226        (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
227        (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
228        (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
229        (JobState::Elsewhere, Some("cancelled")) if !started => {
230            Outcome::new("cancelled in queue", OutcomeKind::Ended)
231        }
232        (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
233            kind: OutcomeKind::Ended,
234            ..Outcome::concluded(conclusion)
235        },
236        (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
237        (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
238        (JobState::Completed, Some(conclusion))
239            if runner_fault && failed_conclusion(conclusion) =>
240        {
241            Outcome::new("runner fault", OutcomeKind::RunnerFault)
242        }
243        (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
244    }
245}
246
247/// A stretch of a job's life.
248#[derive(Debug, Clone, Copy, PartialEq, Eq)]
249pub struct Phase {
250    /// Which.
251    pub kind: PhaseKind,
252    /// When it began.
253    pub from: u64,
254    /// When it ended; `None` while it goes on.
255    pub to: Option<u64>,
256}
257
258impl Phase {
259    /// How long it took, or has taken by `now`.
260    pub fn seconds(&self, now: u64) -> u64 {
261        self.to.unwrap_or(now).saturating_sub(self.from)
262    }
263}
264
265/// The phases of a job, in the order they happen.
266#[derive(Debug, Clone, Copy, PartialEq, Eq)]
267pub enum PhaseKind {
268    /// Queued, without a runner.
269    Waiting,
270    /// A runner took it and is starting.
271    Starting,
272    /// The runner runs it.
273    Running,
274    /// It finished; its container and registration are being removed.
275    CleaningUp,
276}
277
278impl PhaseKind {
279    /// Its name, as every client labels it.
280    pub fn label(self) -> &'static str {
281        match self {
282            PhaseKind::Waiting => "Waiting for a runner",
283            PhaseKind::Starting => "Runner starting",
284            PhaseKind::Running => "Running",
285            PhaseKind::CleaningUp => "Cleaning up",
286        }
287    }
288}
289
290/// A moment of a job's life.
291#[derive(Debug, Clone, Copy, PartialEq, Eq)]
292pub struct Milestone {
293    /// Such as `Container started`.
294    pub label: &'static str,
295    /// When.
296    pub at: u64,
297}
298
299impl Timeline {
300    /// The moments it has times for, earliest first: a warm runner's container started before the job was queued.
301    /// When runnerd first listed it splits its wait in three (ADR 0092): before it, runnerd had not seen the job;
302    /// from it to its container's start, runnerd had; then the runner booted. A listing never comes before the
303    /// queueing, however GitHub's clock and this machine's differ.
304    pub fn milestones(&self) -> Vec<Milestone> {
305        let mut milestones: Vec<Milestone> = [
306            ("Queued", Some(self.queued_at)),
307            (
308                "Seen by runnerd",
309                self.seen_at.map(|seen| seen.max(self.queued_at)),
310            ),
311            ("Container started", self.container_started_at),
312            ("Job started", self.started_at),
313            ("Finished", self.finished_at),
314            ("Cleaned up", self.cleaned_at),
315        ]
316        .into_iter()
317        .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
318        .collect();
319        milestones.sort_by_key(|milestone| milestone.at);
320        milestones
321    }
322}
323
324impl Step {
325    /// Its outcome: how it concluded, else whether it runs or waits. A step that timed out failed, one that was
326    /// cancelled or skipped did not.
327    pub fn outcome(&self) -> Outcome {
328        match (self.conclusion.as_deref(), self.status.as_str()) {
329            (Some(conclusion), _) => Outcome::concluded(conclusion),
330            (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
331            (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
332            (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
333        }
334    }
335
336    /// Whether it runs now.
337    pub fn is_running(&self) -> bool {
338        self.status == "in_progress" && self.conclusion.is_none()
339    }
340
341    /// Seconds it took, until `now` while it runs; `None` before it started and for a step that ended without a
342    /// completion time.
343    pub fn took(&self, now: u64) -> Option<u64> {
344        let started = self.started_at?;
345        let end = match self.completed_at {
346            Some(completed) => completed,
347            None if self.is_running() => now,
348            None => return None,
349        };
350        Some(end.saturating_sub(started))
351    }
352}
353
354/// A job's name as every client writes it, in parts for a page that sets the workflow apart.
355#[derive(Debug, Clone, PartialEq, Eq)]
356pub struct JobTitle<'job> {
357    /// The workflow's name, when known.
358    pub workflow: Option<&'job str>,
359    /// The job's name, or `job <id>` until the daemon knows it.
360    pub name: Cow<'job, str>,
361}
362
363/// `workflow / job`, or the job's name alone; for a job whose name the daemon does not know yet, as for one its
364/// runner took before the daemon saw it queued, `job <id>`, or `a job` without an ID.
365pub fn job_title<'job>(
366    workflow: Option<&'job str>,
367    name: &'job str,
368    id: Option<u64>,
369) -> JobTitle<'job> {
370    let name = match (name, id) {
371        ("", Some(id)) => Cow::Owned(format!("job {id}")),
372        ("", None) => Cow::Borrowed("a job"),
373        (name, _) => Cow::Borrowed(name),
374    };
375    JobTitle { workflow, name }
376}
377
378impl fmt::Display for JobTitle<'_> {
379    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
380        match self.workflow {
381            Some(workflow) => write!(f, "{workflow} / {}", self.name),
382            None => f.write_str(&self.name),
383        }
384    }
385}
386
387impl QueuedJob {
388    /// Its name as [`JobSummary::title`] writes a job's.
389    pub fn title(&self) -> JobTitle<'_> {
390        job_title(self.workflow.as_deref(), &self.name, Some(self.id))
391    }
392}
393
394impl Record {
395    /// Its job's name as [`JobSummary::title`] writes a job's.
396    pub fn title(&self) -> JobTitle<'_> {
397        job_title(
398            self.workflow.as_deref(),
399            self.name.as_deref().unwrap_or_default(),
400            Some(self.job),
401        )
402    }
403}
404
405impl Job {
406    /// Takes a sample of its container as it arrives: while the job runs, its peaks rise to it, high-water marks
407    /// that deleting old samples does not lower. A final job's figures are the daemon's totals and stay.
408    pub fn observe(&mut self, sample: &Sample) {
409        let metrics = &mut self.summary.metrics;
410        if !self.summary.state.is_final() {
411            metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
412            metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
413        }
414    }
415
416    /// Takes the job as read again. While it runs, its peaks keep the highest seen, since the store's lag the
417    /// samples a client saw arrive; once it is final, the read is the daemon's totals.
418    pub fn reread(&mut self, read: Job) {
419        let seen = self.summary.metrics.clone();
420        *self = read;
421        if !self.summary.state.is_final() {
422            let metrics = &mut self.summary.metrics;
423            metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
424            metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
425        }
426    }
427
428    /// How it compares with the same job's latest `runs`, oldest first as given: this job among them as it is now,
429    /// not as the runs were read, since it may have started, finished or peaked higher since.
430    pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
431        let runs: Vec<JobSummary> = runs
432            .iter()
433            .map(|run| {
434                if run.id == self.summary.id {
435                    self.summary.clone()
436                } else {
437                    run.clone()
438                }
439            })
440            .collect();
441        // Another machine's runs are not this daemon's to compare.
442        let finished: Vec<&JobSummary> = runs
443            .iter()
444            .filter(|run| {
445                matches!(run.state, JobState::Completed | JobState::Lost)
446                    && run.timeline.started_at.is_some()
447            })
448            .collect();
449        // The lower of the middle two for an even count: a duration one of the runs took.
450        let median = |mut values: Vec<u64>| {
451            values.sort_unstable();
452            values.get(values.len().saturating_sub(1) / 2).copied()
453        };
454        RunComparison {
455            median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
456            median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
457            finished: finished.len(),
458            failed: finished.iter().filter(|run| run.is_failure()).count(),
459            largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
460            runs,
461        }
462    }
463}
464
465/// A job's latest runs and what they say together, from [`Job::compare`].
466#[derive(Debug, Clone, PartialEq, Eq)]
467pub struct RunComparison {
468    /// Oldest first, this job as it is now.
469    pub runs: Vec<JobSummary>,
470    /// The median of the finished runs' run times.
471    pub median_ran: Option<u64>,
472    /// The median of the finished runs' waits.
473    pub median_waited: Option<u64>,
474    /// The runs that started and ended here: those the medians and failures count.
475    pub finished: usize,
476    /// Of those, the ones that [failed](JobSummary::is_failure).
477    pub failed: usize,
478    /// The largest memory peak of any run.
479    pub largest_peak: Option<u64>,
480}
481
482impl SampleSeries {
483    /// What the container used from `from` to `to`, from the samples taken after `from` up to `to`: each covers the
484    /// interval before it, so steps that meet at a second share no sample. `None` when none was taken in it.
485    pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
486        usage_of(&samples_between(&self.samples, from, to))
487    }
488}
489
490/// The samples of `samples` taken after `from` up to `to`.
491pub(crate) fn samples_between(samples: &[Sample], from: u64, to: u64) -> Vec<&Sample> {
492    samples
493        .iter()
494        .filter(|sample| from < sample.at && sample.at <= to)
495        .collect()
496}
497
498/// What `inside` say together: their highest memory and their CPU averaged; `None` for no sample.
499pub(crate) fn usage_of(inside: &[&Sample]) -> Option<SampleUsage> {
500    let count = u64::try_from(inside.len())
501        .ok()
502        .filter(|count| *count > 0)?;
503    Some(SampleUsage {
504        memory_peak: inside.iter().map(|sample| sample.memory).max()?,
505        cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
506    })
507}
508
509/// What a container used over a stretch of its run, from [`SampleSeries::usage_between`].
510#[derive(Debug, Clone, Copy, PartialEq, Eq)]
511pub struct SampleUsage {
512    /// The highest memory of the samples, without page cache.
513    pub memory_peak: u64,
514    /// The samples' CPU use averaged, in millicores.
515    pub cpu_average: u64,
516}
517
518impl SampleUsage {
519    /// `peak 4.1G`.
520    pub fn memory(&self) -> String {
521        format!("peak {}", crate::config::ByteSize(self.memory_peak))
522    }
523
524    /// `average 1.5 cores`.
525    pub fn cpu(&self) -> String {
526        format!("average {}", super::super::in_cores(self.cpu_average))
527    }
528
529    /// Both, `peak 4.1G, average 1.5 cores`.
530    pub fn description(&self) -> String {
531        format!("{}, {}", self.memory(), self.cpu())
532    }
533}
534
535#[cfg(test)]
536mod tests;