Skip to main content

runnerd_core/api/jobs/
facts.rs

1//! What every client shows of a job: its name, its outcome and its steps', how long it waited and ran, its phases
2//! and milestones, its figures as samples come in, what it used over a step, and how it compares with the same
3//! job's latest runs. Each returns meaning, a label, a figure or an [`OutcomeKind`]; the dashboard, the TUI and
4//! `runnerctl` only draw it, each colouring a kind its own way. Nothing here formats for one client.
5
6use std::borrow::Cow;
7use std::fmt;
8
9use super::{Job, JobState, JobSummary, Sample, SampleSeries, Step, Timeline};
10use crate::api::{QueuedJob, Record};
11
12/// What kind of outcome a job's or a step's is, for each client to colour as it colours that meaning.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum OutcomeKind {
15    /// It succeeded.
16    Succeeded,
17    /// It failed.
18    Failed,
19    /// It is under way.
20    Running,
21    /// It has not begun: waiting for a runner, or a step not reached yet.
22    Waiting,
23    /// It ended without succeeding or failing: cancelled, skipped, run elsewhere, or ended before any runner took it.
24    Ended,
25}
26
27/// A job's or a step's outcome: its word and what it means.
28#[derive(Debug, Clone, PartialEq, Eq)]
29pub struct Outcome {
30    /// Such as `success`, `timed out` or `running`.
31    pub label: Cow<'static, str>,
32    /// What it means.
33    pub kind: OutcomeKind,
34}
35
36impl Outcome {
37    fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
38        Outcome {
39            label: Cow::Borrowed(label),
40            kind,
41        }
42    }
43
44    /// A conclusion of GitHub's, its underscores as spaces: `timed_out` is `timed out`.
45    fn concluded(conclusion: &str) -> Outcome {
46        let kind = match conclusion {
47            "success" => OutcomeKind::Succeeded,
48            failed if failed_conclusion(failed) => OutcomeKind::Failed,
49            _ => OutcomeKind::Ended,
50        };
51        Outcome {
52            label: Cow::Owned(conclusion.replace('_', " ")),
53            kind,
54        }
55    }
56}
57
58/// The conclusions that count as a failure, as `GET /v1/stats` and the insights count them.
59fn failed_conclusion(conclusion: &str) -> bool {
60    matches!(conclusion, "failure" | "timed_out")
61}
62
63impl JobSummary {
64    /// Whether it failed, as `GET /v1/stats` counts failures: it concluded `failure` or `timed_out`, or it lost its
65    /// runner. A job another machine ran is not this daemon's to count.
66    pub fn is_failure(&self) -> bool {
67        match self.state {
68            JobState::Lost => true,
69            JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
70            JobState::Queued | JobState::Running | JobState::Elsewhere => false,
71        }
72    }
73
74    /// Its outcome: where it is while it can change, then how it ended. Its kind is [`OutcomeKind::Failed`] exactly
75    /// when it [failed](JobSummary::is_failure). A job that left the queue without starting, as one a newer push
76    /// cancelled, did not run elsewhere: it reads `cancelled in queue`, or as it concluded.
77    pub fn outcome(&self) -> Outcome {
78        let started = self.timeline.started_at.is_some();
79        match (self.state, self.conclusion.as_deref()) {
80            (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
81            (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
82            (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
83            (JobState::Elsewhere, Some("cancelled")) if !started => {
84                Outcome::new("cancelled in queue", OutcomeKind::Ended)
85            }
86            (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
87                kind: OutcomeKind::Ended,
88                ..Outcome::concluded(conclusion)
89            },
90            (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
91            (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
92            (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
93        }
94    }
95
96    /// Seconds it waited for a runner: until it started, or until `now` while it may still start. `None` for a job
97    /// that ended without starting here.
98    pub fn waited(&self, now: u64) -> Option<u64> {
99        let end = match self.timeline.started_at {
100            Some(started) => started,
101            None if !self.state.is_final() => now,
102            None => return None,
103        };
104        Some(end.saturating_sub(self.timeline.queued_at))
105    }
106
107    /// What of its wait at `now` passed while no runnerd ran, to follow it: `8h 10m of it while runnerd was not
108    /// running`, never more than the wait. `None` when none did.
109    pub fn unseen_note(&self, now: u64) -> Option<String> {
110        let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
111            waited.min(self.timeline.unseen)
112        });
113        (unseen > 0).then(|| {
114            format!(
115                "{} of it while runnerd was not running",
116                crate::api::duration(unseen)
117            )
118        })
119    }
120
121    /// What held its CPU back by `now`: `held at its CPU cap in 18% of its busy time, waited for a CPU 31% of its
122    /// run`. The cap's part is left out without a quota's figures or busy periods, the wait's without pressure stall
123    /// information or a run time; `None` when both are. A share that is not 0 never reads as 0%.
124    pub fn cpu_held(&self, now: u64) -> Option<String> {
125        let metrics = &self.metrics;
126        let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
127            0 => "under 1%".to_owned(),
128            percent => format!("{}%", percent.min(100)),
129        };
130        let cap = metrics
131            .cpu_periods
132            .zip(metrics.cpu_throttled)
133            .filter(|(periods, _)| *periods > 0)
134            .map(|(periods, throttled)| match throttled {
135                0 => "never held at its CPU cap".to_owned(),
136                throttled => format!(
137                    "held at its CPU cap in {} of its busy time",
138                    share(throttled, periods)
139                ),
140            });
141        let waited = metrics
142            .cpu_stalled
143            .zip(self.ran(now).filter(|ran| *ran > 0))
144            .map(|(stalled, ran)| match stalled {
145                0 => "never waited for a CPU".to_owned(),
146                stalled => format!(
147                    "waited for a CPU {} of its run",
148                    share(stalled, ran.saturating_mul(1000))
149                ),
150            });
151        let parts: Vec<String> = cap.into_iter().chain(waited).collect();
152        (!parts.is_empty()).then(|| parts.join(", "))
153    }
154
155    /// Seconds it ran: until it finished, or until `now` while it may still finish. `None` before it started, and
156    /// for a final job without a finish time, which no client claims ran for as long as it has been since.
157    pub fn ran(&self, now: u64) -> Option<u64> {
158        let started = self.timeline.started_at?;
159        let end = match self.timeline.finished_at {
160            Some(finished) => finished,
161            None if !self.state.is_final() => now,
162            None => return None,
163        };
164        Some(end.saturating_sub(started))
165    }
166
167    /// Its phases in order, each from when it began to when it ended, `None` while it goes on. A phase without an
168    /// end goes on while the job can change; a final job's ends when the job finished, or is left out when no time
169    /// says how long it took. Cleaning up follows the finish of a job whose runner started here, until its cleanup.
170    pub fn phases(&self) -> Vec<Phase> {
171        let timeline = &self.timeline;
172        let open = !self.state.is_final();
173        // `Some(end)` for a phase to show, `None` for one to leave out.
174        let until = |end: Option<u64>| match end {
175            Some(end) => Some(Some(end)),
176            None if open => Some(None),
177            None => timeline.finished_at.map(Some),
178        };
179        let runner = timeline.runner_from();
180        let mut phases = Vec::new();
181        if let Some(to) = until(runner.or(timeline.started_at)) {
182            phases.push(Phase {
183                kind: PhaseKind::Waiting,
184                from: timeline.queued_at,
185                to,
186            });
187        }
188        if let Some(from) = runner
189            && let Some(to) = until(timeline.started_at)
190        {
191            phases.push(Phase {
192                kind: PhaseKind::Starting,
193                from,
194                to,
195            });
196        }
197        if let Some(from) = timeline.started_at
198            && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
199        {
200            phases.push(Phase {
201                kind: PhaseKind::Running,
202                from,
203                to,
204            });
205        }
206        if let Some(from) = timeline.finished_at
207            && runner.is_some()
208        {
209            phases.push(Phase {
210                kind: PhaseKind::CleaningUp,
211                from,
212                to: timeline.cleaned_at,
213            });
214        }
215        phases
216    }
217
218    /// Its name as every client writes it: `workflow / job`.
219    pub fn title(&self) -> JobTitle<'_> {
220        job_title(self.workflow.as_deref(), &self.name, Some(self.id))
221    }
222}
223
224/// A stretch of a job's life.
225#[derive(Debug, Clone, Copy, PartialEq, Eq)]
226pub struct Phase {
227    /// Which.
228    pub kind: PhaseKind,
229    /// When it began.
230    pub from: u64,
231    /// When it ended; `None` while it goes on.
232    pub to: Option<u64>,
233}
234
235impl Phase {
236    /// How long it took, or has taken by `now`.
237    pub fn seconds(&self, now: u64) -> u64 {
238        self.to.unwrap_or(now).saturating_sub(self.from)
239    }
240}
241
242/// The phases of a job, in the order they happen.
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub enum PhaseKind {
245    /// Queued, without a runner.
246    Waiting,
247    /// A runner took it and is starting.
248    Starting,
249    /// The runner runs it.
250    Running,
251    /// It finished; its container and registration are being removed.
252    CleaningUp,
253}
254
255impl PhaseKind {
256    /// Its name, as every client labels it.
257    pub fn label(self) -> &'static str {
258        match self {
259            PhaseKind::Waiting => "Waiting for a runner",
260            PhaseKind::Starting => "Runner starting",
261            PhaseKind::Running => "Running",
262            PhaseKind::CleaningUp => "Cleaning up",
263        }
264    }
265}
266
267/// A moment of a job's life.
268#[derive(Debug, Clone, Copy, PartialEq, Eq)]
269pub struct Milestone {
270    /// Such as `Container started`.
271    pub label: &'static str,
272    /// When.
273    pub at: u64,
274}
275
276impl Timeline {
277    /// The moments it has times for, earliest first: a warm runner's container started before the job was queued.
278    /// When runnerd first listed it splits its wait in three (ADR 0092): before it, runnerd had not seen the job;
279    /// from it to its container's start, runnerd had; then the runner booted. A listing never comes before the
280    /// queueing, however GitHub's clock and this machine's differ.
281    pub fn milestones(&self) -> Vec<Milestone> {
282        let mut milestones: Vec<Milestone> = [
283            ("Queued", Some(self.queued_at)),
284            (
285                "Seen by runnerd",
286                self.seen_at.map(|seen| seen.max(self.queued_at)),
287            ),
288            ("Container started", self.container_started_at),
289            ("Job started", self.started_at),
290            ("Finished", self.finished_at),
291            ("Cleaned up", self.cleaned_at),
292        ]
293        .into_iter()
294        .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
295        .collect();
296        milestones.sort_by_key(|milestone| milestone.at);
297        milestones
298    }
299}
300
301impl Step {
302    /// Its outcome: how it concluded, else whether it runs or waits. A step that timed out failed, one that was
303    /// cancelled or skipped did not.
304    pub fn outcome(&self) -> Outcome {
305        match (self.conclusion.as_deref(), self.status.as_str()) {
306            (Some(conclusion), _) => Outcome::concluded(conclusion),
307            (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
308            (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
309            (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
310        }
311    }
312
313    /// Whether it runs now.
314    pub fn is_running(&self) -> bool {
315        self.status == "in_progress" && self.conclusion.is_none()
316    }
317
318    /// Seconds it took, until `now` while it runs; `None` before it started and for a step that ended without a
319    /// completion time.
320    pub fn took(&self, now: u64) -> Option<u64> {
321        let started = self.started_at?;
322        let end = match self.completed_at {
323            Some(completed) => completed,
324            None if self.is_running() => now,
325            None => return None,
326        };
327        Some(end.saturating_sub(started))
328    }
329}
330
331/// A job's name as every client writes it, in parts for a page that sets the workflow apart.
332#[derive(Debug, Clone, PartialEq, Eq)]
333pub struct JobTitle<'job> {
334    /// The workflow's name, when known.
335    pub workflow: Option<&'job str>,
336    /// The job's name, or `job <id>` until the daemon knows it.
337    pub name: Cow<'job, str>,
338}
339
340/// `workflow / job`, or the job's name alone; for a job whose name the daemon does not know yet, as for one its
341/// runner took before the daemon saw it queued, `job <id>`, or `a job` without an ID.
342pub fn job_title<'job>(
343    workflow: Option<&'job str>,
344    name: &'job str,
345    id: Option<u64>,
346) -> JobTitle<'job> {
347    let name = match (name, id) {
348        ("", Some(id)) => Cow::Owned(format!("job {id}")),
349        ("", None) => Cow::Borrowed("a job"),
350        (name, _) => Cow::Borrowed(name),
351    };
352    JobTitle { workflow, name }
353}
354
355impl fmt::Display for JobTitle<'_> {
356    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
357        match self.workflow {
358            Some(workflow) => write!(f, "{workflow} / {}", self.name),
359            None => f.write_str(&self.name),
360        }
361    }
362}
363
364impl QueuedJob {
365    /// Its name as [`JobSummary::title`] writes a job's.
366    pub fn title(&self) -> JobTitle<'_> {
367        job_title(self.workflow.as_deref(), &self.name, Some(self.id))
368    }
369}
370
371impl Record {
372    /// Its job's name as [`JobSummary::title`] writes a job's.
373    pub fn title(&self) -> JobTitle<'_> {
374        job_title(
375            self.workflow.as_deref(),
376            self.name.as_deref().unwrap_or_default(),
377            Some(self.job),
378        )
379    }
380}
381
382impl Job {
383    /// Takes a sample of its container as it arrives: while the job runs, its peaks rise to it, high-water marks
384    /// that deleting old samples does not lower. A final job's figures are the daemon's totals and stay.
385    pub fn observe(&mut self, sample: &Sample) {
386        let metrics = &mut self.summary.metrics;
387        if !self.summary.state.is_final() {
388            metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
389            metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
390        }
391    }
392
393    /// Takes the job as read again. While it runs, its peaks keep the highest seen, since the store's lag the
394    /// samples a client saw arrive; once it is final, the read is the daemon's totals.
395    pub fn reread(&mut self, read: Job) {
396        let seen = self.summary.metrics.clone();
397        *self = read;
398        if !self.summary.state.is_final() {
399            let metrics = &mut self.summary.metrics;
400            metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
401            metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
402        }
403    }
404
405    /// How it compares with the same job's latest `runs`, oldest first as given: this job among them as it is now,
406    /// not as the runs were read, since it may have started, finished or peaked higher since.
407    pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
408        let runs: Vec<JobSummary> = runs
409            .iter()
410            .map(|run| {
411                if run.id == self.summary.id {
412                    self.summary.clone()
413                } else {
414                    run.clone()
415                }
416            })
417            .collect();
418        // Another machine's runs are not this daemon's to compare.
419        let finished: Vec<&JobSummary> = runs
420            .iter()
421            .filter(|run| {
422                matches!(run.state, JobState::Completed | JobState::Lost)
423                    && run.timeline.started_at.is_some()
424            })
425            .collect();
426        // The lower of the middle two for an even count: a duration one of the runs took.
427        let median = |mut values: Vec<u64>| {
428            values.sort_unstable();
429            values.get(values.len().saturating_sub(1) / 2).copied()
430        };
431        RunComparison {
432            median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
433            median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
434            finished: finished.len(),
435            failed: finished.iter().filter(|run| run.is_failure()).count(),
436            largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
437            runs,
438        }
439    }
440}
441
442/// A job's latest runs and what they say together, from [`Job::compare`].
443#[derive(Debug, Clone, PartialEq, Eq)]
444pub struct RunComparison {
445    /// Oldest first, this job as it is now.
446    pub runs: Vec<JobSummary>,
447    /// The median of the finished runs' run times.
448    pub median_ran: Option<u64>,
449    /// The median of the finished runs' waits.
450    pub median_waited: Option<u64>,
451    /// The runs that started and ended here: those the medians and failures count.
452    pub finished: usize,
453    /// Of those, the ones that [failed](JobSummary::is_failure).
454    pub failed: usize,
455    /// The largest memory peak of any run.
456    pub largest_peak: Option<u64>,
457}
458
459impl SampleSeries {
460    /// What the container used from `from` to `to`, from the samples taken after `from` up to `to`: each covers the
461    /// interval before it, so steps that meet at a second share no sample. `None` when none was taken in it.
462    pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
463        let inside: Vec<&Sample> = self
464            .samples
465            .iter()
466            .filter(|sample| from < sample.at && sample.at <= to)
467            .collect();
468        let count = u64::try_from(inside.len())
469            .ok()
470            .filter(|count| *count > 0)?;
471        Some(SampleUsage {
472            memory_peak: inside.iter().map(|sample| sample.memory).max()?,
473            cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
474        })
475    }
476}
477
478/// What a container used over a stretch of its run, from [`SampleSeries::usage_between`].
479#[derive(Debug, Clone, Copy, PartialEq, Eq)]
480pub struct SampleUsage {
481    /// The highest memory of the samples, without page cache.
482    pub memory_peak: u64,
483    /// The samples' CPU use averaged, in millicores.
484    pub cpu_average: u64,
485}
486
487impl SampleUsage {
488    /// `peak 4.1G`.
489    pub fn memory(&self) -> String {
490        format!("peak {}", crate::config::ByteSize(self.memory_peak))
491    }
492
493    /// `average 1.5 cores`.
494    pub fn cpu(&self) -> String {
495        format!("average {}", super::super::in_cores(self.cpu_average))
496    }
497
498    /// Both, `peak 4.1G, average 1.5 cores`.
499    pub fn description(&self) -> String {
500        format!("{}, {}", self.memory(), self.cpu())
501    }
502}
503
504#[cfg(test)]
505mod tests;