1use std::borrow::Cow;
7use std::fmt;
8
9use super::{
10 FailureCause, Job, JobState, JobSummary, RunnerProbe, Sample, SampleSeries, Step, Timeline,
11};
12use crate::api::{QueuedJob, Record, Tone, Toned};
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub enum OutcomeKind {
17 Succeeded,
19 Failed,
21 RunnerFault,
24 Running,
26 Waiting,
28 Ended,
30}
31
32#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct Outcome {
35 pub label: Cow<'static, str>,
37 pub kind: OutcomeKind,
39}
40
41impl Outcome {
42 fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
43 Outcome {
44 label: Cow::Borrowed(label),
45 kind,
46 }
47 }
48
49 fn concluded(conclusion: &str) -> Outcome {
51 let kind = match conclusion {
52 "success" => OutcomeKind::Succeeded,
53 failed if failed_conclusion(failed) => OutcomeKind::Failed,
54 _ => OutcomeKind::Ended,
55 };
56 Outcome {
57 label: Cow::Owned(conclusion.replace('_', " ")),
58 kind,
59 }
60 }
61}
62
63impl FailureCause {
64 pub fn label(self) -> Option<&'static str> {
67 match self {
68 FailureCause::ActionArchive => Some("action download"),
69 FailureCause::ActionResolve => Some("action resolution"),
70 FailureCause::Checkout => Some("checkout"),
71 FailureCause::Unknown => None,
72 }
73 }
74
75 pub fn line(self) -> Option<&'static str> {
77 match self {
78 FailureCause::ActionArchive => {
79 Some("the runner could not download or unpack an action while it set up the job")
80 }
81 FailureCause::ActionResolve => {
82 Some("the runner could not resolve an action with GitHub while it set up the job")
83 }
84 FailureCause::Checkout => Some("actions/checkout could not fetch the repository"),
85 FailureCause::Unknown => None,
86 }
87 }
88}
89
90fn failed_conclusion(conclusion: &str) -> bool {
92 matches!(conclusion, "failure" | "timed_out")
93}
94
95impl JobSummary {
96 pub fn is_failure(&self) -> bool {
99 match self.state {
100 JobState::Lost => true,
101 JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
102 JobState::Queued | JobState::Running | JobState::Elsewhere => false,
103 }
104 }
105
106 pub fn outcome(&self) -> Outcome {
111 outcome_of(
112 self.state,
113 self.conclusion.as_deref(),
114 self.timeline.started_at.is_some(),
115 self.runner_fault,
116 )
117 }
118
119 pub fn failure_line(&self) -> Option<&'static str> {
123 self.failure_cause
124 .filter(|_| self.is_failure())
125 .and_then(FailureCause::line)
126 }
127
128 pub fn rerun_line(&self) -> Option<String> {
131 self.rerun_at.map(|_| {
132 format!(
133 "runnerd re-ran the run's failed jobs as attempt {}, since the runner failed this one",
134 self.run_attempt.saturating_add(1)
135 )
136 })
137 }
138
139 pub fn probe_line(&self) -> Option<Toned> {
144 let found = |text: &str, tone| {
145 Some(Toned {
146 text: text.to_owned(),
147 tone,
148 })
149 };
150 match self.probe {
151 None if self.is_failure() => found("no probe verdict recorded", Tone::Quiet),
152 None | Some(RunnerProbe::Unknown) => None,
153 Some(RunnerProbe::Clean | RunnerProbe::Broken) if !self.is_failure() => None,
154 Some(RunnerProbe::Clean) => found(
155 "the probe's sampled mount and /proc calls succeeded",
156 Tone::Quiet,
157 ),
158 Some(RunnerProbe::Broken) => found("a probe after the job could not run", Tone::Quiet),
159 Some(RunnerProbe::Mount) => found(
160 "a probe after the job found the runner's mounts failing",
161 Tone::Warn,
162 ),
163 Some(RunnerProbe::Procfs) => found(
164 "a probe after the job found the runner's /proc failing to open",
165 Tone::Warn,
166 ),
167 Some(RunnerProbe::MountAndProcfs) => found(
168 "a probe after the job found the runner's mounts and /proc failing",
169 Tone::Warn,
170 ),
171 Some(RunnerProbe::TimedOut) => found(
172 "a probe after the job did not finish within 3 seconds",
173 Tone::Warn,
174 ),
175 }
176 }
177
178 pub fn waited(&self, now: u64) -> Option<u64> {
181 let end = match self.timeline.started_at {
182 Some(started) => started,
183 None if !self.state.is_final() => now,
184 None => return None,
185 };
186 Some(end.saturating_sub(self.timeline.queued_at))
187 }
188
189 pub fn unseen_note(&self, now: u64) -> Option<String> {
192 let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
193 waited.min(self.timeline.unseen)
194 });
195 (unseen > 0).then(|| {
196 format!(
197 "{} of it while runnerd was not running",
198 crate::api::duration(unseen)
199 )
200 })
201 }
202
203 pub fn cpu_held(&self, now: u64) -> Option<String> {
207 let metrics = &self.metrics;
208 let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
209 0 => "under 1%".to_owned(),
210 percent => format!("{}%", percent.min(100)),
211 };
212 let cap = metrics
213 .cpu_periods
214 .zip(metrics.cpu_throttled)
215 .filter(|(periods, _)| *periods > 0)
216 .map(|(periods, throttled)| match throttled {
217 0 => "never held at its CPU cap".to_owned(),
218 throttled => format!(
219 "held at its CPU cap in {} of its busy time",
220 share(throttled, periods)
221 ),
222 });
223 let waited = metrics
224 .cpu_stalled
225 .zip(self.ran(now).filter(|ran| *ran > 0))
226 .map(|(stalled, ran)| match stalled {
227 0 => "never waited for a CPU".to_owned(),
228 stalled => format!(
229 "waited for a CPU {} of its run",
230 share(stalled, ran.saturating_mul(1000))
231 ),
232 });
233 let parts: Vec<String> = cap.into_iter().chain(waited).collect();
234 (!parts.is_empty()).then(|| parts.join(", "))
235 }
236
237 pub fn ran(&self, now: u64) -> Option<u64> {
240 let started = self.timeline.started_at?;
241 let end = match self.timeline.finished_at {
242 Some(finished) => finished,
243 None if !self.state.is_final() => now,
244 None => return None,
245 };
246 Some(end.saturating_sub(started))
247 }
248
249 pub fn phases(&self) -> Vec<Phase> {
253 let timeline = &self.timeline;
254 let open = !self.state.is_final();
255 let until = |end: Option<u64>| match end {
257 Some(end) => Some(Some(end)),
258 None if open => Some(None),
259 None => timeline.finished_at.map(Some),
260 };
261 let runner = timeline.runner_from();
262 let mut phases = Vec::new();
263 if let Some(to) = until(runner.or(timeline.started_at)) {
264 phases.push(Phase {
265 kind: PhaseKind::Waiting,
266 from: timeline.queued_at,
267 to,
268 });
269 }
270 if let Some(from) = runner
271 && let Some(to) = until(timeline.started_at)
272 {
273 phases.push(Phase {
274 kind: PhaseKind::Starting,
275 from,
276 to,
277 });
278 }
279 if let Some(from) = timeline.started_at
280 && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
281 {
282 phases.push(Phase {
283 kind: PhaseKind::Running,
284 from,
285 to,
286 });
287 }
288 if let Some(from) = timeline.finished_at
289 && runner.is_some()
290 {
291 phases.push(Phase {
292 kind: PhaseKind::CleaningUp,
293 from,
294 to: timeline.cleaned_at,
295 });
296 }
297 phases
298 }
299
300 pub fn title(&self) -> JobTitle<'_> {
302 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
303 }
304}
305
306pub(crate) fn outcome_of(
308 state: JobState,
309 conclusion: Option<&str>,
310 started: bool,
311 runner_fault: bool,
312) -> Outcome {
313 match (state, conclusion) {
314 (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
315 (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
316 (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
317 (JobState::Elsewhere, Some("cancelled")) if !started => {
318 Outcome::new("cancelled in queue", OutcomeKind::Ended)
319 }
320 (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
321 kind: OutcomeKind::Ended,
322 ..Outcome::concluded(conclusion)
323 },
324 (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
325 (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
326 (JobState::Completed, Some(conclusion))
327 if runner_fault && failed_conclusion(conclusion) =>
328 {
329 Outcome::new("runner fault", OutcomeKind::RunnerFault)
330 }
331 (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
332 }
333}
334
335#[derive(Debug, Clone, Copy, PartialEq, Eq)]
337pub struct Phase {
338 pub kind: PhaseKind,
340 pub from: u64,
342 pub to: Option<u64>,
344}
345
346impl Phase {
347 pub fn seconds(&self, now: u64) -> u64 {
349 self.to.unwrap_or(now).saturating_sub(self.from)
350 }
351}
352
353#[derive(Debug, Clone, Copy, PartialEq, Eq)]
355pub enum PhaseKind {
356 Waiting,
358 Starting,
360 Running,
362 CleaningUp,
364}
365
366impl PhaseKind {
367 pub fn label(self) -> &'static str {
369 match self {
370 PhaseKind::Waiting => "Waiting for a runner",
371 PhaseKind::Starting => "Runner starting",
372 PhaseKind::Running => "Running",
373 PhaseKind::CleaningUp => "Cleaning up",
374 }
375 }
376}
377
378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub struct Milestone {
381 pub label: &'static str,
383 pub at: u64,
385}
386
387impl Timeline {
388 pub fn milestones(&self) -> Vec<Milestone> {
393 let mut milestones: Vec<Milestone> = [
394 ("Queued", Some(self.queued_at)),
395 (
396 "Seen by runnerd",
397 self.seen_at.map(|seen| seen.max(self.queued_at)),
398 ),
399 ("Container started", self.container_started_at),
400 ("Job started", self.started_at),
401 ("Finished", self.finished_at),
402 ("Cleaned up", self.cleaned_at),
403 ]
404 .into_iter()
405 .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
406 .collect();
407 milestones.sort_by_key(|milestone| milestone.at);
408 milestones
409 }
410}
411
412impl Step {
413 pub fn outcome(&self) -> Outcome {
416 match (self.conclusion.as_deref(), self.status.as_str()) {
417 (Some(conclusion), _) => Outcome::concluded(conclusion),
418 (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
419 (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
420 (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
421 }
422 }
423
424 pub fn is_running(&self) -> bool {
426 self.status == "in_progress" && self.conclusion.is_none()
427 }
428
429 pub fn took(&self, now: u64) -> Option<u64> {
432 let started = self.started_at?;
433 let end = match self.completed_at {
434 Some(completed) => completed,
435 None if self.is_running() => now,
436 None => return None,
437 };
438 Some(end.saturating_sub(started))
439 }
440}
441
442#[derive(Debug, Clone, PartialEq, Eq)]
444pub struct JobTitle<'job> {
445 pub workflow: Option<&'job str>,
447 pub name: Cow<'job, str>,
449}
450
451pub fn job_title<'job>(
454 workflow: Option<&'job str>,
455 name: &'job str,
456 id: Option<u64>,
457) -> JobTitle<'job> {
458 let name = match (name, id) {
459 ("", Some(id)) => Cow::Owned(format!("job {id}")),
460 ("", None) => Cow::Borrowed("a job"),
461 (name, _) => Cow::Borrowed(name),
462 };
463 JobTitle { workflow, name }
464}
465
466impl fmt::Display for JobTitle<'_> {
467 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
468 match self.workflow {
469 Some(workflow) => write!(f, "{workflow} / {}", self.name),
470 None => f.write_str(&self.name),
471 }
472 }
473}
474
475impl QueuedJob {
476 pub fn title(&self) -> JobTitle<'_> {
478 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
479 }
480}
481
482impl Record {
483 pub fn title(&self) -> JobTitle<'_> {
485 job_title(
486 self.workflow.as_deref(),
487 self.name.as_deref().unwrap_or_default(),
488 Some(self.job),
489 )
490 }
491}
492
493impl Job {
494 pub fn observe(&mut self, sample: &Sample) {
497 let metrics = &mut self.summary.metrics;
498 if !self.summary.state.is_final() {
499 metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
500 metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
501 }
502 }
503
504 pub fn reread(&mut self, read: Job) {
507 let seen = self.summary.metrics.clone();
508 *self = read;
509 if !self.summary.state.is_final() {
510 let metrics = &mut self.summary.metrics;
511 metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
512 metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
513 }
514 }
515
516 pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
519 let runs: Vec<JobSummary> = runs
520 .iter()
521 .map(|run| {
522 if run.id == self.summary.id {
523 self.summary.clone()
524 } else {
525 run.clone()
526 }
527 })
528 .collect();
529 let finished: Vec<&JobSummary> = runs
531 .iter()
532 .filter(|run| {
533 matches!(run.state, JobState::Completed | JobState::Lost)
534 && run.timeline.started_at.is_some()
535 })
536 .collect();
537 let median = |mut values: Vec<u64>| {
539 values.sort_unstable();
540 values.get(values.len().saturating_sub(1) / 2).copied()
541 };
542 RunComparison {
543 median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
544 median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
545 finished: finished.len(),
546 failed: finished.iter().filter(|run| run.is_failure()).count(),
547 largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
548 runs,
549 }
550 }
551}
552
553#[derive(Debug, Clone, PartialEq, Eq)]
555pub struct RunComparison {
556 pub runs: Vec<JobSummary>,
558 pub median_ran: Option<u64>,
560 pub median_waited: Option<u64>,
562 pub finished: usize,
564 pub failed: usize,
566 pub largest_peak: Option<u64>,
568}
569
570impl SampleSeries {
571 pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
574 usage_of(&samples_between(&self.samples, from, to))
575 }
576}
577
578pub(crate) fn samples_between(samples: &[Sample], from: u64, to: u64) -> Vec<&Sample> {
580 samples
581 .iter()
582 .filter(|sample| from < sample.at && sample.at <= to)
583 .collect()
584}
585
586pub(crate) fn usage_of(inside: &[&Sample]) -> Option<SampleUsage> {
588 let count = u64::try_from(inside.len())
589 .ok()
590 .filter(|count| *count > 0)?;
591 Some(SampleUsage {
592 memory_peak: inside.iter().map(|sample| sample.memory).max()?,
593 cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
594 })
595}
596
597#[derive(Debug, Clone, Copy, PartialEq, Eq)]
599pub struct SampleUsage {
600 pub memory_peak: u64,
602 pub cpu_average: u64,
604}
605
606impl SampleUsage {
607 pub fn memory(&self) -> String {
609 format!("peak {}", crate::config::ByteSize(self.memory_peak))
610 }
611
612 pub fn cpu(&self) -> String {
614 format!("average {}", super::super::in_cores(self.cpu_average))
615 }
616
617 pub fn description(&self) -> String {
619 format!("{}, {}", self.memory(), self.cpu())
620 }
621}
622
623#[cfg(test)]
624mod tests;