1use std::borrow::Cow;
7use std::fmt;
8
9use super::{Job, JobState, JobSummary, Sample, SampleSeries, Step, Timeline};
10use crate::api::{QueuedJob, Record};
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum OutcomeKind {
15 Succeeded,
17 Failed,
19 Running,
21 Waiting,
23 Ended,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq)]
29pub struct Outcome {
30 pub label: Cow<'static, str>,
32 pub kind: OutcomeKind,
34}
35
36impl Outcome {
37 fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
38 Outcome {
39 label: Cow::Borrowed(label),
40 kind,
41 }
42 }
43
44 fn concluded(conclusion: &str) -> Outcome {
46 let kind = match conclusion {
47 "success" => OutcomeKind::Succeeded,
48 failed if failed_conclusion(failed) => OutcomeKind::Failed,
49 _ => OutcomeKind::Ended,
50 };
51 Outcome {
52 label: Cow::Owned(conclusion.replace('_', " ")),
53 kind,
54 }
55 }
56}
57
58fn failed_conclusion(conclusion: &str) -> bool {
60 matches!(conclusion, "failure" | "timed_out")
61}
62
63impl JobSummary {
64 pub fn is_failure(&self) -> bool {
67 match self.state {
68 JobState::Lost => true,
69 JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
70 JobState::Queued | JobState::Running | JobState::Elsewhere => false,
71 }
72 }
73
74 pub fn outcome(&self) -> Outcome {
78 let started = self.timeline.started_at.is_some();
79 match (self.state, self.conclusion.as_deref()) {
80 (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
81 (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
82 (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
83 (JobState::Elsewhere, Some("cancelled")) if !started => {
84 Outcome::new("cancelled in queue", OutcomeKind::Ended)
85 }
86 (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
87 kind: OutcomeKind::Ended,
88 ..Outcome::concluded(conclusion)
89 },
90 (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
91 (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
92 (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
93 }
94 }
95
96 pub fn waited(&self, now: u64) -> Option<u64> {
99 let end = match self.timeline.started_at {
100 Some(started) => started,
101 None if !self.state.is_final() => now,
102 None => return None,
103 };
104 Some(end.saturating_sub(self.timeline.queued_at))
105 }
106
107 pub fn unseen_note(&self, now: u64) -> Option<String> {
110 let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
111 waited.min(self.timeline.unseen)
112 });
113 (unseen > 0).then(|| {
114 format!(
115 "{} of it while runnerd was not running",
116 crate::api::duration(unseen)
117 )
118 })
119 }
120
121 pub fn cpu_held(&self, now: u64) -> Option<String> {
125 let metrics = &self.metrics;
126 let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
127 0 => "under 1%".to_owned(),
128 percent => format!("{}%", percent.min(100)),
129 };
130 let cap = metrics
131 .cpu_periods
132 .zip(metrics.cpu_throttled)
133 .filter(|(periods, _)| *periods > 0)
134 .map(|(periods, throttled)| match throttled {
135 0 => "never held at its CPU cap".to_owned(),
136 throttled => format!(
137 "held at its CPU cap in {} of its busy time",
138 share(throttled, periods)
139 ),
140 });
141 let waited = metrics
142 .cpu_stalled
143 .zip(self.ran(now).filter(|ran| *ran > 0))
144 .map(|(stalled, ran)| match stalled {
145 0 => "never waited for a CPU".to_owned(),
146 stalled => format!(
147 "waited for a CPU {} of its run",
148 share(stalled, ran.saturating_mul(1000))
149 ),
150 });
151 let parts: Vec<String> = cap.into_iter().chain(waited).collect();
152 (!parts.is_empty()).then(|| parts.join(", "))
153 }
154
155 pub fn ran(&self, now: u64) -> Option<u64> {
158 let started = self.timeline.started_at?;
159 let end = match self.timeline.finished_at {
160 Some(finished) => finished,
161 None if !self.state.is_final() => now,
162 None => return None,
163 };
164 Some(end.saturating_sub(started))
165 }
166
167 pub fn phases(&self) -> Vec<Phase> {
171 let timeline = &self.timeline;
172 let open = !self.state.is_final();
173 let until = |end: Option<u64>| match end {
175 Some(end) => Some(Some(end)),
176 None if open => Some(None),
177 None => timeline.finished_at.map(Some),
178 };
179 let runner = timeline.runner_from();
180 let mut phases = Vec::new();
181 if let Some(to) = until(runner.or(timeline.started_at)) {
182 phases.push(Phase {
183 kind: PhaseKind::Waiting,
184 from: timeline.queued_at,
185 to,
186 });
187 }
188 if let Some(from) = runner
189 && let Some(to) = until(timeline.started_at)
190 {
191 phases.push(Phase {
192 kind: PhaseKind::Starting,
193 from,
194 to,
195 });
196 }
197 if let Some(from) = timeline.started_at
198 && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
199 {
200 phases.push(Phase {
201 kind: PhaseKind::Running,
202 from,
203 to,
204 });
205 }
206 if let Some(from) = timeline.finished_at
207 && runner.is_some()
208 {
209 phases.push(Phase {
210 kind: PhaseKind::CleaningUp,
211 from,
212 to: timeline.cleaned_at,
213 });
214 }
215 phases
216 }
217
218 pub fn title(&self) -> JobTitle<'_> {
220 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
221 }
222}
223
224#[derive(Debug, Clone, Copy, PartialEq, Eq)]
226pub struct Phase {
227 pub kind: PhaseKind,
229 pub from: u64,
231 pub to: Option<u64>,
233}
234
235impl Phase {
236 pub fn seconds(&self, now: u64) -> u64 {
238 self.to.unwrap_or(now).saturating_sub(self.from)
239 }
240}
241
242#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub enum PhaseKind {
245 Waiting,
247 Starting,
249 Running,
251 CleaningUp,
253}
254
255impl PhaseKind {
256 pub fn label(self) -> &'static str {
258 match self {
259 PhaseKind::Waiting => "Waiting for a runner",
260 PhaseKind::Starting => "Runner starting",
261 PhaseKind::Running => "Running",
262 PhaseKind::CleaningUp => "Cleaning up",
263 }
264 }
265}
266
267#[derive(Debug, Clone, Copy, PartialEq, Eq)]
269pub struct Milestone {
270 pub label: &'static str,
272 pub at: u64,
274}
275
276impl Timeline {
277 pub fn milestones(&self) -> Vec<Milestone> {
282 let mut milestones: Vec<Milestone> = [
283 ("Queued", Some(self.queued_at)),
284 (
285 "Seen by runnerd",
286 self.seen_at.map(|seen| seen.max(self.queued_at)),
287 ),
288 ("Container started", self.container_started_at),
289 ("Job started", self.started_at),
290 ("Finished", self.finished_at),
291 ("Cleaned up", self.cleaned_at),
292 ]
293 .into_iter()
294 .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
295 .collect();
296 milestones.sort_by_key(|milestone| milestone.at);
297 milestones
298 }
299}
300
301impl Step {
302 pub fn outcome(&self) -> Outcome {
305 match (self.conclusion.as_deref(), self.status.as_str()) {
306 (Some(conclusion), _) => Outcome::concluded(conclusion),
307 (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
308 (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
309 (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
310 }
311 }
312
313 pub fn is_running(&self) -> bool {
315 self.status == "in_progress" && self.conclusion.is_none()
316 }
317
318 pub fn took(&self, now: u64) -> Option<u64> {
321 let started = self.started_at?;
322 let end = match self.completed_at {
323 Some(completed) => completed,
324 None if self.is_running() => now,
325 None => return None,
326 };
327 Some(end.saturating_sub(started))
328 }
329}
330
331#[derive(Debug, Clone, PartialEq, Eq)]
333pub struct JobTitle<'job> {
334 pub workflow: Option<&'job str>,
336 pub name: Cow<'job, str>,
338}
339
340pub fn job_title<'job>(
343 workflow: Option<&'job str>,
344 name: &'job str,
345 id: Option<u64>,
346) -> JobTitle<'job> {
347 let name = match (name, id) {
348 ("", Some(id)) => Cow::Owned(format!("job {id}")),
349 ("", None) => Cow::Borrowed("a job"),
350 (name, _) => Cow::Borrowed(name),
351 };
352 JobTitle { workflow, name }
353}
354
355impl fmt::Display for JobTitle<'_> {
356 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
357 match self.workflow {
358 Some(workflow) => write!(f, "{workflow} / {}", self.name),
359 None => f.write_str(&self.name),
360 }
361 }
362}
363
364impl QueuedJob {
365 pub fn title(&self) -> JobTitle<'_> {
367 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
368 }
369}
370
371impl Record {
372 pub fn title(&self) -> JobTitle<'_> {
374 job_title(
375 self.workflow.as_deref(),
376 self.name.as_deref().unwrap_or_default(),
377 Some(self.job),
378 )
379 }
380}
381
382impl Job {
383 pub fn observe(&mut self, sample: &Sample) {
386 let metrics = &mut self.summary.metrics;
387 if !self.summary.state.is_final() {
388 metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
389 metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
390 }
391 }
392
393 pub fn reread(&mut self, read: Job) {
396 let seen = self.summary.metrics.clone();
397 *self = read;
398 if !self.summary.state.is_final() {
399 let metrics = &mut self.summary.metrics;
400 metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
401 metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
402 }
403 }
404
405 pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
408 let runs: Vec<JobSummary> = runs
409 .iter()
410 .map(|run| {
411 if run.id == self.summary.id {
412 self.summary.clone()
413 } else {
414 run.clone()
415 }
416 })
417 .collect();
418 let finished: Vec<&JobSummary> = runs
420 .iter()
421 .filter(|run| {
422 matches!(run.state, JobState::Completed | JobState::Lost)
423 && run.timeline.started_at.is_some()
424 })
425 .collect();
426 let median = |mut values: Vec<u64>| {
428 values.sort_unstable();
429 values.get(values.len().saturating_sub(1) / 2).copied()
430 };
431 RunComparison {
432 median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
433 median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
434 finished: finished.len(),
435 failed: finished.iter().filter(|run| run.is_failure()).count(),
436 largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
437 runs,
438 }
439 }
440}
441
442#[derive(Debug, Clone, PartialEq, Eq)]
444pub struct RunComparison {
445 pub runs: Vec<JobSummary>,
447 pub median_ran: Option<u64>,
449 pub median_waited: Option<u64>,
451 pub finished: usize,
453 pub failed: usize,
455 pub largest_peak: Option<u64>,
457}
458
459impl SampleSeries {
460 pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
463 let inside: Vec<&Sample> = self
464 .samples
465 .iter()
466 .filter(|sample| from < sample.at && sample.at <= to)
467 .collect();
468 let count = u64::try_from(inside.len())
469 .ok()
470 .filter(|count| *count > 0)?;
471 Some(SampleUsage {
472 memory_peak: inside.iter().map(|sample| sample.memory).max()?,
473 cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
474 })
475 }
476}
477
478#[derive(Debug, Clone, Copy, PartialEq, Eq)]
480pub struct SampleUsage {
481 pub memory_peak: u64,
483 pub cpu_average: u64,
485}
486
487impl SampleUsage {
488 pub fn memory(&self) -> String {
490 format!("peak {}", crate::config::ByteSize(self.memory_peak))
491 }
492
493 pub fn cpu(&self) -> String {
495 format!("average {}", super::super::in_cores(self.cpu_average))
496 }
497
498 pub fn description(&self) -> String {
500 format!("{}, {}", self.memory(), self.cpu())
501 }
502}
503
504#[cfg(test)]
505mod tests;