1use std::borrow::Cow;
7use std::fmt;
8
9use super::{Job, JobState, JobSummary, Sample, SampleSeries, Step, Timeline};
10use crate::api::{QueuedJob, Record};
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum OutcomeKind {
15 Succeeded,
17 Failed,
19 RunnerFault,
22 Running,
24 Waiting,
26 Ended,
28}
29
30#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct Outcome {
33 pub label: Cow<'static, str>,
35 pub kind: OutcomeKind,
37}
38
39impl Outcome {
40 fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
41 Outcome {
42 label: Cow::Borrowed(label),
43 kind,
44 }
45 }
46
47 fn concluded(conclusion: &str) -> Outcome {
49 let kind = match conclusion {
50 "success" => OutcomeKind::Succeeded,
51 failed if failed_conclusion(failed) => OutcomeKind::Failed,
52 _ => OutcomeKind::Ended,
53 };
54 Outcome {
55 label: Cow::Owned(conclusion.replace('_', " ")),
56 kind,
57 }
58 }
59}
60
61fn failed_conclusion(conclusion: &str) -> bool {
63 matches!(conclusion, "failure" | "timed_out")
64}
65
66impl JobSummary {
67 pub fn is_failure(&self) -> bool {
70 match self.state {
71 JobState::Lost => true,
72 JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
73 JobState::Queued | JobState::Running | JobState::Elsewhere => false,
74 }
75 }
76
77 pub fn outcome(&self) -> Outcome {
82 outcome_of(
83 self.state,
84 self.conclusion.as_deref(),
85 self.timeline.started_at.is_some(),
86 self.runner_fault,
87 )
88 }
89
90 pub fn waited(&self, now: u64) -> Option<u64> {
93 let end = match self.timeline.started_at {
94 Some(started) => started,
95 None if !self.state.is_final() => now,
96 None => return None,
97 };
98 Some(end.saturating_sub(self.timeline.queued_at))
99 }
100
101 pub fn unseen_note(&self, now: u64) -> Option<String> {
104 let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
105 waited.min(self.timeline.unseen)
106 });
107 (unseen > 0).then(|| {
108 format!(
109 "{} of it while runnerd was not running",
110 crate::api::duration(unseen)
111 )
112 })
113 }
114
115 pub fn cpu_held(&self, now: u64) -> Option<String> {
119 let metrics = &self.metrics;
120 let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
121 0 => "under 1%".to_owned(),
122 percent => format!("{}%", percent.min(100)),
123 };
124 let cap = metrics
125 .cpu_periods
126 .zip(metrics.cpu_throttled)
127 .filter(|(periods, _)| *periods > 0)
128 .map(|(periods, throttled)| match throttled {
129 0 => "never held at its CPU cap".to_owned(),
130 throttled => format!(
131 "held at its CPU cap in {} of its busy time",
132 share(throttled, periods)
133 ),
134 });
135 let waited = metrics
136 .cpu_stalled
137 .zip(self.ran(now).filter(|ran| *ran > 0))
138 .map(|(stalled, ran)| match stalled {
139 0 => "never waited for a CPU".to_owned(),
140 stalled => format!(
141 "waited for a CPU {} of its run",
142 share(stalled, ran.saturating_mul(1000))
143 ),
144 });
145 let parts: Vec<String> = cap.into_iter().chain(waited).collect();
146 (!parts.is_empty()).then(|| parts.join(", "))
147 }
148
149 pub fn ran(&self, now: u64) -> Option<u64> {
152 let started = self.timeline.started_at?;
153 let end = match self.timeline.finished_at {
154 Some(finished) => finished,
155 None if !self.state.is_final() => now,
156 None => return None,
157 };
158 Some(end.saturating_sub(started))
159 }
160
161 pub fn phases(&self) -> Vec<Phase> {
165 let timeline = &self.timeline;
166 let open = !self.state.is_final();
167 let until = |end: Option<u64>| match end {
169 Some(end) => Some(Some(end)),
170 None if open => Some(None),
171 None => timeline.finished_at.map(Some),
172 };
173 let runner = timeline.runner_from();
174 let mut phases = Vec::new();
175 if let Some(to) = until(runner.or(timeline.started_at)) {
176 phases.push(Phase {
177 kind: PhaseKind::Waiting,
178 from: timeline.queued_at,
179 to,
180 });
181 }
182 if let Some(from) = runner
183 && let Some(to) = until(timeline.started_at)
184 {
185 phases.push(Phase {
186 kind: PhaseKind::Starting,
187 from,
188 to,
189 });
190 }
191 if let Some(from) = timeline.started_at
192 && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
193 {
194 phases.push(Phase {
195 kind: PhaseKind::Running,
196 from,
197 to,
198 });
199 }
200 if let Some(from) = timeline.finished_at
201 && runner.is_some()
202 {
203 phases.push(Phase {
204 kind: PhaseKind::CleaningUp,
205 from,
206 to: timeline.cleaned_at,
207 });
208 }
209 phases
210 }
211
212 pub fn title(&self) -> JobTitle<'_> {
214 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
215 }
216}
217
218pub(crate) fn outcome_of(
220 state: JobState,
221 conclusion: Option<&str>,
222 started: bool,
223 runner_fault: bool,
224) -> Outcome {
225 match (state, conclusion) {
226 (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
227 (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
228 (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
229 (JobState::Elsewhere, Some("cancelled")) if !started => {
230 Outcome::new("cancelled in queue", OutcomeKind::Ended)
231 }
232 (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
233 kind: OutcomeKind::Ended,
234 ..Outcome::concluded(conclusion)
235 },
236 (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
237 (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
238 (JobState::Completed, Some(conclusion))
239 if runner_fault && failed_conclusion(conclusion) =>
240 {
241 Outcome::new("runner fault", OutcomeKind::RunnerFault)
242 }
243 (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
244 }
245}
246
247#[derive(Debug, Clone, Copy, PartialEq, Eq)]
249pub struct Phase {
250 pub kind: PhaseKind,
252 pub from: u64,
254 pub to: Option<u64>,
256}
257
258impl Phase {
259 pub fn seconds(&self, now: u64) -> u64 {
261 self.to.unwrap_or(now).saturating_sub(self.from)
262 }
263}
264
265#[derive(Debug, Clone, Copy, PartialEq, Eq)]
267pub enum PhaseKind {
268 Waiting,
270 Starting,
272 Running,
274 CleaningUp,
276}
277
278impl PhaseKind {
279 pub fn label(self) -> &'static str {
281 match self {
282 PhaseKind::Waiting => "Waiting for a runner",
283 PhaseKind::Starting => "Runner starting",
284 PhaseKind::Running => "Running",
285 PhaseKind::CleaningUp => "Cleaning up",
286 }
287 }
288}
289
290#[derive(Debug, Clone, Copy, PartialEq, Eq)]
292pub struct Milestone {
293 pub label: &'static str,
295 pub at: u64,
297}
298
299impl Timeline {
300 pub fn milestones(&self) -> Vec<Milestone> {
305 let mut milestones: Vec<Milestone> = [
306 ("Queued", Some(self.queued_at)),
307 (
308 "Seen by runnerd",
309 self.seen_at.map(|seen| seen.max(self.queued_at)),
310 ),
311 ("Container started", self.container_started_at),
312 ("Job started", self.started_at),
313 ("Finished", self.finished_at),
314 ("Cleaned up", self.cleaned_at),
315 ]
316 .into_iter()
317 .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
318 .collect();
319 milestones.sort_by_key(|milestone| milestone.at);
320 milestones
321 }
322}
323
324impl Step {
325 pub fn outcome(&self) -> Outcome {
328 match (self.conclusion.as_deref(), self.status.as_str()) {
329 (Some(conclusion), _) => Outcome::concluded(conclusion),
330 (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
331 (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
332 (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
333 }
334 }
335
336 pub fn is_running(&self) -> bool {
338 self.status == "in_progress" && self.conclusion.is_none()
339 }
340
341 pub fn took(&self, now: u64) -> Option<u64> {
344 let started = self.started_at?;
345 let end = match self.completed_at {
346 Some(completed) => completed,
347 None if self.is_running() => now,
348 None => return None,
349 };
350 Some(end.saturating_sub(started))
351 }
352}
353
354#[derive(Debug, Clone, PartialEq, Eq)]
356pub struct JobTitle<'job> {
357 pub workflow: Option<&'job str>,
359 pub name: Cow<'job, str>,
361}
362
363pub fn job_title<'job>(
366 workflow: Option<&'job str>,
367 name: &'job str,
368 id: Option<u64>,
369) -> JobTitle<'job> {
370 let name = match (name, id) {
371 ("", Some(id)) => Cow::Owned(format!("job {id}")),
372 ("", None) => Cow::Borrowed("a job"),
373 (name, _) => Cow::Borrowed(name),
374 };
375 JobTitle { workflow, name }
376}
377
378impl fmt::Display for JobTitle<'_> {
379 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
380 match self.workflow {
381 Some(workflow) => write!(f, "{workflow} / {}", self.name),
382 None => f.write_str(&self.name),
383 }
384 }
385}
386
387impl QueuedJob {
388 pub fn title(&self) -> JobTitle<'_> {
390 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
391 }
392}
393
394impl Record {
395 pub fn title(&self) -> JobTitle<'_> {
397 job_title(
398 self.workflow.as_deref(),
399 self.name.as_deref().unwrap_or_default(),
400 Some(self.job),
401 )
402 }
403}
404
405impl Job {
406 pub fn observe(&mut self, sample: &Sample) {
409 let metrics = &mut self.summary.metrics;
410 if !self.summary.state.is_final() {
411 metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
412 metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
413 }
414 }
415
416 pub fn reread(&mut self, read: Job) {
419 let seen = self.summary.metrics.clone();
420 *self = read;
421 if !self.summary.state.is_final() {
422 let metrics = &mut self.summary.metrics;
423 metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
424 metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
425 }
426 }
427
428 pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
431 let runs: Vec<JobSummary> = runs
432 .iter()
433 .map(|run| {
434 if run.id == self.summary.id {
435 self.summary.clone()
436 } else {
437 run.clone()
438 }
439 })
440 .collect();
441 let finished: Vec<&JobSummary> = runs
443 .iter()
444 .filter(|run| {
445 matches!(run.state, JobState::Completed | JobState::Lost)
446 && run.timeline.started_at.is_some()
447 })
448 .collect();
449 let median = |mut values: Vec<u64>| {
451 values.sort_unstable();
452 values.get(values.len().saturating_sub(1) / 2).copied()
453 };
454 RunComparison {
455 median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
456 median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
457 finished: finished.len(),
458 failed: finished.iter().filter(|run| run.is_failure()).count(),
459 largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
460 runs,
461 }
462 }
463}
464
465#[derive(Debug, Clone, PartialEq, Eq)]
467pub struct RunComparison {
468 pub runs: Vec<JobSummary>,
470 pub median_ran: Option<u64>,
472 pub median_waited: Option<u64>,
474 pub finished: usize,
476 pub failed: usize,
478 pub largest_peak: Option<u64>,
480}
481
482impl SampleSeries {
483 pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
486 usage_of(&samples_between(&self.samples, from, to))
487 }
488}
489
490pub(crate) fn samples_between(samples: &[Sample], from: u64, to: u64) -> Vec<&Sample> {
492 samples
493 .iter()
494 .filter(|sample| from < sample.at && sample.at <= to)
495 .collect()
496}
497
498pub(crate) fn usage_of(inside: &[&Sample]) -> Option<SampleUsage> {
500 let count = u64::try_from(inside.len())
501 .ok()
502 .filter(|count| *count > 0)?;
503 Some(SampleUsage {
504 memory_peak: inside.iter().map(|sample| sample.memory).max()?,
505 cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
506 })
507}
508
509#[derive(Debug, Clone, Copy, PartialEq, Eq)]
511pub struct SampleUsage {
512 pub memory_peak: u64,
514 pub cpu_average: u64,
516}
517
518impl SampleUsage {
519 pub fn memory(&self) -> String {
521 format!("peak {}", crate::config::ByteSize(self.memory_peak))
522 }
523
524 pub fn cpu(&self) -> String {
526 format!("average {}", super::super::in_cores(self.cpu_average))
527 }
528
529 pub fn description(&self) -> String {
531 format!("{}, {}", self.memory(), self.cpu())
532 }
533}
534
535#[cfg(test)]
536mod tests;