1use std::borrow::Cow;
7use std::fmt;
8
9use super::{Job, JobState, JobSummary, Sample, SampleSeries, Step, Timeline};
10use crate::api::{QueuedJob, Record};
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum OutcomeKind {
15 Succeeded,
17 Failed,
19 RunnerFault,
22 Running,
24 Waiting,
26 Ended,
28}
29
30#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct Outcome {
33 pub label: Cow<'static, str>,
35 pub kind: OutcomeKind,
37}
38
39impl Outcome {
40 fn new(label: &'static str, kind: OutcomeKind) -> Outcome {
41 Outcome {
42 label: Cow::Borrowed(label),
43 kind,
44 }
45 }
46
47 fn concluded(conclusion: &str) -> Outcome {
49 let kind = match conclusion {
50 "success" => OutcomeKind::Succeeded,
51 failed if failed_conclusion(failed) => OutcomeKind::Failed,
52 _ => OutcomeKind::Ended,
53 };
54 Outcome {
55 label: Cow::Owned(conclusion.replace('_', " ")),
56 kind,
57 }
58 }
59}
60
61fn failed_conclusion(conclusion: &str) -> bool {
63 matches!(conclusion, "failure" | "timed_out")
64}
65
66impl JobSummary {
67 pub fn is_failure(&self) -> bool {
70 match self.state {
71 JobState::Lost => true,
72 JobState::Completed => self.conclusion.as_deref().is_some_and(failed_conclusion),
73 JobState::Queued | JobState::Running | JobState::Elsewhere => false,
74 }
75 }
76
77 pub fn outcome(&self) -> Outcome {
82 let started = self.timeline.started_at.is_some();
83 match (self.state, self.conclusion.as_deref()) {
84 (JobState::Queued, _) => Outcome::new("queued", OutcomeKind::Waiting),
85 (JobState::Running, _) => Outcome::new("running", OutcomeKind::Running),
86 (JobState::Lost, _) => Outcome::new("lost", OutcomeKind::Failed),
87 (JobState::Elsewhere, Some("cancelled")) if !started => {
88 Outcome::new("cancelled in queue", OutcomeKind::Ended)
89 }
90 (JobState::Elsewhere, Some(conclusion)) if !started => Outcome {
91 kind: OutcomeKind::Ended,
92 ..Outcome::concluded(conclusion)
93 },
94 (JobState::Elsewhere, _) => Outcome::new("elsewhere", OutcomeKind::Ended),
95 (JobState::Completed, None) => Outcome::new("completed", OutcomeKind::Ended),
96 (JobState::Completed, Some(conclusion))
97 if self.runner_fault && failed_conclusion(conclusion) =>
98 {
99 Outcome::new("runner fault", OutcomeKind::RunnerFault)
100 }
101 (JobState::Completed, Some(conclusion)) => Outcome::concluded(conclusion),
102 }
103 }
104
105 pub fn waited(&self, now: u64) -> Option<u64> {
108 let end = match self.timeline.started_at {
109 Some(started) => started,
110 None if !self.state.is_final() => now,
111 None => return None,
112 };
113 Some(end.saturating_sub(self.timeline.queued_at))
114 }
115
116 pub fn unseen_note(&self, now: u64) -> Option<String> {
119 let unseen = self.waited(now).map_or(self.timeline.unseen, |waited| {
120 waited.min(self.timeline.unseen)
121 });
122 (unseen > 0).then(|| {
123 format!(
124 "{} of it while runnerd was not running",
125 crate::api::duration(unseen)
126 )
127 })
128 }
129
130 pub fn cpu_held(&self, now: u64) -> Option<String> {
134 let metrics = &self.metrics;
135 let share = |part: u64, whole: u64| match part.saturating_mul(100) / whole.max(1) {
136 0 => "under 1%".to_owned(),
137 percent => format!("{}%", percent.min(100)),
138 };
139 let cap = metrics
140 .cpu_periods
141 .zip(metrics.cpu_throttled)
142 .filter(|(periods, _)| *periods > 0)
143 .map(|(periods, throttled)| match throttled {
144 0 => "never held at its CPU cap".to_owned(),
145 throttled => format!(
146 "held at its CPU cap in {} of its busy time",
147 share(throttled, periods)
148 ),
149 });
150 let waited = metrics
151 .cpu_stalled
152 .zip(self.ran(now).filter(|ran| *ran > 0))
153 .map(|(stalled, ran)| match stalled {
154 0 => "never waited for a CPU".to_owned(),
155 stalled => format!(
156 "waited for a CPU {} of its run",
157 share(stalled, ran.saturating_mul(1000))
158 ),
159 });
160 let parts: Vec<String> = cap.into_iter().chain(waited).collect();
161 (!parts.is_empty()).then(|| parts.join(", "))
162 }
163
164 pub fn ran(&self, now: u64) -> Option<u64> {
167 let started = self.timeline.started_at?;
168 let end = match self.timeline.finished_at {
169 Some(finished) => finished,
170 None if !self.state.is_final() => now,
171 None => return None,
172 };
173 Some(end.saturating_sub(started))
174 }
175
176 pub fn phases(&self) -> Vec<Phase> {
180 let timeline = &self.timeline;
181 let open = !self.state.is_final();
182 let until = |end: Option<u64>| match end {
184 Some(end) => Some(Some(end)),
185 None if open => Some(None),
186 None => timeline.finished_at.map(Some),
187 };
188 let runner = timeline.runner_from();
189 let mut phases = Vec::new();
190 if let Some(to) = until(runner.or(timeline.started_at)) {
191 phases.push(Phase {
192 kind: PhaseKind::Waiting,
193 from: timeline.queued_at,
194 to,
195 });
196 }
197 if let Some(from) = runner
198 && let Some(to) = until(timeline.started_at)
199 {
200 phases.push(Phase {
201 kind: PhaseKind::Starting,
202 from,
203 to,
204 });
205 }
206 if let Some(from) = timeline.started_at
207 && let Some(to) = timeline.finished_at.map(Some).or(open.then_some(None))
208 {
209 phases.push(Phase {
210 kind: PhaseKind::Running,
211 from,
212 to,
213 });
214 }
215 if let Some(from) = timeline.finished_at
216 && runner.is_some()
217 {
218 phases.push(Phase {
219 kind: PhaseKind::CleaningUp,
220 from,
221 to: timeline.cleaned_at,
222 });
223 }
224 phases
225 }
226
227 pub fn title(&self) -> JobTitle<'_> {
229 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
230 }
231}
232
233#[derive(Debug, Clone, Copy, PartialEq, Eq)]
235pub struct Phase {
236 pub kind: PhaseKind,
238 pub from: u64,
240 pub to: Option<u64>,
242}
243
244impl Phase {
245 pub fn seconds(&self, now: u64) -> u64 {
247 self.to.unwrap_or(now).saturating_sub(self.from)
248 }
249}
250
251#[derive(Debug, Clone, Copy, PartialEq, Eq)]
253pub enum PhaseKind {
254 Waiting,
256 Starting,
258 Running,
260 CleaningUp,
262}
263
264impl PhaseKind {
265 pub fn label(self) -> &'static str {
267 match self {
268 PhaseKind::Waiting => "Waiting for a runner",
269 PhaseKind::Starting => "Runner starting",
270 PhaseKind::Running => "Running",
271 PhaseKind::CleaningUp => "Cleaning up",
272 }
273 }
274}
275
276#[derive(Debug, Clone, Copy, PartialEq, Eq)]
278pub struct Milestone {
279 pub label: &'static str,
281 pub at: u64,
283}
284
285impl Timeline {
286 pub fn milestones(&self) -> Vec<Milestone> {
291 let mut milestones: Vec<Milestone> = [
292 ("Queued", Some(self.queued_at)),
293 (
294 "Seen by runnerd",
295 self.seen_at.map(|seen| seen.max(self.queued_at)),
296 ),
297 ("Container started", self.container_started_at),
298 ("Job started", self.started_at),
299 ("Finished", self.finished_at),
300 ("Cleaned up", self.cleaned_at),
301 ]
302 .into_iter()
303 .filter_map(|(label, at)| Some(Milestone { label, at: at? }))
304 .collect();
305 milestones.sort_by_key(|milestone| milestone.at);
306 milestones
307 }
308}
309
310impl Step {
311 pub fn outcome(&self) -> Outcome {
314 match (self.conclusion.as_deref(), self.status.as_str()) {
315 (Some(conclusion), _) => Outcome::concluded(conclusion),
316 (None, "in_progress") => Outcome::new("running", OutcomeKind::Running),
317 (None, "completed") => Outcome::new("completed", OutcomeKind::Ended),
318 (None, _) => Outcome::new("pending", OutcomeKind::Waiting),
319 }
320 }
321
322 pub fn is_running(&self) -> bool {
324 self.status == "in_progress" && self.conclusion.is_none()
325 }
326
327 pub fn took(&self, now: u64) -> Option<u64> {
330 let started = self.started_at?;
331 let end = match self.completed_at {
332 Some(completed) => completed,
333 None if self.is_running() => now,
334 None => return None,
335 };
336 Some(end.saturating_sub(started))
337 }
338}
339
340#[derive(Debug, Clone, PartialEq, Eq)]
342pub struct JobTitle<'job> {
343 pub workflow: Option<&'job str>,
345 pub name: Cow<'job, str>,
347}
348
349pub fn job_title<'job>(
352 workflow: Option<&'job str>,
353 name: &'job str,
354 id: Option<u64>,
355) -> JobTitle<'job> {
356 let name = match (name, id) {
357 ("", Some(id)) => Cow::Owned(format!("job {id}")),
358 ("", None) => Cow::Borrowed("a job"),
359 (name, _) => Cow::Borrowed(name),
360 };
361 JobTitle { workflow, name }
362}
363
364impl fmt::Display for JobTitle<'_> {
365 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
366 match self.workflow {
367 Some(workflow) => write!(f, "{workflow} / {}", self.name),
368 None => f.write_str(&self.name),
369 }
370 }
371}
372
373impl QueuedJob {
374 pub fn title(&self) -> JobTitle<'_> {
376 job_title(self.workflow.as_deref(), &self.name, Some(self.id))
377 }
378}
379
380impl Record {
381 pub fn title(&self) -> JobTitle<'_> {
383 job_title(
384 self.workflow.as_deref(),
385 self.name.as_deref().unwrap_or_default(),
386 Some(self.job),
387 )
388 }
389}
390
391impl Job {
392 pub fn observe(&mut self, sample: &Sample) {
395 let metrics = &mut self.summary.metrics;
396 if !self.summary.state.is_final() {
397 metrics.memory_peak = metrics.memory_peak.max(Some(sample.memory));
398 metrics.cpu_peak = metrics.cpu_peak.max(Some(sample.cpu));
399 }
400 }
401
402 pub fn reread(&mut self, read: Job) {
405 let seen = self.summary.metrics.clone();
406 *self = read;
407 if !self.summary.state.is_final() {
408 let metrics = &mut self.summary.metrics;
409 metrics.memory_peak = metrics.memory_peak.max(seen.memory_peak);
410 metrics.cpu_peak = metrics.cpu_peak.max(seen.cpu_peak);
411 }
412 }
413
414 pub fn compare(&self, runs: &[JobSummary], now: u64) -> RunComparison {
417 let runs: Vec<JobSummary> = runs
418 .iter()
419 .map(|run| {
420 if run.id == self.summary.id {
421 self.summary.clone()
422 } else {
423 run.clone()
424 }
425 })
426 .collect();
427 let finished: Vec<&JobSummary> = runs
429 .iter()
430 .filter(|run| {
431 matches!(run.state, JobState::Completed | JobState::Lost)
432 && run.timeline.started_at.is_some()
433 })
434 .collect();
435 let median = |mut values: Vec<u64>| {
437 values.sort_unstable();
438 values.get(values.len().saturating_sub(1) / 2).copied()
439 };
440 RunComparison {
441 median_ran: median(finished.iter().filter_map(|run| run.ran(now)).collect()),
442 median_waited: median(finished.iter().filter_map(|run| run.waited(now)).collect()),
443 finished: finished.len(),
444 failed: finished.iter().filter(|run| run.is_failure()).count(),
445 largest_peak: runs.iter().filter_map(|run| run.metrics.memory_peak).max(),
446 runs,
447 }
448 }
449}
450
451#[derive(Debug, Clone, PartialEq, Eq)]
453pub struct RunComparison {
454 pub runs: Vec<JobSummary>,
456 pub median_ran: Option<u64>,
458 pub median_waited: Option<u64>,
460 pub finished: usize,
462 pub failed: usize,
464 pub largest_peak: Option<u64>,
466}
467
468impl SampleSeries {
469 pub fn usage_between(&self, from: u64, to: u64) -> Option<SampleUsage> {
472 let inside: Vec<&Sample> = self
473 .samples
474 .iter()
475 .filter(|sample| from < sample.at && sample.at <= to)
476 .collect();
477 let count = u64::try_from(inside.len())
478 .ok()
479 .filter(|count| *count > 0)?;
480 Some(SampleUsage {
481 memory_peak: inside.iter().map(|sample| sample.memory).max()?,
482 cpu_average: inside.iter().map(|sample| sample.cpu).sum::<u64>() / count,
483 })
484 }
485}
486
487#[derive(Debug, Clone, Copy, PartialEq, Eq)]
489pub struct SampleUsage {
490 pub memory_peak: u64,
492 pub cpu_average: u64,
494}
495
496impl SampleUsage {
497 pub fn memory(&self) -> String {
499 format!("peak {}", crate::config::ByteSize(self.memory_peak))
500 }
501
502 pub fn cpu(&self) -> String {
504 format!("average {}", super::super::in_cores(self.cpu_average))
505 }
506
507 pub fn description(&self) -> String {
509 format!("{}, {}", self.memory(), self.cpu())
510 }
511}
512
513#[cfg(test)]
514mod tests;