runnerd_core/api/jobs.rs
1//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
2//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
3//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).
4
5use serde::{Deserialize, Serialize};
6
7mod facts;
8
9pub use facts::{
10 JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
11 job_title,
12};
13
14/// Where a job is in its life, as this daemon saw it.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "kebab-case")]
17pub enum JobState {
18 /// Waiting for a runner.
19 Queued,
20 /// A runner of this daemon took it.
21 Running,
22 /// GitHub reports it completed; `conclusion` says how.
23 Completed,
24 /// Its runner's container is gone and GitHub did not report the job completed in time.
25 Lost,
26 /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
27 /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
28 Elsewhere,
29}
30
31impl JobState {
32 /// Whether the job can still change.
33 pub fn is_final(self) -> bool {
34 matches!(
35 self,
36 JobState::Completed | JobState::Lost | JobState::Elsewhere
37 )
38 }
39}
40
41/// A job without its steps: what lists and events carry.
42#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
43pub struct JobSummary {
44 /// GitHub's job ID, also its check run ID.
45 pub id: u64,
46 /// `owner/name` on GitHub.
47 pub repository: String,
48 /// The workflow run it belongs to.
49 pub run_id: u64,
50 /// 1 for the first attempt of the run, higher for re-runs.
51 pub run_attempt: u32,
52 /// The workflow's name.
53 pub workflow: Option<String>,
54 /// The job's name in the workflow.
55 pub name: String,
56 /// The branch the run is for.
57 pub branch: Option<String>,
58 /// The job's page on GitHub.
59 pub url: Option<String>,
60 /// The workflow's `runs-on` labels.
61 pub labels: Vec<String>,
62 /// Where the job is.
63 pub state: JobState,
64 /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
65 pub conclusion: Option<String>,
66 /// The runner that ran it.
67 pub runner: Option<JobRunner>,
68 /// When each phase started.
69 pub timeline: Timeline,
70 /// CPU, memory and network use while it ran.
71 pub metrics: JobMetrics,
72 /// How much of its log is stored.
73 pub log: LogSummary,
74}
75
76/// `GET /v1/jobs/{id}`: a job with its steps.
77#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
78pub struct Job {
79 /// Everything but the steps.
80 #[serde(flatten)]
81 pub summary: JobSummary,
82 /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
83 pub steps: Vec<Step>,
84 /// Its record, for a failed job of a profile that records failures.
85 #[serde(default)]
86 pub record: Option<super::RecordNote>,
87 /// How its runner was admitted; `None` for one started before runnerd recorded it.
88 #[serde(default)]
89 pub admission: Option<super::Admission>,
90}
91
92impl Job {
93 /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
94 /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
95 /// limit as configured now. A reservation of the whole cap is one line.
96 pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
97 let limits = self.limits(profile);
98 let admission = self.admission.as_ref();
99 let name = profile.map_or("profile", |profile| profile.name.as_str());
100 ChartLines {
101 memory: chart_lines(
102 limits.memory,
103 admission.map(|admission| admission.reserved),
104 |value| crate::config::ByteSize(value).to_string(),
105 name,
106 ),
107 cpu: chart_lines(
108 limits.cpu,
109 admission
110 .and_then(|admission| admission.cpu)
111 .map(|cpu| cpu.reserved),
112 super::cpus,
113 name,
114 ),
115 }
116 }
117
118 /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
119 /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
120 /// reload that changes the profile never replaces a cap the record has.
121 pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
122 let runner = self.summary.runner.as_ref();
123 let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
124 (Some(value), _) => Some(Limit {
125 value,
126 source: LimitSource::Cap,
127 }),
128 (None, Some(value)) => Some(Limit {
129 value,
130 source: LimitSource::ProfileNow,
131 }),
132 (None, None) => None,
133 };
134 Limits {
135 memory: limit(
136 runner.and_then(|runner| runner.memory_cap),
137 profile.map(|profile| profile.memory),
138 ),
139 cpu: limit(
140 runner.and_then(|runner| runner.cpu_cap),
141 profile.map(|profile| u64::from(profile.cpus) * 1000),
142 ),
143 }
144 }
145}
146
147/// What a job's container may use, from `Job::limits`.
148#[derive(Debug, Clone, Copy, PartialEq, Eq)]
149pub struct Limits {
150 /// In bytes.
151 pub memory: Option<Limit>,
152 /// In millicores.
153 pub cpu: Option<Limit>,
154}
155
156impl Limits {
157 /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
158 /// now 6G and 4 CPUs`; `None` without either.
159 pub fn line(&self) -> Option<String> {
160 let memory = self.memory.map(|limit| {
161 (
162 limit.source,
163 crate::config::ByteSize(limit.value).to_string(),
164 )
165 });
166 let cpu = self
167 .cpu
168 .map(|limit| (limit.source, super::cpus(limit.value)));
169 let clause = |source: LimitSource, figures: &str| match source {
170 LimitSource::Cap => format!("capped at {figures}"),
171 LimitSource::ProfileNow => format!("profile now {figures}"),
172 };
173 match (memory, cpu) {
174 (Some((memory_source, memory)), Some((cpu_source, cpu)))
175 if memory_source == cpu_source =>
176 {
177 Some(clause(memory_source, &format!("{memory} and {cpu}")))
178 }
179 (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
180 "{}; {}",
181 clause(memory_source, &memory),
182 clause(cpu_source, &cpu)
183 )),
184 (Some((source, figure)), None) | (None, Some((source, figure))) => {
185 Some(clause(source, &figure))
186 }
187 (None, None) => None,
188 }
189 }
190}
191
192/// One resource's limit for a job's container.
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194pub struct Limit {
195 /// Bytes or millicores.
196 pub value: u64,
197 /// Where it comes from.
198 pub source: LimitSource,
199}
200
201/// Where a job's limit comes from.
202#[derive(Debug, Clone, Copy, PartialEq, Eq)]
203pub enum LimitSource {
204 /// The cap its container ran with.
205 Cap,
206 /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
207 /// changed it.
208 ProfileNow,
209}
210
211/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
212fn chart_lines(
213 limit: Option<Limit>,
214 reserved: Option<u64>,
215 value: fn(u64) -> String,
216 profile: &str,
217) -> Vec<ChartLine> {
218 let line = |figure, kind, label: &str| ChartLine {
219 value: figure,
220 kind,
221 label: format!("{label} {}", value(figure)),
222 };
223 let mut lines = vec![];
224 match limit {
225 Some(Limit {
226 value: cap,
227 source: LimitSource::Cap,
228 }) if reserved == Some(cap) => {
229 return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
230 }
231 Some(Limit {
232 value: cap,
233 source: LimitSource::Cap,
234 }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
235 Some(Limit {
236 value: now,
237 source: LimitSource::ProfileNow,
238 }) => lines.push(line(
239 now,
240 ChartLineKind::ProfileNow,
241 &format!("{profile} now"),
242 )),
243 None => {}
244 }
245 if let Some(reserved) = reserved {
246 lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
247 }
248 lines
249}
250
251/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
252#[derive(Debug, Clone, PartialEq, Eq)]
253pub struct ChartLines {
254 /// In bytes.
255 pub memory: Vec<ChartLine>,
256 /// In millicores.
257 pub cpu: Vec<ChartLine>,
258}
259
260/// A line across a job's memory or CPU chart.
261#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct ChartLine {
263 /// Bytes or millicores.
264 pub value: u64,
265 /// What it stands for.
266 pub kind: ChartLineKind,
267 /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
268 pub label: String,
269}
270
271/// What a line across a job's chart stands for.
272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
273pub enum ChartLineKind {
274 /// The most its container may use: its cap.
275 Cap,
276 /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
277 /// changed it.
278 ProfileNow,
279 /// What admission counted against the budget for its runner.
280 Reserved,
281 /// A reservation of the whole cap, as admission by caps makes.
282 CapAndReserved,
283}
284
285/// The runner a job ran on.
286#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
287pub struct JobRunner {
288 /// The runner's name, also its container's.
289 pub name: String,
290 /// The profile it ran in.
291 pub profile: String,
292 /// The slot it ran in, 1-based.
293 pub slot: usize,
294 /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
295 /// from before runnerd kept it.
296 pub memory_cap: Option<u64>,
297 /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
298 /// records from before runnerd kept it, or when Docker reports no limit.
299 pub cpu_cap: Option<u64>,
300}
301
302/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
303#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
304pub struct Timeline {
305 /// GitHub queued the job.
306 pub queued_at: u64,
307 /// The container of the runner that took it started.
308 pub container_started_at: Option<u64>,
309 /// The runner started the job.
310 pub started_at: Option<u64>,
311 /// The job finished, by GitHub's clock; or when its container was found gone.
312 pub finished_at: Option<u64>,
313 /// The container and the runner's registration were removed.
314 pub cleaned_at: Option<u64>,
315 /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
316 /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
317 #[serde(default)]
318 pub unseen: u64,
319}
320
321impl Timeline {
322 /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
323 /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
324 pub fn runner_from(&self) -> Option<u64> {
325 self.container_started_at
326 .map(|started| started.max(self.queued_at))
327 }
328}
329
330/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
331#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
332pub struct JobMetrics {
333 /// CPU time, in milliseconds.
334 pub cpu_time: Option<u64>,
335 /// The highest CPU use of a sample interval, in millicores.
336 pub cpu_peak: Option<u64>,
337 /// CPU time over wall time, in millicores.
338 pub cpu_average: Option<u64>,
339 /// The highest memory use without page cache.
340 pub memory_peak: Option<u64>,
341 /// The average memory use without page cache.
342 pub memory_average: Option<u64>,
343 /// Bytes the container received.
344 pub network_received: Option<u64>,
345 /// Bytes the container sent.
346 pub network_sent: Option<u64>,
347 /// Of the bytes received, those the Docker Hub mirror served.
348 pub mirror_bytes: u64,
349 /// What the shared build cache served it and took from it.
350 pub build_cache: BuildCacheCounts,
351 /// Times the kernel held its container's sockets' receive windows small because the container's memory was
352 /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
353 /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
354 #[serde(default, skip_serializing_if = "Option::is_none")]
355 pub sock_throttled: Option<u64>,
356 /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
357 /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
358 #[serde(default, skip_serializing_if = "Option::is_none")]
359 pub cpu_periods: Option<u64>,
360 /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
361 /// cap held it back.
362 #[serde(default, skip_serializing_if = "Option::is_none")]
363 pub cpu_throttled: Option<u64>,
364 /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
365 /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
366 #[serde(default, skip_serializing_if = "Option::is_none")]
367 pub cpu_stalled: Option<u64>,
368}
369
370/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
371#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
372pub struct BuildCacheCounts {
373 /// Lookups that found an entry.
374 pub hits: u64,
375 /// Lookups that found none.
376 pub misses: u64,
377 /// Bytes of the entries it served.
378 pub read_bytes: u64,
379 /// Bytes of the entries it stored.
380 pub written_bytes: u64,
381 /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
382 /// cache was full.
383 pub refused: u64,
384 /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
385 pub writer: bool,
386}
387
388impl BuildCacheCounts {
389 /// Hits and misses.
390 pub fn lookups(&self) -> u64 {
391 self.hits + self.misses
392 }
393
394 /// Hits in percent of lookups, `None` without any.
395 pub fn hit_percent(&self) -> Option<u64> {
396 let lookups = self.lookups();
397 (lookups > 0).then(|| self.hits * 100 / lookups)
398 }
399
400 /// Adds `other`'s counts; `writer` stays.
401 pub fn add(&mut self, other: &BuildCacheCounts) {
402 self.hits += other.hits;
403 self.misses += other.misses;
404 self.read_bytes += other.read_bytes;
405 self.written_bytes += other.written_bytes;
406 self.refused += other.refused;
407 }
408}
409
410/// A job's stored log.
411#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
412pub struct LogSummary {
413 /// Lines stored.
414 pub lines: u64,
415 /// Bytes stored.
416 pub bytes: u64,
417 /// Set once the log reached `history.log_max` and later lines were dropped.
418 pub truncated: bool,
419}
420
421/// One step of a job, as GitHub reports it.
422#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
423pub struct Step {
424 /// GitHub's step number; numbers may skip.
425 pub number: u32,
426 /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
427 pub name: String,
428 /// `pending`, `queued`, `in_progress` or `completed`.
429 pub status: String,
430 /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
431 pub conclusion: Option<String>,
432 /// When it started.
433 pub started_at: Option<u64>,
434 /// When it completed.
435 pub completed_at: Option<u64>,
436}
437
438/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time).
439#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
440pub struct JobQuery {
441 /// Only this `owner/name`.
442 pub repository: Option<String>,
443 /// Only jobs in this state.
444 pub state: Option<JobState>,
445 /// Only jobs with this conclusion, such as `failure`.
446 pub conclusion: Option<String>,
447 /// Only jobs of the workflow with this name, such as `CI`.
448 pub workflow: Option<String>,
449 /// Only jobs with this name in their workflow, such as `build`.
450 pub name: Option<String>,
451 /// Only jobs queued at or after this time.
452 pub since: Option<u64>,
453 /// Only jobs queued before this time.
454 pub until: Option<u64>,
455 /// Only jobs with a smaller ID: the `next` of the previous page.
456 pub before: Option<u64>,
457 /// At most this many jobs; default 50, at most 500.
458 pub limit: Option<u32>,
459}
460
461/// `GET /v1/jobs`: one page of jobs.
462#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
463pub struct JobPage {
464 /// Newest first.
465 pub jobs: Vec<JobSummary>,
466 /// Pass as `before` for the next page; `None` on the last page.
467 pub next: Option<u64>,
468}
469
470/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
471#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
472pub struct LogQuery {
473 /// Start at this line, 0-based.
474 pub from: Option<u64>,
475 /// Keep the response open and send lines as they are stored, until the job finishes.
476 #[serde(default)]
477 pub follow: bool,
478}
479
480/// One sample of a container's use.
481#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
482pub struct Sample {
483 /// When it was taken.
484 pub at: u64,
485 /// CPU use since the previous sample, in millicores.
486 pub cpu: u64,
487 /// Memory use without page cache.
488 pub memory: u64,
489 /// Bytes received since the container started.
490 pub network_received: u64,
491 /// Bytes sent since the container started.
492 pub network_sent: u64,
493}
494
495/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
496/// older than `history.samples` are deleted; the job's metrics stay.
497#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
498pub struct SampleSeries {
499 /// The job.
500 pub job: u64,
501 /// Seconds between samples.
502 pub interval: u64,
503 /// Oldest first.
504 pub samples: Vec<Sample>,
505}
506
507/// One sample of the whole slice runners run in.
508#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
509pub struct HostSample {
510 /// When it was taken.
511 pub at: u64,
512 /// CPU use of the slice since the previous sample, in millicores.
513 pub cpu: u64,
514 /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
515 /// since active page cache still counts, which the high-water mark of measured admission compares to
516 /// `MemoryMax`.
517 pub memory: u64,
518}
519
520/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
521/// (ADR 0055). Read every few seconds; not kept in the history.
522#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
523pub struct HostMemory {
524 /// When it was read.
525 pub at: u64,
526 /// `MemTotal` in `/proc/meminfo`, in bytes.
527 pub total: u64,
528 /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
529 pub available: u64,
530 /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
531 /// out.
532 pub slice: u64,
533 /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
534 /// kernel without pressure stall information.
535 pub some: Option<Pressure>,
536 /// Its `full` line: the share of time every task that could run waited for memory.
537 pub full: Option<Pressure>,
538 /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
539 /// Read less often than the rest. Empty when none is that large.
540 #[serde(default, skip_serializing_if = "Vec::is_empty")]
541 pub holders: Vec<Holder>,
542 /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
543 /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
544 #[serde(default, skip_serializing_if = "Option::is_none")]
545 pub ceiling: Option<u64>,
546}
547
548impl HostMemory {
549 /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
550 pub fn grown(&self) -> u64 {
551 self.ceiling
552 .map_or(self.total, |ceiling| ceiling.max(self.total))
553 }
554
555 /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
556 /// reached it.
557 pub fn growable(&self) -> u64 {
558 self.grown().saturating_sub(self.total)
559 }
560
561 /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
562 /// page cache it can drop.
563 pub fn outside(&self) -> u64 {
564 self.total
565 .saturating_sub(self.available)
566 .saturating_sub(self.slice)
567 }
568
569 /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
570 /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
571 pub fn outside_line(&self) -> String {
572 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
573 let line = format!("programs outside CI hold {}", size(self.outside()));
574 let named: Vec<String> = self
575 .holders
576 .iter()
577 .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
578 .collect();
579 match named.as_slice() {
580 [] => line,
581 [one] => format!("{line}, most {one}"),
582 [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
583 }
584 }
585}
586
587/// A program outside the slice and the memory its processes hold.
588#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
589pub struct Holder {
590 /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
591 pub name: String,
592 /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
593 pub memory: u64,
594}
595
596/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
597#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
598pub struct Pressure {
599 /// Over the last 10 seconds.
600 pub avg10: u32,
601 /// Over the last 60 seconds.
602 pub avg60: u32,
603}
604
605/// `GET /v1/host/samples` query parameters.
606#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
607pub struct HostSampleQuery {
608 /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
609 pub since: Option<u64>,
610}
611
612/// `GET /v1/host/samples`: the slice's use over time, oldest first.
613#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
614pub struct HostSamples {
615 /// Seconds between samples.
616 pub interval: u64,
617 /// Oldest first.
618 pub samples: Vec<HostSample>,
619}
620
621/// How `GET /v1/stats` groups jobs in time.
622#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
623#[serde(rename_all = "kebab-case")]
624pub enum Bucket {
625 /// By UTC hour.
626 Hour,
627 /// By UTC day.
628 #[default]
629 Day,
630}
631
632/// `GET /v1/activity` query parameters.
633#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
634pub struct ActivityQuery {
635 /// The start of the window; default 24 hours ago. It ends now.
636 pub since: Option<u64>,
637}
638
639/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
640/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
641#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
642pub struct Activity {
643 /// The start of the window.
644 pub since: u64,
645 /// When it was read: the end of the window.
646 pub until: u64,
647 /// Each job that started in the window, oldest start first.
648 pub started: Vec<Started>,
649 /// The jobs that finished in the window: completed, or lost with their runner.
650 pub finished: u64,
651 /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
652 pub failed: u64,
653 /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
654 /// after the kernel killed a process of theirs for memory at their cap.
655 pub runner_failed: u64,
656}
657
658impl Activity {
659 /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
660 /// started.
661 pub fn wait_p95(&self) -> Option<u64> {
662 let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
663 waits.sort_unstable();
664 let rank = (waits.len() * 95).div_ceil(100);
665 waits.get(rank.checked_sub(1)?).copied()
666 }
667
668 /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
669 /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
670 pub fn wait_hours(&self) -> [u64; 24] {
671 const HOUR: u64 = 3600;
672 let since = self.until.saturating_sub(24 * HOUR);
673 let mut hours = [0u64; 24];
674 for start in self
675 .started
676 .iter()
677 .filter(|start| (since..=self.until).contains(&start.at))
678 {
679 // A start exactly 24 hours ago is in the oldest hour.
680 let back =
681 usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
682 hours[23 - back] = hours[23 - back].max(start.waited);
683 }
684 hours
685 }
686}
687
688/// A job that started: when, and how long it had waited since it was queued.
689#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
690pub struct Started {
691 /// When it started.
692 pub at: u64,
693 /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
694 pub waited: u64,
695}
696
697/// `GET /v1/stats` query parameters.
698#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
699pub struct StatsQuery {
700 /// Only this `owner/name`.
701 pub repository: Option<String>,
702 /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
703 /// are counted: 31 days of hours, two years of days.
704 pub since: Option<u64>,
705 /// Only jobs queued before this time.
706 pub until: Option<u64>,
707 /// The bucket size; default a day.
708 #[serde(default)]
709 pub bucket: Bucket,
710}
711
712/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
713#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
714pub struct Stats {
715 /// The bucket size.
716 pub bucket: Bucket,
717 /// By bucket start, then repository.
718 pub rows: Vec<StatsRow>,
719}
720
721/// The finished jobs of one repository in one bucket.
722#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
723pub struct StatsRow {
724 /// The bucket's start.
725 pub start: u64,
726 /// `owner/name`.
727 pub repository: String,
728 /// Jobs completed or lost.
729 pub jobs: u64,
730 /// Of those, concluded `success`.
731 pub succeeded: u64,
732 /// Of those, concluded `failure` or `timed_out`, or lost.
733 pub failed: u64,
734 /// Of those, concluded `cancelled`.
735 pub cancelled: u64,
736 /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
737 /// ran (ADR 0081).
738 pub queue_seconds: u64,
739 /// Seconds from started to finished, summed.
740 pub run_seconds: u64,
741 /// CPU time, in milliseconds, summed.
742 pub cpu_time: u64,
743 /// The highest memory peak of a job.
744 pub memory_peak: u64,
745 /// Bytes received, summed.
746 pub network_received: u64,
747 /// Bytes from the mirror, summed.
748 pub mirror_bytes: u64,
749}
750
751/// What a cache volume holds.
752#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
753#[serde(rename_all = "kebab-case", tag = "kind")]
754pub enum CacheKind {
755 /// A profile's read-only snapshot.
756 Snapshot {
757 /// The profile.
758 profile: String,
759 },
760 /// A slot's layer changes and Docker Engine data.
761 Slot {
762 /// The profile.
763 profile: String,
764 /// 1-based.
765 slot: usize,
766 },
767 /// The pnpm store all slots share.
768 Pnpm,
769 /// The Maven repository all slots share.
770 Maven,
771 /// The Docker Hub mirror's storage.
772 Mirror,
773 /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
774 Build,
775}
776
777/// One cache volume and its size.
778#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
779pub struct CacheVolume {
780 /// The Docker volume.
781 pub name: String,
782 /// What it holds.
783 #[serde(flatten)]
784 pub kind: CacheKind,
785 /// Its size at `measured_at`.
786 pub bytes: u64,
787 /// The size above which runnerd empties it, or for a snapshot starts it over at its next reseed, if it has one.
788 pub limit: Option<u64>,
789}
790
791/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
792/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
793#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
794pub struct Caches {
795 /// When the sizes were measured; `None` before the first measurement.
796 pub measured_at: Option<u64>,
797 /// By name.
798 pub volumes: Vec<CacheVolume>,
799 /// The shared build cache now, when it is on.
800 pub build: Option<BuildCacheStats>,
801}
802
803/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
804#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
805pub struct BuildCacheStats {
806 /// The entries' bytes.
807 pub bytes: u64,
808 /// How many entries it holds.
809 pub entries: u64,
810 /// `[caches.build] max_size`.
811 pub limit: u64,
812 /// The runners of the last 7 days, summed.
813 pub week: BuildCacheCounts,
814 /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
815 /// daemon older than it, which never served it.
816 #[serde(default)]
817 pub actions_cache: bool,
818}
819
820/// `GET /v1/caches/history` query parameters.
821#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
822pub struct CacheHistoryQuery {
823 /// Only measurements at or after this time; default 30 days ago.
824 pub since: Option<u64>,
825}
826
827/// One volume's size at one measurement.
828#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
829pub struct CacheSize {
830 /// When it was measured.
831 pub at: u64,
832 /// The Docker volume.
833 pub name: String,
834 /// Its size.
835 pub bytes: u64,
836}
837
838/// `GET /v1/caches/history`: every measurement, oldest first.
839#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
840pub struct CacheHistory {
841 /// Oldest first.
842 pub sizes: Vec<CacheSize>,
843}