runnerd_core/api/jobs.rs
1//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
2//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
3//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).
4
5use serde::{Deserialize, Serialize};
6
7use super::{cpu_time, in_cores};
8use crate::config::ByteSize;
9
10mod facts;
11
12pub use facts::{
13 JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
14 job_title,
15};
16
17/// Where a job is in its life, as this daemon saw it.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "kebab-case")]
20pub enum JobState {
21 /// Waiting for a runner.
22 Queued,
23 /// A runner of this daemon took it.
24 Running,
25 /// GitHub reports it completed; `conclusion` says how.
26 Completed,
27 /// Its runner's container is gone and GitHub did not report the job completed in time.
28 Lost,
29 /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
30 /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
31 Elsewhere,
32}
33
34impl JobState {
35 /// Whether the job can still change.
36 pub fn is_final(self) -> bool {
37 matches!(
38 self,
39 JobState::Completed | JobState::Lost | JobState::Elsewhere
40 )
41 }
42}
43
44/// A job without its steps: what lists and events carry.
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
46pub struct JobSummary {
47 /// GitHub's job ID, also its check run ID.
48 pub id: u64,
49 /// `owner/name` on GitHub.
50 pub repository: String,
51 /// The workflow run it belongs to.
52 pub run_id: u64,
53 /// 1 for the first attempt of the run, higher for re-runs.
54 pub run_attempt: u32,
55 /// The workflow's name.
56 pub workflow: Option<String>,
57 /// The job's name in the workflow.
58 pub name: String,
59 /// The branch the run is for.
60 pub branch: Option<String>,
61 /// The job's page on GitHub.
62 pub url: Option<String>,
63 /// The workflow's `runs-on` labels.
64 pub labels: Vec<String>,
65 /// Where the job is.
66 pub state: JobState,
67 /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
68 pub conclusion: Option<String>,
69 /// The runner that ran it.
70 pub runner: Option<JobRunner>,
71 /// When each phase started.
72 pub timeline: Timeline,
73 /// CPU, memory and network use while it ran.
74 pub metrics: JobMetrics,
75 /// How much of its log is stored.
76 pub log: LogSummary,
77 /// Whether the runner failed it rather than its code (`record::runner_fault`): Sysbox lost the runner's seccomp
78 /// listener, so what its failed step printed has `function not implemented` on a mount, umount or openat2, or
79 /// `unsafe procfs detected` with `invalid argument`. Read from the log once the record is final; false for a job
80 /// that has not concluded failed, and in records from before it was kept.
81 #[serde(default)]
82 pub runner_fault: bool,
83}
84
85/// `GET /v1/jobs/{id}`: a job with its steps.
86#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
87pub struct Job {
88 /// Everything but the steps.
89 #[serde(flatten)]
90 pub summary: JobSummary,
91 /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
92 pub steps: Vec<Step>,
93 /// Its record, for a failed job of a profile that records failures.
94 #[serde(default)]
95 pub record: Option<super::RecordNote>,
96 /// How its runner was admitted; `None` for one started before runnerd recorded it.
97 #[serde(default)]
98 pub admission: Option<super::Admission>,
99}
100
101impl Job {
102 /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
103 /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
104 /// limit as configured now. A reservation of the whole cap is one line.
105 pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
106 let limits = self.limits(profile);
107 let admission = self.admission.as_ref();
108 let name = profile.map_or("profile", |profile| profile.name.as_str());
109 ChartLines {
110 memory: chart_lines(
111 limits.memory,
112 admission.map(|admission| admission.reserved),
113 |value| ByteSize(value).to_string(),
114 name,
115 ),
116 cpu: chart_lines(
117 limits.cpu,
118 admission
119 .and_then(|admission| admission.cpu)
120 .map(|cpu| cpu.reserved),
121 super::cpus,
122 name,
123 ),
124 }
125 }
126
127 /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
128 /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
129 /// reload that changes the profile never replaces a cap the record has.
130 pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
131 let runner = self.summary.runner.as_ref();
132 let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
133 (Some(value), _) => Some(Limit {
134 value,
135 source: LimitSource::Cap,
136 }),
137 (None, Some(value)) => Some(Limit {
138 value,
139 source: LimitSource::ProfileNow,
140 }),
141 (None, None) => None,
142 };
143 Limits {
144 memory: limit(
145 runner.and_then(|runner| runner.memory_cap),
146 profile.map(|profile| profile.memory),
147 ),
148 cpu: limit(
149 runner.and_then(|runner| runner.cpu_cap),
150 profile.map(|profile| u64::from(profile.cpus) * 1000),
151 ),
152 }
153 }
154}
155
156/// What a job's container may use, from `Job::limits`.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct Limits {
159 /// In bytes.
160 pub memory: Option<Limit>,
161 /// In millicores.
162 pub cpu: Option<Limit>,
163}
164
165impl Limits {
166 /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
167 /// now 6G and 4 CPUs`; `None` without either.
168 pub fn line(&self) -> Option<String> {
169 let memory = self
170 .memory
171 .map(|limit| (limit.source, ByteSize(limit.value).to_string()));
172 let cpu = self
173 .cpu
174 .map(|limit| (limit.source, super::cpus(limit.value)));
175 let clause = |source: LimitSource, figures: &str| match source {
176 LimitSource::Cap => format!("capped at {figures}"),
177 LimitSource::ProfileNow => format!("profile now {figures}"),
178 };
179 match (memory, cpu) {
180 (Some((memory_source, memory)), Some((cpu_source, cpu)))
181 if memory_source == cpu_source =>
182 {
183 Some(clause(memory_source, &format!("{memory} and {cpu}")))
184 }
185 (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
186 "{}; {}",
187 clause(memory_source, &memory),
188 clause(cpu_source, &cpu)
189 )),
190 (Some((source, figure)), None) | (None, Some((source, figure))) => {
191 Some(clause(source, &figure))
192 }
193 (None, None) => None,
194 }
195 }
196}
197
198/// One resource's limit for a job's container.
199#[derive(Debug, Clone, Copy, PartialEq, Eq)]
200pub struct Limit {
201 /// Bytes or millicores.
202 pub value: u64,
203 /// Where it comes from.
204 pub source: LimitSource,
205}
206
207/// Where a job's limit comes from.
208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
209pub enum LimitSource {
210 /// The cap its container ran with.
211 Cap,
212 /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
213 /// changed it.
214 ProfileNow,
215}
216
217/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
218fn chart_lines(
219 limit: Option<Limit>,
220 reserved: Option<u64>,
221 value: fn(u64) -> String,
222 profile: &str,
223) -> Vec<ChartLine> {
224 let line = |figure, kind, label: &str| ChartLine {
225 value: figure,
226 kind,
227 label: format!("{label} {}", value(figure)),
228 };
229 let mut lines = vec![];
230 match limit {
231 Some(Limit {
232 value: cap,
233 source: LimitSource::Cap,
234 }) if reserved == Some(cap) => {
235 return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
236 }
237 Some(Limit {
238 value: cap,
239 source: LimitSource::Cap,
240 }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
241 Some(Limit {
242 value: now,
243 source: LimitSource::ProfileNow,
244 }) => lines.push(line(
245 now,
246 ChartLineKind::ProfileNow,
247 &format!("{profile} now"),
248 )),
249 None => {}
250 }
251 if let Some(reserved) = reserved {
252 lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
253 }
254 lines
255}
256
257/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
258#[derive(Debug, Clone, PartialEq, Eq)]
259pub struct ChartLines {
260 /// In bytes.
261 pub memory: Vec<ChartLine>,
262 /// In millicores.
263 pub cpu: Vec<ChartLine>,
264}
265
266/// A line across a job's memory or CPU chart.
267#[derive(Debug, Clone, PartialEq, Eq)]
268pub struct ChartLine {
269 /// Bytes or millicores.
270 pub value: u64,
271 /// What it stands for.
272 pub kind: ChartLineKind,
273 /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
274 pub label: String,
275}
276
277/// What a line across a job's chart stands for.
278#[derive(Debug, Clone, Copy, PartialEq, Eq)]
279pub enum ChartLineKind {
280 /// The most its container may use: its cap.
281 Cap,
282 /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
283 /// changed it.
284 ProfileNow,
285 /// What admission counted against the budget for its runner.
286 Reserved,
287 /// A reservation of the whole cap, as admission by caps makes.
288 CapAndReserved,
289}
290
291/// The runner a job ran on.
292#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
293pub struct JobRunner {
294 /// The runner's name, also its container's.
295 pub name: String,
296 /// The profile it ran in.
297 pub profile: String,
298 /// The slot it ran in, 1-based.
299 pub slot: usize,
300 /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
301 /// from before runnerd kept it.
302 pub memory_cap: Option<u64>,
303 /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
304 /// records from before runnerd kept it, or when Docker reports no limit.
305 pub cpu_cap: Option<u64>,
306}
307
308/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
309#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
310pub struct Timeline {
311 /// GitHub queued the job.
312 pub queued_at: u64,
313 /// The container of the runner that took it started.
314 pub container_started_at: Option<u64>,
315 /// The runner started the job.
316 pub started_at: Option<u64>,
317 /// The job finished, by GitHub's clock; or when its container was found gone.
318 pub finished_at: Option<u64>,
319 /// The container and the runner's registration were removed.
320 pub cleaned_at: Option<u64>,
321 /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
322 /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
323 #[serde(default)]
324 pub unseen: u64,
325 /// When runnerd first listed it queued (ADR 0092). `None` for a job no listing showed queued, such as one a
326 /// warm runner took first, and for one an older runnerd recorded.
327 #[serde(default)]
328 pub seen_at: Option<u64>,
329}
330
331impl Timeline {
332 /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
333 /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
334 pub fn runner_from(&self) -> Option<u64> {
335 self.container_started_at
336 .map(|started| started.max(self.queued_at))
337 }
338}
339
340/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
341#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
342pub struct JobMetrics {
343 /// CPU time, in milliseconds.
344 pub cpu_time: Option<u64>,
345 /// The highest CPU use of a sample interval, in millicores.
346 pub cpu_peak: Option<u64>,
347 /// CPU time over wall time, in millicores.
348 pub cpu_average: Option<u64>,
349 /// The highest memory use without page cache.
350 pub memory_peak: Option<u64>,
351 /// The average memory use without page cache.
352 pub memory_average: Option<u64>,
353 /// Bytes the container received.
354 pub network_received: Option<u64>,
355 /// Bytes the container sent.
356 pub network_sent: Option<u64>,
357 /// Of the bytes received, those the Docker Hub mirror served.
358 pub mirror_bytes: u64,
359 /// What the shared build cache served it and took from it.
360 pub build_cache: BuildCacheCounts,
361 /// Times the kernel held its container's sockets' receive windows small because the container's memory was
362 /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
363 /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
364 #[serde(default, skip_serializing_if = "Option::is_none")]
365 pub sock_throttled: Option<u64>,
366 /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
367 /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
368 #[serde(default, skip_serializing_if = "Option::is_none")]
369 pub cpu_periods: Option<u64>,
370 /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
371 /// cap held it back.
372 #[serde(default, skip_serializing_if = "Option::is_none")]
373 pub cpu_throttled: Option<u64>,
374 /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
375 /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
376 #[serde(default, skip_serializing_if = "Option::is_none")]
377 pub cpu_stalled: Option<u64>,
378}
379
380/// One figure of what a job used, worded for every client.
381#[derive(Debug, Clone, PartialEq, Eq)]
382pub struct Total {
383 /// What it measures: `Memory peak`, `Received`.
384 pub label: &'static str,
385 /// The figure with its unit.
386 pub value: String,
387}
388
389impl JobMetrics {
390 /// What the job used, over its whole life: memory, CPU and network, each where the daemon measured it. The
391 /// peaks are of the samples taken every few seconds, so the figures' note says that
392 /// ([`sampled_note`](super::sampled_note)).
393 pub fn totals(&self) -> Vec<Total> {
394 let mut totals = Vec::new();
395 let mut add = |label: &'static str, value: Option<String>| {
396 totals.extend(value.map(|value| Total { label, value }));
397 };
398 let bytes = |bytes: u64| ByteSize(bytes).to_string();
399 add("Memory peak", self.memory_peak.map(bytes));
400 add("Average memory", self.memory_average.map(bytes));
401 add("CPU peak", self.cpu_peak.map(in_cores));
402 add("Average CPU", self.cpu_average.map(in_cores));
403 add("CPU time", self.cpu_time.map(cpu_time));
404 add("Received", self.network_received.map(bytes));
405 add("Sent", self.network_sent.map(bytes));
406 // Counted from the mirror's own log, so a job without network samples still has it.
407 add(
408 "From the mirror",
409 (self.network_received.is_some() || self.mirror_bytes > 0)
410 .then(|| bytes(self.mirror_bytes)),
411 );
412 add(
413 "Throttled for memory pressure",
414 self.sock_throttled
415 .filter(|times| *times > 0)
416 .map(|times| format!("{times} times")),
417 );
418 totals
419 }
420}
421
422/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
423#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
424pub struct BuildCacheCounts {
425 /// Lookups that found an entry.
426 pub hits: u64,
427 /// Lookups that found none.
428 pub misses: u64,
429 /// Bytes of the entries it served.
430 pub read_bytes: u64,
431 /// Bytes of the entries it stored.
432 pub written_bytes: u64,
433 /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
434 /// cache was full.
435 pub refused: u64,
436 /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
437 pub writer: bool,
438}
439
440impl BuildCacheCounts {
441 /// Hits and misses.
442 pub fn lookups(&self) -> u64 {
443 self.hits + self.misses
444 }
445
446 /// Hits in percent of lookups, `None` without any.
447 pub fn hit_percent(&self) -> Option<u64> {
448 let lookups = self.lookups();
449 (lookups > 0).then(|| self.hits * 100 / lookups)
450 }
451
452 /// Adds `other`'s counts; `writer` stays.
453 pub fn add(&mut self, other: &BuildCacheCounts) {
454 self.hits += other.hits;
455 self.misses += other.misses;
456 self.read_bytes += other.read_bytes;
457 self.written_bytes += other.written_bytes;
458 self.refused += other.refused;
459 }
460}
461
462/// A job's stored log.
463#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
464pub struct LogSummary {
465 /// Lines stored.
466 pub lines: u64,
467 /// Bytes stored.
468 pub bytes: u64,
469 /// Set once the log reached `history.log_max` and later lines were dropped.
470 pub truncated: bool,
471}
472
473/// One step of a job, as GitHub reports it.
474#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
475pub struct Step {
476 /// GitHub's step number; numbers may skip.
477 pub number: u32,
478 /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
479 pub name: String,
480 /// `pending`, `queued`, `in_progress` or `completed`.
481 pub status: String,
482 /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
483 pub conclusion: Option<String>,
484 /// When it started.
485 pub started_at: Option<u64>,
486 /// When it completed.
487 pub completed_at: Option<u64>,
488}
489
490/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time).
491#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
492pub struct JobQuery {
493 /// Only this `owner/name`.
494 pub repository: Option<String>,
495 /// Only jobs in this state.
496 pub state: Option<JobState>,
497 /// Only jobs with this conclusion, such as `failure`.
498 pub conclusion: Option<String>,
499 /// Only jobs of the workflow with this name, such as `CI`.
500 pub workflow: Option<String>,
501 /// Only jobs with this name in their workflow, such as `build`.
502 pub name: Option<String>,
503 /// Only jobs queued at or after this time.
504 pub since: Option<u64>,
505 /// Only jobs queued before this time.
506 pub until: Option<u64>,
507 /// Only jobs with a smaller ID: the `next` of the previous page.
508 pub before: Option<u64>,
509 /// At most this many jobs; default 50, at most 500.
510 pub limit: Option<u32>,
511}
512
513/// `GET /v1/jobs`: one page of jobs.
514#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
515pub struct JobPage {
516 /// Newest first.
517 pub jobs: Vec<JobSummary>,
518 /// Pass as `before` for the next page; `None` on the last page.
519 pub next: Option<u64>,
520}
521
522/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
523#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
524pub struct LogQuery {
525 /// Start at this line, 0-based.
526 pub from: Option<u64>,
527 /// Keep the response open and send lines as they are stored, until the job finishes.
528 #[serde(default)]
529 pub follow: bool,
530}
531
532/// One sample of a container's use.
533#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
534pub struct Sample {
535 /// When it was taken.
536 pub at: u64,
537 /// CPU use since the previous sample, in millicores.
538 pub cpu: u64,
539 /// Memory use without page cache.
540 pub memory: u64,
541 /// Bytes received since the container started.
542 pub network_received: u64,
543 /// Bytes sent since the container started.
544 pub network_sent: u64,
545}
546
547/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
548/// older than `history.samples` are deleted; the job's metrics stay.
549#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
550pub struct SampleSeries {
551 /// The job.
552 pub job: u64,
553 /// Seconds between samples.
554 pub interval: u64,
555 /// Oldest first.
556 pub samples: Vec<Sample>,
557}
558
559/// One sample of the whole slice runners run in.
560#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
561pub struct HostSample {
562 /// When it was taken.
563 pub at: u64,
564 /// CPU use of the slice since the previous sample, in millicores.
565 pub cpu: u64,
566 /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
567 /// since active page cache still counts, which the high-water mark of measured admission compares to
568 /// `MemoryMax`.
569 pub memory: u64,
570}
571
572/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
573/// (ADR 0055). Read every few seconds; not kept in the history.
574#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
575pub struct HostMemory {
576 /// When it was read.
577 pub at: u64,
578 /// `MemTotal` in `/proc/meminfo`, in bytes.
579 pub total: u64,
580 /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
581 pub available: u64,
582 /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
583 /// out.
584 pub slice: u64,
585 /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
586 /// kernel without pressure stall information.
587 pub some: Option<Pressure>,
588 /// Its `full` line: the share of time every task that could run waited for memory.
589 pub full: Option<Pressure>,
590 /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
591 /// Read less often than the rest. Empty when none is that large.
592 #[serde(default, skip_serializing_if = "Vec::is_empty")]
593 pub holders: Vec<Holder>,
594 /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
595 /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
596 #[serde(default, skip_serializing_if = "Option::is_none")]
597 pub ceiling: Option<u64>,
598 /// `SwapTotal` and `SwapFree`, from the same read of `/proc/meminfo`; absent on a host without swap (ADR 0101).
599 /// Admission counts none of it.
600 #[serde(default, skip_serializing_if = "Option::is_none")]
601 pub swap: Option<Swap>,
602}
603
604/// A host's swap at one reading: zram and zswap's device included, as `/proc/meminfo` counts it.
605#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
606pub struct Swap {
607 /// `SwapTotal`, in bytes. Never 0: a host without swap has none.
608 pub total: u64,
609 /// `SwapFree`, in bytes.
610 pub free: u64,
611}
612
613impl Swap {
614 /// What is in use, in bytes.
615 pub fn used(&self) -> u64 {
616 self.total.saturating_sub(self.free)
617 }
618}
619
620/// The margin kept free at least: 2G.
621const MARGIN_MIN: u64 = 2 << 30;
622
623/// The margin as a share of the host's memory, as a divisor: a sixteenth, 4G of 64G.
624const MARGIN_SHARE: u64 = 16;
625
626/// The plain margin `MemAvailable` keeps beside the runners on a host with `total` memory: the larger of `MARGIN_MIN`
627/// and `total / MARGIN_SHARE` (ADR 0055).
628pub fn host_margin(total: u64) -> u64 {
629 MARGIN_MIN.max(total / MARGIN_SHARE)
630}
631
632impl HostMemory {
633 /// What the host has for a new runner this moment, in bytes: `MemAvailable` and what it can still grow by, less
634 /// the plain margin. Unlike `HostBudget::spare` it does not count what runners reserve past what they hold, which
635 /// is memory they may yet take, not memory taken.
636 pub fn room(&self) -> u64 {
637 self.available
638 .saturating_add(self.growable())
639 .saturating_sub(host_margin(self.grown()))
640 }
641
642 /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
643 pub fn grown(&self) -> u64 {
644 self.ceiling
645 .map_or(self.total, |ceiling| ceiling.max(self.total))
646 }
647
648 /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
649 /// reached it.
650 pub fn growable(&self) -> u64 {
651 self.grown().saturating_sub(self.total)
652 }
653
654 /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
655 /// page cache it can drop.
656 pub fn outside(&self) -> u64 {
657 self.total
658 .saturating_sub(self.available)
659 .saturating_sub(self.slice)
660 }
661
662 /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
663 /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
664 pub fn outside_line(&self) -> String {
665 let size = |bytes: u64| ByteSize(bytes).to_string();
666 let line = format!("programs outside CI hold {}", size(self.outside()));
667 let named: Vec<String> = self
668 .holders
669 .iter()
670 .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
671 .collect();
672 match named.as_slice() {
673 [] => line,
674 [one] => format!("{line}, most {one}"),
675 [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
676 }
677 }
678}
679
680/// A program outside the slice and the memory its processes hold.
681#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
682pub struct Holder {
683 /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
684 pub name: String,
685 /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
686 pub memory: u64,
687}
688
689/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
690#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
691pub struct Pressure {
692 /// Over the last 10 seconds.
693 pub avg10: u32,
694 /// Over the last 60 seconds.
695 pub avg60: u32,
696}
697
698/// `GET /v1/host/samples` query parameters.
699#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
700pub struct HostSampleQuery {
701 /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
702 pub since: Option<u64>,
703}
704
705/// `GET /v1/host/samples`: the slice's use over time, oldest first.
706#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
707pub struct HostSamples {
708 /// Seconds between samples.
709 pub interval: u64,
710 /// Oldest first.
711 pub samples: Vec<HostSample>,
712}
713
714/// How `GET /v1/stats` groups jobs in time.
715#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
716#[serde(rename_all = "kebab-case")]
717pub enum Bucket {
718 /// By UTC hour.
719 Hour,
720 /// By UTC day.
721 #[default]
722 Day,
723}
724
725/// `GET /v1/activity` query parameters.
726#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
727pub struct ActivityQuery {
728 /// The start of the window; default 24 hours ago. It ends now.
729 pub since: Option<u64>,
730}
731
732/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
733/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
734#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
735pub struct Activity {
736 /// The start of the window.
737 pub since: u64,
738 /// When it was read: the end of the window.
739 pub until: u64,
740 /// Each job that started in the window, oldest start first.
741 pub started: Vec<Started>,
742 /// The jobs that finished in the window: completed, or lost with their runner.
743 pub finished: u64,
744 /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
745 pub failed: u64,
746 /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
747 /// after the kernel killed a process of theirs for memory at their cap.
748 pub runner_failed: u64,
749}
750
751impl Activity {
752 /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
753 /// started.
754 pub fn wait_p95(&self) -> Option<u64> {
755 let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
756 waits.sort_unstable();
757 let rank = (waits.len() * 95).div_ceil(100);
758 waits.get(rank.checked_sub(1)?).copied()
759 }
760
761 /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
762 /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
763 pub fn wait_hours(&self) -> [u64; 24] {
764 const HOUR: u64 = 3600;
765 let since = self.until.saturating_sub(24 * HOUR);
766 let mut hours = [0u64; 24];
767 for start in self
768 .started
769 .iter()
770 .filter(|start| (since..=self.until).contains(&start.at))
771 {
772 // A start exactly 24 hours ago is in the oldest hour.
773 let back =
774 usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
775 hours[23 - back] = hours[23 - back].max(start.waited);
776 }
777 hours
778 }
779}
780
781/// A job that started: when, and how long it had waited since it was queued.
782#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
783pub struct Started {
784 /// When it started.
785 pub at: u64,
786 /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
787 pub waited: u64,
788}
789
790/// `GET /v1/stats` query parameters.
791#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
792pub struct StatsQuery {
793 /// Only this `owner/name`.
794 pub repository: Option<String>,
795 /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
796 /// are counted: 31 days of hours, two years of days.
797 pub since: Option<u64>,
798 /// Only jobs queued before this time.
799 pub until: Option<u64>,
800 /// The bucket size; default a day.
801 #[serde(default)]
802 pub bucket: Bucket,
803}
804
805/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
806#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
807pub struct Stats {
808 /// The bucket size.
809 pub bucket: Bucket,
810 /// By bucket start, then repository.
811 pub rows: Vec<StatsRow>,
812}
813
814/// The finished jobs of one repository in one bucket.
815#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
816pub struct StatsRow {
817 /// The bucket's start.
818 pub start: u64,
819 /// `owner/name`.
820 pub repository: String,
821 /// Jobs completed or lost.
822 pub jobs: u64,
823 /// Of those, concluded `success`.
824 pub succeeded: u64,
825 /// Of those, concluded `failure` or `timed_out`, or lost.
826 pub failed: u64,
827 /// Of those, concluded `cancelled`.
828 pub cancelled: u64,
829 /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
830 /// ran (ADR 0081).
831 pub queue_seconds: u64,
832 /// Seconds from started to finished, summed.
833 pub run_seconds: u64,
834 /// CPU time, in milliseconds, summed.
835 pub cpu_time: u64,
836 /// The highest memory peak of a job.
837 pub memory_peak: u64,
838 /// Bytes received, summed.
839 pub network_received: u64,
840 /// Bytes from the mirror, summed.
841 pub mirror_bytes: u64,
842}
843
844/// What a cache volume holds.
845#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
846#[serde(rename_all = "kebab-case", tag = "kind")]
847pub enum CacheKind {
848 /// A profile's read-only snapshot.
849 Snapshot {
850 /// The profile.
851 profile: String,
852 },
853 /// A slot's layer changes and Docker Engine data.
854 Slot {
855 /// The profile.
856 profile: String,
857 /// 1-based.
858 slot: usize,
859 },
860 /// The pnpm store all slots share.
861 Pnpm,
862 /// The Maven repository all slots share.
863 Maven,
864 /// The Docker Hub mirror's storage.
865 Mirror,
866 /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
867 Build,
868}
869
870/// One cache volume and its size.
871#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
872pub struct CacheVolume {
873 /// The Docker volume.
874 pub name: String,
875 /// What it holds.
876 #[serde(flatten)]
877 pub kind: CacheKind,
878 /// Its size at `measured_at`.
879 pub bytes: u64,
880 /// The size above which runnerd empties it, or for a snapshot starts it over at its next reseed, if it has one.
881 pub limit: Option<u64>,
882}
883
884/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
885/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
886#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
887pub struct Caches {
888 /// When the sizes were measured; `None` before the first measurement.
889 pub measured_at: Option<u64>,
890 /// By name.
891 pub volumes: Vec<CacheVolume>,
892 /// The shared build cache now, when it is on.
893 pub build: Option<BuildCacheStats>,
894}
895
896/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
897#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
898pub struct BuildCacheStats {
899 /// The entries' bytes.
900 pub bytes: u64,
901 /// How many entries it holds.
902 pub entries: u64,
903 /// `[caches.build] max_size`.
904 pub limit: u64,
905 /// The runners of the last 7 days, summed.
906 pub week: BuildCacheCounts,
907 /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
908 /// daemon older than it, which never served it.
909 #[serde(default)]
910 pub actions_cache: bool,
911}
912
913/// `GET /v1/caches/history` query parameters.
914#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
915pub struct CacheHistoryQuery {
916 /// Only measurements at or after this time; default 30 days ago.
917 pub since: Option<u64>,
918}
919
920/// One volume's size at one measurement.
921#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
922pub struct CacheSize {
923 /// When it was measured.
924 pub at: u64,
925 /// The Docker volume.
926 pub name: String,
927 /// Its size.
928 pub bytes: u64,
929}
930
931/// `GET /v1/caches/history`: every measurement, oldest first.
932#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
933pub struct CacheHistory {
934 /// Oldest first.
935 pub sizes: Vec<CacheSize>,
936}