Skip to main content

runnerd_core/api/
jobs.rs

1//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
2//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
3//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).
4
5use serde::{Deserialize, Serialize};
6
7use super::{cpu_time, in_cores};
8use crate::config::ByteSize;
9
10mod facts;
11
12pub use facts::{
13    JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
14    job_title,
15};
16
17/// Where a job is in its life, as this daemon saw it.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "kebab-case")]
20pub enum JobState {
21    /// Waiting for a runner.
22    Queued,
23    /// A runner of this daemon took it.
24    Running,
25    /// GitHub reports it completed; `conclusion` says how.
26    Completed,
27    /// Its runner's container is gone and GitHub did not report the job completed in time.
28    Lost,
29    /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
30    /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
31    Elsewhere,
32}
33
34impl JobState {
35    /// Whether the job can still change.
36    pub fn is_final(self) -> bool {
37        matches!(
38            self,
39            JobState::Completed | JobState::Lost | JobState::Elsewhere
40        )
41    }
42}
43
44/// A job without its steps: what lists and events carry.
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
46pub struct JobSummary {
47    /// GitHub's job ID, also its check run ID.
48    pub id: u64,
49    /// `owner/name` on GitHub.
50    pub repository: String,
51    /// The workflow run it belongs to.
52    pub run_id: u64,
53    /// 1 for the first attempt of the run, higher for re-runs.
54    pub run_attempt: u32,
55    /// The workflow's name.
56    pub workflow: Option<String>,
57    /// The job's name in the workflow.
58    pub name: String,
59    /// The branch the run is for.
60    pub branch: Option<String>,
61    /// The job's page on GitHub.
62    pub url: Option<String>,
63    /// The workflow's `runs-on` labels.
64    pub labels: Vec<String>,
65    /// Where the job is.
66    pub state: JobState,
67    /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
68    pub conclusion: Option<String>,
69    /// The runner that ran it.
70    pub runner: Option<JobRunner>,
71    /// When each phase started.
72    pub timeline: Timeline,
73    /// CPU, memory and network use while it ran.
74    pub metrics: JobMetrics,
75    /// How much of its log is stored.
76    pub log: LogSummary,
77    /// Whether the runner failed it rather than its code (`record::runner_fault`): Sysbox lost the runner's seccomp
78    /// listener, so what its failed step printed has `function not implemented` on a mount, umount or openat2, or
79    /// `unsafe procfs detected` with `invalid argument`. Read from the log once the record is final; false for a job
80    /// that has not concluded failed, and in records from before it was kept.
81    #[serde(default)]
82    pub runner_fault: bool,
83}
84
85/// `GET /v1/jobs/{id}`: a job with its steps.
86#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
87pub struct Job {
88    /// Everything but the steps.
89    #[serde(flatten)]
90    pub summary: JobSummary,
91    /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
92    pub steps: Vec<Step>,
93    /// Its record, for a failed job of a profile that records failures.
94    #[serde(default)]
95    pub record: Option<super::RecordNote>,
96    /// How its runner was admitted; `None` for one started before runnerd recorded it.
97    #[serde(default)]
98    pub admission: Option<super::Admission>,
99}
100
101impl Job {
102    /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
103    /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
104    /// limit as configured now. A reservation of the whole cap is one line.
105    pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
106        let limits = self.limits(profile);
107        let admission = self.admission.as_ref();
108        let name = profile.map_or("profile", |profile| profile.name.as_str());
109        ChartLines {
110            memory: chart_lines(
111                limits.memory,
112                admission.map(|admission| admission.reserved),
113                |value| ByteSize(value).to_string(),
114                name,
115            ),
116            cpu: chart_lines(
117                limits.cpu,
118                admission
119                    .and_then(|admission| admission.cpu)
120                    .map(|cpu| cpu.reserved),
121                super::cpus,
122                name,
123            ),
124        }
125    }
126
127    /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
128    /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
129    /// reload that changes the profile never replaces a cap the record has.
130    pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
131        let runner = self.summary.runner.as_ref();
132        let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
133            (Some(value), _) => Some(Limit {
134                value,
135                source: LimitSource::Cap,
136            }),
137            (None, Some(value)) => Some(Limit {
138                value,
139                source: LimitSource::ProfileNow,
140            }),
141            (None, None) => None,
142        };
143        Limits {
144            memory: limit(
145                runner.and_then(|runner| runner.memory_cap),
146                profile.map(|profile| profile.memory),
147            ),
148            cpu: limit(
149                runner.and_then(|runner| runner.cpu_cap),
150                profile.map(|profile| u64::from(profile.cpus) * 1000),
151            ),
152        }
153    }
154}
155
156/// What a job's container may use, from `Job::limits`.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct Limits {
159    /// In bytes.
160    pub memory: Option<Limit>,
161    /// In millicores.
162    pub cpu: Option<Limit>,
163}
164
165impl Limits {
166    /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
167    /// now 6G and 4 CPUs`; `None` without either.
168    pub fn line(&self) -> Option<String> {
169        let memory = self
170            .memory
171            .map(|limit| (limit.source, ByteSize(limit.value).to_string()));
172        let cpu = self
173            .cpu
174            .map(|limit| (limit.source, super::cpus(limit.value)));
175        let clause = |source: LimitSource, figures: &str| match source {
176            LimitSource::Cap => format!("capped at {figures}"),
177            LimitSource::ProfileNow => format!("profile now {figures}"),
178        };
179        match (memory, cpu) {
180            (Some((memory_source, memory)), Some((cpu_source, cpu)))
181                if memory_source == cpu_source =>
182            {
183                Some(clause(memory_source, &format!("{memory} and {cpu}")))
184            }
185            (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
186                "{}; {}",
187                clause(memory_source, &memory),
188                clause(cpu_source, &cpu)
189            )),
190            (Some((source, figure)), None) | (None, Some((source, figure))) => {
191                Some(clause(source, &figure))
192            }
193            (None, None) => None,
194        }
195    }
196}
197
198/// One resource's limit for a job's container.
199#[derive(Debug, Clone, Copy, PartialEq, Eq)]
200pub struct Limit {
201    /// Bytes or millicores.
202    pub value: u64,
203    /// Where it comes from.
204    pub source: LimitSource,
205}
206
207/// Where a job's limit comes from.
208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
209pub enum LimitSource {
210    /// The cap its container ran with.
211    Cap,
212    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
213    /// changed it.
214    ProfileNow,
215}
216
217/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
218fn chart_lines(
219    limit: Option<Limit>,
220    reserved: Option<u64>,
221    value: fn(u64) -> String,
222    profile: &str,
223) -> Vec<ChartLine> {
224    let line = |figure, kind, label: &str| ChartLine {
225        value: figure,
226        kind,
227        label: format!("{label} {}", value(figure)),
228    };
229    let mut lines = vec![];
230    match limit {
231        Some(Limit {
232            value: cap,
233            source: LimitSource::Cap,
234        }) if reserved == Some(cap) => {
235            return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
236        }
237        Some(Limit {
238            value: cap,
239            source: LimitSource::Cap,
240        }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
241        Some(Limit {
242            value: now,
243            source: LimitSource::ProfileNow,
244        }) => lines.push(line(
245            now,
246            ChartLineKind::ProfileNow,
247            &format!("{profile} now"),
248        )),
249        None => {}
250    }
251    if let Some(reserved) = reserved {
252        lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
253    }
254    lines
255}
256
257/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
258#[derive(Debug, Clone, PartialEq, Eq)]
259pub struct ChartLines {
260    /// In bytes.
261    pub memory: Vec<ChartLine>,
262    /// In millicores.
263    pub cpu: Vec<ChartLine>,
264}
265
266/// A line across a job's memory or CPU chart.
267#[derive(Debug, Clone, PartialEq, Eq)]
268pub struct ChartLine {
269    /// Bytes or millicores.
270    pub value: u64,
271    /// What it stands for.
272    pub kind: ChartLineKind,
273    /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
274    pub label: String,
275}
276
277/// What a line across a job's chart stands for.
278#[derive(Debug, Clone, Copy, PartialEq, Eq)]
279pub enum ChartLineKind {
280    /// The most its container may use: its cap.
281    Cap,
282    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
283    /// changed it.
284    ProfileNow,
285    /// What admission counted against the budget for its runner.
286    Reserved,
287    /// A reservation of the whole cap, as admission by caps makes.
288    CapAndReserved,
289}
290
291/// The runner a job ran on.
292#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
293pub struct JobRunner {
294    /// The runner's name, also its container's.
295    pub name: String,
296    /// The profile it ran in.
297    pub profile: String,
298    /// The slot it ran in, 1-based.
299    pub slot: usize,
300    /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
301    /// from before runnerd kept it.
302    pub memory_cap: Option<u64>,
303    /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
304    /// records from before runnerd kept it, or when Docker reports no limit.
305    pub cpu_cap: Option<u64>,
306}
307
308/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
309#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
310pub struct Timeline {
311    /// GitHub queued the job.
312    pub queued_at: u64,
313    /// The container of the runner that took it started.
314    pub container_started_at: Option<u64>,
315    /// The runner started the job.
316    pub started_at: Option<u64>,
317    /// The job finished, by GitHub's clock; or when its container was found gone.
318    pub finished_at: Option<u64>,
319    /// The container and the runner's registration were removed.
320    pub cleaned_at: Option<u64>,
321    /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
322    /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
323    #[serde(default)]
324    pub unseen: u64,
325    /// When runnerd first listed it queued (ADR 0092). `None` for a job no listing showed queued, such as one a
326    /// warm runner took first, and for one an older runnerd recorded.
327    #[serde(default)]
328    pub seen_at: Option<u64>,
329}
330
331impl Timeline {
332    /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
333    /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
334    pub fn runner_from(&self) -> Option<u64> {
335        self.container_started_at
336            .map(|started| started.max(self.queued_at))
337    }
338}
339
340/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
341#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
342pub struct JobMetrics {
343    /// CPU time, in milliseconds.
344    pub cpu_time: Option<u64>,
345    /// The highest CPU use of a sample interval, in millicores.
346    pub cpu_peak: Option<u64>,
347    /// CPU time over wall time, in millicores.
348    pub cpu_average: Option<u64>,
349    /// The highest memory use without page cache.
350    pub memory_peak: Option<u64>,
351    /// The average memory use without page cache.
352    pub memory_average: Option<u64>,
353    /// Bytes the container received.
354    pub network_received: Option<u64>,
355    /// Bytes the container sent.
356    pub network_sent: Option<u64>,
357    /// Of the bytes received, those the Docker Hub mirror served.
358    pub mirror_bytes: u64,
359    /// What the shared build cache served it and took from it.
360    pub build_cache: BuildCacheCounts,
361    /// Times the kernel held its container's sockets' receive windows small because the container's memory was
362    /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
363    /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
364    #[serde(default, skip_serializing_if = "Option::is_none")]
365    pub sock_throttled: Option<u64>,
366    /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
367    /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
368    #[serde(default, skip_serializing_if = "Option::is_none")]
369    pub cpu_periods: Option<u64>,
370    /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
371    /// cap held it back.
372    #[serde(default, skip_serializing_if = "Option::is_none")]
373    pub cpu_throttled: Option<u64>,
374    /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
375    /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
376    #[serde(default, skip_serializing_if = "Option::is_none")]
377    pub cpu_stalled: Option<u64>,
378}
379
380/// One figure of what a job used, worded for every client.
381#[derive(Debug, Clone, PartialEq, Eq)]
382pub struct Total {
383    /// What it measures: `Memory peak`, `Received`.
384    pub label: &'static str,
385    /// The figure with its unit.
386    pub value: String,
387}
388
389impl JobMetrics {
390    /// What the job used, over its whole life: memory, CPU and network, each where the daemon measured it. The
391    /// peaks are of the samples taken every few seconds, so the figures' note says that
392    /// ([`sampled_note`](super::sampled_note)).
393    pub fn totals(&self) -> Vec<Total> {
394        let mut totals = Vec::new();
395        let mut add = |label: &'static str, value: Option<String>| {
396            totals.extend(value.map(|value| Total { label, value }));
397        };
398        let bytes = |bytes: u64| ByteSize(bytes).to_string();
399        add("Memory peak", self.memory_peak.map(bytes));
400        add("Average memory", self.memory_average.map(bytes));
401        add("CPU peak", self.cpu_peak.map(in_cores));
402        add("Average CPU", self.cpu_average.map(in_cores));
403        add("CPU time", self.cpu_time.map(cpu_time));
404        add("Received", self.network_received.map(bytes));
405        add("Sent", self.network_sent.map(bytes));
406        // Counted from the mirror's own log, so a job without network samples still has it.
407        add(
408            "From the mirror",
409            (self.network_received.is_some() || self.mirror_bytes > 0)
410                .then(|| bytes(self.mirror_bytes)),
411        );
412        add(
413            "Throttled for memory pressure",
414            self.sock_throttled
415                .filter(|times| *times > 0)
416                .map(|times| format!("{times} times")),
417        );
418        totals
419    }
420}
421
422/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
423#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
424pub struct BuildCacheCounts {
425    /// Lookups that found an entry.
426    pub hits: u64,
427    /// Lookups that found none.
428    pub misses: u64,
429    /// Bytes of the entries it served.
430    pub read_bytes: u64,
431    /// Bytes of the entries it stored.
432    pub written_bytes: u64,
433    /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
434    /// cache was full.
435    pub refused: u64,
436    /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
437    pub writer: bool,
438}
439
440impl BuildCacheCounts {
441    /// Hits and misses.
442    pub fn lookups(&self) -> u64 {
443        self.hits + self.misses
444    }
445
446    /// Hits in percent of lookups, `None` without any.
447    pub fn hit_percent(&self) -> Option<u64> {
448        let lookups = self.lookups();
449        (lookups > 0).then(|| self.hits * 100 / lookups)
450    }
451
452    /// Adds `other`'s counts; `writer` stays.
453    pub fn add(&mut self, other: &BuildCacheCounts) {
454        self.hits += other.hits;
455        self.misses += other.misses;
456        self.read_bytes += other.read_bytes;
457        self.written_bytes += other.written_bytes;
458        self.refused += other.refused;
459    }
460}
461
462/// A job's stored log.
463#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
464pub struct LogSummary {
465    /// Lines stored.
466    pub lines: u64,
467    /// Bytes stored.
468    pub bytes: u64,
469    /// Set once the log reached `history.log_max` and later lines were dropped.
470    pub truncated: bool,
471}
472
473/// One step of a job, as GitHub reports it.
474#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
475pub struct Step {
476    /// GitHub's step number; numbers may skip.
477    pub number: u32,
478    /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
479    pub name: String,
480    /// `pending`, `queued`, `in_progress` or `completed`.
481    pub status: String,
482    /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
483    pub conclusion: Option<String>,
484    /// When it started.
485    pub started_at: Option<u64>,
486    /// When it completed.
487    pub completed_at: Option<u64>,
488}
489
490/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time).
491#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
492pub struct JobQuery {
493    /// Only this `owner/name`.
494    pub repository: Option<String>,
495    /// Only jobs in this state.
496    pub state: Option<JobState>,
497    /// Only jobs with this conclusion, such as `failure`.
498    pub conclusion: Option<String>,
499    /// Only jobs of the workflow with this name, such as `CI`.
500    pub workflow: Option<String>,
501    /// Only jobs with this name in their workflow, such as `build`.
502    pub name: Option<String>,
503    /// Only jobs queued at or after this time.
504    pub since: Option<u64>,
505    /// Only jobs queued before this time.
506    pub until: Option<u64>,
507    /// Only jobs with a smaller ID: the `next` of the previous page.
508    pub before: Option<u64>,
509    /// At most this many jobs; default 50, at most 500.
510    pub limit: Option<u32>,
511}
512
513/// `GET /v1/jobs`: one page of jobs.
514#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
515pub struct JobPage {
516    /// Newest first.
517    pub jobs: Vec<JobSummary>,
518    /// Pass as `before` for the next page; `None` on the last page.
519    pub next: Option<u64>,
520}
521
522/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
523#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
524pub struct LogQuery {
525    /// Start at this line, 0-based.
526    pub from: Option<u64>,
527    /// Keep the response open and send lines as they are stored, until the job finishes.
528    #[serde(default)]
529    pub follow: bool,
530}
531
532/// One sample of a container's use.
533#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
534pub struct Sample {
535    /// When it was taken.
536    pub at: u64,
537    /// CPU use since the previous sample, in millicores.
538    pub cpu: u64,
539    /// Memory use without page cache.
540    pub memory: u64,
541    /// Bytes received since the container started.
542    pub network_received: u64,
543    /// Bytes sent since the container started.
544    pub network_sent: u64,
545}
546
547/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
548/// older than `history.samples` are deleted; the job's metrics stay.
549#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
550pub struct SampleSeries {
551    /// The job.
552    pub job: u64,
553    /// Seconds between samples.
554    pub interval: u64,
555    /// Oldest first.
556    pub samples: Vec<Sample>,
557}
558
559/// One sample of the whole slice runners run in.
560#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
561pub struct HostSample {
562    /// When it was taken.
563    pub at: u64,
564    /// CPU use of the slice since the previous sample, in millicores.
565    pub cpu: u64,
566    /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
567    /// since active page cache still counts, which the high-water mark of measured admission compares to
568    /// `MemoryMax`.
569    pub memory: u64,
570}
571
572/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
573/// (ADR 0055). Read every few seconds; not kept in the history.
574#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
575pub struct HostMemory {
576    /// When it was read.
577    pub at: u64,
578    /// `MemTotal` in `/proc/meminfo`, in bytes.
579    pub total: u64,
580    /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
581    pub available: u64,
582    /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
583    /// out.
584    pub slice: u64,
585    /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
586    /// kernel without pressure stall information.
587    pub some: Option<Pressure>,
588    /// Its `full` line: the share of time every task that could run waited for memory.
589    pub full: Option<Pressure>,
590    /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
591    /// Read less often than the rest. Empty when none is that large.
592    #[serde(default, skip_serializing_if = "Vec::is_empty")]
593    pub holders: Vec<Holder>,
594    /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
595    /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
596    #[serde(default, skip_serializing_if = "Option::is_none")]
597    pub ceiling: Option<u64>,
598    /// `SwapTotal` and `SwapFree`, from the same read of `/proc/meminfo`; absent on a host without swap (ADR 0101).
599    /// Admission counts none of it.
600    #[serde(default, skip_serializing_if = "Option::is_none")]
601    pub swap: Option<Swap>,
602}
603
604/// A host's swap at one reading: zram and zswap's device included, as `/proc/meminfo` counts it.
605#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
606pub struct Swap {
607    /// `SwapTotal`, in bytes. Never 0: a host without swap has none.
608    pub total: u64,
609    /// `SwapFree`, in bytes.
610    pub free: u64,
611}
612
613impl Swap {
614    /// What is in use, in bytes.
615    pub fn used(&self) -> u64 {
616        self.total.saturating_sub(self.free)
617    }
618}
619
620/// The margin kept free at least: 2G.
621const MARGIN_MIN: u64 = 2 << 30;
622
623/// The margin as a share of the host's memory, as a divisor: a sixteenth, 4G of 64G.
624const MARGIN_SHARE: u64 = 16;
625
626/// The plain margin `MemAvailable` keeps beside the runners on a host with `total` memory: the larger of `MARGIN_MIN`
627/// and `total / MARGIN_SHARE` (ADR 0055).
628pub fn host_margin(total: u64) -> u64 {
629    MARGIN_MIN.max(total / MARGIN_SHARE)
630}
631
632impl HostMemory {
633    /// What the host has for a new runner this moment, in bytes: `MemAvailable` and what it can still grow by, less
634    /// the plain margin. Unlike `HostBudget::spare` it does not count what runners reserve past what they hold, which
635    /// is memory they may yet take, not memory taken.
636    pub fn room(&self) -> u64 {
637        self.available
638            .saturating_add(self.growable())
639            .saturating_sub(host_margin(self.grown()))
640    }
641
642    /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
643    pub fn grown(&self) -> u64 {
644        self.ceiling
645            .map_or(self.total, |ceiling| ceiling.max(self.total))
646    }
647
648    /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
649    /// reached it.
650    pub fn growable(&self) -> u64 {
651        self.grown().saturating_sub(self.total)
652    }
653
654    /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
655    /// page cache it can drop.
656    pub fn outside(&self) -> u64 {
657        self.total
658            .saturating_sub(self.available)
659            .saturating_sub(self.slice)
660    }
661
662    /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
663    /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
664    pub fn outside_line(&self) -> String {
665        let size = |bytes: u64| ByteSize(bytes).to_string();
666        let line = format!("programs outside CI hold {}", size(self.outside()));
667        let named: Vec<String> = self
668            .holders
669            .iter()
670            .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
671            .collect();
672        match named.as_slice() {
673            [] => line,
674            [one] => format!("{line}, most {one}"),
675            [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
676        }
677    }
678}
679
680/// A program outside the slice and the memory its processes hold.
681#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
682pub struct Holder {
683    /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
684    pub name: String,
685    /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
686    pub memory: u64,
687}
688
689/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
690#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
691pub struct Pressure {
692    /// Over the last 10 seconds.
693    pub avg10: u32,
694    /// Over the last 60 seconds.
695    pub avg60: u32,
696}
697
698/// `GET /v1/host/samples` query parameters.
699#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
700pub struct HostSampleQuery {
701    /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
702    pub since: Option<u64>,
703}
704
705/// `GET /v1/host/samples`: the slice's use over time, oldest first.
706#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
707pub struct HostSamples {
708    /// Seconds between samples.
709    pub interval: u64,
710    /// Oldest first.
711    pub samples: Vec<HostSample>,
712}
713
714/// How `GET /v1/stats` groups jobs in time.
715#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
716#[serde(rename_all = "kebab-case")]
717pub enum Bucket {
718    /// By UTC hour.
719    Hour,
720    /// By UTC day.
721    #[default]
722    Day,
723}
724
725/// `GET /v1/activity` query parameters.
726#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
727pub struct ActivityQuery {
728    /// The start of the window; default 24 hours ago. It ends now.
729    pub since: Option<u64>,
730}
731
732/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
733/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
734#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
735pub struct Activity {
736    /// The start of the window.
737    pub since: u64,
738    /// When it was read: the end of the window.
739    pub until: u64,
740    /// Each job that started in the window, oldest start first.
741    pub started: Vec<Started>,
742    /// The jobs that finished in the window: completed, or lost with their runner.
743    pub finished: u64,
744    /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
745    pub failed: u64,
746    /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
747    /// after the kernel killed a process of theirs for memory at their cap.
748    pub runner_failed: u64,
749}
750
751impl Activity {
752    /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
753    /// started.
754    pub fn wait_p95(&self) -> Option<u64> {
755        let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
756        waits.sort_unstable();
757        let rank = (waits.len() * 95).div_ceil(100);
758        waits.get(rank.checked_sub(1)?).copied()
759    }
760
761    /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
762    /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
763    pub fn wait_hours(&self) -> [u64; 24] {
764        const HOUR: u64 = 3600;
765        let since = self.until.saturating_sub(24 * HOUR);
766        let mut hours = [0u64; 24];
767        for start in self
768            .started
769            .iter()
770            .filter(|start| (since..=self.until).contains(&start.at))
771        {
772            // A start exactly 24 hours ago is in the oldest hour.
773            let back =
774                usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
775            hours[23 - back] = hours[23 - back].max(start.waited);
776        }
777        hours
778    }
779}
780
781/// A job that started: when, and how long it had waited since it was queued.
782#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
783pub struct Started {
784    /// When it started.
785    pub at: u64,
786    /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
787    pub waited: u64,
788}
789
790/// `GET /v1/stats` query parameters.
791#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
792pub struct StatsQuery {
793    /// Only this `owner/name`.
794    pub repository: Option<String>,
795    /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
796    /// are counted: 31 days of hours, two years of days.
797    pub since: Option<u64>,
798    /// Only jobs queued before this time.
799    pub until: Option<u64>,
800    /// The bucket size; default a day.
801    #[serde(default)]
802    pub bucket: Bucket,
803}
804
805/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
806#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
807pub struct Stats {
808    /// The bucket size.
809    pub bucket: Bucket,
810    /// By bucket start, then repository.
811    pub rows: Vec<StatsRow>,
812}
813
814/// The finished jobs of one repository in one bucket.
815#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
816pub struct StatsRow {
817    /// The bucket's start.
818    pub start: u64,
819    /// `owner/name`.
820    pub repository: String,
821    /// Jobs completed or lost.
822    pub jobs: u64,
823    /// Of those, concluded `success`.
824    pub succeeded: u64,
825    /// Of those, concluded `failure` or `timed_out`, or lost.
826    pub failed: u64,
827    /// Of those, concluded `cancelled`.
828    pub cancelled: u64,
829    /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
830    /// ran (ADR 0081).
831    pub queue_seconds: u64,
832    /// Seconds from started to finished, summed.
833    pub run_seconds: u64,
834    /// CPU time, in milliseconds, summed.
835    pub cpu_time: u64,
836    /// The highest memory peak of a job.
837    pub memory_peak: u64,
838    /// Bytes received, summed.
839    pub network_received: u64,
840    /// Bytes from the mirror, summed.
841    pub mirror_bytes: u64,
842}
843
844/// What a cache volume holds.
845#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
846#[serde(rename_all = "kebab-case", tag = "kind")]
847pub enum CacheKind {
848    /// A profile's read-only snapshot.
849    Snapshot {
850        /// The profile.
851        profile: String,
852    },
853    /// A slot's layer changes and Docker Engine data.
854    Slot {
855        /// The profile.
856        profile: String,
857        /// 1-based.
858        slot: usize,
859    },
860    /// The pnpm store all slots share.
861    Pnpm,
862    /// The Maven repository all slots share.
863    Maven,
864    /// The Docker Hub mirror's storage.
865    Mirror,
866    /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
867    Build,
868}
869
870/// One cache volume and its size.
871#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
872pub struct CacheVolume {
873    /// The Docker volume.
874    pub name: String,
875    /// What it holds.
876    #[serde(flatten)]
877    pub kind: CacheKind,
878    /// Its size at `measured_at`.
879    pub bytes: u64,
880    /// The size above which runnerd empties it, or for a snapshot starts it over at its next reseed, if it has one.
881    pub limit: Option<u64>,
882}
883
884/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
885/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
886#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
887pub struct Caches {
888    /// When the sizes were measured; `None` before the first measurement.
889    pub measured_at: Option<u64>,
890    /// By name.
891    pub volumes: Vec<CacheVolume>,
892    /// The shared build cache now, when it is on.
893    pub build: Option<BuildCacheStats>,
894}
895
896/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
897#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
898pub struct BuildCacheStats {
899    /// The entries' bytes.
900    pub bytes: u64,
901    /// How many entries it holds.
902    pub entries: u64,
903    /// `[caches.build] max_size`.
904    pub limit: u64,
905    /// The runners of the last 7 days, summed.
906    pub week: BuildCacheCounts,
907    /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
908    /// daemon older than it, which never served it.
909    #[serde(default)]
910    pub actions_cache: bool,
911}
912
913/// `GET /v1/caches/history` query parameters.
914#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
915pub struct CacheHistoryQuery {
916    /// Only measurements at or after this time; default 30 days ago.
917    pub since: Option<u64>,
918}
919
920/// One volume's size at one measurement.
921#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
922pub struct CacheSize {
923    /// When it was measured.
924    pub at: u64,
925    /// The Docker volume.
926    pub name: String,
927    /// Its size.
928    pub bytes: u64,
929}
930
931/// `GET /v1/caches/history`: every measurement, oldest first.
932#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
933pub struct CacheHistory {
934    /// Oldest first.
935    pub sizes: Vec<CacheSize>,
936}