Skip to main content

runnerd_core/api/
jobs.rs

1//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
2//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
3//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).
4
5use serde::{Deserialize, Serialize};
6
7use super::{cpu_time, in_cores};
8use crate::config::ByteSize;
9
10mod facts;
11
12pub use facts::{
13    JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
14    job_title,
15};
16
17/// Where a job is in its life, as this daemon saw it.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "kebab-case")]
20pub enum JobState {
21    /// Waiting for a runner.
22    Queued,
23    /// A runner of this daemon took it.
24    Running,
25    /// GitHub reports it completed; `conclusion` says how.
26    Completed,
27    /// Its runner's container is gone and GitHub did not report the job completed in time.
28    Lost,
29    /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
30    /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
31    Elsewhere,
32}
33
34impl JobState {
35    /// Whether the job can still change.
36    pub fn is_final(self) -> bool {
37        matches!(
38            self,
39            JobState::Completed | JobState::Lost | JobState::Elsewhere
40        )
41    }
42}
43
44/// A job without its steps: what lists and events carry.
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
46pub struct JobSummary {
47    /// GitHub's job ID, also its check run ID.
48    pub id: u64,
49    /// `owner/name` on GitHub.
50    pub repository: String,
51    /// The workflow run it belongs to.
52    pub run_id: u64,
53    /// 1 for the first attempt of the run, higher for re-runs.
54    pub run_attempt: u32,
55    /// The workflow's name.
56    pub workflow: Option<String>,
57    /// The job's name in the workflow.
58    pub name: String,
59    /// The branch the run is for.
60    pub branch: Option<String>,
61    /// The job's page on GitHub.
62    pub url: Option<String>,
63    /// The workflow's `runs-on` labels.
64    pub labels: Vec<String>,
65    /// Where the job is.
66    pub state: JobState,
67    /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
68    pub conclusion: Option<String>,
69    /// The runner that ran it.
70    pub runner: Option<JobRunner>,
71    /// When each phase started.
72    pub timeline: Timeline,
73    /// CPU, memory and network use while it ran.
74    pub metrics: JobMetrics,
75    /// How much of its log is stored.
76    pub log: LogSummary,
77}
78
79/// `GET /v1/jobs/{id}`: a job with its steps.
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub struct Job {
82    /// Everything but the steps.
83    #[serde(flatten)]
84    pub summary: JobSummary,
85    /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
86    pub steps: Vec<Step>,
87    /// Its record, for a failed job of a profile that records failures.
88    #[serde(default)]
89    pub record: Option<super::RecordNote>,
90    /// How its runner was admitted; `None` for one started before runnerd recorded it.
91    #[serde(default)]
92    pub admission: Option<super::Admission>,
93}
94
95impl Job {
96    /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
97    /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
98    /// limit as configured now. A reservation of the whole cap is one line.
99    pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
100        let limits = self.limits(profile);
101        let admission = self.admission.as_ref();
102        let name = profile.map_or("profile", |profile| profile.name.as_str());
103        ChartLines {
104            memory: chart_lines(
105                limits.memory,
106                admission.map(|admission| admission.reserved),
107                |value| ByteSize(value).to_string(),
108                name,
109            ),
110            cpu: chart_lines(
111                limits.cpu,
112                admission
113                    .and_then(|admission| admission.cpu)
114                    .map(|cpu| cpu.reserved),
115                super::cpus,
116                name,
117            ),
118        }
119    }
120
121    /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
122    /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
123    /// reload that changes the profile never replaces a cap the record has.
124    pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
125        let runner = self.summary.runner.as_ref();
126        let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
127            (Some(value), _) => Some(Limit {
128                value,
129                source: LimitSource::Cap,
130            }),
131            (None, Some(value)) => Some(Limit {
132                value,
133                source: LimitSource::ProfileNow,
134            }),
135            (None, None) => None,
136        };
137        Limits {
138            memory: limit(
139                runner.and_then(|runner| runner.memory_cap),
140                profile.map(|profile| profile.memory),
141            ),
142            cpu: limit(
143                runner.and_then(|runner| runner.cpu_cap),
144                profile.map(|profile| u64::from(profile.cpus) * 1000),
145            ),
146        }
147    }
148}
149
150/// What a job's container may use, from `Job::limits`.
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
152pub struct Limits {
153    /// In bytes.
154    pub memory: Option<Limit>,
155    /// In millicores.
156    pub cpu: Option<Limit>,
157}
158
159impl Limits {
160    /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
161    /// now 6G and 4 CPUs`; `None` without either.
162    pub fn line(&self) -> Option<String> {
163        let memory = self
164            .memory
165            .map(|limit| (limit.source, ByteSize(limit.value).to_string()));
166        let cpu = self
167            .cpu
168            .map(|limit| (limit.source, super::cpus(limit.value)));
169        let clause = |source: LimitSource, figures: &str| match source {
170            LimitSource::Cap => format!("capped at {figures}"),
171            LimitSource::ProfileNow => format!("profile now {figures}"),
172        };
173        match (memory, cpu) {
174            (Some((memory_source, memory)), Some((cpu_source, cpu)))
175                if memory_source == cpu_source =>
176            {
177                Some(clause(memory_source, &format!("{memory} and {cpu}")))
178            }
179            (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
180                "{}; {}",
181                clause(memory_source, &memory),
182                clause(cpu_source, &cpu)
183            )),
184            (Some((source, figure)), None) | (None, Some((source, figure))) => {
185                Some(clause(source, &figure))
186            }
187            (None, None) => None,
188        }
189    }
190}
191
192/// One resource's limit for a job's container.
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194pub struct Limit {
195    /// Bytes or millicores.
196    pub value: u64,
197    /// Where it comes from.
198    pub source: LimitSource,
199}
200
201/// Where a job's limit comes from.
202#[derive(Debug, Clone, Copy, PartialEq, Eq)]
203pub enum LimitSource {
204    /// The cap its container ran with.
205    Cap,
206    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
207    /// changed it.
208    ProfileNow,
209}
210
211/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
212fn chart_lines(
213    limit: Option<Limit>,
214    reserved: Option<u64>,
215    value: fn(u64) -> String,
216    profile: &str,
217) -> Vec<ChartLine> {
218    let line = |figure, kind, label: &str| ChartLine {
219        value: figure,
220        kind,
221        label: format!("{label} {}", value(figure)),
222    };
223    let mut lines = vec![];
224    match limit {
225        Some(Limit {
226            value: cap,
227            source: LimitSource::Cap,
228        }) if reserved == Some(cap) => {
229            return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
230        }
231        Some(Limit {
232            value: cap,
233            source: LimitSource::Cap,
234        }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
235        Some(Limit {
236            value: now,
237            source: LimitSource::ProfileNow,
238        }) => lines.push(line(
239            now,
240            ChartLineKind::ProfileNow,
241            &format!("{profile} now"),
242        )),
243        None => {}
244    }
245    if let Some(reserved) = reserved {
246        lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
247    }
248    lines
249}
250
251/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
252#[derive(Debug, Clone, PartialEq, Eq)]
253pub struct ChartLines {
254    /// In bytes.
255    pub memory: Vec<ChartLine>,
256    /// In millicores.
257    pub cpu: Vec<ChartLine>,
258}
259
260/// A line across a job's memory or CPU chart.
261#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct ChartLine {
263    /// Bytes or millicores.
264    pub value: u64,
265    /// What it stands for.
266    pub kind: ChartLineKind,
267    /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
268    pub label: String,
269}
270
271/// What a line across a job's chart stands for.
272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
273pub enum ChartLineKind {
274    /// The most its container may use: its cap.
275    Cap,
276    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
277    /// changed it.
278    ProfileNow,
279    /// What admission counted against the budget for its runner.
280    Reserved,
281    /// A reservation of the whole cap, as admission by caps makes.
282    CapAndReserved,
283}
284
285/// The runner a job ran on.
286#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
287pub struct JobRunner {
288    /// The runner's name, also its container's.
289    pub name: String,
290    /// The profile it ran in.
291    pub profile: String,
292    /// The slot it ran in, 1-based.
293    pub slot: usize,
294    /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
295    /// from before runnerd kept it.
296    pub memory_cap: Option<u64>,
297    /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
298    /// records from before runnerd kept it, or when Docker reports no limit.
299    pub cpu_cap: Option<u64>,
300}
301
302/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
303#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
304pub struct Timeline {
305    /// GitHub queued the job.
306    pub queued_at: u64,
307    /// The container of the runner that took it started.
308    pub container_started_at: Option<u64>,
309    /// The runner started the job.
310    pub started_at: Option<u64>,
311    /// The job finished, by GitHub's clock; or when its container was found gone.
312    pub finished_at: Option<u64>,
313    /// The container and the runner's registration were removed.
314    pub cleaned_at: Option<u64>,
315    /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
316    /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
317    #[serde(default)]
318    pub unseen: u64,
319    /// When runnerd first listed it queued (ADR 0092). `None` for a job no listing showed queued, such as one a
320    /// warm runner took first, and for one an older runnerd recorded.
321    #[serde(default)]
322    pub seen_at: Option<u64>,
323}
324
325impl Timeline {
326    /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
327    /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
328    pub fn runner_from(&self) -> Option<u64> {
329        self.container_started_at
330            .map(|started| started.max(self.queued_at))
331    }
332}
333
334/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
335#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
336pub struct JobMetrics {
337    /// CPU time, in milliseconds.
338    pub cpu_time: Option<u64>,
339    /// The highest CPU use of a sample interval, in millicores.
340    pub cpu_peak: Option<u64>,
341    /// CPU time over wall time, in millicores.
342    pub cpu_average: Option<u64>,
343    /// The highest memory use without page cache.
344    pub memory_peak: Option<u64>,
345    /// The average memory use without page cache.
346    pub memory_average: Option<u64>,
347    /// Bytes the container received.
348    pub network_received: Option<u64>,
349    /// Bytes the container sent.
350    pub network_sent: Option<u64>,
351    /// Of the bytes received, those the Docker Hub mirror served.
352    pub mirror_bytes: u64,
353    /// What the shared build cache served it and took from it.
354    pub build_cache: BuildCacheCounts,
355    /// Times the kernel held its container's sockets' receive windows small because the container's memory was
356    /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
357    /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
358    #[serde(default, skip_serializing_if = "Option::is_none")]
359    pub sock_throttled: Option<u64>,
360    /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
361    /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
362    #[serde(default, skip_serializing_if = "Option::is_none")]
363    pub cpu_periods: Option<u64>,
364    /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
365    /// cap held it back.
366    #[serde(default, skip_serializing_if = "Option::is_none")]
367    pub cpu_throttled: Option<u64>,
368    /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
369    /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
370    #[serde(default, skip_serializing_if = "Option::is_none")]
371    pub cpu_stalled: Option<u64>,
372}
373
374/// One figure of what a job used, worded for every client.
375#[derive(Debug, Clone, PartialEq, Eq)]
376pub struct Total {
377    /// What it measures: `Memory peak`, `Received`.
378    pub label: &'static str,
379    /// The figure with its unit.
380    pub value: String,
381}
382
383impl JobMetrics {
384    /// What the job used, over its whole life: memory, CPU and network, each where the daemon measured it. The
385    /// peaks are of the samples taken every few seconds, so the figures' note says that
386    /// ([`sampled_note`](super::sampled_note)).
387    pub fn totals(&self) -> Vec<Total> {
388        let mut totals = Vec::new();
389        let mut add = |label: &'static str, value: Option<String>| {
390            totals.extend(value.map(|value| Total { label, value }));
391        };
392        let bytes = |bytes: u64| ByteSize(bytes).to_string();
393        add("Memory peak", self.memory_peak.map(bytes));
394        add("Average memory", self.memory_average.map(bytes));
395        add("CPU peak", self.cpu_peak.map(in_cores));
396        add("Average CPU", self.cpu_average.map(in_cores));
397        add("CPU time", self.cpu_time.map(cpu_time));
398        add("Received", self.network_received.map(bytes));
399        add("Sent", self.network_sent.map(bytes));
400        // Counted from the mirror's own log, so a job without network samples still has it.
401        add(
402            "From the mirror",
403            (self.network_received.is_some() || self.mirror_bytes > 0)
404                .then(|| bytes(self.mirror_bytes)),
405        );
406        add(
407            "Throttled for memory pressure",
408            self.sock_throttled
409                .filter(|times| *times > 0)
410                .map(|times| format!("{times} times")),
411        );
412        totals
413    }
414}
415
416/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
417#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
418pub struct BuildCacheCounts {
419    /// Lookups that found an entry.
420    pub hits: u64,
421    /// Lookups that found none.
422    pub misses: u64,
423    /// Bytes of the entries it served.
424    pub read_bytes: u64,
425    /// Bytes of the entries it stored.
426    pub written_bytes: u64,
427    /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
428    /// cache was full.
429    pub refused: u64,
430    /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
431    pub writer: bool,
432}
433
434impl BuildCacheCounts {
435    /// Hits and misses.
436    pub fn lookups(&self) -> u64 {
437        self.hits + self.misses
438    }
439
440    /// Hits in percent of lookups, `None` without any.
441    pub fn hit_percent(&self) -> Option<u64> {
442        let lookups = self.lookups();
443        (lookups > 0).then(|| self.hits * 100 / lookups)
444    }
445
446    /// Adds `other`'s counts; `writer` stays.
447    pub fn add(&mut self, other: &BuildCacheCounts) {
448        self.hits += other.hits;
449        self.misses += other.misses;
450        self.read_bytes += other.read_bytes;
451        self.written_bytes += other.written_bytes;
452        self.refused += other.refused;
453    }
454}
455
456/// A job's stored log.
457#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
458pub struct LogSummary {
459    /// Lines stored.
460    pub lines: u64,
461    /// Bytes stored.
462    pub bytes: u64,
463    /// Set once the log reached `history.log_max` and later lines were dropped.
464    pub truncated: bool,
465}
466
467/// One step of a job, as GitHub reports it.
468#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
469pub struct Step {
470    /// GitHub's step number; numbers may skip.
471    pub number: u32,
472    /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
473    pub name: String,
474    /// `pending`, `queued`, `in_progress` or `completed`.
475    pub status: String,
476    /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
477    pub conclusion: Option<String>,
478    /// When it started.
479    pub started_at: Option<u64>,
480    /// When it completed.
481    pub completed_at: Option<u64>,
482}
483
484/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time).
485#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
486pub struct JobQuery {
487    /// Only this `owner/name`.
488    pub repository: Option<String>,
489    /// Only jobs in this state.
490    pub state: Option<JobState>,
491    /// Only jobs with this conclusion, such as `failure`.
492    pub conclusion: Option<String>,
493    /// Only jobs of the workflow with this name, such as `CI`.
494    pub workflow: Option<String>,
495    /// Only jobs with this name in their workflow, such as `build`.
496    pub name: Option<String>,
497    /// Only jobs queued at or after this time.
498    pub since: Option<u64>,
499    /// Only jobs queued before this time.
500    pub until: Option<u64>,
501    /// Only jobs with a smaller ID: the `next` of the previous page.
502    pub before: Option<u64>,
503    /// At most this many jobs; default 50, at most 500.
504    pub limit: Option<u32>,
505}
506
507/// `GET /v1/jobs`: one page of jobs.
508#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
509pub struct JobPage {
510    /// Newest first.
511    pub jobs: Vec<JobSummary>,
512    /// Pass as `before` for the next page; `None` on the last page.
513    pub next: Option<u64>,
514}
515
516/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
517#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
518pub struct LogQuery {
519    /// Start at this line, 0-based.
520    pub from: Option<u64>,
521    /// Keep the response open and send lines as they are stored, until the job finishes.
522    #[serde(default)]
523    pub follow: bool,
524}
525
526/// One sample of a container's use.
527#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
528pub struct Sample {
529    /// When it was taken.
530    pub at: u64,
531    /// CPU use since the previous sample, in millicores.
532    pub cpu: u64,
533    /// Memory use without page cache.
534    pub memory: u64,
535    /// Bytes received since the container started.
536    pub network_received: u64,
537    /// Bytes sent since the container started.
538    pub network_sent: u64,
539}
540
541/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
542/// older than `history.samples` are deleted; the job's metrics stay.
543#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
544pub struct SampleSeries {
545    /// The job.
546    pub job: u64,
547    /// Seconds between samples.
548    pub interval: u64,
549    /// Oldest first.
550    pub samples: Vec<Sample>,
551}
552
553/// One sample of the whole slice runners run in.
554#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
555pub struct HostSample {
556    /// When it was taken.
557    pub at: u64,
558    /// CPU use of the slice since the previous sample, in millicores.
559    pub cpu: u64,
560    /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
561    /// since active page cache still counts, which the high-water mark of measured admission compares to
562    /// `MemoryMax`.
563    pub memory: u64,
564}
565
566/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
567/// (ADR 0055). Read every few seconds; not kept in the history.
568#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
569pub struct HostMemory {
570    /// When it was read.
571    pub at: u64,
572    /// `MemTotal` in `/proc/meminfo`, in bytes.
573    pub total: u64,
574    /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
575    pub available: u64,
576    /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
577    /// out.
578    pub slice: u64,
579    /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
580    /// kernel without pressure stall information.
581    pub some: Option<Pressure>,
582    /// Its `full` line: the share of time every task that could run waited for memory.
583    pub full: Option<Pressure>,
584    /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
585    /// Read less often than the rest. Empty when none is that large.
586    #[serde(default, skip_serializing_if = "Vec::is_empty")]
587    pub holders: Vec<Holder>,
588    /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
589    /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
590    #[serde(default, skip_serializing_if = "Option::is_none")]
591    pub ceiling: Option<u64>,
592    /// `SwapTotal` and `SwapFree`, from the same read of `/proc/meminfo`; absent on a host without swap (ADR 0101).
593    /// Admission counts none of it.
594    #[serde(default, skip_serializing_if = "Option::is_none")]
595    pub swap: Option<Swap>,
596}
597
598/// A host's swap at one reading: zram and zswap's device included, as `/proc/meminfo` counts it.
599#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
600pub struct Swap {
601    /// `SwapTotal`, in bytes. Never 0: a host without swap has none.
602    pub total: u64,
603    /// `SwapFree`, in bytes.
604    pub free: u64,
605}
606
607impl Swap {
608    /// What is in use, in bytes.
609    pub fn used(&self) -> u64 {
610        self.total.saturating_sub(self.free)
611    }
612}
613
614/// The margin kept free at least: 2G.
615const MARGIN_MIN: u64 = 2 << 30;
616
617/// The margin as a share of the host's memory, as a divisor: a sixteenth, 4G of 64G.
618const MARGIN_SHARE: u64 = 16;
619
620/// The plain margin `MemAvailable` keeps beside the runners on a host with `total` memory: the larger of `MARGIN_MIN`
621/// and `total / MARGIN_SHARE` (ADR 0055).
622pub fn host_margin(total: u64) -> u64 {
623    MARGIN_MIN.max(total / MARGIN_SHARE)
624}
625
626impl HostMemory {
627    /// What the host has for a new runner this moment, in bytes: `MemAvailable` and what it can still grow by, less
628    /// the plain margin. Unlike `HostBudget::spare` it does not count what runners reserve past what they hold, which
629    /// is memory they may yet take, not memory taken.
630    pub fn room(&self) -> u64 {
631        self.available
632            .saturating_add(self.growable())
633            .saturating_sub(host_margin(self.grown()))
634    }
635
636    /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
637    pub fn grown(&self) -> u64 {
638        self.ceiling
639            .map_or(self.total, |ceiling| ceiling.max(self.total))
640    }
641
642    /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
643    /// reached it.
644    pub fn growable(&self) -> u64 {
645        self.grown().saturating_sub(self.total)
646    }
647
648    /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
649    /// page cache it can drop.
650    pub fn outside(&self) -> u64 {
651        self.total
652            .saturating_sub(self.available)
653            .saturating_sub(self.slice)
654    }
655
656    /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
657    /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
658    pub fn outside_line(&self) -> String {
659        let size = |bytes: u64| ByteSize(bytes).to_string();
660        let line = format!("programs outside CI hold {}", size(self.outside()));
661        let named: Vec<String> = self
662            .holders
663            .iter()
664            .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
665            .collect();
666        match named.as_slice() {
667            [] => line,
668            [one] => format!("{line}, most {one}"),
669            [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
670        }
671    }
672}
673
674/// A program outside the slice and the memory its processes hold.
675#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
676pub struct Holder {
677    /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
678    pub name: String,
679    /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
680    pub memory: u64,
681}
682
683/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
684#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
685pub struct Pressure {
686    /// Over the last 10 seconds.
687    pub avg10: u32,
688    /// Over the last 60 seconds.
689    pub avg60: u32,
690}
691
692/// `GET /v1/host/samples` query parameters.
693#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
694pub struct HostSampleQuery {
695    /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
696    pub since: Option<u64>,
697}
698
699/// `GET /v1/host/samples`: the slice's use over time, oldest first.
700#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
701pub struct HostSamples {
702    /// Seconds between samples.
703    pub interval: u64,
704    /// Oldest first.
705    pub samples: Vec<HostSample>,
706}
707
708/// How `GET /v1/stats` groups jobs in time.
709#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
710#[serde(rename_all = "kebab-case")]
711pub enum Bucket {
712    /// By UTC hour.
713    Hour,
714    /// By UTC day.
715    #[default]
716    Day,
717}
718
719/// `GET /v1/activity` query parameters.
720#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
721pub struct ActivityQuery {
722    /// The start of the window; default 24 hours ago. It ends now.
723    pub since: Option<u64>,
724}
725
726/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
727/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
728#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
729pub struct Activity {
730    /// The start of the window.
731    pub since: u64,
732    /// When it was read: the end of the window.
733    pub until: u64,
734    /// Each job that started in the window, oldest start first.
735    pub started: Vec<Started>,
736    /// The jobs that finished in the window: completed, or lost with their runner.
737    pub finished: u64,
738    /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
739    pub failed: u64,
740    /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
741    /// after the kernel killed a process of theirs for memory at their cap.
742    pub runner_failed: u64,
743}
744
745impl Activity {
746    /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
747    /// started.
748    pub fn wait_p95(&self) -> Option<u64> {
749        let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
750        waits.sort_unstable();
751        let rank = (waits.len() * 95).div_ceil(100);
752        waits.get(rank.checked_sub(1)?).copied()
753    }
754
755    /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
756    /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
757    pub fn wait_hours(&self) -> [u64; 24] {
758        const HOUR: u64 = 3600;
759        let since = self.until.saturating_sub(24 * HOUR);
760        let mut hours = [0u64; 24];
761        for start in self
762            .started
763            .iter()
764            .filter(|start| (since..=self.until).contains(&start.at))
765        {
766            // A start exactly 24 hours ago is in the oldest hour.
767            let back =
768                usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
769            hours[23 - back] = hours[23 - back].max(start.waited);
770        }
771        hours
772    }
773}
774
775/// A job that started: when, and how long it had waited since it was queued.
776#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
777pub struct Started {
778    /// When it started.
779    pub at: u64,
780    /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
781    pub waited: u64,
782}
783
784/// `GET /v1/stats` query parameters.
785#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
786pub struct StatsQuery {
787    /// Only this `owner/name`.
788    pub repository: Option<String>,
789    /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
790    /// are counted: 31 days of hours, two years of days.
791    pub since: Option<u64>,
792    /// Only jobs queued before this time.
793    pub until: Option<u64>,
794    /// The bucket size; default a day.
795    #[serde(default)]
796    pub bucket: Bucket,
797}
798
799/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
800#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
801pub struct Stats {
802    /// The bucket size.
803    pub bucket: Bucket,
804    /// By bucket start, then repository.
805    pub rows: Vec<StatsRow>,
806}
807
808/// The finished jobs of one repository in one bucket.
809#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
810pub struct StatsRow {
811    /// The bucket's start.
812    pub start: u64,
813    /// `owner/name`.
814    pub repository: String,
815    /// Jobs completed or lost.
816    pub jobs: u64,
817    /// Of those, concluded `success`.
818    pub succeeded: u64,
819    /// Of those, concluded `failure` or `timed_out`, or lost.
820    pub failed: u64,
821    /// Of those, concluded `cancelled`.
822    pub cancelled: u64,
823    /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
824    /// ran (ADR 0081).
825    pub queue_seconds: u64,
826    /// Seconds from started to finished, summed.
827    pub run_seconds: u64,
828    /// CPU time, in milliseconds, summed.
829    pub cpu_time: u64,
830    /// The highest memory peak of a job.
831    pub memory_peak: u64,
832    /// Bytes received, summed.
833    pub network_received: u64,
834    /// Bytes from the mirror, summed.
835    pub mirror_bytes: u64,
836}
837
838/// What a cache volume holds.
839#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
840#[serde(rename_all = "kebab-case", tag = "kind")]
841pub enum CacheKind {
842    /// A profile's read-only snapshot.
843    Snapshot {
844        /// The profile.
845        profile: String,
846    },
847    /// A slot's layer changes and Docker Engine data.
848    Slot {
849        /// The profile.
850        profile: String,
851        /// 1-based.
852        slot: usize,
853    },
854    /// The pnpm store all slots share.
855    Pnpm,
856    /// The Maven repository all slots share.
857    Maven,
858    /// The Docker Hub mirror's storage.
859    Mirror,
860    /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
861    Build,
862}
863
864/// One cache volume and its size.
865#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
866pub struct CacheVolume {
867    /// The Docker volume.
868    pub name: String,
869    /// What it holds.
870    #[serde(flatten)]
871    pub kind: CacheKind,
872    /// Its size at `measured_at`.
873    pub bytes: u64,
874    /// The size above which runnerd empties it, or for a snapshot starts it over at its next reseed, if it has one.
875    pub limit: Option<u64>,
876}
877
878/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
879/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
880#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
881pub struct Caches {
882    /// When the sizes were measured; `None` before the first measurement.
883    pub measured_at: Option<u64>,
884    /// By name.
885    pub volumes: Vec<CacheVolume>,
886    /// The shared build cache now, when it is on.
887    pub build: Option<BuildCacheStats>,
888}
889
890/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
891#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
892pub struct BuildCacheStats {
893    /// The entries' bytes.
894    pub bytes: u64,
895    /// How many entries it holds.
896    pub entries: u64,
897    /// `[caches.build] max_size`.
898    pub limit: u64,
899    /// The runners of the last 7 days, summed.
900    pub week: BuildCacheCounts,
901    /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
902    /// daemon older than it, which never served it.
903    #[serde(default)]
904    pub actions_cache: bool,
905}
906
907/// `GET /v1/caches/history` query parameters.
908#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
909pub struct CacheHistoryQuery {
910    /// Only measurements at or after this time; default 30 days ago.
911    pub since: Option<u64>,
912}
913
914/// One volume's size at one measurement.
915#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
916pub struct CacheSize {
917    /// When it was measured.
918    pub at: u64,
919    /// The Docker volume.
920    pub name: String,
921    /// Its size.
922    pub bytes: u64,
923}
924
925/// `GET /v1/caches/history`: every measurement, oldest first.
926#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
927pub struct CacheHistory {
928    /// Oldest first.
929    pub sizes: Vec<CacheSize>,
930}