Skip to main content

runnerd_core/api/
jobs.rs

1//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
2//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
3//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).
4
5use serde::{Deserialize, Serialize};
6
7use super::{cpu_time, in_cores};
8use crate::config::ByteSize;
9
10mod compare;
11mod facts;
12mod log;
13mod query;
14mod timeline;
15
16pub(crate) use log::plain;
17
18#[cfg(feature = "host")]
19pub(crate) use compare::SAMPLE_GAP_INTERVALS;
20pub use compare::{
21    Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind, CompareQuery,
22    ComparedSteps, FigureDelta, FigureKind, JobComparison, StepDelta, StepMatch, StepUsage,
23    Unavailable,
24};
25pub use facts::{
26    JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
27    job_title,
28};
29#[cfg(feature = "host")]
30pub(crate) use log::TITLE_SOURCE_MAX;
31pub use log::{
32    Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
33    LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
34    LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
35    SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
36};
37pub use query::{
38    JOB_LABELS_MAX, JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT,
39    JOBS_PAGE_MAX, JobOrder, JobRange, JobSort, RecordFilter, RecordMark, SortOrder, row_bytes,
40};
41pub use timeline::{
42    BarEnd, Gap, GapKind, Lane, RunnerSpan, SlotKey, SlotTimeline, SlottedBar, SlottedRunner, Span,
43    TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
44    TimelineRows,
45};
46
47/// Where a job is in its life, as this daemon saw it.
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
49#[serde(rename_all = "kebab-case")]
50pub enum JobState {
51    /// Waiting for a runner.
52    Queued,
53    /// A runner of this daemon took it.
54    Running,
55    /// GitHub reports it completed; `conclusion` says how.
56    Completed,
57    /// Its runner's container is gone and GitHub did not report the job completed in time.
58    Lost,
59    /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
60    /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
61    Elsewhere,
62}
63
64impl JobState {
65    /// Whether the job can still change.
66    pub fn is_final(self) -> bool {
67        matches!(
68            self,
69            JobState::Completed | JobState::Lost | JobState::Elsewhere
70        )
71    }
72}
73
74/// A job without its steps: what lists and events carry.
75#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
76pub struct JobSummary {
77    /// GitHub's job ID, also its check run ID.
78    pub id: u64,
79    /// `owner/name` on GitHub.
80    pub repository: String,
81    /// The workflow run it belongs to.
82    pub run_id: u64,
83    /// 1 for the first attempt of the run, higher for re-runs.
84    pub run_attempt: u32,
85    /// The workflow's name.
86    pub workflow: Option<String>,
87    /// The job's name in the workflow.
88    pub name: String,
89    /// The branch the run is for.
90    pub branch: Option<String>,
91    /// The job's page on GitHub.
92    pub url: Option<String>,
93    /// The workflow's `runs-on` labels.
94    pub labels: Vec<String>,
95    /// Where the job is.
96    pub state: JobState,
97    /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
98    pub conclusion: Option<String>,
99    /// The runner that ran it.
100    pub runner: Option<JobRunner>,
101    /// When each phase started.
102    pub timeline: Timeline,
103    /// CPU, memory and network use while it ran.
104    pub metrics: JobMetrics,
105    /// How much of its log is stored.
106    pub log: LogSummary,
107    /// Whether the runner failed it rather than its code (`record::runner_fault`): Sysbox lost the runner's seccomp
108    /// listener, so what its failed step printed has `function not implemented` on a mount, umount or openat2, or
109    /// `unsafe procfs detected` with `invalid argument`. Read from the log once the record is final; false for a job
110    /// that has not concluded failed, and in records from before it was kept.
111    #[serde(default)]
112    pub runner_fault: bool,
113    /// What its log says failed it, for a job that concluded failed and whose log shows one of the failures runnerd
114    /// tells apart (`record::failure_cause`): an action the runner could not download or unpack, one it could not
115    /// resolve, or `actions/checkout`. Read once the record is final; `None` for every other job, from a daemon older
116    /// than it, and where the log does not say.
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub failure_cause: Option<FailureCause>,
119    /// When runnerd asked GitHub to re-run its run's failed jobs because its runner failed it (ADR 0153), in seconds:
120    /// GitHub accepted, and the run went on to its next attempt. `None` for every other job, from a daemon older than
121    /// it, and where a person re-ran the run first.
122    #[serde(default, skip_serializing_if = "Option::is_none")]
123    pub rerun_at: Option<u64>,
124    /// What the runner image's probe found after the runner exited (ADR 0155): whether the calls Sysbox serves still
125    /// worked in its container when sampled. `None` where no verdict was recorded, including when collection is
126    /// disabled, a container stops early or the daemon is older. Recorded only; it decides nothing.
127    #[serde(default, skip_serializing_if = "Option::is_none")]
128    pub probe: Option<RunnerProbe>,
129    /// What became of its record, for a job the history notes one for: kept, removed or not made (ADR 0135).
130    /// `None` for a job without a record note, and from a daemon older than the mark. Not `record`, which a `Job`
131    /// that flattens this summary holds in full.
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub record_mark: Option<RecordMark>,
134}
135
136/// What a failed job's log says failed it (`record::failure_cause`), apart from its own code.
137#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
138#[serde(rename_all = "kebab-case")]
139pub enum FailureCause {
140    /// While it set up, the runner could not download an action's archive after its retries, or could not unpack
141    /// the archive it had.
142    ActionArchive,
143    /// While it set up, the runner could not resolve an action: GitHub's API did not say where to download it.
144    ActionResolve,
145    /// The step that failed is `actions/checkout`: the repository could not be fetched.
146    Checkout,
147    /// A cause a newer daemon tells apart.
148    #[serde(other)]
149    Unknown,
150}
151
152/// What the probe a runner's entrypoint runs as root after the runner exited found (ADR 0155): it mounts a tmpfs,
153/// a proc and a sysfs and binds one in a mount namespace of its own, then opens a file under `/proc/sys` with
154/// `openat2`, the calls Sysbox serves and a runner fault breaks.
155#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
156#[serde(rename_all = "kebab-case")]
157pub enum RunnerProbe {
158    /// Every call worked.
159    Clean,
160    /// A mount or an unmount failed; `/proc` opened.
161    Mount,
162    /// `/proc` did not open with `openat2`; the mounts worked.
163    Procfs,
164    /// Both failed.
165    MountAndProcfs,
166    /// It did not finish within its time.
167    TimedOut,
168    /// It ran but could not test: it failed before its first call.
169    Broken,
170    /// A verdict a newer daemon tells apart.
171    #[serde(other)]
172    Unknown,
173}
174
175/// `GET /v1/jobs/{id}`: a job with its steps.
176#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
177pub struct Job {
178    /// Everything but the steps.
179    #[serde(flatten)]
180    pub summary: JobSummary,
181    /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
182    pub steps: Vec<Step>,
183    /// Its record, for a failed job of a profile that records failures.
184    #[serde(default)]
185    pub record: Option<super::RecordNote>,
186    /// How its runner was admitted; `None` for one started before runnerd recorded it.
187    #[serde(default)]
188    pub admission: Option<super::Admission>,
189}
190
191impl Job {
192    /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
193    /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
194    /// limit as configured now. A reservation of the whole cap is one line.
195    pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
196        let limits = self.limits(profile);
197        let admission = self.admission.as_ref();
198        let name = profile.map_or("profile", |profile| profile.name.as_str());
199        ChartLines {
200            memory: chart_lines(
201                limits.memory,
202                admission.map(|admission| admission.reserved),
203                |value| ByteSize(value).to_string(),
204                name,
205            ),
206            cpu: chart_lines(
207                limits.cpu,
208                admission
209                    .and_then(|admission| admission.cpu)
210                    .map(|cpu| cpu.reserved),
211                super::cpus,
212                name,
213            ),
214        }
215    }
216
217    /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
218    /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
219    /// reload that changes the profile never replaces a cap the record has.
220    pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
221        let runner = self.summary.runner.as_ref();
222        let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
223            (Some(value), _) => Some(Limit {
224                value,
225                source: LimitSource::Cap,
226            }),
227            (None, Some(value)) => Some(Limit {
228                value,
229                source: LimitSource::ProfileNow,
230            }),
231            (None, None) => None,
232        };
233        Limits {
234            memory: limit(
235                runner.and_then(|runner| runner.memory_cap),
236                profile.map(|profile| profile.memory),
237            ),
238            cpu: limit(
239                runner.and_then(|runner| runner.cpu_cap),
240                profile.map(|profile| u64::from(profile.cpus) * 1000),
241            ),
242        }
243    }
244}
245
246/// What a job's container may use, from `Job::limits`.
247#[derive(Debug, Clone, Copy, PartialEq, Eq)]
248pub struct Limits {
249    /// In bytes.
250    pub memory: Option<Limit>,
251    /// In millicores.
252    pub cpu: Option<Limit>,
253}
254
255impl Limits {
256    /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
257    /// now 6G and 4 CPUs`; `None` without either.
258    pub fn line(&self) -> Option<String> {
259        let memory = self
260            .memory
261            .map(|limit| (limit.source, ByteSize(limit.value).to_string()));
262        let cpu = self
263            .cpu
264            .map(|limit| (limit.source, super::cpus(limit.value)));
265        let clause = |source: LimitSource, figures: &str| match source {
266            LimitSource::Cap => format!("capped at {figures}"),
267            LimitSource::ProfileNow => format!("profile now {figures}"),
268        };
269        match (memory, cpu) {
270            (Some((memory_source, memory)), Some((cpu_source, cpu)))
271                if memory_source == cpu_source =>
272            {
273                Some(clause(memory_source, &format!("{memory} and {cpu}")))
274            }
275            (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
276                "{}; {}",
277                clause(memory_source, &memory),
278                clause(cpu_source, &cpu)
279            )),
280            (Some((source, figure)), None) | (None, Some((source, figure))) => {
281                Some(clause(source, &figure))
282            }
283            (None, None) => None,
284        }
285    }
286}
287
288/// One resource's limit for a job's container.
289#[derive(Debug, Clone, Copy, PartialEq, Eq)]
290pub struct Limit {
291    /// Bytes or millicores.
292    pub value: u64,
293    /// Where it comes from.
294    pub source: LimitSource,
295}
296
297/// Where a job's limit comes from.
298#[derive(Debug, Clone, Copy, PartialEq, Eq)]
299pub enum LimitSource {
300    /// The cap its container ran with.
301    Cap,
302    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
303    /// changed it.
304    ProfileNow,
305}
306
307/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
308fn chart_lines(
309    limit: Option<Limit>,
310    reserved: Option<u64>,
311    value: fn(u64) -> String,
312    profile: &str,
313) -> Vec<ChartLine> {
314    let line = |figure, kind, label: &str| ChartLine {
315        value: figure,
316        kind,
317        label: format!("{label} {}", value(figure)),
318    };
319    let mut lines = vec![];
320    match limit {
321        Some(Limit {
322            value: cap,
323            source: LimitSource::Cap,
324        }) if reserved == Some(cap) => {
325            return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
326        }
327        Some(Limit {
328            value: cap,
329            source: LimitSource::Cap,
330        }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
331        Some(Limit {
332            value: now,
333            source: LimitSource::ProfileNow,
334        }) => lines.push(line(
335            now,
336            ChartLineKind::ProfileNow,
337            &format!("{profile} now"),
338        )),
339        None => {}
340    }
341    if let Some(reserved) = reserved {
342        lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
343    }
344    lines
345}
346
347/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
348#[derive(Debug, Clone, PartialEq, Eq)]
349pub struct ChartLines {
350    /// In bytes.
351    pub memory: Vec<ChartLine>,
352    /// In millicores.
353    pub cpu: Vec<ChartLine>,
354}
355
356/// A line across a job's memory or CPU chart.
357#[derive(Debug, Clone, PartialEq, Eq)]
358pub struct ChartLine {
359    /// Bytes or millicores.
360    pub value: u64,
361    /// What it stands for.
362    pub kind: ChartLineKind,
363    /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
364    pub label: String,
365}
366
367/// What a line across a job's chart stands for.
368#[derive(Debug, Clone, Copy, PartialEq, Eq)]
369pub enum ChartLineKind {
370    /// The most its container may use: its cap.
371    Cap,
372    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
373    /// changed it.
374    ProfileNow,
375    /// What admission counted against the budget for its runner.
376    Reserved,
377    /// A reservation of the whole cap, as admission by caps makes.
378    CapAndReserved,
379}
380
381/// The runner a job ran on.
382#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
383pub struct JobRunner {
384    /// The runner's name, also its container's.
385    pub name: String,
386    /// The profile it ran in.
387    pub profile: String,
388    /// The slot it ran in, 1-based.
389    pub slot: usize,
390    /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
391    /// from before runnerd kept it.
392    pub memory_cap: Option<u64>,
393    /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
394    /// records from before runnerd kept it, or when Docker reports no limit.
395    pub cpu_cap: Option<u64>,
396}
397
398/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
399#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
400pub struct Timeline {
401    /// GitHub queued the job.
402    pub queued_at: u64,
403    /// The container of the runner that took it started.
404    pub container_started_at: Option<u64>,
405    /// The runner started the job.
406    pub started_at: Option<u64>,
407    /// The job finished, by GitHub's clock; or when its container was found gone.
408    pub finished_at: Option<u64>,
409    /// The container and the runner's registration were removed.
410    pub cleaned_at: Option<u64>,
411    /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
412    /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
413    #[serde(default)]
414    pub unseen: u64,
415    /// When runnerd first listed it queued (ADR 0092). `None` for a job no listing showed queued, such as one a
416    /// warm runner took first, and for one an older runnerd recorded.
417    #[serde(default)]
418    pub seen_at: Option<u64>,
419}
420
421impl Timeline {
422    /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
423    /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
424    pub fn runner_from(&self) -> Option<u64> {
425        self.container_started_at
426            .map(|started| started.max(self.queued_at))
427    }
428}
429
430/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
431#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
432pub struct JobMetrics {
433    /// CPU time, in milliseconds.
434    pub cpu_time: Option<u64>,
435    /// The highest CPU use of a sample interval, in millicores.
436    pub cpu_peak: Option<u64>,
437    /// CPU time over wall time, in millicores.
438    pub cpu_average: Option<u64>,
439    /// The highest memory use without page cache.
440    pub memory_peak: Option<u64>,
441    /// The average memory use without page cache.
442    pub memory_average: Option<u64>,
443    /// Bytes the container received.
444    pub network_received: Option<u64>,
445    /// Bytes the container sent.
446    pub network_sent: Option<u64>,
447    /// Of the bytes received, those the Docker Hub mirror served.
448    pub mirror_bytes: u64,
449    /// What the shared build cache served it and took from it.
450    pub build_cache: BuildCacheCounts,
451    /// Times the kernel held its container's sockets' receive windows small because the container's memory was
452    /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
453    /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
454    #[serde(default, skip_serializing_if = "Option::is_none")]
455    pub sock_throttled: Option<u64>,
456    /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
457    /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
458    #[serde(default, skip_serializing_if = "Option::is_none")]
459    pub cpu_periods: Option<u64>,
460    /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
461    /// cap held it back.
462    #[serde(default, skip_serializing_if = "Option::is_none")]
463    pub cpu_throttled: Option<u64>,
464    /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
465    /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
466    #[serde(default, skip_serializing_if = "Option::is_none")]
467    pub cpu_stalled: Option<u64>,
468    /// What its runner container wrote to the disks under Docker's root (ADR 0143). `None` where no reading measured
469    /// it, as without the io controller, and in records from before.
470    #[serde(default, skip_serializing_if = "Option::is_none")]
471    pub disk: Option<super::DiskWritten>,
472}
473
474/// One figure of what a job used, worded for every client.
475#[derive(Debug, Clone, PartialEq, Eq)]
476pub struct Total {
477    /// What it measures: `Memory peak`, `Received`.
478    pub label: &'static str,
479    /// The figure with its unit.
480    pub value: String,
481}
482
483impl JobMetrics {
484    /// What the job used, over its whole life: memory, CPU and network, each where the daemon measured it. The
485    /// peaks are of the samples taken every few seconds, so the figures' note says that
486    /// ([`sampled_note`](super::sampled_note)).
487    pub fn totals(&self) -> Vec<Total> {
488        let mut totals = Vec::new();
489        let mut add = |label: &'static str, value: Option<String>| {
490            totals.extend(value.map(|value| Total { label, value }));
491        };
492        let bytes = |bytes: u64| ByteSize(bytes).to_string();
493        add("Memory peak", self.memory_peak.map(bytes));
494        add("Average memory", self.memory_average.map(bytes));
495        add("CPU peak", self.cpu_peak.map(in_cores));
496        add("Average CPU", self.cpu_average.map(in_cores));
497        add("CPU time", self.cpu_time.map(cpu_time));
498        add("Received", self.network_received.map(bytes));
499        add("Sent", self.network_sent.map(bytes));
500        add("Written to disk", self.disk.map(|disk| disk.line()));
501        // Counted from the mirror's own log, so a job without network samples still has it.
502        add(
503            "From the mirror",
504            (self.network_received.is_some() || self.mirror_bytes > 0)
505                .then(|| bytes(self.mirror_bytes)),
506        );
507        add(
508            "Throttled for memory pressure",
509            self.sock_throttled
510                .filter(|times| *times > 0)
511                .map(|times| format!("{times} times")),
512        );
513        totals
514    }
515}
516
517/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
518#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
519pub struct BuildCacheCounts {
520    /// Lookups that found an entry.
521    pub hits: u64,
522    /// Lookups that found none.
523    pub misses: u64,
524    /// Bytes of the entries it served.
525    pub read_bytes: u64,
526    /// Bytes of the entries it stored.
527    pub written_bytes: u64,
528    /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
529    /// cache was full.
530    pub refused: u64,
531    /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
532    pub writer: bool,
533}
534
535impl BuildCacheCounts {
536    /// Hits and misses.
537    pub fn lookups(&self) -> u64 {
538        self.hits + self.misses
539    }
540
541    /// Hits in percent of lookups, `None` without any.
542    pub fn hit_percent(&self) -> Option<u64> {
543        let lookups = self.lookups();
544        (lookups > 0).then(|| self.hits * 100 / lookups)
545    }
546
547    /// Adds `other`'s counts; `writer` stays.
548    pub fn add(&mut self, other: &BuildCacheCounts) {
549        self.hits += other.hits;
550        self.misses += other.misses;
551        self.read_bytes += other.read_bytes;
552        self.written_bytes += other.written_bytes;
553        self.refused += other.refused;
554    }
555}
556
557/// A job's stored log.
558#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
559pub struct LogSummary {
560    /// Lines stored.
561    pub lines: u64,
562    /// Bytes stored.
563    pub bytes: u64,
564    /// Set once the log reached `history.log_max` and later lines were dropped.
565    pub truncated: bool,
566}
567
568/// One step of a job, as GitHub reports it.
569#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
570pub struct Step {
571    /// GitHub's step number; numbers may skip.
572    pub number: u32,
573    /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
574    pub name: String,
575    /// `pending`, `queued`, `in_progress` or `completed`.
576    pub status: String,
577    /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
578    pub conclusion: Option<String>,
579    /// When it started.
580    pub started_at: Option<u64>,
581    /// When it completed.
582    pub completed_at: Option<u64>,
583}
584
585/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time), or by `sort` (ADR 0135).
586/// [`JobQuery::check`] says which queries the daemon refuses.
587#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
588pub struct JobQuery {
589    /// Only this `owner/name`.
590    pub repository: Option<String>,
591    /// Only jobs in this state.
592    pub state: Option<JobState>,
593    /// Only jobs with this conclusion, such as `failure`.
594    pub conclusion: Option<String>,
595    /// Only jobs of the workflow with this name, such as `CI`.
596    pub workflow: Option<String>,
597    /// Only jobs with this name in their workflow, such as `build`.
598    pub name: Option<String>,
599    /// Only jobs queued at or after this time.
600    pub since: Option<u64>,
601    /// Only jobs queued before this time.
602    pub until: Option<u64>,
603    /// Only jobs with a smaller ID: the `next` of the previous page. Only in the default order, newest first.
604    pub before: Option<u64>,
605    /// At most this many jobs; default 50, at most 500. A page also ends once it holds
606    /// [`JOBS_PAGE_BYTES`] of text.
607    pub limit: Option<u32>,
608    /// Only jobs whose name, workflow or branch holds this text, as written: 1 to [`JOB_SEARCH_MAX`] bytes, ASCII
609    /// letters in either case. The history keeps no commit, so a commit is not searched.
610    #[serde(default, skip_serializing_if = "Option::is_none")]
611    pub q: Option<String>,
612    /// Only jobs of runs for this branch.
613    #[serde(default, skip_serializing_if = "Option::is_none")]
614    pub branch: Option<String>,
615    /// Only jobs whose record this says became of.
616    #[serde(default, skip_serializing_if = "Option::is_none")]
617    pub record: Option<RecordFilter>,
618    /// Sort by this; a job whose figure is unknown comes last either way, ties by ID. `queued`, newest first, is
619    /// the default order, which pages with `before`; any other order is one page, the first `limit` jobs.
620    #[serde(default, skip_serializing_if = "Option::is_none")]
621    pub sort: Option<JobSort>,
622    /// Which way; default largest or latest first.
623    #[serde(default, skip_serializing_if = "Option::is_none")]
624    pub order: Option<SortOrder>,
625    /// Refused: a sorted list is one page, which a search or the filters narrow. Read only to say so.
626    #[serde(default, skip_serializing_if = "Option::is_none")]
627    pub cursor: Option<String>,
628}
629
630/// One page of jobs, as `GET /v1/jobs` answered before it searched and sorted: what [`JobList`] holds.
631#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
632pub struct JobPage {
633    /// Newest first, or in the query's sort.
634    pub jobs: Vec<JobSummary>,
635    /// In the default order: pass as `before` for the next page; `None` on the last page, and in any other order.
636    pub next: Option<u64>,
637}
638
639/// `GET /v1/jobs`: one page of jobs (ADR 0135). A [`JobPage`] with fields more, so a client that reads a `JobPage`
640/// reads it too, and one that reads a `JobList` from an older daemon finds them absent.
641#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
642pub struct JobList {
643    /// The jobs and, in the default order, `next`.
644    #[serde(flatten)]
645    pub page: JobPage,
646    /// In any order but the default: how many jobs the filters admit, of which the page shows the first
647    /// ([`JobList::shown_words`]).
648    #[serde(default, skip_serializing_if = "Option::is_none")]
649    pub total: Option<u64>,
650    /// The jobs whose name, workflow, branch, repository, URL or labels were longer than a list shows
651    /// ([`JOB_TEXT_MAX`] bytes each, [`JOB_LABELS_MAX`] bytes of labels) and are cut here; their own page has them
652    /// whole.
653    #[serde(default, skip_serializing_if = "Vec::is_empty")]
654    pub shortened: Vec<u64>,
655    /// The query as the daemon read it, its limit and direction filled in; `None` from a daemon that predates
656    /// search and sorting, which ignored them ([`JobList::understood`]).
657    #[serde(default, skip_serializing_if = "Option::is_none")]
658    pub applied: Option<JobQuery>,
659}
660
661/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
662#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
663pub struct LogQuery {
664    /// Start at this line, 0-based.
665    pub from: Option<u64>,
666    /// Keep the response open and send lines as they are stored, until the job finishes.
667    #[serde(default)]
668    pub follow: bool,
669}
670
671/// One sample of a container's use.
672#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
673pub struct Sample {
674    /// When it was taken.
675    pub at: u64,
676    /// CPU use since the previous sample, in millicores.
677    pub cpu: u64,
678    /// Memory use without page cache.
679    pub memory: u64,
680    /// Bytes received since the container started.
681    pub network_received: u64,
682    /// Bytes sent since the container started.
683    pub network_sent: u64,
684}
685
686/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
687/// older than `history.samples` are deleted; the job's metrics stay.
688#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
689pub struct SampleSeries {
690    /// The job.
691    pub job: u64,
692    /// Seconds between samples.
693    pub interval: u64,
694    /// Oldest first.
695    pub samples: Vec<Sample>,
696}
697
698/// One sample of the whole slice runners run in.
699#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
700pub struct HostSample {
701    /// When it was taken.
702    pub at: u64,
703    /// CPU use of the slice since the previous sample, in millicores.
704    pub cpu: u64,
705    /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
706    /// since active page cache still counts, which the high-water mark of measured admission compares to
707    /// `MemoryMax`.
708    pub memory: u64,
709}
710
711/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
712/// (ADR 0055). Read every few seconds; not kept in the history.
713#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
714pub struct HostMemory {
715    /// When it was read.
716    pub at: u64,
717    /// `MemTotal` in `/proc/meminfo`, in bytes.
718    pub total: u64,
719    /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
720    pub available: u64,
721    /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
722    /// out.
723    pub slice: u64,
724    /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
725    /// kernel without pressure stall information.
726    pub some: Option<Pressure>,
727    /// Its `full` line: the share of time every task that could run waited for memory.
728    pub full: Option<Pressure>,
729    /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
730    /// Read less often than the rest. Empty when none is that large.
731    #[serde(default, skip_serializing_if = "Vec::is_empty")]
732    pub holders: Vec<Holder>,
733    /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
734    /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
735    #[serde(default, skip_serializing_if = "Option::is_none")]
736    pub ceiling: Option<u64>,
737    /// `SwapTotal` and `SwapFree`, from the same read of `/proc/meminfo`; absent on a host without swap (ADR 0101).
738    /// Admission counts none of it.
739    #[serde(default, skip_serializing_if = "Option::is_none")]
740    pub swap: Option<Swap>,
741}
742
743/// A host's swap at one reading: zram and zswap's device included, as `/proc/meminfo` counts it.
744#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
745pub struct Swap {
746    /// `SwapTotal`, in bytes. Never 0: a host without swap has none.
747    pub total: u64,
748    /// `SwapFree`, in bytes.
749    pub free: u64,
750}
751
752impl Swap {
753    /// What is in use, in bytes.
754    pub fn used(&self) -> u64 {
755        self.total.saturating_sub(self.free)
756    }
757}
758
759/// The margin kept free at least: 2G.
760const MARGIN_MIN: u64 = 2 << 30;
761
762/// The margin as a share of the host's memory, as a divisor: a sixteenth, 4G of 64G.
763const MARGIN_SHARE: u64 = 16;
764
765/// The plain margin `MemAvailable` keeps beside the runners on a host with `total` memory: the larger of `MARGIN_MIN`
766/// and `total / MARGIN_SHARE` (ADR 0055).
767pub fn host_margin(total: u64) -> u64 {
768    MARGIN_MIN.max(total / MARGIN_SHARE)
769}
770
771impl HostMemory {
772    /// What the host has for a new runner this moment, in bytes: `MemAvailable` and what it can still grow by, less
773    /// the plain margin. Unlike `HostBudget::spare` it does not count what runners reserve past what they hold, which
774    /// is memory they may yet take, not memory taken.
775    pub fn room(&self) -> u64 {
776        self.available
777            .saturating_add(self.growable())
778            .saturating_sub(host_margin(self.grown()))
779    }
780
781    /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
782    pub fn grown(&self) -> u64 {
783        self.ceiling
784            .map_or(self.total, |ceiling| ceiling.max(self.total))
785    }
786
787    /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
788    /// reached it.
789    pub fn growable(&self) -> u64 {
790        self.grown().saturating_sub(self.total)
791    }
792
793    /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
794    /// page cache it can drop.
795    pub fn outside(&self) -> u64 {
796        self.total
797            .saturating_sub(self.available)
798            .saturating_sub(self.slice)
799    }
800
801    /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
802    /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
803    pub fn outside_line(&self) -> String {
804        let size = |bytes: u64| ByteSize(bytes).to_string();
805        let line = format!("programs outside CI hold {}", size(self.outside()));
806        let named: Vec<String> = self
807            .holders
808            .iter()
809            .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
810            .collect();
811        match named.as_slice() {
812            [] => line,
813            [one] => format!("{line}, most {one}"),
814            [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
815        }
816    }
817}
818
819/// A program outside the slice and the memory its processes hold.
820#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
821pub struct Holder {
822    /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
823    pub name: String,
824    /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
825    pub memory: u64,
826}
827
828/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
829#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
830pub struct Pressure {
831    /// Over the last 10 seconds.
832    pub avg10: u32,
833    /// Over the last 60 seconds.
834    pub avg60: u32,
835}
836
837/// `GET /v1/host/samples` query parameters.
838#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
839pub struct HostSampleQuery {
840    /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
841    pub since: Option<u64>,
842}
843
844/// `GET /v1/host/samples`: the slice's use over time, oldest first.
845#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
846pub struct HostSamples {
847    /// Seconds between samples.
848    pub interval: u64,
849    /// Oldest first.
850    pub samples: Vec<HostSample>,
851}
852
853/// How `GET /v1/stats` groups jobs in time.
854#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
855#[serde(rename_all = "kebab-case")]
856pub enum Bucket {
857    /// By UTC hour.
858    Hour,
859    /// By UTC day.
860    #[default]
861    Day,
862}
863
864/// `GET /v1/activity` query parameters.
865#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
866pub struct ActivityQuery {
867    /// The start of the window; default 24 hours ago. It ends now.
868    pub since: Option<u64>,
869}
870
871/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
872/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
873#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
874pub struct Activity {
875    /// The start of the window.
876    pub since: u64,
877    /// When it was read: the end of the window.
878    pub until: u64,
879    /// Each job that started in the window, oldest start first.
880    pub started: Vec<Started>,
881    /// The jobs that finished in the window: completed, or lost with their runner.
882    pub finished: u64,
883    /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
884    pub failed: u64,
885    /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
886    /// after the kernel killed a process of theirs for memory at their cap.
887    pub runner_failed: u64,
888}
889
890impl Activity {
891    /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
892    /// started.
893    pub fn wait_p95(&self) -> Option<u64> {
894        let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
895        waits.sort_unstable();
896        let rank = (waits.len() * 95).div_ceil(100);
897        waits.get(rank.checked_sub(1)?).copied()
898    }
899
900    /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
901    /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
902    pub fn wait_hours(&self) -> [u64; 24] {
903        const HOUR: u64 = 3600;
904        let since = self.until.saturating_sub(24 * HOUR);
905        let mut hours = [0u64; 24];
906        for start in self
907            .started
908            .iter()
909            .filter(|start| (since..=self.until).contains(&start.at))
910        {
911            // A start exactly 24 hours ago is in the oldest hour.
912            let back =
913                usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
914            hours[23 - back] = hours[23 - back].max(start.waited);
915        }
916        hours
917    }
918}
919
920/// A job that started: when, and how long it had waited since it was queued.
921#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
922pub struct Started {
923    /// When it started.
924    pub at: u64,
925    /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
926    pub waited: u64,
927}
928
929/// `GET /v1/stats` query parameters.
930#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
931pub struct StatsQuery {
932    /// Only this `owner/name`.
933    pub repository: Option<String>,
934    /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
935    /// are counted: 31 days of hours, two years of days.
936    pub since: Option<u64>,
937    /// Only jobs queued before this time.
938    pub until: Option<u64>,
939    /// The bucket size; default a day.
940    #[serde(default)]
941    pub bucket: Bucket,
942}
943
944/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
945#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
946pub struct Stats {
947    /// The bucket size.
948    pub bucket: Bucket,
949    /// By bucket start, then repository.
950    pub rows: Vec<StatsRow>,
951}
952
953/// The finished jobs of one repository in one bucket.
954#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
955pub struct StatsRow {
956    /// The bucket's start.
957    pub start: u64,
958    /// `owner/name`.
959    pub repository: String,
960    /// Jobs completed or lost.
961    pub jobs: u64,
962    /// Of those, concluded `success`.
963    pub succeeded: u64,
964    /// Of those, concluded `failure` or `timed_out`, or lost.
965    pub failed: u64,
966    /// Of those, concluded `cancelled`.
967    pub cancelled: u64,
968    /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
969    /// ran (ADR 0081).
970    pub queue_seconds: u64,
971    /// Seconds from started to finished, summed.
972    pub run_seconds: u64,
973    /// CPU time, in milliseconds, summed.
974    pub cpu_time: u64,
975    /// The highest memory peak of a job.
976    pub memory_peak: u64,
977    /// Bytes received, summed.
978    pub network_received: u64,
979    /// Bytes from the mirror, summed.
980    pub mirror_bytes: u64,
981}
982
983/// What a cache volume holds.
984#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
985#[serde(rename_all = "kebab-case", tag = "kind")]
986pub enum CacheKind {
987    /// A profile's read-only snapshot.
988    Snapshot {
989        /// The profile.
990        profile: String,
991    },
992    /// A slot's layer changes and Docker Engine data.
993    Slot {
994        /// The profile.
995        profile: String,
996        /// 1-based.
997        slot: usize,
998    },
999    /// The pnpm store all slots share.
1000    Pnpm,
1001    /// The Maven repository all slots share.
1002    Maven,
1003    /// The Docker Hub mirror's storage.
1004    Mirror,
1005    /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
1006    Build,
1007}
1008
1009/// One cache volume and its size.
1010#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1011pub struct CacheVolume {
1012    /// The Docker volume.
1013    pub name: String,
1014    /// What it holds.
1015    #[serde(flatten)]
1016    pub kind: CacheKind,
1017    /// Its size at `measured_at`.
1018    pub bytes: u64,
1019    /// The size above which runnerd empties it, or for a snapshot cuts it back at its next reseed, if it has one.
1020    pub limit: Option<u64>,
1021}
1022
1023/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
1024/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
1025#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1026pub struct Caches {
1027    /// When the sizes were measured; `None` before the first measurement.
1028    pub measured_at: Option<u64>,
1029    /// By name.
1030    pub volumes: Vec<CacheVolume>,
1031    /// The shared build cache now, when it is on.
1032    pub build: Option<BuildCacheStats>,
1033    /// The action archives runnerd keeps for its runners, when `[caches.actions]` is on (ADR 0147); absent from an
1034    /// older daemon.
1035    #[serde(default, skip_serializing_if = "Option::is_none")]
1036    pub actions: Option<super::ActionArchives>,
1037}
1038
1039/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
1040#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
1041pub struct BuildCacheStats {
1042    /// The entries' bytes.
1043    pub bytes: u64,
1044    /// How many entries it holds.
1045    pub entries: u64,
1046    /// `[caches.build] max_size`.
1047    pub limit: u64,
1048    /// The runners of the last 7 days, summed.
1049    pub week: BuildCacheCounts,
1050    /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
1051    /// daemon older than it, which never served it.
1052    #[serde(default)]
1053    pub actions_cache: bool,
1054}
1055
1056/// `GET /v1/caches/history` query parameters.
1057#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
1058pub struct CacheHistoryQuery {
1059    /// Only measurements at or after this time; default 30 days ago.
1060    pub since: Option<u64>,
1061}
1062
1063/// One volume's size at one measurement.
1064#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1065pub struct CacheSize {
1066    /// When it was measured.
1067    pub at: u64,
1068    /// The Docker volume.
1069    pub name: String,
1070    /// Its size.
1071    pub bytes: u64,
1072}
1073
1074/// `GET /v1/caches/history`: every measurement, oldest first.
1075#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1076pub struct CacheHistory {
1077    /// Oldest first.
1078    pub sizes: Vec<CacheSize>,
1079}