runnerd_core/api/jobs.rs
1//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
2//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
3//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).
4
5use serde::{Deserialize, Serialize};
6
7use super::{cpu_time, in_cores};
8use crate::config::ByteSize;
9
10mod compare;
11mod facts;
12mod log;
13mod query;
14mod timeline;
15
16pub(crate) use log::plain;
17
18#[cfg(feature = "host")]
19pub(crate) use compare::SAMPLE_GAP_INTERVALS;
20pub use compare::{
21 Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind, CompareQuery,
22 ComparedSteps, FigureDelta, FigureKind, JobComparison, StepDelta, StepMatch, StepUsage,
23 Unavailable,
24};
25pub use facts::{
26 JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
27 job_title,
28};
29#[cfg(feature = "host")]
30pub(crate) use log::TITLE_SOURCE_MAX;
31pub use log::{
32 Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
33 LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
34 LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
35 SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
36};
37pub use query::{
38 JOB_LABELS_MAX, JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT,
39 JOBS_PAGE_MAX, JobOrder, JobRange, JobSort, RecordFilter, RecordMark, SortOrder, row_bytes,
40};
41pub use timeline::{
42 BarEnd, Gap, GapKind, Lane, RunnerSpan, SlotKey, SlotTimeline, SlottedBar, SlottedRunner, Span,
43 TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
44 TimelineRows,
45};
46
47/// Where a job is in its life, as this daemon saw it.
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
49#[serde(rename_all = "kebab-case")]
50pub enum JobState {
51 /// Waiting for a runner.
52 Queued,
53 /// A runner of this daemon took it.
54 Running,
55 /// GitHub reports it completed; `conclusion` says how.
56 Completed,
57 /// Its runner's container is gone and GitHub did not report the job completed in time.
58 Lost,
59 /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
60 /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
61 Elsewhere,
62}
63
64impl JobState {
65 /// Whether the job can still change.
66 pub fn is_final(self) -> bool {
67 matches!(
68 self,
69 JobState::Completed | JobState::Lost | JobState::Elsewhere
70 )
71 }
72}
73
74/// A job without its steps: what lists and events carry.
75#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
76pub struct JobSummary {
77 /// GitHub's job ID, also its check run ID.
78 pub id: u64,
79 /// `owner/name` on GitHub.
80 pub repository: String,
81 /// The workflow run it belongs to.
82 pub run_id: u64,
83 /// 1 for the first attempt of the run, higher for re-runs.
84 pub run_attempt: u32,
85 /// The workflow's name.
86 pub workflow: Option<String>,
87 /// The job's name in the workflow.
88 pub name: String,
89 /// The branch the run is for.
90 pub branch: Option<String>,
91 /// The job's page on GitHub.
92 pub url: Option<String>,
93 /// The workflow's `runs-on` labels.
94 pub labels: Vec<String>,
95 /// Where the job is.
96 pub state: JobState,
97 /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
98 pub conclusion: Option<String>,
99 /// The runner that ran it.
100 pub runner: Option<JobRunner>,
101 /// When each phase started.
102 pub timeline: Timeline,
103 /// CPU, memory and network use while it ran.
104 pub metrics: JobMetrics,
105 /// How much of its log is stored.
106 pub log: LogSummary,
107 /// Whether the runner failed it rather than its code (`record::runner_fault`): Sysbox lost the runner's seccomp
108 /// listener, so what its failed step printed has `function not implemented` on a mount, umount or openat2, or
109 /// `unsafe procfs detected` with `invalid argument`. Read from the log once the record is final; false for a job
110 /// that has not concluded failed, and in records from before it was kept.
111 #[serde(default)]
112 pub runner_fault: bool,
113 /// What became of its record, for a job the history notes one for: kept, removed or not made (ADR 0135).
114 /// `None` for a job without a record note, and from a daemon older than the mark. Not `record`, which a `Job`
115 /// that flattens this summary holds in full.
116 #[serde(default, skip_serializing_if = "Option::is_none")]
117 pub record_mark: Option<RecordMark>,
118}
119
120/// `GET /v1/jobs/{id}`: a job with its steps.
121#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
122pub struct Job {
123 /// Everything but the steps.
124 #[serde(flatten)]
125 pub summary: JobSummary,
126 /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
127 pub steps: Vec<Step>,
128 /// Its record, for a failed job of a profile that records failures.
129 #[serde(default)]
130 pub record: Option<super::RecordNote>,
131 /// How its runner was admitted; `None` for one started before runnerd recorded it.
132 #[serde(default)]
133 pub admission: Option<super::Admission>,
134}
135
136impl Job {
137 /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
138 /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
139 /// limit as configured now. A reservation of the whole cap is one line.
140 pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
141 let limits = self.limits(profile);
142 let admission = self.admission.as_ref();
143 let name = profile.map_or("profile", |profile| profile.name.as_str());
144 ChartLines {
145 memory: chart_lines(
146 limits.memory,
147 admission.map(|admission| admission.reserved),
148 |value| ByteSize(value).to_string(),
149 name,
150 ),
151 cpu: chart_lines(
152 limits.cpu,
153 admission
154 .and_then(|admission| admission.cpu)
155 .map(|cpu| cpu.reserved),
156 super::cpus,
157 name,
158 ),
159 }
160 }
161
162 /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
163 /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
164 /// reload that changes the profile never replaces a cap the record has.
165 pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
166 let runner = self.summary.runner.as_ref();
167 let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
168 (Some(value), _) => Some(Limit {
169 value,
170 source: LimitSource::Cap,
171 }),
172 (None, Some(value)) => Some(Limit {
173 value,
174 source: LimitSource::ProfileNow,
175 }),
176 (None, None) => None,
177 };
178 Limits {
179 memory: limit(
180 runner.and_then(|runner| runner.memory_cap),
181 profile.map(|profile| profile.memory),
182 ),
183 cpu: limit(
184 runner.and_then(|runner| runner.cpu_cap),
185 profile.map(|profile| u64::from(profile.cpus) * 1000),
186 ),
187 }
188 }
189}
190
191/// What a job's container may use, from `Job::limits`.
192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
193pub struct Limits {
194 /// In bytes.
195 pub memory: Option<Limit>,
196 /// In millicores.
197 pub cpu: Option<Limit>,
198}
199
200impl Limits {
201 /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
202 /// now 6G and 4 CPUs`; `None` without either.
203 pub fn line(&self) -> Option<String> {
204 let memory = self
205 .memory
206 .map(|limit| (limit.source, ByteSize(limit.value).to_string()));
207 let cpu = self
208 .cpu
209 .map(|limit| (limit.source, super::cpus(limit.value)));
210 let clause = |source: LimitSource, figures: &str| match source {
211 LimitSource::Cap => format!("capped at {figures}"),
212 LimitSource::ProfileNow => format!("profile now {figures}"),
213 };
214 match (memory, cpu) {
215 (Some((memory_source, memory)), Some((cpu_source, cpu)))
216 if memory_source == cpu_source =>
217 {
218 Some(clause(memory_source, &format!("{memory} and {cpu}")))
219 }
220 (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
221 "{}; {}",
222 clause(memory_source, &memory),
223 clause(cpu_source, &cpu)
224 )),
225 (Some((source, figure)), None) | (None, Some((source, figure))) => {
226 Some(clause(source, &figure))
227 }
228 (None, None) => None,
229 }
230 }
231}
232
233/// One resource's limit for a job's container.
234#[derive(Debug, Clone, Copy, PartialEq, Eq)]
235pub struct Limit {
236 /// Bytes or millicores.
237 pub value: u64,
238 /// Where it comes from.
239 pub source: LimitSource,
240}
241
242/// Where a job's limit comes from.
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub enum LimitSource {
245 /// The cap its container ran with.
246 Cap,
247 /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
248 /// changed it.
249 ProfileNow,
250}
251
252/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
253fn chart_lines(
254 limit: Option<Limit>,
255 reserved: Option<u64>,
256 value: fn(u64) -> String,
257 profile: &str,
258) -> Vec<ChartLine> {
259 let line = |figure, kind, label: &str| ChartLine {
260 value: figure,
261 kind,
262 label: format!("{label} {}", value(figure)),
263 };
264 let mut lines = vec![];
265 match limit {
266 Some(Limit {
267 value: cap,
268 source: LimitSource::Cap,
269 }) if reserved == Some(cap) => {
270 return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
271 }
272 Some(Limit {
273 value: cap,
274 source: LimitSource::Cap,
275 }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
276 Some(Limit {
277 value: now,
278 source: LimitSource::ProfileNow,
279 }) => lines.push(line(
280 now,
281 ChartLineKind::ProfileNow,
282 &format!("{profile} now"),
283 )),
284 None => {}
285 }
286 if let Some(reserved) = reserved {
287 lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
288 }
289 lines
290}
291
292/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
293#[derive(Debug, Clone, PartialEq, Eq)]
294pub struct ChartLines {
295 /// In bytes.
296 pub memory: Vec<ChartLine>,
297 /// In millicores.
298 pub cpu: Vec<ChartLine>,
299}
300
301/// A line across a job's memory or CPU chart.
302#[derive(Debug, Clone, PartialEq, Eq)]
303pub struct ChartLine {
304 /// Bytes or millicores.
305 pub value: u64,
306 /// What it stands for.
307 pub kind: ChartLineKind,
308 /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
309 pub label: String,
310}
311
312/// What a line across a job's chart stands for.
313#[derive(Debug, Clone, Copy, PartialEq, Eq)]
314pub enum ChartLineKind {
315 /// The most its container may use: its cap.
316 Cap,
317 /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
318 /// changed it.
319 ProfileNow,
320 /// What admission counted against the budget for its runner.
321 Reserved,
322 /// A reservation of the whole cap, as admission by caps makes.
323 CapAndReserved,
324}
325
326/// The runner a job ran on.
327#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
328pub struct JobRunner {
329 /// The runner's name, also its container's.
330 pub name: String,
331 /// The profile it ran in.
332 pub profile: String,
333 /// The slot it ran in, 1-based.
334 pub slot: usize,
335 /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
336 /// from before runnerd kept it.
337 pub memory_cap: Option<u64>,
338 /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
339 /// records from before runnerd kept it, or when Docker reports no limit.
340 pub cpu_cap: Option<u64>,
341}
342
343/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
344#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
345pub struct Timeline {
346 /// GitHub queued the job.
347 pub queued_at: u64,
348 /// The container of the runner that took it started.
349 pub container_started_at: Option<u64>,
350 /// The runner started the job.
351 pub started_at: Option<u64>,
352 /// The job finished, by GitHub's clock; or when its container was found gone.
353 pub finished_at: Option<u64>,
354 /// The container and the runner's registration were removed.
355 pub cleaned_at: Option<u64>,
356 /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
357 /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
358 #[serde(default)]
359 pub unseen: u64,
360 /// When runnerd first listed it queued (ADR 0092). `None` for a job no listing showed queued, such as one a
361 /// warm runner took first, and for one an older runnerd recorded.
362 #[serde(default)]
363 pub seen_at: Option<u64>,
364}
365
366impl Timeline {
367 /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
368 /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
369 pub fn runner_from(&self) -> Option<u64> {
370 self.container_started_at
371 .map(|started| started.max(self.queued_at))
372 }
373}
374
375/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
376#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
377pub struct JobMetrics {
378 /// CPU time, in milliseconds.
379 pub cpu_time: Option<u64>,
380 /// The highest CPU use of a sample interval, in millicores.
381 pub cpu_peak: Option<u64>,
382 /// CPU time over wall time, in millicores.
383 pub cpu_average: Option<u64>,
384 /// The highest memory use without page cache.
385 pub memory_peak: Option<u64>,
386 /// The average memory use without page cache.
387 pub memory_average: Option<u64>,
388 /// Bytes the container received.
389 pub network_received: Option<u64>,
390 /// Bytes the container sent.
391 pub network_sent: Option<u64>,
392 /// Of the bytes received, those the Docker Hub mirror served.
393 pub mirror_bytes: u64,
394 /// What the shared build cache served it and took from it.
395 pub build_cache: BuildCacheCounts,
396 /// Times the kernel held its container's sockets' receive windows small because the container's memory was
397 /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
398 /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
399 #[serde(default, skip_serializing_if = "Option::is_none")]
400 pub sock_throttled: Option<u64>,
401 /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
402 /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
403 #[serde(default, skip_serializing_if = "Option::is_none")]
404 pub cpu_periods: Option<u64>,
405 /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
406 /// cap held it back.
407 #[serde(default, skip_serializing_if = "Option::is_none")]
408 pub cpu_throttled: Option<u64>,
409 /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
410 /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
411 #[serde(default, skip_serializing_if = "Option::is_none")]
412 pub cpu_stalled: Option<u64>,
413 /// What its runner container wrote to the disks under Docker's root (ADR 0143). `None` where no reading measured
414 /// it, as without the io controller, and in records from before.
415 #[serde(default, skip_serializing_if = "Option::is_none")]
416 pub disk: Option<super::DiskWritten>,
417}
418
419/// One figure of what a job used, worded for every client.
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub struct Total {
422 /// What it measures: `Memory peak`, `Received`.
423 pub label: &'static str,
424 /// The figure with its unit.
425 pub value: String,
426}
427
428impl JobMetrics {
429 /// What the job used, over its whole life: memory, CPU and network, each where the daemon measured it. The
430 /// peaks are of the samples taken every few seconds, so the figures' note says that
431 /// ([`sampled_note`](super::sampled_note)).
432 pub fn totals(&self) -> Vec<Total> {
433 let mut totals = Vec::new();
434 let mut add = |label: &'static str, value: Option<String>| {
435 totals.extend(value.map(|value| Total { label, value }));
436 };
437 let bytes = |bytes: u64| ByteSize(bytes).to_string();
438 add("Memory peak", self.memory_peak.map(bytes));
439 add("Average memory", self.memory_average.map(bytes));
440 add("CPU peak", self.cpu_peak.map(in_cores));
441 add("Average CPU", self.cpu_average.map(in_cores));
442 add("CPU time", self.cpu_time.map(cpu_time));
443 add("Received", self.network_received.map(bytes));
444 add("Sent", self.network_sent.map(bytes));
445 add("Written to disk", self.disk.map(|disk| disk.line()));
446 // Counted from the mirror's own log, so a job without network samples still has it.
447 add(
448 "From the mirror",
449 (self.network_received.is_some() || self.mirror_bytes > 0)
450 .then(|| bytes(self.mirror_bytes)),
451 );
452 add(
453 "Throttled for memory pressure",
454 self.sock_throttled
455 .filter(|times| *times > 0)
456 .map(|times| format!("{times} times")),
457 );
458 totals
459 }
460}
461
462/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
463#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
464pub struct BuildCacheCounts {
465 /// Lookups that found an entry.
466 pub hits: u64,
467 /// Lookups that found none.
468 pub misses: u64,
469 /// Bytes of the entries it served.
470 pub read_bytes: u64,
471 /// Bytes of the entries it stored.
472 pub written_bytes: u64,
473 /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
474 /// cache was full.
475 pub refused: u64,
476 /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
477 pub writer: bool,
478}
479
480impl BuildCacheCounts {
481 /// Hits and misses.
482 pub fn lookups(&self) -> u64 {
483 self.hits + self.misses
484 }
485
486 /// Hits in percent of lookups, `None` without any.
487 pub fn hit_percent(&self) -> Option<u64> {
488 let lookups = self.lookups();
489 (lookups > 0).then(|| self.hits * 100 / lookups)
490 }
491
492 /// Adds `other`'s counts; `writer` stays.
493 pub fn add(&mut self, other: &BuildCacheCounts) {
494 self.hits += other.hits;
495 self.misses += other.misses;
496 self.read_bytes += other.read_bytes;
497 self.written_bytes += other.written_bytes;
498 self.refused += other.refused;
499 }
500}
501
502/// A job's stored log.
503#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
504pub struct LogSummary {
505 /// Lines stored.
506 pub lines: u64,
507 /// Bytes stored.
508 pub bytes: u64,
509 /// Set once the log reached `history.log_max` and later lines were dropped.
510 pub truncated: bool,
511}
512
513/// One step of a job, as GitHub reports it.
514#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
515pub struct Step {
516 /// GitHub's step number; numbers may skip.
517 pub number: u32,
518 /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
519 pub name: String,
520 /// `pending`, `queued`, `in_progress` or `completed`.
521 pub status: String,
522 /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
523 pub conclusion: Option<String>,
524 /// When it started.
525 pub started_at: Option<u64>,
526 /// When it completed.
527 pub completed_at: Option<u64>,
528}
529
530/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time), or by `sort` (ADR 0135).
531/// [`JobQuery::check`] says which queries the daemon refuses.
532#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
533pub struct JobQuery {
534 /// Only this `owner/name`.
535 pub repository: Option<String>,
536 /// Only jobs in this state.
537 pub state: Option<JobState>,
538 /// Only jobs with this conclusion, such as `failure`.
539 pub conclusion: Option<String>,
540 /// Only jobs of the workflow with this name, such as `CI`.
541 pub workflow: Option<String>,
542 /// Only jobs with this name in their workflow, such as `build`.
543 pub name: Option<String>,
544 /// Only jobs queued at or after this time.
545 pub since: Option<u64>,
546 /// Only jobs queued before this time.
547 pub until: Option<u64>,
548 /// Only jobs with a smaller ID: the `next` of the previous page. Only in the default order, newest first.
549 pub before: Option<u64>,
550 /// At most this many jobs; default 50, at most 500. A page also ends once it holds
551 /// [`JOBS_PAGE_BYTES`] of text.
552 pub limit: Option<u32>,
553 /// Only jobs whose name, workflow or branch holds this text, as written: 1 to [`JOB_SEARCH_MAX`] bytes, ASCII
554 /// letters in either case. The history keeps no commit, so a commit is not searched.
555 #[serde(default, skip_serializing_if = "Option::is_none")]
556 pub q: Option<String>,
557 /// Only jobs of runs for this branch.
558 #[serde(default, skip_serializing_if = "Option::is_none")]
559 pub branch: Option<String>,
560 /// Only jobs whose record this says became of.
561 #[serde(default, skip_serializing_if = "Option::is_none")]
562 pub record: Option<RecordFilter>,
563 /// Sort by this; a job whose figure is unknown comes last either way, ties by ID. `queued`, newest first, is
564 /// the default order, which pages with `before`; any other order is one page, the first `limit` jobs.
565 #[serde(default, skip_serializing_if = "Option::is_none")]
566 pub sort: Option<JobSort>,
567 /// Which way; default largest or latest first.
568 #[serde(default, skip_serializing_if = "Option::is_none")]
569 pub order: Option<SortOrder>,
570 /// Refused: a sorted list is one page, which a search or the filters narrow. Read only to say so.
571 #[serde(default, skip_serializing_if = "Option::is_none")]
572 pub cursor: Option<String>,
573}
574
575/// One page of jobs, as `GET /v1/jobs` answered before it searched and sorted: what [`JobList`] holds.
576#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
577pub struct JobPage {
578 /// Newest first, or in the query's sort.
579 pub jobs: Vec<JobSummary>,
580 /// In the default order: pass as `before` for the next page; `None` on the last page, and in any other order.
581 pub next: Option<u64>,
582}
583
584/// `GET /v1/jobs`: one page of jobs (ADR 0135). A [`JobPage`] with fields more, so a client that reads a `JobPage`
585/// reads it too, and one that reads a `JobList` from an older daemon finds them absent.
586#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
587pub struct JobList {
588 /// The jobs and, in the default order, `next`.
589 #[serde(flatten)]
590 pub page: JobPage,
591 /// In any order but the default: how many jobs the filters admit, of which the page shows the first
592 /// ([`JobList::shown_words`]).
593 #[serde(default, skip_serializing_if = "Option::is_none")]
594 pub total: Option<u64>,
595 /// The jobs whose name, workflow, branch, repository, URL or labels were longer than a list shows
596 /// ([`JOB_TEXT_MAX`] bytes each, [`JOB_LABELS_MAX`] bytes of labels) and are cut here; their own page has them
597 /// whole.
598 #[serde(default, skip_serializing_if = "Vec::is_empty")]
599 pub shortened: Vec<u64>,
600 /// The query as the daemon read it, its limit and direction filled in; `None` from a daemon that predates
601 /// search and sorting, which ignored them ([`JobList::understood`]).
602 #[serde(default, skip_serializing_if = "Option::is_none")]
603 pub applied: Option<JobQuery>,
604}
605
606/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
607#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
608pub struct LogQuery {
609 /// Start at this line, 0-based.
610 pub from: Option<u64>,
611 /// Keep the response open and send lines as they are stored, until the job finishes.
612 #[serde(default)]
613 pub follow: bool,
614}
615
616/// One sample of a container's use.
617#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
618pub struct Sample {
619 /// When it was taken.
620 pub at: u64,
621 /// CPU use since the previous sample, in millicores.
622 pub cpu: u64,
623 /// Memory use without page cache.
624 pub memory: u64,
625 /// Bytes received since the container started.
626 pub network_received: u64,
627 /// Bytes sent since the container started.
628 pub network_sent: u64,
629}
630
631/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
632/// older than `history.samples` are deleted; the job's metrics stay.
633#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
634pub struct SampleSeries {
635 /// The job.
636 pub job: u64,
637 /// Seconds between samples.
638 pub interval: u64,
639 /// Oldest first.
640 pub samples: Vec<Sample>,
641}
642
643/// One sample of the whole slice runners run in.
644#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
645pub struct HostSample {
646 /// When it was taken.
647 pub at: u64,
648 /// CPU use of the slice since the previous sample, in millicores.
649 pub cpu: u64,
650 /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
651 /// since active page cache still counts, which the high-water mark of measured admission compares to
652 /// `MemoryMax`.
653 pub memory: u64,
654}
655
656/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
657/// (ADR 0055). Read every few seconds; not kept in the history.
658#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
659pub struct HostMemory {
660 /// When it was read.
661 pub at: u64,
662 /// `MemTotal` in `/proc/meminfo`, in bytes.
663 pub total: u64,
664 /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
665 pub available: u64,
666 /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
667 /// out.
668 pub slice: u64,
669 /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
670 /// kernel without pressure stall information.
671 pub some: Option<Pressure>,
672 /// Its `full` line: the share of time every task that could run waited for memory.
673 pub full: Option<Pressure>,
674 /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
675 /// Read less often than the rest. Empty when none is that large.
676 #[serde(default, skip_serializing_if = "Vec::is_empty")]
677 pub holders: Vec<Holder>,
678 /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
679 /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
680 #[serde(default, skip_serializing_if = "Option::is_none")]
681 pub ceiling: Option<u64>,
682 /// `SwapTotal` and `SwapFree`, from the same read of `/proc/meminfo`; absent on a host without swap (ADR 0101).
683 /// Admission counts none of it.
684 #[serde(default, skip_serializing_if = "Option::is_none")]
685 pub swap: Option<Swap>,
686}
687
688/// A host's swap at one reading: zram and zswap's device included, as `/proc/meminfo` counts it.
689#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
690pub struct Swap {
691 /// `SwapTotal`, in bytes. Never 0: a host without swap has none.
692 pub total: u64,
693 /// `SwapFree`, in bytes.
694 pub free: u64,
695}
696
697impl Swap {
698 /// What is in use, in bytes.
699 pub fn used(&self) -> u64 {
700 self.total.saturating_sub(self.free)
701 }
702}
703
704/// The margin kept free at least: 2G.
705const MARGIN_MIN: u64 = 2 << 30;
706
707/// The margin as a share of the host's memory, as a divisor: a sixteenth, 4G of 64G.
708const MARGIN_SHARE: u64 = 16;
709
710/// The plain margin `MemAvailable` keeps beside the runners on a host with `total` memory: the larger of `MARGIN_MIN`
711/// and `total / MARGIN_SHARE` (ADR 0055).
712pub fn host_margin(total: u64) -> u64 {
713 MARGIN_MIN.max(total / MARGIN_SHARE)
714}
715
716impl HostMemory {
717 /// What the host has for a new runner this moment, in bytes: `MemAvailable` and what it can still grow by, less
718 /// the plain margin. Unlike `HostBudget::spare` it does not count what runners reserve past what they hold, which
719 /// is memory they may yet take, not memory taken.
720 pub fn room(&self) -> u64 {
721 self.available
722 .saturating_add(self.growable())
723 .saturating_sub(host_margin(self.grown()))
724 }
725
726 /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
727 pub fn grown(&self) -> u64 {
728 self.ceiling
729 .map_or(self.total, |ceiling| ceiling.max(self.total))
730 }
731
732 /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
733 /// reached it.
734 pub fn growable(&self) -> u64 {
735 self.grown().saturating_sub(self.total)
736 }
737
738 /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
739 /// page cache it can drop.
740 pub fn outside(&self) -> u64 {
741 self.total
742 .saturating_sub(self.available)
743 .saturating_sub(self.slice)
744 }
745
746 /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
747 /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
748 pub fn outside_line(&self) -> String {
749 let size = |bytes: u64| ByteSize(bytes).to_string();
750 let line = format!("programs outside CI hold {}", size(self.outside()));
751 let named: Vec<String> = self
752 .holders
753 .iter()
754 .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
755 .collect();
756 match named.as_slice() {
757 [] => line,
758 [one] => format!("{line}, most {one}"),
759 [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
760 }
761 }
762}
763
764/// A program outside the slice and the memory its processes hold.
765#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
766pub struct Holder {
767 /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
768 pub name: String,
769 /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
770 pub memory: u64,
771}
772
773/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
774#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
775pub struct Pressure {
776 /// Over the last 10 seconds.
777 pub avg10: u32,
778 /// Over the last 60 seconds.
779 pub avg60: u32,
780}
781
782/// `GET /v1/host/samples` query parameters.
783#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
784pub struct HostSampleQuery {
785 /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
786 pub since: Option<u64>,
787}
788
789/// `GET /v1/host/samples`: the slice's use over time, oldest first.
790#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
791pub struct HostSamples {
792 /// Seconds between samples.
793 pub interval: u64,
794 /// Oldest first.
795 pub samples: Vec<HostSample>,
796}
797
798/// How `GET /v1/stats` groups jobs in time.
799#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
800#[serde(rename_all = "kebab-case")]
801pub enum Bucket {
802 /// By UTC hour.
803 Hour,
804 /// By UTC day.
805 #[default]
806 Day,
807}
808
809/// `GET /v1/activity` query parameters.
810#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
811pub struct ActivityQuery {
812 /// The start of the window; default 24 hours ago. It ends now.
813 pub since: Option<u64>,
814}
815
816/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
817/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
818#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
819pub struct Activity {
820 /// The start of the window.
821 pub since: u64,
822 /// When it was read: the end of the window.
823 pub until: u64,
824 /// Each job that started in the window, oldest start first.
825 pub started: Vec<Started>,
826 /// The jobs that finished in the window: completed, or lost with their runner.
827 pub finished: u64,
828 /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
829 pub failed: u64,
830 /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
831 /// after the kernel killed a process of theirs for memory at their cap.
832 pub runner_failed: u64,
833}
834
835impl Activity {
836 /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
837 /// started.
838 pub fn wait_p95(&self) -> Option<u64> {
839 let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
840 waits.sort_unstable();
841 let rank = (waits.len() * 95).div_ceil(100);
842 waits.get(rank.checked_sub(1)?).copied()
843 }
844
845 /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
846 /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
847 pub fn wait_hours(&self) -> [u64; 24] {
848 const HOUR: u64 = 3600;
849 let since = self.until.saturating_sub(24 * HOUR);
850 let mut hours = [0u64; 24];
851 for start in self
852 .started
853 .iter()
854 .filter(|start| (since..=self.until).contains(&start.at))
855 {
856 // A start exactly 24 hours ago is in the oldest hour.
857 let back =
858 usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
859 hours[23 - back] = hours[23 - back].max(start.waited);
860 }
861 hours
862 }
863}
864
865/// A job that started: when, and how long it had waited since it was queued.
866#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
867pub struct Started {
868 /// When it started.
869 pub at: u64,
870 /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
871 pub waited: u64,
872}
873
874/// `GET /v1/stats` query parameters.
875#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
876pub struct StatsQuery {
877 /// Only this `owner/name`.
878 pub repository: Option<String>,
879 /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
880 /// are counted: 31 days of hours, two years of days.
881 pub since: Option<u64>,
882 /// Only jobs queued before this time.
883 pub until: Option<u64>,
884 /// The bucket size; default a day.
885 #[serde(default)]
886 pub bucket: Bucket,
887}
888
889/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
890#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
891pub struct Stats {
892 /// The bucket size.
893 pub bucket: Bucket,
894 /// By bucket start, then repository.
895 pub rows: Vec<StatsRow>,
896}
897
898/// The finished jobs of one repository in one bucket.
899#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
900pub struct StatsRow {
901 /// The bucket's start.
902 pub start: u64,
903 /// `owner/name`.
904 pub repository: String,
905 /// Jobs completed or lost.
906 pub jobs: u64,
907 /// Of those, concluded `success`.
908 pub succeeded: u64,
909 /// Of those, concluded `failure` or `timed_out`, or lost.
910 pub failed: u64,
911 /// Of those, concluded `cancelled`.
912 pub cancelled: u64,
913 /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
914 /// ran (ADR 0081).
915 pub queue_seconds: u64,
916 /// Seconds from started to finished, summed.
917 pub run_seconds: u64,
918 /// CPU time, in milliseconds, summed.
919 pub cpu_time: u64,
920 /// The highest memory peak of a job.
921 pub memory_peak: u64,
922 /// Bytes received, summed.
923 pub network_received: u64,
924 /// Bytes from the mirror, summed.
925 pub mirror_bytes: u64,
926}
927
928/// What a cache volume holds.
929#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
930#[serde(rename_all = "kebab-case", tag = "kind")]
931pub enum CacheKind {
932 /// A profile's read-only snapshot.
933 Snapshot {
934 /// The profile.
935 profile: String,
936 },
937 /// A slot's layer changes and Docker Engine data.
938 Slot {
939 /// The profile.
940 profile: String,
941 /// 1-based.
942 slot: usize,
943 },
944 /// The pnpm store all slots share.
945 Pnpm,
946 /// The Maven repository all slots share.
947 Maven,
948 /// The Docker Hub mirror's storage.
949 Mirror,
950 /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
951 Build,
952}
953
954/// One cache volume and its size.
955#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
956pub struct CacheVolume {
957 /// The Docker volume.
958 pub name: String,
959 /// What it holds.
960 #[serde(flatten)]
961 pub kind: CacheKind,
962 /// Its size at `measured_at`.
963 pub bytes: u64,
964 /// The size above which runnerd empties it, or for a snapshot starts it over at its next reseed, if it has one.
965 pub limit: Option<u64>,
966}
967
968/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
969/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
970#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
971pub struct Caches {
972 /// When the sizes were measured; `None` before the first measurement.
973 pub measured_at: Option<u64>,
974 /// By name.
975 pub volumes: Vec<CacheVolume>,
976 /// The shared build cache now, when it is on.
977 pub build: Option<BuildCacheStats>,
978}
979
980/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
981#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
982pub struct BuildCacheStats {
983 /// The entries' bytes.
984 pub bytes: u64,
985 /// How many entries it holds.
986 pub entries: u64,
987 /// `[caches.build] max_size`.
988 pub limit: u64,
989 /// The runners of the last 7 days, summed.
990 pub week: BuildCacheCounts,
991 /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
992 /// daemon older than it, which never served it.
993 #[serde(default)]
994 pub actions_cache: bool,
995}
996
997/// `GET /v1/caches/history` query parameters.
998#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
999pub struct CacheHistoryQuery {
1000 /// Only measurements at or after this time; default 30 days ago.
1001 pub since: Option<u64>,
1002}
1003
1004/// One volume's size at one measurement.
1005#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1006pub struct CacheSize {
1007 /// When it was measured.
1008 pub at: u64,
1009 /// The Docker volume.
1010 pub name: String,
1011 /// Its size.
1012 pub bytes: u64,
1013}
1014
1015/// `GET /v1/caches/history`: every measurement, oldest first.
1016#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1017pub struct CacheHistory {
1018 /// Oldest first.
1019 pub sizes: Vec<CacheSize>,
1020}