runnerd-core 0.1.1

Config, domain logic and API types shared by runnerd and its clients
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
//! Job history as clients read it: one record per GitHub job a runner of this daemon ran, its timeline, steps,
//! metrics and log size; the queries that list them; sample series for graphs; per-repository aggregates; cache
//! sizes. Times are seconds since the Unix epoch, sizes are bytes, CPU is in millicores (1000 = one core busy).

use serde::{Deserialize, Serialize};

use super::{cpu_time, in_cores};
use crate::config::ByteSize;

mod compare;
mod facts;
mod log;
mod query;
mod timeline;

pub(crate) use log::plain;

#[cfg(feature = "host")]
pub(crate) use compare::SAMPLE_GAP_INTERVALS;
pub use compare::{
    Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind, CompareQuery,
    ComparedSteps, FigureDelta, FigureKind, JobComparison, StepDelta, StepMatch, StepUsage,
    Unavailable,
};
pub use facts::{
    JobTitle, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, RunComparison, SampleUsage,
    job_title,
};
#[cfg(feature = "host")]
pub(crate) use log::TITLE_SOURCE_MAX;
pub use log::{
    Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
    LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
    LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
    SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
};
pub use query::{
    JOB_LABELS_MAX, JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT,
    JOBS_PAGE_MAX, JobOrder, JobRange, JobSort, RecordFilter, RecordMark, SortOrder, row_bytes,
};
pub use timeline::{
    BarEnd, Gap, GapKind, Lane, RunnerSpan, SlotKey, SlotTimeline, SlottedBar, SlottedRunner, Span,
    TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
    TimelineRows,
};

/// Where a job is in its life, as this daemon saw it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum JobState {
    /// Waiting for a runner.
    Queued,
    /// A runner of this daemon took it.
    Running,
    /// GitHub reports it completed; `conclusion` says how.
    Completed,
    /// Its runner's container is gone and GitHub did not report the job completed in time.
    Lost,
    /// A runner of another machine took it, or it ended before any runner did, as when a newer push cancelled it.
    /// One no runner took has no `started_at`, except in records of versions that kept the start GitHub gives it.
    Elsewhere,
}

impl JobState {
    /// Whether the job can still change.
    pub fn is_final(self) -> bool {
        matches!(
            self,
            JobState::Completed | JobState::Lost | JobState::Elsewhere
        )
    }
}

/// A job without its steps: what lists and events carry.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobSummary {
    /// GitHub's job ID, also its check run ID.
    pub id: u64,
    /// `owner/name` on GitHub.
    pub repository: String,
    /// The workflow run it belongs to.
    pub run_id: u64,
    /// 1 for the first attempt of the run, higher for re-runs.
    pub run_attempt: u32,
    /// The workflow's name.
    pub workflow: Option<String>,
    /// The job's name in the workflow.
    pub name: String,
    /// The branch the run is for.
    pub branch: Option<String>,
    /// The job's page on GitHub.
    pub url: Option<String>,
    /// The workflow's `runs-on` labels.
    pub labels: Vec<String>,
    /// Where the job is.
    pub state: JobState,
    /// GitHub's conclusion once completed: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, ….
    pub conclusion: Option<String>,
    /// The runner that ran it.
    pub runner: Option<JobRunner>,
    /// When each phase started.
    pub timeline: Timeline,
    /// CPU, memory and network use while it ran.
    pub metrics: JobMetrics,
    /// How much of its log is stored.
    pub log: LogSummary,
    /// Whether the runner failed it rather than its code (`record::runner_fault`): Sysbox lost the runner's seccomp
    /// listener, so what its failed step printed has `function not implemented` on a mount, umount or openat2, or
    /// `unsafe procfs detected` with `invalid argument`. Read from the log once the record is final; false for a job
    /// that has not concluded failed, and in records from before it was kept.
    #[serde(default)]
    pub runner_fault: bool,
    /// What became of its record, for a job the history notes one for: kept, removed or not made (ADR 0135).
    /// `None` for a job without a record note, and from a daemon older than the mark. Not `record`, which a `Job`
    /// that flattens this summary holds in full.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub record_mark: Option<RecordMark>,
}

/// `GET /v1/jobs/{id}`: a job with its steps.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Job {
    /// Everything but the steps.
    #[serde(flatten)]
    pub summary: JobSummary,
    /// The steps as GitHub numbers them, including `Set up job` and `Complete job`.
    pub steps: Vec<Step>,
    /// Its record, for a failed job of a profile that records failures.
    #[serde(default)]
    pub record: Option<super::RecordNote>,
    /// How its runner was admitted; `None` for one started before runnerd recorded it.
    #[serde(default)]
    pub admission: Option<super::Admission>,
}

impl Job {
    /// The lines across its memory and CPU charts, as every client labels them: what admission reserved of the
    /// budget, when the record has it, and the cap its container ran with, or for a record without it, `profile`'s
    /// limit as configured now. A reservation of the whole cap is one line.
    pub fn chart_lines(&self, profile: Option<&super::ProfileStatus>) -> ChartLines {
        let limits = self.limits(profile);
        let admission = self.admission.as_ref();
        let name = profile.map_or("profile", |profile| profile.name.as_str());
        ChartLines {
            memory: chart_lines(
                limits.memory,
                admission.map(|admission| admission.reserved),
                |value| ByteSize(value).to_string(),
                name,
            ),
            cpu: chart_lines(
                limits.cpu,
                admission
                    .and_then(|admission| admission.cpu)
                    .map(|cpu| cpu.reserved),
                super::cpus,
                name,
            ),
        }
    }

    /// What its container may use of each resource, as every client scales its use against: the cap it ran with,
    /// or, for a record without it, `profile`'s limit as configured now. Each resource falls back on its own, so a
    /// reload that changes the profile never replaces a cap the record has.
    pub fn limits(&self, profile: Option<&super::ProfileStatus>) -> Limits {
        let runner = self.summary.runner.as_ref();
        let limit = |cap: Option<u64>, now: Option<u64>| match (cap, now) {
            (Some(value), _) => Some(Limit {
                value,
                source: LimitSource::Cap,
            }),
            (None, Some(value)) => Some(Limit {
                value,
                source: LimitSource::ProfileNow,
            }),
            (None, None) => None,
        };
        Limits {
            memory: limit(
                runner.and_then(|runner| runner.memory_cap),
                profile.map(|profile| profile.memory),
            ),
            cpu: limit(
                runner.and_then(|runner| runner.cpu_cap),
                profile.map(|profile| u64::from(profile.cpus) * 1000),
            ),
        }
    }
}

/// What a job's container may use, from `Job::limits`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Limits {
    /// In bytes.
    pub memory: Option<Limit>,
    /// In millicores.
    pub cpu: Option<Limit>,
}

impl Limits {
    /// The limits as one clause, such as `capped at 6G and 4 CPUs`, `capped at 6G; profile now 4 CPUs` or `profile
    /// now 6G and 4 CPUs`; `None` without either.
    pub fn line(&self) -> Option<String> {
        let memory = self
            .memory
            .map(|limit| (limit.source, ByteSize(limit.value).to_string()));
        let cpu = self
            .cpu
            .map(|limit| (limit.source, super::cpus(limit.value)));
        let clause = |source: LimitSource, figures: &str| match source {
            LimitSource::Cap => format!("capped at {figures}"),
            LimitSource::ProfileNow => format!("profile now {figures}"),
        };
        match (memory, cpu) {
            (Some((memory_source, memory)), Some((cpu_source, cpu)))
                if memory_source == cpu_source =>
            {
                Some(clause(memory_source, &format!("{memory} and {cpu}")))
            }
            (Some((memory_source, memory)), Some((cpu_source, cpu))) => Some(format!(
                "{}; {}",
                clause(memory_source, &memory),
                clause(cpu_source, &cpu)
            )),
            (Some((source, figure)), None) | (None, Some((source, figure))) => {
                Some(clause(source, &figure))
            }
            (None, None) => None,
        }
    }
}

/// One resource's limit for a job's container.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Limit {
    /// Bytes or millicores.
    pub value: u64,
    /// Where it comes from.
    pub source: LimitSource,
}

/// Where a job's limit comes from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LimitSource {
    /// The cap its container ran with.
    Cap,
    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
    /// changed it.
    ProfileNow,
}

/// One resource's lines: `value` writes a figure, such as `8.5G` or `3.6 CPUs`.
fn chart_lines(
    limit: Option<Limit>,
    reserved: Option<u64>,
    value: fn(u64) -> String,
    profile: &str,
) -> Vec<ChartLine> {
    let line = |figure, kind, label: &str| ChartLine {
        value: figure,
        kind,
        label: format!("{label} {}", value(figure)),
    };
    let mut lines = vec![];
    match limit {
        Some(Limit {
            value: cap,
            source: LimitSource::Cap,
        }) if reserved == Some(cap) => {
            return vec![line(cap, ChartLineKind::CapAndReserved, "cap and reserved")];
        }
        Some(Limit {
            value: cap,
            source: LimitSource::Cap,
        }) => lines.push(line(cap, ChartLineKind::Cap, "cap")),
        Some(Limit {
            value: now,
            source: LimitSource::ProfileNow,
        }) => lines.push(line(
            now,
            ChartLineKind::ProfileNow,
            &format!("{profile} now"),
        )),
        None => {}
    }
    if let Some(reserved) = reserved {
        lines.push(line(reserved, ChartLineKind::Reserved, "reserved"));
    }
    lines
}

/// The lines across a job's memory and CPU charts, from `Job::chart_lines`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChartLines {
    /// In bytes.
    pub memory: Vec<ChartLine>,
    /// In millicores.
    pub cpu: Vec<ChartLine>,
}

/// A line across a job's memory or CPU chart.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChartLine {
    /// Bytes or millicores.
    pub value: u64,
    /// What it stands for.
    pub kind: ChartLineKind,
    /// Its label, such as `cap 8.5G` or `reserved 3.6 CPUs`.
    pub label: String,
}

/// What a line across a job's chart stands for.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChartLineKind {
    /// The most its container may use: its cap.
    Cap,
    /// Its profile's limit in the config now, for a record without its container's cap; a reload since may have
    /// changed it.
    ProfileNow,
    /// What admission counted against the budget for its runner.
    Reserved,
    /// A reservation of the whole cap, as admission by caps makes.
    CapAndReserved,
}

/// The runner a job ran on.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobRunner {
    /// The runner's name, also its container's.
    pub name: String,
    /// The profile it ran in.
    pub profile: String,
    /// The slot it ran in, 1-based.
    pub slot: usize,
    /// Its container's memory cap in bytes, which a reload since may have changed in the profile. `None` in records
    /// from before runnerd kept it.
    pub memory_cap: Option<u64>,
    /// Its container's CPU limit in millicores, the unit of the CPU use in `Sample` and `JobMetrics`. `None` in
    /// records from before runnerd kept it, or when Docker reports no limit.
    pub cpu_cap: Option<u64>,
}

/// A job's phases, in the order they happen. A runner's container starts before GitHub gives it a job.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Timeline {
    /// GitHub queued the job.
    pub queued_at: u64,
    /// The container of the runner that took it started.
    pub container_started_at: Option<u64>,
    /// The runner started the job.
    pub started_at: Option<u64>,
    /// The job finished, by GitHub's clock; or when its container was found gone.
    pub finished_at: Option<u64>,
    /// The container and the runner's registration were removed.
    pub cleaned_at: Option<u64>,
    /// Seconds of its wait that passed while no runnerd ran, as the daemon that started after measured them. Waits
    /// for a slot, as sizing, insights and the history's figures count them, leave them out; its own wait does not.
    #[serde(default)]
    pub unseen: u64,
    /// When runnerd first listed it queued (ADR 0092). `None` for a job no listing showed queued, such as one a
    /// warm runner took first, and for one an older runnerd recorded.
    #[serde(default)]
    pub seen_at: Option<u64>,
}

impl Timeline {
    /// When the runner that took it began to start for it: its container's start, or the job's queueing when that
    /// container waited for jobs before, as a warm runner does. `None` until a runner took it.
    pub fn runner_from(&self) -> Option<u64> {
        self.container_started_at
            .map(|started| started.max(self.queued_at))
    }
}

/// What a job used, over the samples taken while it ran (every few seconds). `None` until a sample exists.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobMetrics {
    /// CPU time, in milliseconds.
    pub cpu_time: Option<u64>,
    /// The highest CPU use of a sample interval, in millicores.
    pub cpu_peak: Option<u64>,
    /// CPU time over wall time, in millicores.
    pub cpu_average: Option<u64>,
    /// The highest memory use without page cache.
    pub memory_peak: Option<u64>,
    /// The average memory use without page cache.
    pub memory_average: Option<u64>,
    /// Bytes the container received.
    pub network_received: Option<u64>,
    /// Bytes the container sent.
    pub network_sent: Option<u64>,
    /// Of the bytes received, those the Docker Hub mirror served.
    pub mirror_bytes: u64,
    /// What the shared build cache served it and took from it.
    pub build_cache: BuildCacheCounts,
    /// Times the kernel held its container's sockets' receive windows small because the container's memory was
    /// under pressure (`memory.events` `sock_throttled`): reclaim went badly, mostly because the host was short of
    /// memory, and its downloads slowed. `None` when the kernel does not count it, or in records from before.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub sock_throttled: Option<u64>,
    /// The periods of its CPU quota, 100 ms each, in which it had work: its busy time (`cpu.stat` `nr_periods`).
    /// `None` where the kernel keeps no CPU quota's figures, or in records from before.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cpu_periods: Option<u64>,
    /// Of `cpu_periods`, those in which it used up its quota and was held until the next (`nr_throttled`): its CPU
    /// cap held it back.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cpu_throttled: Option<u64>,
    /// Milliseconds in which a task of it could run and waited for a CPU (`cpu.pressure` `some`): held by its cap,
    /// or by the other runners and programs on the host's CPUs. `None` without pressure stall information.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cpu_stalled: Option<u64>,
}

/// One figure of what a job used, worded for every client.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Total {
    /// What it measures: `Memory peak`, `Received`.
    pub label: &'static str,
    /// The figure with its unit.
    pub value: String,
}

impl JobMetrics {
    /// What the job used, over its whole life: memory, CPU and network, each where the daemon measured it. The
    /// peaks are of the samples taken every few seconds, so the figures' note says that
    /// ([`sampled_note`](super::sampled_note)).
    pub fn totals(&self) -> Vec<Total> {
        let mut totals = Vec::new();
        let mut add = |label: &'static str, value: Option<String>| {
            totals.extend(value.map(|value| Total { label, value }));
        };
        let bytes = |bytes: u64| ByteSize(bytes).to_string();
        add("Memory peak", self.memory_peak.map(bytes));
        add("Average memory", self.memory_average.map(bytes));
        add("CPU peak", self.cpu_peak.map(in_cores));
        add("Average CPU", self.cpu_average.map(in_cores));
        add("CPU time", self.cpu_time.map(cpu_time));
        add("Received", self.network_received.map(bytes));
        add("Sent", self.network_sent.map(bytes));
        // Counted from the mirror's own log, so a job without network samples still has it.
        add(
            "From the mirror",
            (self.network_received.is_some() || self.mirror_bytes > 0)
                .then(|| bytes(self.mirror_bytes)),
        );
        add(
            "Throttled for memory pressure",
            self.sock_throttled
                .filter(|times| *times > 0)
                .map(|times| format!("{times} times")),
        );
        totals
    }
}

/// What the shared build cache served one runner and took from it (ADR 0052), counted by the cache itself.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct BuildCacheCounts {
    /// Lookups that found an entry.
    pub hits: u64,
    /// Lookups that found none.
    pub misses: u64,
    /// Bytes of the entries it served.
    pub read_bytes: u64,
    /// Bytes of the entries it stored.
    pub written_bytes: u64,
    /// Entries it was sent and did not store: the job may not write (a pull request, another branch), or the
    /// cache was full.
    pub refused: u64,
    /// Whether the job could write: a push, scheduled or manually started run of a branch that may write.
    pub writer: bool,
}

impl BuildCacheCounts {
    /// Hits and misses.
    pub fn lookups(&self) -> u64 {
        self.hits + self.misses
    }

    /// Hits in percent of lookups, `None` without any.
    pub fn hit_percent(&self) -> Option<u64> {
        let lookups = self.lookups();
        (lookups > 0).then(|| self.hits * 100 / lookups)
    }

    /// Adds `other`'s counts; `writer` stays.
    pub fn add(&mut self, other: &BuildCacheCounts) {
        self.hits += other.hits;
        self.misses += other.misses;
        self.read_bytes += other.read_bytes;
        self.written_bytes += other.written_bytes;
        self.refused += other.refused;
    }
}

/// A job's stored log.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogSummary {
    /// Lines stored.
    pub lines: u64,
    /// Bytes stored.
    pub bytes: u64,
    /// Set once the log reached `history.log_max` and later lines were dropped.
    pub truncated: bool,
}

/// One step of a job, as GitHub reports it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Step {
    /// GitHub's step number; numbers may skip.
    pub number: u32,
    /// The name GitHub shows, such as `Set up job` or `Run docker run --rm hello-world`.
    pub name: String,
    /// `pending`, `queued`, `in_progress` or `completed`.
    pub status: String,
    /// Once completed: `success`, `failure`, `skipped`, `cancelled`, ….
    pub conclusion: Option<String>,
    /// When it started.
    pub started_at: Option<u64>,
    /// When it completed.
    pub completed_at: Option<u64>,
}

/// `GET /v1/jobs` query parameters. Jobs come newest first (by ID, which grows with time), or by `sort` (ADR 0135).
/// [`JobQuery::check`] says which queries the daemon refuses.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobQuery {
    /// Only this `owner/name`.
    pub repository: Option<String>,
    /// Only jobs in this state.
    pub state: Option<JobState>,
    /// Only jobs with this conclusion, such as `failure`.
    pub conclusion: Option<String>,
    /// Only jobs of the workflow with this name, such as `CI`.
    pub workflow: Option<String>,
    /// Only jobs with this name in their workflow, such as `build`.
    pub name: Option<String>,
    /// Only jobs queued at or after this time.
    pub since: Option<u64>,
    /// Only jobs queued before this time.
    pub until: Option<u64>,
    /// Only jobs with a smaller ID: the `next` of the previous page. Only in the default order, newest first.
    pub before: Option<u64>,
    /// At most this many jobs; default 50, at most 500. A page also ends once it holds
    /// [`JOBS_PAGE_BYTES`] of text.
    pub limit: Option<u32>,
    /// Only jobs whose name, workflow or branch holds this text, as written: 1 to [`JOB_SEARCH_MAX`] bytes, ASCII
    /// letters in either case. The history keeps no commit, so a commit is not searched.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub q: Option<String>,
    /// Only jobs of runs for this branch.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub branch: Option<String>,
    /// Only jobs whose record this says became of.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub record: Option<RecordFilter>,
    /// Sort by this; a job whose figure is unknown comes last either way, ties by ID. `queued`, newest first, is
    /// the default order, which pages with `before`; any other order is one page, the first `limit` jobs.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub sort: Option<JobSort>,
    /// Which way; default largest or latest first.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub order: Option<SortOrder>,
    /// Refused: a sorted list is one page, which a search or the filters narrow. Read only to say so.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
}

/// One page of jobs, as `GET /v1/jobs` answered before it searched and sorted: what [`JobList`] holds.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobPage {
    /// Newest first, or in the query's sort.
    pub jobs: Vec<JobSummary>,
    /// In the default order: pass as `before` for the next page; `None` on the last page, and in any other order.
    pub next: Option<u64>,
}

/// `GET /v1/jobs`: one page of jobs (ADR 0135). A [`JobPage`] with fields more, so a client that reads a `JobPage`
/// reads it too, and one that reads a `JobList` from an older daemon finds them absent.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobList {
    /// The jobs and, in the default order, `next`.
    #[serde(flatten)]
    pub page: JobPage,
    /// In any order but the default: how many jobs the filters admit, of which the page shows the first
    /// ([`JobList::shown_words`]).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub total: Option<u64>,
    /// The jobs whose name, workflow, branch, repository, URL or labels were longer than a list shows
    /// ([`JOB_TEXT_MAX`] bytes each, [`JOB_LABELS_MAX`] bytes of labels) and are cut here; their own page has them
    /// whole.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub shortened: Vec<u64>,
    /// The query as the daemon read it, its limit and direction filled in; `None` from a daemon that predates
    /// search and sorting, which ignored them ([`JobList::understood`]).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub applied: Option<JobQuery>,
}

/// `GET /v1/jobs/{id}/logs` query parameters. The response is `text/plain`, one log line per line.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogQuery {
    /// Start at this line, 0-based.
    pub from: Option<u64>,
    /// Keep the response open and send lines as they are stored, until the job finishes.
    #[serde(default)]
    pub follow: bool,
}

/// One sample of a container's use.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Sample {
    /// When it was taken.
    pub at: u64,
    /// CPU use since the previous sample, in millicores.
    pub cpu: u64,
    /// Memory use without page cache.
    pub memory: u64,
    /// Bytes received since the container started.
    pub network_received: u64,
    /// Bytes sent since the container started.
    pub network_sent: u64,
}

/// `GET /v1/jobs/{id}/samples`: what the job's container used over time, from its start to its removal. Samples
/// older than `history.samples` are deleted; the job's metrics stay.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SampleSeries {
    /// The job.
    pub job: u64,
    /// Seconds between samples.
    pub interval: u64,
    /// Oldest first.
    pub samples: Vec<Sample>,
}

/// One sample of the whole slice runners run in.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostSample {
    /// When it was taken.
    pub at: u64,
    /// CPU use of the slice since the previous sample, in millicores.
    pub cpu: u64,
    /// Memory use of the slice less its inactive page cache, as `docker stats` counts it: a conservative number,
    /// since active page cache still counts, which the high-water mark of measured admission compares to
    /// `MemoryMax`.
    pub memory: u64,
}

/// The host's memory at one moment, beside the slice's: what admission fits new runners in besides `MemoryMax`
/// (ADR 0055). Read every few seconds; not kept in the history.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostMemory {
    /// When it was read.
    pub at: u64,
    /// `MemTotal` in `/proc/meminfo`, in bytes.
    pub total: u64,
    /// `MemAvailable`: what the host could give without swapping, page cache it can drop included, in bytes.
    pub available: u64,
    /// The slice's memory without page cache and reclaimable slab: what the runners hold that `MemAvailable` leaves
    /// out.
    pub slice: u64,
    /// `/proc/pressure/memory`'s `some` line: the share of time at least one task waited for memory. `None` on a
    /// kernel without pressure stall information.
    pub some: Option<Pressure>,
    /// Its `full` line: the share of time every task that could run waited for memory.
    pub full: Option<Pressure>,
    /// The programs outside the slice that hold the most memory, largest first: at most a few, none under a floor.
    /// Read less often than the rest. Empty when none is that large.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub holders: Vec<Holder>,
    /// `[host] memory_ceiling`: the most memory the host can grow to, in bytes, for a VM whose hypervisor adds
    /// memory as the guest asks for it (ADR 0084). From the config, not from `/proc`; absent unless it is set.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ceiling: Option<u64>,
    /// `SwapTotal` and `SwapFree`, from the same read of `/proc/meminfo`; absent on a host without swap (ADR 0101).
    /// Admission counts none of it.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub swap: Option<Swap>,
}

/// A host's swap at one reading: zram and zswap's device included, as `/proc/meminfo` counts it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct Swap {
    /// `SwapTotal`, in bytes. Never 0: a host without swap has none.
    pub total: u64,
    /// `SwapFree`, in bytes.
    pub free: u64,
}

impl Swap {
    /// What is in use, in bytes.
    pub fn used(&self) -> u64 {
        self.total.saturating_sub(self.free)
    }
}

/// The margin kept free at least: 2G.
const MARGIN_MIN: u64 = 2 << 30;

/// The margin as a share of the host's memory, as a divisor: a sixteenth, 4G of 64G.
const MARGIN_SHARE: u64 = 16;

/// The plain margin `MemAvailable` keeps beside the runners on a host with `total` memory: the larger of `MARGIN_MIN`
/// and `total / MARGIN_SHARE` (ADR 0055).
pub fn host_margin(total: u64) -> u64 {
    MARGIN_MIN.max(total / MARGIN_SHARE)
}

impl HostMemory {
    /// What the host has for a new runner this moment, in bytes: `MemAvailable` and what it can still grow by, less
    /// the plain margin. Unlike `HostBudget::spare` it does not count what runners reserve past what they hold, which
    /// is memory they may yet take, not memory taken.
    pub fn room(&self) -> u64 {
        self.available
            .saturating_add(self.growable())
            .saturating_sub(host_margin(self.grown()))
    }

    /// The host's memory as admission counts it, in bytes: `MemTotal`, or the ceiling when that is more (ADR 0084).
    pub fn grown(&self) -> u64 {
        self.ceiling
            .map_or(self.total, |ceiling| ceiling.max(self.total))
    }

    /// What the host can still grow by before it reaches its ceiling, in bytes: 0 without one, or once `MemTotal`
    /// reached it.
    pub fn growable(&self) -> u64 {
        self.grown().saturating_sub(self.total)
    }

    /// What the host holds outside the slice's runners, in bytes: its other programs, tmpfs and the kernel, without
    /// page cache it can drop.
    pub fn outside(&self) -> u64 {
        self.total
            .saturating_sub(self.available)
            .saturating_sub(self.slice)
    }

    /// What holds the host's memory outside the slice, as the clients say it: `programs outside CI hold 30G` and,
    /// with `holders`, `, most java 9.3G and qemu-system-x86 5.8G`.
    pub fn outside_line(&self) -> String {
        let size = |bytes: u64| ByteSize(bytes).to_string();
        let line = format!("programs outside CI hold {}", size(self.outside()));
        let named: Vec<String> = self
            .holders
            .iter()
            .map(|holder| format!("{} {}", holder.name, size(holder.memory)))
            .collect();
        match named.as_slice() {
            [] => line,
            [one] => format!("{line}, most {one}"),
            [rest @ .., last] => format!("{line}, most {} and {last}", rest.join(", ")),
        }
    }
}

/// A program outside the slice and the memory its processes hold.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Holder {
    /// Its processes' name, as `/proc/<pid>/comm` says it: never a command line, argument or path.
    pub name: String,
    /// What its processes hold that the host can't drop as page cache (`RssAnon` and `RssShmem`), in bytes.
    pub memory: u64,
}

/// One line of a pressure stall file, in hundredths of a percent: 250 is 2.5% of the time.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Pressure {
    /// Over the last 10 seconds.
    pub avg10: u32,
    /// Over the last 60 seconds.
    pub avg60: u32,
}

/// `GET /v1/host/samples` query parameters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostSampleQuery {
    /// Only samples taken at or after this time; default the last hour, at most the last 24 hours.
    pub since: Option<u64>,
}

/// `GET /v1/host/samples`: the slice's use over time, oldest first.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostSamples {
    /// Seconds between samples.
    pub interval: u64,
    /// Oldest first.
    pub samples: Vec<HostSample>,
}

/// How `GET /v1/stats` groups jobs in time.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum Bucket {
    /// By UTC hour.
    Hour,
    /// By UTC day.
    #[default]
    Day,
}

/// `GET /v1/activity` query parameters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActivityQuery {
    /// The start of the window; default 24 hours ago. It ends now.
    pub since: Option<u64>,
}

/// `GET /v1/activity`: every job runnerd's runners started or finished in a window, by when that happened rather
/// than when the job was queued, so a job that waited or ran long counts in the window it started or ended in.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Activity {
    /// The start of the window.
    pub since: u64,
    /// When it was read: the end of the window.
    pub until: u64,
    /// Each job that started in the window, oldest start first.
    pub started: Vec<Started>,
    /// The jobs that finished in the window: completed, or lost with their runner.
    pub finished: u64,
    /// Of those, the ones that failed, timed out or were lost, as `GET /v1/stats` counts failures.
    pub failed: u64,
    /// Of the failed, the ones their runner failed rather than their own steps: lost with their runner, or failed
    /// after the kernel killed a process of theirs for memory at their cap.
    pub runner_failed: u64,
}

impl Activity {
    /// The 95th percentile of the waits of the jobs that started, by nearest rank as sizing's are; `None` when none
    /// started.
    pub fn wait_p95(&self) -> Option<u64> {
        let mut waits: Vec<u64> = self.started.iter().map(|start| start.waited).collect();
        waits.sort_unstable();
        let rank = (waits.len() * 95).div_ceil(100);
        waits.get(rank.checked_sub(1)?).copied()
    }

    /// The longest wait of the jobs that started in each of the 24 hours before `until`, oldest hour first; 0 for
    /// an hour without a start. Counted back from `until`, so the hours are the day the other figures count.
    pub fn wait_hours(&self) -> [u64; 24] {
        const HOUR: u64 = 3600;
        let since = self.until.saturating_sub(24 * HOUR);
        let mut hours = [0u64; 24];
        for start in self
            .started
            .iter()
            .filter(|start| (since..=self.until).contains(&start.at))
        {
            // A start exactly 24 hours ago is in the oldest hour.
            let back =
                usize::try_from((self.until - start.at) / HOUR).map_or(23, |back| back.min(23));
            hours[23 - back] = hours[23 - back].max(start.waited);
        }
        hours
    }
}

/// A job that started: when, and how long it had waited since it was queued.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct Started {
    /// When it started.
    pub at: u64,
    /// Seconds it waited, leaving out the time before the history's start and the time no runnerd ran (ADR 0081).
    pub waited: u64,
}

/// `GET /v1/stats` query parameters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct StatsQuery {
    /// Only this `owner/name`.
    pub repository: Option<String>,
    /// Only jobs queued at or after this time; default 30 days ago. At most 744 buckets before `until` (or now)
    /// are counted: 31 days of hours, two years of days.
    pub since: Option<u64>,
    /// Only jobs queued before this time.
    pub until: Option<u64>,
    /// The bucket size; default a day.
    #[serde(default)]
    pub bucket: Bucket,
}

/// `GET /v1/stats`: finished jobs per repository and time bucket, for trends. Sums, so clients can average.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Stats {
    /// The bucket size.
    pub bucket: Bucket,
    /// By bucket start, then repository.
    pub rows: Vec<StatsRow>,
}

/// The finished jobs of one repository in one bucket.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct StatsRow {
    /// The bucket's start.
    pub start: u64,
    /// `owner/name`.
    pub repository: String,
    /// Jobs completed or lost.
    pub jobs: u64,
    /// Of those, concluded `success`.
    pub succeeded: u64,
    /// Of those, concluded `failure` or `timed_out`, or lost.
    pub failed: u64,
    /// Of those, concluded `cancelled`.
    pub cancelled: u64,
    /// Seconds from queued to started, summed, leaving out the time before the history's start and the time no runnerd
    /// ran (ADR 0081).
    pub queue_seconds: u64,
    /// Seconds from started to finished, summed.
    pub run_seconds: u64,
    /// CPU time, in milliseconds, summed.
    pub cpu_time: u64,
    /// The highest memory peak of a job.
    pub memory_peak: u64,
    /// Bytes received, summed.
    pub network_received: u64,
    /// Bytes from the mirror, summed.
    pub mirror_bytes: u64,
}

/// What a cache volume holds.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "kind")]
pub enum CacheKind {
    /// A profile's read-only snapshot.
    Snapshot {
        /// The profile.
        profile: String,
    },
    /// A slot's layer changes and Docker Engine data.
    Slot {
        /// The profile.
        profile: String,
        /// 1-based.
        slot: usize,
    },
    /// The pnpm store all slots share.
    Pnpm,
    /// The Maven repository all slots share.
    Maven,
    /// The Docker Hub mirror's storage.
    Mirror,
    /// The build cache every runner shares: a directory of the daemon's, not a Docker volume.
    Build,
}

/// One cache volume and its size.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CacheVolume {
    /// The Docker volume.
    pub name: String,
    /// What it holds.
    #[serde(flatten)]
    pub kind: CacheKind,
    /// Its size at `measured_at`.
    pub bytes: u64,
    /// The size above which runnerd empties it, or for a snapshot starts it over at its next reseed, if it has one.
    pub limit: Option<u64>,
}

/// `GET /v1/caches`: every cache volume of the configured profiles, as last measured. Docker measures volumes
/// slowly, so runnerd measures them at start and whenever no runner runs after jobs ended.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Caches {
    /// When the sizes were measured; `None` before the first measurement.
    pub measured_at: Option<u64>,
    /// By name.
    pub volumes: Vec<CacheVolume>,
    /// The shared build cache now, when it is on.
    pub build: Option<BuildCacheStats>,
}

/// The shared build cache (ADR 0052): its size now, and what it served the jobs of the last 7 days.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct BuildCacheStats {
    /// The entries' bytes.
    pub bytes: u64,
    /// How many entries it holds.
    pub entries: u64,
    /// `[caches.build] max_size`.
    pub limit: u64,
    /// The runners of the last 7 days, summed.
    pub week: BuildCacheCounts,
    /// `[caches.build] actions_cache`: whether it also serves GitHub Actions' cache (ADR 0082). `false` from a
    /// daemon older than it, which never served it.
    #[serde(default)]
    pub actions_cache: bool,
}

/// `GET /v1/caches/history` query parameters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct CacheHistoryQuery {
    /// Only measurements at or after this time; default 30 days ago.
    pub since: Option<u64>,
}

/// One volume's size at one measurement.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CacheSize {
    /// When it was measured.
    pub at: u64,
    /// The Docker volume.
    pub name: String,
    /// Its size.
    pub bytes: u64,
}

/// `GET /v1/caches/history`: every measurement, oldest first.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CacheHistory {
    /// Oldest first.
    pub sizes: Vec<CacheSize>,
}