runnerd-core 0.0.29

Config, domain logic and API types shared by runnerd and its clients
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
//! through `Control`) and never logs the token or a JIT config.
//!
//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
//! runner's side of the shared build cache and whether its job may write), `history` (what
//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
//! `status` (what it publishes).

mod admission;
mod build_cache;
mod caches;
mod clock;
mod control;
mod exits;
mod history;
mod images;
mod records;
mod recover;
mod runners;
mod status;
mod tick;
mod warm;

use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};

use tokio::sync::{mpsc, watch};
use tokio::task::JoinHandle;
use tracing::{info, warn};

use self::clock::Clock;
pub use self::control::{
    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
    Target, same_slice,
};
use self::history::{CAPTURE_DRAIN, Closing, Found};
use self::images::Images;
pub use self::records::RecordSelection;
use super::{Hold, SlotRef, Waits};
use crate::admission::Key;
use crate::api;
use crate::buildcache::BuildCache;
use crate::cache::{self, Names, Trimmed};
use crate::config::Config;
use crate::docker::{self, Docker, DockerApi};
use crate::github::{Github, GithubApi, QueuedJob};
use crate::isolation::{self, Isolation};
use crate::observe::metrics::{self, Paths, Sampler, Tracked};
use crate::observe::mirror::Leases;
use crate::store::{Announced, MirrorPosition, Store};

/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
/// identifies the machine, so a changed host name still matches the runners started before the change, and two
/// machines with the same or overlapping host names never take each other's runners for their own.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HostIdentity {
    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
    id: String,
    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
    host: String,
}

impl HostIdentity {
    /// This machine's identity, from `/etc/machine-id` and its host name.
    pub fn read() -> std::io::Result<HostIdentity> {
        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
        let machine_id = std::fs::read_to_string("/etc/machine-id")
            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
            .map_err(|error| {
                std::io::Error::new(
                    error.kind(),
                    format!("cannot read /etc/machine-id: {error}"),
                )
            })?;
        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
            std::io::Error::new(
                std::io::ErrorKind::InvalidData,
                "/etc/machine-id does not hold a machine ID",
            )
        })
    }

    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
        let machine_id = machine_id.trim();
        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
            return None;
        }
        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
            .bytes()
            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
            });
        let host: String = hostname
            .trim()
            .chars()
            .map(|c| {
                if c.is_ascii_alphanumeric() {
                    c.to_ascii_lowercase()
                } else {
                    '-'
                }
            })
            .take(20)
            .collect();
        let host = host.trim_matches('-');
        Some(HostIdentity {
            id: format!("{:08x}", hash >> 32),
            host: if host.is_empty() { "host" } else { host }.to_owned(),
        })
    }

    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
    /// machine's names.
    pub fn prefix(&self) -> String {
        format!("runnerd-{}-", self.id)
    }

    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
    }
}

/// What the API needs from a running engine.
#[derive(Debug, Clone)]
pub struct Handle {
    /// The latest status, replaced after every tick.
    pub status: watch::Receiver<api::Status>,
    /// Pause, resume and shutdown requests.
    pub control: Arc<Control>,
    /// The job history.
    pub store: Store,
    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
    /// config is.
    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
    /// The records of failed jobs, replaced whenever one is made or removed.
    pub records: watch::Receiver<api::Records>,
    /// The shared build cache, for its size now.
    pub build_cache: Option<BuildCache>,
    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
    /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
    pub disk: watch::Receiver<Option<u64>>,
}

impl Handle {
    /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
    /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
    pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
        self.control.shutdown(mode).stopping()
    }
}

/// Where the engine runs, besides its config and its GitHub and Docker clients.
#[derive(Debug)]
pub struct Setup {
    /// The names of what runnerd creates in Docker.
    pub names: Names,
    /// This machine, which names its runners, so a restarted daemon recognizes them.
    pub identity: HostIdentity,
    /// The slice's `MemoryMax`.
    pub memory_max: Option<u64>,
    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
    pub cpus: Option<u64>,
    /// The job history.
    pub store: Store,
    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
    /// announces only the difference, including what the config no longer has.
    pub announced: Announced,
    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
    pub paused: Vec<String>,
    /// Where the cgroup and proc file systems are.
    pub paths: Paths,
    /// The shared build cache, unless the daemon could not open it.
    pub build_cache: Option<BuildCache>,
}

/// Why a runner starts, as its start's log line and its record say (ADR 0092).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Purpose {
    /// For a queued job.
    Job,
    /// As one of its repository's configured warm runners, for jobs not queued yet.
    Warm,
    /// As a warm runner past those, for a job its runs are expected to queue still.
    Burst,
}

impl Purpose {
    /// How the history stores it.
    fn as_str(self) -> &'static str {
        match self {
            Purpose::Job => "job",
            Purpose::Warm => "warm",
            Purpose::Burst => "burst",
        }
    }
}

/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Meanwhile {
    /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
    Turnover,
    /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
    Commands,
}

/// Whether Docker answers a stop.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DockerInStop {
    /// It answers, if with errors, or no stop was asked for.
    Answers,
    /// It did not answer a call the stop needed: the stop calls it no more.
    Failed,
}

/// A slot's runner and its container.
#[derive(Debug, Clone)]
struct Occupant {
    repository: String,
    runner_name: String,
    runner_id: u64,
    container: String,
    busy: bool,
    /// Whether the last runners listing was the first to show it busy, or GitHub said so since: the queued jobs
    /// listing after it may still show the job it took, as GitHub lists one queued for a few seconds after.
    newly_busy: bool,
    started_at: SystemTime,
    /// Its container's memory cap, which a reload does not change.
    memory: u64,
    /// Its container's CPU limit in millicores, which a reload does not change.
    cpus: u64,
    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
    /// one whose record says nothing of it, which reserves its cap.
    admission: Option<api::Admission>,
    /// The job it took, once its log capture found it.
    job: Option<u64>,
    /// When this daemon's capture found that job; `None` before, and for a job found again after a restart only by
    /// an adopted runner, whose `purpose` says so.
    job_started: Option<SystemTime>,
    /// That job as admission tells jobs apart, once the history had its record.
    key: Option<Key>,
    /// The labels it registered with; `None` for one adopted from a container that does not say.
    labels: Option<Vec<String>>,
    /// The image it runs; `None` for one adopted from a container that does not say.
    image: Option<String>,
    /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
    /// label.
    spec: Option<String>,
    /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
    /// container, which each tick tries again.
    observed: bool,
    /// Why this daemon started it; `None` for one a restart adopted.
    purpose: Option<Purpose>,
}

impl Occupant {
    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
    /// the profile's. An unknown label set counts as the same.
    fn outdated(
        &self,
        repository: &crate::config::Repository,
        profile: &crate::config::Profile,
    ) -> bool {
        let differ = self.labels.as_ref().is_some_and(|labels| {
            let set = |labels: &[String]| -> BTreeSet<String> {
                labels
                    .iter()
                    .map(|label| label.to_ascii_lowercase())
                    .collect()
            };
            set(labels) != set(&repository.labels)
        });
        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
    }

    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
    fn other_image(&self, profile: &crate::config::Profile) -> bool {
        self.image
            .as_ref()
            .is_some_and(|image| *image != profile.image.reference())
    }

    /// Whether another release started it, with another container spec: one that does not say is older.
    fn other_spec(&self) -> bool {
        self.spec.as_deref() != Some(runners::THIS_SPEC)
    }

    /// What it reserves of the budget under measured admission: its cap if it has no admission.
    fn reserved(&self) -> u64 {
        self.admission
            .map_or(self.memory, |admission| admission.reserved)
    }

    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
    fn reserved_cpu(&self) -> u64 {
        self.admission
            .and_then(|admission| admission.cpu)
            .map_or(self.cpus, |cpu| cpu.reserved)
    }

    fn running_job(&self) -> api::RunningJob {
        api::RunningJob {
            repository: self.repository.clone(),
            runner: self.runner_name.clone(),
            job: self.job,
        }
    }
}

/// A runner gone that may have taken one of its repository's listed jobs, which no capture named.
#[derive(Debug)]
struct Unnamed {
    /// The profile of its pool.
    profile: String,
    /// The `Engine::listing` begun last when it went.
    listing: u64,
}

/// What a repository's listings may show queued that this machine's runners took (ADR 0018, 0095). It outlives the
/// repository's removal from the config: a listing begun before the removal may answer after it is added back.
#[derive(Debug, Default)]
struct Claims {
    /// Its runners gone that may have taken a listed job without the engine knowing which: one ended before its
    /// log capture named its job, or one whose start failed may have run. Each counts against one listed job of its
    /// pool until a listing begun after it went answers.
    unnamed: Vec<Unnamed>,
    /// The jobs this machine's runners took whose runners are gone, by the `listing` begun last when they went:
    /// GitHub lists a job queued for a while after a runner took it, so a listing begun before that, or one GitHub
    /// lags on, still shows it. A listing begun after that and showing it no longer drops it.
    consumed: BTreeMap<u64, u64>,
}

/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
#[derive(Debug, Default)]
struct RepositoryState {
    queued: Vec<QueuedJob>,
    /// Whether the last poll of its queued jobs succeeded.
    listed: bool,
    /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
    /// not hold back the others (ADR 0058).
    holds: BTreeMap<String, PoolHold>,
    /// By the profile of each of its pools, what that pool's next job has waited for and since when (ADR 0096): from
    /// the first plan of a tick or a pass that held it, not one a status runs, until a runner of the pool is in its
    /// slot, the pool may no longer start one, or a listing that succeeded no longer shows the job. A plan that holds
    /// it for a slot or for the other resource does not make it younger, nor does a listing that failed. With them,
    /// whether a barrier has served the pool (ADR 0098), until a listing that succeeded shows its queue empty or the
    /// pool may no longer start a runner. Not kept over a restart.
    waits: BTreeMap<String, Waits>,
    /// The profiles of its pools whose next job the last plan held for CPU.
    cpu_held: BTreeSet<String>,
}

/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
#[derive(Debug, Default)]
struct PoolHold {
    held: Option<Hold>,
    /// Runners in a row that ended without taking a job.
    failures: u32,
    /// Runners in a row that connected and ended without a job, or waited `IDLE_TIMEOUT` for one (ADR 0104). Those
    /// `left_beside` holds count with the one before them.
    idle_exits: u32,
    /// The pool's other runners when the last of those ended: already up, they say nothing of a later start.
    left_beside: BTreeSet<String>,
}

/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
/// a single Docker request (120s) could otherwise outlast.
pub const STOP_DEADLINE: Duration = Duration::from_secs(30);

/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
#[derive(Debug)]
pub struct Engine<G = Github, D = Docker> {
    config: Config,
    /// Shared with the listings a tick reads while it answers commands.
    github: Arc<G>,
    docker: Arc<D>,
    names: Names,
    isolation: &'static dyn Isolation,
    identity: HostIdentity,
    runner_prefix: String,
    /// What a sample's or a host reading's age goes by.
    clock: Clock,
    memory_max: Option<u64>,
    /// The CPUs the slice may use, in millicores, read at start.
    cpus: Option<u64>,
    slots: BTreeMap<String, Vec<Option<Occupant>>>,
    repositories: BTreeMap<String, RepositoryState>,
    reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
    /// The pulled images of the config's profiles, and the removal of those no profile runs.
    images: Images,
    /// The memory cap each running reseed's container started with, which a reload does not change.
    reseed_memory: BTreeMap<String, u64>,
    /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
    reseed_requested: BTreeSet<String>,
    /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
    draining: BTreeSet<String>,
    /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
    /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
    drained_once: BTreeSet<String>,
    /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
    /// each tick tries again.
    leftover: BTreeMap<String, String>,
    /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
    /// failed jobs' records: the first tick after each one's backoff tries again.
    unread: Vec<recover::Left>,
    /// Profiles a reload removed, kept while their runners or reseed still run.
    retired: BTreeMap<String, crate::config::Profile>,
    /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
    measure_due: BTreeSet<String>,
    /// The measures of slot 1 running, by profile.
    measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
    /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
    measured_at: BTreeMap<String, tokio::time::Instant>,
    /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
    /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
    /// `snapshot_max`.
    reseed_due: BTreeSet<String>,
    /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
    /// little slot 1 changed.
    oversized: BTreeSet<String>,
    /// When each profile's last reseed finished: a check of the caches that started before then measured the old
    /// snapshot.
    reseeded_at: BTreeMap<String, SystemTime>,
    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
    ran: BTreeMap<SlotRef, Key>,
    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
    /// `None` until one measured Docker's disk, when each image keeps its own default.
    docker_disk: watch::Sender<Option<u64>>,
    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
    /// Whether a job ended since the shared caches were last checked.
    trim_due: bool,
    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
    /// unless runners were adopted.
    checked_at: SystemTime,
    mirror_up: bool,
    /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
    docker_in_stop: DockerInStop,
    /// The memory cap the running mirror started with.
    mirror_memory: u64,
    rate_limit_logged: bool,
    /// Runners without a container that GitHub still holds busy, already logged.
    busy_orphans: BTreeSet<u64>,
    /// Runners this engine released or killed, by ID, with their repository, until a listing of that repository no
    /// longer has them: GitHub holding one busy then is no orphan to warn of.
    ended_busy: BTreeMap<u64, String>,
    status: watch::Sender<api::Status>,
    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
    announced_profiles: Option<Vec<api::ProfileStatus>>,
    control: Arc<Control>,
    commands: Commands,
    caches: watch::Sender<Vec<api::CacheVolume>>,
    store: Store,
    paths: Paths,
    /// The containers the sampler reads.
    tracked: watch::Sender<Vec<Tracked>>,
    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
    captures: BTreeMap<String, JoinHandle<history::Capture>>,
    /// The runners whose log capture broke off, until they are gone.
    recaptures: BTreeMap<String, history::Recapture>,
    /// Who held which address, for the mirror's log.
    leases: Leases,
    /// How far the mirror's log was counted.
    mirror_until: MirrorPosition,
    /// Jobs recorded queued and not yet taken or closed, with their repository.
    queued_jobs: BTreeMap<u64, String>,
    /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
    cancelling: BTreeMap<String, tokio::time::Instant>,
    /// Jobs whose runner is gone, waiting for GitHub to report them completed.
    closing: BTreeMap<u64, Closing>,
    last_prune: Option<SystemTime>,
    /// The records of failed jobs and the shells open in them.
    records: records::RecordState,
    /// The predictor, the high-water mark of measured admission, and what admission saw of the host.
    measured: admission::Measured,
    /// What warm runners learned from the history, and the workflow runs seen lately.
    warm: warm::Warm,
    /// The shared build cache.
    build_cache: Option<BuildCache>,
    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
    /// cache when `[caches.build] branches` names none.
    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
    /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
    /// `build_cache::ENDED_WAIT`, by runner.
    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
    /// the answers.
    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
    visibility_asking: Option<JoinHandle<()>>,
    /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
    exits: Arc<exits::Exits>,
    /// The task following Docker's events into `exits`, while the engine schedules.
    watcher: Option<exits::Watcher>,
    /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
    /// pass the event wakes while the tick waits for GitHub starts runners into their room.
    released: BTreeSet<String>,
    /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
    /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
    passes_in_tick: Duration,
    /// How many times the engine began listing GitHub's queued jobs: the listing each answer comes from, and how
    /// old a claim on a listed job is (`Claims`).
    listing: u64,
    /// By repository, the jobs its listings may show that this machine's runners took.
    claims: BTreeMap<String, Claims>,
    /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
    /// a name, which GitHub, Docker and the history know them by. A restart takes it from the containers and the
    /// history, so a name ahead of the clock, or a clock that stepped back, is not used again.
    named: u64,
}

fn unix_secs(time: SystemTime) -> u64 {
    time.duration_since(SystemTime::UNIX_EPOCH)
        .unwrap_or_default()
        .as_secs()
}

impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
    /// them; `memory_max` is the slice's.
    #[allow(
        clippy::too_many_lines,
        reason = "one line per field of the engine's state, which split up would only be listed twice"
    )]
    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
        let Setup {
            names,
            identity,
            memory_max,
            cpus,
            store,
            announced,
            paused,
            paths,
            build_cache,
        } = setup;
        let slots = config
            .profiles
            .iter()
            .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
            .collect();
        let repositories = config
            .repositories
            .iter()
            .map(|repository| (repository.name.clone(), RepositoryState::default()))
            .collect();
        let (control, commands) = Control::new(
            config
                .repositories
                .iter()
                .map(|repository| repository.name.clone()),
            paused,
            store.clone(),
        );
        let control = Arc::new(control);
        let (status, receiver) = watch::channel(api::Status {
            version: env!("CARGO_PKG_VERSION").to_owned(),
            shutting_down: false,
            stopping: None,
            budget: api::Budget {
                slice: config.host.slice.clone(),
                memory_max,
                ..api::Budget::default()
            },
            profiles: Vec::new(),
            // What clients resuming after the last run's events have, so the first publish announces the difference.
            repositories: announced.repositories,
            queued: Vec::new(),
        });
        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
        let (records, records_receiver) = records::RecordState::new();
        let (docker_disk, disk_receiver) = watch::channel(None);
        let mut engine = Engine {
            clock: Clock::default(),
            isolation: isolation::backend(config.host.isolation),
            // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
            mirror_memory: config.mirror_memory(),
            config,
            github: Arc::new(github),
            docker: Arc::new(docker),
            names,
            runner_prefix: identity.prefix(),
            identity,
            memory_max,
            cpus,
            slots,
            repositories,
            reseeds: BTreeMap::new(),
            images: Images::new(),
            reseed_memory: BTreeMap::new(),
            reseed_requested: BTreeSet::new(),
            retired: BTreeMap::new(),
            draining: BTreeSet::new(),
            drained_once: BTreeSet::new(),
            leftover: BTreeMap::new(),
            unread: Vec::new(),
            measure_due: BTreeSet::new(),
            measures: BTreeMap::new(),
            measured_at: BTreeMap::new(),
            reseed_due: BTreeSet::new(),
            oversized: BTreeSet::new(),
            reseeded_at: BTreeMap::new(),
            ran: BTreeMap::new(),
            docker_disk,
            disk: None,
            // So the caches are measured once at start.
            trim_due: true,
            trim: None,
            checked_at: SystemTime::now(),
            // Unknown: an earlier run may have left it up, so the first idle tick stops it.
            mirror_up: true,
            docker_in_stop: DockerInStop::Answers,
            rate_limit_logged: false,
            busy_orphans: BTreeSet::new(),
            ended_busy: BTreeMap::new(),
            status,
            announced_profiles: Some(announced.profiles),
            control: control.clone(),
            commands,
            caches,
            store: store.clone(),
            paths,
            tracked: watch::channel(Vec::new()).0,
            found: mpsc::unbounded_channel(),
            captures: BTreeMap::new(),
            recaptures: BTreeMap::new(),
            leases: Leases::default(),
            mirror_until: MirrorPosition::default(),
            queued_jobs: BTreeMap::new(),
            cancelling: BTreeMap::new(),
            closing: BTreeMap::new(),
            last_prune: None,
            measured: admission::Measured::default(),
            warm: warm::Warm::default(),
            records,
            build_cache: build_cache.clone(),
            default_branches: BTreeMap::new(),
            ended_uploads: BTreeMap::new(),
            visibility_due: Arc::default(),
            visibility_asking: None,
            exits: Arc::default(),
            watcher: None,
            released: BTreeSet::new(),
            passes_in_tick: Duration::ZERO,
            listing: 0,
            named: 0,
            claims: BTreeMap::new(),
        };
        engine.publish();
        (
            engine,
            Handle {
                status: receiver,
                control,
                store,
                caches: caches_receiver,
                records: records_receiver,
                build_cache,
                disk: disk_receiver,
            },
        )
    }

    /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
    /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
    pub async fn run(mut self) {
        tokio::spawn(metrics::run(
            Sampler::new(self.paths.clone(), &self.config.host.slice),
            self.tracked.subscribe(),
            self.store.clone(),
        ));
        let control = self.control.clone();
        let deadline = async {
            control.hurried().await;
            tokio::time::sleep(STOP_DEADLINE).await;
        };
        tokio::select! {
            () = self.serve() => {}
            () = deadline => warn!(
                "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
                STOP_DEADLINE.as_secs()
            ),
        }
    }

    /// Recovers, then schedules until a shutdown request is carried out.
    async fn serve(&mut self) {
        // Until Docker answers: scheduling without knowing the containers already running would start a second
        // runner on a slot's volume and delete the running one's registration.
        while let Err(error) = self.recover().await {
            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
            if self.control.mode() != Shutdown::No {
                return;
            }
            tokio::select! {
                () = tokio::time::sleep(Duration::from_secs(5)) => {}
                () = self.control.wake.notified() => {}
            }
        }
        self.schedule().await;
    }

    /// Schedules until a shutdown request is carried out, from a recovered engine. Between ticks, a runner whose
    /// container Docker says stopped is released at once, and its slot taken by a job the last poll listed
    /// (`turnover`, ADR 0095); that leaves the next tick when it was due.
    async fn schedule(&mut self) {
        self.watcher = Some(exits::Watcher::start(
            self.docker.clone(),
            self.exits.clone(),
        ));
        let mut shown = Shutdown::No;
        let mode = loop {
            let mode = self.stopping();
            // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
            // Docker: until then status would show a daemon that is not stopping.
            if mode != shown {
                self.publish();
                shown = mode;
            }
            if mode == Shutdown::Detach {
                self.remove_every_idle().await;
                self.collect_recordings().await;
            } else {
                self.tick(mode).await;
            }
            self.publish();
            // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
            // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
            if self.stopping() != mode {
                continue;
            }
            if self.stopped(mode) {
                break mode;
            }
            let interval = if mode == Shutdown::No {
                self.config.host.poll_interval
            } else {
                Duration::from_secs(5)
            };
            self.wait_for_tick(mode, tokio::time::Instant::now() + interval)
                .await;
        };
        // Nothing more is started or released: what Docker says now is the next start's.
        self.watcher = None;
        // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
        self.collect_reseeds().await;
        if self.docker_in_stop == DockerInStop::Failed {
            info!("stopped; the next start removes or adopts what Docker did not answer for");
            return;
        }
        self.close_shells().await;
        if mode == Shutdown::Detach {
            // The mirror stays up only for the busy runners left behind.
            let mirror = if self.config.caches.mirror.enabled {
                " and the mirror"
            } else {
                ""
            };
            match self.occupied().count() {
                0 => info!("stopped; no job was running"),
                1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
                runners => {
                    info!("stopped, leaving {runners} busy runners{mirror} to the next start");
                }
            }
            return;
        }
        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
            warn!("cannot stop the mirror: {error}");
        }
        info!("stopped");
    }

    /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
    /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
    /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
    async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) {
        loop {
            let exits = self.exits.clone();
            // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
            // keep turning over.
            tokio::select! {
                biased;
                () = tokio::time::sleep_until(due) => return,
                () = self.control.wake.notified() => return,
                // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
                Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
                    self.command(command).await;
                    self.publish();
                    return;
                }
                // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
                // it was due. A stop returns to see whether it is done.
                Some(()) = self.records.finished.recv() => {
                    self.collect_recordings().await;
                    self.publish();
                    if mode != Shutdown::No || self.stopping() != mode {
                        return;
                    }
                }
                // A capture named its runner's job: status says the runner runs it now, not after the next poll.
                Some(found) = self.found.1.recv() => {
                    self.note_found(&found);
                    self.publish();
                }
                () = exits.noted(), if mode != Shutdown::Detach => {
                    if mode != Shutdown::No {
                        return;
                    }
                    self.turnover().await;
                    self.publish();
                    if self.stopping() != mode {
                        return;
                    }
                }
            }
        }
    }

    /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
    fn stopping(&self) -> Shutdown {
        match self.control.mode() {
            Shutdown::No => Shutdown::No,
            _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
            mode => mode,
        }
    }

    /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
    /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
    fn stopped(&self, mode: Shutdown) -> bool {
        let idle = self.slots.values().flatten().all(Option::is_none);
        let recorded = self.records.recordings.is_empty();
        match mode {
            Shutdown::No => false,
            Shutdown::Detach => recorded,
            Shutdown::Graceful | Shutdown::Force => idle && recorded,
        }
    }

    /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
    /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
    /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
    /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
    fn docker_failed(&mut self, error: &docker::Error) {
        if self.control.mode() != Shutdown::No
            && self.docker_in_stop == DockerInStop::Answers
            && docker::is_unreachable(error)
        {
            warn!("Docker does not answer during the stop; leaving the rest to the next start");
            self.docker_in_stop = DockerInStop::Failed;
            // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
            // running jobs.
            self.control.shutdown(Shutdown::Detach);
        }
    }

    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
    /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
    /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
    /// while, and the jobs captures name are noted as they come.
    async fn serving<T: Send + 'static>(
        &mut self,
        work: impl Future<Output = T> + Send + 'static,
        meanwhile: Meanwhile,
    ) -> T {
        let mut task = tokio::spawn(work);
        loop {
            let exits = self.exits.clone();
            tokio::select! {
                biased;
                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
                    Ok(output) => return output,
                    // As if it had panicked here.
                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
                    // Only while the runtime shuts down, which drops this task too.
                    Err(Err(_)) => std::future::pending::<()>().await,
                },
                Some(command) = self.commands.recv() => {
                    self.command(command).await;
                    self.publish();
                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
                    // polls its repositories then, not a poll interval later.
                    self.control.wake.notify_one();
                }
                () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
                    let began = tokio::time::Instant::now();
                    self.turnover().await;
                    self.passes_in_tick += began.elapsed();
                    self.publish();
                }
                Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
                    self.note_found(&found);
                    self.publish();
                }
            }
        }
    }

    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
        self.slots.iter().flat_map(|(profile, slots)| {
            slots.iter().enumerate().filter_map(|(index, slot)| {
                slot.as_ref().map(|occupant| {
                    (
                        SlotRef {
                            profile: profile.clone(),
                            index,
                        },
                        occupant,
                    )
                })
            })
        })
    }

    /// The runner in `slot`, if one is.
    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
        self.slots
            .get(&slot.profile)
            .and_then(|slots| slots.get(slot.index))
            .and_then(Option::as_ref)
    }

    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
    /// runs in them.
    fn drop_closed_slots(&mut self) {
        for (name, slots) in &mut self.slots {
            let count = self
                .config
                .profiles
                .get(name)
                .map_or(0, |profile| profile.slots as usize);
            while slots.len() > count && slots.last().is_some_and(Option::is_none) {
                slots.pop();
            }
        }
        let profiles = &self.config.profiles;
        self.slots
            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
        // The volumes of the slots dropped go at the next check of the caches.
        let slots = &self.slots;
        self.ran.retain(|slot, _| {
            slots
                .get(&slot.profile)
                .is_some_and(|slots| slot.index < slots.len())
        });
        let (slots, reseeds) = (&self.slots, &self.reseeds);
        self.retired
            .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
    }
}

#[cfg(test)]
mod tests;