Skip to main content

runnerd_core/scheduler/
engine.rs

1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `leftovers` (the containers still to read or remove once nothing occupies them), `listing` (what a repository's last listing of queued jobs showed), `listing_pace` (when a repository is listed next), `ledger` (the jobs followed on GitHub, queued or closing), `claims` (what a repository's listings may show that this machine's runners took), `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `upkeep` (what each
11//! profile's measures and reseeds are doing and have due), `presence` (which repositories are listed or draining), `registrations` (what is remembered of a registration with no slot), `watches` (how each runner's container is watched: its log capture and what broken ones left), `mirror` (whether the mirror runs and at what cap), `build_cache` (each runner's side of the shared build cache and whether its job
12//! may write), `history` (what the store records), `warm` (how many warm runners each repository wants), `control` (the
13//! API's commands), `stop` (how far the daemon's stop has got) and `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod claims;
19mod clock;
20mod control;
21mod engagement;
22mod exits;
23mod governor;
24mod history;
25mod images;
26mod ledger;
27mod leftovers;
28mod listing;
29mod listing_pace;
30mod mirror;
31mod open_waits;
32mod pool_health;
33mod presence;
34mod records;
35mod recover;
36mod registrations;
37mod repository_state;
38mod runner_exit;
39mod runners;
40mod running_reads;
41mod slot;
42mod start;
43mod status;
44mod stop;
45mod tick;
46mod unsynced;
47mod upkeep;
48mod warm;
49mod watches;
50
51use std::collections::{BTreeMap, BTreeSet};
52use std::path::PathBuf;
53use std::sync::{Arc, Mutex};
54use std::time::{Duration, SystemTime};
55
56use tokio::sync::{mpsc, oneshot, watch};
57use tokio::task::JoinHandle;
58use tracing::{info, warn};
59
60use self::claims::Claims;
61use self::clock::Clock;
62pub use self::control::{
63    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
64    Target, same_slice,
65};
66use self::engagement::{Engagement, Mark};
67use self::history::{CAPTURE_DRAIN, Found};
68use self::images::Images;
69use self::ledger::Ledger;
70use self::leftovers::Leftovers;
71use self::mirror::Mirror;
72use self::presence::Repositories;
73pub use self::records::RecordSelection;
74use self::registrations::Registrations;
75use self::repository_state::RepositoryState;
76use self::slot::Slot;
77use self::tick::Scope;
78pub use self::unsynced::Durability;
79use self::unsynced::Unsynced;
80use self::upkeep::Upkeep;
81use self::watches::Watches;
82use super::SlotRef;
83use crate::actions::JobActions;
84use crate::admission::Key;
85use crate::api;
86use crate::archives;
87use crate::buildcache::BuildCache;
88use crate::cache::{self, Names, Trimmed};
89use crate::config::Config;
90use crate::docker::{self, Docker, DockerApi};
91use crate::github::{Github, GithubApi};
92use crate::isolation::{self, Isolation};
93use crate::observe::disk::Disks;
94use crate::observe::metrics::{self, Paths, Sampler, Tracked};
95use crate::observe::mirror::Leases;
96use crate::store::{Announced, MirrorPosition, Store};
97
98/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
99/// identifies the machine, so a changed host name still matches the runners started before the change, and two
100/// machines with the same or overlapping host names never take each other's runners for their own.
101#[derive(Debug, Clone, PartialEq, Eq)]
102pub struct HostIdentity {
103    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
104    id: String,
105    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
106    host: String,
107}
108
109impl HostIdentity {
110    /// This machine's identity, from `/etc/machine-id` and its host name.
111    pub fn read() -> std::io::Result<HostIdentity> {
112        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
113        let machine_id = std::fs::read_to_string("/etc/machine-id")
114            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
115            .map_err(|error| {
116                std::io::Error::new(
117                    error.kind(),
118                    format!("cannot read /etc/machine-id: {error}"),
119                )
120            })?;
121        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
122            std::io::Error::new(
123                std::io::ErrorKind::InvalidData,
124                "/etc/machine-id does not hold a machine ID",
125            )
126        })
127    }
128
129    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
130    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
131        let machine_id = machine_id.trim();
132        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
133            return None;
134        }
135        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
136        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
137            .bytes()
138            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
139                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
140            });
141        let host: String = hostname
142            .trim()
143            .chars()
144            .map(|c| {
145                if c.is_ascii_alphanumeric() {
146                    c.to_ascii_lowercase()
147                } else {
148                    '-'
149                }
150            })
151            .take(20)
152            .collect();
153        let host = host.trim_matches('-');
154        Some(HostIdentity {
155            id: format!("{:08x}", hash >> 32),
156            host: if host.is_empty() { "host" } else { host }.to_owned(),
157        })
158    }
159
160    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
161    /// machine's names.
162    pub fn prefix(&self) -> String {
163        format!("runnerd-{}-", self.id)
164    }
165
166    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
167    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
168        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
169    }
170}
171
172/// What the API needs from a running engine.
173#[derive(Debug, Clone)]
174pub struct Handle {
175    /// The latest status, replaced after every tick.
176    pub status: watch::Receiver<api::Status>,
177    /// Pause, resume and shutdown requests.
178    pub control: Arc<Control>,
179    /// The job history.
180    pub store: Store,
181    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
182    /// config is.
183    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
184    /// The records of failed jobs, replaced whenever one is made or removed.
185    pub records: watch::Receiver<api::Records>,
186    /// The shared build cache, for its size now.
187    pub build_cache: Option<BuildCache>,
188    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
189    /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
190    pub disk: watch::Receiver<Option<u64>>,
191    /// The owner's job actions, sent through the engine's GitHub client but never on its loop (ADR 0134).
192    pub actions: JobActions,
193}
194
195impl Handle {
196    /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
197    /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
198    pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
199        self.control.shutdown(mode).stopping()
200    }
201}
202
203/// Where the engine runs, besides its config and its GitHub and Docker clients.
204#[derive(Debug)]
205pub struct Setup {
206    /// The names of what runnerd creates in Docker.
207    pub names: Names,
208    /// This machine, which names its runners, so a restarted daemon recognizes them.
209    pub identity: HostIdentity,
210    /// The slice's `MemoryMax`.
211    pub memory_max: Option<u64>,
212    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
213    pub cpus: Option<u64>,
214    /// The job history.
215    pub store: Store,
216    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
217    /// announces only the difference, including what the config no longer has.
218    pub announced: Announced,
219    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
220    pub paused: Vec<String>,
221    /// Where the cgroup and proc file systems are.
222    pub paths: Paths,
223    /// The shared build cache, unless the daemon could not open it.
224    pub build_cache: Option<BuildCache>,
225    /// Where the action archives are kept (`[caches.actions] directory` or its default), unless the daemon could not
226    /// name a directory: then runners download their actions themselves.
227    pub action_archives: Option<PathBuf>,
228    /// Where the engine keeps whether what runners wrote without syncing may be lost to a host crash (ADR 0141), such
229    /// as `unsynced` beside the job history.
230    pub unsynced: PathBuf,
231    /// Writes what the kernel holds in memory to disk, before the engine says nothing is owed, and tells whether a
232    /// past boot ended in an orderly shutdown: `sync(2)` and the systemd journal.
233    pub durability: Durability,
234}
235
236/// Why a runner starts, as its start's log line and its record say (ADR 0092).
237#[derive(Debug, Clone, Copy, PartialEq, Eq)]
238enum Purpose {
239    /// For a queued job.
240    Job,
241    /// As one of its repository's configured warm runners, for jobs not queued yet.
242    Warm,
243    /// As a warm runner past those, for a job its runs are expected to queue still.
244    Burst,
245}
246
247impl Purpose {
248    /// How the history stores it.
249    fn as_str(self) -> &'static str {
250        match self {
251            Purpose::Job => "job",
252            Purpose::Warm => "warm",
253            Purpose::Burst => "burst",
254        }
255    }
256}
257
258/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
259#[derive(Debug, Clone, Copy, PartialEq, Eq)]
260enum Meanwhile {
261    /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
262    Turnover,
263    /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
264    Commands,
265}
266
267/// What a runner, or a start in flight, takes of the budgets: what `held_by` and `reserves_cpu` count it by, so a
268/// start counts as the runner it becomes.
269#[derive(Debug, Clone, Copy)]
270struct Footprint<'slot> {
271    runner_name: &'slot str,
272    /// Its container's memory cap.
273    memory: u64,
274    /// What it reserves of memory under measured admission.
275    reserved: u64,
276    /// Its container's CPU limit, in millicores.
277    cpus: u64,
278    /// What it reserves of the CPU budget, in millicores.
279    reserved_cpu: u64,
280    /// When it started, or its start was launched.
281    started_at: SystemTime,
282    /// Its job, as admission tells jobs apart.
283    key: Option<&'slot Key>,
284    /// When it took its job, if this daemon saw it do so.
285    job_found_here: Option<SystemTime>,
286    /// What its reservation went by.
287    basis: Option<api::Basis>,
288    /// Whether it is a runner yet.
289    stage: Stage,
290}
291
292/// Whether a footprint is a runner's or a start's in flight.
293#[derive(Debug, Clone, Copy, PartialEq, Eq)]
294enum Stage {
295    /// Named and admitted, registering its runner or creating its container: there is nothing to sample yet.
296    InFlight,
297    /// A runner in its slot, with its container.
298    Placed,
299}
300
301/// A slot's runner and its container.
302#[derive(Debug, Clone)]
303struct Occupant {
304    repository: String,
305    runner_name: String,
306    runner_id: u64,
307    container: String,
308    started_at: SystemTime,
309    /// Its container's memory cap, which a reload does not change.
310    memory: u64,
311    /// Its container's CPU limit in millicores, which a reload does not change.
312    cpus: u64,
313    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
314    /// one whose record says nothing of it, which reserves its cap.
315    admission: Option<api::Admission>,
316    /// The labels it registered with; `None` for one adopted from a container that does not say.
317    labels: Option<Vec<String>>,
318    /// The image it runs; `None` for one adopted from a container that does not say.
319    image: Option<String>,
320    /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
321    /// label.
322    spec: Option<String>,
323    /// Why it is there, whether it holds a job and which.
324    engagement: Engagement,
325}
326
327impl Occupant {
328    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
329    /// the profile's. An unknown label set counts as the same.
330    fn outdated(
331        &self,
332        repository: &crate::config::Repository,
333        profile: &crate::config::Profile,
334    ) -> bool {
335        let differ = self.labels.as_ref().is_some_and(|labels| {
336            let set = |labels: &[String]| -> BTreeSet<String> {
337                labels
338                    .iter()
339                    .map(|label| label.to_ascii_lowercase())
340                    .collect()
341            };
342            set(labels) != set(&repository.labels)
343        });
344        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
345    }
346
347    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
348    fn other_image(&self, profile: &crate::config::Profile) -> bool {
349        self.image
350            .as_ref()
351            .is_some_and(|image| *image != profile.image.reference())
352    }
353
354    /// Whether another release started it, with another container spec: one that does not say is older.
355    fn other_spec(&self) -> bool {
356        self.spec.as_deref() != Some(runners::THIS_SPEC)
357    }
358
359    /// What it takes of the budgets.
360    fn footprint(&self) -> Footprint<'_> {
361        Footprint {
362            runner_name: &self.runner_name,
363            memory: self.memory,
364            reserved: self.reserved(),
365            cpus: self.cpus,
366            reserved_cpu: self.reserved_cpu(),
367            started_at: self.started_at,
368            key: self.key(),
369            job_found_here: self.job_found_here(),
370            basis: self.admission.map(|admission| admission.basis),
371            stage: Stage::Placed,
372        }
373    }
374
375    /// What it reserves of the budget under measured admission: its cap if it has no admission.
376    fn reserved(&self) -> u64 {
377        self.admission
378            .map_or(self.memory, |admission| admission.reserved)
379    }
380
381    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
382    fn reserved_cpu(&self) -> u64 {
383        self.admission
384            .and_then(|admission| admission.cpu)
385            .map_or(self.cpus, |cpu| cpu.reserved)
386    }
387
388    /// Whether GitHub's last runners listing showed it busy, or GitHub said so since (`Engagement::listed_busy`).
389    fn listed_busy(&self) -> bool {
390        self.engagement.listed_busy()
391    }
392
393    /// The job its capture or its history row named (`Engagement::named_job`).
394    fn named_job(&self) -> Option<u64> {
395        self.engagement.named_job()
396    }
397
398    /// Whether `job` is the one named for it.
399    fn runs(&self, job: u64) -> bool {
400        self.engagement.runs(job)
401    }
402
403    /// Whether it holds a job as far as this daemon knows (`Engagement::engaged`).
404    fn engaged(&self) -> bool {
405        self.engagement.engaged()
406    }
407
408    /// Whether it waits for a job: not `engaged`.
409    fn idle(&self) -> bool {
410        !self.engaged()
411    }
412
413    /// Whether it may be running one of its pool's listed jobs that no capture named yet
414    /// (`Engagement::claims_a_listed_job`).
415    fn claims_a_listed_job(&self) -> bool {
416        self.engagement.claims_a_listed_job()
417    }
418
419    /// Whether a detaching stop removes it (`Engagement::goes_when_detaching`).
420    fn goes_when_detaching(&self) -> bool {
421        self.engagement.goes_when_detaching()
422    }
423
424    /// Why this daemon started it; `None` for one a restart adopted.
425    fn purpose(&self) -> Option<Purpose> {
426        self.engagement.purpose()
427    }
428
429    /// When this daemon's capture found its job, if it started under this daemon (`Engagement::job_found_here`).
430    fn job_found_here(&self) -> Option<SystemTime> {
431        self.engagement.job_found_here()
432    }
433
434    /// Its job as admission tells jobs apart, once the history had its record.
435    fn key(&self) -> Option<&Key> {
436        self.engagement.key()
437    }
438
439    /// Whether admission can tell its job apart yet. A job named with no record yet has no key.
440    fn has_key(&self) -> bool {
441        self.key().is_some()
442    }
443
444    /// Whether it holds a job admission cannot tell apart yet, as when its log capture failed.
445    fn runs_unknown_job(&self) -> bool {
446        self.engaged() && !self.has_key()
447    }
448
449    fn running_job(&self) -> api::RunningJob {
450        api::RunningJob {
451            repository: self.repository.clone(),
452            runner: self.runner_name.clone(),
453            job: self.named_job(),
454        }
455    }
456}
457
458/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
459/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
460/// a single Docker request (120s) could otherwise outlast.
461pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
462
463/// How long GitHub and Docker rest at least between the end of a tick and the start of the next, when a tick outlasts
464/// the interval between their starts (ADR 0111).
465const TICK_REST: Duration = Duration::from_secs(2);
466
467/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
468#[derive(Debug)]
469pub struct Engine<G = Github, D = Docker> {
470    config: Config,
471    /// Shared with the listings a tick reads while it answers commands.
472    github: Arc<G>,
473    docker: Arc<D>,
474    names: Names,
475    isolation: &'static dyn Isolation,
476    identity: HostIdentity,
477    runner_prefix: String,
478    /// What a sample's or a host reading's age goes by.
479    clock: Clock,
480    memory_max: Option<u64>,
481    /// The CPUs the slice may use, in millicores, read at start.
482    cpus: Option<u64>,
483    slots: BTreeMap<String, Vec<Slot>>,
484    /// The repositories, listed by the config or draining after a reload removed them.
485    repositories: Repositories,
486    /// What each profile's measures and reseeds are doing and have due.
487    upkeep: Upkeep,
488    /// Whether what runners wrote without syncing may be lost to a host crash.
489    unsynced: Unsynced,
490    /// The pulled images of the config's profiles, and the removal of those no profile runs.
491    images: Images,
492    /// The containers still to read or remove once nothing occupies them.
493    leftovers: Leftovers,
494    /// Profiles a reload removed, kept while their runners or reseed still run.
495    retired: BTreeMap<String, crate::config::Profile>,
496    /// The slots whose last start failed preparing its caches. Not kept over a restart.
497    prepare_failures: BTreeMap<SlotRef, runners::PrepareFailures>,
498    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
499    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
500    ran: BTreeMap<SlotRef, Key>,
501    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
502    /// `None` until one measured Docker's disk, when each image keeps its own default.
503    docker_disk: watch::Sender<Option<u64>>,
504    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
505    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
506    /// Whether a job ended since the shared caches were last checked.
507    trim_due: bool,
508    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
509    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
510    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
511    /// unless runners were adopted.
512    checked_at: SystemTime,
513    mirror: Mirror,
514    /// Until when GitHub's rate limit is used up, from the last answer that said so: no runner starts before.
515    github_limited_until: Option<SystemTime>,
516    /// What is known of the registrations with no slot.
517    registrations: Registrations,
518    status: watch::Sender<api::Status>,
519    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
520    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
521    announced_profiles: Option<Vec<api::ProfileStatus>>,
522    control: Arc<Control>,
523    commands: Commands,
524    caches: watch::Sender<Vec<api::CacheVolume>>,
525    store: Store,
526    paths: Paths,
527    /// The containers the sampler reads.
528    tracked: watch::Sender<Vec<Tracked>>,
529    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
530    /// How each runner's container is watched: its log capture, and what broken ones left.
531    watches: Watches,
532    /// Who held which address, for the mirror's log.
533    leases: Leases,
534    /// How far the mirror's log was counted.
535    mirror_until: MirrorPosition,
536    /// The jobs followed on GitHub that no runner of this machine runs now: queued, or closing once their runner
537    /// went.
538    ledger: Ledger,
539    last_prune: Option<SystemTime>,
540    /// When the running jobs were last read from GitHub, which is once every `poll_interval` (ADR 0124).
541    running_reads: running_reads::RunningReads,
542    /// The pace of the reads by what is left of GitHub's rate limit (ADR 0127).
543    governor: governor::Governor,
544    /// The records of failed jobs and the shells open in them.
545    records: records::RecordState,
546    /// The predictor, the high-water mark of measured admission, and what admission saw of the host.
547    measured: admission::Measured,
548    /// What warm runners learned from the history, and the workflow runs seen lately.
549    warm: warm::Warm,
550    /// The shared build cache.
551    build_cache: Option<BuildCache>,
552    /// The action archives' store: where it is, its settings and what it last published (ADR 0147).
553    archives: archives::Link,
554    /// The job actions, for the re-runs of runner faults the engine starts and the refusals its status shows.
555    actions: Arc<JobActions>,
556    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
557    /// cache when `[caches.build] branches` names none.
558    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
559    /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
560    /// `build_cache::ENDED_WAIT`, by runner.
561    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
562    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
563    /// the answers.
564    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
565    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
566    visibility_asking: Option<JoinHandle<()>>,
567    /// Released runners' registration deletes, sent in the background after their containers went (ADR 0113): each
568    /// tick reaps the finished ones, and a stop waits for the rest, up to `STOP_DEADLINE`.
569    deletes: Arc<Mutex<tokio::task::JoinSet<()>>>,
570    /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
571    exits: Arc<exits::Exits>,
572    /// The task following Docker's events into `exits`, while the engine schedules.
573    watcher: Option<exits::Watcher>,
574    /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
575    /// pass the event wakes while the tick waits for GitHub starts runners into their room.
576    released: BTreeSet<String>,
577    /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
578    /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
579    passes_in_tick: Duration,
580    /// The listing the engine began last, counted from 1 since its start, and when it began: the listing each answer
581    /// comes from, and how old a claim on a listed job is (`Claims`, `Engagement`).
582    listing: Mark,
583    /// By repository, the jobs its listings may show that this machine's runners took.
584    claims: Claims,
585    /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
586    /// a name, which GitHub, Docker and the history know them by. The history keeps each before the name is used,
587    /// and a restart takes it from there and the containers, so a name ahead of the clock, or a clock that stepped
588    /// back, is not used again.
589    named: u64,
590    /// The runners each batch launched, in launch order: what a test's reading taken mid-batch binds.
591    #[cfg(test)]
592    launches: start::Launches,
593}
594
595fn unix_secs(time: SystemTime) -> u64 {
596    time.duration_since(SystemTime::UNIX_EPOCH)
597        .unwrap_or_default()
598        .as_secs()
599}
600
601impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
602    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
603    /// them; `memory_max` is the slice's.
604    #[allow(
605        clippy::too_many_lines,
606        reason = "one line per field of the engine's state, which split up would only be listed twice"
607    )]
608    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
609        let Setup {
610            names,
611            identity,
612            memory_max,
613            cpus,
614            store,
615            announced,
616            paused,
617            paths,
618            build_cache,
619            action_archives,
620            unsynced,
621            durability,
622        } = setup;
623        let unsynced = Unsynced::open(unsynced, &paths.proc, durability);
624        let slots = config
625            .profiles
626            .iter()
627            .map(|(name, profile)| {
628                let mut slots = Vec::new();
629                slots.resize_with(profile.slots as usize, Slot::default);
630                (name.clone(), slots)
631            })
632            .collect();
633        let repositories = Repositories::new(
634            config
635                .repositories
636                .iter()
637                .map(|repository| repository.name.clone()),
638        );
639        let (control, commands) = Control::new(
640            config
641                .repositories
642                .iter()
643                .map(|repository| repository.name.clone()),
644            paused,
645            store.clone(),
646        );
647        let control = Arc::new(control);
648        control.set_action_policy(crate::actions::policy::Allowed::of(&config));
649        let github = Arc::new(github);
650        let actions = JobActions::new(
651            github.clone(),
652            store.clone(),
653            control.clone(),
654            crate::actions::LIMITS,
655        );
656        let (status, receiver) = watch::channel(api::Status {
657            job_actions: api::JobActions::configured(config.github.job_actions),
658            version: env!("CARGO_PKG_VERSION").to_owned(),
659            shutting_down: false,
660            stopping: None,
661            github_limited_until: None,
662            budget: api::Budget {
663                slice: config.host.slice.clone(),
664                memory_max,
665                ..api::Budget::default()
666            },
667            profiles: Vec::new(),
668            // What clients resuming after the last run's events have, so the first publish announces the difference.
669            repositories: announced.repositories,
670            queued: Vec::new(),
671            action_archives: None,
672        });
673        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
674        let (records, records_receiver) = records::RecordState::new();
675        let (docker_disk, disk_receiver) = watch::channel(None);
676        let archives = archives::Link::new(action_archives, &config);
677        let mut engine = Engine {
678            clock: Clock::default(),
679            isolation: isolation::backend(config.host.isolation),
680            // An earlier run may have left the mirror up, so the first idle tick stops it.
681            mirror: Mirror::unknown(config.mirror_memory()),
682            config,
683            github,
684            docker: Arc::new(docker),
685            names,
686            runner_prefix: identity.prefix(),
687            identity,
688            memory_max,
689            cpus,
690            slots,
691            repositories,
692            upkeep: Upkeep::default(),
693            unsynced,
694            images: Images::new(),
695            retired: BTreeMap::new(),
696            leftovers: Leftovers::default(),
697            prepare_failures: BTreeMap::new(),
698            ran: BTreeMap::new(),
699            docker_disk,
700            disk: None,
701            // So the caches are measured once at start.
702            trim_due: true,
703            trim: None,
704            checked_at: SystemTime::now(),
705            github_limited_until: None,
706            registrations: Registrations::default(),
707            status,
708            announced_profiles: Some(announced.profiles),
709            control: control.clone(),
710            commands,
711            caches,
712            store: store.clone(),
713            paths,
714            tracked: watch::channel(Vec::new()).0,
715            found: mpsc::unbounded_channel(),
716            watches: Watches::default(),
717            leases: Leases::default(),
718            mirror_until: MirrorPosition::default(),
719            ledger: Ledger::default(),
720            last_prune: None,
721            running_reads: running_reads::RunningReads::default(),
722            governor: governor::Governor::default(),
723            measured: admission::Measured::new(SystemTime::now()),
724            warm: warm::Warm::default(),
725            records,
726            build_cache: build_cache.clone(),
727            archives,
728            actions: Arc::new(actions.clone()),
729            default_branches: BTreeMap::new(),
730            ended_uploads: BTreeMap::new(),
731            visibility_due: Arc::default(),
732            visibility_asking: None,
733            deletes: Arc::default(),
734            exits: Arc::default(),
735            watcher: None,
736            released: BTreeSet::new(),
737            passes_in_tick: Duration::ZERO,
738            listing: Mark {
739                listing: 0,
740                at: tokio::time::Instant::now(),
741            },
742            named: 0,
743            #[cfg(test)]
744            launches: start::Launches::default(),
745            claims: Claims::default(),
746        };
747        engine.publish();
748        (
749            engine,
750            Handle {
751                status: receiver,
752                control,
753                store,
754                caches: caches_receiver,
755                records: records_receiver,
756                build_cache,
757                disk: disk_receiver,
758                actions,
759            },
760        )
761    }
762
763    /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
764    /// began, whichever comes first. Either way it returns, once the sampler stored what it was adding up, so the
765    /// daemon still closes its history.
766    pub async fn run(mut self) {
767        let (disk, disk_read) = oneshot::channel();
768        let sampler = tokio::spawn(metrics::run(
769            Sampler::new(self.paths.clone(), &self.config.host.slice),
770            self.tracked.subscribe(),
771            self.store.clone(),
772            disk_read,
773        ));
774        tokio::spawn(disk_target(self.docker.clone(), self.paths.clone(), disk));
775        // Stopped when this returns, as `archives` goes with it.
776        let archives = self.archives.start(self.github.clone());
777        // Ends at a stop, as it waits for one.
778        tokio::spawn(JobActions::clone(&self.actions).rerun_faults());
779        let control = self.control.clone();
780        let deadline = async {
781            control.hurried().await;
782            tokio::time::sleep(STOP_DEADLINE).await;
783        };
784        tokio::select! {
785            () = self.serve() => {}
786            () = deadline => warn!(
787                "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
788                STOP_DEADLINE.as_secs()
789            ),
790        }
791        // Written at the stop's start already, unless the deadline cut a tick short before the loop saw the stop.
792        self.note_stop_causes();
793        // Letting go of the tracked containers stops the sampler, which stores the drive's window it was adding up:
794        // the daemon closes its history right after this returns.
795        drop(self);
796        if let Some(archives) = archives {
797            archives.stop().await;
798        }
799        let _ = sampler.await;
800    }
801
802    /// Recovers, then schedules until a shutdown request is carried out.
803    async fn serve(&mut self) {
804        // Until Docker answers: scheduling without knowing the containers already running would start a second
805        // runner on a slot's volume and delete the running one's registration.
806        while let Err(error) = self.recover().await {
807            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
808            if self.control.mode() != Shutdown::No {
809                return;
810            }
811            tokio::select! {
812                () = tokio::time::sleep(Duration::from_secs(5)) => {}
813                () = self.control.wake.notified() => {}
814            }
815        }
816        self.schedule().await;
817    }
818
819    /// Schedules until a shutdown request is carried out, from a recovered engine. A tick is due when the first
820    /// repository is, each an interval after its last listing started, and at least `TICK_REST` after the last tick
821    /// ended (ADR 0111, 0128); it lists the repositories due. One a command or a wake-up brings sooner lists every
822    /// repository. Between ticks, a runner whose container Docker says stopped is released at once, and its slot taken
823    /// by a job the last poll listed (`turnover`, ADR 0095); that leaves the next tick when it was due.
824    async fn schedule(&mut self) {
825        self.watcher = Some(exits::Watcher::start(
826            self.docker.clone(),
827            self.exits.clone(),
828        ));
829        let mut shown = Shutdown::No;
830        // The first tick lists every repository: none was listed yet.
831        let mut scope = Scope::Every;
832        let mode = loop {
833            let mode = self.control.mode();
834            if mode != Shutdown::No && shown == Shutdown::No {
835                self.note_stop_causes();
836            }
837            // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
838            // Docker: until then status would show a daemon that is not stopping.
839            if mode != shown {
840                self.publish();
841                shown = mode;
842            }
843            // What the next tick's due time counts from: this one's listing, or, in a detach, which lists nothing,
844            // its start.
845            let anchor = if mode == Shutdown::Detach {
846                let began = tokio::time::Instant::now();
847                self.remove_every_idle().await;
848                self.collect_recordings().await;
849                began
850            } else {
851                self.tick_listing(mode, scope).await
852            };
853            self.publish();
854            // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
855            // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
856            if self.control.mode() != mode {
857                continue;
858            }
859            if self.stopped(mode) {
860                break mode;
861            }
862            // From the listings' starts, so the tick's own length does not push the next listing out, and two listings
863            // of a repository are never closer than its interval, however long either tick took to collect before
864            // listing.
865            let due = if mode == Shutdown::No {
866                self.next_tick_due(anchor)
867            } else {
868                let interval = Duration::from_secs(5);
869                (anchor + interval).max(tokio::time::Instant::now() + interval.min(TICK_REST))
870            };
871            scope = self.wait_for_tick(mode, due).await;
872        };
873        // Nothing more is started or released: what Docker says now is the next start's.
874        self.watcher = None;
875        // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
876        self.collect_reseeds().await;
877        self.finish_deletes().await;
878        if self.control.docker_gone() {
879            info!("stopped; the next start removes or adopts what Docker did not answer for");
880            return;
881        }
882        self.close_shells().await;
883        // What the runners wrote without syncing is on disk once the stop leaves none running (ADR 0141).
884        if self.occupied().next().is_none()
885            && let Err(error) = self.unsynced.stopped_clean()
886        {
887            warn!(
888                "cannot note that what runners wrote is on disk; the next boot empties their caches: {error}"
889            );
890        }
891        if mode == Shutdown::Detach {
892            // The mirror stays up only for the busy runners left behind.
893            let mirror = if self.config.caches.mirror.enabled {
894                " and the mirror"
895            } else {
896                ""
897            };
898            match self.occupied().count() {
899                0 => info!("stopped; no job was running"),
900                1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
901                runners => {
902                    info!("stopped, leaving {runners} busy runners{mirror} to the next start");
903                }
904            }
905            return;
906        }
907        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
908            warn!("cannot stop the mirror: {error}");
909        }
910        info!("stopped");
911    }
912
913    /// Looks at what GitHub's rate limit says before a tick and moves the pace of the reads a step if it calls for it
914    /// (ADR 0127). Registrations and deletes do not ask.
915    fn pace_by_limit(&mut self) {
916        let spend = self.github.spend();
917        if let Some(change) =
918            self.governor
919                .observe(tokio::time::Instant::now(), SystemTime::now(), spend)
920        {
921            info!("{}", change.line(self.config.host.poll_interval));
922        }
923    }
924
925    /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
926    /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
927    /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
928    /// Says what the tick lists: the repositories due when it came when due, and every repository when it came sooner.
929    async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) -> Scope {
930        loop {
931            let exits = self.exits.clone();
932            // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
933            // keep turning over.
934            tokio::select! {
935                biased;
936                () = tokio::time::sleep_until(due) => return Scope::Due,
937                () = self.control.wake.notified() => return Scope::Every,
938                // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
939                Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
940                    self.command(command).await;
941                    self.publish();
942                    return Scope::Every;
943                }
944                // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
945                // it was due. A stop returns to see whether it is done.
946                Some(()) = self.records.finished.recv() => {
947                    self.collect_recordings().await;
948                    self.publish();
949                    if mode != Shutdown::No || self.control.mode() != mode {
950                        return Scope::Every;
951                    }
952                }
953                // A capture named its runner's job: status says the runner runs it now, not after the next poll.
954                Some(found) = self.found.1.recv() => {
955                    self.note_found(&found);
956                    self.publish();
957                }
958                () = exits.noted(), if mode != Shutdown::Detach => {
959                    if mode != Shutdown::No {
960                        return Scope::Every;
961                    }
962                    self.turnover().await;
963                    self.publish();
964                    if self.control.mode() != mode {
965                        return Scope::Every;
966                    }
967                }
968            }
969        }
970    }
971
972    /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
973    /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
974    fn stopped(&self, mode: Shutdown) -> bool {
975        let idle = self.slots.values().flatten().all(Slot::is_free);
976        let recorded = !self.records.is_recording();
977        match mode {
978            Shutdown::No => false,
979            Shutdown::Detach => recorded,
980            Shutdown::Graceful | Shutdown::Force => idle && recorded,
981        }
982    }
983
984    /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
985    /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
986    /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
987    /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
988    fn docker_failed(&self, error: &docker::Error) {
989        if docker::is_unreachable(error) && self.control.note_docker_gone() {
990            warn!("Docker does not answer during the stop; leaving the rest to the next start");
991        }
992    }
993
994    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
995    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
996    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
997    /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
998    /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
999    /// while, and the jobs captures name are noted as they come.
1000    async fn serving<T: Send + 'static>(
1001        &mut self,
1002        work: impl Future<Output = T> + Send + 'static,
1003        meanwhile: Meanwhile,
1004    ) -> T {
1005        let mut task = tokio::spawn(work);
1006        loop {
1007            let exits = self.exits.clone();
1008            tokio::select! {
1009                biased;
1010                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
1011                    Ok(output) => return output,
1012                    // As if it had panicked here.
1013                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
1014                    // Only while the runtime shuts down, which drops this task too.
1015                    Err(Err(_)) => std::future::pending::<()>().await,
1016                },
1017                Some(command) = self.commands.recv() => {
1018                    self.command(command).await;
1019                    self.publish();
1020                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
1021                    // polls its repositories then, not a poll interval later.
1022                    self.control.wake.notify_one();
1023                }
1024                () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
1025                    let began = tokio::time::Instant::now();
1026                    self.turnover().await;
1027                    self.passes_in_tick += began.elapsed();
1028                    self.publish();
1029                }
1030                Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
1031                    self.note_found(&found);
1032                    self.publish();
1033                }
1034            }
1035        }
1036    }
1037
1038    /// The jobs this machine's runners took, as their slots' occupants name them: computed where it is read, as a
1039    /// runner may take a job, or go, while the engine waits for GitHub.
1040    fn taken(&self) -> BTreeSet<u64> {
1041        self.occupied()
1042            .filter_map(|(_, occupant)| occupant.named_job())
1043            .collect()
1044    }
1045
1046    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
1047        self.slots.iter().flat_map(|(profile, slots)| {
1048            slots.iter().enumerate().filter_map(|(index, slot)| {
1049                slot.occupant().map(|occupant| {
1050                    (
1051                        SlotRef {
1052                            profile: profile.clone(),
1053                            index,
1054                        },
1055                        occupant,
1056                    )
1057                })
1058            })
1059        })
1060    }
1061
1062    /// What each runner and each start in flight takes of the budgets.
1063    fn footprints(&self) -> impl Iterator<Item = Footprint<'_>> {
1064        self.slots.values().flatten().filter_map(Slot::footprint)
1065    }
1066
1067    /// Each runner and each start in flight with its slot.
1068    fn slot_footprints(&self) -> impl Iterator<Item = (SlotRef, Footprint<'_>)> {
1069        self.slots.iter().flat_map(|(profile, slots)| {
1070            slots.iter().enumerate().filter_map(|(index, slot)| {
1071                slot.footprint().map(|footprint| {
1072                    (
1073                        SlotRef {
1074                            profile: profile.clone(),
1075                            index,
1076                        },
1077                        footprint,
1078                    )
1079                })
1080            })
1081        })
1082    }
1083
1084    /// The runner in `slot`, if one is.
1085    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
1086        self.slots
1087            .get(&slot.profile)
1088            .and_then(|slots| slots.get(slot.index))
1089            .and_then(Slot::occupant)
1090    }
1091
1092    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
1093    /// runs in them.
1094    fn drop_closed_slots(&mut self) {
1095        for (name, slots) in &mut self.slots {
1096            let count = self
1097                .config
1098                .profiles
1099                .get(name)
1100                .map_or(0, |profile| profile.slots as usize);
1101            while slots.len() > count && slots.last().is_some_and(Slot::is_free) {
1102                slots.pop();
1103            }
1104        }
1105        let profiles = &self.config.profiles;
1106        self.slots
1107            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
1108        // The volumes of the slots dropped go at the next check of the caches.
1109        let slots = &self.slots;
1110        self.ran.retain(|slot, _| {
1111            slots
1112                .get(&slot.profile)
1113                .is_some_and(|slots| slot.index < slots.len())
1114        });
1115        let (slots, upkeep) = (&self.slots, &self.upkeep);
1116        self.retired
1117            .retain(|name, _| slots.contains_key(name) || upkeep.reseed_runs(name));
1118    }
1119}
1120
1121/// Asks Docker for its root until it answers, every 5 s, and sends the sampler the disks it is on (ADR 0143). Stops
1122/// once the sampler is gone. Disks that can't be told are logged once, and nothing is sent: writes stay unknown.
1123async fn disk_target<D: DockerApi>(docker: Arc<D>, paths: Paths, disk: oneshot::Sender<Disks>) {
1124    let root = loop {
1125        if disk.is_closed() {
1126            return;
1127        }
1128        match docker.root_dir().await {
1129            Ok(root) => break root,
1130            Err(_) => tokio::time::sleep(Duration::from_secs(5)).await,
1131        }
1132    };
1133    let resolved = tokio::task::spawn_blocking({
1134        let root = root.clone();
1135        move || Disks::resolve(&paths, std::path::Path::new(&root))
1136    })
1137    .await;
1138    if let Ok(Some(target)) = resolved {
1139        info!("disk writes are read from {}, under {root}", target.name());
1140        let _ = disk.send(target);
1141    } else {
1142        warn!("cannot tell the disks under Docker's root {root}: what runners write stays unknown");
1143    }
1144}
1145
1146#[cfg(test)]
1147mod tests;