Skip to main content

runnerd_core/scheduler/
engine.rs

1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod history;
20mod images;
21mod records;
22mod recover;
23mod runners;
24mod status;
25mod tick;
26mod warm;
27
28use std::collections::{BTreeMap, BTreeSet};
29use std::sync::{Arc, Mutex};
30use std::time::{Duration, SystemTime};
31
32use tokio::sync::{mpsc, watch};
33use tokio::task::JoinHandle;
34use tracing::{info, warn};
35
36pub use self::control::{
37    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
38    Target, same_slice,
39};
40use self::history::{Closing, Found};
41use self::images::Images;
42pub use self::records::RecordSelection;
43use super::{Hold, SlotRef};
44use crate::admission::Key;
45use crate::api;
46use crate::buildcache::BuildCache;
47use crate::cache::{self, Names, Trimmed};
48use crate::config::Config;
49use crate::docker::{self, Docker, DockerApi};
50use crate::github::{Github, GithubApi, QueuedJob};
51use crate::isolation::{self, Isolation};
52use crate::observe::metrics::{self, Paths, Sampler, Tracked};
53use crate::observe::mirror::Leases;
54use crate::store::{Announced, MirrorPosition, Store};
55
56/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
57/// identifies the machine, so a changed host name still matches the runners started before the change, and two
58/// machines with the same or overlapping host names never take each other's runners for their own.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct HostIdentity {
61    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
62    id: String,
63    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
64    host: String,
65}
66
67impl HostIdentity {
68    /// This machine's identity, from `/etc/machine-id` and its host name.
69    pub fn read() -> std::io::Result<HostIdentity> {
70        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
71        let machine_id = std::fs::read_to_string("/etc/machine-id")
72            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
73            .map_err(|error| {
74                std::io::Error::new(
75                    error.kind(),
76                    format!("cannot read /etc/machine-id: {error}"),
77                )
78            })?;
79        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
80            std::io::Error::new(
81                std::io::ErrorKind::InvalidData,
82                "/etc/machine-id does not hold a machine ID",
83            )
84        })
85    }
86
87    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
88    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
89        let machine_id = machine_id.trim();
90        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
91            return None;
92        }
93        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
94        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
95            .bytes()
96            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
97                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
98            });
99        let host: String = hostname
100            .trim()
101            .chars()
102            .map(|c| {
103                if c.is_ascii_alphanumeric() {
104                    c.to_ascii_lowercase()
105                } else {
106                    '-'
107                }
108            })
109            .take(20)
110            .collect();
111        let host = host.trim_matches('-');
112        Some(HostIdentity {
113            id: format!("{:08x}", hash >> 32),
114            host: if host.is_empty() { "host" } else { host }.to_owned(),
115        })
116    }
117
118    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
119    /// machine's names.
120    pub fn prefix(&self) -> String {
121        format!("runnerd-{}-", self.id)
122    }
123
124    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
125    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
126        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
127    }
128}
129
130/// What the API needs from a running engine.
131#[derive(Debug, Clone)]
132pub struct Handle {
133    /// The latest status, replaced after every tick.
134    pub status: watch::Receiver<api::Status>,
135    /// Pause, resume and shutdown requests.
136    pub control: Arc<Control>,
137    /// The job history.
138    pub store: Store,
139    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
140    /// config is.
141    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
142    /// The records of failed jobs, replaced whenever one is made or removed.
143    pub records: watch::Receiver<api::Records>,
144    /// The shared build cache, for its size now.
145    pub build_cache: Option<BuildCache>,
146    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
147    /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
148    pub disk: watch::Receiver<Option<u64>>,
149}
150
151impl Handle {
152    /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
153    /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
154    pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
155        self.control.shutdown(mode).stopping()
156    }
157}
158
159/// Where the engine runs, besides its config and its GitHub and Docker clients.
160#[derive(Debug)]
161pub struct Setup {
162    /// The names of what runnerd creates in Docker.
163    pub names: Names,
164    /// This machine, which names its runners, so a restarted daemon recognizes them.
165    pub identity: HostIdentity,
166    /// The slice's `MemoryMax`.
167    pub memory_max: Option<u64>,
168    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
169    pub cpus: Option<u64>,
170    /// The job history.
171    pub store: Store,
172    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
173    /// announces only the difference, including what the config no longer has.
174    pub announced: Announced,
175    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
176    pub paused: Vec<String>,
177    /// Where the cgroup and proc file systems are.
178    pub paths: Paths,
179    /// The shared build cache, unless the daemon could not open it.
180    pub build_cache: Option<BuildCache>,
181}
182
183/// Why a runner starts, as its start's log line and its record say (ADR 0092).
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
185enum Purpose {
186    /// For a queued job.
187    Job,
188    /// As one of its repository's configured warm runners, for jobs not queued yet.
189    Warm,
190    /// As a warm runner past those, for a job its runs are expected to queue still.
191    Burst,
192}
193
194impl Purpose {
195    /// How the history stores it.
196    fn as_str(self) -> &'static str {
197        match self {
198            Purpose::Job => "job",
199            Purpose::Warm => "warm",
200            Purpose::Burst => "burst",
201        }
202    }
203}
204
205/// Whether Docker answers a stop.
206#[derive(Debug, Clone, Copy, PartialEq, Eq)]
207enum DockerInStop {
208    /// It answers, if with errors, or no stop was asked for.
209    Answers,
210    /// It did not answer a call the stop needed: the stop calls it no more.
211    Failed,
212}
213
214/// A slot's runner and its container.
215#[derive(Debug, Clone)]
216struct Occupant {
217    repository: String,
218    runner_name: String,
219    runner_id: u64,
220    container: String,
221    busy: bool,
222    started_at: SystemTime,
223    /// Its container's memory cap, which a reload does not change.
224    memory: u64,
225    /// Its container's CPU limit in millicores, which a reload does not change.
226    cpus: u64,
227    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
228    /// one whose record says nothing of it, which reserves its cap.
229    admission: Option<api::Admission>,
230    /// The job it took, once its log capture found it.
231    job: Option<u64>,
232    /// That job as admission tells jobs apart, once the history had its record.
233    key: Option<Key>,
234    /// The labels it registered with; `None` for one adopted from a container that does not say.
235    labels: Option<Vec<String>>,
236    /// The image it runs; `None` for one adopted from a container that does not say.
237    image: Option<String>,
238    /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
239    /// label.
240    spec: Option<String>,
241    /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
242    /// container, which each tick tries again.
243    observed: bool,
244    /// Why this daemon started it; `None` for one a restart adopted.
245    purpose: Option<Purpose>,
246}
247
248impl Occupant {
249    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
250    /// the profile's. An unknown label set counts as the same.
251    fn outdated(
252        &self,
253        repository: &crate::config::Repository,
254        profile: &crate::config::Profile,
255    ) -> bool {
256        let differ = self.labels.as_ref().is_some_and(|labels| {
257            let set = |labels: &[String]| -> BTreeSet<String> {
258                labels
259                    .iter()
260                    .map(|label| label.to_ascii_lowercase())
261                    .collect()
262            };
263            set(labels) != set(&repository.labels)
264        });
265        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
266    }
267
268    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
269    fn other_image(&self, profile: &crate::config::Profile) -> bool {
270        self.image
271            .as_ref()
272            .is_some_and(|image| *image != profile.image.reference())
273    }
274
275    /// Whether another release started it, with another container spec: one that does not say is older.
276    fn other_spec(&self) -> bool {
277        self.spec.as_deref() != Some(runners::THIS_SPEC)
278    }
279
280    /// What it reserves of the budget under measured admission: its cap if it has no admission.
281    fn reserved(&self) -> u64 {
282        self.admission
283            .map_or(self.memory, |admission| admission.reserved)
284    }
285
286    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
287    fn reserved_cpu(&self) -> u64 {
288        self.admission
289            .and_then(|admission| admission.cpu)
290            .map_or(self.cpus, |cpu| cpu.reserved)
291    }
292
293    fn running_job(&self) -> api::RunningJob {
294        api::RunningJob {
295            repository: self.repository.clone(),
296            runner: self.runner_name.clone(),
297            job: self.job,
298        }
299    }
300}
301
302/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
303#[derive(Debug, Default)]
304struct RepositoryState {
305    queued: Vec<QueuedJob>,
306    /// Whether the last poll of its queued jobs succeeded.
307    listed: bool,
308    /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
309    /// not hold back the others (ADR 0058).
310    holds: BTreeMap<String, PoolHold>,
311    /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
312    /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
313    /// make it younger.
314    cpu_wait: BTreeMap<String, CpuWait>,
315    /// The profiles of its pools whose next job the last plan held for CPU.
316    cpu_held: BTreeSet<String>,
317}
318
319/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
320#[derive(Debug, Default)]
321struct PoolHold {
322    held: Option<Hold>,
323    /// Runners in a row that ended without taking a job.
324    failures: u32,
325    /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
326    idle_exits: u32,
327}
328
329/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
330#[derive(Debug, Clone, Copy)]
331struct CpuWait {
332    /// The job's ID.
333    job: u64,
334    /// When a plan first held it for CPU.
335    since: SystemTime,
336}
337
338/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
339/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
340/// a single Docker request (120s) could otherwise outlast.
341pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
342
343/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
344#[derive(Debug)]
345pub struct Engine<G = Github, D = Docker> {
346    config: Config,
347    /// Shared with the listings a tick reads while it answers commands.
348    github: Arc<G>,
349    docker: Arc<D>,
350    names: Names,
351    isolation: &'static dyn Isolation,
352    identity: HostIdentity,
353    runner_prefix: String,
354    memory_max: Option<u64>,
355    /// The CPUs the slice may use, in millicores, read at start.
356    cpus: Option<u64>,
357    slots: BTreeMap<String, Vec<Option<Occupant>>>,
358    repositories: BTreeMap<String, RepositoryState>,
359    reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
360    /// The pulled images of the config's profiles, and the removal of those no profile runs.
361    images: Images,
362    /// The memory cap each running reseed's container started with, which a reload does not change.
363    reseed_memory: BTreeMap<String, u64>,
364    /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
365    reseed_requested: BTreeSet<String>,
366    /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
367    draining: BTreeSet<String>,
368    /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
369    /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
370    drained_once: BTreeSet<String>,
371    /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
372    /// each tick tries again.
373    leftover: BTreeMap<String, String>,
374    /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
375    /// failed jobs' records: the first tick after each one's backoff tries again.
376    unread: Vec<recover::Left>,
377    /// Profiles a reload removed, kept while their runners or reseed still run.
378    retired: BTreeMap<String, crate::config::Profile>,
379    /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
380    measure_due: BTreeSet<String>,
381    /// The measures of slot 1 running, by profile.
382    measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
383    /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
384    measured_at: BTreeMap<String, tokio::time::Instant>,
385    /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
386    /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
387    /// `snapshot_max`.
388    reseed_due: BTreeSet<String>,
389    /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
390    /// little slot 1 changed.
391    oversized: BTreeSet<String>,
392    /// When each profile's last reseed finished: a check of the caches that started before then measured the old
393    /// snapshot.
394    reseeded_at: BTreeMap<String, SystemTime>,
395    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
396    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
397    ran: BTreeMap<SlotRef, Key>,
398    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
399    /// `None` until one measured Docker's disk, when each image keeps its own default.
400    docker_disk: watch::Sender<Option<u64>>,
401    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
402    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
403    /// Whether a job ended since the shared caches were last checked.
404    trim_due: bool,
405    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
406    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
407    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
408    /// unless runners were adopted.
409    checked_at: SystemTime,
410    mirror_up: bool,
411    /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
412    docker_in_stop: DockerInStop,
413    /// The memory cap the running mirror started with.
414    mirror_memory: u64,
415    rate_limit_logged: bool,
416    /// Runners without a container that GitHub still holds busy, already logged.
417    busy_orphans: BTreeSet<u64>,
418    status: watch::Sender<api::Status>,
419    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
420    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
421    announced_profiles: Option<Vec<api::ProfileStatus>>,
422    control: Arc<Control>,
423    commands: Commands,
424    caches: watch::Sender<Vec<api::CacheVolume>>,
425    store: Store,
426    paths: Paths,
427    /// The containers the sampler reads.
428    tracked: watch::Sender<Vec<Tracked>>,
429    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
430    captures: BTreeMap<String, JoinHandle<history::Capture>>,
431    /// The runners whose log capture broke off, until they are gone.
432    recaptures: BTreeMap<String, history::Recapture>,
433    /// Who held which address, for the mirror's log.
434    leases: Leases,
435    /// How far the mirror's log was counted.
436    mirror_until: MirrorPosition,
437    /// Jobs recorded queued and not yet taken or closed, with their repository.
438    queued_jobs: BTreeMap<u64, String>,
439    /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
440    cancelling: BTreeMap<String, tokio::time::Instant>,
441    /// Jobs whose runner is gone, waiting for GitHub to report them completed.
442    closing: BTreeMap<u64, Closing>,
443    last_prune: Option<SystemTime>,
444    /// The records of failed jobs and the shells open in them.
445    records: records::RecordState,
446    /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
447    measured: admission::Measured,
448    /// What warm runners learned from the history, and the workflow runs seen lately.
449    warm: warm::Warm,
450    /// The shared build cache.
451    build_cache: Option<BuildCache>,
452    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
453    /// cache when `[caches.build] branches` names none.
454    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
455    /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
456    /// `build_cache::ENDED_WAIT`, by runner.
457    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
458    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
459    /// the answers.
460    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
461    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
462    visibility_asking: Option<JoinHandle<()>>,
463}
464
465fn unix_secs(time: SystemTime) -> u64 {
466    time.duration_since(SystemTime::UNIX_EPOCH)
467        .unwrap_or_default()
468        .as_secs()
469}
470
471impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
472    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
473    /// them; `memory_max` is the slice's.
474    #[allow(
475        clippy::too_many_lines,
476        reason = "one line per field of the engine's state, which split up would only be listed twice"
477    )]
478    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
479        let Setup {
480            names,
481            identity,
482            memory_max,
483            cpus,
484            store,
485            announced,
486            paused,
487            paths,
488            build_cache,
489        } = setup;
490        let slots = config
491            .profiles
492            .iter()
493            .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
494            .collect();
495        let repositories = config
496            .repositories
497            .iter()
498            .map(|repository| (repository.name.clone(), RepositoryState::default()))
499            .collect();
500        let (control, commands) = Control::new(
501            config
502                .repositories
503                .iter()
504                .map(|repository| repository.name.clone()),
505            paused,
506            store.clone(),
507        );
508        let control = Arc::new(control);
509        let (status, receiver) = watch::channel(api::Status {
510            version: env!("CARGO_PKG_VERSION").to_owned(),
511            shutting_down: false,
512            stopping: None,
513            budget: api::Budget {
514                slice: config.host.slice.clone(),
515                memory_max,
516                ..api::Budget::default()
517            },
518            profiles: Vec::new(),
519            // What clients resuming after the last run's events have, so the first publish announces the difference.
520            repositories: announced.repositories,
521            queued: Vec::new(),
522        });
523        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
524        let (records, records_receiver) = records::RecordState::new();
525        let (docker_disk, disk_receiver) = watch::channel(None);
526        let mut engine = Engine {
527            isolation: isolation::backend(config.host.isolation),
528            // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
529            mirror_memory: config.mirror_memory(),
530            config,
531            github: Arc::new(github),
532            docker: Arc::new(docker),
533            names,
534            runner_prefix: identity.prefix(),
535            identity,
536            memory_max,
537            cpus,
538            slots,
539            repositories,
540            reseeds: BTreeMap::new(),
541            images: Images::new(),
542            reseed_memory: BTreeMap::new(),
543            reseed_requested: BTreeSet::new(),
544            retired: BTreeMap::new(),
545            draining: BTreeSet::new(),
546            drained_once: BTreeSet::new(),
547            leftover: BTreeMap::new(),
548            unread: Vec::new(),
549            measure_due: BTreeSet::new(),
550            measures: BTreeMap::new(),
551            measured_at: BTreeMap::new(),
552            reseed_due: BTreeSet::new(),
553            oversized: BTreeSet::new(),
554            reseeded_at: BTreeMap::new(),
555            ran: BTreeMap::new(),
556            docker_disk,
557            disk: None,
558            // So the caches are measured once at start.
559            trim_due: true,
560            trim: None,
561            checked_at: SystemTime::now(),
562            // Unknown: an earlier run may have left it up, so the first idle tick stops it.
563            mirror_up: true,
564            docker_in_stop: DockerInStop::Answers,
565            rate_limit_logged: false,
566            busy_orphans: BTreeSet::new(),
567            status,
568            announced_profiles: Some(announced.profiles),
569            control: control.clone(),
570            commands,
571            caches,
572            store: store.clone(),
573            paths,
574            tracked: watch::channel(Vec::new()).0,
575            found: mpsc::unbounded_channel(),
576            captures: BTreeMap::new(),
577            recaptures: BTreeMap::new(),
578            leases: Leases::default(),
579            mirror_until: MirrorPosition::default(),
580            queued_jobs: BTreeMap::new(),
581            cancelling: BTreeMap::new(),
582            closing: BTreeMap::new(),
583            last_prune: None,
584            measured: admission::Measured::default(),
585            warm: warm::Warm::default(),
586            records,
587            build_cache: build_cache.clone(),
588            default_branches: BTreeMap::new(),
589            ended_uploads: BTreeMap::new(),
590            visibility_due: Arc::default(),
591            visibility_asking: None,
592        };
593        engine.publish();
594        (
595            engine,
596            Handle {
597                status: receiver,
598                control,
599                store,
600                caches: caches_receiver,
601                records: records_receiver,
602                build_cache,
603                disk: disk_receiver,
604            },
605        )
606    }
607
608    /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
609    /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
610    pub async fn run(mut self) {
611        tokio::spawn(metrics::run(
612            Sampler::new(self.paths.clone(), &self.config.host.slice),
613            self.tracked.subscribe(),
614            self.store.clone(),
615        ));
616        let control = self.control.clone();
617        let deadline = async {
618            control.hurried().await;
619            tokio::time::sleep(STOP_DEADLINE).await;
620        };
621        tokio::select! {
622            () = self.serve() => {}
623            () = deadline => warn!(
624                "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
625                STOP_DEADLINE.as_secs()
626            ),
627        }
628    }
629
630    /// Recovers, then schedules until a shutdown request is carried out.
631    async fn serve(&mut self) {
632        // Until Docker answers: scheduling without knowing the containers already running would start a second
633        // runner on a slot's volume and delete the running one's registration.
634        while let Err(error) = self.recover().await {
635            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
636            if self.control.mode() != Shutdown::No {
637                return;
638            }
639            tokio::select! {
640                () = tokio::time::sleep(Duration::from_secs(5)) => {}
641                () = self.control.wake.notified() => {}
642            }
643        }
644        self.schedule().await;
645    }
646
647    /// Schedules until a shutdown request is carried out, from a recovered engine.
648    async fn schedule(&mut self) {
649        let mut shown = Shutdown::No;
650        let mode = loop {
651            let mode = self.stopping();
652            // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
653            // Docker: until then status would show a daemon that is not stopping.
654            if mode != shown {
655                self.publish();
656                shown = mode;
657            }
658            if mode == Shutdown::Detach {
659                self.remove_every_idle().await;
660                self.collect_recordings().await;
661            } else {
662                self.tick(mode).await;
663            }
664            self.publish();
665            // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
666            // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
667            if self.stopping() != mode {
668                continue;
669            }
670            if self.stopped(mode) {
671                break mode;
672            }
673            let interval = if mode == Shutdown::No {
674                self.config.host.poll_interval
675            } else {
676                Duration::from_secs(5)
677            };
678            tokio::select! {
679                () = tokio::time::sleep(interval) => {}
680                () = self.control.wake.notified() => {}
681                // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
682                Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
683                    self.command(command).await;
684                    self.publish();
685                }
686                Some(()) = self.records.finished.recv() => {
687                    self.collect_recordings().await;
688                    self.publish();
689                }
690            }
691        };
692        // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
693        self.collect_reseeds().await;
694        if self.docker_in_stop == DockerInStop::Failed {
695            info!("stopped; the next start removes or adopts what Docker did not answer for");
696            return;
697        }
698        self.close_shells().await;
699        if mode == Shutdown::Detach {
700            // The mirror stays up only for the busy runners left behind.
701            let mirror = if self.config.caches.mirror.enabled {
702                " and the mirror"
703            } else {
704                ""
705            };
706            match self.occupied().count() {
707                0 => info!("stopped; no job was running"),
708                1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
709                runners => {
710                    info!("stopped, leaving {runners} busy runners{mirror} to the next start");
711                }
712            }
713            return;
714        }
715        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
716            warn!("cannot stop the mirror: {error}");
717        }
718        info!("stopped");
719    }
720
721    /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
722    fn stopping(&self) -> Shutdown {
723        match self.control.mode() {
724            Shutdown::No => Shutdown::No,
725            _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
726            mode => mode,
727        }
728    }
729
730    /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
731    /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
732    fn stopped(&self, mode: Shutdown) -> bool {
733        let idle = self.slots.values().flatten().all(Option::is_none);
734        let recorded = self.records.recordings.is_empty();
735        match mode {
736            Shutdown::No => false,
737            Shutdown::Detach => recorded,
738            Shutdown::Graceful | Shutdown::Force => idle && recorded,
739        }
740    }
741
742    /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
743    /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
744    /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
745    /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
746    fn docker_failed(&mut self, error: &docker::Error) {
747        if self.control.mode() != Shutdown::No
748            && self.docker_in_stop == DockerInStop::Answers
749            && docker::is_unreachable(error)
750        {
751            warn!("Docker does not answer during the stop; leaving the rest to the next start");
752            self.docker_in_stop = DockerInStop::Failed;
753            // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
754            // running jobs.
755            self.control.shutdown(Shutdown::Detach);
756        }
757    }
758
759    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
760    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
761    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
762    /// the clock, not on polls.
763    async fn serving<T: Send + 'static>(
764        &mut self,
765        work: impl Future<Output = T> + Send + 'static,
766    ) -> T {
767        let mut task = tokio::spawn(work);
768        loop {
769            tokio::select! {
770                biased;
771                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
772                    Ok(output) => return output,
773                    // As if it had panicked here.
774                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
775                    // Only while the runtime shuts down, which drops this task too.
776                    Err(Err(_)) => std::future::pending::<()>().await,
777                },
778                Some(command) = self.commands.recv() => {
779                    self.command(command).await;
780                    self.publish();
781                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
782                    // polls its repositories then, not a poll interval later.
783                    self.control.wake.notify_one();
784                }
785            }
786        }
787    }
788
789    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
790        self.slots.iter().flat_map(|(profile, slots)| {
791            slots.iter().enumerate().filter_map(|(index, slot)| {
792                slot.as_ref().map(|occupant| {
793                    (
794                        SlotRef {
795                            profile: profile.clone(),
796                            index,
797                        },
798                        occupant,
799                    )
800                })
801            })
802        })
803    }
804
805    /// The runner in `slot`, if one is.
806    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
807        self.slots
808            .get(&slot.profile)
809            .and_then(|slots| slots.get(slot.index))
810            .and_then(Option::as_ref)
811    }
812
813    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
814    /// runs in them.
815    fn drop_closed_slots(&mut self) {
816        for (name, slots) in &mut self.slots {
817            let count = self
818                .config
819                .profiles
820                .get(name)
821                .map_or(0, |profile| profile.slots as usize);
822            while slots.len() > count && slots.last().is_some_and(Option::is_none) {
823                slots.pop();
824            }
825        }
826        let profiles = &self.config.profiles;
827        self.slots
828            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
829        // The volumes of the slots dropped go at the next check of the caches.
830        let slots = &self.slots;
831        self.ran.retain(|slot, _| {
832            slots
833                .get(&slot.profile)
834                .is_some_and(|slots| slot.index < slots.len())
835        });
836        let (slots, reseeds) = (&self.slots, &self.reseeds);
837        self.retired
838            .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
839    }
840}
841
842#[cfg(test)]
843mod tests;