Skip to main content

runnerd_core/scheduler/
engine.rs

1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod history;
20mod images;
21mod records;
22mod recover;
23mod runners;
24mod status;
25mod tick;
26mod warm;
27
28use std::collections::{BTreeMap, BTreeSet};
29use std::sync::{Arc, Mutex};
30use std::time::{Duration, SystemTime};
31
32use tokio::sync::{mpsc, watch};
33use tokio::task::JoinHandle;
34use tracing::{info, warn};
35
36pub use self::control::{
37    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
38    Target, same_slice,
39};
40use self::history::{Closing, Found};
41use self::images::Images;
42pub use self::records::RecordSelection;
43use super::{Hold, SlotRef};
44use crate::admission::Key;
45use crate::api;
46use crate::buildcache::BuildCache;
47use crate::cache::{self, Names, Trimmed};
48use crate::config::Config;
49use crate::docker::{self, Docker, DockerApi};
50use crate::github::{Github, GithubApi, QueuedJob};
51use crate::isolation::{self, Isolation};
52use crate::observe::metrics::{self, Paths, Sampler, Tracked};
53use crate::observe::mirror::Leases;
54use crate::store::{Announced, MirrorPosition, Store};
55
56/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
57/// identifies the machine, so a changed host name still matches the runners started before the change, and two
58/// machines with the same or overlapping host names never take each other's runners for their own.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct HostIdentity {
61    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
62    id: String,
63    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
64    host: String,
65}
66
67impl HostIdentity {
68    /// This machine's identity, from `/etc/machine-id` and its host name.
69    pub fn read() -> std::io::Result<HostIdentity> {
70        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
71        let machine_id = std::fs::read_to_string("/etc/machine-id")
72            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
73            .map_err(|error| {
74                std::io::Error::new(
75                    error.kind(),
76                    format!("cannot read /etc/machine-id: {error}"),
77                )
78            })?;
79        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
80            std::io::Error::new(
81                std::io::ErrorKind::InvalidData,
82                "/etc/machine-id does not hold a machine ID",
83            )
84        })
85    }
86
87    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
88    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
89        let machine_id = machine_id.trim();
90        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
91            return None;
92        }
93        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
94        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
95            .bytes()
96            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
97                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
98            });
99        let host: String = hostname
100            .trim()
101            .chars()
102            .map(|c| {
103                if c.is_ascii_alphanumeric() {
104                    c.to_ascii_lowercase()
105                } else {
106                    '-'
107                }
108            })
109            .take(20)
110            .collect();
111        let host = host.trim_matches('-');
112        Some(HostIdentity {
113            id: format!("{:08x}", hash >> 32),
114            host: if host.is_empty() { "host" } else { host }.to_owned(),
115        })
116    }
117
118    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
119    /// machine's names.
120    pub fn prefix(&self) -> String {
121        format!("runnerd-{}-", self.id)
122    }
123
124    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
125    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
126        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
127    }
128}
129
130/// What the API needs from a running engine.
131#[derive(Debug, Clone)]
132pub struct Handle {
133    /// The latest status, replaced after every tick.
134    pub status: watch::Receiver<api::Status>,
135    /// Pause, resume and shutdown requests.
136    pub control: Arc<Control>,
137    /// The job history.
138    pub store: Store,
139    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
140    /// config is.
141    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
142    /// The records of failed jobs, replaced whenever one is made or removed.
143    pub records: watch::Receiver<api::Records>,
144    /// The shared build cache, for its size now.
145    pub build_cache: Option<BuildCache>,
146    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
147    /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
148    pub disk: watch::Receiver<Option<u64>>,
149}
150
151impl Handle {
152    /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
153    /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
154    pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
155        self.control.shutdown(mode).stopping()
156    }
157}
158
159/// Where the engine runs, besides its config and its GitHub and Docker clients.
160#[derive(Debug)]
161pub struct Setup {
162    /// The names of what runnerd creates in Docker.
163    pub names: Names,
164    /// This machine, which names its runners, so a restarted daemon recognizes them.
165    pub identity: HostIdentity,
166    /// The slice's `MemoryMax`.
167    pub memory_max: Option<u64>,
168    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
169    pub cpus: Option<u64>,
170    /// The job history.
171    pub store: Store,
172    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
173    /// announces only the difference, including what the config no longer has.
174    pub announced: Announced,
175    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
176    pub paused: Vec<String>,
177    /// Where the cgroup and proc file systems are.
178    pub paths: Paths,
179    /// The shared build cache, unless the daemon could not open it.
180    pub build_cache: Option<BuildCache>,
181}
182
183/// Why a runner starts.
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
185enum Purpose {
186    /// For a queued job.
187    Job,
188    /// As a warm runner, for jobs not queued yet.
189    Warm,
190}
191
192/// Whether Docker answers a stop.
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194enum DockerInStop {
195    /// It answers, if with errors, or no stop was asked for.
196    Answers,
197    /// It did not answer a call the stop needed: the stop calls it no more.
198    Failed,
199}
200
201/// A slot's runner and its container.
202#[derive(Debug, Clone)]
203struct Occupant {
204    repository: String,
205    runner_name: String,
206    runner_id: u64,
207    container: String,
208    busy: bool,
209    started_at: SystemTime,
210    /// Its container's memory cap, which a reload does not change.
211    memory: u64,
212    /// Its container's CPU limit in millicores, which a reload does not change.
213    cpus: u64,
214    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
215    /// one whose record says nothing of it, which reserves its cap.
216    admission: Option<api::Admission>,
217    /// The job it took, once its log capture found it.
218    job: Option<u64>,
219    /// That job as admission tells jobs apart, once the history had its record.
220    key: Option<Key>,
221    /// The labels it registered with; `None` for one adopted from a container that does not say.
222    labels: Option<Vec<String>>,
223    /// The image it runs; `None` for one adopted from a container that does not say.
224    image: Option<String>,
225    /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
226    /// label.
227    spec: Option<String>,
228    /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
229    /// container, which each tick tries again.
230    observed: bool,
231}
232
233impl Occupant {
234    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
235    /// the profile's. An unknown label set counts as the same.
236    fn outdated(
237        &self,
238        repository: &crate::config::Repository,
239        profile: &crate::config::Profile,
240    ) -> bool {
241        let differ = self.labels.as_ref().is_some_and(|labels| {
242            let set = |labels: &[String]| -> BTreeSet<String> {
243                labels
244                    .iter()
245                    .map(|label| label.to_ascii_lowercase())
246                    .collect()
247            };
248            set(labels) != set(&repository.labels)
249        });
250        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
251    }
252
253    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
254    fn other_image(&self, profile: &crate::config::Profile) -> bool {
255        self.image
256            .as_ref()
257            .is_some_and(|image| *image != profile.image.reference())
258    }
259
260    /// Whether another release started it, with another container spec: one that does not say is older.
261    fn other_spec(&self) -> bool {
262        self.spec.as_deref() != Some(runners::THIS_SPEC)
263    }
264
265    /// What it reserves of the budget under measured admission: its cap if it has no admission.
266    fn reserved(&self) -> u64 {
267        self.admission
268            .map_or(self.memory, |admission| admission.reserved)
269    }
270
271    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
272    fn reserved_cpu(&self) -> u64 {
273        self.admission
274            .and_then(|admission| admission.cpu)
275            .map_or(self.cpus, |cpu| cpu.reserved)
276    }
277
278    fn running_job(&self) -> api::RunningJob {
279        api::RunningJob {
280            repository: self.repository.clone(),
281            runner: self.runner_name.clone(),
282            job: self.job,
283        }
284    }
285}
286
287/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
288#[derive(Debug, Default)]
289struct RepositoryState {
290    queued: Vec<QueuedJob>,
291    /// Whether the last poll of its queued jobs succeeded.
292    listed: bool,
293    /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
294    /// not hold back the others (ADR 0058).
295    holds: BTreeMap<String, PoolHold>,
296    /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
297    /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
298    /// make it younger.
299    cpu_wait: BTreeMap<String, CpuWait>,
300    /// The profiles of its pools whose next job the last plan held for CPU.
301    cpu_held: BTreeSet<String>,
302}
303
304/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
305#[derive(Debug, Default)]
306struct PoolHold {
307    held: Option<Hold>,
308    /// Runners in a row that ended without taking a job.
309    failures: u32,
310    /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
311    idle_exits: u32,
312}
313
314/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
315#[derive(Debug, Clone, Copy)]
316struct CpuWait {
317    /// The job's ID.
318    job: u64,
319    /// When a plan first held it for CPU.
320    since: SystemTime,
321}
322
323/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
324/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
325/// a single Docker request (120s) could otherwise outlast.
326pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
327
328/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
329#[derive(Debug)]
330pub struct Engine<G = Github, D = Docker> {
331    config: Config,
332    /// Shared with the listings a tick reads while it answers commands.
333    github: Arc<G>,
334    docker: Arc<D>,
335    names: Names,
336    isolation: &'static dyn Isolation,
337    identity: HostIdentity,
338    runner_prefix: String,
339    memory_max: Option<u64>,
340    /// The CPUs the slice may use, in millicores, read at start.
341    cpus: Option<u64>,
342    slots: BTreeMap<String, Vec<Option<Occupant>>>,
343    repositories: BTreeMap<String, RepositoryState>,
344    reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
345    /// The pulled images of the config's profiles, and the removal of those no profile runs.
346    images: Images,
347    /// The memory cap each running reseed's container started with, which a reload does not change.
348    reseed_memory: BTreeMap<String, u64>,
349    /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
350    reseed_requested: BTreeSet<String>,
351    /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
352    draining: BTreeSet<String>,
353    /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
354    /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
355    drained_once: BTreeSet<String>,
356    /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
357    /// each tick tries again.
358    leftover: BTreeMap<String, String>,
359    /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
360    /// failed jobs' records: the first tick after each one's backoff tries again.
361    unread: Vec<recover::Left>,
362    /// Profiles a reload removed, kept while their runners or reseed still run.
363    retired: BTreeMap<String, crate::config::Profile>,
364    /// Profiles whose slot 1 ran a job since their last reseed check.
365    reseed_due: BTreeSet<String>,
366    /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
367    /// little slot 1 changed.
368    oversized: BTreeSet<String>,
369    /// When each profile's last reseed finished: a check of the caches that started before then measured the old
370    /// snapshot.
371    reseeded_at: BTreeMap<String, SystemTime>,
372    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
373    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
374    ran: BTreeMap<SlotRef, Key>,
375    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
376    /// `None` until one measured Docker's disk, when each image keeps its own default.
377    docker_disk: watch::Sender<Option<u64>>,
378    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
379    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
380    /// Whether a job ended since the shared caches were last checked.
381    trim_due: bool,
382    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
383    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
384    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
385    /// unless runners were adopted.
386    checked_at: SystemTime,
387    mirror_up: bool,
388    /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
389    docker_in_stop: DockerInStop,
390    /// The memory cap the running mirror started with.
391    mirror_memory: u64,
392    rate_limit_logged: bool,
393    /// Runners without a container that GitHub still holds busy, already logged.
394    busy_orphans: BTreeSet<u64>,
395    status: watch::Sender<api::Status>,
396    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
397    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
398    announced_profiles: Option<Vec<api::ProfileStatus>>,
399    control: Arc<Control>,
400    commands: Commands,
401    caches: watch::Sender<Vec<api::CacheVolume>>,
402    store: Store,
403    paths: Paths,
404    /// The containers the sampler reads.
405    tracked: watch::Sender<Vec<Tracked>>,
406    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
407    captures: BTreeMap<String, JoinHandle<history::Capture>>,
408    /// The runners whose log capture broke off, until they are gone.
409    recaptures: BTreeMap<String, history::Recapture>,
410    /// Who held which address, for the mirror's log.
411    leases: Leases,
412    /// How far the mirror's log was counted.
413    mirror_until: MirrorPosition,
414    /// Jobs recorded queued and not yet taken or closed, with their repository.
415    queued_jobs: BTreeMap<u64, String>,
416    /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
417    cancelling: BTreeMap<String, tokio::time::Instant>,
418    /// Jobs whose runner is gone, waiting for GitHub to report them completed.
419    closing: BTreeMap<u64, Closing>,
420    last_prune: Option<SystemTime>,
421    /// The records of failed jobs and the shells open in them.
422    records: records::RecordState,
423    /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
424    measured: admission::Measured,
425    /// What warm runners learned from the history, and the workflow runs seen lately.
426    warm: warm::Warm,
427    /// The shared build cache.
428    build_cache: Option<BuildCache>,
429    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
430    /// cache when `[caches.build] branches` names none.
431    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
432    /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
433    /// `build_cache::ENDED_WAIT`, by runner.
434    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
435    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
436    /// the answers.
437    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
438    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
439    visibility_asking: Option<JoinHandle<()>>,
440}
441
442fn unix_secs(time: SystemTime) -> u64 {
443    time.duration_since(SystemTime::UNIX_EPOCH)
444        .unwrap_or_default()
445        .as_secs()
446}
447
448impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
449    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
450    /// them; `memory_max` is the slice's.
451    #[allow(
452        clippy::too_many_lines,
453        reason = "one line per field of the engine's state, which split up would only be listed twice"
454    )]
455    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
456        let Setup {
457            names,
458            identity,
459            memory_max,
460            cpus,
461            store,
462            announced,
463            paused,
464            paths,
465            build_cache,
466        } = setup;
467        let slots = config
468            .profiles
469            .iter()
470            .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
471            .collect();
472        let repositories = config
473            .repositories
474            .iter()
475            .map(|repository| (repository.name.clone(), RepositoryState::default()))
476            .collect();
477        let (control, commands) = Control::new(
478            config
479                .repositories
480                .iter()
481                .map(|repository| repository.name.clone()),
482            paused,
483            store.clone(),
484        );
485        let control = Arc::new(control);
486        let (status, receiver) = watch::channel(api::Status {
487            version: env!("CARGO_PKG_VERSION").to_owned(),
488            shutting_down: false,
489            stopping: None,
490            budget: api::Budget {
491                slice: config.host.slice.clone(),
492                memory_max,
493                ..api::Budget::default()
494            },
495            profiles: Vec::new(),
496            // What clients resuming after the last run's events have, so the first publish announces the difference.
497            repositories: announced.repositories,
498            queued: Vec::new(),
499        });
500        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
501        let (records, records_receiver) = records::RecordState::new();
502        let (docker_disk, disk_receiver) = watch::channel(None);
503        let mut engine = Engine {
504            isolation: isolation::backend(config.host.isolation),
505            // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
506            mirror_memory: config.mirror_memory(),
507            config,
508            github: Arc::new(github),
509            docker: Arc::new(docker),
510            names,
511            runner_prefix: identity.prefix(),
512            identity,
513            memory_max,
514            cpus,
515            slots,
516            repositories,
517            reseeds: BTreeMap::new(),
518            images: Images::new(),
519            reseed_memory: BTreeMap::new(),
520            reseed_requested: BTreeSet::new(),
521            retired: BTreeMap::new(),
522            draining: BTreeSet::new(),
523            drained_once: BTreeSet::new(),
524            leftover: BTreeMap::new(),
525            unread: Vec::new(),
526            reseed_due: BTreeSet::new(),
527            oversized: BTreeSet::new(),
528            reseeded_at: BTreeMap::new(),
529            ran: BTreeMap::new(),
530            docker_disk,
531            disk: None,
532            // So the caches are measured once at start.
533            trim_due: true,
534            trim: None,
535            checked_at: SystemTime::now(),
536            // Unknown: an earlier run may have left it up, so the first idle tick stops it.
537            mirror_up: true,
538            docker_in_stop: DockerInStop::Answers,
539            rate_limit_logged: false,
540            busy_orphans: BTreeSet::new(),
541            status,
542            announced_profiles: Some(announced.profiles),
543            control: control.clone(),
544            commands,
545            caches,
546            store: store.clone(),
547            paths,
548            tracked: watch::channel(Vec::new()).0,
549            found: mpsc::unbounded_channel(),
550            captures: BTreeMap::new(),
551            recaptures: BTreeMap::new(),
552            leases: Leases::default(),
553            mirror_until: MirrorPosition::default(),
554            queued_jobs: BTreeMap::new(),
555            cancelling: BTreeMap::new(),
556            closing: BTreeMap::new(),
557            last_prune: None,
558            measured: admission::Measured::default(),
559            warm: warm::Warm::default(),
560            records,
561            build_cache: build_cache.clone(),
562            default_branches: BTreeMap::new(),
563            ended_uploads: BTreeMap::new(),
564            visibility_due: Arc::default(),
565            visibility_asking: None,
566        };
567        engine.publish();
568        (
569            engine,
570            Handle {
571                status: receiver,
572                control,
573                store,
574                caches: caches_receiver,
575                records: records_receiver,
576                build_cache,
577                disk: disk_receiver,
578            },
579        )
580    }
581
582    /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
583    /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
584    pub async fn run(mut self) {
585        tokio::spawn(metrics::run(
586            Sampler::new(self.paths.clone(), &self.config.host.slice),
587            self.tracked.subscribe(),
588            self.store.clone(),
589        ));
590        let control = self.control.clone();
591        let deadline = async {
592            control.hurried().await;
593            tokio::time::sleep(STOP_DEADLINE).await;
594        };
595        tokio::select! {
596            () = self.serve() => {}
597            () = deadline => warn!(
598                "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
599                STOP_DEADLINE.as_secs()
600            ),
601        }
602    }
603
604    /// Recovers, then schedules until a shutdown request is carried out.
605    async fn serve(&mut self) {
606        // Until Docker answers: scheduling without knowing the containers already running would start a second
607        // runner on a slot's volume and delete the running one's registration.
608        while let Err(error) = self.recover().await {
609            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
610            if self.control.mode() != Shutdown::No {
611                return;
612            }
613            tokio::select! {
614                () = tokio::time::sleep(Duration::from_secs(5)) => {}
615                () = self.control.wake.notified() => {}
616            }
617        }
618        self.schedule().await;
619    }
620
621    /// Schedules until a shutdown request is carried out, from a recovered engine.
622    async fn schedule(&mut self) {
623        let mut shown = Shutdown::No;
624        let mode = loop {
625            let mode = self.stopping();
626            // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
627            // Docker: until then status would show a daemon that is not stopping.
628            if mode != shown {
629                self.publish();
630                shown = mode;
631            }
632            if mode == Shutdown::Detach {
633                self.remove_idle().await;
634                self.collect_recordings().await;
635            } else {
636                self.tick(mode).await;
637            }
638            self.publish();
639            // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
640            // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
641            if self.stopping() != mode {
642                continue;
643            }
644            if self.stopped(mode) {
645                break mode;
646            }
647            let interval = if mode == Shutdown::No {
648                self.config.host.poll_interval
649            } else {
650                Duration::from_secs(5)
651            };
652            tokio::select! {
653                () = tokio::time::sleep(interval) => {}
654                () = self.control.wake.notified() => {}
655                // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
656                Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
657                    self.command(command).await;
658                    self.publish();
659                }
660                Some(()) = self.records.finished.recv() => {
661                    self.collect_recordings().await;
662                    self.publish();
663                }
664            }
665        };
666        // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
667        self.collect_reseeds().await;
668        if self.docker_in_stop == DockerInStop::Failed {
669            info!("stopped; the next start removes or adopts what Docker did not answer for");
670            return;
671        }
672        self.close_shells().await;
673        if mode == Shutdown::Detach {
674            // The mirror stays up only for the busy runners left behind.
675            let mirror = if self.config.caches.mirror.enabled {
676                " and the mirror"
677            } else {
678                ""
679            };
680            match self.occupied().count() {
681                0 => info!("stopped; no job was running"),
682                1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
683                runners => {
684                    info!("stopped, leaving {runners} busy runners{mirror} to the next start");
685                }
686            }
687            return;
688        }
689        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
690            warn!("cannot stop the mirror: {error}");
691        }
692        info!("stopped");
693    }
694
695    /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
696    fn stopping(&self) -> Shutdown {
697        match self.control.mode() {
698            Shutdown::No => Shutdown::No,
699            _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
700            mode => mode,
701        }
702    }
703
704    /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
705    /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
706    fn stopped(&self, mode: Shutdown) -> bool {
707        let idle = self.slots.values().flatten().all(Option::is_none);
708        let recorded = self.records.recordings.is_empty();
709        match mode {
710            Shutdown::No => false,
711            Shutdown::Detach => recorded,
712            Shutdown::Graceful | Shutdown::Force => idle && recorded,
713        }
714    }
715
716    /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
717    /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
718    /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
719    /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
720    fn docker_failed(&mut self, error: &docker::Error) {
721        if self.control.mode() != Shutdown::No
722            && self.docker_in_stop == DockerInStop::Answers
723            && docker::is_unreachable(error)
724        {
725            warn!("Docker does not answer during the stop; leaving the rest to the next start");
726            self.docker_in_stop = DockerInStop::Failed;
727            // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
728            // running jobs.
729            self.control.shutdown(Shutdown::Detach);
730        }
731    }
732
733    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
734    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
735    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
736    /// the clock, not on polls.
737    async fn serving<T: Send + 'static>(
738        &mut self,
739        work: impl Future<Output = T> + Send + 'static,
740    ) -> T {
741        let mut task = tokio::spawn(work);
742        loop {
743            tokio::select! {
744                biased;
745                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
746                    Ok(output) => return output,
747                    // As if it had panicked here.
748                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
749                    // Only while the runtime shuts down, which drops this task too.
750                    Err(Err(_)) => std::future::pending::<()>().await,
751                },
752                Some(command) = self.commands.recv() => {
753                    self.command(command).await;
754                    self.publish();
755                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
756                    // polls its repositories then, not a poll interval later.
757                    self.control.wake.notify_one();
758                }
759            }
760        }
761    }
762
763    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
764        self.slots.iter().flat_map(|(profile, slots)| {
765            slots.iter().enumerate().filter_map(|(index, slot)| {
766                slot.as_ref().map(|occupant| {
767                    (
768                        SlotRef {
769                            profile: profile.clone(),
770                            index,
771                        },
772                        occupant,
773                    )
774                })
775            })
776        })
777    }
778
779    /// The runner in `slot`, if one is.
780    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
781        self.slots
782            .get(&slot.profile)
783            .and_then(|slots| slots.get(slot.index))
784            .and_then(Option::as_ref)
785    }
786
787    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
788    /// runs in them.
789    fn drop_closed_slots(&mut self) {
790        for (name, slots) in &mut self.slots {
791            let count = self
792                .config
793                .profiles
794                .get(name)
795                .map_or(0, |profile| profile.slots as usize);
796            while slots.len() > count && slots.last().is_some_and(Option::is_none) {
797                slots.pop();
798            }
799        }
800        let profiles = &self.config.profiles;
801        self.slots
802            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
803        // The volumes of the slots dropped go at the next check of the caches.
804        let slots = &self.slots;
805        self.ran.retain(|slot, _| {
806            slots
807                .get(&slot.profile)
808                .is_some_and(|slots| slot.index < slots.len())
809        });
810        let (slots, reseeds) = (&self.slots, &self.reseeds);
811        self.retired
812            .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
813    }
814}
815
816#[cfg(test)]
817mod tests;