Skip to main content

runnerd_core/scheduler/
engine.rs

1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod exits;
20mod history;
21mod images;
22mod records;
23mod recover;
24mod runners;
25mod status;
26mod tick;
27mod warm;
28
29use std::collections::{BTreeMap, BTreeSet};
30use std::sync::{Arc, Mutex};
31use std::time::{Duration, SystemTime};
32
33use tokio::sync::{mpsc, watch};
34use tokio::task::JoinHandle;
35use tracing::{info, warn};
36
37pub use self::control::{
38    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
39    Target, same_slice,
40};
41use self::history::{CAPTURE_DRAIN, Closing, Found};
42use self::images::Images;
43pub use self::records::RecordSelection;
44use super::{Hold, SlotRef};
45use crate::admission::Key;
46use crate::api;
47use crate::buildcache::BuildCache;
48use crate::cache::{self, Names, Trimmed};
49use crate::config::Config;
50use crate::docker::{self, Docker, DockerApi};
51use crate::github::{Github, GithubApi, QueuedJob};
52use crate::isolation::{self, Isolation};
53use crate::observe::metrics::{self, Paths, Sampler, Tracked};
54use crate::observe::mirror::Leases;
55use crate::store::{Announced, MirrorPosition, Store};
56
57/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
58/// identifies the machine, so a changed host name still matches the runners started before the change, and two
59/// machines with the same or overlapping host names never take each other's runners for their own.
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub struct HostIdentity {
62    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
63    id: String,
64    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
65    host: String,
66}
67
68impl HostIdentity {
69    /// This machine's identity, from `/etc/machine-id` and its host name.
70    pub fn read() -> std::io::Result<HostIdentity> {
71        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
72        let machine_id = std::fs::read_to_string("/etc/machine-id")
73            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
74            .map_err(|error| {
75                std::io::Error::new(
76                    error.kind(),
77                    format!("cannot read /etc/machine-id: {error}"),
78                )
79            })?;
80        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
81            std::io::Error::new(
82                std::io::ErrorKind::InvalidData,
83                "/etc/machine-id does not hold a machine ID",
84            )
85        })
86    }
87
88    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
89    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
90        let machine_id = machine_id.trim();
91        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
92            return None;
93        }
94        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
95        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
96            .bytes()
97            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
98                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
99            });
100        let host: String = hostname
101            .trim()
102            .chars()
103            .map(|c| {
104                if c.is_ascii_alphanumeric() {
105                    c.to_ascii_lowercase()
106                } else {
107                    '-'
108                }
109            })
110            .take(20)
111            .collect();
112        let host = host.trim_matches('-');
113        Some(HostIdentity {
114            id: format!("{:08x}", hash >> 32),
115            host: if host.is_empty() { "host" } else { host }.to_owned(),
116        })
117    }
118
119    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
120    /// machine's names.
121    pub fn prefix(&self) -> String {
122        format!("runnerd-{}-", self.id)
123    }
124
125    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
126    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
127        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
128    }
129}
130
131/// What the API needs from a running engine.
132#[derive(Debug, Clone)]
133pub struct Handle {
134    /// The latest status, replaced after every tick.
135    pub status: watch::Receiver<api::Status>,
136    /// Pause, resume and shutdown requests.
137    pub control: Arc<Control>,
138    /// The job history.
139    pub store: Store,
140    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
141    /// config is.
142    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
143    /// The records of failed jobs, replaced whenever one is made or removed.
144    pub records: watch::Receiver<api::Records>,
145    /// The shared build cache, for its size now.
146    pub build_cache: Option<BuildCache>,
147    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
148    /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
149    pub disk: watch::Receiver<Option<u64>>,
150}
151
152impl Handle {
153    /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
154    /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
155    pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
156        self.control.shutdown(mode).stopping()
157    }
158}
159
160/// Where the engine runs, besides its config and its GitHub and Docker clients.
161#[derive(Debug)]
162pub struct Setup {
163    /// The names of what runnerd creates in Docker.
164    pub names: Names,
165    /// This machine, which names its runners, so a restarted daemon recognizes them.
166    pub identity: HostIdentity,
167    /// The slice's `MemoryMax`.
168    pub memory_max: Option<u64>,
169    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
170    pub cpus: Option<u64>,
171    /// The job history.
172    pub store: Store,
173    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
174    /// announces only the difference, including what the config no longer has.
175    pub announced: Announced,
176    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
177    pub paused: Vec<String>,
178    /// Where the cgroup and proc file systems are.
179    pub paths: Paths,
180    /// The shared build cache, unless the daemon could not open it.
181    pub build_cache: Option<BuildCache>,
182}
183
184/// Why a runner starts, as its start's log line and its record say (ADR 0092).
185#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186enum Purpose {
187    /// For a queued job.
188    Job,
189    /// As one of its repository's configured warm runners, for jobs not queued yet.
190    Warm,
191    /// As a warm runner past those, for a job its runs are expected to queue still.
192    Burst,
193}
194
195impl Purpose {
196    /// How the history stores it.
197    fn as_str(self) -> &'static str {
198        match self {
199            Purpose::Job => "job",
200            Purpose::Warm => "warm",
201            Purpose::Burst => "burst",
202        }
203    }
204}
205
206/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
207#[derive(Debug, Clone, Copy, PartialEq, Eq)]
208enum Meanwhile {
209    /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
210    Turnover,
211    /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
212    Commands,
213}
214
215/// Whether Docker answers a stop.
216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
217enum DockerInStop {
218    /// It answers, if with errors, or no stop was asked for.
219    Answers,
220    /// It did not answer a call the stop needed: the stop calls it no more.
221    Failed,
222}
223
224/// A slot's runner and its container.
225#[derive(Debug, Clone)]
226struct Occupant {
227    repository: String,
228    runner_name: String,
229    runner_id: u64,
230    container: String,
231    busy: bool,
232    /// Whether the last runners listing was the first to show it busy, or GitHub said so since: the queued jobs
233    /// listing after it may still show the job it took, as GitHub lists one queued for a few seconds after.
234    newly_busy: bool,
235    started_at: SystemTime,
236    /// Its container's memory cap, which a reload does not change.
237    memory: u64,
238    /// Its container's CPU limit in millicores, which a reload does not change.
239    cpus: u64,
240    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
241    /// one whose record says nothing of it, which reserves its cap.
242    admission: Option<api::Admission>,
243    /// The job it took, once its log capture found it.
244    job: Option<u64>,
245    /// That job as admission tells jobs apart, once the history had its record.
246    key: Option<Key>,
247    /// The labels it registered with; `None` for one adopted from a container that does not say.
248    labels: Option<Vec<String>>,
249    /// The image it runs; `None` for one adopted from a container that does not say.
250    image: Option<String>,
251    /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
252    /// label.
253    spec: Option<String>,
254    /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
255    /// container, which each tick tries again.
256    observed: bool,
257    /// Why this daemon started it; `None` for one a restart adopted.
258    purpose: Option<Purpose>,
259}
260
261impl Occupant {
262    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
263    /// the profile's. An unknown label set counts as the same.
264    fn outdated(
265        &self,
266        repository: &crate::config::Repository,
267        profile: &crate::config::Profile,
268    ) -> bool {
269        let differ = self.labels.as_ref().is_some_and(|labels| {
270            let set = |labels: &[String]| -> BTreeSet<String> {
271                labels
272                    .iter()
273                    .map(|label| label.to_ascii_lowercase())
274                    .collect()
275            };
276            set(labels) != set(&repository.labels)
277        });
278        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
279    }
280
281    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
282    fn other_image(&self, profile: &crate::config::Profile) -> bool {
283        self.image
284            .as_ref()
285            .is_some_and(|image| *image != profile.image.reference())
286    }
287
288    /// Whether another release started it, with another container spec: one that does not say is older.
289    fn other_spec(&self) -> bool {
290        self.spec.as_deref() != Some(runners::THIS_SPEC)
291    }
292
293    /// What it reserves of the budget under measured admission: its cap if it has no admission.
294    fn reserved(&self) -> u64 {
295        self.admission
296            .map_or(self.memory, |admission| admission.reserved)
297    }
298
299    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
300    fn reserved_cpu(&self) -> u64 {
301        self.admission
302            .and_then(|admission| admission.cpu)
303            .map_or(self.cpus, |cpu| cpu.reserved)
304    }
305
306    fn running_job(&self) -> api::RunningJob {
307        api::RunningJob {
308            repository: self.repository.clone(),
309            runner: self.runner_name.clone(),
310            job: self.job,
311        }
312    }
313}
314
315/// A runner gone that may have taken one of its repository's listed jobs, which no capture named.
316#[derive(Debug)]
317struct Unnamed {
318    /// The profile of its pool.
319    profile: String,
320    /// The `Engine::listing` begun last when it went.
321    listing: u64,
322}
323
324/// What a repository's listings may show queued that this machine's runners took (ADR 0018, 0095). It outlives the
325/// repository's removal from the config: a listing begun before the removal may answer after it is added back.
326#[derive(Debug, Default)]
327struct Claims {
328    /// Its runners gone that may have taken a listed job without the engine knowing which: one ended before its
329    /// log capture named its job, or one whose start failed may have run. Each counts against one listed job of its
330    /// pool until a listing begun after it went answers.
331    unnamed: Vec<Unnamed>,
332    /// The jobs this machine's runners took whose runners are gone, by the `listing` begun last when they went:
333    /// GitHub lists a job queued for a while after a runner took it, so a listing begun before that, or one GitHub
334    /// lags on, still shows it. A listing begun after that and showing it no longer drops it.
335    consumed: BTreeMap<u64, u64>,
336}
337
338/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
339#[derive(Debug, Default)]
340struct RepositoryState {
341    queued: Vec<QueuedJob>,
342    /// Whether the last poll of its queued jobs succeeded.
343    listed: bool,
344    /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
345    /// not hold back the others (ADR 0058).
346    holds: BTreeMap<String, PoolHold>,
347    /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
348    /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
349    /// make it younger.
350    cpu_wait: BTreeMap<String, CpuWait>,
351    /// The profiles of its pools whose next job the last plan held for CPU.
352    cpu_held: BTreeSet<String>,
353}
354
355/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
356#[derive(Debug, Default)]
357struct PoolHold {
358    held: Option<Hold>,
359    /// Runners in a row that ended without taking a job.
360    failures: u32,
361    /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
362    idle_exits: u32,
363}
364
365/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
366#[derive(Debug, Clone, Copy)]
367struct CpuWait {
368    /// The job's ID.
369    job: u64,
370    /// When a plan first held it for CPU.
371    since: SystemTime,
372}
373
374/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
375/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
376/// a single Docker request (120s) could otherwise outlast.
377pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
378
379/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
380#[derive(Debug)]
381pub struct Engine<G = Github, D = Docker> {
382    config: Config,
383    /// Shared with the listings a tick reads while it answers commands.
384    github: Arc<G>,
385    docker: Arc<D>,
386    names: Names,
387    isolation: &'static dyn Isolation,
388    identity: HostIdentity,
389    runner_prefix: String,
390    memory_max: Option<u64>,
391    /// The CPUs the slice may use, in millicores, read at start.
392    cpus: Option<u64>,
393    slots: BTreeMap<String, Vec<Option<Occupant>>>,
394    repositories: BTreeMap<String, RepositoryState>,
395    reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
396    /// The pulled images of the config's profiles, and the removal of those no profile runs.
397    images: Images,
398    /// The memory cap each running reseed's container started with, which a reload does not change.
399    reseed_memory: BTreeMap<String, u64>,
400    /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
401    reseed_requested: BTreeSet<String>,
402    /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
403    draining: BTreeSet<String>,
404    /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
405    /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
406    drained_once: BTreeSet<String>,
407    /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
408    /// each tick tries again.
409    leftover: BTreeMap<String, String>,
410    /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
411    /// failed jobs' records: the first tick after each one's backoff tries again.
412    unread: Vec<recover::Left>,
413    /// Profiles a reload removed, kept while their runners or reseed still run.
414    retired: BTreeMap<String, crate::config::Profile>,
415    /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
416    measure_due: BTreeSet<String>,
417    /// The measures of slot 1 running, by profile.
418    measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
419    /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
420    measured_at: BTreeMap<String, tokio::time::Instant>,
421    /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
422    /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
423    /// `snapshot_max`.
424    reseed_due: BTreeSet<String>,
425    /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
426    /// little slot 1 changed.
427    oversized: BTreeSet<String>,
428    /// When each profile's last reseed finished: a check of the caches that started before then measured the old
429    /// snapshot.
430    reseeded_at: BTreeMap<String, SystemTime>,
431    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
432    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
433    ran: BTreeMap<SlotRef, Key>,
434    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
435    /// `None` until one measured Docker's disk, when each image keeps its own default.
436    docker_disk: watch::Sender<Option<u64>>,
437    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
438    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
439    /// Whether a job ended since the shared caches were last checked.
440    trim_due: bool,
441    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
442    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
443    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
444    /// unless runners were adopted.
445    checked_at: SystemTime,
446    mirror_up: bool,
447    /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
448    docker_in_stop: DockerInStop,
449    /// The memory cap the running mirror started with.
450    mirror_memory: u64,
451    rate_limit_logged: bool,
452    /// Runners without a container that GitHub still holds busy, already logged.
453    busy_orphans: BTreeSet<u64>,
454    status: watch::Sender<api::Status>,
455    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
456    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
457    announced_profiles: Option<Vec<api::ProfileStatus>>,
458    control: Arc<Control>,
459    commands: Commands,
460    caches: watch::Sender<Vec<api::CacheVolume>>,
461    store: Store,
462    paths: Paths,
463    /// The containers the sampler reads.
464    tracked: watch::Sender<Vec<Tracked>>,
465    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
466    captures: BTreeMap<String, JoinHandle<history::Capture>>,
467    /// The runners whose log capture broke off, until they are gone.
468    recaptures: BTreeMap<String, history::Recapture>,
469    /// Who held which address, for the mirror's log.
470    leases: Leases,
471    /// How far the mirror's log was counted.
472    mirror_until: MirrorPosition,
473    /// Jobs recorded queued and not yet taken or closed, with their repository.
474    queued_jobs: BTreeMap<u64, String>,
475    /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
476    cancelling: BTreeMap<String, tokio::time::Instant>,
477    /// Jobs whose runner is gone, waiting for GitHub to report them completed.
478    closing: BTreeMap<u64, Closing>,
479    last_prune: Option<SystemTime>,
480    /// The records of failed jobs and the shells open in them.
481    records: records::RecordState,
482    /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
483    measured: admission::Measured,
484    /// What warm runners learned from the history, and the workflow runs seen lately.
485    warm: warm::Warm,
486    /// The shared build cache.
487    build_cache: Option<BuildCache>,
488    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
489    /// cache when `[caches.build] branches` names none.
490    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
491    /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
492    /// `build_cache::ENDED_WAIT`, by runner.
493    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
494    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
495    /// the answers.
496    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
497    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
498    visibility_asking: Option<JoinHandle<()>>,
499    /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
500    exits: Arc<exits::Exits>,
501    /// The task following Docker's events into `exits`, while the engine schedules.
502    watcher: Option<exits::Watcher>,
503    /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
504    /// pass the event wakes while the tick waits for GitHub starts runners into their room.
505    released: BTreeSet<String>,
506    /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
507    /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
508    passes_in_tick: Duration,
509    /// How many times the engine began listing GitHub's queued jobs: the listing each answer comes from, and how
510    /// old a claim on a listed job is (`Claims`).
511    listing: u64,
512    /// By repository, the jobs its listings may show that this machine's runners took.
513    claims: BTreeMap<String, Claims>,
514    /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
515    /// a name, which GitHub, Docker and the history know them by. A restart takes it from the containers and the
516    /// history, so a name ahead of the clock, or a clock that stepped back, is not used again.
517    named: u64,
518}
519
520fn unix_secs(time: SystemTime) -> u64 {
521    time.duration_since(SystemTime::UNIX_EPOCH)
522        .unwrap_or_default()
523        .as_secs()
524}
525
526impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
527    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
528    /// them; `memory_max` is the slice's.
529    #[allow(
530        clippy::too_many_lines,
531        reason = "one line per field of the engine's state, which split up would only be listed twice"
532    )]
533    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
534        let Setup {
535            names,
536            identity,
537            memory_max,
538            cpus,
539            store,
540            announced,
541            paused,
542            paths,
543            build_cache,
544        } = setup;
545        let slots = config
546            .profiles
547            .iter()
548            .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
549            .collect();
550        let repositories = config
551            .repositories
552            .iter()
553            .map(|repository| (repository.name.clone(), RepositoryState::default()))
554            .collect();
555        let (control, commands) = Control::new(
556            config
557                .repositories
558                .iter()
559                .map(|repository| repository.name.clone()),
560            paused,
561            store.clone(),
562        );
563        let control = Arc::new(control);
564        let (status, receiver) = watch::channel(api::Status {
565            version: env!("CARGO_PKG_VERSION").to_owned(),
566            shutting_down: false,
567            stopping: None,
568            budget: api::Budget {
569                slice: config.host.slice.clone(),
570                memory_max,
571                ..api::Budget::default()
572            },
573            profiles: Vec::new(),
574            // What clients resuming after the last run's events have, so the first publish announces the difference.
575            repositories: announced.repositories,
576            queued: Vec::new(),
577        });
578        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
579        let (records, records_receiver) = records::RecordState::new();
580        let (docker_disk, disk_receiver) = watch::channel(None);
581        let mut engine = Engine {
582            isolation: isolation::backend(config.host.isolation),
583            // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
584            mirror_memory: config.mirror_memory(),
585            config,
586            github: Arc::new(github),
587            docker: Arc::new(docker),
588            names,
589            runner_prefix: identity.prefix(),
590            identity,
591            memory_max,
592            cpus,
593            slots,
594            repositories,
595            reseeds: BTreeMap::new(),
596            images: Images::new(),
597            reseed_memory: BTreeMap::new(),
598            reseed_requested: BTreeSet::new(),
599            retired: BTreeMap::new(),
600            draining: BTreeSet::new(),
601            drained_once: BTreeSet::new(),
602            leftover: BTreeMap::new(),
603            unread: Vec::new(),
604            measure_due: BTreeSet::new(),
605            measures: BTreeMap::new(),
606            measured_at: BTreeMap::new(),
607            reseed_due: BTreeSet::new(),
608            oversized: BTreeSet::new(),
609            reseeded_at: BTreeMap::new(),
610            ran: BTreeMap::new(),
611            docker_disk,
612            disk: None,
613            // So the caches are measured once at start.
614            trim_due: true,
615            trim: None,
616            checked_at: SystemTime::now(),
617            // Unknown: an earlier run may have left it up, so the first idle tick stops it.
618            mirror_up: true,
619            docker_in_stop: DockerInStop::Answers,
620            rate_limit_logged: false,
621            busy_orphans: BTreeSet::new(),
622            status,
623            announced_profiles: Some(announced.profiles),
624            control: control.clone(),
625            commands,
626            caches,
627            store: store.clone(),
628            paths,
629            tracked: watch::channel(Vec::new()).0,
630            found: mpsc::unbounded_channel(),
631            captures: BTreeMap::new(),
632            recaptures: BTreeMap::new(),
633            leases: Leases::default(),
634            mirror_until: MirrorPosition::default(),
635            queued_jobs: BTreeMap::new(),
636            cancelling: BTreeMap::new(),
637            closing: BTreeMap::new(),
638            last_prune: None,
639            measured: admission::Measured::default(),
640            warm: warm::Warm::default(),
641            records,
642            build_cache: build_cache.clone(),
643            default_branches: BTreeMap::new(),
644            ended_uploads: BTreeMap::new(),
645            visibility_due: Arc::default(),
646            visibility_asking: None,
647            exits: Arc::default(),
648            watcher: None,
649            released: BTreeSet::new(),
650            passes_in_tick: Duration::ZERO,
651            listing: 0,
652            named: 0,
653            claims: BTreeMap::new(),
654        };
655        engine.publish();
656        (
657            engine,
658            Handle {
659                status: receiver,
660                control,
661                store,
662                caches: caches_receiver,
663                records: records_receiver,
664                build_cache,
665                disk: disk_receiver,
666            },
667        )
668    }
669
670    /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
671    /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
672    pub async fn run(mut self) {
673        tokio::spawn(metrics::run(
674            Sampler::new(self.paths.clone(), &self.config.host.slice),
675            self.tracked.subscribe(),
676            self.store.clone(),
677        ));
678        let control = self.control.clone();
679        let deadline = async {
680            control.hurried().await;
681            tokio::time::sleep(STOP_DEADLINE).await;
682        };
683        tokio::select! {
684            () = self.serve() => {}
685            () = deadline => warn!(
686                "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
687                STOP_DEADLINE.as_secs()
688            ),
689        }
690    }
691
692    /// Recovers, then schedules until a shutdown request is carried out.
693    async fn serve(&mut self) {
694        // Until Docker answers: scheduling without knowing the containers already running would start a second
695        // runner on a slot's volume and delete the running one's registration.
696        while let Err(error) = self.recover().await {
697            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
698            if self.control.mode() != Shutdown::No {
699                return;
700            }
701            tokio::select! {
702                () = tokio::time::sleep(Duration::from_secs(5)) => {}
703                () = self.control.wake.notified() => {}
704            }
705        }
706        self.schedule().await;
707    }
708
709    /// Schedules until a shutdown request is carried out, from a recovered engine. Between ticks, a runner whose
710    /// container Docker says stopped is released at once, and its slot taken by a job the last poll listed
711    /// (`turnover`, ADR 0095); that leaves the next tick when it was due.
712    async fn schedule(&mut self) {
713        self.watcher = Some(exits::Watcher::start(
714            self.docker.clone(),
715            self.exits.clone(),
716        ));
717        let mut shown = Shutdown::No;
718        let mode = loop {
719            let mode = self.stopping();
720            // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
721            // Docker: until then status would show a daemon that is not stopping.
722            if mode != shown {
723                self.publish();
724                shown = mode;
725            }
726            if mode == Shutdown::Detach {
727                self.remove_every_idle().await;
728                self.collect_recordings().await;
729            } else {
730                self.tick(mode).await;
731            }
732            self.publish();
733            // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
734            // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
735            if self.stopping() != mode {
736                continue;
737            }
738            if self.stopped(mode) {
739                break mode;
740            }
741            let interval = if mode == Shutdown::No {
742                self.config.host.poll_interval
743            } else {
744                Duration::from_secs(5)
745            };
746            self.wait_for_tick(mode, tokio::time::Instant::now() + interval)
747                .await;
748        };
749        // Nothing more is started or released: what Docker says now is the next start's.
750        self.watcher = None;
751        // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
752        self.collect_reseeds().await;
753        if self.docker_in_stop == DockerInStop::Failed {
754            info!("stopped; the next start removes or adopts what Docker did not answer for");
755            return;
756        }
757        self.close_shells().await;
758        if mode == Shutdown::Detach {
759            // The mirror stays up only for the busy runners left behind.
760            let mirror = if self.config.caches.mirror.enabled {
761                " and the mirror"
762            } else {
763                ""
764            };
765            match self.occupied().count() {
766                0 => info!("stopped; no job was running"),
767                1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
768                runners => {
769                    info!("stopped, leaving {runners} busy runners{mirror} to the next start");
770                }
771            }
772            return;
773        }
774        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
775            warn!("cannot stop the mirror: {error}");
776        }
777        info!("stopped");
778    }
779
780    /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
781    /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
782    /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
783    async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) {
784        loop {
785            let exits = self.exits.clone();
786            // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
787            // keep turning over.
788            tokio::select! {
789                biased;
790                () = tokio::time::sleep_until(due) => return,
791                () = self.control.wake.notified() => return,
792                // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
793                Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
794                    self.command(command).await;
795                    self.publish();
796                    return;
797                }
798                // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
799                // it was due. A stop returns to see whether it is done.
800                Some(()) = self.records.finished.recv() => {
801                    self.collect_recordings().await;
802                    self.publish();
803                    if mode != Shutdown::No || self.stopping() != mode {
804                        return;
805                    }
806                }
807                // A capture named its runner's job: status says the runner runs it now, not after the next poll.
808                Some(found) = self.found.1.recv() => {
809                    self.note_found(&found);
810                    self.publish();
811                }
812                () = exits.noted(), if mode != Shutdown::Detach => {
813                    if mode != Shutdown::No {
814                        return;
815                    }
816                    self.turnover().await;
817                    self.publish();
818                    if self.stopping() != mode {
819                        return;
820                    }
821                }
822            }
823        }
824    }
825
826    /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
827    fn stopping(&self) -> Shutdown {
828        match self.control.mode() {
829            Shutdown::No => Shutdown::No,
830            _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
831            mode => mode,
832        }
833    }
834
835    /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
836    /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
837    fn stopped(&self, mode: Shutdown) -> bool {
838        let idle = self.slots.values().flatten().all(Option::is_none);
839        let recorded = self.records.recordings.is_empty();
840        match mode {
841            Shutdown::No => false,
842            Shutdown::Detach => recorded,
843            Shutdown::Graceful | Shutdown::Force => idle && recorded,
844        }
845    }
846
847    /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
848    /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
849    /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
850    /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
851    fn docker_failed(&mut self, error: &docker::Error) {
852        if self.control.mode() != Shutdown::No
853            && self.docker_in_stop == DockerInStop::Answers
854            && docker::is_unreachable(error)
855        {
856            warn!("Docker does not answer during the stop; leaving the rest to the next start");
857            self.docker_in_stop = DockerInStop::Failed;
858            // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
859            // running jobs.
860            self.control.shutdown(Shutdown::Detach);
861        }
862    }
863
864    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
865    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
866    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
867    /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
868    /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
869    /// while, and the jobs captures name are noted as they come.
870    async fn serving<T: Send + 'static>(
871        &mut self,
872        work: impl Future<Output = T> + Send + 'static,
873        meanwhile: Meanwhile,
874    ) -> T {
875        let mut task = tokio::spawn(work);
876        loop {
877            let exits = self.exits.clone();
878            tokio::select! {
879                biased;
880                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
881                    Ok(output) => return output,
882                    // As if it had panicked here.
883                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
884                    // Only while the runtime shuts down, which drops this task too.
885                    Err(Err(_)) => std::future::pending::<()>().await,
886                },
887                Some(command) = self.commands.recv() => {
888                    self.command(command).await;
889                    self.publish();
890                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
891                    // polls its repositories then, not a poll interval later.
892                    self.control.wake.notify_one();
893                }
894                () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
895                    let began = tokio::time::Instant::now();
896                    self.turnover().await;
897                    self.passes_in_tick += began.elapsed();
898                    self.publish();
899                }
900                Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
901                    self.note_found(&found);
902                    self.publish();
903                }
904            }
905        }
906    }
907
908    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
909        self.slots.iter().flat_map(|(profile, slots)| {
910            slots.iter().enumerate().filter_map(|(index, slot)| {
911                slot.as_ref().map(|occupant| {
912                    (
913                        SlotRef {
914                            profile: profile.clone(),
915                            index,
916                        },
917                        occupant,
918                    )
919                })
920            })
921        })
922    }
923
924    /// The runner in `slot`, if one is.
925    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
926        self.slots
927            .get(&slot.profile)
928            .and_then(|slots| slots.get(slot.index))
929            .and_then(Option::as_ref)
930    }
931
932    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
933    /// runs in them.
934    fn drop_closed_slots(&mut self) {
935        for (name, slots) in &mut self.slots {
936            let count = self
937                .config
938                .profiles
939                .get(name)
940                .map_or(0, |profile| profile.slots as usize);
941            while slots.len() > count && slots.last().is_some_and(Option::is_none) {
942                slots.pop();
943            }
944        }
945        let profiles = &self.config.profiles;
946        self.slots
947            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
948        // The volumes of the slots dropped go at the next check of the caches.
949        let slots = &self.slots;
950        self.ran.retain(|slot, _| {
951            slots
952                .get(&slot.profile)
953                .is_some_and(|slots| slot.index < slots.len())
954        });
955        let (slots, reseeds) = (&self.slots, &self.reseeds);
956        self.retired
957            .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
958    }
959}
960
961#[cfg(test)]
962mod tests;