Skip to main content

runnerd_core/scheduler/
engine.rs

1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod clock;
19mod control;
20mod exits;
21mod history;
22mod images;
23mod records;
24mod recover;
25mod runners;
26mod status;
27mod tick;
28mod warm;
29
30use std::collections::{BTreeMap, BTreeSet};
31use std::sync::{Arc, Mutex};
32use std::time::{Duration, SystemTime};
33
34use tokio::sync::{mpsc, watch};
35use tokio::task::JoinHandle;
36use tracing::{info, warn};
37
38use self::clock::Clock;
39pub use self::control::{
40    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
41    Target, same_slice,
42};
43use self::history::{CAPTURE_DRAIN, Closing, Found};
44use self::images::Images;
45pub use self::records::RecordSelection;
46use super::{Hold, SlotRef, Waits};
47use crate::admission::Key;
48use crate::api;
49use crate::buildcache::BuildCache;
50use crate::cache::{self, Names, Trimmed};
51use crate::config::Config;
52use crate::docker::{self, Docker, DockerApi};
53use crate::github::{Github, GithubApi, QueuedJob};
54use crate::isolation::{self, Isolation};
55use crate::observe::metrics::{self, Paths, Sampler, Tracked};
56use crate::observe::mirror::Leases;
57use crate::store::{Announced, MirrorPosition, Store};
58
59/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
60/// identifies the machine, so a changed host name still matches the runners started before the change, and two
61/// machines with the same or overlapping host names never take each other's runners for their own.
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct HostIdentity {
64    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
65    id: String,
66    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
67    host: String,
68}
69
70impl HostIdentity {
71    /// This machine's identity, from `/etc/machine-id` and its host name.
72    pub fn read() -> std::io::Result<HostIdentity> {
73        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
74        let machine_id = std::fs::read_to_string("/etc/machine-id")
75            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
76            .map_err(|error| {
77                std::io::Error::new(
78                    error.kind(),
79                    format!("cannot read /etc/machine-id: {error}"),
80                )
81            })?;
82        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
83            std::io::Error::new(
84                std::io::ErrorKind::InvalidData,
85                "/etc/machine-id does not hold a machine ID",
86            )
87        })
88    }
89
90    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
91    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
92        let machine_id = machine_id.trim();
93        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
94            return None;
95        }
96        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
97        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
98            .bytes()
99            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
100                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
101            });
102        let host: String = hostname
103            .trim()
104            .chars()
105            .map(|c| {
106                if c.is_ascii_alphanumeric() {
107                    c.to_ascii_lowercase()
108                } else {
109                    '-'
110                }
111            })
112            .take(20)
113            .collect();
114        let host = host.trim_matches('-');
115        Some(HostIdentity {
116            id: format!("{:08x}", hash >> 32),
117            host: if host.is_empty() { "host" } else { host }.to_owned(),
118        })
119    }
120
121    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
122    /// machine's names.
123    pub fn prefix(&self) -> String {
124        format!("runnerd-{}-", self.id)
125    }
126
127    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
128    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
129        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
130    }
131}
132
133/// What the API needs from a running engine.
134#[derive(Debug, Clone)]
135pub struct Handle {
136    /// The latest status, replaced after every tick.
137    pub status: watch::Receiver<api::Status>,
138    /// Pause, resume and shutdown requests.
139    pub control: Arc<Control>,
140    /// The job history.
141    pub store: Store,
142    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
143    /// config is.
144    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
145    /// The records of failed jobs, replaced whenever one is made or removed.
146    pub records: watch::Receiver<api::Records>,
147    /// The shared build cache, for its size now.
148    pub build_cache: Option<BuildCache>,
149    /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
150    /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
151    pub disk: watch::Receiver<Option<u64>>,
152}
153
154impl Handle {
155    /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
156    /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
157    pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
158        self.control.shutdown(mode).stopping()
159    }
160}
161
162/// Where the engine runs, besides its config and its GitHub and Docker clients.
163#[derive(Debug)]
164pub struct Setup {
165    /// The names of what runnerd creates in Docker.
166    pub names: Names,
167    /// This machine, which names its runners, so a restarted daemon recognizes them.
168    pub identity: HostIdentity,
169    /// The slice's `MemoryMax`.
170    pub memory_max: Option<u64>,
171    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
172    pub cpus: Option<u64>,
173    /// The job history.
174    pub store: Store,
175    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
176    /// announces only the difference, including what the config no longer has.
177    pub announced: Announced,
178    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
179    pub paused: Vec<String>,
180    /// Where the cgroup and proc file systems are.
181    pub paths: Paths,
182    /// The shared build cache, unless the daemon could not open it.
183    pub build_cache: Option<BuildCache>,
184}
185
186/// Why a runner starts, as its start's log line and its record say (ADR 0092).
187#[derive(Debug, Clone, Copy, PartialEq, Eq)]
188enum Purpose {
189    /// For a queued job.
190    Job,
191    /// As one of its repository's configured warm runners, for jobs not queued yet.
192    Warm,
193    /// As a warm runner past those, for a job its runs are expected to queue still.
194    Burst,
195}
196
197impl Purpose {
198    /// How the history stores it.
199    fn as_str(self) -> &'static str {
200        match self {
201            Purpose::Job => "job",
202            Purpose::Warm => "warm",
203            Purpose::Burst => "burst",
204        }
205    }
206}
207
208/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
209#[derive(Debug, Clone, Copy, PartialEq, Eq)]
210enum Meanwhile {
211    /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
212    Turnover,
213    /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
214    Commands,
215}
216
217/// Whether Docker answers a stop.
218#[derive(Debug, Clone, Copy, PartialEq, Eq)]
219enum DockerInStop {
220    /// It answers, if with errors, or no stop was asked for.
221    Answers,
222    /// It did not answer a call the stop needed: the stop calls it no more.
223    Failed,
224}
225
226/// A slot's runner and its container.
227#[derive(Debug, Clone)]
228struct Occupant {
229    repository: String,
230    runner_name: String,
231    runner_id: u64,
232    container: String,
233    busy: bool,
234    /// Whether the last runners listing was the first to show it busy, or GitHub said so since: the queued jobs
235    /// listing after it may still show the job it took, as GitHub lists one queued for a few seconds after.
236    newly_busy: bool,
237    started_at: SystemTime,
238    /// Its container's memory cap, which a reload does not change.
239    memory: u64,
240    /// Its container's CPU limit in millicores, which a reload does not change.
241    cpus: u64,
242    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
243    /// one whose record says nothing of it, which reserves its cap.
244    admission: Option<api::Admission>,
245    /// The job it took, once its log capture found it.
246    job: Option<u64>,
247    /// When this daemon's capture found that job; `None` before, and for a job found again after a restart only by
248    /// an adopted runner, whose `purpose` says so.
249    job_started: Option<SystemTime>,
250    /// That job as admission tells jobs apart, once the history had its record.
251    key: Option<Key>,
252    /// The labels it registered with; `None` for one adopted from a container that does not say.
253    labels: Option<Vec<String>>,
254    /// The image it runs; `None` for one adopted from a container that does not say.
255    image: Option<String>,
256    /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
257    /// label.
258    spec: Option<String>,
259    /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
260    /// container, which each tick tries again.
261    observed: bool,
262    /// Why this daemon started it; `None` for one a restart adopted.
263    purpose: Option<Purpose>,
264}
265
266impl Occupant {
267    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
268    /// the profile's. An unknown label set counts as the same.
269    fn outdated(
270        &self,
271        repository: &crate::config::Repository,
272        profile: &crate::config::Profile,
273    ) -> bool {
274        let differ = self.labels.as_ref().is_some_and(|labels| {
275            let set = |labels: &[String]| -> BTreeSet<String> {
276                labels
277                    .iter()
278                    .map(|label| label.to_ascii_lowercase())
279                    .collect()
280            };
281            set(labels) != set(&repository.labels)
282        });
283        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
284    }
285
286    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
287    fn other_image(&self, profile: &crate::config::Profile) -> bool {
288        self.image
289            .as_ref()
290            .is_some_and(|image| *image != profile.image.reference())
291    }
292
293    /// Whether another release started it, with another container spec: one that does not say is older.
294    fn other_spec(&self) -> bool {
295        self.spec.as_deref() != Some(runners::THIS_SPEC)
296    }
297
298    /// What it reserves of the budget under measured admission: its cap if it has no admission.
299    fn reserved(&self) -> u64 {
300        self.admission
301            .map_or(self.memory, |admission| admission.reserved)
302    }
303
304    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
305    fn reserved_cpu(&self) -> u64 {
306        self.admission
307            .and_then(|admission| admission.cpu)
308            .map_or(self.cpus, |cpu| cpu.reserved)
309    }
310
311    fn running_job(&self) -> api::RunningJob {
312        api::RunningJob {
313            repository: self.repository.clone(),
314            runner: self.runner_name.clone(),
315            job: self.job,
316        }
317    }
318}
319
320/// A runner gone that may have taken one of its repository's listed jobs, which no capture named.
321#[derive(Debug)]
322struct Unnamed {
323    /// The profile of its pool.
324    profile: String,
325    /// The `Engine::listing` begun last when it went.
326    listing: u64,
327}
328
329/// What a repository's listings may show queued that this machine's runners took (ADR 0018, 0095). It outlives the
330/// repository's removal from the config: a listing begun before the removal may answer after it is added back.
331#[derive(Debug, Default)]
332struct Claims {
333    /// Its runners gone that may have taken a listed job without the engine knowing which: one ended before its
334    /// log capture named its job, or one whose start failed may have run. Each counts against one listed job of its
335    /// pool until a listing begun after it went answers.
336    unnamed: Vec<Unnamed>,
337    /// The jobs this machine's runners took whose runners are gone, by the `listing` begun last when they went:
338    /// GitHub lists a job queued for a while after a runner took it, so a listing begun before that, or one GitHub
339    /// lags on, still shows it. A listing begun after that and showing it no longer drops it.
340    consumed: BTreeMap<u64, u64>,
341}
342
343/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
344#[derive(Debug, Default)]
345struct RepositoryState {
346    queued: Vec<QueuedJob>,
347    /// Whether the last poll of its queued jobs succeeded.
348    listed: bool,
349    /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
350    /// not hold back the others (ADR 0058).
351    holds: BTreeMap<String, PoolHold>,
352    /// By the profile of each of its pools, what that pool's next job has waited for and since when (ADR 0096): from
353    /// the first plan of a tick or a pass that held it, not one a status runs, until a runner of the pool is in its
354    /// slot, the pool may no longer start one, or a listing that succeeded no longer shows the job. A plan that holds
355    /// it for a slot or for the other resource does not make it younger, nor does a listing that failed. With them,
356    /// whether a barrier has served the pool (ADR 0098), until a listing that succeeded shows its queue empty or the
357    /// pool may no longer start a runner. Not kept over a restart.
358    waits: BTreeMap<String, Waits>,
359    /// The profiles of its pools whose next job the last plan held for CPU.
360    cpu_held: BTreeSet<String>,
361}
362
363/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
364#[derive(Debug, Default)]
365struct PoolHold {
366    held: Option<Hold>,
367    /// Runners in a row that ended without taking a job.
368    failures: u32,
369    /// Runners in a row that connected and ended without a job, or waited `IDLE_TIMEOUT` for one (ADR 0104). Those
370    /// `left_beside` holds count with the one before them.
371    idle_exits: u32,
372    /// The pool's other runners when the last of those ended: already up, they say nothing of a later start.
373    left_beside: BTreeSet<String>,
374}
375
376/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
377/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
378/// a single Docker request (120s) could otherwise outlast.
379pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
380
381/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
382#[derive(Debug)]
383pub struct Engine<G = Github, D = Docker> {
384    config: Config,
385    /// Shared with the listings a tick reads while it answers commands.
386    github: Arc<G>,
387    docker: Arc<D>,
388    names: Names,
389    isolation: &'static dyn Isolation,
390    identity: HostIdentity,
391    runner_prefix: String,
392    /// What a sample's or a host reading's age goes by.
393    clock: Clock,
394    memory_max: Option<u64>,
395    /// The CPUs the slice may use, in millicores, read at start.
396    cpus: Option<u64>,
397    slots: BTreeMap<String, Vec<Option<Occupant>>>,
398    repositories: BTreeMap<String, RepositoryState>,
399    reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
400    /// The pulled images of the config's profiles, and the removal of those no profile runs.
401    images: Images,
402    /// The memory cap each running reseed's container started with, which a reload does not change.
403    reseed_memory: BTreeMap<String, u64>,
404    /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
405    reseed_requested: BTreeSet<String>,
406    /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
407    draining: BTreeSet<String>,
408    /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
409    /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
410    drained_once: BTreeSet<String>,
411    /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
412    /// each tick tries again.
413    leftover: BTreeMap<String, String>,
414    /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
415    /// failed jobs' records: the first tick after each one's backoff tries again.
416    unread: Vec<recover::Left>,
417    /// Profiles a reload removed, kept while their runners or reseed still run.
418    retired: BTreeMap<String, crate::config::Profile>,
419    /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
420    measure_due: BTreeSet<String>,
421    /// The measures of slot 1 running, by profile.
422    measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
423    /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
424    measured_at: BTreeMap<String, tokio::time::Instant>,
425    /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
426    /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
427    /// `snapshot_max`.
428    reseed_due: BTreeSet<String>,
429    /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
430    /// little slot 1 changed.
431    oversized: BTreeSet<String>,
432    /// When each profile's last reseed finished: a check of the caches that started before then measured the old
433    /// snapshot.
434    reseeded_at: BTreeMap<String, SystemTime>,
435    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
436    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
437    ran: BTreeMap<SlotRef, Key>,
438    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
439    /// `None` until one measured Docker's disk, when each image keeps its own default.
440    docker_disk: watch::Sender<Option<u64>>,
441    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
442    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
443    /// Whether a job ended since the shared caches were last checked.
444    trim_due: bool,
445    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
446    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
447    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
448    /// unless runners were adopted.
449    checked_at: SystemTime,
450    mirror_up: bool,
451    /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
452    docker_in_stop: DockerInStop,
453    /// The memory cap the running mirror started with.
454    mirror_memory: u64,
455    rate_limit_logged: bool,
456    /// Runners without a container that GitHub still holds busy, already logged.
457    busy_orphans: BTreeSet<u64>,
458    /// Runners this engine released or killed, by ID, with their repository, until a listing of that repository no
459    /// longer has them: GitHub holding one busy then is no orphan to warn of.
460    ended_busy: BTreeMap<u64, String>,
461    status: watch::Sender<api::Status>,
462    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
463    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
464    announced_profiles: Option<Vec<api::ProfileStatus>>,
465    control: Arc<Control>,
466    commands: Commands,
467    caches: watch::Sender<Vec<api::CacheVolume>>,
468    store: Store,
469    paths: Paths,
470    /// The containers the sampler reads.
471    tracked: watch::Sender<Vec<Tracked>>,
472    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
473    captures: BTreeMap<String, JoinHandle<history::Capture>>,
474    /// The runners whose log capture broke off, until they are gone.
475    recaptures: BTreeMap<String, history::Recapture>,
476    /// Who held which address, for the mirror's log.
477    leases: Leases,
478    /// How far the mirror's log was counted.
479    mirror_until: MirrorPosition,
480    /// Jobs recorded queued and not yet taken or closed, with their repository.
481    queued_jobs: BTreeMap<u64, String>,
482    /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
483    cancelling: BTreeMap<String, tokio::time::Instant>,
484    /// Jobs whose runner is gone, waiting for GitHub to report them completed.
485    closing: BTreeMap<u64, Closing>,
486    last_prune: Option<SystemTime>,
487    /// The records of failed jobs and the shells open in them.
488    records: records::RecordState,
489    /// The predictor, the high-water mark of measured admission, and what admission saw of the host.
490    measured: admission::Measured,
491    /// What warm runners learned from the history, and the workflow runs seen lately.
492    warm: warm::Warm,
493    /// The shared build cache.
494    build_cache: Option<BuildCache>,
495    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
496    /// cache when `[caches.build] branches` names none.
497    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
498    /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
499    /// `build_cache::ENDED_WAIT`, by runner.
500    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
501    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
502    /// the answers.
503    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
504    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
505    visibility_asking: Option<JoinHandle<()>>,
506    /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
507    exits: Arc<exits::Exits>,
508    /// The task following Docker's events into `exits`, while the engine schedules.
509    watcher: Option<exits::Watcher>,
510    /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
511    /// pass the event wakes while the tick waits for GitHub starts runners into their room.
512    released: BTreeSet<String>,
513    /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
514    /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
515    passes_in_tick: Duration,
516    /// How many times the engine began listing GitHub's queued jobs: the listing each answer comes from, and how
517    /// old a claim on a listed job is (`Claims`).
518    listing: u64,
519    /// By repository, the jobs its listings may show that this machine's runners took.
520    claims: BTreeMap<String, Claims>,
521    /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
522    /// a name, which GitHub, Docker and the history know them by. A restart takes it from the containers and the
523    /// history, so a name ahead of the clock, or a clock that stepped back, is not used again.
524    named: u64,
525}
526
527fn unix_secs(time: SystemTime) -> u64 {
528    time.duration_since(SystemTime::UNIX_EPOCH)
529        .unwrap_or_default()
530        .as_secs()
531}
532
533impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
534    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
535    /// them; `memory_max` is the slice's.
536    #[allow(
537        clippy::too_many_lines,
538        reason = "one line per field of the engine's state, which split up would only be listed twice"
539    )]
540    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
541        let Setup {
542            names,
543            identity,
544            memory_max,
545            cpus,
546            store,
547            announced,
548            paused,
549            paths,
550            build_cache,
551        } = setup;
552        let slots = config
553            .profiles
554            .iter()
555            .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
556            .collect();
557        let repositories = config
558            .repositories
559            .iter()
560            .map(|repository| (repository.name.clone(), RepositoryState::default()))
561            .collect();
562        let (control, commands) = Control::new(
563            config
564                .repositories
565                .iter()
566                .map(|repository| repository.name.clone()),
567            paused,
568            store.clone(),
569        );
570        let control = Arc::new(control);
571        let (status, receiver) = watch::channel(api::Status {
572            version: env!("CARGO_PKG_VERSION").to_owned(),
573            shutting_down: false,
574            stopping: None,
575            budget: api::Budget {
576                slice: config.host.slice.clone(),
577                memory_max,
578                ..api::Budget::default()
579            },
580            profiles: Vec::new(),
581            // What clients resuming after the last run's events have, so the first publish announces the difference.
582            repositories: announced.repositories,
583            queued: Vec::new(),
584        });
585        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
586        let (records, records_receiver) = records::RecordState::new();
587        let (docker_disk, disk_receiver) = watch::channel(None);
588        let mut engine = Engine {
589            clock: Clock::default(),
590            isolation: isolation::backend(config.host.isolation),
591            // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
592            mirror_memory: config.mirror_memory(),
593            config,
594            github: Arc::new(github),
595            docker: Arc::new(docker),
596            names,
597            runner_prefix: identity.prefix(),
598            identity,
599            memory_max,
600            cpus,
601            slots,
602            repositories,
603            reseeds: BTreeMap::new(),
604            images: Images::new(),
605            reseed_memory: BTreeMap::new(),
606            reseed_requested: BTreeSet::new(),
607            retired: BTreeMap::new(),
608            draining: BTreeSet::new(),
609            drained_once: BTreeSet::new(),
610            leftover: BTreeMap::new(),
611            unread: Vec::new(),
612            measure_due: BTreeSet::new(),
613            measures: BTreeMap::new(),
614            measured_at: BTreeMap::new(),
615            reseed_due: BTreeSet::new(),
616            oversized: BTreeSet::new(),
617            reseeded_at: BTreeMap::new(),
618            ran: BTreeMap::new(),
619            docker_disk,
620            disk: None,
621            // So the caches are measured once at start.
622            trim_due: true,
623            trim: None,
624            checked_at: SystemTime::now(),
625            // Unknown: an earlier run may have left it up, so the first idle tick stops it.
626            mirror_up: true,
627            docker_in_stop: DockerInStop::Answers,
628            rate_limit_logged: false,
629            busy_orphans: BTreeSet::new(),
630            ended_busy: BTreeMap::new(),
631            status,
632            announced_profiles: Some(announced.profiles),
633            control: control.clone(),
634            commands,
635            caches,
636            store: store.clone(),
637            paths,
638            tracked: watch::channel(Vec::new()).0,
639            found: mpsc::unbounded_channel(),
640            captures: BTreeMap::new(),
641            recaptures: BTreeMap::new(),
642            leases: Leases::default(),
643            mirror_until: MirrorPosition::default(),
644            queued_jobs: BTreeMap::new(),
645            cancelling: BTreeMap::new(),
646            closing: BTreeMap::new(),
647            last_prune: None,
648            measured: admission::Measured::default(),
649            warm: warm::Warm::default(),
650            records,
651            build_cache: build_cache.clone(),
652            default_branches: BTreeMap::new(),
653            ended_uploads: BTreeMap::new(),
654            visibility_due: Arc::default(),
655            visibility_asking: None,
656            exits: Arc::default(),
657            watcher: None,
658            released: BTreeSet::new(),
659            passes_in_tick: Duration::ZERO,
660            listing: 0,
661            named: 0,
662            claims: BTreeMap::new(),
663        };
664        engine.publish();
665        (
666            engine,
667            Handle {
668                status: receiver,
669                control,
670                store,
671                caches: caches_receiver,
672                records: records_receiver,
673                build_cache,
674                disk: disk_receiver,
675            },
676        )
677    }
678
679    /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
680    /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
681    pub async fn run(mut self) {
682        tokio::spawn(metrics::run(
683            Sampler::new(self.paths.clone(), &self.config.host.slice),
684            self.tracked.subscribe(),
685            self.store.clone(),
686        ));
687        let control = self.control.clone();
688        let deadline = async {
689            control.hurried().await;
690            tokio::time::sleep(STOP_DEADLINE).await;
691        };
692        tokio::select! {
693            () = self.serve() => {}
694            () = deadline => warn!(
695                "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
696                STOP_DEADLINE.as_secs()
697            ),
698        }
699    }
700
701    /// Recovers, then schedules until a shutdown request is carried out.
702    async fn serve(&mut self) {
703        // Until Docker answers: scheduling without knowing the containers already running would start a second
704        // runner on a slot's volume and delete the running one's registration.
705        while let Err(error) = self.recover().await {
706            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
707            if self.control.mode() != Shutdown::No {
708                return;
709            }
710            tokio::select! {
711                () = tokio::time::sleep(Duration::from_secs(5)) => {}
712                () = self.control.wake.notified() => {}
713            }
714        }
715        self.schedule().await;
716    }
717
718    /// Schedules until a shutdown request is carried out, from a recovered engine. Between ticks, a runner whose
719    /// container Docker says stopped is released at once, and its slot taken by a job the last poll listed
720    /// (`turnover`, ADR 0095); that leaves the next tick when it was due.
721    async fn schedule(&mut self) {
722        self.watcher = Some(exits::Watcher::start(
723            self.docker.clone(),
724            self.exits.clone(),
725        ));
726        let mut shown = Shutdown::No;
727        let mode = loop {
728            let mode = self.stopping();
729            // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
730            // Docker: until then status would show a daemon that is not stopping.
731            if mode != shown {
732                self.publish();
733                shown = mode;
734            }
735            if mode == Shutdown::Detach {
736                self.remove_every_idle().await;
737                self.collect_recordings().await;
738            } else {
739                self.tick(mode).await;
740            }
741            self.publish();
742            // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
743            // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
744            if self.stopping() != mode {
745                continue;
746            }
747            if self.stopped(mode) {
748                break mode;
749            }
750            let interval = if mode == Shutdown::No {
751                self.config.host.poll_interval
752            } else {
753                Duration::from_secs(5)
754            };
755            self.wait_for_tick(mode, tokio::time::Instant::now() + interval)
756                .await;
757        };
758        // Nothing more is started or released: what Docker says now is the next start's.
759        self.watcher = None;
760        // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
761        self.collect_reseeds().await;
762        if self.docker_in_stop == DockerInStop::Failed {
763            info!("stopped; the next start removes or adopts what Docker did not answer for");
764            return;
765        }
766        self.close_shells().await;
767        if mode == Shutdown::Detach {
768            // The mirror stays up only for the busy runners left behind.
769            let mirror = if self.config.caches.mirror.enabled {
770                " and the mirror"
771            } else {
772                ""
773            };
774            match self.occupied().count() {
775                0 => info!("stopped; no job was running"),
776                1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
777                runners => {
778                    info!("stopped, leaving {runners} busy runners{mirror} to the next start");
779                }
780            }
781            return;
782        }
783        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
784            warn!("cannot stop the mirror: {error}");
785        }
786        info!("stopped");
787    }
788
789    /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
790    /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
791    /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
792    async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) {
793        loop {
794            let exits = self.exits.clone();
795            // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
796            // keep turning over.
797            tokio::select! {
798                biased;
799                () = tokio::time::sleep_until(due) => return,
800                () = self.control.wake.notified() => return,
801                // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
802                Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
803                    self.command(command).await;
804                    self.publish();
805                    return;
806                }
807                // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
808                // it was due. A stop returns to see whether it is done.
809                Some(()) = self.records.finished.recv() => {
810                    self.collect_recordings().await;
811                    self.publish();
812                    if mode != Shutdown::No || self.stopping() != mode {
813                        return;
814                    }
815                }
816                // A capture named its runner's job: status says the runner runs it now, not after the next poll.
817                Some(found) = self.found.1.recv() => {
818                    self.note_found(&found);
819                    self.publish();
820                }
821                () = exits.noted(), if mode != Shutdown::Detach => {
822                    if mode != Shutdown::No {
823                        return;
824                    }
825                    self.turnover().await;
826                    self.publish();
827                    if self.stopping() != mode {
828                        return;
829                    }
830                }
831            }
832        }
833    }
834
835    /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
836    fn stopping(&self) -> Shutdown {
837        match self.control.mode() {
838            Shutdown::No => Shutdown::No,
839            _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
840            mode => mode,
841        }
842    }
843
844    /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
845    /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
846    fn stopped(&self, mode: Shutdown) -> bool {
847        let idle = self.slots.values().flatten().all(Option::is_none);
848        let recorded = self.records.recordings.is_empty();
849        match mode {
850            Shutdown::No => false,
851            Shutdown::Detach => recorded,
852            Shutdown::Graceful | Shutdown::Force => idle && recorded,
853        }
854    }
855
856    /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
857    /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
858    /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
859    /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
860    fn docker_failed(&mut self, error: &docker::Error) {
861        if self.control.mode() != Shutdown::No
862            && self.docker_in_stop == DockerInStop::Answers
863            && docker::is_unreachable(error)
864        {
865            warn!("Docker does not answer during the stop; leaving the rest to the next start");
866            self.docker_in_stop = DockerInStop::Failed;
867            // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
868            // running jobs.
869            self.control.shutdown(Shutdown::Detach);
870        }
871    }
872
873    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
874    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
875    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
876    /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
877    /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
878    /// while, and the jobs captures name are noted as they come.
879    async fn serving<T: Send + 'static>(
880        &mut self,
881        work: impl Future<Output = T> + Send + 'static,
882        meanwhile: Meanwhile,
883    ) -> T {
884        let mut task = tokio::spawn(work);
885        loop {
886            let exits = self.exits.clone();
887            tokio::select! {
888                biased;
889                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
890                    Ok(output) => return output,
891                    // As if it had panicked here.
892                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
893                    // Only while the runtime shuts down, which drops this task too.
894                    Err(Err(_)) => std::future::pending::<()>().await,
895                },
896                Some(command) = self.commands.recv() => {
897                    self.command(command).await;
898                    self.publish();
899                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
900                    // polls its repositories then, not a poll interval later.
901                    self.control.wake.notify_one();
902                }
903                () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
904                    let began = tokio::time::Instant::now();
905                    self.turnover().await;
906                    self.passes_in_tick += began.elapsed();
907                    self.publish();
908                }
909                Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
910                    self.note_found(&found);
911                    self.publish();
912                }
913            }
914        }
915    }
916
917    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
918        self.slots.iter().flat_map(|(profile, slots)| {
919            slots.iter().enumerate().filter_map(|(index, slot)| {
920                slot.as_ref().map(|occupant| {
921                    (
922                        SlotRef {
923                            profile: profile.clone(),
924                            index,
925                        },
926                        occupant,
927                    )
928                })
929            })
930        })
931    }
932
933    /// The runner in `slot`, if one is.
934    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
935        self.slots
936            .get(&slot.profile)
937            .and_then(|slots| slots.get(slot.index))
938            .and_then(Option::as_ref)
939    }
940
941    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
942    /// runs in them.
943    fn drop_closed_slots(&mut self) {
944        for (name, slots) in &mut self.slots {
945            let count = self
946                .config
947                .profiles
948                .get(name)
949                .map_or(0, |profile| profile.slots as usize);
950            while slots.len() > count && slots.last().is_some_and(Option::is_none) {
951                slots.pop();
952            }
953        }
954        let profiles = &self.config.profiles;
955        self.slots
956            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
957        // The volumes of the slots dropped go at the next check of the caches.
958        let slots = &self.slots;
959        self.ran.retain(|slot, _| {
960            slots
961                .get(&slot.profile)
962                .is_some_and(|slots| slot.index < slots.len())
963        });
964        let (slots, reseeds) = (&self.slots, &self.reseeds);
965        self.retired
966            .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
967    }
968}
969
970#[cfg(test)]
971mod tests;