Skip to main content

runnerd_core/scheduler/
engine.rs

1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod history;
20mod images;
21mod records;
22mod recover;
23mod runners;
24mod status;
25mod tick;
26mod warm;
27
28use std::collections::{BTreeMap, BTreeSet};
29use std::sync::{Arc, Mutex};
30use std::time::{Duration, SystemTime};
31
32use tokio::sync::{mpsc, watch};
33use tokio::task::JoinHandle;
34use tracing::{info, warn};
35
36pub use self::control::{
37    Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
38    Target, same_slice,
39};
40use self::history::{Closing, Found};
41use self::images::Images;
42pub use self::records::RecordSelection;
43use super::{Hold, SlotRef};
44use crate::admission::Key;
45use crate::api;
46use crate::buildcache::BuildCache;
47use crate::cache::{self, Names, Trimmed};
48use crate::config::Config;
49use crate::docker::{self, Docker, DockerApi};
50use crate::github::{Github, GithubApi, QueuedJob};
51use crate::isolation::{self, Isolation};
52use crate::observe::metrics::{self, Paths, Sampler, Tracked};
53use crate::observe::mirror::Leases;
54use crate::store::{Announced, MirrorPosition, Store};
55
56/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
57/// identifies the machine, so a changed host name still matches the runners started before the change, and two
58/// machines with the same or overlapping host names never take each other's runners for their own.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct HostIdentity {
61    /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
62    id: String,
63    /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
64    host: String,
65}
66
67impl HostIdentity {
68    /// This machine's identity, from `/etc/machine-id` and its host name.
69    pub fn read() -> std::io::Result<HostIdentity> {
70        let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
71        let machine_id = std::fs::read_to_string("/etc/machine-id")
72            .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
73            .map_err(|error| {
74                std::io::Error::new(
75                    error.kind(),
76                    format!("cannot read /etc/machine-id: {error}"),
77                )
78            })?;
79        HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
80            std::io::Error::new(
81                std::io::ErrorKind::InvalidData,
82                "/etc/machine-id does not hold a machine ID",
83            )
84        })
85    }
86
87    /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
88    pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
89        let machine_id = machine_id.trim();
90        if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
91            return None;
92        }
93        // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
94        let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
95            .bytes()
96            .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
97                (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
98            });
99        let host: String = hostname
100            .trim()
101            .chars()
102            .map(|c| {
103                if c.is_ascii_alphanumeric() {
104                    c.to_ascii_lowercase()
105                } else {
106                    '-'
107                }
108            })
109            .take(20)
110            .collect();
111        let host = host.trim_matches('-');
112        Some(HostIdentity {
113            id: format!("{:08x}", hash >> 32),
114            host: if host.is_empty() { "host" } else { host }.to_owned(),
115        })
116    }
117
118    /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
119    /// machine's names.
120    pub fn prefix(&self) -> String {
121        format!("runnerd-{}-", self.id)
122    }
123
124    /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
125    pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
126        format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
127    }
128}
129
130/// What the API needs from a running engine.
131#[derive(Debug, Clone)]
132pub struct Handle {
133    /// The latest status, replaced after every tick.
134    pub status: watch::Receiver<api::Status>,
135    /// Pause, resume and shutdown requests.
136    pub control: Arc<Control>,
137    /// The job history.
138    pub store: Store,
139    /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
140    /// config is.
141    pub caches: watch::Receiver<Vec<api::CacheVolume>>,
142    /// The records of failed jobs, replaced whenever one is made or removed.
143    pub records: watch::Receiver<api::Records>,
144    /// The shared build cache, for its size now.
145    pub build_cache: Option<BuildCache>,
146}
147
148/// Where the engine runs, besides its config and its GitHub and Docker clients.
149#[derive(Debug)]
150pub struct Setup {
151    /// The names of what runnerd creates in Docker.
152    pub names: Names,
153    /// This machine, which names its runners, so a restarted daemon recognizes them.
154    pub identity: HostIdentity,
155    /// The slice's `MemoryMax`.
156    pub memory_max: Option<u64>,
157    /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
158    pub cpus: Option<u64>,
159    /// The job history.
160    pub store: Store,
161    /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
162    /// announces only the difference, including what the config no longer has.
163    pub announced: Announced,
164    /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
165    pub paused: Vec<String>,
166    /// Where the cgroup and proc file systems are.
167    pub paths: Paths,
168    /// The shared build cache, unless the daemon could not open it.
169    pub build_cache: Option<BuildCache>,
170}
171
172/// Why a runner starts.
173#[derive(Debug, Clone, Copy, PartialEq, Eq)]
174enum Purpose {
175    /// For a queued job.
176    Job,
177    /// As a warm runner, for jobs not queued yet.
178    Warm,
179}
180
181/// A slot's runner and its container.
182#[derive(Debug, Clone)]
183struct Occupant {
184    repository: String,
185    runner_name: String,
186    runner_id: u64,
187    container: String,
188    busy: bool,
189    started_at: SystemTime,
190    /// Its container's memory cap, which a reload does not change.
191    memory: u64,
192    /// Its container's CPU limit in millicores, which a reload does not change.
193    cpus: u64,
194    /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
195    /// one whose record says nothing of it, which reserves its cap.
196    admission: Option<api::Admission>,
197    /// The job it took, once its log capture found it.
198    job: Option<u64>,
199    /// That job as admission tells jobs apart, once the history had its record.
200    key: Option<Key>,
201    /// The labels it registered with; `None` for one adopted from a container that does not say.
202    labels: Option<Vec<String>>,
203    /// The image it runs; `None` for one adopted from a container that does not say.
204    image: Option<String>,
205    /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
206    /// container, which each tick tries again.
207    observed: bool,
208}
209
210impl Occupant {
211    /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
212    /// the profile's. An unknown label set counts as the same.
213    fn outdated(
214        &self,
215        repository: &crate::config::Repository,
216        profile: &crate::config::Profile,
217    ) -> bool {
218        let differ = self.labels.as_ref().is_some_and(|labels| {
219            let set = |labels: &[String]| -> BTreeSet<String> {
220                labels
221                    .iter()
222                    .map(|label| label.to_ascii_lowercase())
223                    .collect()
224            };
225            set(labels) != set(&repository.labels)
226        });
227        differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
228    }
229
230    /// Whether it runs another image than `profile` names. An unknown image counts as the same.
231    fn other_image(&self, profile: &crate::config::Profile) -> bool {
232        self.image
233            .as_ref()
234            .is_some_and(|image| *image != profile.image.reference())
235    }
236
237    /// What it reserves of the budget under measured admission: its cap if it has no admission.
238    fn reserved(&self) -> u64 {
239        self.admission
240            .map_or(self.memory, |admission| admission.reserved)
241    }
242
243    /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
244    fn reserved_cpu(&self) -> u64 {
245        self.admission
246            .and_then(|admission| admission.cpu)
247            .map_or(self.cpus, |cpu| cpu.reserved)
248    }
249
250    fn running_job(&self) -> api::RunningJob {
251        api::RunningJob {
252            repository: self.repository.clone(),
253            runner: self.runner_name.clone(),
254            job: self.job,
255        }
256    }
257}
258
259/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
260#[derive(Debug, Default)]
261struct RepositoryState {
262    queued: Vec<QueuedJob>,
263    /// Whether the last poll of its queued jobs succeeded.
264    listed: bool,
265    /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
266    /// not hold back the others (ADR 0058).
267    holds: BTreeMap<String, PoolHold>,
268    /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
269    /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
270    /// make it younger.
271    cpu_wait: BTreeMap<String, CpuWait>,
272    /// The profiles of its pools whose next job the last plan held for CPU.
273    cpu_held: BTreeSet<String>,
274}
275
276/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
277#[derive(Debug, Default)]
278struct PoolHold {
279    held: Option<Hold>,
280    /// Runners in a row that ended without taking a job.
281    failures: u32,
282    /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
283    idle_exits: u32,
284}
285
286/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
287#[derive(Debug, Clone, Copy)]
288struct CpuWait {
289    /// The job's ID.
290    job: u64,
291    /// When a plan first held it for CPU.
292    since: SystemTime,
293}
294
295/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
296#[derive(Debug)]
297pub struct Engine<G = Github, D = Docker> {
298    config: Config,
299    /// Shared with the listings a tick reads while it answers commands.
300    github: Arc<G>,
301    docker: Arc<D>,
302    names: Names,
303    isolation: &'static dyn Isolation,
304    identity: HostIdentity,
305    runner_prefix: String,
306    memory_max: Option<u64>,
307    /// The CPUs the slice may use, in millicores, read at start.
308    cpus: Option<u64>,
309    slots: BTreeMap<String, Vec<Option<Occupant>>>,
310    repositories: BTreeMap<String, RepositoryState>,
311    reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
312    /// The pulled images of the config's profiles, and the removal of those no profile runs.
313    images: Images,
314    /// The memory cap each running reseed's container started with, which a reload does not change.
315    reseed_memory: BTreeMap<String, u64>,
316    /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
317    reseed_requested: BTreeSet<String>,
318    /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
319    draining: BTreeSet<String>,
320    /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
321    /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
322    drained_once: BTreeSet<String>,
323    /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
324    /// each tick tries again.
325    leftover: BTreeMap<String, String>,
326    /// Profiles a reload removed, kept while their runners or reseed still run.
327    retired: BTreeMap<String, crate::config::Profile>,
328    /// Profiles whose slot 1 ran a job since their last reseed check.
329    reseed_due: BTreeSet<String>,
330    /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
331    /// little slot 1 changed.
332    oversized: BTreeSet<String>,
333    /// When each profile's last reseed finished: a check of the caches that started before then measured the old
334    /// snapshot.
335    reseeded_at: BTreeMap<String, SystemTime>,
336    /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
337    /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
338    ran: BTreeMap<SlotRef, Key>,
339    /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
340    /// `None` until one measured Docker's disk, when each image keeps its own default.
341    docker_disk: Option<u64>,
342    /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
343    disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
344    /// Whether a job ended since the shared caches were last checked.
345    trim_due: bool,
346    /// The running check of the shared caches: Docker takes many seconds to measure volumes.
347    trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
348    /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
349    /// unless runners were adopted.
350    checked_at: SystemTime,
351    mirror_up: bool,
352    /// The memory cap the running mirror started with.
353    mirror_memory: u64,
354    rate_limit_logged: bool,
355    /// Runners without a container that GitHub still holds busy, already logged.
356    busy_orphans: BTreeSet<u64>,
357    status: watch::Sender<api::Status>,
358    /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
359    /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
360    announced_profiles: Option<Vec<api::ProfileStatus>>,
361    control: Arc<Control>,
362    commands: Commands,
363    caches: watch::Sender<Vec<api::CacheVolume>>,
364    store: Store,
365    paths: Paths,
366    /// The containers the sampler reads.
367    tracked: watch::Sender<Vec<Tracked>>,
368    found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
369    captures: BTreeMap<String, JoinHandle<history::Capture>>,
370    /// The runners whose log capture broke off, until they are gone.
371    recaptures: BTreeMap<String, history::Recapture>,
372    /// Who held which address, for the mirror's log.
373    leases: Leases,
374    /// How far the mirror's log was counted.
375    mirror_until: MirrorPosition,
376    /// Jobs recorded queued and not yet taken or closed, with their repository.
377    queued_jobs: BTreeMap<u64, String>,
378    /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
379    cancelling: BTreeMap<String, tokio::time::Instant>,
380    /// Jobs whose runner is gone, waiting for GitHub to report them completed.
381    closing: BTreeMap<u64, Closing>,
382    last_prune: Option<SystemTime>,
383    /// The records of failed jobs and the shells open in them.
384    records: records::RecordState,
385    /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
386    measured: admission::Measured,
387    /// What warm runners learned from the history, and the workflow runs seen lately.
388    warm: warm::Warm,
389    /// The shared build cache.
390    build_cache: Option<BuildCache>,
391    /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
392    /// cache when `[caches.build] branches` names none.
393    default_branches: BTreeMap<String, build_cache::DefaultBranch>,
394    /// Removed runners' build cache uploads held until `follow_jobs` asks GitHub of their jobs, by runner.
395    ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
396    /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
397    /// the answers.
398    visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
399    /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
400    visibility_asking: Option<JoinHandle<()>>,
401}
402
403fn unix_secs(time: SystemTime) -> u64 {
404    time.duration_since(SystemTime::UNIX_EPOCH)
405        .unwrap_or_default()
406        .as_secs()
407}
408
409impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
410    /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
411    /// them; `memory_max` is the slice's.
412    #[allow(
413        clippy::too_many_lines,
414        reason = "one line per field of the engine's state, which split up would only be listed twice"
415    )]
416    pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
417        let Setup {
418            names,
419            identity,
420            memory_max,
421            cpus,
422            store,
423            announced,
424            paused,
425            paths,
426            build_cache,
427        } = setup;
428        let slots = config
429            .profiles
430            .iter()
431            .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
432            .collect();
433        let repositories = config
434            .repositories
435            .iter()
436            .map(|repository| (repository.name.clone(), RepositoryState::default()))
437            .collect();
438        let (control, commands) = Control::new(
439            config
440                .repositories
441                .iter()
442                .map(|repository| repository.name.clone()),
443            paused,
444            store.clone(),
445        );
446        let control = Arc::new(control);
447        let (status, receiver) = watch::channel(api::Status {
448            version: env!("CARGO_PKG_VERSION").to_owned(),
449            shutting_down: false,
450            budget: api::Budget {
451                slice: config.host.slice.clone(),
452                memory_max,
453                ..api::Budget::default()
454            },
455            profiles: Vec::new(),
456            // What clients resuming after the last run's events have, so the first publish announces the difference.
457            repositories: announced.repositories,
458            queued: Vec::new(),
459        });
460        let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
461        let (records, records_receiver) = records::RecordState::new();
462        let mut engine = Engine {
463            isolation: isolation::backend(config.host.isolation),
464            // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
465            mirror_memory: config.mirror_memory(),
466            config,
467            github: Arc::new(github),
468            docker: Arc::new(docker),
469            names,
470            runner_prefix: identity.prefix(),
471            identity,
472            memory_max,
473            cpus,
474            slots,
475            repositories,
476            reseeds: BTreeMap::new(),
477            images: Images::new(),
478            reseed_memory: BTreeMap::new(),
479            reseed_requested: BTreeSet::new(),
480            retired: BTreeMap::new(),
481            draining: BTreeSet::new(),
482            drained_once: BTreeSet::new(),
483            leftover: BTreeMap::new(),
484            reseed_due: BTreeSet::new(),
485            oversized: BTreeSet::new(),
486            reseeded_at: BTreeMap::new(),
487            ran: BTreeMap::new(),
488            docker_disk: None,
489            disk: None,
490            // So the caches are measured once at start.
491            trim_due: true,
492            trim: None,
493            checked_at: SystemTime::now(),
494            // Unknown: an earlier run may have left it up, so the first idle tick stops it.
495            mirror_up: true,
496            rate_limit_logged: false,
497            busy_orphans: BTreeSet::new(),
498            status,
499            announced_profiles: Some(announced.profiles),
500            control: control.clone(),
501            commands,
502            caches,
503            store: store.clone(),
504            paths,
505            tracked: watch::channel(Vec::new()).0,
506            found: mpsc::unbounded_channel(),
507            captures: BTreeMap::new(),
508            recaptures: BTreeMap::new(),
509            leases: Leases::default(),
510            mirror_until: MirrorPosition::default(),
511            queued_jobs: BTreeMap::new(),
512            cancelling: BTreeMap::new(),
513            closing: BTreeMap::new(),
514            last_prune: None,
515            measured: admission::Measured::default(),
516            warm: warm::Warm::default(),
517            records,
518            build_cache: build_cache.clone(),
519            default_branches: BTreeMap::new(),
520            ended_uploads: BTreeMap::new(),
521            visibility_due: Arc::default(),
522            visibility_asking: None,
523        };
524        engine.publish();
525        (
526            engine,
527            Handle {
528                status: receiver,
529                control,
530                store,
531                caches: caches_receiver,
532                records: records_receiver,
533                build_cache,
534            },
535        )
536    }
537
538    /// Runs until a shutdown request is carried out.
539    pub async fn run(mut self) {
540        tokio::spawn(metrics::run(
541            Sampler::new(self.paths.clone(), &self.config.host.slice),
542            self.tracked.subscribe(),
543            self.store.clone(),
544        ));
545        // Until Docker answers: scheduling without knowing the containers already running would start a second
546        // runner on a slot's volume and delete the running one's registration.
547        while let Err(error) = self.recover().await {
548            warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
549            if self.control.mode() != Shutdown::No {
550                return;
551            }
552            tokio::select! {
553                () = tokio::time::sleep(Duration::from_secs(5)) => {}
554                () = self.control.wake.notified() => {}
555            }
556        }
557        loop {
558            let mode = self.control.mode();
559            self.tick(mode).await;
560            self.publish();
561            let idle = self.slots.values().flatten().all(Option::is_none);
562            if mode != Shutdown::No
563                && idle
564                && self.reseeds.is_empty()
565                && self.trim.is_none()
566                && self.records.recordings.is_empty()
567            {
568                break;
569            }
570            let interval = if mode == Shutdown::No {
571                self.config.host.poll_interval
572            } else {
573                Duration::from_secs(5)
574            };
575            tokio::select! {
576                () = tokio::time::sleep(interval) => {}
577                () = self.control.wake.notified() => {}
578                Some(command) = self.commands.recv() => {
579                    self.command(command).await;
580                    self.publish();
581                }
582                Some(()) = self.records.finished.recv() => {
583                    self.collect_recordings().await;
584                    self.publish();
585                }
586            }
587        }
588        self.close_shells().await;
589        if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
590            warn!("cannot stop the mirror: {error}");
591        }
592        info!("stopped");
593    }
594
595    /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
596    /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
597    /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
598    /// the clock, not on polls.
599    async fn serving<T: Send + 'static>(
600        &mut self,
601        work: impl Future<Output = T> + Send + 'static,
602    ) -> T {
603        let mut task = tokio::spawn(work);
604        loop {
605            tokio::select! {
606                biased;
607                done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
608                    Ok(output) => return output,
609                    // As if it had panicked here.
610                    Err(Ok(panic)) => std::panic::resume_unwind(panic),
611                    // Only while the runtime shuts down, which drops this task too.
612                    Err(Err(_)) => std::future::pending::<()>().await,
613                },
614                Some(command) = self.commands.recv() => {
615                    self.command(command).await;
616                    self.publish();
617                    // A tick follows it, as one follows a command answered between ticks: a config it replaced
618                    // polls its repositories then, not a poll interval later.
619                    self.control.wake.notify_one();
620                }
621            }
622        }
623    }
624
625    fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
626        self.slots.iter().flat_map(|(profile, slots)| {
627            slots.iter().enumerate().filter_map(|(index, slot)| {
628                slot.as_ref().map(|occupant| {
629                    (
630                        SlotRef {
631                            profile: profile.clone(),
632                            index,
633                        },
634                        occupant,
635                    )
636                })
637            })
638        })
639    }
640
641    /// The runner in `slot`, if one is.
642    fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
643        self.slots
644            .get(&slot.profile)
645            .and_then(|slots| slots.get(slot.index))
646            .and_then(Option::as_ref)
647    }
648
649    /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
650    /// runs in them.
651    fn drop_closed_slots(&mut self) {
652        for (name, slots) in &mut self.slots {
653            let count = self
654                .config
655                .profiles
656                .get(name)
657                .map_or(0, |profile| profile.slots as usize);
658            while slots.len() > count && slots.last().is_some_and(Option::is_none) {
659                slots.pop();
660            }
661        }
662        let profiles = &self.config.profiles;
663        self.slots
664            .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
665        // The volumes of the slots dropped go at the next check of the caches.
666        let slots = &self.slots;
667        self.ran.retain(|slot, _| {
668            slots
669                .get(&slot.profile)
670                .is_some_and(|slots| slot.index < slots.len())
671        });
672        let (slots, reseeds) = (&self.slots, &self.reseeds);
673        self.retired
674            .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
675    }
676}
677
678#[cfg(test)]
679mod tests;