runnerd_core/scheduler/engine.rs
1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod history;
20mod images;
21mod records;
22mod recover;
23mod runners;
24mod status;
25mod tick;
26mod warm;
27
28use std::collections::{BTreeMap, BTreeSet};
29use std::sync::{Arc, Mutex};
30use std::time::{Duration, SystemTime};
31
32use tokio::sync::{mpsc, watch};
33use tokio::task::JoinHandle;
34use tracing::{info, warn};
35
36pub use self::control::{
37 Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
38 Target, same_slice,
39};
40use self::history::{Closing, Found};
41use self::images::Images;
42pub use self::records::RecordSelection;
43use super::{Hold, SlotRef};
44use crate::admission::Key;
45use crate::api;
46use crate::buildcache::BuildCache;
47use crate::cache::{self, Names, Trimmed};
48use crate::config::Config;
49use crate::docker::{self, Docker, DockerApi};
50use crate::github::{Github, GithubApi, QueuedJob};
51use crate::isolation::{self, Isolation};
52use crate::observe::metrics::{self, Paths, Sampler, Tracked};
53use crate::observe::mirror::Leases;
54use crate::store::{Announced, MirrorPosition, Store};
55
56/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
57/// identifies the machine, so a changed host name still matches the runners started before the change, and two
58/// machines with the same or overlapping host names never take each other's runners for their own.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct HostIdentity {
61 /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
62 id: String,
63 /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
64 host: String,
65}
66
67impl HostIdentity {
68 /// This machine's identity, from `/etc/machine-id` and its host name.
69 pub fn read() -> std::io::Result<HostIdentity> {
70 let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
71 let machine_id = std::fs::read_to_string("/etc/machine-id")
72 .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
73 .map_err(|error| {
74 std::io::Error::new(
75 error.kind(),
76 format!("cannot read /etc/machine-id: {error}"),
77 )
78 })?;
79 HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
80 std::io::Error::new(
81 std::io::ErrorKind::InvalidData,
82 "/etc/machine-id does not hold a machine ID",
83 )
84 })
85 }
86
87 /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
88 pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
89 let machine_id = machine_id.trim();
90 if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
91 return None;
92 }
93 // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
94 let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
95 .bytes()
96 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
97 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
98 });
99 let host: String = hostname
100 .trim()
101 .chars()
102 .map(|c| {
103 if c.is_ascii_alphanumeric() {
104 c.to_ascii_lowercase()
105 } else {
106 '-'
107 }
108 })
109 .take(20)
110 .collect();
111 let host = host.trim_matches('-');
112 Some(HostIdentity {
113 id: format!("{:08x}", hash >> 32),
114 host: if host.is_empty() { "host" } else { host }.to_owned(),
115 })
116 }
117
118 /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
119 /// machine's names.
120 pub fn prefix(&self) -> String {
121 format!("runnerd-{}-", self.id)
122 }
123
124 /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
125 pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
126 format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
127 }
128}
129
130/// What the API needs from a running engine.
131#[derive(Debug, Clone)]
132pub struct Handle {
133 /// The latest status, replaced after every tick.
134 pub status: watch::Receiver<api::Status>,
135 /// Pause, resume and shutdown requests.
136 pub control: Arc<Control>,
137 /// The job history.
138 pub store: Store,
139 /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
140 /// config is.
141 pub caches: watch::Receiver<Vec<api::CacheVolume>>,
142 /// The records of failed jobs, replaced whenever one is made or removed.
143 pub records: watch::Receiver<api::Records>,
144 /// The shared build cache, for its size now.
145 pub build_cache: Option<BuildCache>,
146}
147
148/// Where the engine runs, besides its config and its GitHub and Docker clients.
149#[derive(Debug)]
150pub struct Setup {
151 /// The names of what runnerd creates in Docker.
152 pub names: Names,
153 /// This machine, which names its runners, so a restarted daemon recognizes them.
154 pub identity: HostIdentity,
155 /// The slice's `MemoryMax`.
156 pub memory_max: Option<u64>,
157 /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
158 pub cpus: Option<u64>,
159 /// The job history.
160 pub store: Store,
161 /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
162 /// announces only the difference, including what the config no longer has.
163 pub announced: Announced,
164 /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
165 pub paused: Vec<String>,
166 /// Where the cgroup and proc file systems are.
167 pub paths: Paths,
168 /// The shared build cache, unless the daemon could not open it.
169 pub build_cache: Option<BuildCache>,
170}
171
172/// Why a runner starts.
173#[derive(Debug, Clone, Copy, PartialEq, Eq)]
174enum Purpose {
175 /// For a queued job.
176 Job,
177 /// As a warm runner, for jobs not queued yet.
178 Warm,
179}
180
181/// A slot's runner and its container.
182#[derive(Debug, Clone)]
183struct Occupant {
184 repository: String,
185 runner_name: String,
186 runner_id: u64,
187 container: String,
188 busy: bool,
189 started_at: SystemTime,
190 /// Its container's memory cap, which a reload does not change.
191 memory: u64,
192 /// Its container's CPU limit in millicores, which a reload does not change.
193 cpus: u64,
194 /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
195 /// one whose record says nothing of it, which reserves its cap.
196 admission: Option<api::Admission>,
197 /// The job it took, once its log capture found it.
198 job: Option<u64>,
199 /// That job as admission tells jobs apart, once the history had its record.
200 key: Option<Key>,
201 /// The labels it registered with; `None` for one adopted from a container that does not say.
202 labels: Option<Vec<String>>,
203 /// The image it runs; `None` for one adopted from a container that does not say.
204 image: Option<String>,
205 /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
206 /// container, which each tick tries again.
207 observed: bool,
208}
209
210impl Occupant {
211 /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
212 /// the profile's. An unknown label set counts as the same.
213 fn outdated(
214 &self,
215 repository: &crate::config::Repository,
216 profile: &crate::config::Profile,
217 ) -> bool {
218 let differ = self.labels.as_ref().is_some_and(|labels| {
219 let set = |labels: &[String]| -> BTreeSet<String> {
220 labels
221 .iter()
222 .map(|label| label.to_ascii_lowercase())
223 .collect()
224 };
225 set(labels) != set(&repository.labels)
226 });
227 differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
228 }
229
230 /// Whether it runs another image than `profile` names. An unknown image counts as the same.
231 fn other_image(&self, profile: &crate::config::Profile) -> bool {
232 self.image
233 .as_ref()
234 .is_some_and(|image| *image != profile.image.reference())
235 }
236
237 /// What it reserves of the budget under measured admission: its cap if it has no admission.
238 fn reserved(&self) -> u64 {
239 self.admission
240 .map_or(self.memory, |admission| admission.reserved)
241 }
242
243 /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
244 fn reserved_cpu(&self) -> u64 {
245 self.admission
246 .and_then(|admission| admission.cpu)
247 .map_or(self.cpus, |cpu| cpu.reserved)
248 }
249
250 fn running_job(&self) -> api::RunningJob {
251 api::RunningJob {
252 repository: self.repository.clone(),
253 runner: self.runner_name.clone(),
254 job: self.job,
255 }
256 }
257}
258
259/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
260#[derive(Debug, Default)]
261struct RepositoryState {
262 queued: Vec<QueuedJob>,
263 /// Whether the last poll of its queued jobs succeeded.
264 listed: bool,
265 /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
266 /// not hold back the others (ADR 0058).
267 holds: BTreeMap<String, PoolHold>,
268 /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
269 /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
270 /// make it younger.
271 cpu_wait: BTreeMap<String, CpuWait>,
272 /// The profiles of its pools whose next job the last plan held for CPU.
273 cpu_held: BTreeSet<String>,
274}
275
276/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
277#[derive(Debug, Default)]
278struct PoolHold {
279 held: Option<Hold>,
280 /// Runners in a row that ended without taking a job.
281 failures: u32,
282 /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
283 idle_exits: u32,
284}
285
286/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
287#[derive(Debug, Clone, Copy)]
288struct CpuWait {
289 /// The job's ID.
290 job: u64,
291 /// When a plan first held it for CPU.
292 since: SystemTime,
293}
294
295/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
296#[derive(Debug)]
297pub struct Engine<G = Github, D = Docker> {
298 config: Config,
299 /// Shared with the listings a tick reads while it answers commands.
300 github: Arc<G>,
301 docker: Arc<D>,
302 names: Names,
303 isolation: &'static dyn Isolation,
304 identity: HostIdentity,
305 runner_prefix: String,
306 memory_max: Option<u64>,
307 /// The CPUs the slice may use, in millicores, read at start.
308 cpus: Option<u64>,
309 slots: BTreeMap<String, Vec<Option<Occupant>>>,
310 repositories: BTreeMap<String, RepositoryState>,
311 reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
312 /// The pulled images of the config's profiles, and the removal of those no profile runs.
313 images: Images,
314 /// The memory cap each running reseed's container started with, which a reload does not change.
315 reseed_memory: BTreeMap<String, u64>,
316 /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
317 reseed_requested: BTreeSet<String>,
318 /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
319 draining: BTreeSet<String>,
320 /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
321 /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
322 drained_once: BTreeSet<String>,
323 /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
324 /// each tick tries again.
325 leftover: BTreeMap<String, String>,
326 /// Profiles a reload removed, kept while their runners or reseed still run.
327 retired: BTreeMap<String, crate::config::Profile>,
328 /// Profiles whose slot 1 ran a job since their last reseed check.
329 reseed_due: BTreeSet<String>,
330 /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
331 /// little slot 1 changed.
332 oversized: BTreeSet<String>,
333 /// When each profile's last reseed finished: a check of the caches that started before then measured the old
334 /// snapshot.
335 reseeded_at: BTreeMap<String, SystemTime>,
336 /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
337 /// `None` until one measured Docker's disk, when each image keeps its own default.
338 docker_disk: Option<u64>,
339 /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
340 disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
341 /// Whether a job ended since the shared caches were last checked.
342 trim_due: bool,
343 /// The running check of the shared caches: Docker takes many seconds to measure volumes.
344 trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
345 /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
346 /// unless runners were adopted.
347 checked_at: SystemTime,
348 mirror_up: bool,
349 /// The memory cap the running mirror started with.
350 mirror_memory: u64,
351 rate_limit_logged: bool,
352 /// Runners without a container that GitHub still holds busy, already logged.
353 busy_orphans: BTreeSet<u64>,
354 status: watch::Sender<api::Status>,
355 /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
356 /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
357 announced_profiles: Option<Vec<api::ProfileStatus>>,
358 control: Arc<Control>,
359 commands: Commands,
360 caches: watch::Sender<Vec<api::CacheVolume>>,
361 store: Store,
362 paths: Paths,
363 /// The containers the sampler reads.
364 tracked: watch::Sender<Vec<Tracked>>,
365 found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
366 captures: BTreeMap<String, JoinHandle<history::Capture>>,
367 /// The runners whose log capture broke off, until they are gone.
368 recaptures: BTreeMap<String, history::Recapture>,
369 /// Who held which address, for the mirror's log.
370 leases: Leases,
371 /// How far the mirror's log was counted.
372 mirror_until: MirrorPosition,
373 /// Jobs recorded queued and not yet taken or closed, with their repository.
374 queued_jobs: BTreeMap<u64, String>,
375 /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
376 cancelling: BTreeMap<String, tokio::time::Instant>,
377 /// Jobs whose runner is gone, waiting for GitHub to report them completed.
378 closing: BTreeMap<u64, Closing>,
379 last_prune: Option<SystemTime>,
380 /// The records of failed jobs and the shells open in them.
381 records: records::RecordState,
382 /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
383 measured: admission::Measured,
384 /// What warm runners learned from the history, and the workflow runs seen lately.
385 warm: warm::Warm,
386 /// The shared build cache.
387 build_cache: Option<BuildCache>,
388 /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
389 /// cache when `[caches.build] branches` names none.
390 default_branches: BTreeMap<String, build_cache::DefaultBranch>,
391 /// Removed runners' build cache uploads held until `follow_jobs` asks GitHub of their jobs, by runner.
392 ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
393 /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
394 /// the answers.
395 visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
396 /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
397 visibility_asking: Option<JoinHandle<()>>,
398}
399
400fn unix_secs(time: SystemTime) -> u64 {
401 time.duration_since(SystemTime::UNIX_EPOCH)
402 .unwrap_or_default()
403 .as_secs()
404}
405
406impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
407 /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
408 /// them; `memory_max` is the slice's.
409 #[allow(
410 clippy::too_many_lines,
411 reason = "one line per field of the engine's state, which split up would only be listed twice"
412 )]
413 pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
414 let Setup {
415 names,
416 identity,
417 memory_max,
418 cpus,
419 store,
420 announced,
421 paused,
422 paths,
423 build_cache,
424 } = setup;
425 let slots = config
426 .profiles
427 .iter()
428 .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
429 .collect();
430 let repositories = config
431 .repositories
432 .iter()
433 .map(|repository| (repository.name.clone(), RepositoryState::default()))
434 .collect();
435 let (control, commands) = Control::new(
436 config
437 .repositories
438 .iter()
439 .map(|repository| repository.name.clone()),
440 paused,
441 store.clone(),
442 );
443 let control = Arc::new(control);
444 let (status, receiver) = watch::channel(api::Status {
445 version: env!("CARGO_PKG_VERSION").to_owned(),
446 shutting_down: false,
447 budget: api::Budget {
448 slice: config.host.slice.clone(),
449 memory_max,
450 ..api::Budget::default()
451 },
452 profiles: Vec::new(),
453 // What clients resuming after the last run's events have, so the first publish announces the difference.
454 repositories: announced.repositories,
455 queued: Vec::new(),
456 });
457 let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
458 let (records, records_receiver) = records::RecordState::new();
459 let mut engine = Engine {
460 isolation: isolation::backend(config.host.isolation),
461 // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
462 mirror_memory: config.mirror_memory(),
463 config,
464 github: Arc::new(github),
465 docker: Arc::new(docker),
466 names,
467 runner_prefix: identity.prefix(),
468 identity,
469 memory_max,
470 cpus,
471 slots,
472 repositories,
473 reseeds: BTreeMap::new(),
474 images: Images::new(),
475 reseed_memory: BTreeMap::new(),
476 reseed_requested: BTreeSet::new(),
477 retired: BTreeMap::new(),
478 draining: BTreeSet::new(),
479 drained_once: BTreeSet::new(),
480 leftover: BTreeMap::new(),
481 reseed_due: BTreeSet::new(),
482 oversized: BTreeSet::new(),
483 reseeded_at: BTreeMap::new(),
484 docker_disk: None,
485 disk: None,
486 // So the caches are measured once at start.
487 trim_due: true,
488 trim: None,
489 checked_at: SystemTime::now(),
490 // Unknown: an earlier run may have left it up, so the first idle tick stops it.
491 mirror_up: true,
492 rate_limit_logged: false,
493 busy_orphans: BTreeSet::new(),
494 status,
495 announced_profiles: Some(announced.profiles),
496 control: control.clone(),
497 commands,
498 caches,
499 store: store.clone(),
500 paths,
501 tracked: watch::channel(Vec::new()).0,
502 found: mpsc::unbounded_channel(),
503 captures: BTreeMap::new(),
504 recaptures: BTreeMap::new(),
505 leases: Leases::default(),
506 mirror_until: MirrorPosition::default(),
507 queued_jobs: BTreeMap::new(),
508 cancelling: BTreeMap::new(),
509 closing: BTreeMap::new(),
510 last_prune: None,
511 measured: admission::Measured::default(),
512 warm: warm::Warm::default(),
513 records,
514 build_cache: build_cache.clone(),
515 default_branches: BTreeMap::new(),
516 ended_uploads: BTreeMap::new(),
517 visibility_due: Arc::default(),
518 visibility_asking: None,
519 };
520 engine.publish();
521 (
522 engine,
523 Handle {
524 status: receiver,
525 control,
526 store,
527 caches: caches_receiver,
528 records: records_receiver,
529 build_cache,
530 },
531 )
532 }
533
534 /// Runs until a shutdown request is carried out.
535 pub async fn run(mut self) {
536 tokio::spawn(metrics::run(
537 Sampler::new(self.paths.clone(), &self.config.host.slice),
538 self.tracked.subscribe(),
539 self.store.clone(),
540 ));
541 // Until Docker answers: scheduling without knowing the containers already running would start a second
542 // runner on a slot's volume and delete the running one's registration.
543 while let Err(error) = self.recover().await {
544 warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
545 if self.control.mode() != Shutdown::No {
546 return;
547 }
548 tokio::select! {
549 () = tokio::time::sleep(Duration::from_secs(5)) => {}
550 () = self.control.wake.notified() => {}
551 }
552 }
553 loop {
554 let mode = self.control.mode();
555 self.tick(mode).await;
556 self.publish();
557 let idle = self.slots.values().flatten().all(Option::is_none);
558 if mode != Shutdown::No
559 && idle
560 && self.reseeds.is_empty()
561 && self.trim.is_none()
562 && self.records.recordings.is_empty()
563 {
564 break;
565 }
566 let interval = if mode == Shutdown::No {
567 self.config.host.poll_interval
568 } else {
569 Duration::from_secs(5)
570 };
571 tokio::select! {
572 () = tokio::time::sleep(interval) => {}
573 () = self.control.wake.notified() => {}
574 Some(command) = self.commands.recv() => {
575 self.command(command).await;
576 self.publish();
577 }
578 Some(()) = self.records.finished.recv() => {
579 self.collect_recordings().await;
580 self.publish();
581 }
582 }
583 }
584 self.close_shells().await;
585 if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
586 warn!("cannot stop the mirror: {error}");
587 }
588 info!("stopped");
589 }
590
591 /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
592 /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
593 /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
594 /// the clock, not on polls.
595 async fn serving<T: Send + 'static>(
596 &mut self,
597 work: impl Future<Output = T> + Send + 'static,
598 ) -> T {
599 let mut task = tokio::spawn(work);
600 loop {
601 tokio::select! {
602 biased;
603 done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
604 Ok(output) => return output,
605 // As if it had panicked here.
606 Err(Ok(panic)) => std::panic::resume_unwind(panic),
607 // Only while the runtime shuts down, which drops this task too.
608 Err(Err(_)) => std::future::pending::<()>().await,
609 },
610 Some(command) = self.commands.recv() => {
611 self.command(command).await;
612 self.publish();
613 // A tick follows it, as one follows a command answered between ticks: a config it replaced
614 // polls its repositories then, not a poll interval later.
615 self.control.wake.notify_one();
616 }
617 }
618 }
619 }
620
621 fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
622 self.slots.iter().flat_map(|(profile, slots)| {
623 slots.iter().enumerate().filter_map(|(index, slot)| {
624 slot.as_ref().map(|occupant| {
625 (
626 SlotRef {
627 profile: profile.clone(),
628 index,
629 },
630 occupant,
631 )
632 })
633 })
634 })
635 }
636
637 /// The runner in `slot`, if one is.
638 fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
639 self.slots
640 .get(&slot.profile)
641 .and_then(|slots| slots.get(slot.index))
642 .and_then(Option::as_ref)
643 }
644
645 /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
646 /// runs in them.
647 fn drop_closed_slots(&mut self) {
648 for (name, slots) in &mut self.slots {
649 let count = self
650 .config
651 .profiles
652 .get(name)
653 .map_or(0, |profile| profile.slots as usize);
654 while slots.len() > count && slots.last().is_some_and(Option::is_none) {
655 slots.pop();
656 }
657 }
658 let profiles = &self.config.profiles;
659 self.slots
660 .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
661 let (slots, reseeds) = (&self.slots, &self.reseeds);
662 self.retired
663 .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
664 }
665}
666
667#[cfg(test)]
668mod tests;