runnerd_core/scheduler/engine.rs
1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod history;
20mod images;
21mod records;
22mod recover;
23mod runners;
24mod status;
25mod tick;
26mod warm;
27
28use std::collections::{BTreeMap, BTreeSet};
29use std::sync::{Arc, Mutex};
30use std::time::{Duration, SystemTime};
31
32use tokio::sync::{mpsc, watch};
33use tokio::task::JoinHandle;
34use tracing::{info, warn};
35
36pub use self::control::{
37 Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
38 Target, same_slice,
39};
40use self::history::{Closing, Found};
41use self::images::Images;
42pub use self::records::RecordSelection;
43use super::{Hold, SlotRef};
44use crate::admission::Key;
45use crate::api;
46use crate::buildcache::BuildCache;
47use crate::cache::{self, Names, Trimmed};
48use crate::config::Config;
49use crate::docker::{self, Docker, DockerApi};
50use crate::github::{Github, GithubApi, QueuedJob};
51use crate::isolation::{self, Isolation};
52use crate::observe::metrics::{self, Paths, Sampler, Tracked};
53use crate::observe::mirror::Leases;
54use crate::store::{Announced, MirrorPosition, Store};
55
56/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
57/// identifies the machine, so a changed host name still matches the runners started before the change, and two
58/// machines with the same or overlapping host names never take each other's runners for their own.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct HostIdentity {
61 /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
62 id: String,
63 /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
64 host: String,
65}
66
67impl HostIdentity {
68 /// This machine's identity, from `/etc/machine-id` and its host name.
69 pub fn read() -> std::io::Result<HostIdentity> {
70 let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
71 let machine_id = std::fs::read_to_string("/etc/machine-id")
72 .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
73 .map_err(|error| {
74 std::io::Error::new(
75 error.kind(),
76 format!("cannot read /etc/machine-id: {error}"),
77 )
78 })?;
79 HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
80 std::io::Error::new(
81 std::io::ErrorKind::InvalidData,
82 "/etc/machine-id does not hold a machine ID",
83 )
84 })
85 }
86
87 /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
88 pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
89 let machine_id = machine_id.trim();
90 if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
91 return None;
92 }
93 // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
94 let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
95 .bytes()
96 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
97 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
98 });
99 let host: String = hostname
100 .trim()
101 .chars()
102 .map(|c| {
103 if c.is_ascii_alphanumeric() {
104 c.to_ascii_lowercase()
105 } else {
106 '-'
107 }
108 })
109 .take(20)
110 .collect();
111 let host = host.trim_matches('-');
112 Some(HostIdentity {
113 id: format!("{:08x}", hash >> 32),
114 host: if host.is_empty() { "host" } else { host }.to_owned(),
115 })
116 }
117
118 /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
119 /// machine's names.
120 pub fn prefix(&self) -> String {
121 format!("runnerd-{}-", self.id)
122 }
123
124 /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
125 pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
126 format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
127 }
128}
129
130/// What the API needs from a running engine.
131#[derive(Debug, Clone)]
132pub struct Handle {
133 /// The latest status, replaced after every tick.
134 pub status: watch::Receiver<api::Status>,
135 /// Pause, resume and shutdown requests.
136 pub control: Arc<Control>,
137 /// The job history.
138 pub store: Store,
139 /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
140 /// config is.
141 pub caches: watch::Receiver<Vec<api::CacheVolume>>,
142 /// The records of failed jobs, replaced whenever one is made or removed.
143 pub records: watch::Receiver<api::Records>,
144 /// The shared build cache, for its size now.
145 pub build_cache: Option<BuildCache>,
146 /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
147 /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
148 pub disk: watch::Receiver<Option<u64>>,
149}
150
151impl Handle {
152 /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
153 /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
154 pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
155 self.control.shutdown(mode).stopping()
156 }
157}
158
159/// Where the engine runs, besides its config and its GitHub and Docker clients.
160#[derive(Debug)]
161pub struct Setup {
162 /// The names of what runnerd creates in Docker.
163 pub names: Names,
164 /// This machine, which names its runners, so a restarted daemon recognizes them.
165 pub identity: HostIdentity,
166 /// The slice's `MemoryMax`.
167 pub memory_max: Option<u64>,
168 /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
169 pub cpus: Option<u64>,
170 /// The job history.
171 pub store: Store,
172 /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
173 /// announces only the difference, including what the config no longer has.
174 pub announced: Announced,
175 /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
176 pub paused: Vec<String>,
177 /// Where the cgroup and proc file systems are.
178 pub paths: Paths,
179 /// The shared build cache, unless the daemon could not open it.
180 pub build_cache: Option<BuildCache>,
181}
182
183/// Why a runner starts.
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
185enum Purpose {
186 /// For a queued job.
187 Job,
188 /// As a warm runner, for jobs not queued yet.
189 Warm,
190}
191
192/// Whether Docker answers a stop.
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194enum DockerInStop {
195 /// It answers, if with errors, or no stop was asked for.
196 Answers,
197 /// It did not answer a call the stop needed: the stop calls it no more.
198 Failed,
199}
200
201/// A slot's runner and its container.
202#[derive(Debug, Clone)]
203struct Occupant {
204 repository: String,
205 runner_name: String,
206 runner_id: u64,
207 container: String,
208 busy: bool,
209 started_at: SystemTime,
210 /// Its container's memory cap, which a reload does not change.
211 memory: u64,
212 /// Its container's CPU limit in millicores, which a reload does not change.
213 cpus: u64,
214 /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
215 /// one whose record says nothing of it, which reserves its cap.
216 admission: Option<api::Admission>,
217 /// The job it took, once its log capture found it.
218 job: Option<u64>,
219 /// That job as admission tells jobs apart, once the history had its record.
220 key: Option<Key>,
221 /// The labels it registered with; `None` for one adopted from a container that does not say.
222 labels: Option<Vec<String>>,
223 /// The image it runs; `None` for one adopted from a container that does not say.
224 image: Option<String>,
225 /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
226 /// label.
227 spec: Option<String>,
228 /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
229 /// container, which each tick tries again.
230 observed: bool,
231}
232
233impl Occupant {
234 /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
235 /// the profile's. An unknown label set counts as the same.
236 fn outdated(
237 &self,
238 repository: &crate::config::Repository,
239 profile: &crate::config::Profile,
240 ) -> bool {
241 let differ = self.labels.as_ref().is_some_and(|labels| {
242 let set = |labels: &[String]| -> BTreeSet<String> {
243 labels
244 .iter()
245 .map(|label| label.to_ascii_lowercase())
246 .collect()
247 };
248 set(labels) != set(&repository.labels)
249 });
250 differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
251 }
252
253 /// Whether it runs another image than `profile` names. An unknown image counts as the same.
254 fn other_image(&self, profile: &crate::config::Profile) -> bool {
255 self.image
256 .as_ref()
257 .is_some_and(|image| *image != profile.image.reference())
258 }
259
260 /// Whether another release started it, with another container spec: one that does not say is older.
261 fn other_spec(&self) -> bool {
262 self.spec.as_deref() != Some(runners::THIS_SPEC)
263 }
264
265 /// What it reserves of the budget under measured admission: its cap if it has no admission.
266 fn reserved(&self) -> u64 {
267 self.admission
268 .map_or(self.memory, |admission| admission.reserved)
269 }
270
271 /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
272 fn reserved_cpu(&self) -> u64 {
273 self.admission
274 .and_then(|admission| admission.cpu)
275 .map_or(self.cpus, |cpu| cpu.reserved)
276 }
277
278 fn running_job(&self) -> api::RunningJob {
279 api::RunningJob {
280 repository: self.repository.clone(),
281 runner: self.runner_name.clone(),
282 job: self.job,
283 }
284 }
285}
286
287/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
288#[derive(Debug, Default)]
289struct RepositoryState {
290 queued: Vec<QueuedJob>,
291 /// Whether the last poll of its queued jobs succeeded.
292 listed: bool,
293 /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
294 /// not hold back the others (ADR 0058).
295 holds: BTreeMap<String, PoolHold>,
296 /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
297 /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
298 /// make it younger.
299 cpu_wait: BTreeMap<String, CpuWait>,
300 /// The profiles of its pools whose next job the last plan held for CPU.
301 cpu_held: BTreeSet<String>,
302}
303
304/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
305#[derive(Debug, Default)]
306struct PoolHold {
307 held: Option<Hold>,
308 /// Runners in a row that ended without taking a job.
309 failures: u32,
310 /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
311 idle_exits: u32,
312}
313
314/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
315#[derive(Debug, Clone, Copy)]
316struct CpuWait {
317 /// The job's ID.
318 job: u64,
319 /// When a plan first held it for CPU.
320 since: SystemTime,
321}
322
323/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
324/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
325/// a single Docker request (120s) could otherwise outlast.
326pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
327
328/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
329#[derive(Debug)]
330pub struct Engine<G = Github, D = Docker> {
331 config: Config,
332 /// Shared with the listings a tick reads while it answers commands.
333 github: Arc<G>,
334 docker: Arc<D>,
335 names: Names,
336 isolation: &'static dyn Isolation,
337 identity: HostIdentity,
338 runner_prefix: String,
339 memory_max: Option<u64>,
340 /// The CPUs the slice may use, in millicores, read at start.
341 cpus: Option<u64>,
342 slots: BTreeMap<String, Vec<Option<Occupant>>>,
343 repositories: BTreeMap<String, RepositoryState>,
344 reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
345 /// The pulled images of the config's profiles, and the removal of those no profile runs.
346 images: Images,
347 /// The memory cap each running reseed's container started with, which a reload does not change.
348 reseed_memory: BTreeMap<String, u64>,
349 /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
350 reseed_requested: BTreeSet<String>,
351 /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
352 draining: BTreeSet<String>,
353 /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
354 /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
355 drained_once: BTreeSet<String>,
356 /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
357 /// each tick tries again.
358 leftover: BTreeMap<String, String>,
359 /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
360 /// failed jobs' records: the first tick after each one's backoff tries again.
361 unread: Vec<recover::Left>,
362 /// Profiles a reload removed, kept while their runners or reseed still run.
363 retired: BTreeMap<String, crate::config::Profile>,
364 /// Profiles whose slot 1 ran a job since their last reseed check.
365 reseed_due: BTreeSet<String>,
366 /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
367 /// little slot 1 changed.
368 oversized: BTreeSet<String>,
369 /// When each profile's last reseed finished: a check of the caches that started before then measured the old
370 /// snapshot.
371 reseeded_at: BTreeMap<String, SystemTime>,
372 /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
373 /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
374 ran: BTreeMap<SlotRef, Key>,
375 /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
376 /// `None` until one measured Docker's disk, when each image keeps its own default.
377 docker_disk: watch::Sender<Option<u64>>,
378 /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
379 disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
380 /// Whether a job ended since the shared caches were last checked.
381 trim_due: bool,
382 /// The running check of the shared caches: Docker takes many seconds to measure volumes.
383 trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
384 /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
385 /// unless runners were adopted.
386 checked_at: SystemTime,
387 mirror_up: bool,
388 /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
389 docker_in_stop: DockerInStop,
390 /// The memory cap the running mirror started with.
391 mirror_memory: u64,
392 rate_limit_logged: bool,
393 /// Runners without a container that GitHub still holds busy, already logged.
394 busy_orphans: BTreeSet<u64>,
395 status: watch::Sender<api::Status>,
396 /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
397 /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
398 announced_profiles: Option<Vec<api::ProfileStatus>>,
399 control: Arc<Control>,
400 commands: Commands,
401 caches: watch::Sender<Vec<api::CacheVolume>>,
402 store: Store,
403 paths: Paths,
404 /// The containers the sampler reads.
405 tracked: watch::Sender<Vec<Tracked>>,
406 found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
407 captures: BTreeMap<String, JoinHandle<history::Capture>>,
408 /// The runners whose log capture broke off, until they are gone.
409 recaptures: BTreeMap<String, history::Recapture>,
410 /// Who held which address, for the mirror's log.
411 leases: Leases,
412 /// How far the mirror's log was counted.
413 mirror_until: MirrorPosition,
414 /// Jobs recorded queued and not yet taken or closed, with their repository.
415 queued_jobs: BTreeMap<u64, String>,
416 /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
417 cancelling: BTreeMap<String, tokio::time::Instant>,
418 /// Jobs whose runner is gone, waiting for GitHub to report them completed.
419 closing: BTreeMap<u64, Closing>,
420 last_prune: Option<SystemTime>,
421 /// The records of failed jobs and the shells open in them.
422 records: records::RecordState,
423 /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
424 measured: admission::Measured,
425 /// What warm runners learned from the history, and the workflow runs seen lately.
426 warm: warm::Warm,
427 /// The shared build cache.
428 build_cache: Option<BuildCache>,
429 /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
430 /// cache when `[caches.build] branches` names none.
431 default_branches: BTreeMap<String, build_cache::DefaultBranch>,
432 /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
433 /// `build_cache::ENDED_WAIT`, by runner.
434 ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
435 /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
436 /// the answers.
437 visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
438 /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
439 visibility_asking: Option<JoinHandle<()>>,
440}
441
442fn unix_secs(time: SystemTime) -> u64 {
443 time.duration_since(SystemTime::UNIX_EPOCH)
444 .unwrap_or_default()
445 .as_secs()
446}
447
448impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
449 /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
450 /// them; `memory_max` is the slice's.
451 #[allow(
452 clippy::too_many_lines,
453 reason = "one line per field of the engine's state, which split up would only be listed twice"
454 )]
455 pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
456 let Setup {
457 names,
458 identity,
459 memory_max,
460 cpus,
461 store,
462 announced,
463 paused,
464 paths,
465 build_cache,
466 } = setup;
467 let slots = config
468 .profiles
469 .iter()
470 .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
471 .collect();
472 let repositories = config
473 .repositories
474 .iter()
475 .map(|repository| (repository.name.clone(), RepositoryState::default()))
476 .collect();
477 let (control, commands) = Control::new(
478 config
479 .repositories
480 .iter()
481 .map(|repository| repository.name.clone()),
482 paused,
483 store.clone(),
484 );
485 let control = Arc::new(control);
486 let (status, receiver) = watch::channel(api::Status {
487 version: env!("CARGO_PKG_VERSION").to_owned(),
488 shutting_down: false,
489 stopping: None,
490 budget: api::Budget {
491 slice: config.host.slice.clone(),
492 memory_max,
493 ..api::Budget::default()
494 },
495 profiles: Vec::new(),
496 // What clients resuming after the last run's events have, so the first publish announces the difference.
497 repositories: announced.repositories,
498 queued: Vec::new(),
499 });
500 let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
501 let (records, records_receiver) = records::RecordState::new();
502 let (docker_disk, disk_receiver) = watch::channel(None);
503 let mut engine = Engine {
504 isolation: isolation::backend(config.host.isolation),
505 // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
506 mirror_memory: config.mirror_memory(),
507 config,
508 github: Arc::new(github),
509 docker: Arc::new(docker),
510 names,
511 runner_prefix: identity.prefix(),
512 identity,
513 memory_max,
514 cpus,
515 slots,
516 repositories,
517 reseeds: BTreeMap::new(),
518 images: Images::new(),
519 reseed_memory: BTreeMap::new(),
520 reseed_requested: BTreeSet::new(),
521 retired: BTreeMap::new(),
522 draining: BTreeSet::new(),
523 drained_once: BTreeSet::new(),
524 leftover: BTreeMap::new(),
525 unread: Vec::new(),
526 reseed_due: BTreeSet::new(),
527 oversized: BTreeSet::new(),
528 reseeded_at: BTreeMap::new(),
529 ran: BTreeMap::new(),
530 docker_disk,
531 disk: None,
532 // So the caches are measured once at start.
533 trim_due: true,
534 trim: None,
535 checked_at: SystemTime::now(),
536 // Unknown: an earlier run may have left it up, so the first idle tick stops it.
537 mirror_up: true,
538 docker_in_stop: DockerInStop::Answers,
539 rate_limit_logged: false,
540 busy_orphans: BTreeSet::new(),
541 status,
542 announced_profiles: Some(announced.profiles),
543 control: control.clone(),
544 commands,
545 caches,
546 store: store.clone(),
547 paths,
548 tracked: watch::channel(Vec::new()).0,
549 found: mpsc::unbounded_channel(),
550 captures: BTreeMap::new(),
551 recaptures: BTreeMap::new(),
552 leases: Leases::default(),
553 mirror_until: MirrorPosition::default(),
554 queued_jobs: BTreeMap::new(),
555 cancelling: BTreeMap::new(),
556 closing: BTreeMap::new(),
557 last_prune: None,
558 measured: admission::Measured::default(),
559 warm: warm::Warm::default(),
560 records,
561 build_cache: build_cache.clone(),
562 default_branches: BTreeMap::new(),
563 ended_uploads: BTreeMap::new(),
564 visibility_due: Arc::default(),
565 visibility_asking: None,
566 };
567 engine.publish();
568 (
569 engine,
570 Handle {
571 status: receiver,
572 control,
573 store,
574 caches: caches_receiver,
575 records: records_receiver,
576 build_cache,
577 disk: disk_receiver,
578 },
579 )
580 }
581
582 /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
583 /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
584 pub async fn run(mut self) {
585 tokio::spawn(metrics::run(
586 Sampler::new(self.paths.clone(), &self.config.host.slice),
587 self.tracked.subscribe(),
588 self.store.clone(),
589 ));
590 let control = self.control.clone();
591 let deadline = async {
592 control.hurried().await;
593 tokio::time::sleep(STOP_DEADLINE).await;
594 };
595 tokio::select! {
596 () = self.serve() => {}
597 () = deadline => warn!(
598 "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
599 STOP_DEADLINE.as_secs()
600 ),
601 }
602 }
603
604 /// Recovers, then schedules until a shutdown request is carried out.
605 async fn serve(&mut self) {
606 // Until Docker answers: scheduling without knowing the containers already running would start a second
607 // runner on a slot's volume and delete the running one's registration.
608 while let Err(error) = self.recover().await {
609 warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
610 if self.control.mode() != Shutdown::No {
611 return;
612 }
613 tokio::select! {
614 () = tokio::time::sleep(Duration::from_secs(5)) => {}
615 () = self.control.wake.notified() => {}
616 }
617 }
618 self.schedule().await;
619 }
620
621 /// Schedules until a shutdown request is carried out, from a recovered engine.
622 async fn schedule(&mut self) {
623 let mut shown = Shutdown::No;
624 let mode = loop {
625 let mode = self.stopping();
626 // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
627 // Docker: until then status would show a daemon that is not stopping.
628 if mode != shown {
629 self.publish();
630 shown = mode;
631 }
632 if mode == Shutdown::Detach {
633 self.remove_idle().await;
634 self.collect_recordings().await;
635 } else {
636 self.tick(mode).await;
637 }
638 self.publish();
639 // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
640 // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
641 if self.stopping() != mode {
642 continue;
643 }
644 if self.stopped(mode) {
645 break mode;
646 }
647 let interval = if mode == Shutdown::No {
648 self.config.host.poll_interval
649 } else {
650 Duration::from_secs(5)
651 };
652 tokio::select! {
653 () = tokio::time::sleep(interval) => {}
654 () = self.control.wake.notified() => {}
655 // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
656 Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
657 self.command(command).await;
658 self.publish();
659 }
660 Some(()) = self.records.finished.recv() => {
661 self.collect_recordings().await;
662 self.publish();
663 }
664 }
665 };
666 // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
667 self.collect_reseeds().await;
668 if self.docker_in_stop == DockerInStop::Failed {
669 info!("stopped; the next start removes or adopts what Docker did not answer for");
670 return;
671 }
672 self.close_shells().await;
673 if mode == Shutdown::Detach {
674 // The mirror stays up only for the busy runners left behind.
675 let mirror = if self.config.caches.mirror.enabled {
676 " and the mirror"
677 } else {
678 ""
679 };
680 match self.occupied().count() {
681 0 => info!("stopped; no job was running"),
682 1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
683 runners => {
684 info!("stopped, leaving {runners} busy runners{mirror} to the next start");
685 }
686 }
687 return;
688 }
689 if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
690 warn!("cannot stop the mirror: {error}");
691 }
692 info!("stopped");
693 }
694
695 /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
696 fn stopping(&self) -> Shutdown {
697 match self.control.mode() {
698 Shutdown::No => Shutdown::No,
699 _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
700 mode => mode,
701 }
702 }
703
704 /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
705 /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
706 fn stopped(&self, mode: Shutdown) -> bool {
707 let idle = self.slots.values().flatten().all(Option::is_none);
708 let recorded = self.records.recordings.is_empty();
709 match mode {
710 Shutdown::No => false,
711 Shutdown::Detach => recorded,
712 Shutdown::Graceful | Shutdown::Force => idle && recorded,
713 }
714 }
715
716 /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
717 /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
718 /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
719 /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
720 fn docker_failed(&mut self, error: &docker::Error) {
721 if self.control.mode() != Shutdown::No
722 && self.docker_in_stop == DockerInStop::Answers
723 && docker::is_unreachable(error)
724 {
725 warn!("Docker does not answer during the stop; leaving the rest to the next start");
726 self.docker_in_stop = DockerInStop::Failed;
727 // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
728 // running jobs.
729 self.control.shutdown(Shutdown::Detach);
730 }
731 }
732
733 /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
734 /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
735 /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
736 /// the clock, not on polls.
737 async fn serving<T: Send + 'static>(
738 &mut self,
739 work: impl Future<Output = T> + Send + 'static,
740 ) -> T {
741 let mut task = tokio::spawn(work);
742 loop {
743 tokio::select! {
744 biased;
745 done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
746 Ok(output) => return output,
747 // As if it had panicked here.
748 Err(Ok(panic)) => std::panic::resume_unwind(panic),
749 // Only while the runtime shuts down, which drops this task too.
750 Err(Err(_)) => std::future::pending::<()>().await,
751 },
752 Some(command) = self.commands.recv() => {
753 self.command(command).await;
754 self.publish();
755 // A tick follows it, as one follows a command answered between ticks: a config it replaced
756 // polls its repositories then, not a poll interval later.
757 self.control.wake.notify_one();
758 }
759 }
760 }
761 }
762
763 fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
764 self.slots.iter().flat_map(|(profile, slots)| {
765 slots.iter().enumerate().filter_map(|(index, slot)| {
766 slot.as_ref().map(|occupant| {
767 (
768 SlotRef {
769 profile: profile.clone(),
770 index,
771 },
772 occupant,
773 )
774 })
775 })
776 })
777 }
778
779 /// The runner in `slot`, if one is.
780 fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
781 self.slots
782 .get(&slot.profile)
783 .and_then(|slots| slots.get(slot.index))
784 .and_then(Option::as_ref)
785 }
786
787 /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
788 /// runs in them.
789 fn drop_closed_slots(&mut self) {
790 for (name, slots) in &mut self.slots {
791 let count = self
792 .config
793 .profiles
794 .get(name)
795 .map_or(0, |profile| profile.slots as usize);
796 while slots.len() > count && slots.last().is_some_and(Option::is_none) {
797 slots.pop();
798 }
799 }
800 let profiles = &self.config.profiles;
801 self.slots
802 .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
803 // The volumes of the slots dropped go at the next check of the caches.
804 let slots = &self.slots;
805 self.ran.retain(|slot, _| {
806 slots
807 .get(&slot.profile)
808 .is_some_and(|slots| slot.index < slots.len())
809 });
810 let (slots, reseeds) = (&self.slots, &self.reseeds);
811 self.retired
812 .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
813 }
814}
815
816#[cfg(test)]
817mod tests;