runnerd_core/scheduler/engine.rs
1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod history;
20mod images;
21mod records;
22mod recover;
23mod runners;
24mod status;
25mod tick;
26mod warm;
27
28use std::collections::{BTreeMap, BTreeSet};
29use std::sync::{Arc, Mutex};
30use std::time::{Duration, SystemTime};
31
32use tokio::sync::{mpsc, watch};
33use tokio::task::JoinHandle;
34use tracing::{info, warn};
35
36pub use self::control::{
37 Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
38 Target, same_slice,
39};
40use self::history::{Closing, Found};
41use self::images::Images;
42pub use self::records::RecordSelection;
43use super::{Hold, SlotRef};
44use crate::admission::Key;
45use crate::api;
46use crate::buildcache::BuildCache;
47use crate::cache::{self, Names, Trimmed};
48use crate::config::Config;
49use crate::docker::{self, Docker, DockerApi};
50use crate::github::{Github, GithubApi, QueuedJob};
51use crate::isolation::{self, Isolation};
52use crate::observe::metrics::{self, Paths, Sampler, Tracked};
53use crate::observe::mirror::Leases;
54use crate::store::{Announced, MirrorPosition, Store};
55
56/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
57/// identifies the machine, so a changed host name still matches the runners started before the change, and two
58/// machines with the same or overlapping host names never take each other's runners for their own.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct HostIdentity {
61 /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
62 id: String,
63 /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
64 host: String,
65}
66
67impl HostIdentity {
68 /// This machine's identity, from `/etc/machine-id` and its host name.
69 pub fn read() -> std::io::Result<HostIdentity> {
70 let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
71 let machine_id = std::fs::read_to_string("/etc/machine-id")
72 .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
73 .map_err(|error| {
74 std::io::Error::new(
75 error.kind(),
76 format!("cannot read /etc/machine-id: {error}"),
77 )
78 })?;
79 HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
80 std::io::Error::new(
81 std::io::ErrorKind::InvalidData,
82 "/etc/machine-id does not hold a machine ID",
83 )
84 })
85 }
86
87 /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
88 pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
89 let machine_id = machine_id.trim();
90 if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
91 return None;
92 }
93 // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
94 let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
95 .bytes()
96 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
97 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
98 });
99 let host: String = hostname
100 .trim()
101 .chars()
102 .map(|c| {
103 if c.is_ascii_alphanumeric() {
104 c.to_ascii_lowercase()
105 } else {
106 '-'
107 }
108 })
109 .take(20)
110 .collect();
111 let host = host.trim_matches('-');
112 Some(HostIdentity {
113 id: format!("{:08x}", hash >> 32),
114 host: if host.is_empty() { "host" } else { host }.to_owned(),
115 })
116 }
117
118 /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
119 /// machine's names.
120 pub fn prefix(&self) -> String {
121 format!("runnerd-{}-", self.id)
122 }
123
124 /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
125 pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
126 format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
127 }
128}
129
130/// What the API needs from a running engine.
131#[derive(Debug, Clone)]
132pub struct Handle {
133 /// The latest status, replaced after every tick.
134 pub status: watch::Receiver<api::Status>,
135 /// Pause, resume and shutdown requests.
136 pub control: Arc<Control>,
137 /// The job history.
138 pub store: Store,
139 /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
140 /// config is.
141 pub caches: watch::Receiver<Vec<api::CacheVolume>>,
142 /// The records of failed jobs, replaced whenever one is made or removed.
143 pub records: watch::Receiver<api::Records>,
144 /// The shared build cache, for its size now.
145 pub build_cache: Option<BuildCache>,
146 /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
147 /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
148 pub disk: watch::Receiver<Option<u64>>,
149}
150
151impl Handle {
152 /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
153 /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
154 pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
155 self.control.shutdown(mode).stopping()
156 }
157}
158
159/// Where the engine runs, besides its config and its GitHub and Docker clients.
160#[derive(Debug)]
161pub struct Setup {
162 /// The names of what runnerd creates in Docker.
163 pub names: Names,
164 /// This machine, which names its runners, so a restarted daemon recognizes them.
165 pub identity: HostIdentity,
166 /// The slice's `MemoryMax`.
167 pub memory_max: Option<u64>,
168 /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
169 pub cpus: Option<u64>,
170 /// The job history.
171 pub store: Store,
172 /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
173 /// announces only the difference, including what the config no longer has.
174 pub announced: Announced,
175 /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
176 pub paused: Vec<String>,
177 /// Where the cgroup and proc file systems are.
178 pub paths: Paths,
179 /// The shared build cache, unless the daemon could not open it.
180 pub build_cache: Option<BuildCache>,
181}
182
183/// Why a runner starts, as its start's log line and its record say (ADR 0092).
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
185enum Purpose {
186 /// For a queued job.
187 Job,
188 /// As one of its repository's configured warm runners, for jobs not queued yet.
189 Warm,
190 /// As a warm runner past those, for a job its runs are expected to queue still.
191 Burst,
192}
193
194impl Purpose {
195 /// How the history stores it.
196 fn as_str(self) -> &'static str {
197 match self {
198 Purpose::Job => "job",
199 Purpose::Warm => "warm",
200 Purpose::Burst => "burst",
201 }
202 }
203}
204
205/// Whether Docker answers a stop.
206#[derive(Debug, Clone, Copy, PartialEq, Eq)]
207enum DockerInStop {
208 /// It answers, if with errors, or no stop was asked for.
209 Answers,
210 /// It did not answer a call the stop needed: the stop calls it no more.
211 Failed,
212}
213
214/// A slot's runner and its container.
215#[derive(Debug, Clone)]
216struct Occupant {
217 repository: String,
218 runner_name: String,
219 runner_id: u64,
220 container: String,
221 busy: bool,
222 started_at: SystemTime,
223 /// Its container's memory cap, which a reload does not change.
224 memory: u64,
225 /// Its container's CPU limit in millicores, which a reload does not change.
226 cpus: u64,
227 /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
228 /// one whose record says nothing of it, which reserves its cap.
229 admission: Option<api::Admission>,
230 /// The job it took, once its log capture found it.
231 job: Option<u64>,
232 /// That job as admission tells jobs apart, once the history had its record.
233 key: Option<Key>,
234 /// The labels it registered with; `None` for one adopted from a container that does not say.
235 labels: Option<Vec<String>>,
236 /// The image it runs; `None` for one adopted from a container that does not say.
237 image: Option<String>,
238 /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
239 /// label.
240 spec: Option<String>,
241 /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
242 /// container, which each tick tries again.
243 observed: bool,
244 /// Why this daemon started it; `None` for one a restart adopted.
245 purpose: Option<Purpose>,
246}
247
248impl Occupant {
249 /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
250 /// the profile's. An unknown label set counts as the same.
251 fn outdated(
252 &self,
253 repository: &crate::config::Repository,
254 profile: &crate::config::Profile,
255 ) -> bool {
256 let differ = self.labels.as_ref().is_some_and(|labels| {
257 let set = |labels: &[String]| -> BTreeSet<String> {
258 labels
259 .iter()
260 .map(|label| label.to_ascii_lowercase())
261 .collect()
262 };
263 set(labels) != set(&repository.labels)
264 });
265 differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
266 }
267
268 /// Whether it runs another image than `profile` names. An unknown image counts as the same.
269 fn other_image(&self, profile: &crate::config::Profile) -> bool {
270 self.image
271 .as_ref()
272 .is_some_and(|image| *image != profile.image.reference())
273 }
274
275 /// Whether another release started it, with another container spec: one that does not say is older.
276 fn other_spec(&self) -> bool {
277 self.spec.as_deref() != Some(runners::THIS_SPEC)
278 }
279
280 /// What it reserves of the budget under measured admission: its cap if it has no admission.
281 fn reserved(&self) -> u64 {
282 self.admission
283 .map_or(self.memory, |admission| admission.reserved)
284 }
285
286 /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
287 fn reserved_cpu(&self) -> u64 {
288 self.admission
289 .and_then(|admission| admission.cpu)
290 .map_or(self.cpus, |cpu| cpu.reserved)
291 }
292
293 fn running_job(&self) -> api::RunningJob {
294 api::RunningJob {
295 repository: self.repository.clone(),
296 runner: self.runner_name.clone(),
297 job: self.job,
298 }
299 }
300}
301
302/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
303#[derive(Debug, Default)]
304struct RepositoryState {
305 queued: Vec<QueuedJob>,
306 /// Whether the last poll of its queued jobs succeeded.
307 listed: bool,
308 /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
309 /// not hold back the others (ADR 0058).
310 holds: BTreeMap<String, PoolHold>,
311 /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
312 /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
313 /// make it younger.
314 cpu_wait: BTreeMap<String, CpuWait>,
315 /// The profiles of its pools whose next job the last plan held for CPU.
316 cpu_held: BTreeSet<String>,
317}
318
319/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
320#[derive(Debug, Default)]
321struct PoolHold {
322 held: Option<Hold>,
323 /// Runners in a row that ended without taking a job.
324 failures: u32,
325 /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
326 idle_exits: u32,
327}
328
329/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
330#[derive(Debug, Clone, Copy)]
331struct CpuWait {
332 /// The job's ID.
333 job: u64,
334 /// When a plan first held it for CPU.
335 since: SystemTime,
336}
337
338/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
339/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
340/// a single Docker request (120s) could otherwise outlast.
341pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
342
343/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
344#[derive(Debug)]
345pub struct Engine<G = Github, D = Docker> {
346 config: Config,
347 /// Shared with the listings a tick reads while it answers commands.
348 github: Arc<G>,
349 docker: Arc<D>,
350 names: Names,
351 isolation: &'static dyn Isolation,
352 identity: HostIdentity,
353 runner_prefix: String,
354 memory_max: Option<u64>,
355 /// The CPUs the slice may use, in millicores, read at start.
356 cpus: Option<u64>,
357 slots: BTreeMap<String, Vec<Option<Occupant>>>,
358 repositories: BTreeMap<String, RepositoryState>,
359 reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
360 /// The pulled images of the config's profiles, and the removal of those no profile runs.
361 images: Images,
362 /// The memory cap each running reseed's container started with, which a reload does not change.
363 reseed_memory: BTreeMap<String, u64>,
364 /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
365 reseed_requested: BTreeSet<String>,
366 /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
367 draining: BTreeSet<String>,
368 /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
369 /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
370 drained_once: BTreeSet<String>,
371 /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
372 /// each tick tries again.
373 leftover: BTreeMap<String, String>,
374 /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
375 /// failed jobs' records: the first tick after each one's backoff tries again.
376 unread: Vec<recover::Left>,
377 /// Profiles a reload removed, kept while their runners or reseed still run.
378 retired: BTreeMap<String, crate::config::Profile>,
379 /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
380 measure_due: BTreeSet<String>,
381 /// The measures of slot 1 running, by profile.
382 measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
383 /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
384 measured_at: BTreeMap<String, tokio::time::Instant>,
385 /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
386 /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
387 /// `snapshot_max`.
388 reseed_due: BTreeSet<String>,
389 /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
390 /// little slot 1 changed.
391 oversized: BTreeSet<String>,
392 /// When each profile's last reseed finished: a check of the caches that started before then measured the old
393 /// snapshot.
394 reseeded_at: BTreeMap<String, SystemTime>,
395 /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
396 /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
397 ran: BTreeMap<SlotRef, Key>,
398 /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
399 /// `None` until one measured Docker's disk, when each image keeps its own default.
400 docker_disk: watch::Sender<Option<u64>>,
401 /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
402 disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
403 /// Whether a job ended since the shared caches were last checked.
404 trim_due: bool,
405 /// The running check of the shared caches: Docker takes many seconds to measure volumes.
406 trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
407 /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
408 /// unless runners were adopted.
409 checked_at: SystemTime,
410 mirror_up: bool,
411 /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
412 docker_in_stop: DockerInStop,
413 /// The memory cap the running mirror started with.
414 mirror_memory: u64,
415 rate_limit_logged: bool,
416 /// Runners without a container that GitHub still holds busy, already logged.
417 busy_orphans: BTreeSet<u64>,
418 status: watch::Sender<api::Status>,
419 /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
420 /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
421 announced_profiles: Option<Vec<api::ProfileStatus>>,
422 control: Arc<Control>,
423 commands: Commands,
424 caches: watch::Sender<Vec<api::CacheVolume>>,
425 store: Store,
426 paths: Paths,
427 /// The containers the sampler reads.
428 tracked: watch::Sender<Vec<Tracked>>,
429 found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
430 captures: BTreeMap<String, JoinHandle<history::Capture>>,
431 /// The runners whose log capture broke off, until they are gone.
432 recaptures: BTreeMap<String, history::Recapture>,
433 /// Who held which address, for the mirror's log.
434 leases: Leases,
435 /// How far the mirror's log was counted.
436 mirror_until: MirrorPosition,
437 /// Jobs recorded queued and not yet taken or closed, with their repository.
438 queued_jobs: BTreeMap<u64, String>,
439 /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
440 cancelling: BTreeMap<String, tokio::time::Instant>,
441 /// Jobs whose runner is gone, waiting for GitHub to report them completed.
442 closing: BTreeMap<u64, Closing>,
443 last_prune: Option<SystemTime>,
444 /// The records of failed jobs and the shells open in them.
445 records: records::RecordState,
446 /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
447 measured: admission::Measured,
448 /// What warm runners learned from the history, and the workflow runs seen lately.
449 warm: warm::Warm,
450 /// The shared build cache.
451 build_cache: Option<BuildCache>,
452 /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
453 /// cache when `[caches.build] branches` names none.
454 default_branches: BTreeMap<String, build_cache::DefaultBranch>,
455 /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
456 /// `build_cache::ENDED_WAIT`, by runner.
457 ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
458 /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
459 /// the answers.
460 visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
461 /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
462 visibility_asking: Option<JoinHandle<()>>,
463}
464
465fn unix_secs(time: SystemTime) -> u64 {
466 time.duration_since(SystemTime::UNIX_EPOCH)
467 .unwrap_or_default()
468 .as_secs()
469}
470
471impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
472 /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
473 /// them; `memory_max` is the slice's.
474 #[allow(
475 clippy::too_many_lines,
476 reason = "one line per field of the engine's state, which split up would only be listed twice"
477 )]
478 pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
479 let Setup {
480 names,
481 identity,
482 memory_max,
483 cpus,
484 store,
485 announced,
486 paused,
487 paths,
488 build_cache,
489 } = setup;
490 let slots = config
491 .profiles
492 .iter()
493 .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
494 .collect();
495 let repositories = config
496 .repositories
497 .iter()
498 .map(|repository| (repository.name.clone(), RepositoryState::default()))
499 .collect();
500 let (control, commands) = Control::new(
501 config
502 .repositories
503 .iter()
504 .map(|repository| repository.name.clone()),
505 paused,
506 store.clone(),
507 );
508 let control = Arc::new(control);
509 let (status, receiver) = watch::channel(api::Status {
510 version: env!("CARGO_PKG_VERSION").to_owned(),
511 shutting_down: false,
512 stopping: None,
513 budget: api::Budget {
514 slice: config.host.slice.clone(),
515 memory_max,
516 ..api::Budget::default()
517 },
518 profiles: Vec::new(),
519 // What clients resuming after the last run's events have, so the first publish announces the difference.
520 repositories: announced.repositories,
521 queued: Vec::new(),
522 });
523 let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
524 let (records, records_receiver) = records::RecordState::new();
525 let (docker_disk, disk_receiver) = watch::channel(None);
526 let mut engine = Engine {
527 isolation: isolation::backend(config.host.isolation),
528 // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
529 mirror_memory: config.mirror_memory(),
530 config,
531 github: Arc::new(github),
532 docker: Arc::new(docker),
533 names,
534 runner_prefix: identity.prefix(),
535 identity,
536 memory_max,
537 cpus,
538 slots,
539 repositories,
540 reseeds: BTreeMap::new(),
541 images: Images::new(),
542 reseed_memory: BTreeMap::new(),
543 reseed_requested: BTreeSet::new(),
544 retired: BTreeMap::new(),
545 draining: BTreeSet::new(),
546 drained_once: BTreeSet::new(),
547 leftover: BTreeMap::new(),
548 unread: Vec::new(),
549 measure_due: BTreeSet::new(),
550 measures: BTreeMap::new(),
551 measured_at: BTreeMap::new(),
552 reseed_due: BTreeSet::new(),
553 oversized: BTreeSet::new(),
554 reseeded_at: BTreeMap::new(),
555 ran: BTreeMap::new(),
556 docker_disk,
557 disk: None,
558 // So the caches are measured once at start.
559 trim_due: true,
560 trim: None,
561 checked_at: SystemTime::now(),
562 // Unknown: an earlier run may have left it up, so the first idle tick stops it.
563 mirror_up: true,
564 docker_in_stop: DockerInStop::Answers,
565 rate_limit_logged: false,
566 busy_orphans: BTreeSet::new(),
567 status,
568 announced_profiles: Some(announced.profiles),
569 control: control.clone(),
570 commands,
571 caches,
572 store: store.clone(),
573 paths,
574 tracked: watch::channel(Vec::new()).0,
575 found: mpsc::unbounded_channel(),
576 captures: BTreeMap::new(),
577 recaptures: BTreeMap::new(),
578 leases: Leases::default(),
579 mirror_until: MirrorPosition::default(),
580 queued_jobs: BTreeMap::new(),
581 cancelling: BTreeMap::new(),
582 closing: BTreeMap::new(),
583 last_prune: None,
584 measured: admission::Measured::default(),
585 warm: warm::Warm::default(),
586 records,
587 build_cache: build_cache.clone(),
588 default_branches: BTreeMap::new(),
589 ended_uploads: BTreeMap::new(),
590 visibility_due: Arc::default(),
591 visibility_asking: None,
592 };
593 engine.publish();
594 (
595 engine,
596 Handle {
597 status: receiver,
598 control,
599 store,
600 caches: caches_receiver,
601 records: records_receiver,
602 build_cache,
603 disk: disk_receiver,
604 },
605 )
606 }
607
608 /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
609 /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
610 pub async fn run(mut self) {
611 tokio::spawn(metrics::run(
612 Sampler::new(self.paths.clone(), &self.config.host.slice),
613 self.tracked.subscribe(),
614 self.store.clone(),
615 ));
616 let control = self.control.clone();
617 let deadline = async {
618 control.hurried().await;
619 tokio::time::sleep(STOP_DEADLINE).await;
620 };
621 tokio::select! {
622 () = self.serve() => {}
623 () = deadline => warn!(
624 "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
625 STOP_DEADLINE.as_secs()
626 ),
627 }
628 }
629
630 /// Recovers, then schedules until a shutdown request is carried out.
631 async fn serve(&mut self) {
632 // Until Docker answers: scheduling without knowing the containers already running would start a second
633 // runner on a slot's volume and delete the running one's registration.
634 while let Err(error) = self.recover().await {
635 warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
636 if self.control.mode() != Shutdown::No {
637 return;
638 }
639 tokio::select! {
640 () = tokio::time::sleep(Duration::from_secs(5)) => {}
641 () = self.control.wake.notified() => {}
642 }
643 }
644 self.schedule().await;
645 }
646
647 /// Schedules until a shutdown request is carried out, from a recovered engine.
648 async fn schedule(&mut self) {
649 let mut shown = Shutdown::No;
650 let mode = loop {
651 let mode = self.stopping();
652 // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
653 // Docker: until then status would show a daemon that is not stopping.
654 if mode != shown {
655 self.publish();
656 shown = mode;
657 }
658 if mode == Shutdown::Detach {
659 self.remove_every_idle().await;
660 self.collect_recordings().await;
661 } else {
662 self.tick(mode).await;
663 }
664 self.publish();
665 // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
666 // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
667 if self.stopping() != mode {
668 continue;
669 }
670 if self.stopped(mode) {
671 break mode;
672 }
673 let interval = if mode == Shutdown::No {
674 self.config.host.poll_interval
675 } else {
676 Duration::from_secs(5)
677 };
678 tokio::select! {
679 () = tokio::time::sleep(interval) => {}
680 () = self.control.wake.notified() => {}
681 // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
682 Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
683 self.command(command).await;
684 self.publish();
685 }
686 Some(()) = self.records.finished.recv() => {
687 self.collect_recordings().await;
688 self.publish();
689 }
690 }
691 };
692 // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
693 self.collect_reseeds().await;
694 if self.docker_in_stop == DockerInStop::Failed {
695 info!("stopped; the next start removes or adopts what Docker did not answer for");
696 return;
697 }
698 self.close_shells().await;
699 if mode == Shutdown::Detach {
700 // The mirror stays up only for the busy runners left behind.
701 let mirror = if self.config.caches.mirror.enabled {
702 " and the mirror"
703 } else {
704 ""
705 };
706 match self.occupied().count() {
707 0 => info!("stopped; no job was running"),
708 1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
709 runners => {
710 info!("stopped, leaving {runners} busy runners{mirror} to the next start");
711 }
712 }
713 return;
714 }
715 if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
716 warn!("cannot stop the mirror: {error}");
717 }
718 info!("stopped");
719 }
720
721 /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
722 fn stopping(&self) -> Shutdown {
723 match self.control.mode() {
724 Shutdown::No => Shutdown::No,
725 _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
726 mode => mode,
727 }
728 }
729
730 /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
731 /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
732 fn stopped(&self, mode: Shutdown) -> bool {
733 let idle = self.slots.values().flatten().all(Option::is_none);
734 let recorded = self.records.recordings.is_empty();
735 match mode {
736 Shutdown::No => false,
737 Shutdown::Detach => recorded,
738 Shutdown::Graceful | Shutdown::Force => idle && recorded,
739 }
740 }
741
742 /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
743 /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
744 /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
745 /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
746 fn docker_failed(&mut self, error: &docker::Error) {
747 if self.control.mode() != Shutdown::No
748 && self.docker_in_stop == DockerInStop::Answers
749 && docker::is_unreachable(error)
750 {
751 warn!("Docker does not answer during the stop; leaving the rest to the next start");
752 self.docker_in_stop = DockerInStop::Failed;
753 // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
754 // running jobs.
755 self.control.shutdown(Shutdown::Detach);
756 }
757 }
758
759 /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
760 /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
761 /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
762 /// the clock, not on polls.
763 async fn serving<T: Send + 'static>(
764 &mut self,
765 work: impl Future<Output = T> + Send + 'static,
766 ) -> T {
767 let mut task = tokio::spawn(work);
768 loop {
769 tokio::select! {
770 biased;
771 done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
772 Ok(output) => return output,
773 // As if it had panicked here.
774 Err(Ok(panic)) => std::panic::resume_unwind(panic),
775 // Only while the runtime shuts down, which drops this task too.
776 Err(Err(_)) => std::future::pending::<()>().await,
777 },
778 Some(command) = self.commands.recv() => {
779 self.command(command).await;
780 self.publish();
781 // A tick follows it, as one follows a command answered between ticks: a config it replaced
782 // polls its repositories then, not a poll interval later.
783 self.control.wake.notify_one();
784 }
785 }
786 }
787 }
788
789 fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
790 self.slots.iter().flat_map(|(profile, slots)| {
791 slots.iter().enumerate().filter_map(|(index, slot)| {
792 slot.as_ref().map(|occupant| {
793 (
794 SlotRef {
795 profile: profile.clone(),
796 index,
797 },
798 occupant,
799 )
800 })
801 })
802 })
803 }
804
805 /// The runner in `slot`, if one is.
806 fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
807 self.slots
808 .get(&slot.profile)
809 .and_then(|slots| slots.get(slot.index))
810 .and_then(Option::as_ref)
811 }
812
813 /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
814 /// runs in them.
815 fn drop_closed_slots(&mut self) {
816 for (name, slots) in &mut self.slots {
817 let count = self
818 .config
819 .profiles
820 .get(name)
821 .map_or(0, |profile| profile.slots as usize);
822 while slots.len() > count && slots.last().is_some_and(Option::is_none) {
823 slots.pop();
824 }
825 }
826 let profiles = &self.config.profiles;
827 self.slots
828 .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
829 // The volumes of the slots dropped go at the next check of the caches.
830 let slots = &self.slots;
831 self.ran.retain(|slot, _| {
832 slots
833 .get(&slot.profile)
834 .is_some_and(|slots| slot.index < slots.len())
835 });
836 let (slots, reseeds) = (&self.slots, &self.reseeds);
837 self.retired
838 .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
839 }
840}
841
842#[cfg(test)]
843mod tests;