runnerd_core/scheduler/engine.rs
1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod control;
19mod exits;
20mod history;
21mod images;
22mod records;
23mod recover;
24mod runners;
25mod status;
26mod tick;
27mod warm;
28
29use std::collections::{BTreeMap, BTreeSet};
30use std::sync::{Arc, Mutex};
31use std::time::{Duration, SystemTime};
32
33use tokio::sync::{mpsc, watch};
34use tokio::task::JoinHandle;
35use tracing::{info, warn};
36
37pub use self::control::{
38 Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
39 Target, same_slice,
40};
41use self::history::{CAPTURE_DRAIN, Closing, Found};
42use self::images::Images;
43pub use self::records::RecordSelection;
44use super::{Hold, SlotRef};
45use crate::admission::Key;
46use crate::api;
47use crate::buildcache::BuildCache;
48use crate::cache::{self, Names, Trimmed};
49use crate::config::Config;
50use crate::docker::{self, Docker, DockerApi};
51use crate::github::{Github, GithubApi, QueuedJob};
52use crate::isolation::{self, Isolation};
53use crate::observe::metrics::{self, Paths, Sampler, Tracked};
54use crate::observe::mirror::Leases;
55use crate::store::{Announced, MirrorPosition, Store};
56
57/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
58/// identifies the machine, so a changed host name still matches the runners started before the change, and two
59/// machines with the same or overlapping host names never take each other's runners for their own.
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub struct HostIdentity {
62 /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
63 id: String,
64 /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
65 host: String,
66}
67
68impl HostIdentity {
69 /// This machine's identity, from `/etc/machine-id` and its host name.
70 pub fn read() -> std::io::Result<HostIdentity> {
71 let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
72 let machine_id = std::fs::read_to_string("/etc/machine-id")
73 .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
74 .map_err(|error| {
75 std::io::Error::new(
76 error.kind(),
77 format!("cannot read /etc/machine-id: {error}"),
78 )
79 })?;
80 HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
81 std::io::Error::new(
82 std::io::ErrorKind::InvalidData,
83 "/etc/machine-id does not hold a machine ID",
84 )
85 })
86 }
87
88 /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
89 pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
90 let machine_id = machine_id.trim();
91 if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
92 return None;
93 }
94 // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
95 let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
96 .bytes()
97 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
98 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
99 });
100 let host: String = hostname
101 .trim()
102 .chars()
103 .map(|c| {
104 if c.is_ascii_alphanumeric() {
105 c.to_ascii_lowercase()
106 } else {
107 '-'
108 }
109 })
110 .take(20)
111 .collect();
112 let host = host.trim_matches('-');
113 Some(HostIdentity {
114 id: format!("{:08x}", hash >> 32),
115 host: if host.is_empty() { "host" } else { host }.to_owned(),
116 })
117 }
118
119 /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
120 /// machine's names.
121 pub fn prefix(&self) -> String {
122 format!("runnerd-{}-", self.id)
123 }
124
125 /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
126 pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
127 format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
128 }
129}
130
131/// What the API needs from a running engine.
132#[derive(Debug, Clone)]
133pub struct Handle {
134 /// The latest status, replaced after every tick.
135 pub status: watch::Receiver<api::Status>,
136 /// Pause, resume and shutdown requests.
137 pub control: Arc<Control>,
138 /// The job history.
139 pub store: Store,
140 /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
141 /// config is.
142 pub caches: watch::Receiver<Vec<api::CacheVolume>>,
143 /// The records of failed jobs, replaced whenever one is made or removed.
144 pub records: watch::Receiver<api::Records>,
145 /// The shared build cache, for its size now.
146 pub build_cache: Option<BuildCache>,
147 /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
148 /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
149 pub disk: watch::Receiver<Option<u64>>,
150}
151
152impl Handle {
153 /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
154 /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
155 pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
156 self.control.shutdown(mode).stopping()
157 }
158}
159
160/// Where the engine runs, besides its config and its GitHub and Docker clients.
161#[derive(Debug)]
162pub struct Setup {
163 /// The names of what runnerd creates in Docker.
164 pub names: Names,
165 /// This machine, which names its runners, so a restarted daemon recognizes them.
166 pub identity: HostIdentity,
167 /// The slice's `MemoryMax`.
168 pub memory_max: Option<u64>,
169 /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
170 pub cpus: Option<u64>,
171 /// The job history.
172 pub store: Store,
173 /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
174 /// announces only the difference, including what the config no longer has.
175 pub announced: Announced,
176 /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
177 pub paused: Vec<String>,
178 /// Where the cgroup and proc file systems are.
179 pub paths: Paths,
180 /// The shared build cache, unless the daemon could not open it.
181 pub build_cache: Option<BuildCache>,
182}
183
184/// Why a runner starts, as its start's log line and its record say (ADR 0092).
185#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186enum Purpose {
187 /// For a queued job.
188 Job,
189 /// As one of its repository's configured warm runners, for jobs not queued yet.
190 Warm,
191 /// As a warm runner past those, for a job its runs are expected to queue still.
192 Burst,
193}
194
195impl Purpose {
196 /// How the history stores it.
197 fn as_str(self) -> &'static str {
198 match self {
199 Purpose::Job => "job",
200 Purpose::Warm => "warm",
201 Purpose::Burst => "burst",
202 }
203 }
204}
205
206/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
207#[derive(Debug, Clone, Copy, PartialEq, Eq)]
208enum Meanwhile {
209 /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
210 Turnover,
211 /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
212 Commands,
213}
214
215/// Whether Docker answers a stop.
216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
217enum DockerInStop {
218 /// It answers, if with errors, or no stop was asked for.
219 Answers,
220 /// It did not answer a call the stop needed: the stop calls it no more.
221 Failed,
222}
223
224/// A slot's runner and its container.
225#[derive(Debug, Clone)]
226struct Occupant {
227 repository: String,
228 runner_name: String,
229 runner_id: u64,
230 container: String,
231 busy: bool,
232 /// Whether the last runners listing was the first to show it busy, or GitHub said so since: the queued jobs
233 /// listing after it may still show the job it took, as GitHub lists one queued for a few seconds after.
234 newly_busy: bool,
235 started_at: SystemTime,
236 /// Its container's memory cap, which a reload does not change.
237 memory: u64,
238 /// Its container's CPU limit in millicores, which a reload does not change.
239 cpus: u64,
240 /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
241 /// one whose record says nothing of it, which reserves its cap.
242 admission: Option<api::Admission>,
243 /// The job it took, once its log capture found it.
244 job: Option<u64>,
245 /// That job as admission tells jobs apart, once the history had its record.
246 key: Option<Key>,
247 /// The labels it registered with; `None` for one adopted from a container that does not say.
248 labels: Option<Vec<String>>,
249 /// The image it runs; `None` for one adopted from a container that does not say.
250 image: Option<String>,
251 /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
252 /// label.
253 spec: Option<String>,
254 /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
255 /// container, which each tick tries again.
256 observed: bool,
257 /// Why this daemon started it; `None` for one a restart adopted.
258 purpose: Option<Purpose>,
259}
260
261impl Occupant {
262 /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
263 /// the profile's. An unknown label set counts as the same.
264 fn outdated(
265 &self,
266 repository: &crate::config::Repository,
267 profile: &crate::config::Profile,
268 ) -> bool {
269 let differ = self.labels.as_ref().is_some_and(|labels| {
270 let set = |labels: &[String]| -> BTreeSet<String> {
271 labels
272 .iter()
273 .map(|label| label.to_ascii_lowercase())
274 .collect()
275 };
276 set(labels) != set(&repository.labels)
277 });
278 differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
279 }
280
281 /// Whether it runs another image than `profile` names. An unknown image counts as the same.
282 fn other_image(&self, profile: &crate::config::Profile) -> bool {
283 self.image
284 .as_ref()
285 .is_some_and(|image| *image != profile.image.reference())
286 }
287
288 /// Whether another release started it, with another container spec: one that does not say is older.
289 fn other_spec(&self) -> bool {
290 self.spec.as_deref() != Some(runners::THIS_SPEC)
291 }
292
293 /// What it reserves of the budget under measured admission: its cap if it has no admission.
294 fn reserved(&self) -> u64 {
295 self.admission
296 .map_or(self.memory, |admission| admission.reserved)
297 }
298
299 /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
300 fn reserved_cpu(&self) -> u64 {
301 self.admission
302 .and_then(|admission| admission.cpu)
303 .map_or(self.cpus, |cpu| cpu.reserved)
304 }
305
306 fn running_job(&self) -> api::RunningJob {
307 api::RunningJob {
308 repository: self.repository.clone(),
309 runner: self.runner_name.clone(),
310 job: self.job,
311 }
312 }
313}
314
315/// A runner gone that may have taken one of its repository's listed jobs, which no capture named.
316#[derive(Debug)]
317struct Unnamed {
318 /// The profile of its pool.
319 profile: String,
320 /// The `Engine::listing` begun last when it went.
321 listing: u64,
322}
323
324/// What a repository's listings may show queued that this machine's runners took (ADR 0018, 0095). It outlives the
325/// repository's removal from the config: a listing begun before the removal may answer after it is added back.
326#[derive(Debug, Default)]
327struct Claims {
328 /// Its runners gone that may have taken a listed job without the engine knowing which: one ended before its
329 /// log capture named its job, or one whose start failed may have run. Each counts against one listed job of its
330 /// pool until a listing begun after it went answers.
331 unnamed: Vec<Unnamed>,
332 /// The jobs this machine's runners took whose runners are gone, by the `listing` begun last when they went:
333 /// GitHub lists a job queued for a while after a runner took it, so a listing begun before that, or one GitHub
334 /// lags on, still shows it. A listing begun after that and showing it no longer drops it.
335 consumed: BTreeMap<u64, u64>,
336}
337
338/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
339#[derive(Debug, Default)]
340struct RepositoryState {
341 queued: Vec<QueuedJob>,
342 /// Whether the last poll of its queued jobs succeeded.
343 listed: bool,
344 /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
345 /// not hold back the others (ADR 0058).
346 holds: BTreeMap<String, PoolHold>,
347 /// By the profile of each of its pools, that pool's oldest job held for CPU that memory and a slot had room
348 /// for, and since when, kept while that job stays queued: a tick held by memory or slots in between does not
349 /// make it younger.
350 cpu_wait: BTreeMap<String, CpuWait>,
351 /// The profiles of its pools whose next job the last plan held for CPU.
352 cpu_held: BTreeSet<String>,
353}
354
355/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
356#[derive(Debug, Default)]
357struct PoolHold {
358 held: Option<Hold>,
359 /// Runners in a row that ended without taking a job.
360 failures: u32,
361 /// Runners in a row that ended without a job after waiting `IDLE_TIMEOUT` for one.
362 idle_exits: u32,
363}
364
365/// A job waiting for CPU, as `RepositoryState::cpu_wait` keeps it.
366#[derive(Debug, Clone, Copy)]
367struct CpuWait {
368 /// The job's ID.
369 job: u64,
370 /// When a plan first held it for CPU.
371 since: SystemTime,
372}
373
374/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
375/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
376/// a single Docker request (120s) could otherwise outlast.
377pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
378
379/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
380#[derive(Debug)]
381pub struct Engine<G = Github, D = Docker> {
382 config: Config,
383 /// Shared with the listings a tick reads while it answers commands.
384 github: Arc<G>,
385 docker: Arc<D>,
386 names: Names,
387 isolation: &'static dyn Isolation,
388 identity: HostIdentity,
389 runner_prefix: String,
390 memory_max: Option<u64>,
391 /// The CPUs the slice may use, in millicores, read at start.
392 cpus: Option<u64>,
393 slots: BTreeMap<String, Vec<Option<Occupant>>>,
394 repositories: BTreeMap<String, RepositoryState>,
395 reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
396 /// The pulled images of the config's profiles, and the removal of those no profile runs.
397 images: Images,
398 /// The memory cap each running reseed's container started with, which a reload does not change.
399 reseed_memory: BTreeMap<String, u64>,
400 /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
401 reseed_requested: BTreeSet<String>,
402 /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
403 draining: BTreeSet<String>,
404 /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
405 /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
406 drained_once: BTreeSet<String>,
407 /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
408 /// each tick tries again.
409 leftover: BTreeMap<String, String>,
410 /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
411 /// failed jobs' records: the first tick after each one's backoff tries again.
412 unread: Vec<recover::Left>,
413 /// Profiles a reload removed, kept while their runners or reseed still run.
414 retired: BTreeMap<String, crate::config::Profile>,
415 /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
416 measure_due: BTreeSet<String>,
417 /// The measures of slot 1 running, by profile.
418 measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
419 /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
420 measured_at: BTreeMap<String, tokio::time::Instant>,
421 /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
422 /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
423 /// `snapshot_max`.
424 reseed_due: BTreeSet<String>,
425 /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
426 /// little slot 1 changed.
427 oversized: BTreeSet<String>,
428 /// When each profile's last reseed finished: a check of the caches that started before then measured the old
429 /// snapshot.
430 reseeded_at: BTreeMap<String, SystemTime>,
431 /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
432 /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
433 ran: BTreeMap<SlotRef, Key>,
434 /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
435 /// `None` until one measured Docker's disk, when each image keeps its own default.
436 docker_disk: watch::Sender<Option<u64>>,
437 /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
438 disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
439 /// Whether a job ended since the shared caches were last checked.
440 trim_due: bool,
441 /// The running check of the shared caches: Docker takes many seconds to measure volumes.
442 trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
443 /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
444 /// unless runners were adopted.
445 checked_at: SystemTime,
446 mirror_up: bool,
447 /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
448 docker_in_stop: DockerInStop,
449 /// The memory cap the running mirror started with.
450 mirror_memory: u64,
451 rate_limit_logged: bool,
452 /// Runners without a container that GitHub still holds busy, already logged.
453 busy_orphans: BTreeSet<u64>,
454 status: watch::Sender<api::Status>,
455 /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
456 /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
457 announced_profiles: Option<Vec<api::ProfileStatus>>,
458 control: Arc<Control>,
459 commands: Commands,
460 caches: watch::Sender<Vec<api::CacheVolume>>,
461 store: Store,
462 paths: Paths,
463 /// The containers the sampler reads.
464 tracked: watch::Sender<Vec<Tracked>>,
465 found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
466 captures: BTreeMap<String, JoinHandle<history::Capture>>,
467 /// The runners whose log capture broke off, until they are gone.
468 recaptures: BTreeMap<String, history::Recapture>,
469 /// Who held which address, for the mirror's log.
470 leases: Leases,
471 /// How far the mirror's log was counted.
472 mirror_until: MirrorPosition,
473 /// Jobs recorded queued and not yet taken or closed, with their repository.
474 queued_jobs: BTreeMap<u64, String>,
475 /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
476 cancelling: BTreeMap<String, tokio::time::Instant>,
477 /// Jobs whose runner is gone, waiting for GitHub to report them completed.
478 closing: BTreeMap<u64, Closing>,
479 last_prune: Option<SystemTime>,
480 /// The records of failed jobs and the shells open in them.
481 records: records::RecordState,
482 /// The predictor and the high-water mark of measured admission, and what admission saw of the host.
483 measured: admission::Measured,
484 /// What warm runners learned from the history, and the workflow runs seen lately.
485 warm: warm::Warm,
486 /// The shared build cache.
487 build_cache: Option<BuildCache>,
488 /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
489 /// cache when `[caches.build] branches` names none.
490 default_branches: BTreeMap<String, build_cache::DefaultBranch>,
491 /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
492 /// `build_cache::ENDED_WAIT`, by runner.
493 ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
494 /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
495 /// the answers.
496 visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
497 /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
498 visibility_asking: Option<JoinHandle<()>>,
499 /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
500 exits: Arc<exits::Exits>,
501 /// The task following Docker's events into `exits`, while the engine schedules.
502 watcher: Option<exits::Watcher>,
503 /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
504 /// pass the event wakes while the tick waits for GitHub starts runners into their room.
505 released: BTreeSet<String>,
506 /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
507 /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
508 passes_in_tick: Duration,
509 /// How many times the engine began listing GitHub's queued jobs: the listing each answer comes from, and how
510 /// old a claim on a listed job is (`Claims`).
511 listing: u64,
512 /// By repository, the jobs its listings may show that this machine's runners took.
513 claims: BTreeMap<String, Claims>,
514 /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
515 /// a name, which GitHub, Docker and the history know them by. A restart takes it from the containers and the
516 /// history, so a name ahead of the clock, or a clock that stepped back, is not used again.
517 named: u64,
518}
519
520fn unix_secs(time: SystemTime) -> u64 {
521 time.duration_since(SystemTime::UNIX_EPOCH)
522 .unwrap_or_default()
523 .as_secs()
524}
525
526impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
527 /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
528 /// them; `memory_max` is the slice's.
529 #[allow(
530 clippy::too_many_lines,
531 reason = "one line per field of the engine's state, which split up would only be listed twice"
532 )]
533 pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
534 let Setup {
535 names,
536 identity,
537 memory_max,
538 cpus,
539 store,
540 announced,
541 paused,
542 paths,
543 build_cache,
544 } = setup;
545 let slots = config
546 .profiles
547 .iter()
548 .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
549 .collect();
550 let repositories = config
551 .repositories
552 .iter()
553 .map(|repository| (repository.name.clone(), RepositoryState::default()))
554 .collect();
555 let (control, commands) = Control::new(
556 config
557 .repositories
558 .iter()
559 .map(|repository| repository.name.clone()),
560 paused,
561 store.clone(),
562 );
563 let control = Arc::new(control);
564 let (status, receiver) = watch::channel(api::Status {
565 version: env!("CARGO_PKG_VERSION").to_owned(),
566 shutting_down: false,
567 stopping: None,
568 budget: api::Budget {
569 slice: config.host.slice.clone(),
570 memory_max,
571 ..api::Budget::default()
572 },
573 profiles: Vec::new(),
574 // What clients resuming after the last run's events have, so the first publish announces the difference.
575 repositories: announced.repositories,
576 queued: Vec::new(),
577 });
578 let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
579 let (records, records_receiver) = records::RecordState::new();
580 let (docker_disk, disk_receiver) = watch::channel(None);
581 let mut engine = Engine {
582 isolation: isolation::backend(config.host.isolation),
583 // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
584 mirror_memory: config.mirror_memory(),
585 config,
586 github: Arc::new(github),
587 docker: Arc::new(docker),
588 names,
589 runner_prefix: identity.prefix(),
590 identity,
591 memory_max,
592 cpus,
593 slots,
594 repositories,
595 reseeds: BTreeMap::new(),
596 images: Images::new(),
597 reseed_memory: BTreeMap::new(),
598 reseed_requested: BTreeSet::new(),
599 retired: BTreeMap::new(),
600 draining: BTreeSet::new(),
601 drained_once: BTreeSet::new(),
602 leftover: BTreeMap::new(),
603 unread: Vec::new(),
604 measure_due: BTreeSet::new(),
605 measures: BTreeMap::new(),
606 measured_at: BTreeMap::new(),
607 reseed_due: BTreeSet::new(),
608 oversized: BTreeSet::new(),
609 reseeded_at: BTreeMap::new(),
610 ran: BTreeMap::new(),
611 docker_disk,
612 disk: None,
613 // So the caches are measured once at start.
614 trim_due: true,
615 trim: None,
616 checked_at: SystemTime::now(),
617 // Unknown: an earlier run may have left it up, so the first idle tick stops it.
618 mirror_up: true,
619 docker_in_stop: DockerInStop::Answers,
620 rate_limit_logged: false,
621 busy_orphans: BTreeSet::new(),
622 status,
623 announced_profiles: Some(announced.profiles),
624 control: control.clone(),
625 commands,
626 caches,
627 store: store.clone(),
628 paths,
629 tracked: watch::channel(Vec::new()).0,
630 found: mpsc::unbounded_channel(),
631 captures: BTreeMap::new(),
632 recaptures: BTreeMap::new(),
633 leases: Leases::default(),
634 mirror_until: MirrorPosition::default(),
635 queued_jobs: BTreeMap::new(),
636 cancelling: BTreeMap::new(),
637 closing: BTreeMap::new(),
638 last_prune: None,
639 measured: admission::Measured::default(),
640 warm: warm::Warm::default(),
641 records,
642 build_cache: build_cache.clone(),
643 default_branches: BTreeMap::new(),
644 ended_uploads: BTreeMap::new(),
645 visibility_due: Arc::default(),
646 visibility_asking: None,
647 exits: Arc::default(),
648 watcher: None,
649 released: BTreeSet::new(),
650 passes_in_tick: Duration::ZERO,
651 listing: 0,
652 named: 0,
653 claims: BTreeMap::new(),
654 };
655 engine.publish();
656 (
657 engine,
658 Handle {
659 status: receiver,
660 control,
661 store,
662 caches: caches_receiver,
663 records: records_receiver,
664 build_cache,
665 disk: disk_receiver,
666 },
667 )
668 }
669
670 /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
671 /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
672 pub async fn run(mut self) {
673 tokio::spawn(metrics::run(
674 Sampler::new(self.paths.clone(), &self.config.host.slice),
675 self.tracked.subscribe(),
676 self.store.clone(),
677 ));
678 let control = self.control.clone();
679 let deadline = async {
680 control.hurried().await;
681 tokio::time::sleep(STOP_DEADLINE).await;
682 };
683 tokio::select! {
684 () = self.serve() => {}
685 () = deadline => warn!(
686 "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
687 STOP_DEADLINE.as_secs()
688 ),
689 }
690 }
691
692 /// Recovers, then schedules until a shutdown request is carried out.
693 async fn serve(&mut self) {
694 // Until Docker answers: scheduling without knowing the containers already running would start a second
695 // runner on a slot's volume and delete the running one's registration.
696 while let Err(error) = self.recover().await {
697 warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
698 if self.control.mode() != Shutdown::No {
699 return;
700 }
701 tokio::select! {
702 () = tokio::time::sleep(Duration::from_secs(5)) => {}
703 () = self.control.wake.notified() => {}
704 }
705 }
706 self.schedule().await;
707 }
708
709 /// Schedules until a shutdown request is carried out, from a recovered engine. Between ticks, a runner whose
710 /// container Docker says stopped is released at once, and its slot taken by a job the last poll listed
711 /// (`turnover`, ADR 0095); that leaves the next tick when it was due.
712 async fn schedule(&mut self) {
713 self.watcher = Some(exits::Watcher::start(
714 self.docker.clone(),
715 self.exits.clone(),
716 ));
717 let mut shown = Shutdown::No;
718 let mode = loop {
719 let mode = self.stopping();
720 // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
721 // Docker: until then status would show a daemon that is not stopping.
722 if mode != shown {
723 self.publish();
724 shown = mode;
725 }
726 if mode == Shutdown::Detach {
727 self.remove_every_idle().await;
728 self.collect_recordings().await;
729 } else {
730 self.tick(mode).await;
731 }
732 self.publish();
733 // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
734 // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
735 if self.stopping() != mode {
736 continue;
737 }
738 if self.stopped(mode) {
739 break mode;
740 }
741 let interval = if mode == Shutdown::No {
742 self.config.host.poll_interval
743 } else {
744 Duration::from_secs(5)
745 };
746 self.wait_for_tick(mode, tokio::time::Instant::now() + interval)
747 .await;
748 };
749 // Nothing more is started or released: what Docker says now is the next start's.
750 self.watcher = None;
751 // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
752 self.collect_reseeds().await;
753 if self.docker_in_stop == DockerInStop::Failed {
754 info!("stopped; the next start removes or adopts what Docker did not answer for");
755 return;
756 }
757 self.close_shells().await;
758 if mode == Shutdown::Detach {
759 // The mirror stays up only for the busy runners left behind.
760 let mirror = if self.config.caches.mirror.enabled {
761 " and the mirror"
762 } else {
763 ""
764 };
765 match self.occupied().count() {
766 0 => info!("stopped; no job was running"),
767 1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
768 runners => {
769 info!("stopped, leaving {runners} busy runners{mirror} to the next start");
770 }
771 }
772 return;
773 }
774 if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
775 warn!("cannot stop the mirror: {error}");
776 }
777 info!("stopped");
778 }
779
780 /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
781 /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
782 /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
783 async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) {
784 loop {
785 let exits = self.exits.clone();
786 // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
787 // keep turning over.
788 tokio::select! {
789 biased;
790 () = tokio::time::sleep_until(due) => return,
791 () = self.control.wake.notified() => return,
792 // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
793 Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
794 self.command(command).await;
795 self.publish();
796 return;
797 }
798 // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
799 // it was due. A stop returns to see whether it is done.
800 Some(()) = self.records.finished.recv() => {
801 self.collect_recordings().await;
802 self.publish();
803 if mode != Shutdown::No || self.stopping() != mode {
804 return;
805 }
806 }
807 // A capture named its runner's job: status says the runner runs it now, not after the next poll.
808 Some(found) = self.found.1.recv() => {
809 self.note_found(&found);
810 self.publish();
811 }
812 () = exits.noted(), if mode != Shutdown::Detach => {
813 if mode != Shutdown::No {
814 return;
815 }
816 self.turnover().await;
817 self.publish();
818 if self.stopping() != mode {
819 return;
820 }
821 }
822 }
823 }
824 }
825
826 /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
827 fn stopping(&self) -> Shutdown {
828 match self.control.mode() {
829 Shutdown::No => Shutdown::No,
830 _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
831 mode => mode,
832 }
833 }
834
835 /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
836 /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
837 fn stopped(&self, mode: Shutdown) -> bool {
838 let idle = self.slots.values().flatten().all(Option::is_none);
839 let recorded = self.records.recordings.is_empty();
840 match mode {
841 Shutdown::No => false,
842 Shutdown::Detach => recorded,
843 Shutdown::Graceful | Shutdown::Force => idle && recorded,
844 }
845 }
846
847 /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
848 /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
849 /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
850 /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
851 fn docker_failed(&mut self, error: &docker::Error) {
852 if self.control.mode() != Shutdown::No
853 && self.docker_in_stop == DockerInStop::Answers
854 && docker::is_unreachable(error)
855 {
856 warn!("Docker does not answer during the stop; leaving the rest to the next start");
857 self.docker_in_stop = DockerInStop::Failed;
858 // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
859 // running jobs.
860 self.control.shutdown(Shutdown::Detach);
861 }
862 }
863
864 /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
865 /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
866 /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
867 /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
868 /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
869 /// while, and the jobs captures name are noted as they come.
870 async fn serving<T: Send + 'static>(
871 &mut self,
872 work: impl Future<Output = T> + Send + 'static,
873 meanwhile: Meanwhile,
874 ) -> T {
875 let mut task = tokio::spawn(work);
876 loop {
877 let exits = self.exits.clone();
878 tokio::select! {
879 biased;
880 done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
881 Ok(output) => return output,
882 // As if it had panicked here.
883 Err(Ok(panic)) => std::panic::resume_unwind(panic),
884 // Only while the runtime shuts down, which drops this task too.
885 Err(Err(_)) => std::future::pending::<()>().await,
886 },
887 Some(command) = self.commands.recv() => {
888 self.command(command).await;
889 self.publish();
890 // A tick follows it, as one follows a command answered between ticks: a config it replaced
891 // polls its repositories then, not a poll interval later.
892 self.control.wake.notify_one();
893 }
894 () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
895 let began = tokio::time::Instant::now();
896 self.turnover().await;
897 self.passes_in_tick += began.elapsed();
898 self.publish();
899 }
900 Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
901 self.note_found(&found);
902 self.publish();
903 }
904 }
905 }
906 }
907
908 fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
909 self.slots.iter().flat_map(|(profile, slots)| {
910 slots.iter().enumerate().filter_map(|(index, slot)| {
911 slot.as_ref().map(|occupant| {
912 (
913 SlotRef {
914 profile: profile.clone(),
915 index,
916 },
917 occupant,
918 )
919 })
920 })
921 })
922 }
923
924 /// The runner in `slot`, if one is.
925 fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
926 self.slots
927 .get(&slot.profile)
928 .and_then(|slots| slots.get(slot.index))
929 .and_then(Option::as_ref)
930 }
931
932 /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
933 /// runs in them.
934 fn drop_closed_slots(&mut self) {
935 for (name, slots) in &mut self.slots {
936 let count = self
937 .config
938 .profiles
939 .get(name)
940 .map_or(0, |profile| profile.slots as usize);
941 while slots.len() > count && slots.last().is_some_and(Option::is_none) {
942 slots.pop();
943 }
944 }
945 let profiles = &self.config.profiles;
946 self.slots
947 .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
948 // The volumes of the slots dropped go at the next check of the caches.
949 let slots = &self.slots;
950 self.ran.retain(|slot, _| {
951 slots
952 .get(&slot.profile)
953 .is_some_and(|slots| slot.index < slots.len())
954 });
955 let (slots, reseeds) = (&self.slots, &self.reseeds);
956 self.retired
957 .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
958 }
959}
960
961#[cfg(test)]
962mod tests;