runnerd_core/scheduler/engine.rs
1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `build_cache` (each
11//! runner's side of the shared build cache and whether its job may write), `history` (what
12//! the store records), `warm` (how many warm runners each repository wants), `control` (the API's commands) and
13//! `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod clock;
19mod control;
20mod exits;
21mod history;
22mod images;
23mod records;
24mod recover;
25mod runners;
26mod status;
27mod tick;
28mod warm;
29
30use std::collections::{BTreeMap, BTreeSet};
31use std::sync::{Arc, Mutex};
32use std::time::{Duration, SystemTime};
33
34use tokio::sync::{mpsc, watch};
35use tokio::task::JoinHandle;
36use tracing::{info, warn};
37
38use self::clock::Clock;
39pub use self::control::{
40 Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
41 Target, same_slice,
42};
43use self::history::{CAPTURE_DRAIN, Closing, Found};
44use self::images::Images;
45pub use self::records::RecordSelection;
46use super::{Hold, SlotRef, Waits};
47use crate::admission::Key;
48use crate::api;
49use crate::buildcache::BuildCache;
50use crate::cache::{self, Names, Trimmed};
51use crate::config::Config;
52use crate::docker::{self, Docker, DockerApi};
53use crate::github::{Github, GithubApi, QueuedJob};
54use crate::isolation::{self, Isolation};
55use crate::observe::metrics::{self, Paths, Sampler, Tracked};
56use crate::observe::mirror::Leases;
57use crate::store::{Announced, MirrorPosition, Store};
58
59/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
60/// identifies the machine, so a changed host name still matches the runners started before the change, and two
61/// machines with the same or overlapping host names never take each other's runners for their own.
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct HostIdentity {
64 /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
65 id: String,
66 /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
67 host: String,
68}
69
70impl HostIdentity {
71 /// This machine's identity, from `/etc/machine-id` and its host name.
72 pub fn read() -> std::io::Result<HostIdentity> {
73 let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
74 let machine_id = std::fs::read_to_string("/etc/machine-id")
75 .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
76 .map_err(|error| {
77 std::io::Error::new(
78 error.kind(),
79 format!("cannot read /etc/machine-id: {error}"),
80 )
81 })?;
82 HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
83 std::io::Error::new(
84 std::io::ErrorKind::InvalidData,
85 "/etc/machine-id does not hold a machine ID",
86 )
87 })
88 }
89
90 /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
91 pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
92 let machine_id = machine_id.trim();
93 if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
94 return None;
95 }
96 // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
97 let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
98 .bytes()
99 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
100 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
101 });
102 let host: String = hostname
103 .trim()
104 .chars()
105 .map(|c| {
106 if c.is_ascii_alphanumeric() {
107 c.to_ascii_lowercase()
108 } else {
109 '-'
110 }
111 })
112 .take(20)
113 .collect();
114 let host = host.trim_matches('-');
115 Some(HostIdentity {
116 id: format!("{:08x}", hash >> 32),
117 host: if host.is_empty() { "host" } else { host }.to_owned(),
118 })
119 }
120
121 /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
122 /// machine's names.
123 pub fn prefix(&self) -> String {
124 format!("runnerd-{}-", self.id)
125 }
126
127 /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
128 pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
129 format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
130 }
131}
132
133/// What the API needs from a running engine.
134#[derive(Debug, Clone)]
135pub struct Handle {
136 /// The latest status, replaced after every tick.
137 pub status: watch::Receiver<api::Status>,
138 /// Pause, resume and shutdown requests.
139 pub control: Arc<Control>,
140 /// The job history.
141 pub store: Store,
142 /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
143 /// config is.
144 pub caches: watch::Receiver<Vec<api::CacheVolume>>,
145 /// The records of failed jobs, replaced whenever one is made or removed.
146 pub records: watch::Receiver<api::Records>,
147 /// The shared build cache, for its size now.
148 pub build_cache: Option<BuildCache>,
149 /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
150 /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
151 pub disk: watch::Receiver<Option<u64>>,
152}
153
154impl Handle {
155 /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
156 /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
157 pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
158 self.control.shutdown(mode).stopping()
159 }
160}
161
162/// Where the engine runs, besides its config and its GitHub and Docker clients.
163#[derive(Debug)]
164pub struct Setup {
165 /// The names of what runnerd creates in Docker.
166 pub names: Names,
167 /// This machine, which names its runners, so a restarted daemon recognizes them.
168 pub identity: HostIdentity,
169 /// The slice's `MemoryMax`.
170 pub memory_max: Option<u64>,
171 /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
172 pub cpus: Option<u64>,
173 /// The job history.
174 pub store: Store,
175 /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
176 /// announces only the difference, including what the config no longer has.
177 pub announced: Announced,
178 /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
179 pub paused: Vec<String>,
180 /// Where the cgroup and proc file systems are.
181 pub paths: Paths,
182 /// The shared build cache, unless the daemon could not open it.
183 pub build_cache: Option<BuildCache>,
184}
185
186/// Why a runner starts, as its start's log line and its record say (ADR 0092).
187#[derive(Debug, Clone, Copy, PartialEq, Eq)]
188enum Purpose {
189 /// For a queued job.
190 Job,
191 /// As one of its repository's configured warm runners, for jobs not queued yet.
192 Warm,
193 /// As a warm runner past those, for a job its runs are expected to queue still.
194 Burst,
195}
196
197impl Purpose {
198 /// How the history stores it.
199 fn as_str(self) -> &'static str {
200 match self {
201 Purpose::Job => "job",
202 Purpose::Warm => "warm",
203 Purpose::Burst => "burst",
204 }
205 }
206}
207
208/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
209#[derive(Debug, Clone, Copy, PartialEq, Eq)]
210enum Meanwhile {
211 /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
212 Turnover,
213 /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
214 Commands,
215}
216
217/// Whether Docker answers a stop.
218#[derive(Debug, Clone, Copy, PartialEq, Eq)]
219enum DockerInStop {
220 /// It answers, if with errors, or no stop was asked for.
221 Answers,
222 /// It did not answer a call the stop needed: the stop calls it no more.
223 Failed,
224}
225
226/// A slot's runner and its container.
227#[derive(Debug, Clone)]
228struct Occupant {
229 repository: String,
230 runner_name: String,
231 runner_id: u64,
232 container: String,
233 busy: bool,
234 /// Whether the last runners listing was the first to show it busy, or GitHub said so since: the queued jobs
235 /// listing after it may still show the job it took, as GitHub lists one queued for a few seconds after.
236 newly_busy: bool,
237 started_at: SystemTime,
238 /// Its container's memory cap, which a reload does not change.
239 memory: u64,
240 /// Its container's CPU limit in millicores, which a reload does not change.
241 cpus: u64,
242 /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
243 /// one whose record says nothing of it, which reserves its cap.
244 admission: Option<api::Admission>,
245 /// The job it took, once its log capture found it.
246 job: Option<u64>,
247 /// When this daemon's capture found that job; `None` before, and for a job found again after a restart only by
248 /// an adopted runner, whose `purpose` says so.
249 job_started: Option<SystemTime>,
250 /// That job as admission tells jobs apart, once the history had its record.
251 key: Option<Key>,
252 /// The labels it registered with; `None` for one adopted from a container that does not say.
253 labels: Option<Vec<String>>,
254 /// The image it runs; `None` for one adopted from a container that does not say.
255 image: Option<String>,
256 /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
257 /// label.
258 spec: Option<String>,
259 /// Whether `observe` watches it: recorded, sampled and its job captured. Not while Docker fails to inspect its
260 /// container, which each tick tries again.
261 observed: bool,
262 /// Why this daemon started it; `None` for one a restart adopted.
263 purpose: Option<Purpose>,
264}
265
266impl Occupant {
267 /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
268 /// the profile's. An unknown label set counts as the same.
269 fn outdated(
270 &self,
271 repository: &crate::config::Repository,
272 profile: &crate::config::Profile,
273 ) -> bool {
274 let differ = self.labels.as_ref().is_some_and(|labels| {
275 let set = |labels: &[String]| -> BTreeSet<String> {
276 labels
277 .iter()
278 .map(|label| label.to_ascii_lowercase())
279 .collect()
280 };
281 set(labels) != set(&repository.labels)
282 });
283 differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
284 }
285
286 /// Whether it runs another image than `profile` names. An unknown image counts as the same.
287 fn other_image(&self, profile: &crate::config::Profile) -> bool {
288 self.image
289 .as_ref()
290 .is_some_and(|image| *image != profile.image.reference())
291 }
292
293 /// Whether another release started it, with another container spec: one that does not say is older.
294 fn other_spec(&self) -> bool {
295 self.spec.as_deref() != Some(runners::THIS_SPEC)
296 }
297
298 /// What it reserves of the budget under measured admission: its cap if it has no admission.
299 fn reserved(&self) -> u64 {
300 self.admission
301 .map_or(self.memory, |admission| admission.reserved)
302 }
303
304 /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
305 fn reserved_cpu(&self) -> u64 {
306 self.admission
307 .and_then(|admission| admission.cpu)
308 .map_or(self.cpus, |cpu| cpu.reserved)
309 }
310
311 fn running_job(&self) -> api::RunningJob {
312 api::RunningJob {
313 repository: self.repository.clone(),
314 runner: self.runner_name.clone(),
315 job: self.job,
316 }
317 }
318}
319
320/// A runner gone that may have taken one of its repository's listed jobs, which no capture named.
321#[derive(Debug)]
322struct Unnamed {
323 /// The profile of its pool.
324 profile: String,
325 /// The `Engine::listing` begun last when it went.
326 listing: u64,
327}
328
329/// What a repository's listings may show queued that this machine's runners took (ADR 0018, 0095). It outlives the
330/// repository's removal from the config: a listing begun before the removal may answer after it is added back.
331#[derive(Debug, Default)]
332struct Claims {
333 /// Its runners gone that may have taken a listed job without the engine knowing which: one ended before its
334 /// log capture named its job, or one whose start failed may have run. Each counts against one listed job of its
335 /// pool until a listing begun after it went answers.
336 unnamed: Vec<Unnamed>,
337 /// The jobs this machine's runners took whose runners are gone, by the `listing` begun last when they went:
338 /// GitHub lists a job queued for a while after a runner took it, so a listing begun before that, or one GitHub
339 /// lags on, still shows it. A listing begun after that and showing it no longer drops it.
340 consumed: BTreeMap<u64, u64>,
341}
342
343/// A repository's state, shared by its pools but for their holds and what they wait for CPU.
344#[derive(Debug, Default)]
345struct RepositoryState {
346 queued: Vec<QueuedJob>,
347 /// Whether the last poll of its queued jobs succeeded.
348 listed: bool,
349 /// By the profile of each of its pools, that pool's runners held back after they failed: a broken pool does
350 /// not hold back the others (ADR 0058).
351 holds: BTreeMap<String, PoolHold>,
352 /// By the profile of each of its pools, what that pool's next job has waited for and since when (ADR 0096): from
353 /// the first plan of a tick or a pass that held it, not one a status runs, until a runner of the pool is in its
354 /// slot, the pool may no longer start one, or a listing that succeeded no longer shows the job. A plan that holds
355 /// it for a slot or for the other resource does not make it younger, nor does a listing that failed. With them,
356 /// whether a barrier has served the pool (ADR 0098), until a listing that succeeded shows its queue empty or the
357 /// pool may no longer start a runner. Not kept over a restart.
358 waits: BTreeMap<String, Waits>,
359 /// The profiles of its pools whose next job the last plan held for CPU.
360 cpu_held: BTreeSet<String>,
361}
362
363/// A pool's runners held back after they failed, as `RepositoryState::holds` keeps them.
364#[derive(Debug, Default)]
365struct PoolHold {
366 held: Option<Hold>,
367 /// Runners in a row that ended without taking a job.
368 failures: u32,
369 /// Runners in a row that connected and ended without a job, or waited `IDLE_TIMEOUT` for one (ADR 0104). Those
370 /// `left_beside` holds count with the one before them.
371 idle_exits: u32,
372 /// The pool's other runners when the last of those ended: already up, they say nothing of a later start.
373 left_beside: BTreeSet<String>,
374}
375
376/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
377/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
378/// a single Docker request (120s) could otherwise outlast.
379pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
380
381/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
382#[derive(Debug)]
383pub struct Engine<G = Github, D = Docker> {
384 config: Config,
385 /// Shared with the listings a tick reads while it answers commands.
386 github: Arc<G>,
387 docker: Arc<D>,
388 names: Names,
389 isolation: &'static dyn Isolation,
390 identity: HostIdentity,
391 runner_prefix: String,
392 /// What a sample's or a host reading's age goes by.
393 clock: Clock,
394 memory_max: Option<u64>,
395 /// The CPUs the slice may use, in millicores, read at start.
396 cpus: Option<u64>,
397 slots: BTreeMap<String, Vec<Option<Occupant>>>,
398 repositories: BTreeMap<String, RepositoryState>,
399 reseeds: BTreeMap<String, JoinHandle<Result<String, docker::Error>>>,
400 /// The pulled images of the config's profiles, and the removal of those no profile runs.
401 images: Images,
402 /// The memory cap each running reseed's container started with, which a reload does not change.
403 reseed_memory: BTreeMap<String, u64>,
404 /// Profiles the API asked to reseed: they start nothing until their slots are free and the reseed ran.
405 reseed_requested: BTreeSet<String>,
406 /// Repositories a reload removed, polled until none of this machine's runners is registered on them.
407 draining: BTreeSet<String>,
408 /// Draining repositories whose last runners listing had none of this machine's runners. A runner can go unseen
409 /// in one listing whose pages shift while they are read, so a repository stops draining only after two.
410 drained_once: BTreeSet<String>,
411 /// The runner and shell containers an earlier run left that recovery failed to remove, by ID with their names:
412 /// each tick tries again.
413 leftover: BTreeMap<String, String>,
414 /// The stopped runner containers an earlier run left that Docker could not tell recovery whether to keep as their
415 /// failed jobs' records: the first tick after each one's backoff tries again.
416 unread: Vec<recover::Left>,
417 /// Profiles a reload removed, kept while their runners or reseed still run.
418 retired: BTreeMap<String, crate::config::Profile>,
419 /// Profiles whose slot 1 ran a job since their last measure or reseed (ADR 0094).
420 measure_due: BTreeSet<String>,
421 /// The measures of slot 1 running, by profile.
422 measures: BTreeMap<String, JoinHandle<Result<cache::Measured, docker::Error>>>,
423 /// When each profile's last measure started: the next waits `MEASURE_EVERY`.
424 measured_at: BTreeMap<String, tokio::time::Instant>,
425 /// Profiles whose upkeep reseeds once their runners are idle: a measure found slot 1 changed `reseed_at` or more,
426 /// or the last reseed unfinished, or could not tell, or the last check of the caches found the snapshot over
427 /// `snapshot_max`.
428 reseed_due: BTreeSet<String>,
429 /// Profiles whose snapshot the last check of the caches measured over `snapshot_max`: due, and reseeded however
430 /// little slot 1 changed.
431 oversized: BTreeSet<String>,
432 /// When each profile's last reseed finished: a check of the caches that started before then measured the old
433 /// snapshot.
434 reseeded_at: BTreeMap<String, SystemTime>,
435 /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
436 /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
437 ran: BTreeMap<SlotRef, Key>,
438 /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
439 /// `None` until one measured Docker's disk, when each image keeps its own default.
440 docker_disk: watch::Sender<Option<u64>>,
441 /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
442 disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
443 /// Whether a job ended since the shared caches were last checked.
444 trim_due: bool,
445 /// The running check of the shared caches: Docker takes many seconds to measure volumes.
446 trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
447 /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
448 /// unless runners were adopted.
449 checked_at: SystemTime,
450 mirror_up: bool,
451 /// Whether Docker stopped answering a stop: the stop then leaves the rest to the next start, as a detach does.
452 docker_in_stop: DockerInStop,
453 /// The memory cap the running mirror started with.
454 mirror_memory: u64,
455 rate_limit_logged: bool,
456 /// Runners without a container that GitHub still holds busy, already logged.
457 busy_orphans: BTreeSet<u64>,
458 /// Runners this engine released or killed, by ID, with their repository, until a listing of that repository no
459 /// longer has them: GitHub holding one busy then is no orphan to warn of.
460 ended_busy: BTreeMap<u64, String>,
461 status: watch::Sender<api::Status>,
462 /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
463 /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
464 announced_profiles: Option<Vec<api::ProfileStatus>>,
465 control: Arc<Control>,
466 commands: Commands,
467 caches: watch::Sender<Vec<api::CacheVolume>>,
468 store: Store,
469 paths: Paths,
470 /// The containers the sampler reads.
471 tracked: watch::Sender<Vec<Tracked>>,
472 found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
473 captures: BTreeMap<String, JoinHandle<history::Capture>>,
474 /// The runners whose log capture broke off, until they are gone.
475 recaptures: BTreeMap<String, history::Recapture>,
476 /// Who held which address, for the mirror's log.
477 leases: Leases,
478 /// How far the mirror's log was counted.
479 mirror_until: MirrorPosition,
480 /// Jobs recorded queued and not yet taken or closed, with their repository.
481 queued_jobs: BTreeMap<u64, String>,
482 /// Runners of runs a forced pause cancelled, with when they are removed if they still run.
483 cancelling: BTreeMap<String, tokio::time::Instant>,
484 /// Jobs whose runner is gone, waiting for GitHub to report them completed.
485 closing: BTreeMap<u64, Closing>,
486 last_prune: Option<SystemTime>,
487 /// The records of failed jobs and the shells open in them.
488 records: records::RecordState,
489 /// The predictor, the high-water mark of measured admission, and what admission saw of the host.
490 measured: admission::Measured,
491 /// What warm runners learned from the history, and the workflow runs seen lately.
492 warm: warm::Warm,
493 /// The shared build cache.
494 build_cache: Option<BuildCache>,
495 /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
496 /// cache when `[caches.build] branches` names none.
497 default_branches: BTreeMap<String, build_cache::DefaultBranch>,
498 /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
499 /// `build_cache::ENDED_WAIT`, by runner.
500 ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
501 /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
502 /// the answers.
503 visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
504 /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
505 visibility_asking: Option<JoinHandle<()>>,
506 /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
507 exits: Arc<exits::Exits>,
508 /// The task following Docker's events into `exits`, while the engine schedules.
509 watcher: Option<exits::Watcher>,
510 /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
511 /// pass the event wakes while the tick waits for GitHub starts runners into their room.
512 released: BTreeSet<String>,
513 /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
514 /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
515 passes_in_tick: Duration,
516 /// How many times the engine began listing GitHub's queued jobs: the listing each answer comes from, and how
517 /// old a claim on a listed job is (`Claims`).
518 listing: u64,
519 /// By repository, the jobs its listings may show that this machine's runners took.
520 claims: BTreeMap<String, Claims>,
521 /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
522 /// a name, which GitHub, Docker and the history know them by. A restart takes it from the containers and the
523 /// history, so a name ahead of the clock, or a clock that stepped back, is not used again.
524 named: u64,
525}
526
527fn unix_secs(time: SystemTime) -> u64 {
528 time.duration_since(SystemTime::UNIX_EPOCH)
529 .unwrap_or_default()
530 .as_secs()
531}
532
533impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
534 /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
535 /// them; `memory_max` is the slice's.
536 #[allow(
537 clippy::too_many_lines,
538 reason = "one line per field of the engine's state, which split up would only be listed twice"
539 )]
540 pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
541 let Setup {
542 names,
543 identity,
544 memory_max,
545 cpus,
546 store,
547 announced,
548 paused,
549 paths,
550 build_cache,
551 } = setup;
552 let slots = config
553 .profiles
554 .iter()
555 .map(|(name, profile)| (name.clone(), vec![None; profile.slots as usize]))
556 .collect();
557 let repositories = config
558 .repositories
559 .iter()
560 .map(|repository| (repository.name.clone(), RepositoryState::default()))
561 .collect();
562 let (control, commands) = Control::new(
563 config
564 .repositories
565 .iter()
566 .map(|repository| repository.name.clone()),
567 paused,
568 store.clone(),
569 );
570 let control = Arc::new(control);
571 let (status, receiver) = watch::channel(api::Status {
572 version: env!("CARGO_PKG_VERSION").to_owned(),
573 shutting_down: false,
574 stopping: None,
575 budget: api::Budget {
576 slice: config.host.slice.clone(),
577 memory_max,
578 ..api::Budget::default()
579 },
580 profiles: Vec::new(),
581 // What clients resuming after the last run's events have, so the first publish announces the difference.
582 repositories: announced.repositories,
583 queued: Vec::new(),
584 });
585 let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
586 let (records, records_receiver) = records::RecordState::new();
587 let (docker_disk, disk_receiver) = watch::channel(None);
588 let mut engine = Engine {
589 clock: Clock::default(),
590 isolation: isolation::backend(config.host.isolation),
591 // Unknown: an earlier run may have left the mirror up, with the configured cap at best.
592 mirror_memory: config.mirror_memory(),
593 config,
594 github: Arc::new(github),
595 docker: Arc::new(docker),
596 names,
597 runner_prefix: identity.prefix(),
598 identity,
599 memory_max,
600 cpus,
601 slots,
602 repositories,
603 reseeds: BTreeMap::new(),
604 images: Images::new(),
605 reseed_memory: BTreeMap::new(),
606 reseed_requested: BTreeSet::new(),
607 retired: BTreeMap::new(),
608 draining: BTreeSet::new(),
609 drained_once: BTreeSet::new(),
610 leftover: BTreeMap::new(),
611 unread: Vec::new(),
612 measure_due: BTreeSet::new(),
613 measures: BTreeMap::new(),
614 measured_at: BTreeMap::new(),
615 reseed_due: BTreeSet::new(),
616 oversized: BTreeSet::new(),
617 reseeded_at: BTreeMap::new(),
618 ran: BTreeMap::new(),
619 docker_disk,
620 disk: None,
621 // So the caches are measured once at start.
622 trim_due: true,
623 trim: None,
624 checked_at: SystemTime::now(),
625 // Unknown: an earlier run may have left it up, so the first idle tick stops it.
626 mirror_up: true,
627 docker_in_stop: DockerInStop::Answers,
628 rate_limit_logged: false,
629 busy_orphans: BTreeSet::new(),
630 ended_busy: BTreeMap::new(),
631 status,
632 announced_profiles: Some(announced.profiles),
633 control: control.clone(),
634 commands,
635 caches,
636 store: store.clone(),
637 paths,
638 tracked: watch::channel(Vec::new()).0,
639 found: mpsc::unbounded_channel(),
640 captures: BTreeMap::new(),
641 recaptures: BTreeMap::new(),
642 leases: Leases::default(),
643 mirror_until: MirrorPosition::default(),
644 queued_jobs: BTreeMap::new(),
645 cancelling: BTreeMap::new(),
646 closing: BTreeMap::new(),
647 last_prune: None,
648 measured: admission::Measured::default(),
649 warm: warm::Warm::default(),
650 records,
651 build_cache: build_cache.clone(),
652 default_branches: BTreeMap::new(),
653 ended_uploads: BTreeMap::new(),
654 visibility_due: Arc::default(),
655 visibility_asking: None,
656 exits: Arc::default(),
657 watcher: None,
658 released: BTreeSet::new(),
659 passes_in_tick: Duration::ZERO,
660 listing: 0,
661 named: 0,
662 claims: BTreeMap::new(),
663 };
664 engine.publish();
665 (
666 engine,
667 Handle {
668 status: receiver,
669 control,
670 store,
671 caches: caches_receiver,
672 records: records_receiver,
673 build_cache,
674 disk: disk_receiver,
675 },
676 )
677 }
678
679 /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
680 /// began, whichever comes first. Either way it returns, so the daemon still closes its history.
681 pub async fn run(mut self) {
682 tokio::spawn(metrics::run(
683 Sampler::new(self.paths.clone(), &self.config.host.slice),
684 self.tracked.subscribe(),
685 self.store.clone(),
686 ));
687 let control = self.control.clone();
688 let deadline = async {
689 control.hurried().await;
690 tokio::time::sleep(STOP_DEADLINE).await;
691 };
692 tokio::select! {
693 () = self.serve() => {}
694 () = deadline => warn!(
695 "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
696 STOP_DEADLINE.as_secs()
697 ),
698 }
699 }
700
701 /// Recovers, then schedules until a shutdown request is carried out.
702 async fn serve(&mut self) {
703 // Until Docker answers: scheduling without knowing the containers already running would start a second
704 // runner on a slot's volume and delete the running one's registration.
705 while let Err(error) = self.recover().await {
706 warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
707 if self.control.mode() != Shutdown::No {
708 return;
709 }
710 tokio::select! {
711 () = tokio::time::sleep(Duration::from_secs(5)) => {}
712 () = self.control.wake.notified() => {}
713 }
714 }
715 self.schedule().await;
716 }
717
718 /// Schedules until a shutdown request is carried out, from a recovered engine. Between ticks, a runner whose
719 /// container Docker says stopped is released at once, and its slot taken by a job the last poll listed
720 /// (`turnover`, ADR 0095); that leaves the next tick when it was due.
721 async fn schedule(&mut self) {
722 self.watcher = Some(exits::Watcher::start(
723 self.docker.clone(),
724 self.exits.clone(),
725 ));
726 let mut shown = Shutdown::No;
727 let mode = loop {
728 let mode = self.stopping();
729 // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
730 // Docker: until then status would show a daemon that is not stopping.
731 if mode != shown {
732 self.publish();
733 shown = mode;
734 }
735 if mode == Shutdown::Detach {
736 self.remove_every_idle().await;
737 self.collect_recordings().await;
738 } else {
739 self.tick(mode).await;
740 }
741 self.publish();
742 // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
743 // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
744 if self.stopping() != mode {
745 continue;
746 }
747 if self.stopped(mode) {
748 break mode;
749 }
750 let interval = if mode == Shutdown::No {
751 self.config.host.poll_interval
752 } else {
753 Duration::from_secs(5)
754 };
755 self.wait_for_tick(mode, tokio::time::Instant::now() + interval)
756 .await;
757 };
758 // Nothing more is started or released: what Docker says now is the next start's.
759 self.watcher = None;
760 // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
761 self.collect_reseeds().await;
762 if self.docker_in_stop == DockerInStop::Failed {
763 info!("stopped; the next start removes or adopts what Docker did not answer for");
764 return;
765 }
766 self.close_shells().await;
767 if mode == Shutdown::Detach {
768 // The mirror stays up only for the busy runners left behind.
769 let mirror = if self.config.caches.mirror.enabled {
770 " and the mirror"
771 } else {
772 ""
773 };
774 match self.occupied().count() {
775 0 => info!("stopped; no job was running"),
776 1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
777 runners => {
778 info!("stopped, leaving {runners} busy runners{mirror} to the next start");
779 }
780 }
781 return;
782 }
783 if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
784 warn!("cannot stop the mirror: {error}");
785 }
786 info!("stopped");
787 }
788
789 /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
790 /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
791 /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
792 async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) {
793 loop {
794 let exits = self.exits.clone();
795 // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
796 // keep turning over.
797 tokio::select! {
798 biased;
799 () = tokio::time::sleep_until(due) => return,
800 () = self.control.wake.notified() => return,
801 // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
802 Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
803 self.command(command).await;
804 self.publish();
805 return;
806 }
807 // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
808 // it was due. A stop returns to see whether it is done.
809 Some(()) = self.records.finished.recv() => {
810 self.collect_recordings().await;
811 self.publish();
812 if mode != Shutdown::No || self.stopping() != mode {
813 return;
814 }
815 }
816 // A capture named its runner's job: status says the runner runs it now, not after the next poll.
817 Some(found) = self.found.1.recv() => {
818 self.note_found(&found);
819 self.publish();
820 }
821 () = exits.noted(), if mode != Shutdown::Detach => {
822 if mode != Shutdown::No {
823 return;
824 }
825 self.turnover().await;
826 self.publish();
827 if self.stopping() != mode {
828 return;
829 }
830 }
831 }
832 }
833 }
834
835 /// How the stop goes on: as requested, or as a detach once Docker stopped answering it.
836 fn stopping(&self) -> Shutdown {
837 match self.control.mode() {
838 Shutdown::No => Shutdown::No,
839 _ if self.docker_in_stop == DockerInStop::Failed => Shutdown::Detach,
840 mode => mode,
841 }
842 }
843
844 /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
845 /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
846 fn stopped(&self, mode: Shutdown) -> bool {
847 let idle = self.slots.values().flatten().all(Option::is_none);
848 let recorded = self.records.recordings.is_empty();
849 match mode {
850 Shutdown::No => false,
851 Shutdown::Detach => recorded,
852 Shutdown::Graceful | Shutdown::Force => idle && recorded,
853 }
854 }
855
856 /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
857 /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
858 /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
859 /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
860 fn docker_failed(&mut self, error: &docker::Error) {
861 if self.control.mode() != Shutdown::No
862 && self.docker_in_stop == DockerInStop::Answers
863 && docker::is_unreachable(error)
864 {
865 warn!("Docker does not answer during the stop; leaving the rest to the next start");
866 self.docker_in_stop = DockerInStop::Failed;
867 // A detach in the control too: its deadline starts, and a later `runnerd stop` answers that it leaves
868 // running jobs.
869 self.control.shutdown(Shutdown::Detach);
870 }
871 }
872
873 /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
874 /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
875 /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
876 /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
877 /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
878 /// while, and the jobs captures name are noted as they come.
879 async fn serving<T: Send + 'static>(
880 &mut self,
881 work: impl Future<Output = T> + Send + 'static,
882 meanwhile: Meanwhile,
883 ) -> T {
884 let mut task = tokio::spawn(work);
885 loop {
886 let exits = self.exits.clone();
887 tokio::select! {
888 biased;
889 done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
890 Ok(output) => return output,
891 // As if it had panicked here.
892 Err(Ok(panic)) => std::panic::resume_unwind(panic),
893 // Only while the runtime shuts down, which drops this task too.
894 Err(Err(_)) => std::future::pending::<()>().await,
895 },
896 Some(command) = self.commands.recv() => {
897 self.command(command).await;
898 self.publish();
899 // A tick follows it, as one follows a command answered between ticks: a config it replaced
900 // polls its repositories then, not a poll interval later.
901 self.control.wake.notify_one();
902 }
903 () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
904 let began = tokio::time::Instant::now();
905 self.turnover().await;
906 self.passes_in_tick += began.elapsed();
907 self.publish();
908 }
909 Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
910 self.note_found(&found);
911 self.publish();
912 }
913 }
914 }
915 }
916
917 fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
918 self.slots.iter().flat_map(|(profile, slots)| {
919 slots.iter().enumerate().filter_map(|(index, slot)| {
920 slot.as_ref().map(|occupant| {
921 (
922 SlotRef {
923 profile: profile.clone(),
924 index,
925 },
926 occupant,
927 )
928 })
929 })
930 })
931 }
932
933 /// The runner in `slot`, if one is.
934 fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
935 self.slots
936 .get(&slot.profile)
937 .and_then(|slots| slots.get(slot.index))
938 .and_then(Option::as_ref)
939 }
940
941 /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
942 /// runs in them.
943 fn drop_closed_slots(&mut self) {
944 for (name, slots) in &mut self.slots {
945 let count = self
946 .config
947 .profiles
948 .get(name)
949 .map_or(0, |profile| profile.slots as usize);
950 while slots.len() > count && slots.last().is_some_and(Option::is_none) {
951 slots.pop();
952 }
953 }
954 let profiles = &self.config.profiles;
955 self.slots
956 .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
957 // The volumes of the slots dropped go at the next check of the caches.
958 let slots = &self.slots;
959 self.ran.retain(|slot, _| {
960 slots
961 .get(&slot.profile)
962 .is_some_and(|slots| slot.index < slots.len())
963 });
964 let (slots, reseeds) = (&self.slots, &self.reseeds);
965 self.retired
966 .retain(|name, _| slots.contains_key(name) || reseeds.contains_key(name));
967 }
968}
969
970#[cfg(test)]
971mod tests;