runnerd_core/scheduler/engine.rs
1//! The scheduling loop: polls GitHub, pulls the profiles' images, applies `plan`'s decisions with GitHub and
2//! Docker, removes each runner's container when its job ends, keeps the mirror up while any runner runs, reseeds
3//! idle profiles and recovers runners and containers a previous daemon left. It also feeds the job history: queued
4//! jobs, the job each runner took (from `observe::capture`), steps from GitHub, mirror bytes, cache sizes and slot
5//! and budget changes. It owns slot state; it never serves the API (it publishes `api::Status` and takes commands
6//! through `Control`) and never logs the token or a JIT config.
7//!
8//! This module holds `Engine`'s state, its construction and its loop; the submodules each add one concern's
9//! methods: `recover` (startup), `tick` (polling and planning), `runners` (a slot's runner from start to removal),
10//! `leftovers` (the containers still to read or remove once nothing occupies them), `listing` (what a repository's last listing of queued jobs showed), `listing_pace` (when a repository is listed next), `ledger` (the jobs followed on GitHub, queued or closing), `claims` (what a repository's listings may show that this machine's runners took), `images` (pulling the profiles' images), `caches` (reseeds, the mirror and the shared caches), `upkeep` (what each
11//! profile's measures and reseeds are doing and have due), `presence` (which repositories are listed or draining), `registrations` (what is remembered of a registration with no slot), `watches` (how each runner's container is watched: its log capture and what broken ones left), `mirror` (whether the mirror runs and at what cap), `build_cache` (each runner's side of the shared build cache and whether its job
12//! may write), `history` (what the store records), `warm` (how many warm runners each repository wants), `control` (the
13//! API's commands), `stop` (how far the daemon's stop has got) and `status` (what it publishes).
14
15mod admission;
16mod build_cache;
17mod caches;
18mod claims;
19mod clock;
20mod control;
21mod engagement;
22mod exits;
23mod governor;
24mod history;
25mod images;
26mod ledger;
27mod leftovers;
28mod listing;
29mod listing_pace;
30mod mirror;
31mod open_waits;
32mod pool_health;
33mod presence;
34mod records;
35mod recover;
36mod registrations;
37mod repository_state;
38mod runner_exit;
39mod runners;
40mod running_reads;
41mod slot;
42mod start;
43mod status;
44mod stop;
45mod tick;
46mod unsynced;
47mod upkeep;
48mod warm;
49mod watches;
50
51use std::collections::{BTreeMap, BTreeSet};
52use std::path::PathBuf;
53use std::sync::{Arc, Mutex};
54use std::time::{Duration, SystemTime};
55
56use tokio::sync::{mpsc, oneshot, watch};
57use tokio::task::JoinHandle;
58use tracing::{info, warn};
59
60use self::claims::Claims;
61use self::clock::Clock;
62pub use self::control::{
63 Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
64 Target, same_slice,
65};
66use self::engagement::{Engagement, Mark};
67use self::history::{CAPTURE_DRAIN, Found};
68use self::images::Images;
69use self::ledger::Ledger;
70use self::leftovers::Leftovers;
71use self::mirror::Mirror;
72use self::presence::Repositories;
73pub use self::records::RecordSelection;
74use self::registrations::Registrations;
75use self::repository_state::RepositoryState;
76use self::slot::Slot;
77use self::tick::Scope;
78pub use self::unsynced::Durability;
79use self::unsynced::Unsynced;
80use self::upkeep::Upkeep;
81use self::watches::Watches;
82use super::SlotRef;
83use crate::actions::JobActions;
84use crate::admission::Key;
85use crate::api;
86use crate::buildcache::BuildCache;
87use crate::cache::{self, Names, Trimmed};
88use crate::config::Config;
89use crate::docker::{self, Docker, DockerApi};
90use crate::github::{Github, GithubApi};
91use crate::isolation::{self, Isolation};
92use crate::observe::disk::Disks;
93use crate::observe::metrics::{self, Paths, Sampler, Tracked};
94use crate::observe::mirror::Leases;
95use crate::store::{Announced, MirrorPosition, Store};
96
97/// This machine as its runner names carry it: `runnerd-<id>-<host>-<profile>-<slot>-<secs>`. Only `runnerd-<id>-`
98/// identifies the machine, so a changed host name still matches the runners started before the change, and two
99/// machines with the same or overlapping host names never take each other's runners for their own.
100#[derive(Debug, Clone, PartialEq, Eq)]
101pub struct HostIdentity {
102 /// 8 hex digits: a hash of the systemd machine ID, never the ID itself, as machine-id(5) asks.
103 id: String,
104 /// The host name in lower case, letters, digits and dashes, at most 20 characters: for people reading names.
105 host: String,
106}
107
108impl HostIdentity {
109 /// This machine's identity, from `/etc/machine-id` and its host name.
110 pub fn read() -> std::io::Result<HostIdentity> {
111 let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
112 let machine_id = std::fs::read_to_string("/etc/machine-id")
113 .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
114 .map_err(|error| {
115 std::io::Error::new(
116 error.kind(),
117 format!("cannot read /etc/machine-id: {error}"),
118 )
119 })?;
120 HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
121 std::io::Error::new(
122 std::io::ErrorKind::InvalidData,
123 "/etc/machine-id does not hold a machine ID",
124 )
125 })
126 }
127
128 /// The identity for a host name and a machine ID; `None` unless the ID is 32 hex digits.
129 pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
130 let machine_id = machine_id.trim();
131 if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
132 return None;
133 }
134 // FNV-1a: stable across Rust releases, unlike std's hasher, so a new build recognizes the runners of the old.
135 let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
136 .bytes()
137 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
138 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
139 });
140 let host: String = hostname
141 .trim()
142 .chars()
143 .map(|c| {
144 if c.is_ascii_alphanumeric() {
145 c.to_ascii_lowercase()
146 } else {
147 '-'
148 }
149 })
150 .take(20)
151 .collect();
152 let host = host.trim_matches('-');
153 Some(HostIdentity {
154 id: format!("{:08x}", hash >> 32),
155 host: if host.is_empty() { "host" } else { host }.to_owned(),
156 })
157 }
158
159 /// What every runner name of this machine starts with. The ID's fixed length keeps it from starting another
160 /// machine's names.
161 pub fn prefix(&self) -> String {
162 format!("runnerd-{}-", self.id)
163 }
164
165 /// The name of a runner, and of its container, started in `slot` (1-based) of `profile` at `secs`.
166 pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
167 format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
168 }
169}
170
171/// What the API needs from a running engine.
172#[derive(Debug, Clone)]
173pub struct Handle {
174 /// The latest status, replaced after every tick.
175 pub status: watch::Receiver<api::Status>,
176 /// Pause, resume and shutdown requests.
177 pub control: Arc<Control>,
178 /// The job history.
179 pub store: Store,
180 /// Every cache volume of the config, with size 0: the API fills in the sizes it measured. Replaced when the
181 /// config is.
182 pub caches: watch::Receiver<Vec<api::CacheVolume>>,
183 /// The records of failed jobs, replaced whenever one is made or removed.
184 pub records: watch::Receiver<api::Records>,
185 /// The shared build cache, for its size now.
186 pub build_cache: Option<BuildCache>,
187 /// What the slots' Docker Engines may keep together, in bytes, from the last measure (`cache::docker_disk`);
188 /// `None` before the first. Auto sizing adds no slot without it (ADR 0090).
189 pub disk: watch::Receiver<Option<u64>>,
190 /// The owner's job actions, sent through the engine's GitHub client but never on its loop (ADR 0134).
191 pub actions: JobActions,
192}
193
194impl Handle {
195 /// Asks the engine to stop in `mode` and says how it stops now: as asked, unless a stop asked for earlier is more
196 /// urgent, or the stop detaches since Docker stopped answering it. `None` for `Shutdown::No` when nothing stops.
197 pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
198 self.control.shutdown(mode).stopping()
199 }
200}
201
202/// Where the engine runs, besides its config and its GitHub and Docker clients.
203#[derive(Debug)]
204pub struct Setup {
205 /// The names of what runnerd creates in Docker.
206 pub names: Names,
207 /// This machine, which names its runners, so a restarted daemon recognizes them.
208 pub identity: HostIdentity,
209 /// The slice's `MemoryMax`.
210 pub memory_max: Option<u64>,
211 /// The CPUs the slice may use, in millicores (`config::slice_cpus`); `None` admits no runner by CPU.
212 pub cpus: Option<u64>,
213 /// The job history.
214 pub store: Store,
215 /// The profiles and repositories as the last run announced them (`Store::announced`), so the first publish
216 /// announces only the difference, including what the config no longer has.
217 pub announced: Announced,
218 /// The repositories recorded paused (`Store::paused_repositories`): they stay paused from the first tick.
219 pub paused: Vec<String>,
220 /// Where the cgroup and proc file systems are.
221 pub paths: Paths,
222 /// The shared build cache, unless the daemon could not open it.
223 pub build_cache: Option<BuildCache>,
224 /// Where the engine keeps whether what runners wrote without syncing may be lost to a host crash (ADR 0141), such
225 /// as `unsynced` beside the job history.
226 pub unsynced: PathBuf,
227 /// Writes what the kernel holds in memory to disk, before the engine says nothing is owed, and tells whether a
228 /// past boot ended in an orderly shutdown: `sync(2)` and the systemd journal.
229 pub durability: Durability,
230}
231
232/// Why a runner starts, as its start's log line and its record say (ADR 0092).
233#[derive(Debug, Clone, Copy, PartialEq, Eq)]
234enum Purpose {
235 /// For a queued job.
236 Job,
237 /// As one of its repository's configured warm runners, for jobs not queued yet.
238 Warm,
239 /// As a warm runner past those, for a job its runs are expected to queue still.
240 Burst,
241}
242
243impl Purpose {
244 /// How the history stores it.
245 fn as_str(self) -> &'static str {
246 match self {
247 Purpose::Job => "job",
248 Purpose::Warm => "warm",
249 Purpose::Burst => "burst",
250 }
251 }
252}
253
254/// What else the engine does while it waits for a tick's work in `serving`, besides answering commands.
255#[derive(Debug, Clone, Copy, PartialEq, Eq)]
256enum Meanwhile {
257 /// Releases runners Docker says stopped and starts runners in their room (`turnover`).
258 Turnover,
259 /// Only answers commands: the work must not see a runner start or go, as a trim of the build cache.
260 Commands,
261}
262
263/// What a runner, or a start in flight, takes of the budgets: what `held_by` and `reserves_cpu` count it by, so a
264/// start counts as the runner it becomes.
265#[derive(Debug, Clone, Copy)]
266struct Footprint<'slot> {
267 runner_name: &'slot str,
268 /// Its container's memory cap.
269 memory: u64,
270 /// What it reserves of memory under measured admission.
271 reserved: u64,
272 /// Its container's CPU limit, in millicores.
273 cpus: u64,
274 /// What it reserves of the CPU budget, in millicores.
275 reserved_cpu: u64,
276 /// When it started, or its start was launched.
277 started_at: SystemTime,
278 /// Its job, as admission tells jobs apart.
279 key: Option<&'slot Key>,
280 /// When it took its job, if this daemon saw it do so.
281 job_found_here: Option<SystemTime>,
282 /// What its reservation went by.
283 basis: Option<api::Basis>,
284 /// Whether it is a runner yet.
285 stage: Stage,
286}
287
288/// Whether a footprint is a runner's or a start's in flight.
289#[derive(Debug, Clone, Copy, PartialEq, Eq)]
290enum Stage {
291 /// Named and admitted, registering its runner or creating its container: there is nothing to sample yet.
292 InFlight,
293 /// A runner in its slot, with its container.
294 Placed,
295}
296
297/// A slot's runner and its container.
298#[derive(Debug, Clone)]
299struct Occupant {
300 repository: String,
301 runner_name: String,
302 runner_id: u64,
303 container: String,
304 started_at: SystemTime,
305 /// Its container's memory cap, which a reload does not change.
306 memory: u64,
307 /// Its container's CPU limit in millicores, which a reload does not change.
308 cpus: u64,
309 /// How it was admitted, with what it reserved of the budget, kept through a restart by its record; `None` for
310 /// one whose record says nothing of it, which reserves its cap.
311 admission: Option<api::Admission>,
312 /// The labels it registered with; `None` for one adopted from a container that does not say.
313 labels: Option<Vec<String>>,
314 /// The image it runs; `None` for one adopted from a container that does not say.
315 image: Option<String>,
316 /// The release whose container spec it runs; `None` for one adopted from a container of a release before the
317 /// label.
318 spec: Option<String>,
319 /// Why it is there, whether it holds a job and which.
320 engagement: Engagement,
321}
322
323impl Occupant {
324 /// Whether it no longer is what `repository`'s runners in `profile` start as: other labels, or other limits than
325 /// the profile's. An unknown label set counts as the same.
326 fn outdated(
327 &self,
328 repository: &crate::config::Repository,
329 profile: &crate::config::Profile,
330 ) -> bool {
331 let differ = self.labels.as_ref().is_some_and(|labels| {
332 let set = |labels: &[String]| -> BTreeSet<String> {
333 labels
334 .iter()
335 .map(|label| label.to_ascii_lowercase())
336 .collect()
337 };
338 set(labels) != set(&repository.labels)
339 });
340 differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
341 }
342
343 /// Whether it runs another image than `profile` names. An unknown image counts as the same.
344 fn other_image(&self, profile: &crate::config::Profile) -> bool {
345 self.image
346 .as_ref()
347 .is_some_and(|image| *image != profile.image.reference())
348 }
349
350 /// Whether another release started it, with another container spec: one that does not say is older.
351 fn other_spec(&self) -> bool {
352 self.spec.as_deref() != Some(runners::THIS_SPEC)
353 }
354
355 /// What it takes of the budgets.
356 fn footprint(&self) -> Footprint<'_> {
357 Footprint {
358 runner_name: &self.runner_name,
359 memory: self.memory,
360 reserved: self.reserved(),
361 cpus: self.cpus,
362 reserved_cpu: self.reserved_cpu(),
363 started_at: self.started_at,
364 key: self.key(),
365 job_found_here: self.job_found_here(),
366 basis: self.admission.map(|admission| admission.basis),
367 stage: Stage::Placed,
368 }
369 }
370
371 /// What it reserves of the budget under measured admission: its cap if it has no admission.
372 fn reserved(&self) -> u64 {
373 self.admission
374 .map_or(self.memory, |admission| admission.reserved)
375 }
376
377 /// The CPU it reserves, in millicores: its CPU limit unless it was admitted by CPU.
378 fn reserved_cpu(&self) -> u64 {
379 self.admission
380 .and_then(|admission| admission.cpu)
381 .map_or(self.cpus, |cpu| cpu.reserved)
382 }
383
384 /// Whether GitHub's last runners listing showed it busy, or GitHub said so since (`Engagement::listed_busy`).
385 fn listed_busy(&self) -> bool {
386 self.engagement.listed_busy()
387 }
388
389 /// The job its capture or its history row named (`Engagement::named_job`).
390 fn named_job(&self) -> Option<u64> {
391 self.engagement.named_job()
392 }
393
394 /// Whether `job` is the one named for it.
395 fn runs(&self, job: u64) -> bool {
396 self.engagement.runs(job)
397 }
398
399 /// Whether it holds a job as far as this daemon knows (`Engagement::engaged`).
400 fn engaged(&self) -> bool {
401 self.engagement.engaged()
402 }
403
404 /// Whether it waits for a job: not `engaged`.
405 fn idle(&self) -> bool {
406 !self.engaged()
407 }
408
409 /// Whether it may be running one of its pool's listed jobs that no capture named yet
410 /// (`Engagement::claims_a_listed_job`).
411 fn claims_a_listed_job(&self) -> bool {
412 self.engagement.claims_a_listed_job()
413 }
414
415 /// Whether a detaching stop removes it (`Engagement::goes_when_detaching`).
416 fn goes_when_detaching(&self) -> bool {
417 self.engagement.goes_when_detaching()
418 }
419
420 /// Why this daemon started it; `None` for one a restart adopted.
421 fn purpose(&self) -> Option<Purpose> {
422 self.engagement.purpose()
423 }
424
425 /// When this daemon's capture found its job, if it started under this daemon (`Engagement::job_found_here`).
426 fn job_found_here(&self) -> Option<SystemTime> {
427 self.engagement.job_found_here()
428 }
429
430 /// Its job as admission tells jobs apart, once the history had its record.
431 fn key(&self) -> Option<&Key> {
432 self.engagement.key()
433 }
434
435 /// Whether admission can tell its job apart yet. A job named with no record yet has no key.
436 fn has_key(&self) -> bool {
437 self.key().is_some()
438 }
439
440 /// Whether it holds a job admission cannot tell apart yet, as when its log capture failed.
441 fn runs_unknown_job(&self) -> bool {
442 self.engaged() && !self.has_key()
443 }
444
445 fn running_job(&self) -> api::RunningJob {
446 api::RunningJob {
447 repository: self.repository.clone(),
448 runner: self.runner_name.clone(),
449 job: self.named_job(),
450 }
451 }
452}
453
454/// How long a stop that detaches or is forced takes at most, from its request: what Docker and GitHub have not
455/// answered by then is left to the next start (ADR 0091). Well within systemd's default stop timeout of 90s, which
456/// a single Docker request (120s) could otherwise outlast.
457pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
458
459/// How long GitHub and Docker rest at least between the end of a tick and the start of the next, when a tick outlasts
460/// the interval between their starts (ADR 0111).
461const TICK_REST: Duration = Duration::from_secs(2);
462
463/// The scheduler with its side effects, on the real GitHub and Docker unless a test passes fakes.
464#[derive(Debug)]
465pub struct Engine<G = Github, D = Docker> {
466 config: Config,
467 /// Shared with the listings a tick reads while it answers commands.
468 github: Arc<G>,
469 docker: Arc<D>,
470 names: Names,
471 isolation: &'static dyn Isolation,
472 identity: HostIdentity,
473 runner_prefix: String,
474 /// What a sample's or a host reading's age goes by.
475 clock: Clock,
476 memory_max: Option<u64>,
477 /// The CPUs the slice may use, in millicores, read at start.
478 cpus: Option<u64>,
479 slots: BTreeMap<String, Vec<Slot>>,
480 /// The repositories, listed by the config or draining after a reload removed them.
481 repositories: Repositories,
482 /// What each profile's measures and reseeds are doing and have due.
483 upkeep: Upkeep,
484 /// Whether what runners wrote without syncing may be lost to a host crash.
485 unsynced: Unsynced,
486 /// The pulled images of the config's profiles, and the removal of those no profile runs.
487 images: Images,
488 /// The containers still to read or remove once nothing occupies them.
489 leftovers: Leftovers,
490 /// Profiles a reload removed, kept while their runners or reseed still run.
491 retired: BTreeMap<String, crate::config::Profile>,
492 /// The slots whose last start failed preparing its caches. Not kept over a restart.
493 prepare_failures: BTreeMap<SlotRef, runners::PrepareFailures>,
494 /// The job each slot ran last, which left its caches in the slot's layer: what `plan` matches queued jobs to
495 /// (ADR 0080). A slot is absent once its last job is not known, or a reseed dropped its layer.
496 ran: BTreeMap<SlotRef, Key>,
497 /// What the slots' Docker Engines may keep together, from the last check of the caches (`cache::docker_disk`);
498 /// `None` until one measured Docker's disk, when each image keeps its own default.
499 docker_disk: watch::Sender<Option<u64>>,
500 /// The measure of Docker's disk recovery started, in the background: Docker measures every volume.
501 disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
502 /// Whether a job ended since the shared caches were last checked.
503 trim_due: bool,
504 /// The running check of the shared caches: Docker takes many seconds to measure volumes.
505 trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
506 /// When the last check of the shared caches started; the daemon's start counts as one, as the first runs then
507 /// unless runners were adopted.
508 checked_at: SystemTime,
509 mirror: Mirror,
510 /// Until when GitHub's rate limit is used up, from the last answer that said so: no runner starts before.
511 github_limited_until: Option<SystemTime>,
512 /// What is known of the registrations with no slot.
513 registrations: Registrations,
514 status: watch::Sender<api::Status>,
515 /// The profiles as the last run announced them, until the first publish compares the profiles with them. Their
516 /// slots are older than the `slot.changed` events clients have, so that publish announces every slot instead.
517 announced_profiles: Option<Vec<api::ProfileStatus>>,
518 control: Arc<Control>,
519 commands: Commands,
520 caches: watch::Sender<Vec<api::CacheVolume>>,
521 store: Store,
522 paths: Paths,
523 /// The containers the sampler reads.
524 tracked: watch::Sender<Vec<Tracked>>,
525 found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
526 /// How each runner's container is watched: its log capture, and what broken ones left.
527 watches: Watches,
528 /// Who held which address, for the mirror's log.
529 leases: Leases,
530 /// How far the mirror's log was counted.
531 mirror_until: MirrorPosition,
532 /// The jobs followed on GitHub that no runner of this machine runs now: queued, or closing once their runner
533 /// went.
534 ledger: Ledger,
535 last_prune: Option<SystemTime>,
536 /// When the running jobs were last read from GitHub, which is once every `poll_interval` (ADR 0124).
537 running_reads: running_reads::RunningReads,
538 /// The pace of the reads by what is left of GitHub's rate limit (ADR 0127).
539 governor: governor::Governor,
540 /// The records of failed jobs and the shells open in them.
541 records: records::RecordState,
542 /// The predictor, the high-water mark of measured admission, and what admission saw of the host.
543 measured: admission::Measured,
544 /// What warm runners learned from the history, and the workflow runs seen lately.
545 warm: warm::Warm,
546 /// The shared build cache.
547 build_cache: Option<BuildCache>,
548 /// Each repository's default branch, once a job of it needed it: the branch whose builds write to the build
549 /// cache when `[caches.build] branches` names none.
550 default_branches: BTreeMap<String, build_cache::DefaultBranch>,
551 /// Removed runners' build cache uploads held until GitHub decides whether their jobs may write, at most
552 /// `build_cache::ENDED_WAIT`, by runner.
553 ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
554 /// When each repository's visibility is next asked of GitHub, which the asking task updates; the store keeps
555 /// the answers.
556 visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
557 /// The task asking GitHub repositories' visibility, off the tick (`follow_visibility`).
558 visibility_asking: Option<JoinHandle<()>>,
559 /// Released runners' registration deletes, sent in the background after their containers went (ADR 0113): each
560 /// tick reaps the finished ones, and a stop waits for the rest, up to `STOP_DEADLINE`.
561 deletes: Arc<Mutex<tokio::task::JoinSet<()>>>,
562 /// The runner containers Docker said stopped since the last pass or tick took them (ADR 0095).
563 exits: Arc<exits::Exits>,
564 /// The task following Docker's events into `exits`, while the engine schedules.
565 watcher: Option<exits::Watcher>,
566 /// The runner containers this tick's inspect found stopped and released before Docker's event for them came: a
567 /// pass the event wakes while the tick waits for GitHub starts runners into their room.
568 released: BTreeSet<String>,
569 /// How long the passes this tick's waits for GitHub held took. Each waits for its slowest release: a tick holds
570 /// passes until they took `CAPTURE_DRAIN`, or while slots keep turning over it would take minutes to its plan.
571 passes_in_tick: Duration,
572 /// The listing the engine began last, counted from 1 since its start, and when it began: the listing each answer
573 /// comes from, and how old a claim on a listed job is (`Claims`, `Engagement`).
574 listing: Mark,
575 /// By repository, the jobs its listings may show that this machine's runners took.
576 claims: Claims,
577 /// The second in the name of the last runner the engine named: the next one's is later, so no two runners share
578 /// a name, which GitHub, Docker and the history know them by. The history keeps each before the name is used,
579 /// and a restart takes it from there and the containers, so a name ahead of the clock, or a clock that stepped
580 /// back, is not used again.
581 named: u64,
582 /// The runners each batch launched, in launch order: what a test's reading taken mid-batch binds.
583 #[cfg(test)]
584 launches: start::Launches,
585}
586
587fn unix_secs(time: SystemTime) -> u64 {
588 time.duration_since(SystemTime::UNIX_EPOCH)
589 .unwrap_or_default()
590 .as_secs()
591}
592
593impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
594 /// An engine for a validated config. `identity` names this machine's runners, so a restarted daemon recognizes
595 /// them; `memory_max` is the slice's.
596 #[allow(
597 clippy::too_many_lines,
598 reason = "one line per field of the engine's state, which split up would only be listed twice"
599 )]
600 pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
601 let Setup {
602 names,
603 identity,
604 memory_max,
605 cpus,
606 store,
607 announced,
608 paused,
609 paths,
610 build_cache,
611 unsynced,
612 durability,
613 } = setup;
614 let unsynced = Unsynced::open(unsynced, &paths.proc, durability);
615 let slots = config
616 .profiles
617 .iter()
618 .map(|(name, profile)| {
619 let mut slots = Vec::new();
620 slots.resize_with(profile.slots as usize, Slot::default);
621 (name.clone(), slots)
622 })
623 .collect();
624 let repositories = Repositories::new(
625 config
626 .repositories
627 .iter()
628 .map(|repository| repository.name.clone()),
629 );
630 let (control, commands) = Control::new(
631 config
632 .repositories
633 .iter()
634 .map(|repository| repository.name.clone()),
635 paused,
636 store.clone(),
637 );
638 let control = Arc::new(control);
639 control.set_action_policy(
640 api::JobActions::configured(config.github.job_actions),
641 config
642 .repositories
643 .iter()
644 .map(|repository| repository.name.clone())
645 .collect(),
646 );
647 let github = Arc::new(github);
648 let actions = JobActions::new(
649 github.clone(),
650 store.clone(),
651 control.clone(),
652 crate::actions::LIMITS,
653 );
654 let (status, receiver) = watch::channel(api::Status {
655 job_actions: api::JobActions::configured(config.github.job_actions),
656 version: env!("CARGO_PKG_VERSION").to_owned(),
657 shutting_down: false,
658 stopping: None,
659 github_limited_until: None,
660 budget: api::Budget {
661 slice: config.host.slice.clone(),
662 memory_max,
663 ..api::Budget::default()
664 },
665 profiles: Vec::new(),
666 // What clients resuming after the last run's events have, so the first publish announces the difference.
667 repositories: announced.repositories,
668 queued: Vec::new(),
669 });
670 let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
671 let (records, records_receiver) = records::RecordState::new();
672 let (docker_disk, disk_receiver) = watch::channel(None);
673 let mut engine = Engine {
674 clock: Clock::default(),
675 isolation: isolation::backend(config.host.isolation),
676 // An earlier run may have left the mirror up, so the first idle tick stops it.
677 mirror: Mirror::unknown(config.mirror_memory()),
678 config,
679 github,
680 docker: Arc::new(docker),
681 names,
682 runner_prefix: identity.prefix(),
683 identity,
684 memory_max,
685 cpus,
686 slots,
687 repositories,
688 upkeep: Upkeep::default(),
689 unsynced,
690 images: Images::new(),
691 retired: BTreeMap::new(),
692 leftovers: Leftovers::default(),
693 prepare_failures: BTreeMap::new(),
694 ran: BTreeMap::new(),
695 docker_disk,
696 disk: None,
697 // So the caches are measured once at start.
698 trim_due: true,
699 trim: None,
700 checked_at: SystemTime::now(),
701 github_limited_until: None,
702 registrations: Registrations::default(),
703 status,
704 announced_profiles: Some(announced.profiles),
705 control: control.clone(),
706 commands,
707 caches,
708 store: store.clone(),
709 paths,
710 tracked: watch::channel(Vec::new()).0,
711 found: mpsc::unbounded_channel(),
712 watches: Watches::default(),
713 leases: Leases::default(),
714 mirror_until: MirrorPosition::default(),
715 ledger: Ledger::default(),
716 last_prune: None,
717 running_reads: running_reads::RunningReads::default(),
718 governor: governor::Governor::default(),
719 measured: admission::Measured::new(SystemTime::now()),
720 warm: warm::Warm::default(),
721 records,
722 build_cache: build_cache.clone(),
723 default_branches: BTreeMap::new(),
724 ended_uploads: BTreeMap::new(),
725 visibility_due: Arc::default(),
726 visibility_asking: None,
727 deletes: Arc::default(),
728 exits: Arc::default(),
729 watcher: None,
730 released: BTreeSet::new(),
731 passes_in_tick: Duration::ZERO,
732 listing: Mark {
733 listing: 0,
734 at: tokio::time::Instant::now(),
735 },
736 named: 0,
737 #[cfg(test)]
738 launches: start::Launches::default(),
739 claims: Claims::default(),
740 };
741 engine.publish();
742 (
743 engine,
744 Handle {
745 status: receiver,
746 control,
747 store,
748 caches: caches_receiver,
749 records: records_receiver,
750 build_cache,
751 disk: disk_receiver,
752 actions,
753 },
754 )
755 }
756
757 /// Runs until a shutdown request is carried out, or `STOP_DEADLINE` after a stop that detaches or is forced
758 /// began, whichever comes first. Either way it returns, once the sampler stored what it was adding up, so the
759 /// daemon still closes its history.
760 pub async fn run(mut self) {
761 let (disk, disk_read) = oneshot::channel();
762 let sampler = tokio::spawn(metrics::run(
763 Sampler::new(self.paths.clone(), &self.config.host.slice),
764 self.tracked.subscribe(),
765 self.store.clone(),
766 disk_read,
767 ));
768 tokio::spawn(disk_target(self.docker.clone(), self.paths.clone(), disk));
769 let control = self.control.clone();
770 let deadline = async {
771 control.hurried().await;
772 tokio::time::sleep(STOP_DEADLINE).await;
773 };
774 tokio::select! {
775 () = self.serve() => {}
776 () = deadline => warn!(
777 "stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
778 STOP_DEADLINE.as_secs()
779 ),
780 }
781 // Letting go of the tracked containers stops the sampler, which stores the drive's window it was adding up:
782 // the daemon closes its history right after this returns.
783 drop(self);
784 let _ = sampler.await;
785 }
786
787 /// Recovers, then schedules until a shutdown request is carried out.
788 async fn serve(&mut self) {
789 // Until Docker answers: scheduling without knowing the containers already running would start a second
790 // runner on a slot's volume and delete the running one's registration.
791 while let Err(error) = self.recover().await {
792 warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
793 if self.control.mode() != Shutdown::No {
794 return;
795 }
796 tokio::select! {
797 () = tokio::time::sleep(Duration::from_secs(5)) => {}
798 () = self.control.wake.notified() => {}
799 }
800 }
801 self.schedule().await;
802 }
803
804 /// Schedules until a shutdown request is carried out, from a recovered engine. A tick is due when the first
805 /// repository is, each an interval after its last listing started, and at least `TICK_REST` after the last tick
806 /// ended (ADR 0111, 0128); it lists the repositories due. One a command or a wake-up brings sooner lists every
807 /// repository. Between ticks, a runner whose container Docker says stopped is released at once, and its slot taken
808 /// by a job the last poll listed (`turnover`, ADR 0095); that leaves the next tick when it was due.
809 async fn schedule(&mut self) {
810 self.watcher = Some(exits::Watcher::start(
811 self.docker.clone(),
812 self.exits.clone(),
813 ));
814 let mut shown = Shutdown::No;
815 // The first tick lists every repository: none was listed yet.
816 let mut scope = Scope::Every;
817 let mode = loop {
818 let mode = self.control.mode();
819 // A stop shows before it removes idle runners, which can take until `STOP_DEADLINE` on a slow GitHub or
820 // Docker: until then status would show a daemon that is not stopping.
821 if mode != shown {
822 self.publish();
823 shown = mode;
824 }
825 // What the next tick's due time counts from: this one's listing, or, in a detach, which lists nothing,
826 // its start.
827 let anchor = if mode == Shutdown::Detach {
828 let began = tokio::time::Instant::now();
829 self.remove_every_idle().await;
830 self.collect_recordings().await;
831 began
832 } else {
833 self.tick_listing(mode, scope).await
834 };
835 self.publish();
836 // A stop asked for or changed meanwhile goes on at once, before this one counts as done: a second signal
837 // while a detach removed idle runners kills the running jobs, and one Docker failed turns into a detach.
838 if self.control.mode() != mode {
839 continue;
840 }
841 if self.stopped(mode) {
842 break mode;
843 }
844 // From the listings' starts, so the tick's own length does not push the next listing out, and two listings
845 // of a repository are never closer than its interval, however long either tick took to collect before
846 // listing.
847 let due = if mode == Shutdown::No {
848 self.next_tick_due(anchor)
849 } else {
850 let interval = Duration::from_secs(5);
851 (anchor + interval).max(tokio::time::Instant::now() + interval.min(TICK_REST))
852 };
853 scope = self.wait_for_tick(mode, due).await;
854 };
855 // Nothing more is started or released: what Docker says now is the next start's.
856 self.watcher = None;
857 // A reseed that finished meanwhile says so; one still running is the next start's (`recover_reseeds`).
858 self.collect_reseeds().await;
859 self.finish_deletes().await;
860 if self.control.docker_gone() {
861 info!("stopped; the next start removes or adopts what Docker did not answer for");
862 return;
863 }
864 self.close_shells().await;
865 // What the runners wrote without syncing is on disk once the stop leaves none running (ADR 0141).
866 if self.occupied().next().is_none()
867 && let Err(error) = self.unsynced.stopped_clean()
868 {
869 warn!(
870 "cannot note that what runners wrote is on disk; the next boot empties their caches: {error}"
871 );
872 }
873 if mode == Shutdown::Detach {
874 // The mirror stays up only for the busy runners left behind.
875 let mirror = if self.config.caches.mirror.enabled {
876 " and the mirror"
877 } else {
878 ""
879 };
880 match self.occupied().count() {
881 0 => info!("stopped; no job was running"),
882 1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
883 runners => {
884 info!("stopped, leaving {runners} busy runners{mirror} to the next start");
885 }
886 }
887 return;
888 }
889 if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
890 warn!("cannot stop the mirror: {error}");
891 }
892 info!("stopped");
893 }
894
895 /// Looks at what GitHub's rate limit says before a tick and moves the pace of the reads a step if it calls for it
896 /// (ADR 0127). Registrations and deletes do not ask.
897 fn pace_by_limit(&mut self) {
898 let spend = self.github.spend();
899 if let Some(change) =
900 self.governor
901 .observe(tokio::time::Instant::now(), SystemTime::now(), spend)
902 {
903 info!("{}", change.line(self.config.host.poll_interval));
904 }
905 }
906
907 /// Waits until the next tick in `mode` is `due`, or something calls for one sooner: a wake-up or a command.
908 /// Runners Docker says stopped meanwhile are released by passes, finished records collected and the jobs
909 /// captures name noted, all leaving `due` as it is; during a stop they call for the tick instead, which a detach does not wait for runners.
910 /// Says what the tick lists: the repositories due when it came when due, and every repository when it came sooner.
911 async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) -> Scope {
912 loop {
913 let exits = self.exits.clone();
914 // A due tick and a command go before a pass: passes back to back would otherwise hold them while slots
915 // keep turning over.
916 tokio::select! {
917 biased;
918 () = tokio::time::sleep_until(due) => return Scope::Due,
919 () = self.control.wake.notified() => return Scope::Every,
920 // A detaching stop changes nothing more: a command waiting for it fails once the engine is gone.
921 Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
922 self.command(command).await;
923 self.publish();
924 return Scope::Every;
925 }
926 // Running on, a record's end changes nothing GitHub must be asked about: the next poll stays when
927 // it was due. A stop returns to see whether it is done.
928 Some(()) = self.records.finished.recv() => {
929 self.collect_recordings().await;
930 self.publish();
931 if mode != Shutdown::No || self.control.mode() != mode {
932 return Scope::Every;
933 }
934 }
935 // A capture named its runner's job: status says the runner runs it now, not after the next poll.
936 Some(found) = self.found.1.recv() => {
937 self.note_found(&found);
938 self.publish();
939 }
940 () = exits.noted(), if mode != Shutdown::Detach => {
941 if mode != Shutdown::No {
942 return Scope::Every;
943 }
944 self.turnover().await;
945 self.publish();
946 if self.control.mode() != mode {
947 return Scope::Every;
948 }
949 }
950 }
951 }
952 }
953
954 /// Whether a stop in `mode` is done: what it waits for has ended. A record being made is waited for; a reseed and
955 /// a check of the caches are not, as the next start adopts the reseed and checks the caches first.
956 fn stopped(&self, mode: Shutdown) -> bool {
957 let idle = self.slots.values().flatten().all(Slot::is_free);
958 let recorded = !self.records.is_recording();
959 match mode {
960 Shutdown::No => false,
961 Shutdown::Detach => recorded,
962 Shutdown::Graceful | Shutdown::Force => idle && recorded,
963 }
964 }
965
966 /// Notes that Docker failed a call a stop needed. If it did not answer at all, as when it is gone during a
967 /// machine's shutdown, the stop calls it no more and leaves the rest to the next start, which removes or adopts
968 /// every container by its labels. An answer that is an error, or a timeout, is tried again as any failed call:
969 /// `runnerd stop` must not leave runners behind for one busy moment of Docker's.
970 fn docker_failed(&self, error: &docker::Error) {
971 if docker::is_unreachable(error) && self.control.note_docker_gone() {
972 warn!("Docker does not answer during the stop; leaving the rest to the next start");
973 }
974 }
975
976 /// Runs `work` as a task of its own, answering commands and publishing after each until it is done, so a command
977 /// waits for a slow GitHub no longer than for the command before it. A tick follows each such command, as one
978 /// follows a command answered between ticks. The task goes on while a command runs: a request's timeout runs on
979 /// the clock, not on polls. With `Meanwhile::Turnover`, runners Docker says stopped are released and their slots
980 /// taken meanwhile too (`turnover`, ADR 0095), so a tick's wait for GitHub does not hold them, for a
981 /// while, and the jobs captures name are noted as they come.
982 async fn serving<T: Send + 'static>(
983 &mut self,
984 work: impl Future<Output = T> + Send + 'static,
985 meanwhile: Meanwhile,
986 ) -> T {
987 let mut task = tokio::spawn(work);
988 loop {
989 let exits = self.exits.clone();
990 tokio::select! {
991 biased;
992 done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
993 Ok(output) => return output,
994 // As if it had panicked here.
995 Err(Ok(panic)) => std::panic::resume_unwind(panic),
996 // Only while the runtime shuts down, which drops this task too.
997 Err(Err(_)) => std::future::pending::<()>().await,
998 },
999 Some(command) = self.commands.recv() => {
1000 self.command(command).await;
1001 self.publish();
1002 // A tick follows it, as one follows a command answered between ticks: a config it replaced
1003 // polls its repositories then, not a poll interval later.
1004 self.control.wake.notify_one();
1005 }
1006 () = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
1007 let began = tokio::time::Instant::now();
1008 self.turnover().await;
1009 self.passes_in_tick += began.elapsed();
1010 self.publish();
1011 }
1012 Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
1013 self.note_found(&found);
1014 self.publish();
1015 }
1016 }
1017 }
1018 }
1019
1020 /// The jobs this machine's runners took, as their slots' occupants name them: computed where it is read, as a
1021 /// runner may take a job, or go, while the engine waits for GitHub.
1022 fn taken(&self) -> BTreeSet<u64> {
1023 self.occupied()
1024 .filter_map(|(_, occupant)| occupant.named_job())
1025 .collect()
1026 }
1027
1028 fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
1029 self.slots.iter().flat_map(|(profile, slots)| {
1030 slots.iter().enumerate().filter_map(|(index, slot)| {
1031 slot.occupant().map(|occupant| {
1032 (
1033 SlotRef {
1034 profile: profile.clone(),
1035 index,
1036 },
1037 occupant,
1038 )
1039 })
1040 })
1041 })
1042 }
1043
1044 /// What each runner and each start in flight takes of the budgets.
1045 fn footprints(&self) -> impl Iterator<Item = Footprint<'_>> {
1046 self.slots.values().flatten().filter_map(Slot::footprint)
1047 }
1048
1049 /// Each runner and each start in flight with its slot.
1050 fn slot_footprints(&self) -> impl Iterator<Item = (SlotRef, Footprint<'_>)> {
1051 self.slots.iter().flat_map(|(profile, slots)| {
1052 slots.iter().enumerate().filter_map(|(index, slot)| {
1053 slot.footprint().map(|footprint| {
1054 (
1055 SlotRef {
1056 profile: profile.clone(),
1057 index,
1058 },
1059 footprint,
1060 )
1061 })
1062 })
1063 })
1064 }
1065
1066 /// The runner in `slot`, if one is.
1067 fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
1068 self.slots
1069 .get(&slot.profile)
1070 .and_then(|slots| slots.get(slot.index))
1071 .and_then(Slot::occupant)
1072 }
1073
1074 /// Drops the free slots past each profile's configured number, and the profiles a reload removed once nothing
1075 /// runs in them.
1076 fn drop_closed_slots(&mut self) {
1077 for (name, slots) in &mut self.slots {
1078 let count = self
1079 .config
1080 .profiles
1081 .get(name)
1082 .map_or(0, |profile| profile.slots as usize);
1083 while slots.len() > count && slots.last().is_some_and(Slot::is_free) {
1084 slots.pop();
1085 }
1086 }
1087 let profiles = &self.config.profiles;
1088 self.slots
1089 .retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
1090 // The volumes of the slots dropped go at the next check of the caches.
1091 let slots = &self.slots;
1092 self.ran.retain(|slot, _| {
1093 slots
1094 .get(&slot.profile)
1095 .is_some_and(|slots| slot.index < slots.len())
1096 });
1097 let (slots, upkeep) = (&self.slots, &self.upkeep);
1098 self.retired
1099 .retain(|name, _| slots.contains_key(name) || upkeep.reseed_runs(name));
1100 }
1101}
1102
1103/// Asks Docker for its root until it answers, every 5 s, and sends the sampler the disks it is on (ADR 0143). Stops
1104/// once the sampler is gone. Disks that can't be told are logged once, and nothing is sent: writes stay unknown.
1105async fn disk_target<D: DockerApi>(docker: Arc<D>, paths: Paths, disk: oneshot::Sender<Disks>) {
1106 let root = loop {
1107 if disk.is_closed() {
1108 return;
1109 }
1110 match docker.root_dir().await {
1111 Ok(root) => break root,
1112 Err(_) => tokio::time::sleep(Duration::from_secs(5)).await,
1113 }
1114 };
1115 let resolved = tokio::task::spawn_blocking({
1116 let root = root.clone();
1117 move || Disks::resolve(&paths, std::path::Path::new(&root))
1118 })
1119 .await;
1120 if let Ok(Some(target)) = resolved {
1121 info!("disk writes are read from {}, under {root}", target.name());
1122 let _ = disk.send(target);
1123 } else {
1124 warn!("cannot tell the disks under Docker's root {root}: what runners write stays unknown");
1125 }
1126}
1127
1128#[cfg(test)]
1129mod tests;