mod admission;
mod build_cache;
mod caches;
mod claims;
mod clock;
mod control;
mod engagement;
mod exits;
mod governor;
mod history;
mod images;
mod ledger;
mod leftovers;
mod listing;
mod listing_pace;
mod mirror;
mod open_waits;
mod pool_health;
mod presence;
mod records;
mod recover;
mod registrations;
mod repository_state;
mod runner_exit;
mod runners;
mod running_reads;
mod slot;
mod start;
mod status;
mod stop;
mod tick;
mod unsynced;
mod upkeep;
mod warm;
mod watches;
use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinHandle;
use tracing::{info, warn};
use self::claims::Claims;
use self::clock::Clock;
pub use self::control::{
Author, Command, Commands, Commit, Control, ControlError, RunningJobs, RunningSlots, Shutdown,
Target, same_slice,
};
use self::engagement::{Engagement, Mark};
use self::history::{CAPTURE_DRAIN, Found};
use self::images::Images;
use self::ledger::Ledger;
use self::leftovers::Leftovers;
use self::mirror::Mirror;
use self::presence::Repositories;
pub use self::records::RecordSelection;
use self::registrations::Registrations;
use self::repository_state::RepositoryState;
use self::slot::Slot;
use self::tick::Scope;
pub use self::unsynced::Durability;
use self::unsynced::Unsynced;
use self::upkeep::Upkeep;
use self::watches::Watches;
use super::SlotRef;
use crate::actions::JobActions;
use crate::admission::Key;
use crate::api;
use crate::archives;
use crate::buildcache::BuildCache;
use crate::cache::{self, Names, Trimmed};
use crate::config::Config;
use crate::docker::{self, Docker, DockerApi};
use crate::github::{Github, GithubApi};
use crate::isolation::{self, Isolation};
use crate::observe::disk::Disks;
use crate::observe::metrics::{self, Paths, Sampler, Tracked};
use crate::observe::mirror::Leases;
use crate::store::{Announced, MirrorPosition, Store};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HostIdentity {
id: String,
host: String,
}
impl HostIdentity {
pub fn read() -> std::io::Result<HostIdentity> {
let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default();
let machine_id = std::fs::read_to_string("/etc/machine-id")
.or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id"))
.map_err(|error| {
std::io::Error::new(
error.kind(),
format!("cannot read /etc/machine-id: {error}"),
)
})?;
HostIdentity::new(&hostname, &machine_id).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"/etc/machine-id does not hold a machine ID",
)
})
}
pub fn new(hostname: &str, machine_id: &str) -> Option<HostIdentity> {
let machine_id = machine_id.trim();
if machine_id.len() != 32 || !machine_id.chars().all(|c| c.is_ascii_hexdigit()) {
return None;
}
let hash = format!("runnerd:{}", machine_id.to_ascii_lowercase())
.bytes()
.fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
(hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
});
let host: String = hostname
.trim()
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() {
c.to_ascii_lowercase()
} else {
'-'
}
})
.take(20)
.collect();
let host = host.trim_matches('-');
Some(HostIdentity {
id: format!("{:08x}", hash >> 32),
host: if host.is_empty() { "host" } else { host }.to_owned(),
})
}
pub fn prefix(&self) -> String {
format!("runnerd-{}-", self.id)
}
pub fn runner_name(&self, profile: &str, slot: usize, secs: u64) -> String {
format!("{}{}-{profile}-{slot}-{secs}", self.prefix(), self.host)
}
}
#[derive(Debug, Clone)]
pub struct Handle {
pub status: watch::Receiver<api::Status>,
pub control: Arc<Control>,
pub store: Store,
pub caches: watch::Receiver<Vec<api::CacheVolume>>,
pub records: watch::Receiver<api::Records>,
pub build_cache: Option<BuildCache>,
pub disk: watch::Receiver<Option<u64>>,
pub actions: JobActions,
}
impl Handle {
pub fn shutdown(&self, mode: Shutdown) -> Option<api::Stopping> {
self.control.shutdown(mode).stopping()
}
}
#[derive(Debug)]
pub struct Setup {
pub names: Names,
pub identity: HostIdentity,
pub memory_max: Option<u64>,
pub cpus: Option<u64>,
pub store: Store,
pub announced: Announced,
pub paused: Vec<String>,
pub paths: Paths,
pub build_cache: Option<BuildCache>,
pub action_archives: Option<PathBuf>,
pub unsynced: PathBuf,
pub durability: Durability,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Purpose {
Job,
Warm,
Burst,
}
impl Purpose {
fn as_str(self) -> &'static str {
match self {
Purpose::Job => "job",
Purpose::Warm => "warm",
Purpose::Burst => "burst",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Meanwhile {
Turnover,
Commands,
}
#[derive(Debug, Clone, Copy)]
struct Footprint<'slot> {
runner_name: &'slot str,
memory: u64,
reserved: u64,
cpus: u64,
reserved_cpu: u64,
started_at: SystemTime,
key: Option<&'slot Key>,
job_found_here: Option<SystemTime>,
basis: Option<api::Basis>,
stage: Stage,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Stage {
InFlight,
Placed,
}
#[derive(Debug, Clone)]
struct Occupant {
repository: String,
runner_name: String,
runner_id: u64,
container: String,
started_at: SystemTime,
memory: u64,
cpus: u64,
admission: Option<api::Admission>,
labels: Option<Vec<String>>,
image: Option<String>,
spec: Option<String>,
engagement: Engagement,
}
impl Occupant {
fn outdated(
&self,
repository: &crate::config::Repository,
profile: &crate::config::Profile,
) -> bool {
let differ = self.labels.as_ref().is_some_and(|labels| {
let set = |labels: &[String]| -> BTreeSet<String> {
labels
.iter()
.map(|label| label.to_ascii_lowercase())
.collect()
};
set(labels) != set(&repository.labels)
});
differ || self.memory != profile.memory.0 || self.cpus != u64::from(profile.cpus) * 1000
}
fn other_image(&self, profile: &crate::config::Profile) -> bool {
self.image
.as_ref()
.is_some_and(|image| *image != profile.image.reference())
}
fn other_spec(&self) -> bool {
self.spec.as_deref() != Some(runners::THIS_SPEC)
}
fn footprint(&self) -> Footprint<'_> {
Footprint {
runner_name: &self.runner_name,
memory: self.memory,
reserved: self.reserved(),
cpus: self.cpus,
reserved_cpu: self.reserved_cpu(),
started_at: self.started_at,
key: self.key(),
job_found_here: self.job_found_here(),
basis: self.admission.map(|admission| admission.basis),
stage: Stage::Placed,
}
}
fn reserved(&self) -> u64 {
self.admission
.map_or(self.memory, |admission| admission.reserved)
}
fn reserved_cpu(&self) -> u64 {
self.admission
.and_then(|admission| admission.cpu)
.map_or(self.cpus, |cpu| cpu.reserved)
}
fn listed_busy(&self) -> bool {
self.engagement.listed_busy()
}
fn named_job(&self) -> Option<u64> {
self.engagement.named_job()
}
fn runs(&self, job: u64) -> bool {
self.engagement.runs(job)
}
fn engaged(&self) -> bool {
self.engagement.engaged()
}
fn idle(&self) -> bool {
!self.engaged()
}
fn claims_a_listed_job(&self) -> bool {
self.engagement.claims_a_listed_job()
}
fn goes_when_detaching(&self) -> bool {
self.engagement.goes_when_detaching()
}
fn purpose(&self) -> Option<Purpose> {
self.engagement.purpose()
}
fn job_found_here(&self) -> Option<SystemTime> {
self.engagement.job_found_here()
}
fn key(&self) -> Option<&Key> {
self.engagement.key()
}
fn has_key(&self) -> bool {
self.key().is_some()
}
fn runs_unknown_job(&self) -> bool {
self.engaged() && !self.has_key()
}
fn running_job(&self) -> api::RunningJob {
api::RunningJob {
repository: self.repository.clone(),
runner: self.runner_name.clone(),
job: self.named_job(),
}
}
}
pub const STOP_DEADLINE: Duration = Duration::from_secs(30);
const TICK_REST: Duration = Duration::from_secs(2);
#[derive(Debug)]
pub struct Engine<G = Github, D = Docker> {
config: Config,
github: Arc<G>,
docker: Arc<D>,
names: Names,
isolation: &'static dyn Isolation,
identity: HostIdentity,
runner_prefix: String,
clock: Clock,
memory_max: Option<u64>,
cpus: Option<u64>,
slots: BTreeMap<String, Vec<Slot>>,
repositories: Repositories,
upkeep: Upkeep,
unsynced: Unsynced,
images: Images,
leftovers: Leftovers,
retired: BTreeMap<String, crate::config::Profile>,
prepare_failures: BTreeMap<SlotRef, runners::PrepareFailures>,
ran: BTreeMap<SlotRef, Key>,
docker_disk: watch::Sender<Option<u64>>,
disk: Option<JoinHandle<Result<caches::Disk, docker::Error>>>,
trim_due: bool,
trim: Option<JoinHandle<Result<Trimmed, docker::Error>>>,
checked_at: SystemTime,
mirror: Mirror,
github_limited_until: Option<SystemTime>,
registrations: Registrations,
status: watch::Sender<api::Status>,
announced_profiles: Option<Vec<api::ProfileStatus>>,
control: Arc<Control>,
commands: Commands,
caches: watch::Sender<Vec<api::CacheVolume>>,
store: Store,
paths: Paths,
tracked: watch::Sender<Vec<Tracked>>,
found: (mpsc::UnboundedSender<Found>, mpsc::UnboundedReceiver<Found>),
watches: Watches,
leases: Leases,
mirror_until: MirrorPosition,
ledger: Ledger,
last_prune: Option<SystemTime>,
running_reads: running_reads::RunningReads,
governor: governor::Governor,
records: records::RecordState,
measured: admission::Measured,
warm: warm::Warm,
build_cache: Option<BuildCache>,
archives: archives::Link,
actions: Arc<JobActions>,
default_branches: BTreeMap<String, build_cache::DefaultBranch>,
ended_uploads: BTreeMap<String, build_cache::EndedUploads>,
visibility_due: Arc<Mutex<BTreeMap<String, SystemTime>>>,
visibility_asking: Option<JoinHandle<()>>,
deletes: Arc<Mutex<tokio::task::JoinSet<()>>>,
exits: Arc<exits::Exits>,
watcher: Option<exits::Watcher>,
released: BTreeSet<String>,
passes_in_tick: Duration,
listing: Mark,
claims: Claims,
named: u64,
#[cfg(test)]
launches: start::Launches,
}
fn unix_secs(time: SystemTime) -> u64 {
time.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
#[allow(
clippy::too_many_lines,
reason = "one line per field of the engine's state, which split up would only be listed twice"
)]
pub fn new(config: Config, github: G, docker: D, setup: Setup) -> (Engine<G, D>, Handle) {
let Setup {
names,
identity,
memory_max,
cpus,
store,
announced,
paused,
paths,
build_cache,
action_archives,
unsynced,
durability,
} = setup;
let unsynced = Unsynced::open(unsynced, &paths.proc, durability);
let slots = config
.profiles
.iter()
.map(|(name, profile)| {
let mut slots = Vec::new();
slots.resize_with(profile.slots as usize, Slot::default);
(name.clone(), slots)
})
.collect();
let repositories = Repositories::new(
config
.repositories
.iter()
.map(|repository| repository.name.clone()),
);
let (control, commands) = Control::new(
config
.repositories
.iter()
.map(|repository| repository.name.clone()),
paused,
store.clone(),
);
let control = Arc::new(control);
control.set_action_policy(crate::actions::policy::Allowed::of(&config));
let github = Arc::new(github);
let actions = JobActions::new(
github.clone(),
store.clone(),
control.clone(),
crate::actions::LIMITS,
);
let (status, receiver) = watch::channel(api::Status {
job_actions: api::JobActions::configured(config.github.job_actions),
version: env!("CARGO_PKG_VERSION").to_owned(),
shutting_down: false,
stopping: None,
github_limited_until: None,
budget: api::Budget {
slice: config.host.slice.clone(),
memory_max,
..api::Budget::default()
},
profiles: Vec::new(),
repositories: announced.repositories,
queued: Vec::new(),
action_archives: None,
});
let (caches, caches_receiver) = watch::channel(cache::volumes(&names, &config));
let (records, records_receiver) = records::RecordState::new();
let (docker_disk, disk_receiver) = watch::channel(None);
let archives = archives::Link::new(action_archives, &config);
let mut engine = Engine {
clock: Clock::default(),
isolation: isolation::backend(config.host.isolation),
mirror: Mirror::unknown(config.mirror_memory()),
config,
github,
docker: Arc::new(docker),
names,
runner_prefix: identity.prefix(),
identity,
memory_max,
cpus,
slots,
repositories,
upkeep: Upkeep::default(),
unsynced,
images: Images::new(),
retired: BTreeMap::new(),
leftovers: Leftovers::default(),
prepare_failures: BTreeMap::new(),
ran: BTreeMap::new(),
docker_disk,
disk: None,
trim_due: true,
trim: None,
checked_at: SystemTime::now(),
github_limited_until: None,
registrations: Registrations::default(),
status,
announced_profiles: Some(announced.profiles),
control: control.clone(),
commands,
caches,
store: store.clone(),
paths,
tracked: watch::channel(Vec::new()).0,
found: mpsc::unbounded_channel(),
watches: Watches::default(),
leases: Leases::default(),
mirror_until: MirrorPosition::default(),
ledger: Ledger::default(),
last_prune: None,
running_reads: running_reads::RunningReads::default(),
governor: governor::Governor::default(),
measured: admission::Measured::new(SystemTime::now()),
warm: warm::Warm::default(),
records,
build_cache: build_cache.clone(),
archives,
actions: Arc::new(actions.clone()),
default_branches: BTreeMap::new(),
ended_uploads: BTreeMap::new(),
visibility_due: Arc::default(),
visibility_asking: None,
deletes: Arc::default(),
exits: Arc::default(),
watcher: None,
released: BTreeSet::new(),
passes_in_tick: Duration::ZERO,
listing: Mark {
listing: 0,
at: tokio::time::Instant::now(),
},
named: 0,
#[cfg(test)]
launches: start::Launches::default(),
claims: Claims::default(),
};
engine.publish();
(
engine,
Handle {
status: receiver,
control,
store,
caches: caches_receiver,
records: records_receiver,
build_cache,
disk: disk_receiver,
actions,
},
)
}
pub async fn run(mut self) {
let (disk, disk_read) = oneshot::channel();
let sampler = tokio::spawn(metrics::run(
Sampler::new(self.paths.clone(), &self.config.host.slice),
self.tracked.subscribe(),
self.store.clone(),
disk_read,
));
tokio::spawn(disk_target(self.docker.clone(), self.paths.clone(), disk));
let archives = self.archives.start(self.github.clone());
tokio::spawn(JobActions::clone(&self.actions).rerun_faults());
let control = self.control.clone();
let deadline = async {
control.hurried().await;
tokio::time::sleep(STOP_DEADLINE).await;
};
tokio::select! {
() = self.serve() => {}
() = deadline => warn!(
"stopped after {}s with Docker or GitHub still busy; the next start takes over what is left",
STOP_DEADLINE.as_secs()
),
}
self.note_stop_causes();
drop(self);
if let Some(archives) = archives {
archives.stop().await;
}
let _ = sampler.await;
}
async fn serve(&mut self) {
while let Err(error) = self.recover().await {
warn!("cannot list the containers an earlier run left; trying again in 5s: {error}");
if self.control.mode() != Shutdown::No {
return;
}
tokio::select! {
() = tokio::time::sleep(Duration::from_secs(5)) => {}
() = self.control.wake.notified() => {}
}
}
self.schedule().await;
}
async fn schedule(&mut self) {
self.watcher = Some(exits::Watcher::start(
self.docker.clone(),
self.exits.clone(),
));
let mut shown = Shutdown::No;
let mut scope = Scope::Every;
let mode = loop {
let mode = self.control.mode();
if mode != Shutdown::No && shown == Shutdown::No {
self.note_stop_causes();
}
if mode != shown {
self.publish();
shown = mode;
}
let anchor = if mode == Shutdown::Detach {
let began = tokio::time::Instant::now();
self.remove_every_idle().await;
self.collect_recordings().await;
began
} else {
self.tick_listing(mode, scope).await
};
self.publish();
if self.control.mode() != mode {
continue;
}
if self.stopped(mode) {
break mode;
}
let due = if mode == Shutdown::No {
self.next_tick_due(anchor)
} else {
let interval = Duration::from_secs(5);
(anchor + interval).max(tokio::time::Instant::now() + interval.min(TICK_REST))
};
scope = self.wait_for_tick(mode, due).await;
};
self.watcher = None;
self.collect_reseeds().await;
self.finish_deletes().await;
if self.control.docker_gone() {
info!("stopped; the next start removes or adopts what Docker did not answer for");
return;
}
self.close_shells().await;
if self.occupied().next().is_none()
&& let Err(error) = self.unsynced.stopped_clean()
{
warn!(
"cannot note that what runners wrote is on disk; the next boot empties their caches: {error}"
);
}
if mode == Shutdown::Detach {
let mirror = if self.config.caches.mirror.enabled {
" and the mirror"
} else {
""
};
match self.occupied().count() {
0 => info!("stopped; no job was running"),
1 => info!("stopped, leaving 1 busy runner{mirror} to the next start"),
runners => {
info!("stopped, leaving {runners} busy runners{mirror} to the next start");
}
}
return;
}
if let Err(error) = cache::mirror_down(&*self.docker, &self.names).await {
warn!("cannot stop the mirror: {error}");
}
info!("stopped");
}
fn pace_by_limit(&mut self) {
let spend = self.github.spend();
if let Some(change) =
self.governor
.observe(tokio::time::Instant::now(), SystemTime::now(), spend)
{
info!("{}", change.line(self.config.host.poll_interval));
}
}
async fn wait_for_tick(&mut self, mode: Shutdown, due: tokio::time::Instant) -> Scope {
loop {
let exits = self.exits.clone();
tokio::select! {
biased;
() = tokio::time::sleep_until(due) => return Scope::Due,
() = self.control.wake.notified() => return Scope::Every,
Some(command) = self.commands.recv(), if mode != Shutdown::Detach => {
self.command(command).await;
self.publish();
return Scope::Every;
}
Some(()) = self.records.finished.recv() => {
self.collect_recordings().await;
self.publish();
if mode != Shutdown::No || self.control.mode() != mode {
return Scope::Every;
}
}
Some(found) = self.found.1.recv() => {
self.note_found(&found);
self.publish();
}
() = exits.noted(), if mode != Shutdown::Detach => {
if mode != Shutdown::No {
return Scope::Every;
}
self.turnover().await;
self.publish();
if self.control.mode() != mode {
return Scope::Every;
}
}
}
}
}
fn stopped(&self, mode: Shutdown) -> bool {
let idle = self.slots.values().flatten().all(Slot::is_free);
let recorded = !self.records.is_recording();
match mode {
Shutdown::No => false,
Shutdown::Detach => recorded,
Shutdown::Graceful | Shutdown::Force => idle && recorded,
}
}
fn docker_failed(&self, error: &docker::Error) {
if docker::is_unreachable(error) && self.control.note_docker_gone() {
warn!("Docker does not answer during the stop; leaving the rest to the next start");
}
}
async fn serving<T: Send + 'static>(
&mut self,
work: impl Future<Output = T> + Send + 'static,
meanwhile: Meanwhile,
) -> T {
let mut task = tokio::spawn(work);
loop {
let exits = self.exits.clone();
tokio::select! {
biased;
done = &mut task => match done.map_err(tokio::task::JoinError::try_into_panic) {
Ok(output) => return output,
Err(Ok(panic)) => std::panic::resume_unwind(panic),
Err(Err(_)) => std::future::pending::<()>().await,
},
Some(command) = self.commands.recv() => {
self.command(command).await;
self.publish();
self.control.wake.notify_one();
}
() = exits.noted(), if meanwhile == Meanwhile::Turnover && self.passes_in_tick < CAPTURE_DRAIN => {
let began = tokio::time::Instant::now();
self.turnover().await;
self.passes_in_tick += began.elapsed();
self.publish();
}
Some(found) = self.found.1.recv(), if meanwhile == Meanwhile::Turnover => {
self.note_found(&found);
self.publish();
}
}
}
}
fn taken(&self) -> BTreeSet<u64> {
self.occupied()
.filter_map(|(_, occupant)| occupant.named_job())
.collect()
}
fn occupied(&self) -> impl Iterator<Item = (SlotRef, &Occupant)> {
self.slots.iter().flat_map(|(profile, slots)| {
slots.iter().enumerate().filter_map(|(index, slot)| {
slot.occupant().map(|occupant| {
(
SlotRef {
profile: profile.clone(),
index,
},
occupant,
)
})
})
})
}
fn footprints(&self) -> impl Iterator<Item = Footprint<'_>> {
self.slots.values().flatten().filter_map(Slot::footprint)
}
fn slot_footprints(&self) -> impl Iterator<Item = (SlotRef, Footprint<'_>)> {
self.slots.iter().flat_map(|(profile, slots)| {
slots.iter().enumerate().filter_map(|(index, slot)| {
slot.footprint().map(|footprint| {
(
SlotRef {
profile: profile.clone(),
index,
},
footprint,
)
})
})
})
}
fn occupant(&self, slot: &SlotRef) -> Option<&Occupant> {
self.slots
.get(&slot.profile)
.and_then(|slots| slots.get(slot.index))
.and_then(Slot::occupant)
}
fn drop_closed_slots(&mut self) {
for (name, slots) in &mut self.slots {
let count = self
.config
.profiles
.get(name)
.map_or(0, |profile| profile.slots as usize);
while slots.len() > count && slots.last().is_some_and(Slot::is_free) {
slots.pop();
}
}
let profiles = &self.config.profiles;
self.slots
.retain(|name, slots| !slots.is_empty() || profiles.contains_key(name));
let slots = &self.slots;
self.ran.retain(|slot, _| {
slots
.get(&slot.profile)
.is_some_and(|slots| slot.index < slots.len())
});
let (slots, upkeep) = (&self.slots, &self.upkeep);
self.retired
.retain(|name, _| slots.contains_key(name) || upkeep.reseed_runs(name));
}
}
async fn disk_target<D: DockerApi>(docker: Arc<D>, paths: Paths, disk: oneshot::Sender<Disks>) {
let root = loop {
if disk.is_closed() {
return;
}
match docker.root_dir().await {
Ok(root) => break root,
Err(_) => tokio::time::sleep(Duration::from_secs(5)).await,
}
};
let resolved = tokio::task::spawn_blocking({
let root = root.clone();
move || Disks::resolve(&paths, std::path::Path::new(&root))
})
.await;
if let Ok(Some(target)) = resolved {
info!("disk writes are read from {}, under {root}", target.name());
let _ = disk.send(target);
} else {
warn!("cannot tell the disks under Docker's root {root}: what runners write stays unknown");
}
}
#[cfg(test)]
mod tests;