use std::collections::HashMap;
use std::sync::Arc;
use std::time::SystemTime;
use tokio::sync::Semaphore;
use tokio::task::{JoinError, JoinSet};
use tracing::{info, warn};
use super::admission::Admitting;
use super::control::Control;
use super::runners::THIS_SPEC;
use super::slot::Slot;
use super::unsynced::Unsynced;
use super::{Engagement, Engine, Footprint, Occupant, Purpose, Shutdown, Stage, unix_secs};
use crate::api::{self, Basis, HoldCause};
use crate::docker::{DockerApi, RunnerSpec, RunnerState};
use crate::github::{self, GithubApi, Runner};
use crate::isolation::{self, Isolation};
use crate::scheduler::{Decision, Pool, SlotRef, StartFor};
pub(super) const CONCURRENT_STARTS: usize = 4;
pub(super) const POST_LANE: usize = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[must_use]
pub(super) enum StartOutcome {
Started,
Failed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Carried {
Done(StartOutcome),
Deferred,
Skipped,
}
#[derive(Debug, Clone)]
pub(super) struct Starting {
repository: String,
runner_name: String,
purpose: Purpose,
behind: Option<u64>,
admission: api::Admission,
memory: u64,
cpus: u64,
launched_at: SystemTime,
labels: Vec<String>,
image: String,
}
impl Starting {
pub(super) fn footprint(&self) -> Footprint<'_> {
Footprint {
runner_name: &self.runner_name,
memory: self.memory,
reserved: self.admission.reserved,
cpus: self.cpus,
reserved_cpu: self.admission.cpu.map_or(self.cpus, |cpu| cpu.reserved),
started_at: self.launched_at,
key: None,
job_found_here: None,
basis: Some(self.admission.basis),
stage: Stage::InFlight,
}
}
pub(super) fn repository(&self) -> &str {
&self.repository
}
pub(super) fn launched_at(&self) -> SystemTime {
self.launched_at
}
pub(super) fn behind(&self) -> Option<u64> {
self.behind
}
pub(super) fn into_occupant(self, runner_id: u64, container: String) -> Occupant {
Occupant {
repository: self.repository,
runner_name: self.runner_name,
runner_id,
container,
started_at: self.launched_at,
memory: self.memory,
cpus: self.cpus,
admission: Some(self.admission),
labels: Some(self.labels),
image: Some(self.image),
spec: Some(THIS_SPEC.to_owned()),
engagement: Engagement::started(self.purpose, self.behind),
}
}
}
#[cfg(test)]
impl Starting {
pub(super) fn new(
repository: &str,
runner_name: &str,
admission: api::Admission,
memory: u64,
cpus: u64,
) -> Starting {
Starting {
repository: repository.to_owned(),
runner_name: runner_name.to_owned(),
purpose: Purpose::Job,
behind: None,
admission,
memory,
cpus,
launched_at: SystemTime::now(),
labels: vec!["self-hosted".to_owned(), "app".to_owned()],
image: "runnerd-runner:app".to_owned(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Filter {
Carried,
Lacking,
}
fn runner_skips(skip_syncs: bool, filter: Filter, unsynced: &mut Unsynced) -> bool {
if !skip_syncs || filter == Filter::Lacking {
return false;
}
let skips = unsynced.runner_skips();
if !skips {
warn!(
"a runner keeps its syncs: that it writes without them could not be kept for after a host crash"
);
}
skips
}
fn container_labels(
name: &str,
repository: &str,
slot: usize,
runner_id: u64,
labels: &[String],
image: &str,
) -> std::collections::BTreeMap<String, String> {
std::collections::BTreeMap::from([
("runnerd.job".to_owned(), name.to_owned()),
("runnerd.repository".to_owned(), repository.to_owned()),
("runnerd.slot".to_owned(), slot.to_string()),
("runnerd.runner-id".to_owned(), runner_id.to_string()),
(super::runners::LABELS.to_owned(), labels.join(",")),
(super::runners::IMAGE.to_owned(), image.to_owned()),
(super::runners::SPEC.to_owned(), THIS_SPEC.to_owned()),
])
}
struct Request {
repository: String,
number: usize,
labels: Vec<String>,
spec: RunnerSpec,
}
enum StartRun {
Unregistered(github::Error),
Stopped,
NoContainer {
runner: Runner,
error: String,
may_have_run: bool,
},
Container {
runner: Runner,
container: String,
made: Made,
},
}
enum Made {
Running(Result<RunnerState, crate::docker::Error>),
Failed(String),
}
async fn run<G: GithubApi, D: DockerApi>(
github: Arc<G>,
docker: Arc<D>,
control: Arc<Control>,
isolation: &'static dyn Isolation,
lane: Arc<Semaphore>,
request: Request,
) -> StartRun {
let (repository, name) = (request.repository.as_str(), request.spec.name.clone());
let registered = {
let Ok(_turn) = lane.acquire().await else {
return StartRun::Stopped;
};
if control.mode() != Shutdown::No {
return StartRun::Stopped;
}
github.jit_config(repository, &name, &request.labels).await
};
let (runner, jit_config) = match registered {
Ok(registered) => registered,
Err(error) => return StartRun::Unregistered(error),
};
if control.mode() != Shutdown::No {
info!("{repository}: stopping; deleting {name} before it starts");
if let Err(error) = github.delete_runner(repository, runner.id).await {
warn!("{repository}: cannot delete {name}: {error}");
}
return StartRun::Stopped;
}
let spec = RunnerSpec {
labels: container_labels(
&name,
repository,
request.number,
runner.id,
&request.labels,
&request.spec.image,
),
jit_config: Some(jit_config),
..request.spec
};
match docker.start(&spec, isolation).await {
Ok(container) => {
let inspected = docker.inspect(&container).await;
StartRun::Container {
runner,
container,
made: Made::Running(inspected),
}
}
Err(failed) => match failed.container {
Some(container) => StartRun::Container {
runner,
container,
made: Made::Failed(failed.error.to_string()),
},
None => StartRun::NoContainer {
runner,
error: failed.error.to_string(),
may_have_run: failed.may_have_run,
},
},
}
}
#[cfg(test)]
#[derive(Debug, Clone, Default)]
pub(super) struct Launches(Arc<std::sync::Mutex<Vec<String>>>);
#[cfg(test)]
impl Launches {
fn note(&self, runner: &str) {
crate::sync::lock(&self.0).push(runner.to_owned());
}
pub(super) fn count(&self) -> usize {
crate::sync::lock(&self.0).len()
}
pub(super) fn after(&self, count: usize) -> Vec<String> {
crate::sync::lock(&self.0)
.get(count..)
.unwrap_or_default()
.to_vec()
}
}
struct Launched {
slot: SlotRef,
repository: String,
job: Option<u64>,
behind: Option<u64>,
}
enum Launch {
Deferred,
Failed(Launched),
Readied(Readied),
}
struct Readied {
launched: Launched,
admission: api::Admission,
purpose: Purpose,
labels: Vec<String>,
name: String,
at: SystemTime,
filter: Filter,
}
enum Committed {
Running(Launched, Box<Request>),
Failed(Launched),
}
struct InFlight {
tasks: JoinSet<StartRun>,
starts: HashMap<tokio::task::Id, (usize, Launched)>,
}
impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
pub(super) async fn start_batch(
&mut self,
starts: Vec<Decision>,
admitting: &mut Admitting,
) -> Vec<Carried> {
let mut carried = vec![Carried::Skipped; starts.len()];
let mut in_flight = InFlight {
tasks: JoinSet::new(),
starts: HashMap::new(),
};
let lane = Arc::new(Semaphore::new(POST_LANE));
let mut deferred = false;
let mut starts = starts.into_iter().enumerate();
loop {
while in_flight.tasks.len() < CONCURRENT_STARTS
&& let Some((index, start)) = starts.next()
{
if let Decision::Start {
slot, repository, ..
} = &start
&& self
.barred_once_folded(&mut in_flight, &mut carried, repository, &slot.profile)
.await
{
continue;
}
if deferred {
carried[index] = Carried::Deferred;
continue;
}
let readied = match self.launch(start, admitting).await {
Launch::Deferred => {
deferred = true;
carried[index] = Carried::Deferred;
continue;
}
Launch::Failed(launched) => {
self.started(&launched, StartOutcome::Failed);
carried[index] = Carried::Done(StartOutcome::Failed);
continue;
}
Launch::Readied(readied) => readied,
};
let launched = &readied.launched;
if self
.barred_once_folded(
&mut in_flight,
&mut carried,
&launched.repository,
&launched.slot.profile,
)
.await
{
continue;
}
let cpu = readied.admission.cpu.as_ref().map_or(0, |cpu| cpu.reserved);
if !self.still_fits(
admitting,
&launched.slot.profile,
readied.admission.reserved,
cpu,
) {
info!(
"{}: its runner in {} no longer fits what runs now; planning again",
launched.repository, launched.slot.profile
);
deferred = true;
carried[index] = Carried::Deferred;
continue;
}
match self.commit(readied) {
Committed::Failed(launched) => {
self.started(&launched, StartOutcome::Failed);
carried[index] = Carried::Done(StartOutcome::Failed);
}
Committed::Running(launched, request) => {
#[cfg(test)]
self.launches.note(&request.spec.name);
let task = in_flight.tasks.spawn(run(
self.github.clone(),
self.docker.clone(),
self.control.clone(),
self.isolation,
lane.clone(),
*request,
));
in_flight.starts.insert(task.id(), (index, launched));
}
}
}
let Some(joined) = in_flight.tasks.join_next_with_id().await else {
break;
};
self.fold_joined(joined, &mut in_flight, &mut carried).await;
}
carried
}
async fn fold_joined(
&mut self,
joined: Result<(tokio::task::Id, StartRun), JoinError>,
in_flight: &mut InFlight,
carried: &mut [Carried],
) {
let (id, ran) = match joined {
Ok(done) => done,
Err(error) => {
if let Ok(panic) = error.try_into_panic() {
std::panic::resume_unwind(panic);
}
return std::future::pending::<()>().await;
}
};
let Some((index, launched)) = in_flight.starts.remove(&id) else {
return;
};
let became = self.fold(&launched.slot, ran).await;
if let Carried::Done(outcome) = became {
self.started(&launched, outcome);
}
carried[index] = became;
}
async fn barred_once_folded(
&mut self,
in_flight: &mut InFlight,
carried: &mut [Carried],
repository: &str,
profile: &str,
) -> bool {
while let Some(joined) = in_flight.tasks.try_join_next_with_id() {
self.fold_joined(joined, in_flight, carried).await;
}
self.launch_barred(repository, profile)
}
fn launch_barred(&self, repository: &str, profile: &str) -> bool {
self.control.mode() != Shutdown::No
|| self.unsynced.is_torn()
|| self.github_limit().is_some()
|| self.backing_off(repository, profile, SystemTime::now())
}
async fn launch(&mut self, start: Decision, admitting: &mut Admitting) -> Launch {
let Decision::Start {
slot,
repository,
memory,
cpu,
for_,
replacing,
behind,
} = start
else {
return Launch::Deferred;
};
if replacing.iter().any(|slot| self.holds_runner(slot)) {
info!("{repository}: a warm runner it would replace took a job; planning again");
return Launch::Deferred;
}
if !self.still_fits(admitting, &slot.profile, memory, cpu) {
info!(
"{repository}: its runner in {} no longer fits what runs now; planning again",
slot.profile
);
return Launch::Deferred;
}
if let Some(owner) = behind {
info!(
"{repository}: a runner in {} starts behind the barrier of job {owner}",
slot.profile
);
}
let pool = Pool {
repository,
profile: slot.profile.clone(),
};
self.refresh_free(admitting);
let mut admission = admitting.admit(&pool, memory, cpu);
let repository = pool.repository;
let (purpose, job) = if let StartFor::Job(job) = for_ {
let held_since = self
.repositories
.listed(&repository)
.and_then(|state| state.waits(&slot.profile)?.cpu())
.filter(|wait| wait.job == job)
.map(|wait| wait.since);
if let Some((cpu, since)) = admission.cpu.as_mut().zip(held_since) {
cpu.waited = Some(since.elapsed().unwrap_or_default().as_secs());
}
(Purpose::Job, Some(job))
} else {
admission.basis = Basis::Cap;
if let Some(cpu) = admission.cpu.as_mut() {
cpu.basis = Basis::Cap;
}
let purpose = self.warm_purpose(&Pool {
repository: repository.clone(),
profile: slot.profile.clone(),
});
(purpose, None)
};
let launched = Launched {
slot,
repository,
job,
behind,
};
self.launch_admitted(launched, admission, purpose).await
}
async fn launch_admitted(
&mut self,
launched: Launched,
admission: api::Admission,
purpose: Purpose,
) -> Launch {
let (slot, repository) = (&launched.slot, launched.repository.as_str());
let Some(labels) = self
.config
.pool_in(repository, &slot.profile)
.map(|pool| pool.labels.clone())
else {
return Launch::Failed(launched);
};
let now = SystemTime::now();
if !self.caches_ready(slot, now).await {
return Launch::Failed(launched);
}
let filter = self.image_filter(&slot.profile).await;
let number = slot.index + 1;
let secs = unix_secs(now).max(self.named + 1);
self.named = secs;
if !self.store.runner_named(secs).await {
warn!("{repository}: no runner starts until the history records its name");
return Launch::Failed(launched);
}
let name = self.identity.runner_name(&slot.profile, number, secs);
self.start_mirror().await;
Launch::Readied(Readied {
launched,
admission,
purpose,
labels,
name,
at: now,
filter,
})
}
async fn image_filter(&self, profile: &str) -> Filter {
let Some(profile) = self.config.profiles.get(profile) else {
return Filter::Lacking;
};
if !self.config.host.skip_syncs {
return Filter::Lacking;
}
match self
.docker
.image_skips_syncs(&profile.image.reference())
.await
{
Ok(true) => Filter::Carried,
Ok(false) | Err(_) => Filter::Lacking,
}
}
fn commit(&mut self, readied: Readied) -> Committed {
let Readied {
launched,
admission,
purpose,
labels,
name,
at: now,
filter,
} = readied;
let (slot, repository) = (&launched.slot, launched.repository.as_str());
let number = slot.index + 1;
let Some(profile) = self.config.profiles.get(&slot.profile) else {
return Committed::Failed(launched);
};
let image = profile.image.reference();
let skips = runner_skips(self.config.host.skip_syncs, filter, &mut self.unsynced);
let spec = RunnerSpec {
name: name.clone(),
image: image.clone(),
profile: slot.profile.clone(),
slice: self.config.host.slice.clone(),
cpus: profile.cpus,
memory: profile.memory,
env: self.runner_env(skips),
mounts: self.runner_mounts(&slot.profile, number, &name, repository),
network: Some(self.names.network()),
labels: std::collections::BTreeMap::new(),
entrypoint: None,
jit_config: None,
};
let starting = Starting {
repository: repository.to_owned(),
runner_name: name,
purpose,
behind: launched.behind,
admission,
memory: spec.memory.0,
cpus: u64::from(spec.cpus) * 1000,
launched_at: now,
labels: labels.clone(),
image,
};
let Some(entry) = self
.slots
.get_mut(&slot.profile)
.and_then(|slots| slots.get_mut(slot.index))
else {
self.close_build_cache(&spec.name);
return Committed::Failed(launched);
};
entry.launch(starting);
self.owe_runners_listing(repository);
let request = Request {
repository: repository.to_owned(),
number,
labels,
spec,
};
Committed::Running(launched, Box::new(request))
}
async fn fold(&mut self, slot: &SlotRef, ran: StartRun) -> Carried {
let Some(starting) = self
.slots
.get_mut(&slot.profile)
.and_then(|slots| slots.get_mut(slot.index))
.and_then(Slot::end_start)
else {
return Carried::Done(StartOutcome::Failed);
};
let repository = starting.repository.clone();
let outcome = match ran {
StartRun::Unregistered(github::Error::RateLimited { until }) => {
self.close_build_cache(&starting.runner_name);
self.github_limited(until);
return Carried::Skipped;
}
StartRun::Unregistered(error) => {
self.close_build_cache(&starting.runner_name);
let failure = format!("cannot register a runner: {error}");
warn!("{repository}: {failure}");
self.hold(&repository, &slot.profile, HoldCause::Register, failure);
StartOutcome::Failed
}
StartRun::Stopped => {
self.close_build_cache(&starting.runner_name);
StartOutcome::Failed
}
StartRun::NoContainer {
runner,
error,
may_have_run,
} => {
let failure = isolation::start_failure(&starting.runner_name, &error);
warn!("{repository}: {failure}");
self.recheck_image(&starting.image);
if may_have_run {
self.spend(&repository, &slot.profile);
}
self.abandon(&slot.profile, &repository, &runner, failure);
StartOutcome::Failed
}
StartRun::Container {
runner,
container,
made,
} => {
let made = match made {
Made::Running(inspected) => Ok(inspected),
Made::Failed(error) => {
let failure = isolation::start_failure(&starting.runner_name, &error);
warn!("{repository}: {failure}");
self.recheck_image(&starting.image);
Err(failure)
}
};
let purpose = starting.purpose;
let occupant = starting.into_occupant(runner.id, container);
self.place(slot, occupant, purpose, made).await
}
};
Carried::Done(outcome)
}
async fn place(
&mut self,
slot: &SlotRef,
occupant: Occupant,
purpose: Purpose,
made: Result<Result<RunnerState, crate::docker::Error>, String>,
) -> StartOutcome {
if let Some(entry) = self
.slots
.get_mut(&slot.profile)
.and_then(|slots| slots.get_mut(slot.index))
{
entry.place(occupant.clone());
}
let (repository, name, number) =
(&occupant.repository, &occupant.runner_name, slot.index + 1);
match made {
Ok(inspected) => {
let warm = match purpose {
Purpose::Job => "",
Purpose::Warm => " warm, for jobs not queued yet",
Purpose::Burst => " warm, for a job its runs are expected to queue",
};
info!(
"{repository}: started {name} in {} slot {number}{warm}",
slot.profile
);
self.watch_inspected(slot, &occupant, inspected).await;
StartOutcome::Started
}
Err(failure) => {
warn!(
"{repository}: {name} may be left in Docker; {} slot {number} stays its until it is removed",
slot.profile
);
self.hold(repository, &slot.profile, HoldCause::Start, failure);
StartOutcome::Failed
}
}
}
fn started(&mut self, launched: &Launched, outcome: StartOutcome) {
let Some(job) = launched.job else {
return;
};
let (slot, repository) = (&launched.slot, &launched.repository);
if self.holds_runner(slot) {
self.note_started(job);
}
if outcome == StartOutcome::Started
&& !self.backing_off(repository, &slot.profile, SystemTime::now())
&& let Some(waits) = self
.repositories
.listed_mut(repository)
.and_then(|state| state.waits_mut(&slot.profile))
&& waits.started()
{
info!(
"{repository}: a barrier served its pool in {}: its next jobs hold sooner until its queue empties",
slot.profile
);
}
}
}