use std::fmt::Write as _;
use std::panic::AssertUnwindSafe;
use std::time::SystemTime;
use futures_util::FutureExt as _;
use super::*;
use crate::admission::host;
use crate::admission::pooled::{self, POOL_AFTER};
use crate::api::{Basis, HoldCause, JobPeak};
use crate::config::Admission;
use crate::observe::metrics::Events;
use crate::scheduler::engine::clock::Clock;
use crate::scheduler::{Decision, IDLE_TIMEOUT, PRIORITY_AFTER};
use crate::store::{RunnerRecord, SampleRecord};
use crate::time;
const G: u64 = 1 << 30;
const M: u64 = 1 << 20;
const MEMORY_MAX: u64 = 8 * G + 512 * M;
const SLICE_CPUS: u64 = 3000;
const HOST_TOTAL: u64 = 64 * G;
const HOST_MARGIN: u64 = HOST_TOTAL / 16;
const SEED_START: &str = "'s seed container";
const CASES: u64 = 64;
const EVENTS: usize = 40;
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
z ^ (z >> 31)
}
fn below(&mut self, n: u64) -> u64 {
self.next() % n
}
fn index(&mut self) -> usize {
usize::try_from(self.below(8)).unwrap()
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
#[expect(
clippy::struct_excessive_bools,
reason = "each is a config choice a reload flips on its own"
)]
struct Shape {
admission: Admission,
cpu_overcommit: bool,
heavy_slots: u32,
light_slots: Option<u32>,
light_pool: bool,
second_repository: bool,
pooled_count: bool,
heavy_memory: u64,
}
struct PoolShape {
repository: &'static str,
profile: &'static str,
labels: &'static [&'static str],
}
impl Shape {
const START: Shape = Shape {
admission: Admission::Caps,
cpu_overcommit: false,
heavy_slots: 2,
light_slots: Some(2),
light_pool: true,
second_repository: true,
pooled_count: true,
heavy_memory: 4,
};
fn pools(&self) -> Vec<PoolShape> {
let mut pools = vec![PoolShape {
repository: "o/r",
profile: "heavy",
labels: &["self-hosted", "app"],
}];
if self.light_pool && self.light_slots.is_some() {
pools.push(PoolShape {
repository: "o/r",
profile: "light",
labels: &["r-light"],
});
}
if self.second_repository {
pools.push(PoolShape {
repository: "o/s",
profile: "heavy",
labels: &["self-hosted", "s-app"],
});
}
pools
}
fn toml(&self) -> String {
let mut host = String::new();
if self.admission == Admission::Measured {
host.push_str("admission = \"measured\"\n");
}
if self.cpu_overcommit {
host.push_str("cpu_overcommit = 1.0\n");
}
if !self.pooled_count {
host.push_str("pooled_count = false\n");
}
let mut config = format!(
r#"
[host]
slice = "ci.slice"
isolation = "sysbox"
{host}
[caches]
mirror = {{ enabled = false, memory = "512M", cpus = 1, ttl = "168h" }}
pnpm = {{ max_size = "5G" }}
maven = {{ max_size = "5G" }}
[profiles.heavy]
image_stage = "app"
cpus = 2
memory = "{}G"
slots = {}
"#,
self.heavy_memory, self.heavy_slots
);
if let Some(slots) = self.light_slots {
let _ = write!(
config,
"\n[profiles.light]\nimage_stage = \"app\"\ncpus = 1\nmemory = \"1G\"\nslots = {slots}\n"
);
}
for pool in self.pools() {
let labels: Vec<String> = pool
.labels
.iter()
.map(|label| format!("{label:?}"))
.collect();
let _ = write!(
config,
"\n[[repositories]]\nname = {:?}\nprofile = {:?}\nlabels = [{}]\nwarm = 0\n",
pool.repository,
pool.profile,
labels.join(", ")
);
}
config
}
fn serves(&self, repository: &str, labels: &[String]) -> Option<&'static str> {
if labels.is_empty() {
return None;
}
let serving: Vec<&'static str> = self
.pools()
.into_iter()
.filter(|pool| {
pool.repository == repository
&& labels.iter().all(|label| {
pool.labels
.iter()
.any(|has| has.eq_ignore_ascii_case(label))
})
})
.map(|pool| pool.profile)
.collect();
assert!(serving.len() <= 1, "the shape has pools sharing labels");
serving.first().copied()
}
fn configures(&self, repository: &str) -> bool {
self.pools()
.iter()
.any(|pool| pool.repository == repository)
}
}
#[derive(Debug, Clone, Copy)]
enum Labels {
Heavy,
Light,
SelfHosted,
Typo,
Empty,
Hosted,
}
impl Labels {
fn of(self, repository: &str) -> Vec<String> {
let labels: &[&str] = match (self, repository) {
(Labels::Heavy, "o/r") => &["self-hosted", "app"],
(Labels::Heavy, _) => &["self-hosted", "s-app"],
(Labels::Light, "o/r") => &["r-light"],
(Labels::Light, _) => &["s-light"],
(Labels::SelfHosted, _) => &["self-hosted"],
(Labels::Typo, _) => &["r-lite"],
(Labels::Empty, _) => &[],
(Labels::Hosted, _) => &["ubuntu-latest"],
};
labels.iter().map(|label| (*label).to_owned()).collect()
}
}
#[derive(Debug, Clone, Copy)]
enum Change {
HeavySlots(u32),
LightSlots(u32),
LightPool,
SecondRepository,
LightProfile,
Policy,
CpuOvercommit,
PooledCount,
HeavyMemory,
}
#[derive(Debug, Clone, Copy)]
enum Step {
Queue {
second: bool,
labels: Labels,
},
Assign(usize),
AssignListed(usize),
Finish(usize),
Cancel(usize),
Lose(usize),
Removing(usize),
StartFails,
StartFailsLate {
takes_job: bool,
cleanup_fails: bool,
},
CaptureFails {
mid_stream: bool,
},
ExitsAtOnce(usize),
ExitsIdle(usize),
ExitsConnected(usize),
InspectFails(usize),
StartedInspectFails,
RemoveFails,
Sample {
percent: u64,
available: u64,
stalled: bool,
},
SampleDuringStart {
percent: u64,
available: u64,
stalled: bool,
},
Reseed {
light: bool,
},
Reload(Change),
Restart,
RestartInspectFails(usize),
Stop {
docker_gone: bool,
},
Elapse,
Settle,
PastRuns,
Tick,
Pass,
}
fn generate(rng: &mut Rng) -> Vec<Step> {
(0..EVENTS)
.map(|_| match rng.below(100) {
0..=17 => Step::Queue {
second: rng.below(3) == 0,
labels: match rng.below(10) {
0..=3 => Labels::Heavy,
4..=5 => Labels::Light,
6 => Labels::SelfHosted,
7 => Labels::Typo,
8 => Labels::Empty,
_ => Labels::Hosted,
},
},
18..=26 => Step::Assign(rng.index()),
27..=29 => Step::AssignListed(rng.index()),
30..=37 => Step::Finish(rng.index()),
38..=39 => Step::Cancel(rng.index()),
40 => Step::Lose(rng.index()),
96 => Step::Removing(rng.index()),
41..=42 => Step::StartFails,
43 => Step::ExitsAtOnce(rng.index()),
44 => Step::ExitsIdle(rng.index()),
95 => Step::ExitsConnected(rng.index()),
45..=46 => Step::InspectFails(rng.index()),
47 => Step::RemoveFails,
48..=54 => Step::Sample {
percent: [10, 50, 90, 120][rng.index() % 4],
available: [3, 6, 8, 9, 20, 50][rng.index() % 6],
stalled: rng.below(5) == 0,
},
55..=62 => Step::Reload(match rng.below(9) {
0 => Change::HeavySlots(1 + u32::try_from(rng.below(3)).unwrap()),
1 => Change::LightSlots(1 + u32::try_from(rng.below(3)).unwrap()),
2 => Change::LightPool,
3 => Change::SecondRepository,
4 => Change::LightProfile,
5 => Change::Policy,
6 => Change::PooledCount,
7 => Change::HeavyMemory,
_ => Change::CpuOvercommit,
}),
63..=65 => Step::Restart,
66..=67 => Step::RestartInspectFails(rng.index()),
68..=71 => Step::Elapse,
72..=73 => Step::Reseed {
light: rng.below(2) == 0,
},
74 => Step::StartedInspectFails,
75..=76 => Step::StartFailsLate {
takes_job: rng.below(2) == 0,
cleanup_fails: rng.below(2) == 0,
},
77..=78 => Step::SampleDuringStart {
percent: [10, 50, 90, 120][rng.index() % 4],
available: [3, 8, 20, 50][rng.index() % 4],
stalled: rng.below(5) == 0,
},
79..=80 => Step::CaptureFails {
mid_stream: rng.below(2) == 0,
},
81..=82 => Step::Stop {
docker_gone: rng.below(2) == 0,
},
83..=89 => Step::Pass,
90..=93 => Step::Settle,
_ => Step::Tick,
})
.collect()
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Listing {
Current,
Stale,
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Exit {
AtOnce,
AfterIdleTimeout,
Connected,
}
#[derive(Debug, Clone)]
struct ExitWatch {
container: String,
repository: String,
profile: String,
exit: Exit,
quiet_before: bool,
}
#[derive(Debug, Clone, PartialEq)]
enum Phase {
Queued,
Running {
container: String,
},
Done {
ran: Option<(String, String)>,
},
}
#[derive(Debug, Clone)]
struct JobModel {
repository: &'static str,
labels: Vec<String>,
phase: Phase,
taken_by: Option<(String, String)>,
listed: bool,
named_by_github: bool,
}
#[derive(Debug, Clone, Copy)]
struct HostReading {
at: u64,
slice: u64,
available: u64,
stalled: bool,
}
impl HostReading {
fn fresh(&self, now: u64) -> bool {
self.at + 3 * api::SAMPLE_INTERVAL >= now / 1000
}
}
#[derive(Debug, Clone)]
struct Violation {
invariant: &'static str,
step: usize,
detail: String,
}
#[derive(Debug, Clone, Copy, Default)]
struct Seen {
barriers: usize,
credits: usize,
bands: usize,
released: usize,
exits_freed: usize,
}
struct Case {
seen: Seen,
clock: Clock,
github: FakeGithub,
docker: FakeDocker,
dir: TestDir,
engine: Engine<FakeGithub, FakeDocker>,
handle: Handle,
shape: Shape,
jobs: BTreeMap<u64, JobModel>,
next_job: u64,
took_job: BTreeSet<String>,
still_listed: BTreeSet<u64>,
cancelled_since_tick: BTreeSet<u64>,
calls_seen: usize,
host: Option<HostReading>,
host_during_start: Arc<Mutex<Option<HostReading>>>,
capture_faults: HashMap<String, u32>,
exits: Vec<ExitWatch>,
busy_before: Vec<(String, String, String)>,
step: usize,
}
fn violation(case: &Case, invariant: &'static str, detail: String) -> Result<(), Violation> {
Err(Violation {
invariant,
step: case.step,
detail,
})
}
impl Case {
fn new() -> Case {
let github = FakeGithub::default();
let docker = FakeDocker::default();
github.0.lock().unwrap().strict = true;
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.seed_exit = Some(0);
}
let dir = TestDir::new("invariants");
let shape = Shape::START;
let clock = Clock::stopped();
let (engine, handle) = Case::engine(&github, &docker, &dir, shape, clock);
Case {
seen: Seen::default(),
clock,
github,
docker,
dir,
engine,
handle,
shape,
jobs: BTreeMap::new(),
next_job: 1,
took_job: BTreeSet::new(),
still_listed: BTreeSet::new(),
cancelled_since_tick: BTreeSet::new(),
calls_seen: 0,
host: None,
host_during_start: Arc::default(),
capture_faults: HashMap::new(),
exits: Vec::new(),
busy_before: Vec::new(),
step: 0,
}
}
fn engine(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
shape: Shape,
clock: Clock,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let (mut engine, handle) = engine_with(github, docker, dir, &shape.toml());
engine.clock = clock;
engine.memory_max = Some(MEMORY_MAX);
engine.cpus = Some(SLICE_CPUS);
(engine, handle)
}
async fn recover(&mut self) -> Result<(), Violation> {
for _ in 0..10 {
if self.engine.recover().await.is_ok() {
return Ok(());
}
}
violation(
self,
"recovery_succeeds",
"Docker never answered".to_owned(),
)
}
fn running_containers(&self) -> Vec<Container> {
self.docker
.0
.lock()
.unwrap()
.containers
.iter()
.filter(|container| container.running && container.labels.contains_key("runnerd.job"))
.cloned()
.collect()
}
fn idle_containers(&self) -> Vec<Container> {
self.running_containers()
.into_iter()
.filter(|container| !self.took_job.contains(&container.id))
.collect()
}
fn queued(&self) -> Vec<u64> {
self.jobs
.iter()
.filter(|(_, job)| job.phase == Phase::Queued)
.map(|(id, _)| *id)
.collect()
}
fn running(&self) -> Vec<(u64, String)> {
self.jobs
.iter()
.filter_map(|(id, job)| match &job.phase {
Phase::Running { container } => Some((*id, container.clone())),
_ => None,
})
.collect()
}
fn status(&mut self) -> api::Status {
self.engine.publish();
self.handle.status.borrow().clone()
}
#[allow(
clippy::too_many_lines,
reason = "one arm per step, which split up would only be listed twice"
)]
async fn apply(&mut self, step: Step) -> Result<(), Violation> {
match step {
Step::Queue { second, labels } => {
let repository = if second { "o/s" } else { "o/r" };
let id = self.next_job;
self.next_job += 1;
let labels = labels.of(repository);
let mut github = self.github.0.lock().unwrap();
github.queued.push(QueuedJob {
id,
run_id: id,
name: "build".to_owned(),
labels: labels.clone(),
created_at: time::now_ms(),
..QueuedJob::default()
});
github.queued_on.insert(id, repository.to_owned());
self.jobs.insert(
id,
JobModel {
repository,
labels,
phase: Phase::Queued,
taken_by: None,
listed: false,
named_by_github: false,
},
);
}
Step::Assign(pick) => self.assign(pick, Listing::Current).await,
Step::AssignListed(pick) => self.assign(pick, Listing::Stale).await,
Step::Finish(pick) => {
let running = self.running();
if let Some((id, container)) = pick_from(&running, pick).cloned() {
self.end_job(id, &container, "success");
if (self.unobserved(&container) || self.capture_unread(id, &container))
&& let Some(job) = self.jobs.get_mut(&id)
{
job.phase = Phase::Done { ran: None };
}
let mut docker = self.docker.0.lock().unwrap();
docker
.container_logs
.insert(container.clone(), "Running job: build\n".to_owned());
set_running(&mut docker, &container, false);
}
}
Step::Cancel(pick) => {
let queued = self.queued();
if let Some(id) = pick_from(&queued, pick).copied() {
let mut github = self.github.0.lock().unwrap();
github.queued.retain(|job| job.id != id);
let mut cancelled = details(id, "completed", Some("cancelled"));
cancelled.runner_name = None;
github.jobs.insert(id, cancelled);
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Done { ran: None };
if job.listed {
self.cancelled_since_tick.insert(id);
}
}
}
Step::Lose(pick) => self.lose(pick),
Step::Removing(pick) => self.removing(pick),
Step::StartFails => self.docker.0.lock().unwrap().fail_starts += 1,
Step::StartFailsLate {
takes_job,
cleanup_fails,
} => {
let failure = LateFailure {
takes_job,
cleanup_fails,
};
self.docker
.0
.lock()
.unwrap()
.late_failures
.push_back(failure);
}
Step::CaptureFails { mid_stream } => {
let mut docker = self.docker.0.lock().unwrap();
*if mid_stream {
&mut docker.break_captures
} else {
&mut docker.fail_execs
} += 1;
}
Step::ExitsAtOnce(pick) => self.exit_idle(pick, Exit::AtOnce),
Step::ExitsIdle(pick) => self.exit_idle(pick, Exit::AfterIdleTimeout),
Step::ExitsConnected(pick) => self.exit_idle(pick, Exit::Connected),
Step::InspectFails(pick) => self.inspect_fails(pick),
Step::StartedInspectFails => self.docker.0.lock().unwrap().fail_started_inspects += 1,
Step::RemoveFails => self.docker.0.lock().unwrap().fail_removes += 1,
Step::Sample {
percent,
available,
stalled,
} => {
let docker = self.docker.0.lock().unwrap();
let reading = sample(
&docker,
&self.handle.store,
&self.took_job,
self.clock.now_ms(),
percent,
available,
stalled,
);
drop(docker);
self.host = Some(reading);
}
Step::SampleDuringStart {
percent,
available,
stalled,
} => self.sample_during_start(percent, available, stalled),
Step::Reseed { light } => {
let _ = self
.engine
.request_reseed(if light { "light" } else { "heavy" });
}
Step::Reload(change) => {
self.reload(change).await?;
self.tick().await?;
}
Step::Restart => {
self.restart().await?;
self.tick().await?;
}
Step::Stop { docker_gone } => {
self.stop(docker_gone).await?;
self.restart().await?;
self.tick().await?;
}
Step::RestartInspectFails(pick) => {
self.inspect_fails(pick);
self.restart().await?;
self.tick().await?;
}
Step::Elapse => self.elapse(),
Step::Settle => self.settle(),
Step::PastRuns => self.past_runs().await,
Step::Tick => self.tick().await?,
Step::Pass => self.pass().await?,
}
self.no_removal_of_a_running_job()
}
fn lose(&mut self, pick: usize) {
if let Some(container) = self.end_running(pick) {
self.docker
.0
.lock()
.unwrap()
.containers
.retain(|candidate| candidate.id != container);
}
}
fn removing(&mut self, pick: usize) {
if let Some(container) = self.end_running(pick) {
let mut docker = self.docker.0.lock().unwrap();
set_running(&mut docker, &container, false);
docker.being_removed.insert(container);
}
}
fn removals_finish(&mut self) {
let mut docker = self.docker.0.lock().unwrap();
let removed = std::mem::take(&mut docker.being_removed);
docker
.containers
.retain(|candidate| !removed.contains(&candidate.id));
}
fn end_running(&mut self, pick: usize) -> Option<String> {
let running = self.running();
let (id, container) = pick_from(&running, pick).cloned()?;
self.end_job(id, &container, "failure");
if (self.unobserved(&container) || self.capture_unread(id, &container))
&& let Some(job) = self.jobs.get_mut(&id)
{
job.phase = Phase::Done { ran: None };
}
Some(container)
}
fn inspect_fails(&mut self, pick: usize) {
let containers = self.running_containers();
if let Some(container) = pick_from(&containers, pick) {
*self
.docker
.0
.lock()
.unwrap()
.fail_inspects
.entry(container.id.clone())
.or_default() += 1;
}
}
async fn assign(&mut self, pick: usize, listing: Listing) {
let idle = self.idle_containers();
let registered: BTreeSet<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
let candidates: Vec<(u64, Container)> = self
.queued()
.into_iter()
.flat_map(|id| {
let job = &self.jobs[&id];
idle.iter()
.filter(|container| {
let label =
|name: &str| container.labels.get(name).cloned().unwrap_or_default();
let labels: Vec<String> = label("runnerd.labels")
.split(',')
.map(str::to_owned)
.collect();
registered.contains(&container.name)
&& label("runnerd.repository") == job.repository
&& !job.labels.is_empty()
&& job.labels.iter().all(|wanted| {
labels.iter().any(|has| has.eq_ignore_ascii_case(wanted))
})
})
.map(move |container| (id, container.clone()))
.collect::<Vec<_>>()
})
.collect();
let Some((id, container)) = pick_from(&candidates, pick).cloned() else {
return;
};
let profile = container.labels["runnerd.profile"].clone();
let repository = container.labels["runnerd.repository"].clone();
{
let mut github = self.github.0.lock().unwrap();
if listing == Listing::Stale {
self.still_listed.insert(id);
} else {
github.queued.retain(|job| job.id != id);
}
let mut runner_id = None;
for runner in &mut github.runners {
if runner.name == container.name {
runner.busy = true;
runner_id = Some(runner.id);
}
}
let mut running = details(id, "in_progress", None);
running.run_id = id;
running.runner_name = Some(container.name.clone());
running.labels = self.jobs[&id].labels.clone();
github.jobs.insert(id, running);
assert!(runner_id.is_some(), "an assigned runner is registered");
}
{
let mut docker = self.docker.0.lock().unwrap();
docker
.container_captures
.insert(container.id.clone(), format!("@@job {id} {id}\n"));
docker
.diag
.insert(container.id.clone(), docker::tests::diag_archive(id));
}
self.took_job.insert(container.id.clone());
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Running {
container: container.id.clone(),
};
job.taken_by = Some((profile, repository));
for _ in 0..100 {
self.handle.store.flush().await;
if self
.handle
.store
.job(id)
.await
.unwrap()
.is_some_and(|job| job.summary.runner.is_some())
{
break;
}
tokio::time::sleep(Duration::from_millis(2)).await;
}
}
fn end_job(&mut self, id: u64, container: &str, conclusion: &str) {
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Done {
ran: job.taken_by.clone(),
};
let mut github = self.github.0.lock().unwrap();
let name = github
.jobs
.get(&id)
.and_then(|details| details.runner_name.clone());
let mut completed = details(id, "completed", Some(conclusion));
completed.run_id = id;
completed.runner_name = name.clone();
github.jobs.insert(id, completed);
github.queued.retain(|job| job.id != id);
self.still_listed.remove(&id);
github
.runners
.retain(|runner| Some(&runner.name) != name.as_ref());
self.took_job.remove(container);
}
fn exit_idle(&mut self, pick: usize, exit: Exit) {
let idle = self.idle_containers();
let Some(container) = pick_from(&idle, pick).cloned() else {
return;
};
if exit == Exit::AfterIdleTimeout {
for occupant in self.engine.slots.values_mut().flatten().flatten() {
if occupant.container == container.id {
occupant.started_at -= IDLE_TIMEOUT;
}
}
}
let watch = self
.engine
.occupied()
.find(|(_, occupant)| occupant.container == container.id)
.map(|(slot, occupant)| ExitWatch {
container: container.id.clone(),
repository: occupant.repository.clone(),
profile: slot.profile,
exit: match occupant.started_at.elapsed() {
Ok(waited) if exit == Exit::AtOnce && waited >= IDLE_TIMEOUT => {
Exit::AfterIdleTimeout
}
_ => exit,
},
quiet_before: false,
});
self.exits.extend(watch.map(|watch| ExitWatch {
quiet_before: self.pool_is_quiet(&watch.repository, &watch.profile),
..watch
}));
let log = match exit {
Exit::Connected => "Listening for Jobs\n",
Exit::AtOnce | Exit::AfterIdleTimeout => "",
};
let mut docker = self.docker.0.lock().unwrap();
docker
.container_logs
.insert(container.id.clone(), log.to_owned());
set_running(&mut docker, &container.id, false);
}
fn note_busy(&mut self) {
self.busy_before = self
.engine
.occupied()
.filter(|(_, occupant)| {
occupant.busy
|| occupant.job.is_some()
|| self.took_job.contains(&occupant.container)
})
.map(|(slot, occupant)| {
let repository = occupant.repository.clone();
(occupant.container.clone(), repository, slot.profile)
})
.collect();
}
fn pool_is_quiet(&self, repository: &str, profile: &str) -> bool {
self.engine
.repositories
.get(repository)
.and_then(|state| state.holds.get(profile))
.is_none_or(|hold| {
hold.idle_exits == 0 && hold.held.is_none_or(|held| held.until <= SystemTime::now())
})
}
fn exits_hold_only_what_failed_to_start(&mut self) -> Result<(), Violation> {
let in_slots: BTreeSet<String> = self
.engine
.occupied()
.map(|(_, occupant)| occupant.container.clone())
.collect();
let (waiting, freed): (Vec<ExitWatch>, Vec<ExitWatch>) = std::mem::take(&mut self.exits)
.into_iter()
.partition(|watch| in_slots.contains(&watch.container));
self.exits = waiting;
let now = SystemTime::now();
for watch in &freed {
let Some(hold) = self
.engine
.repositories
.get(&watch.repository)
.and_then(|state| state.holds.get(&watch.profile))
else {
continue;
};
let standing = hold.held.filter(|held| held.until > now);
let job_ended = self
.busy_before
.iter()
.any(|(container, repository, profile)| {
!in_slots.contains(container)
&& *repository == watch.repository
&& *profile == watch.profile
});
match watch.exit {
Exit::AtOnce if standing.is_none() && !job_ended => {
let detail = format!("{watch:?} failed to start and {hold:?} holds nothing");
return violation(self, "exits_hold_only_what_failed_to_start", detail);
}
Exit::Connected if watch.quiet_before && standing.is_some() => {
let failed = freed.iter().any(|other| {
other.exit == Exit::AtOnce
&& other.repository == watch.repository
&& other.profile == watch.profile
});
let cause = standing.map(|held| held.cause);
if !failed && matches!(cause, Some(HoldCause::NoJob | HoldCause::IdleExits)) {
let detail = format!("{watch:?} connected and {hold:?} holds its pool");
return violation(self, "exits_hold_only_what_failed_to_start", detail);
}
}
_ => {}
}
self.seen.exits_freed += 1;
}
Ok(())
}
fn sample_during_start(&mut self, percent: u64, available: u64, stalled: bool) {
let (store, took_job, taken, clock) = (
self.handle.store.clone(),
self.took_job.clone(),
self.host_during_start.clone(),
self.clock,
);
self.docker.0.lock().unwrap().on_start = Some(Box::new(move |docker| {
let now = clock.now_ms();
let reading = sample(docker, &store, &took_job, now, percent, available, stalled);
*taken.lock().unwrap() = Some(reading);
}));
}
fn unobserved(&self, container: &str) -> bool {
self.engine
.occupied()
.any(|(_, occupant)| occupant.container == container && !occupant.observed)
}
fn capture_unread(&self, id: u64, container: &str) -> bool {
let faults = self.capture_faults_of(container);
let retried = self.capture_faults.get(container).copied().unwrap_or(0);
!self.jobs[&id].named_by_github && (faults >= 2 || faults > retried)
}
fn capture_faults_of(&self, container: &str) -> u32 {
self.docker
.0
.lock()
.unwrap()
.capture_faults
.get(container)
.copied()
.unwrap_or(0)
}
async fn reload(&mut self, change: Change) -> Result<(), Violation> {
let mut shape = self.shape;
match change {
Change::HeavySlots(slots) => shape.heavy_slots = slots,
Change::LightSlots(slots) => {
if shape.light_slots.is_some() {
shape.light_slots = Some(slots);
}
}
Change::LightPool => shape.light_pool = !shape.light_pool,
Change::SecondRepository => shape.second_repository = !shape.second_repository,
Change::LightProfile => {
shape.light_slots = match shape.light_slots {
Some(_) => None,
None => Some(2),
};
}
Change::Policy => {
shape.admission = match shape.admission {
Admission::Caps => Admission::Measured,
Admission::Measured => Admission::Caps,
};
}
Change::CpuOvercommit => shape.cpu_overcommit = !shape.cpu_overcommit,
Change::PooledCount => shape.pooled_count = !shape.pooled_count,
Change::HeavyMemory => shape.heavy_memory = if shape.heavy_memory == 4 { 2 } else { 4 },
}
let (reply, answer) = tokio::sync::oneshot::channel();
self.engine
.command(Command::ReplaceConfig {
config: Box::new(Config::parse(&shape.toml()).unwrap()),
memory_max: Some(MEMORY_MAX),
cpus: Some(SLICE_CPUS),
commit: Commit::new(|| Ok(None)),
running: RunningSlots::Drain,
author: Author::Owner,
reply,
})
.await;
let applied = answer.await;
if applied != Ok(Ok(Vec::new())) {
return violation(
self,
"a_reload_applies",
format!("{change:?} answered {applied:?}"),
);
}
self.shape = shape;
Ok(())
}
async fn stop(&mut self, docker_gone: bool) -> Result<(), Violation> {
let running = self.running();
let mode = if docker_gone {
self.docker.set_outage(Outage::Refused);
Shutdown::Graceful
} else {
Shutdown::Detach
};
self.handle.control.shutdown(mode);
let stopped = tokio::time::timeout(Duration::from_secs(300), self.engine.schedule()).await;
self.docker.set_outage(Outage::None);
if stopped.is_err() {
return violation(self, "a_stop_ends", format!("{mode:?} never ended"));
}
let left: BTreeSet<String> = self
.running_containers()
.into_iter()
.map(|container| container.id)
.collect();
let gone: Vec<&(u64, String)> = running
.iter()
.filter(|(_, container)| !left.contains(container))
.collect();
if !gone.is_empty() {
let detail = format!("{mode:?} took the containers of running jobs {gone:?}");
return violation(self, "a_stop_leaves_running_jobs", detail);
}
Ok(())
}
async fn restart(&mut self) -> Result<(), Violation> {
for (_, capture) in std::mem::take(&mut self.engine.captures) {
capture.abort();
}
self.handle.store.flush().await;
let (engine, handle) = Case::engine(
&self.github,
&self.docker,
&self.dir,
self.shape,
self.clock,
);
self.engine = engine;
self.handle = handle;
self.host = None;
self.exits.clear();
self.docker.0.lock().unwrap().on_start = None;
self.recover().await
}
fn elapse(&mut self) {
let old = self.clock.now_ms() - 1000 * (3 * api::SAMPLE_INTERVAL + 1);
let docker = self.docker.0.lock().unwrap();
let reading = sample(
&docker,
&self.handle.store,
&self.took_job,
old,
50,
50,
false,
);
drop(docker);
self.host = Some(reading);
for occupant in self.engine.slots.values_mut().flatten().flatten() {
occupant.started_at -= IDLE_TIMEOUT;
}
for watch in &mut self.exits {
if watch.exit == Exit::AtOnce {
watch.exit = Exit::AfterIdleTimeout;
}
}
let past = SystemTime::now() - Duration::from_secs(1);
for state in self.engine.repositories.values_mut() {
for hold in state.holds.values_mut() {
if let Some(held) = hold.held.as_mut() {
held.until = past;
}
}
for waits in state.waits.values_mut() {
if let Some(wait) = waits.memory.as_mut() {
wait.since -= PRIORITY_AFTER;
}
if let Some(wait) = waits.cpu.as_mut() {
wait.since -= PRIORITY_AFTER;
}
}
}
}
fn settle(&mut self) {
for occupant in self.engine.slots.values_mut().flatten().flatten() {
occupant.job_started = occupant.job_started.map(|at| at - POOL_AFTER);
if let Some(admission) = occupant.admission.as_mut() {
admission.basis = Basis::Job {
runs: 10,
peak: JobPeak::P95,
};
}
}
}
async fn past_runs(&mut self) {
let at = time::now_ms() - 3600 * 1000;
let store = &self.handle.store;
for (first, repository) in [(1_000_000, "o/r"), (2_000_000, "o/s")] {
for id in first..first + 10 {
let runner = format!("earlier-{id}");
store.queued(
repository,
&QueuedJob {
id,
run_id: id,
name: "build".to_owned(),
created_at: at,
..QueuedJob::default()
},
at,
);
store.runner_started(RunnerRecord {
purpose: None,
name: runner.clone(),
repository: repository.to_owned(),
profile: "heavy".to_owned(),
slot: 1,
container: format!("c-{id}"),
started_at: at,
memory_cap: Some(4 * G),
cpu_cap: Some(2000),
admission: None,
});
store.claimed(id, id, &runner, at + 1000);
store.samples(vec![SampleRecord {
runner: runner.clone(),
at: at + 2000,
cpu: 0,
cpu_usage: 0,
memory: G,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
events: Events::default(),
}]);
let mut ran = details(id, "completed", Some("success"));
ran.run_id = id;
ran.runner_name = Some(runner.clone());
ran.steps = Vec::new();
store.details(ran);
store.runner_removed(&runner, at + 60_000);
}
}
store.flush().await;
self.engine.measured.predictor = None;
}
fn unregistered_runners_exit(&mut self) {
let registered: BTreeSet<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
let mut docker = self.docker.0.lock().unwrap();
for container in &mut docker.containers {
if container.running
&& container.labels.contains_key("runnerd.job")
&& !registered.contains(&container.name)
{
container.running = false;
}
}
}
fn failed_late(&mut self) {
let failed = std::mem::take(&mut self.docker.0.lock().unwrap().late_started);
for (container, job) in failed {
let Some(id) = job else { continue };
let left = self
.running_containers()
.into_iter()
.find(|candidate| candidate.id == container);
if let Some(left) = left {
let label = |name: &str| left.labels.get(name).cloned().unwrap_or_default();
self.took_job.insert(container.clone());
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Running { container };
job.taken_by = Some((label("runnerd.profile"), label("runnerd.repository")));
} else {
self.jobs.get_mut(&id).unwrap().taken_by = None;
self.end_job(id, &container, "failure");
}
}
}
async fn tick(&mut self) -> Result<(), Violation> {
self.unregistered_runners_exit();
let was_short = self.engine.measured.host.short;
let calls_before = self.docker.0.lock().unwrap().calls.len();
let seeds_before: BTreeSet<String> = self.engine.reseed_memory.keys().cloned().collect();
self.capture_faults = self.docker.0.lock().unwrap().capture_faults.clone();
for (id, job) in &mut self.jobs {
match job.phase {
Phase::Queued if self.shape.serves(job.repository, &job.labels).is_some() => {
job.listed = true;
}
Phase::Running { .. } if job.listed && !self.still_listed.contains(id) => {
job.named_by_github = true;
}
_ => {}
}
}
self.cancelled_since_tick.clear();
self.note_busy();
self.engine.tick(Shutdown::No).await;
self.removals_finish();
self.failed_late();
self.exits_hold_only_what_failed_to_start()?;
let after = self.status();
let listed = std::mem::take(&mut self.still_listed);
self.github
.0
.lock()
.unwrap()
.queued
.retain(|job| !listed.contains(&job.id));
let running: BTreeSet<String> = self
.running_containers()
.into_iter()
.map(|container| container.name)
.collect();
let starts: Vec<String> = self.docker.0.lock().unwrap().calls[calls_before..]
.iter()
.filter_map(|call| call.strip_prefix("start "))
.filter(|name| running.contains(*name))
.map(str::to_owned)
.collect();
let mut started: BTreeSet<String> = starts.iter().cloned().collect();
self.no_pool_gets_more_runners_than_its_queued_jobs(&after, &started)?;
let seeds = self
.engine
.reseed_memory
.keys()
.filter(|profile| !seeds_before.contains(*profile))
.map(|profile| format!("{profile}{SEED_START}"))
.collect::<Vec<_>>();
let read_during_start = self.starts_after_a_reading(&starts);
let same_reading = read_during_start.is_none();
let seeded = !seeds.is_empty();
if let Some(after) = read_during_start {
started = after;
}
started.extend(seeds);
self.starts_fit_the_budget(&after, &started)?;
self.docker_runs_no_runner_the_engine_lost(&after)?;
self.only_unserved_jobs_wait_for_a_pool(&after)?;
self.one_barrier_and_only_for_a_queued_job(&after)?;
self.the_count_keeps_its_bounds(&after)?;
if same_reading {
self.the_band_follows_the_waiters(was_short, seeded)?;
}
self.no_removal_of_a_running_job()
}
fn starts_after_a_reading(&mut self, starts: &[String]) -> Option<BTreeSet<String>> {
let hooked = self.docker.0.lock().unwrap().hooked_start.take();
let during = self.host_during_start.lock().unwrap().take();
let (hooked, reading) = hooked.zip(during)?;
self.host = Some(reading);
let after = starts.iter().skip_while(|name| **name != hooked).skip(1);
Some(after.cloned().collect())
}
async fn pass(&mut self) -> Result<(), Violation> {
self.unregistered_runners_exit();
let calls_before = self.docker.0.lock().unwrap().calls.len();
self.engine.exits.any();
self.note_busy();
self.engine.turnover().await;
self.removals_finish();
self.failed_late();
self.exits_hold_only_what_failed_to_start()?;
let after = self.status();
let running: BTreeSet<String> = self
.running_containers()
.into_iter()
.map(|container| container.name)
.collect();
let order: Vec<String> = self.docker.0.lock().unwrap().calls[calls_before..]
.iter()
.filter_map(|call| call.strip_prefix("start "))
.filter(|name| running.contains(*name))
.map(str::to_owned)
.collect();
let starts: BTreeSet<String> = order.iter().cloned().collect();
self.no_pass_starts_a_second_runner_for_a_job(&after, &starts)?;
let bound = self.starts_after_a_reading(&order);
self.starts_fit_the_budget(&after, bound.as_ref().unwrap_or(&starts))?;
self.docker_runs_no_runner_the_engine_lost(&after)?;
self.one_barrier_and_only_for_a_queued_job(&after)?;
self.the_count_keeps_its_bounds(&after)?;
self.no_removal_of_a_running_job()
}
fn no_pass_starts_a_second_runner_for_a_job(
&self,
status: &api::Status,
started: &BTreeSet<String>,
) -> Result<(), Violation> {
let idle: BTreeSet<String> = self.idle_containers().into_iter().map(|c| c.id).collect();
let pools: BTreeSet<(String, String)> = runners_of(status)
.filter(|(_, runner)| started.contains(&runner.name))
.map(|(profile, runner)| (runner.repository.clone(), profile.to_owned()))
.collect();
for (repository, profile) in pools {
let waiting = runners_of(status)
.filter(|(in_profile, runner)| {
*in_profile == profile
&& runner.repository == repository
&& idle.contains(&runner.container)
})
.count();
let wanted = self
.queued()
.into_iter()
.chain(self.cancelled_since_tick.iter().copied())
.filter(|id| {
let job = &self.jobs[id];
job.repository == repository
&& self.shape.serves(job.repository, &job.labels) == Some(profile.as_str())
})
.count();
if waiting > wanted {
return violation(
self,
"no_pass_starts_a_second_runner_for_a_job",
format!(
"{repository} in {profile}: {waiting} runners wait for {wanted} jobs queued or cancelled"
),
);
}
}
Ok(())
}
fn no_pool_gets_more_runners_than_its_queued_jobs(
&self,
status: &api::Status,
started: &BTreeSet<String>,
) -> Result<(), Violation> {
let mut pools: BTreeSet<(String, String)> = BTreeSet::new();
for (profile, runner) in runners_of(status) {
if started.contains(&runner.name) {
pools.insert((runner.repository.clone(), profile.to_owned()));
}
}
let idle: BTreeSet<String> = self.idle_containers().into_iter().map(|c| c.id).collect();
for (repository, profile) in pools {
let waiting = runners_of(status)
.filter(|(in_profile, runner)| {
*in_profile == profile
&& runner.repository == repository
&& idle.contains(&runner.container)
})
.count();
let queued = self
.queued()
.into_iter()
.filter(|id| {
let job = &self.jobs[id];
job.repository == repository
&& self.shape.serves(job.repository, &job.labels) == Some(profile.as_str())
})
.count();
if waiting > queued {
return violation(
self,
"no_pool_gets_more_runners_than_its_queued_jobs",
format!(
"{repository} in {profile}: {waiting} runners wait for {queued} queued jobs"
),
);
}
}
Ok(())
}
fn starts_fit_the_budget(
&self,
status: &api::Status,
started: &BTreeSet<String>,
) -> Result<(), Violation> {
if started.is_empty() {
return Ok(());
}
let budget = &status.budget;
let runner_started = started.iter().any(|name| !name.ends_with(SEED_START));
if self.shape.admission == Admission::Caps {
let running = self.running_containers();
let docker = self.docker.0.lock().unwrap();
let caps: u64 = running
.iter()
.map(|container| docker.memory.get(&container.id).copied().unwrap_or(0))
.sum();
let cpus: u64 = running
.iter()
.map(|container| docker.cpus.get(&container.id).copied().unwrap_or(0))
.sum();
drop(docker);
if runner_started && self.shape.cpu_overcommit && cpus > SLICE_CPUS {
return violation(
self,
"starts_fit_the_budget",
format!("started {started:?} with CPU limits of {cpus} in {SLICE_CPUS}"),
);
}
let seeds: u64 = budget.seed_memory.iter().map(|seed| seed.memory).sum();
if caps.saturating_add(seeds) > MEMORY_MAX {
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} with caps of {caps} and seeds of {seeds} in {MEMORY_MAX}"
),
);
}
}
if let Some(max) = budget.memory_max
&& budget.counted() > max
{
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} with {} of {max} counted, seed containers {:?}",
budget.counted(),
budget.seed_memory
),
);
}
if let Some(cpu) = &budget.cpu
&& runner_started
&& cpu.in_use > cpu.budget
{
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} with {} of {} millicores reserved",
cpu.in_use, cpu.budget
),
);
}
let now = self.clock.now_ms();
if let Some(reading) = self.host.filter(|reading| reading.fresh(now)) {
let limit = reading
.slice
.saturating_add(reading.available)
.saturating_sub(HOST_MARGIN);
let seeds: u64 = self.engine.reseed_memory.values().sum();
let credit = budget.pooled.map_or(0, |pooled| pooled.credit);
let reserved = runners_of(status)
.map(|(_, runner)| runner.reserved)
.fold(seeds, u64::saturating_add)
.saturating_sub(credit);
if reserved > limit {
return violation(
self,
"starts_fit_the_budget",
format!("started {started:?} with {reserved} reserved past the host's {limit}"),
);
}
if reading.stalled && reading.available < 2 * HOST_MARGIN {
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} while the host stalls with {} available",
reading.available
),
);
}
}
Ok(())
}
fn the_count_keeps_its_bounds(&mut self, status: &api::Status) -> Result<(), Violation> {
let budget = &status.budget;
let credit = budget.pooled.map_or(0, |pooled| pooled.credit);
let measured = self.shape.admission == Admission::Measured;
if credit > pooled::MOST_CREDIT || (credit > 0 && !(measured && self.shape.pooled_count)) {
let detail = format!("{budget:?} under {:?}", self.shape);
return violation(self, "the_count_keeps_its_bounds", detail);
}
let admitting = self.engine.admitting(&mut [], None);
let plan = self.engine.plan_budget(&admitting);
let left = budget
.memory_max
.map(|max| crate::scheduler::left(max, budget.counted()));
if !admitting.held && plan.memory != left {
let detail = format!("a plan reads {:?}, the status {left:?}", plan.memory);
return violation(self, "the_count_keeps_its_bounds", detail);
}
let host = admitting
.host
.as_ref()
.map(|host| host::left(host.limit, budget.memory_in_use, host.stalled));
if !admitting.held && plan.host != host {
let detail = format!(
"a plan's host reads {:?}, the plain count {host:?}",
plan.host
);
return violation(self, "the_count_keeps_its_bounds", detail);
}
if credit == 0 {
return Ok(());
}
self.seen.credits += 1;
let now = SystemTime::now();
let fresh_after = self
.clock
.now_ms()
.saturating_sub(3000 * api::SAMPLE_INTERVAL);
let floor: u64 = self
.engine
.occupied()
.map(|(_, occupant)| {
let sample = self.handle.store.latest_sample(&occupant.runner_name);
let settled = occupant
.job_started
.and_then(|started| now.duration_since(started).ok())
.is_some_and(|ran| ran >= POOL_AFTER);
let predicted = occupant.admission.is_some_and(|admission| {
matches!(
admission.basis,
Basis::Job { .. } | Basis::Profile { .. } | Basis::ProfileMax { .. }
)
});
let known = occupant.key.is_some() && occupant.purpose.is_some();
match sample.filter(|sample| sample.at >= fresh_after) {
Some(sample) if settled && predicted && known => sample.memory,
_ => self.engine.takes(occupant),
}
})
.sum();
let floor = floor + self.engine.committed_memory();
if budget.counted() < floor {
let detail = format!("{budget:?} counts under {floor}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
let host = budget.host.as_ref();
let keeps = HOST_MARGIN + HOST_TOTAL / 4;
if host.is_none_or(|host| host.stalled || host.memory.available < keeps + credit) {
let detail = format!("{credit} of credit beside the host's {host:?}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
let slice = self
.handle
.store
.latest_host_sample()
.map(|sample| sample.memory);
let room = slice.map(|slice| {
(MEMORY_MAX * 90 / 100)
.saturating_sub(slice)
.saturating_sub(pooled::HIGH_WATER_RESERVE)
});
if room.is_none_or(|room| credit > room) {
let detail = format!("{credit} of credit beside the slice's {slice:?}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
Ok(())
}
fn the_band_follows_the_waiters(
&mut self,
was_short: bool,
seeded: bool,
) -> Result<(), Violation> {
let remembered = self.engine.measured.host;
let (plan, admitting) = self.engine.plan();
let Some(host) = admitting.host.as_ref() else {
if remembered.short {
let detail = "the larger margin with no reading".to_owned();
return violation(self, "the_band_follows_the_waiters", detail);
}
return Ok(());
};
if host.stalled {
if !remembered.short {
let detail = format!("a stall without the larger margin: {:?}", host.memory);
return violation(self, "the_band_follows_the_waiters", detail);
}
return Ok(());
}
let in_use = self.engine.count(self.engine.host_remembered()).plain;
let room = host::left(host::limit(&host.memory, false), in_use, false);
let needs: Vec<i64> = plan
.waiting
.values()
.filter_map(|reason| match reason {
api::Waiting::HostMemory { needs, .. } => i64::try_from(*needs).ok(),
_ => None,
})
.collect();
let fits = needs.iter().any(|needs| *needs <= room);
let starts = plan
.decisions
.iter()
.any(|decision| matches!(decision, Decision::Start { .. }));
if remembered.short && fits {
let detail = format!("{needs:?} wait, {room} is left by the plain margin");
return violation(self, "the_band_follows_the_waiters", detail);
}
if !remembered.short && !fits && !needs.is_empty() && !starts && !seeded {
let detail = format!("{needs:?} wait, {room} is left, and the margin is plain");
return violation(self, "the_band_follows_the_waiters", detail);
}
self.seen.bands += usize::from(remembered.short && !needs.is_empty());
self.seen.released += usize::from(was_short && !remembered.short && starts);
Ok(())
}
fn one_barrier_and_only_for_a_queued_job(
&mut self,
status: &api::Status,
) -> Result<(), Violation> {
let behind = |reason: &Option<api::Waiting>| match reason {
Some(api::Waiting::BehindMemory { job } | api::Waiting::BehindCpu { job }) => {
Some(*job)
}
_ => None,
};
let owners: BTreeSet<u64> = status
.queued
.iter()
.filter_map(|job| behind(&job.waiting))
.collect();
self.seen.barriers += owners.len();
if owners.len() > 1 {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("jobs wait behind each of {owners:?}"),
);
}
for owner in &owners {
let Some(queued) = status.queued.iter().find(|job| job.id == *owner) else {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("jobs wait behind {owner}, which no longer waits"),
);
};
if behind(&queued.waiting).is_some() {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("{owner} holds others back and says {:?}", queued.waiting),
);
}
if let Some(younger) = status
.queued
.iter()
.find(|job| behind(&job.waiting).is_some() && job.id <= *owner)
{
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!(
"{} is older than {owner}, which it waits behind",
younger.id
),
);
}
}
for (name, state) in self.engine.repositories.iter().filter(|(_, s)| s.listed) {
let waits = state.waits.values();
let kept = waits.flat_map(|waits| waits.memory.iter().chain(&waits.cpu));
if let Some(wait) = kept
.into_iter()
.find(|wait| !state.queued.iter().any(|job| job.id == wait.job))
{
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("{name} keeps a wait for job {}, listed no more", wait.job),
);
}
if state.queued.is_empty() && state.waits.values().any(|waits| waits.served) {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("{name} keeps a served mark with nothing queued"),
);
}
}
Ok(())
}
fn no_removal_of_a_running_job(&mut self) -> Result<(), Violation> {
let calls = self.docker.calls();
let running: BTreeMap<String, u64> = self
.running()
.into_iter()
.map(|(id, container)| (container, id))
.collect();
for call in &calls[self.calls_seen..] {
if let Some(container) = call.strip_prefix("remove ")
&& let Some(job) = running.get(container)
{
let detail = format!("removed {container}, which runs job {job}");
self.calls_seen = calls.len();
return violation(self, "no_removal_of_a_running_job", detail);
}
}
self.calls_seen = calls.len();
Ok(())
}
fn docker_runs_no_runner_the_engine_lost(&self, status: &api::Status) -> Result<(), Violation> {
let held: BTreeSet<&str> = runners_of(status)
.map(|(_, runner)| runner.container.as_str())
.collect();
let lost: Vec<String> = self
.running_containers()
.into_iter()
.filter(|container| !held.contains(container.id.as_str()))
.map(|container| container.id)
.collect();
if lost.is_empty() {
Ok(())
} else {
violation(
self,
"docker_runs_no_runner_the_engine_lost",
format!("no slot holds {lost:?}"),
)
}
}
fn only_unserved_jobs_wait_for_a_pool(&self, status: &api::Status) -> Result<(), Violation> {
for listed in &status.queued {
let job = &self.jobs[&listed.id];
let hosted = job.labels == ["ubuntu-latest"];
let unserved = self.shape.serves(job.repository, &job.labels).is_none();
let no_pool = matches!(listed.waiting, Some(api::Waiting::NoPool { .. }));
if hosted || unserved != no_pool {
return violation(
self,
"only_unserved_jobs_wait_for_a_pool",
format!(
"job {} asking for {:?} waits with {:?}",
listed.id, job.labels, listed.waiting
),
);
}
}
for id in self.queued() {
let job = &self.jobs[&id];
if self.shape.configures(job.repository)
&& job.labels != ["ubuntu-latest"]
&& !status.queued.iter().any(|listed| listed.id == id)
{
return violation(
self,
"only_unserved_jobs_wait_for_a_pool",
format!("queued job {id} asking for {:?} is not listed", job.labels),
);
}
}
Ok(())
}
async fn drain(&mut self) -> Result<(), Violation> {
self.step += 1;
while !self.queued().is_empty() {
self.apply(Step::Cancel(0)).await?;
}
while !self.running().is_empty() {
self.apply(Step::Finish(0)).await?;
}
for _ in 0..6 {
self.elapse();
self.tick().await?;
}
self.no_runner_is_left()?;
self.history_names_the_pool_that_ran_each_job().await?;
self.no_job_stays_running().await
}
fn no_runner_is_left(&mut self) -> Result<(), Violation> {
let status = self.status();
let containers: Vec<String> = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.filter(|container| container.labels.contains_key("runnerd.job"))
.map(|container| container.id.clone())
.collect();
let slots: Vec<String> = runners_of(&status)
.map(|(_, runner)| runner.name.clone())
.collect();
let registered: Vec<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
if containers.is_empty() && slots.is_empty() && registered.is_empty() {
Ok(())
} else {
violation(
self,
"no_runner_is_left",
format!("containers {containers:?}, slots {slots:?}, registered {registered:?}"),
)
}
}
async fn history_names_the_pool_that_ran_each_job(&mut self) -> Result<(), Violation> {
self.handle.store.flush().await;
let ran: Vec<(u64, (String, String))> = self
.jobs
.iter()
.filter_map(|(id, job)| match &job.phase {
Phase::Done { ran: Some(ran) } => Some((*id, ran.clone())),
_ => None,
})
.collect();
for (id, (profile, repository)) in ran {
let recorded = self.handle.store.job(id).await.unwrap();
let seen = recorded.as_ref().map(|job| {
(
job.summary
.runner
.as_ref()
.map(|runner| runner.profile.clone()),
job.summary.repository.clone(),
)
});
if seen != Some((Some(profile.clone()), repository.clone())) {
return violation(
self,
"history_names_the_pool_that_ran_each_job",
format!("job {id} ran in {repository} {profile}; the history says {seen:?}"),
);
}
}
Ok(())
}
async fn no_job_stays_running(&mut self) -> Result<(), Violation> {
for id in self.jobs.keys().copied().collect::<Vec<_>>() {
let recorded = self.handle.store.job(id).await.unwrap();
if let Some(job) = recorded.filter(|job| job.summary.state == JobState::Running) {
return violation(
self,
"no_job_stays_running",
format!("job {id} is still running on {:?}", job.summary.runner),
);
}
}
Ok(())
}
}
fn sample(
docker: &DockerState,
store: &Store,
took_job: &BTreeSet<String>,
now: u64,
percent: u64,
available: u64,
stalled: bool,
) -> HostReading {
let mut slice = 0;
let mut samples = Vec::new();
for container in docker
.containers
.iter()
.filter(|container| container.running && container.labels.contains_key("runnerd.job"))
{
let cap = docker.memory.get(&container.id).copied().unwrap_or(G);
let memory = if took_job.contains(&container.id) {
cap / 100 * percent
} else {
100 * M
};
slice += memory;
samples.push(SampleRecord {
runner: container.name.clone(),
at: now,
cpu: 0,
cpu_usage: 0,
memory,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
events: Events::default(),
});
}
store.samples(samples);
store.host_sample(api::HostSample {
at: now / 1000,
cpu: 0,
memory: slice,
});
store.host_memory(api::HostMemory {
at: now / 1000,
total: HOST_TOTAL,
available: available * G,
slice,
some: Some(api::Pressure::default()),
full: Some(api::Pressure {
avg10: if stalled { 600 } else { 0 },
avg60: 0,
}),
holders: Vec::new(),
ceiling: None,
swap: None,
});
HostReading {
at: now / 1000,
slice,
available: available * G,
stalled,
}
}
fn pick_from<T>(items: &[T], pick: usize) -> Option<&T> {
(!items.is_empty()).then(|| &items[pick % items.len()])
}
fn set_running(docker: &mut DockerState, container: &str, running: bool) {
for candidate in &mut docker.containers {
if candidate.id == container {
candidate.running = running;
}
}
}
fn runners_of(status: &api::Status) -> impl Iterator<Item = (&str, &api::RunnerStatus)> {
status.profiles.iter().flat_map(|profile| {
profile
.slots
.iter()
.filter_map(|slot| slot.runner.as_ref())
.map(|runner| (profile.name.as_str(), runner))
})
}
async fn run(steps: &[Step]) -> Result<Seen, Violation> {
let mut case = Case::new();
case.recover().await?;
case.tick().await?;
for (index, step) in steps.iter().enumerate() {
case.step = index;
case.apply(*step).await?;
}
case.step = steps.len();
case.drain().await?;
Ok(case.seen)
}
async fn attempt(steps: &[Step]) -> Result<Seen, Violation> {
match AssertUnwindSafe(run(steps)).catch_unwind().await {
Ok(outcome) => outcome,
Err(panic) => Err(Violation {
invariant: "no_panic",
step: 0,
detail: panic
.downcast_ref::<&str>()
.map(|message| (*message).to_owned())
.or_else(|| panic.downcast_ref::<String>().cloned())
.unwrap_or_default(),
}),
}
}
async fn shrink(mut steps: Vec<Step>, found: Violation) -> (Vec<Step>, Violation) {
let mut last = found;
let mut chunk = steps.len() / 2;
while chunk >= 1 {
let mut start = 0;
while start < steps.len() {
let mut fewer = steps.clone();
fewer.drain(start..(start + chunk).min(steps.len()));
match attempt(&fewer).await {
Err(found) if found.invariant == last.invariant => {
steps = fewer;
last = found;
}
_ => start += chunk,
}
}
chunk /= 2;
}
(steps, last)
}
const BAND_RELEASED: [Step; 10] = [
Step::Reload(Change::Policy),
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Sample {
percent: 10,
available: 20,
stalled: false,
},
Step::Tick,
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Sample {
percent: 10,
available: 3,
stalled: false,
},
Step::Tick,
Step::Sample {
percent: 10,
available: 9,
stalled: false,
},
Step::Tick,
Step::Tick,
];
const BAND_STANDS: usize = 7;
const TWO_JOBS_SETTLED: [Step; 11] = [
Step::PastRuns,
Step::Reload(Change::Policy),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Assign(0),
Step::Assign(0),
Step::Tick,
Step::Settle,
Step::Sample {
percent: 10,
available: 50,
stalled: false,
},
Step::Tick,
];
fn env_u64(name: &str) -> Option<u64> {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
}
#[tokio::test]
#[expect(
clippy::print_stderr,
reason = "names the failing seed before shrinking, which takes a while and may itself fail"
)]
async fn the_scheduler_keeps_its_invariants_over_generated_sequences() {
let seeds: Vec<u64> = match env_u64("RUNNERD_INVARIANT_SEED") {
Some(seed) => vec![seed],
None => (0..env_u64("RUNNERD_INVARIANT_CASES").unwrap_or(CASES)).collect(),
};
let mut seen = Seen::default();
for batch in seeds.chunks(64) {
let runs = batch.iter().map(|&seed| async move {
let mut steps = generate(&mut Rng(seed));
if seed % 2 == 1 {
steps.splice(0..0, TWO_JOBS_SETTLED);
}
match seed % 8 {
2 => drop(steps.splice(0..0, BAND_RELEASED[..BAND_STANDS].iter().copied())),
6 => drop(steps.splice(0..0, BAND_RELEASED)),
_ => {}
}
(seed, attempt(&steps).await, steps)
});
for (seed, outcome, steps) in futures_util::future::join_all(runs).await {
if let Ok(case) = &outcome {
seen.barriers += case.barriers;
seen.exits_freed += case.exits_freed;
assert!(
seed % 8 != 2 || case.bands > 0,
"seed {seed} starts from a job no margin fits, so the larger margin stands"
);
assert!(
seed % 8 != 6 || case.released > 0,
"seed {seed} starts from a job between the margins, so the band lets it go"
);
assert!(
seed % 2 == 0 || case.credits > 0,
"seed {seed} starts from two settled jobs, so it reaches a credit"
);
}
if let Err(found) = outcome {
eprintln!(
"seed {seed} breaks {} at step {}: {}; shrinking",
found.invariant, found.step, found.detail
);
let (invariant, detail) = (found.invariant, found.detail.clone());
let (shortest, violation) = shrink(steps, found).await;
panic!(
"seed {seed} breaks {invariant}: {detail}\nshrunk to {shortest:#?}\nwhich breaks it at step {}: {}",
violation.step, violation.detail
);
}
}
}
assert!(
seen.barriers > 0 || seeds.len() == 1,
"the generated runs reach a barrier, or its invariant checks nothing"
);
assert!(
seen.exits_freed > 0 || seeds.len() == 1,
"the generated runs free a runner that left without a job, or its invariant checks nothing"
);
}
async fn broken(steps: &[Step]) -> Option<(&'static str, String)> {
run(steps)
.await
.err()
.map(|found| (found.invariant, found.detail))
}
#[tokio::test]
async fn a_job_taking_a_waiting_runners_room_still_fits_what_the_host_leaves() {
let steps = [
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Cancel(0),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Sample {
percent: 10,
available: 3,
stalled: false,
},
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_that_ends_before_its_broken_capture_comes_back_keeps_its_runner_in_the_history() {
let broke = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::CaptureFails { mid_stream: true },
Step::RestartInspectFails(0),
Step::InspectFails(1),
Step::InspectFails(6),
];
let tick = [Step::Tick, Step::Tick, Step::Assign(3)];
assert_eq!(broken(&[&broke[..], &tick].concat()).await, None);
let pass = [
Step::StartFails,
Step::Tick,
Step::Tick,
Step::Assign(3),
Step::Finish(0),
Step::Pass,
];
assert_eq!(broken(&[&broke[..], &pass].concat()).await, None);
let restart = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Reload(Change::Policy),
Step::InspectFails(5),
Step::CaptureFails { mid_stream: false },
Step::Reload(Change::SecondRepository),
Step::AssignListed(4),
Step::Finish(4),
Step::RestartInspectFails(5),
];
assert_eq!(broken(&restart).await, None);
}
#[tokio::test]
async fn a_runner_whose_inspect_fails_after_a_reload_lowered_its_memory_counts_at_its_own_cap() {
let heavy = |second| Step::Queue {
second,
labels: Labels::Heavy,
};
let steps = [
heavy(false),
heavy(true),
Step::Tick,
Step::Assign(0),
Step::Assign(0),
Step::Reload(Change::HeavyMemory),
Step::RestartInspectFails(7),
heavy(false),
Step::Reload(Change::HeavySlots(3)),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_stopped_runner_that_recovery_cannot_remove_is_removed_later() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Tick,
Step::Assign(0),
Step::RemoveFails,
Step::Finish(0),
Step::Restart,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_runner_watched_through_a_failed_inspect_still_records_the_job_it_runs() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Tick,
Step::InspectFails(0),
Step::RestartInspectFails(0),
Step::Assign(0),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_taking_a_waiting_runners_room_still_fits_the_cpu_budget() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Reload(Change::CpuOvercommit),
Step::AssignListed(1),
Step::InspectFails(1),
Step::Finish(2),
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_seed_container_starts_past_a_cpu_budget_a_reload_lowered() {
let steps = [
Step::Assign(6),
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Reload(Change::HeavyMemory),
Step::AssignListed(4),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Finish(5),
Step::Reload(Change::LightSlots(1)),
Step::Reload(Change::HeavySlots(3)),
Step::Reload(Change::CpuOvercommit),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_taking_a_waiting_runners_room_still_fits_the_slice_under_measured_admission() {
let steps = [
Step::Reload(Change::Policy),
Step::Reload(Change::HeavySlots(3)),
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Cancel(0),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Tick,
Step::Assign(0),
Step::Sample {
percent: 120,
available: 50,
stalled: false,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_reseed_starts_only_where_its_seed_container_fits_the_budget() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Tick,
Step::Assign(0),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Finish(0),
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_runner_first_listed_busy_beside_its_job_still_listed_gets_no_twin() {
let steps = [
Step::StartedInspectFails,
Step::CaptureFails { mid_stream: false },
Step::Queue {
second: false,
labels: Labels::SelfHosted,
},
Step::Restart,
Step::AssignListed(2),
Step::Reload(Change::LightPool),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_pass_starts_no_runner_for_a_job_a_failed_start_may_have_taken() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::StartFailsLate {
takes_job: false,
cleanup_fails: true,
},
Step::Sample {
percent: 120,
available: 8,
stalled: false,
},
Step::StartFailsLate {
takes_job: true,
cleanup_fails: false,
},
Step::Tick,
Step::Restart,
Step::Assign(7),
Step::Lose(5),
Step::Pass,
];
assert_eq!(broken(&steps).await, None);
}