use std::fmt::Write as _;
use std::panic::AssertUnwindSafe;
use std::time::SystemTime;
use futures_util::FutureExt as _;
use super::*;
use crate::admission::host;
use crate::admission::pooled::{self, POOL_AFTER};
use crate::api::{Basis, HoldCause, JobPeak};
use crate::config::Admission;
use crate::observe::metrics::Events;
use crate::scheduler::engine::clock::Clock;
use crate::scheduler::engine::control::RunningJobs;
use crate::scheduler::engine::tick::Scope;
use crate::scheduler::{Decision, IDLE_TIMEOUT, PRIORITY_AFTER};
use crate::store::{RunnerRecord, SampleRecord};
use crate::time;
const G: u64 = 1 << 30;
const M: u64 = 1 << 20;
const MEMORY_MAX: u64 = 8 * G + 512 * M;
const SLICE_CPUS: u64 = 3000;
const HOST_TOTAL: u64 = 64 * G;
const HOST_MARGIN: u64 = HOST_TOTAL / 16;
const SEED_START: &str = "'s seed container";
const CASES: u64 = 64;
const EVENTS: usize = 40;
const TICK_EVERY: Duration = Duration::from_secs(5);
const POLL_INTERVAL: Duration = Duration::from_secs(15);
pub(super) struct Rng(pub(super) u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
z ^ (z >> 31)
}
pub(super) fn below(&mut self, n: u64) -> u64 {
self.next() % n
}
fn index(&mut self) -> usize {
usize::try_from(self.below(8)).unwrap()
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
#[expect(
clippy::struct_excessive_bools,
reason = "each is a config choice a reload flips on its own"
)]
struct Shape {
admission: Admission,
cpu_overcommit: bool,
heavy_slots: u32,
light_slots: Option<u32>,
light_pool: bool,
second_repository: bool,
pooled_count: bool,
heavy_memory: u64,
mirror: bool,
record_failures: bool,
build_cache: bool,
}
struct PoolShape {
repository: &'static str,
profile: &'static str,
labels: &'static [&'static str],
}
impl Shape {
const START: Shape = Shape {
admission: Admission::Caps,
cpu_overcommit: false,
heavy_slots: 2,
light_slots: Some(2),
light_pool: true,
second_repository: true,
pooled_count: true,
heavy_memory: 4,
mirror: false,
record_failures: false,
build_cache: false,
};
fn pools(&self) -> Vec<PoolShape> {
let mut pools = vec![PoolShape {
repository: "o/r",
profile: "heavy",
labels: &["self-hosted", "app"],
}];
if self.light_pool && self.light_slots.is_some() {
pools.push(PoolShape {
repository: "o/r",
profile: "light",
labels: &["r-light"],
});
}
if self.second_repository {
pools.push(PoolShape {
repository: "o/s",
profile: "heavy",
labels: &["self-hosted", "s-app"],
});
}
pools.push(PoolShape {
repository: "o/t",
profile: "heavy",
labels: &["self-hosted", "t-app"],
});
pools
}
fn toml(&self) -> String {
let mut host = String::new();
host.push_str(match self.admission {
Admission::Caps => "admission = \"caps\"\n",
Admission::Measured => "admission = \"measured\"\n",
});
if self.cpu_overcommit {
host.push_str("cpu_overcommit = 1.0\n");
}
if !self.pooled_count {
host.push_str("pooled_count = false\n");
}
let mut config = format!(
r#"
[host]
slice = "ci.slice"
isolation = "sysbox"
{host}
[caches]
mirror = {{ enabled = {}, memory = "512M", cpus = 1, ttl = "168h" }}
pnpm = {{ max_size = "5G" }}
maven = {{ max_size = "5G" }}
[profiles.heavy]
image_stage = "app"
cpus = 2
memory = "{}G"
slots = {}
record_failures = {}
"#,
self.mirror, self.heavy_memory, self.heavy_slots, self.record_failures
);
if let Some(slots) = self.light_slots {
let _ = write!(
config,
"\n[profiles.light]\nimage_stage = \"app\"\ncpus = 1\nmemory = \"1G\"\nslots = {slots}\n"
);
}
for pool in self.pools() {
let labels: Vec<String> = pool
.labels
.iter()
.map(|label| format!("{label:?}"))
.collect();
let _ = write!(
config,
"\n[[repositories]]\nname = {:?}\nprofile = {:?}\nlabels = [{}]\nwarm = 0\n",
pool.repository,
pool.profile,
labels.join(", ")
);
}
config
}
fn serves(&self, repository: &str, labels: &[String]) -> Option<&'static str> {
if labels.is_empty() {
return None;
}
let serving: Vec<&'static str> = self
.pools()
.into_iter()
.filter(|pool| {
pool.repository == repository
&& labels.iter().all(|label| {
pool.labels
.iter()
.any(|has| has.eq_ignore_ascii_case(label))
})
})
.map(|pool| pool.profile)
.collect();
assert!(serving.len() <= 1, "the shape has pools sharing labels");
serving.first().copied()
}
fn configures(&self, repository: &str) -> bool {
self.pools()
.iter()
.any(|pool| pool.repository == repository)
}
}
#[derive(Debug, Clone, Copy)]
enum Labels {
Heavy,
Light,
SelfHosted,
Typo,
Empty,
Hosted,
}
impl Labels {
fn of(self, repository: &str) -> Vec<String> {
let labels: &[&str] = match (self, repository) {
(Labels::Heavy, "o/r") => &["self-hosted", "app"],
(Labels::Heavy, "o/t") => &["self-hosted", "t-app"],
(Labels::Heavy, _) => &["self-hosted", "s-app"],
(Labels::Light, "o/r") => &["r-light"],
(Labels::Light, _) => &["s-light"],
(Labels::SelfHosted, _) => &["self-hosted"],
(Labels::Typo, _) => &["r-lite"],
(Labels::Empty, _) => &[],
(Labels::Hosted, _) => &["ubuntu-latest"],
};
labels.iter().map(|label| (*label).to_owned()).collect()
}
}
#[derive(Debug, Clone, Copy)]
enum Change {
HeavySlots(u32),
LightSlots(u32),
LightPool,
SecondRepository,
LightProfile,
Policy,
CpuOvercommit,
PooledCount,
HeavyMemory,
Mirror,
RecordFailures,
}
#[derive(Debug, Clone, Copy)]
enum Step {
Queue {
second: bool,
labels: Labels,
},
Assign(usize),
AssignListed(usize),
QueueQuiet {
labels: Labels,
},
TickSoon {
after: u64,
},
Quiet,
Finish(usize),
Cancel(usize),
Lose(usize),
Removing(usize),
StartFails,
StartFailsLate {
takes_job: bool,
cleanup_fails: bool,
},
CaptureFails {
mid_stream: bool,
},
ExitsAtOnce(usize),
ExitsIdle(usize),
ExitsConnected(usize),
InspectFails(usize),
StartedInspectFails,
RemoveFails,
Sample {
percent: u64,
available: u64,
stalled: bool,
},
SampleDuringStart {
percent: u64,
available: u64,
stalled: bool,
},
Reseed {
light: bool,
},
Reload(Change),
Restart,
RestartThenStop,
RestartInspectFails(usize),
Stop {
docker_gone: bool,
},
Elapse,
GrowthWindow {
percent: u64,
},
Settle,
PastRuns,
PastLargeRuns,
Tick,
Pass,
RunnerGoes(usize),
RunnerGoesStillListed(usize),
OpenJobCompletes(usize),
ForcedPause {
second: bool,
},
Resume {
second: bool,
},
BusyOrphan,
OversizedSnapshot,
RestartLogsFail,
UseBuildCache,
SaveUnnamed(usize),
RunnerNamed(usize),
JobReadFails {
pick: usize,
how: ReadFailure,
},
RegisterFailsMidBatch {
after: u32,
},
RateLimitedDuringStart {
after: u32,
},
PrepareFailsMidBatch {
after: u32,
},
StopDuringStart {
docker_gone: bool,
after: u32,
at: StopAt,
},
SlowStart,
SliceSampleStallsDuringStart,
AbortMidBatch,
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum StopAt {
Registration,
Launch,
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum ReadFailure {
Gone,
Limited,
Other,
}
fn generate(rng: &mut Rng) -> Vec<Step> {
(0..EVENTS)
.map(|_| match rng.below(125) {
0..=17 => Step::Queue {
second: rng.below(3) == 0,
labels: match rng.below(10) {
0..=3 => Labels::Heavy,
4..=5 => Labels::Light,
6 => Labels::SelfHosted,
7 => Labels::Typo,
8 => Labels::Empty,
_ => Labels::Hosted,
},
},
18..=26 => Step::Assign(rng.index()),
27..=29 => Step::AssignListed(rng.index()),
30..=37 => Step::Finish(rng.index()),
38..=39 => Step::Cancel(rng.index()),
40 => Step::Lose(rng.index()),
96 => Step::Removing(rng.index()),
41..=42 => Step::StartFails,
43 => Step::ExitsAtOnce(rng.index()),
44 => Step::ExitsIdle(rng.index()),
95 => Step::ExitsConnected(rng.index()),
45..=46 => Step::InspectFails(rng.index()),
47 => Step::RemoveFails,
48..=54 => Step::Sample {
percent: [10, 50, 90, 120][rng.index() % 4],
available: [3, 6, 8, 9, 20, 50][rng.index() % 6],
stalled: rng.below(5) == 0,
},
55..=62 => Step::Reload(reload(rng)),
63..=64 => Step::Restart,
65 => Step::RestartThenStop,
66..=67 => Step::RestartInspectFails(rng.index()),
68..=71 => Step::Elapse,
97..=98 => Step::GrowthWindow {
percent: [10, 50, 90][rng.index() % 3],
},
72..=73 => Step::Reseed {
light: rng.below(2) == 0,
},
74 => Step::StartedInspectFails,
75..=76 => Step::StartFailsLate {
takes_job: rng.below(2) == 0,
cleanup_fails: rng.below(2) == 0,
},
77..=78 => Step::SampleDuringStart {
percent: [10, 50, 90, 120][rng.index() % 4],
available: [3, 8, 20, 50][rng.index() % 4],
stalled: rng.below(5) == 0,
},
79..=80 => Step::CaptureFails {
mid_stream: rng.below(2) == 0,
},
81..=82 => Step::Stop {
docker_gone: rng.below(2) == 0,
},
83..=89 => Step::Pass,
90..=93 => Step::Settle,
100..=101 => Step::RunnerGoes(rng.index()),
102 => Step::RunnerGoesStillListed(rng.index()),
103..=104 => Step::OpenJobCompletes(rng.index()),
105 => Step::ForcedPause {
second: rng.below(3) == 0,
},
106 => Step::Resume {
second: rng.below(3) == 0,
},
107 => Step::BusyOrphan,
108 => Step::OversizedSnapshot,
109 => Step::Reload(Change::Mirror),
110 => Step::Reload(Change::RecordFailures),
111 => Step::RestartLogsFail,
99 => Step::JobReadFails {
pick: rng.index(),
how: [ReadFailure::Gone, ReadFailure::Limited, ReadFailure::Other][rng.index() % 3],
},
fault @ 112..=118 => mid_batch(rng, fault),
119..=121 => Step::TickSoon {
after: [2, 3, 5][rng.index() % 3],
},
122 => Step::Quiet,
123 => Step::QueueQuiet {
labels: if rng.below(4) == 0 {
Labels::Typo
} else {
Labels::Heavy
},
},
_ => Step::Tick,
})
.collect()
}
fn reload(rng: &mut Rng) -> Change {
match rng.below(10) {
0 => Change::HeavySlots(1 + u32::try_from(rng.below(3)).unwrap()),
1 => Change::LightSlots(1 + u32::try_from(rng.below(3)).unwrap()),
2 => Change::LightPool,
3 => Change::SecondRepository,
4 => Change::LightProfile,
5 => Change::Policy,
6 => Change::PooledCount,
7 => Change::HeavyMemory,
_ => Change::CpuOvercommit,
}
}
fn mid_batch(rng: &mut Rng, fault: u64) -> Step {
match fault {
112 => Step::RegisterFailsMidBatch {
after: u32::try_from(rng.below(4)).unwrap(),
},
113 => Step::RateLimitedDuringStart {
after: u32::try_from(rng.below(4)).unwrap(),
},
114 => Step::PrepareFailsMidBatch {
after: u32::try_from(rng.below(4)).unwrap(),
},
115 => Step::StopDuringStart {
docker_gone: rng.below(2) == 0,
after: u32::try_from(rng.below(4)).unwrap(),
at: if rng.below(2) == 0 {
StopAt::Registration
} else {
StopAt::Launch
},
},
116 => Step::SlowStart,
117 => Step::SliceSampleStallsDuringStart,
_ => Step::AbortMidBatch,
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Comes {
Due,
Soon(Duration),
Command,
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Listing {
Current,
Stale,
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Exit {
AtOnce,
AfterIdleTimeout,
Connected,
}
#[derive(Debug, Clone)]
struct ExitWatch {
container: String,
repository: String,
profile: String,
exit: Exit,
quiet_before: bool,
}
#[derive(Debug, Clone, PartialEq)]
enum Phase {
Queued,
Running {
container: String,
},
Open {
ran: Option<(String, String)>,
},
Done {
ran: Option<(String, String)>,
},
}
#[derive(Debug, Clone)]
struct JobModel {
repository: &'static str,
name: &'static str,
labels: Vec<String>,
phase: Phase,
taken_by: Option<(String, String)>,
listed: bool,
shown: bool,
named_by_github: bool,
}
#[derive(Debug, Clone, Copy)]
struct HostReading {
at: u64,
slice: u64,
available: u64,
stalled: bool,
}
impl HostReading {
fn fresh(&self, now: u64) -> bool {
self.at + 3 * api::SAMPLE_INTERVAL >= now / 1000
}
}
#[derive(Debug, Clone)]
struct Violation {
invariant: &'static str,
step: usize,
detail: String,
}
#[derive(Debug, Clone, Copy)]
struct MidBatch {
reading: Option<HostReading>,
launched: usize,
}
#[derive(Debug, Clone)]
struct AskedStop {
docker_gone: bool,
at: StopAt,
launched: usize,
calls: usize,
}
#[derive(Debug, Clone, Copy, Default)]
struct Seen {
barriers: usize,
credits: usize,
host_credits: usize,
growth_counts: usize,
bands: usize,
released: usize,
exits_freed: usize,
runners_starting: usize,
closing: usize,
ended_uploads: usize,
decided_uploads: usize,
lost_closes: usize,
unanswered_closes: usize,
cancelling: usize,
busy_orphans: usize,
beside_still_listed: usize,
one_due: usize,
oversized: usize,
unread: usize,
mirrors: usize,
split_waits: usize,
captures_with_recaptures: usize,
registrations_failed: usize,
registrations_limited: usize,
prepares_failed: usize,
stops_during_start: usize,
stops_between_launches: usize,
slow_starts: usize,
stalls_during_start: usize,
aborts_mid_batch: usize,
}
impl Seen {
fn add(&mut self, case: Seen) {
self.barriers += case.barriers;
self.exits_freed += case.exits_freed;
self.runners_starting += case.runners_starting;
self.closing += case.closing;
self.ended_uploads += case.ended_uploads;
self.decided_uploads += case.decided_uploads;
self.lost_closes += case.lost_closes;
self.unanswered_closes += case.unanswered_closes;
self.cancelling += case.cancelling;
self.busy_orphans += case.busy_orphans;
self.beside_still_listed += case.beside_still_listed;
self.one_due += case.one_due;
self.oversized += case.oversized;
self.unread += case.unread;
self.mirrors += case.mirrors;
self.split_waits += case.split_waits;
self.captures_with_recaptures += case.captures_with_recaptures;
self.registrations_failed += case.registrations_failed;
self.registrations_limited += case.registrations_limited;
self.prepares_failed += case.prepares_failed;
self.stops_during_start += case.stops_during_start;
self.stops_between_launches += case.stops_between_launches;
self.slow_starts += case.slow_starts;
self.stalls_during_start += case.stalls_during_start;
self.aborts_mid_batch += case.aborts_mid_batch;
self.growth_counts += case.growth_counts;
}
fn unreached(&self) -> Vec<&'static str> {
[
("a barrier", self.barriers),
("the growth count counting", self.growth_counts),
("a runner that left without a job", self.exits_freed),
("a runner started before its job", self.runners_starting),
("a job whose runner went, still open", self.closing),
("uploads waiting for GitHub", self.ended_uploads),
("uploads GitHub's word decided", self.decided_uploads),
("a job lost", self.lost_closes),
("a job GitHub stopped answering for", self.unanswered_closes),
("a cancelled run's runner", self.cancelling),
("a busy orphan registration", self.busy_orphans),
(
"a listing beside a repository whose taken job GitHub still lists",
self.beside_still_listed,
),
("a tick listing one repository alone", self.one_due),
("an oversized snapshot", self.oversized),
("an unread leftover", self.unread),
("the mirror running", self.mirrors),
(
"a pool's memory and CPU waits on different jobs",
self.split_waits,
),
(
"a capture beside a recapture",
self.captures_with_recaptures,
),
("a registration that failed", self.registrations_failed),
(
"a registration refused for the rate limit",
self.registrations_limited,
),
("caches that failed to prepare", self.prepares_failed),
("a stop while a runner registers", self.stops_during_start),
("a stop between two launches", self.stops_between_launches),
("a slow start beside others", self.slow_starts),
("a slice sample stale mid-batch", self.stalls_during_start),
("a daemon gone mid-batch", self.aborts_mid_batch),
]
.into_iter()
.filter_map(|(state, count)| (count == 0).then_some(state))
.collect()
}
}
struct Case {
seen: Seen,
clock: Clock,
github: FakeGithub,
docker: FakeDocker,
dir: TestDir,
engine: Engine<FakeGithub, FakeDocker>,
handle: Handle,
shape: Shape,
jobs: BTreeMap<u64, JobModel>,
next_job: u64,
took_job: BTreeSet<String>,
still_listed: BTreeMap<u64, Option<Duration>>,
cancelled_since_listed: BTreeMap<&'static str, BTreeSet<u64>>,
model: Duration,
calls_seen: usize,
host: Option<HostReading>,
mid_batch: Arc<Mutex<Option<MidBatch>>>,
stop_during_start: Arc<Mutex<Option<AskedStop>>>,
capture_faults: HashMap<String, u32>,
exits: Vec<ExitWatch>,
busy_before: Vec<(String, String, String)>,
ended_between: Vec<(String, String, String)>,
listed_after_runner: BTreeSet<u64>,
forced: BTreeSet<String>,
orphans: Vec<u64>,
unnamed_saves: BTreeMap<u64, String>,
began_ms: u64,
trace: Option<Vec<String>>,
step: usize,
names_from: u64,
}
fn violation(case: &Case, invariant: &'static str, detail: String) -> Result<(), Violation> {
Err(Violation {
invariant,
step: case.step,
detail,
})
}
impl Case {
fn new() -> Case {
let github = FakeGithub::default();
let docker = FakeDocker::default();
github.0.lock().unwrap().strict = true;
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.seed_exit = Some(0);
}
let dir = TestDir::new("invariants");
let shape = Shape::START;
let clock = Clock::stopped();
let (engine, handle) = Case::engine(&github, &docker, &dir, shape, clock.clone());
Case {
seen: Seen::default(),
clock,
github,
docker,
dir,
engine,
handle,
shape,
jobs: BTreeMap::new(),
next_job: 1,
took_job: BTreeSet::new(),
still_listed: BTreeMap::new(),
cancelled_since_listed: BTreeMap::new(),
model: Duration::ZERO,
calls_seen: 0,
host: None,
mid_batch: Arc::default(),
stop_during_start: Arc::default(),
capture_faults: HashMap::new(),
exits: Vec::new(),
busy_before: Vec::new(),
ended_between: Vec::new(),
listed_after_runner: BTreeSet::new(),
forced: BTreeSet::new(),
orphans: Vec::new(),
unnamed_saves: BTreeMap::new(),
began_ms: time::now_ms(),
trace: None,
step: 0,
names_from: 0,
}
}
fn engine(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
shape: Shape,
clock: Clock,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let (mut engine, handle) = if shape.build_cache {
let (engine, handle, _cache) =
build_cache::engine_with_cache(github, docker, dir, &shape.toml());
(engine, handle)
} else {
engine_with(github, docker, dir, &shape.toml())
};
engine.clock = clock;
engine.measured.restart_growth(engine.now());
engine.memory_max = Some(MEMORY_MAX);
engine.cpus = Some(SLICE_CPUS);
(engine, handle)
}
async fn recover(&mut self) -> Result<(), Violation> {
for _ in 0..10 {
if self.engine.recover().await.is_ok() {
return Ok(());
}
}
violation(
self,
"recovery_succeeds",
"Docker never answered".to_owned(),
)
}
fn running_containers(&self) -> Vec<Container> {
self.docker
.0
.lock()
.unwrap()
.containers
.iter()
.filter(|container| container.running && container.labels.contains_key("runnerd.job"))
.cloned()
.collect()
}
fn idle_containers(&self) -> Vec<Container> {
self.running_containers()
.into_iter()
.filter(|container| !self.took_job.contains(&container.id))
.collect()
}
fn queued(&self) -> Vec<u64> {
self.jobs
.iter()
.filter(|(_, job)| job.phase == Phase::Queued)
.map(|(id, _)| *id)
.collect()
}
fn running(&self) -> Vec<(u64, String)> {
self.jobs
.iter()
.filter_map(|(id, job)| match &job.phase {
Phase::Running { container } => Some((*id, container.clone())),
_ => None,
})
.collect()
}
async fn trace_step(&mut self, step: &str) {
if self.trace.is_none() {
return;
}
self.settle_background().await;
let status = steady(&steady_status(&self.status()).to_string());
let github = std::mem::take(&mut self.github.0.lock().unwrap().trace);
let (docker, execs) = {
let mut state = self.docker.0.lock().unwrap();
(
std::mem::take(&mut state.trace),
std::mem::take(&mut state.exec_trace),
)
};
if let Some(trace) = self.trace.as_mut() {
trace.push(format!("-- {step}"));
trace.push(format!("status {status}"));
trace.extend(github.iter().map(|call| format!("github {}", steady(call))));
trace.extend(docker.iter().map(|call| format!("docker {}", steady(call))));
trace.extend(execs.iter().map(|call| format!("capture {}", steady(call))));
}
}
async fn settle_background(&mut self) {
let mut quiet = 0;
let mut last = (usize::MAX, usize::MAX);
while quiet < QUIET_ROUNDS {
self.handle.store.flush().await;
for _ in 0..16 {
tokio::task::yield_now().await;
}
tokio::time::sleep(Duration::from_millis(1)).await;
let now = (self.github.0.lock().unwrap().trace.len(), {
let state = self.docker.0.lock().unwrap();
state.trace.len() + state.exec_trace.len()
});
quiet = if now == last { quiet + 1 } else { 0 };
last = now;
}
}
async fn between_ticks(&mut self) {
let calls = || {
(
self.github.calls().len(),
self.docker.0.lock().unwrap().calls.len(),
)
};
let (mut last, mut quiet) = (calls(), 0);
for _ in 0..QUIET_ROUNDS {
for _ in 0..16 {
tokio::task::yield_now().await;
}
tokio::time::sleep(Duration::from_millis(1)).await;
let now = calls();
quiet = if now == last { quiet + 1 } else { 0 };
if quiet == 2 {
return;
}
last = now;
}
}
async fn note_closes(&mut self) {
self.handle.store.flush().await;
let db = rusqlite::Connection::open(self.dir.0.join("history.db")).unwrap();
let count = |condition: &str| {
db.query_row(
&format!("SELECT count(*) FROM jobs WHERE {condition}"),
[],
|row| row.get::<_, i64>(0),
)
.map(|count| usize::try_from(count).unwrap())
.unwrap()
};
self.seen.lost_closes += count("state = 'lost'");
self.seen.unanswered_closes += count("state = 'elsewhere' AND conclusion IS NULL");
}
fn note_start_faults(&mut self) {
let github = self.github.0.lock().unwrap();
self.seen.registrations_failed += github.registrations_failed;
self.seen.registrations_limited += github.registrations_limited_once;
drop(github);
self.seen.prepares_failed += self.docker.0.lock().unwrap().prepares_failed;
}
async fn finished_trace(mut self) -> Vec<String> {
self.handle.store.flush().await;
let mut trace = self.trace.take().unwrap_or_default();
trace.push("-- jobs".to_owned());
let db = rusqlite::Connection::open(self.dir.0.join("history.db")).unwrap();
let mut rows = db
.prepare(
"SELECT id, repository, state, conclusion, github_completed, runner, \
started_at IS NOT NULL, finished_at IS NOT NULL, slot_waited > 0, host_waited > 0, \
CASE WHEN state = 'lost' OR (state = 'elsewhere' AND conclusion IS NULL) THEN finished_at END \
FROM jobs ORDER BY id",
)
.unwrap();
let began = i64::try_from(self.began_ms).unwrap();
let jobs = rows
.query_map([], |row| {
Ok(format!(
"{} {} {} {:?} {} {:?} {} {} slot_waited {} host_waited {} closed {:?}",
row.get::<_, i64>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, Option<String>>(3)?,
row.get::<_, i64>(4)?,
row.get::<_, Option<String>>(5)?.map(|name| steady(&name)),
row.get::<_, bool>(6)?,
row.get::<_, bool>(7)?,
row.get::<_, bool>(8)?,
row.get::<_, bool>(9)?,
row.get::<_, Option<i64>>(10)?
.map(|at| (at - began + 30_000).div_euclid(60_000)),
))
})
.unwrap();
trace.extend(jobs.map(Result::unwrap));
trace
}
fn status(&mut self) -> api::Status {
self.engine.publish();
self.handle.status.borrow().clone()
}
#[allow(
clippy::too_many_lines,
reason = "one arm per step, which split up would only be listed twice"
)]
async fn apply(&mut self, step: Step) -> Result<(), Violation> {
match step {
Step::Queue { second, labels } => {
self.queue(if second { "o/s" } else { "o/r" }, labels);
}
Step::QueueQuiet { labels } => self.queue("o/t", labels),
Step::TickSoon { after } => {
self.tick_as(Comes::Soon(Duration::from_secs(after)))
.await?;
}
Step::Quiet => self
.engine
.warm
.testing_learned(crate::scheduler::warm::Learned::default()),
Step::Assign(pick) => self.assign(pick, Listing::Current).await,
Step::AssignListed(pick) => self.assign(pick, Listing::Stale).await,
Step::Finish(pick) => {
let running = self.running();
if let Some((id, container)) = pick_from(&running, pick).cloned() {
self.end_job(id, &container, "success");
if (self.unobserved(&container) || self.capture_unread(id, &container))
&& let Some(job) = self.jobs.get_mut(&id)
{
job.phase = Phase::Done { ran: None };
}
let name = self.jobs[&id].name;
let mut docker = self.docker.0.lock().unwrap();
docker
.container_logs
.insert(container.clone(), format!("Running job: {name}\n"));
set_running(&mut docker, &container, false);
}
}
Step::Cancel(pick) => {
let queued = self.queued();
if let Some(id) = pick_from(&queued, pick).copied() {
let mut github = self.github.0.lock().unwrap();
github.queued.retain(|job| job.id != id);
let mut cancelled = details(id, "completed", Some("cancelled"));
cancelled.runner_name = None;
github.jobs.insert(id, cancelled);
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Done { ran: None };
if job.listed {
self.cancelled_since_listed
.entry(job.repository)
.or_default()
.insert(id);
}
}
}
Step::Lose(pick) => self.lose(pick),
Step::Removing(pick) => self.removing(pick),
Step::StartFails => self.docker.0.lock().unwrap().fail_starts += 1,
Step::StartFailsLate {
takes_job,
cleanup_fails,
} => {
let failure = LateFailure {
takes_job,
cleanup_fails,
};
self.docker
.0
.lock()
.unwrap()
.late_failures
.push_back(failure);
}
Step::CaptureFails { mid_stream } => {
let mut docker = self.docker.0.lock().unwrap();
*if mid_stream {
&mut docker.break_captures
} else {
&mut docker.fail_execs
} += 1;
}
Step::ExitsAtOnce(pick) => self.exit_idle(pick, Exit::AtOnce),
Step::ExitsIdle(pick) => self.exit_idle(pick, Exit::AfterIdleTimeout),
Step::ExitsConnected(pick) => self.exit_idle(pick, Exit::Connected),
Step::InspectFails(pick) => self.inspect_fails(pick),
Step::StartedInspectFails => self.docker.0.lock().unwrap().fail_started_inspects += 1,
Step::RemoveFails => self.docker.0.lock().unwrap().fail_removes += 1,
Step::Sample {
percent,
available,
stalled,
} => {
let docker = self.docker.0.lock().unwrap();
let reading = sample(
&docker,
&self.handle.store,
&self.took_job,
self.clock.now_ms(),
percent,
available,
stalled,
);
drop(docker);
self.host = Some(reading);
}
Step::SampleDuringStart {
percent,
available,
stalled,
} => self.sample_during_start(percent, available, stalled),
Step::Reseed { light } => {
let _ = self
.engine
.request_reseed(if light { "light" } else { "heavy" });
}
Step::Reload(change) => {
self.reload(change).await?;
self.tick_as(Comes::Command).await?;
}
Step::Restart => {
self.restart().await?;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()?;
}
Step::RestartThenStop => {
self.restart().await?;
self.stop(false).await?;
self.restart().await?;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()?;
}
Step::Stop { docker_gone } => {
self.stop(docker_gone).await?;
self.restart().await?;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()?;
}
Step::RestartInspectFails(pick) => {
self.inspect_fails(pick);
self.restart().await?;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()?;
}
Step::Elapse => self.elapse(),
Step::GrowthWindow { percent } => {
self.engine
.measured
.age_growth(api::GROWTH_WINDOW + Duration::from_secs(1));
let docker = self.docker.0.lock().unwrap();
let reading = sample(
&docker,
&self.handle.store,
&self.took_job,
self.clock.now_ms(),
percent,
50,
false,
);
drop(docker);
self.host = Some(reading);
}
Step::Settle => self.settle(),
Step::PastRuns => {
self.past_runs(&[("o/r", "build", "heavy", G), ("o/s", "build", "heavy", G)])
.await;
}
Step::PastLargeRuns => {
let runs = [
("o/r", "build", "heavy", 3 * G),
("o/s", "build", "heavy", 3 * G),
("o/r", "lint", "light", 768 * M),
];
self.past_runs(&runs).await;
}
Step::Tick => self.tick().await?,
Step::Pass => self.pass().await?,
Step::RunnerGoes(pick) => self.runner_goes(pick, Listing::Current),
Step::RunnerGoesStillListed(pick) => self.runner_goes(pick, Listing::Stale),
Step::OpenJobCompletes(pick) => self.open_job_completes(pick),
Step::ForcedPause { second } => self.forced_pause(second).await,
Step::Resume { second } => self.resume(second).await,
Step::BusyOrphan => self.busy_orphan(),
Step::OversizedSnapshot => self.oversized_snapshot(),
Step::RestartLogsFail => {
self.docker.0.lock().unwrap().fail_logs = 1;
self.restart().await?;
self.docker.0.lock().unwrap().fail_logs = 0;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()?;
}
Step::UseBuildCache => {
self.shape.build_cache = true;
self.restart().await?;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()?;
}
Step::SaveUnnamed(pick) => self.save_unnamed(pick).await,
Step::RunnerNamed(pick) => self.runner_named(pick),
Step::JobReadFails { pick, how } => self.job_read_fails(pick, how),
Step::RegisterFailsMidBatch { after } => {
let mut github = self.github.0.lock().unwrap();
github.pass_before_fault = after;
github.fail_registrations += 1;
}
Step::RateLimitedDuringStart { after } => {
let mut github = self.github.0.lock().unwrap();
github.pass_before_fault = after;
github.limit_registrations += 1;
}
Step::PrepareFailsMidBatch { after } => {
let mut docker = self.docker.0.lock().unwrap();
docker.prepares_before_fault = after;
docker.fail_prepares += 1;
}
Step::StopDuringStart {
docker_gone,
after,
at,
} => self.stop_during_start(docker_gone, after, at),
Step::SlowStart => {
self.docker.0.lock().unwrap().slow_start = Some(Duration::from_millis(200));
}
Step::SliceSampleStallsDuringStart => self.slice_sample_stalls_during_start(),
Step::AbortMidBatch => {
self.abort_mid_batch().await;
self.restart().await?;
self.tick().await?;
self.a_restart_cleans_up_an_aborted_batch().await?;
self.a_restart_takes_the_host_credit_back()?;
}
}
self.finish_stop_during_start().await?;
self.no_removal_of_a_running_job()
}
fn queue(&mut self, repository: &'static str, labels: Labels) {
let id = self.next_job;
self.next_job += 1;
let name = if matches!(labels, Labels::Light) {
"lint"
} else {
"build"
};
let labels = labels.of(repository);
let mut github = self.github.0.lock().unwrap();
github.queued.push(QueuedJob {
id,
run_id: id,
name: name.to_owned(),
labels: labels.clone(),
created_at: time::now_ms(),
..QueuedJob::default()
});
github.queued_on.insert(id, repository.to_owned());
if id.is_multiple_of(2) {
github
.triggers
.insert(id, build_cache::trigger("push", "main"));
}
self.jobs.insert(
id,
JobModel {
repository,
name,
labels,
phase: Phase::Queued,
taken_by: None,
listed: false,
shown: false,
named_by_github: false,
},
);
}
fn stop_during_start(&mut self, docker_gone: bool, after: u32, at: StopAt) {
let (control, asked, launches, github) = (
self.handle.control.clone(),
self.stop_during_start.clone(),
self.engine.launches(),
self.github.clone(),
);
let ask = move |calls: usize| {
control.shutdown(if docker_gone {
Shutdown::Graceful
} else {
Shutdown::Detach
});
*asked.lock().unwrap() = Some(AskedStop {
docker_gone,
at,
launched: launches.count(),
calls,
});
};
match at {
StopAt::Registration => {
let docker = self.docker.clone();
let mut state = self.github.0.lock().unwrap();
state.register_hook_after = after;
state.on_register = Some(Box::new(move |_| {
if docker_gone {
docker.set_outage(Outage::Refused);
}
ask(usize::MAX);
}));
}
StopAt::Launch => {
let mut state = self.docker.0.lock().unwrap();
state.prepare_hook_after = after;
state.on_prepare = Some(Box::new(move |docker| {
if docker_gone {
docker.outage = Outage::Refused;
}
ask(github.calls().len());
}));
}
}
}
async fn finish_stop_during_start(&mut self) -> Result<(), Violation> {
let Some(asked) = self.stop_during_start.lock().unwrap().take() else {
return Ok(());
};
match asked.at {
StopAt::Registration => self.seen.stops_during_start += 1,
StopAt::Launch => self.seen.stops_between_launches += 1,
}
self.no_start_after_a_stop(&asked)?;
self.stop(asked.docker_gone).await?;
self.restart().await?;
self.tick().await?;
self.a_restart_takes_the_host_credit_back()
}
fn no_start_after_a_stop(&self, asked: &AskedStop) -> Result<(), Violation> {
let mut launched = self.engine.launches().after(asked.launched);
if asked.at == StopAt::Launch && !launched.is_empty() {
launched.remove(0);
}
if !launched.is_empty() {
return violation(
self,
"no_start_after_a_stop",
format!("{launched:?} launched after the stop"),
);
}
let (calls, hooked) = {
let github = self.github.0.lock().unwrap();
(github.calls.clone(), github.hooked_registration.clone())
};
let from = match asked.at {
StopAt::Registration => calls
.iter()
.position(|call| {
hooked
.as_ref()
.is_some_and(|name| *call == format!("register {name}"))
})
.map_or(calls.len(), |at| at + 1),
StopAt::Launch => asked.calls.min(calls.len()),
};
let registered: Vec<&String> = calls[from..]
.iter()
.filter(|call| call.starts_with("register "))
.collect();
if !registered.is_empty() {
return violation(
self,
"no_start_after_a_stop",
format!("{registered:?} after the stop"),
);
}
if let Some(name) = hooked.filter(|_| asked.at == StopAt::Registration)
&& self
.docker
.calls()
.iter()
.any(|call| *call == format!("start {name}"))
{
return violation(
self,
"no_start_after_a_stop",
format!("{name}, registered as the stop came, started its container"),
);
}
Ok(())
}
fn slice_sample_stalls_during_start(&mut self) {
let (clock, launches, mid_batch) = (
self.clock.clone(),
self.engine.launches(),
self.mid_batch.clone(),
);
self.docker.0.lock().unwrap().on_start = Some(Box::new(move |_| {
clock.advance(Duration::from_secs(3 * api::SAMPLE_INTERVAL + 1));
*mid_batch.lock().unwrap() = Some(MidBatch {
reading: None,
launched: launches.count(),
});
}));
}
async fn abort_mid_batch(&mut self) {
let (github, docker) = (self.github.clone(), self.docker.clone());
docker.hold_started();
github.0.lock().unwrap().answered.send_replace(Some(1));
let all_held = async {
loop {
let (starts, in_docker) = {
let state = docker.0.lock().unwrap();
(state.starts_waiting, state.runner_starts)
};
let (answers, registering) = {
let state = github.0.lock().unwrap();
(state.waiting, state.registering)
};
if starts + answers > 0 && in_docker == starts && (answers > 0 || registering == 0)
{
return answers > 0 && starts > 0;
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
};
let both = tokio::select! {
_ = self.engine.tick(Shutdown::No) => false,
both = all_held => both,
};
if both {
self.seen.aborts_mid_batch += 1;
}
github.0.lock().unwrap().answered.send_replace(None);
docker.release_started();
}
async fn a_restart_cleans_up_an_aborted_batch(&mut self) -> Result<(), Violation> {
self.settle_background().await;
let containers: BTreeSet<String> = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.filter(|container| container.labels.contains_key("runnerd.job"))
.map(|container| container.name.clone())
.collect();
let bare: Vec<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.filter(|runner| {
!self.orphans.contains(&runner.id) && !containers.contains(&runner.name)
})
.map(|runner| runner.name.clone())
.collect();
if !bare.is_empty() {
return violation(
self,
"a_restart_cleans_up_an_aborted_batch",
format!("{bare:?} stay registered with no container"),
);
}
self.no_build_cache_left_open_without_a_runner()
}
fn no_build_cache_left_open_without_a_runner(&self) -> Result<(), Violation> {
let held: BTreeSet<&str> = self
.engine
.occupied()
.map(|(_, occupant)| occupant.runner_name.as_str())
.collect();
let ended = self.engine.ended_upload_runners();
let open: Vec<String> = self
.engine
.build_cache_runners()
.into_iter()
.filter(|runner| !held.contains(runner.as_str()) && !ended.contains(runner))
.collect();
if open.is_empty() {
Ok(())
} else {
violation(
self,
"no_build_cache_left_open_without_a_runner",
format!("{open:?} are open with no runner"),
)
}
}
fn runner_names_are_unique(&self) -> Result<(), Violation> {
let reused = self.github.0.lock().unwrap().duplicate_names.clone();
if !reused.is_empty() {
return violation(
self,
"runner_names_are_unique",
format!("GitHub was asked to register {reused:?} again"),
);
}
let mut names = BTreeSet::new();
let twice: Vec<String> = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.filter(|container| container.labels.contains_key("runnerd.job"))
.filter(|container| !names.insert(container.name.clone()))
.map(|container| container.name.clone())
.collect();
if twice.is_empty() {
Ok(())
} else {
violation(
self,
"runner_names_are_unique",
format!("Docker runs two containers named {twice:?}"),
)
}
}
fn runner_goes(&mut self, pick: usize, listing: Listing) {
let running = self.running();
let Some((id, container)) = pick_from(&running, pick).cloned() else {
return;
};
let ran = if self.unobserved(&container) || self.capture_unread(id, &container) {
None
} else {
self.jobs[&id].taken_by.clone()
};
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Open { ran };
let (name, labels) = (job.name, job.labels.clone());
let mut docker = self.docker.0.lock().unwrap();
let runner = docker
.containers
.iter()
.find(|candidate| candidate.id == container)
.map(|candidate| candidate.name.clone());
docker
.containers
.retain(|candidate| candidate.id != container);
drop(docker);
let mut github = self.github.0.lock().unwrap();
github
.runners
.retain(|registered| Some(®istered.name) != runner.as_ref());
github.queued.retain(|queued| queued.id != id);
if listing == Listing::Stale {
github.queued.push(QueuedJob {
id,
run_id: id,
name: name.to_owned(),
labels,
created_at: time::now_ms(),
..QueuedJob::default()
});
self.listed_after_runner.insert(id);
}
self.still_listed.remove(&id);
drop(github);
self.took_job.remove(&container);
if let Some((slot, occupant)) = self
.engine
.occupied()
.find(|(_, occupant)| occupant.container == container)
{
let repository = occupant.repository.clone();
self.ended_between
.push((container, repository, slot.profile));
}
}
fn open(&self) -> Vec<u64> {
self.jobs
.iter()
.filter(|(_, job)| matches!(job.phase, Phase::Open { .. }))
.map(|(id, _)| *id)
.collect()
}
fn open_job_completes(&mut self, pick: usize) {
let open = self.open();
let Some(id) = pick_from(&open, pick).copied() else {
return;
};
let job = self.jobs.get_mut(&id).unwrap();
let Phase::Open { ran } = job.phase.clone() else {
return;
};
job.phase = Phase::Done { ran };
let name = job.name;
let mut github = self.github.0.lock().unwrap();
let runner = github.jobs.get(&id).and_then(|job| job.runner_name.clone());
let mut completed = details(id, "completed", Some("success"));
completed.run_id = id;
completed.name = name.to_owned();
completed.runner_name = runner;
github.jobs.insert(id, completed);
github.queued.retain(|job| job.id != id);
drop(github);
self.listed_after_runner.remove(&id);
}
async fn forced_pause(&mut self, second: bool) {
let repository = if second { "o/s" } else { "o/r" };
let running = self.running();
self.forced.extend(
running
.into_iter()
.filter(|(id, _)| self.jobs[id].repository == repository)
.map(|(_, container)| container),
);
let (reply, answer) = tokio::sync::oneshot::channel();
self.engine
.command(Command::Pause {
target: Target::Repository(repository.to_owned()),
jobs: RunningJobs::Kill,
reply,
})
.await;
let _ = answer.await;
}
async fn resume(&mut self, second: bool) {
let repository = if second { "o/s" } else { "o/r" };
let (reply, answer) = tokio::sync::oneshot::channel();
self.engine
.command(Command::Resume {
target: Target::Repository(repository.to_owned()),
reply,
})
.await;
let _ = answer.await;
}
fn forced_jobs_end(&mut self) {
let gone: Vec<(u64, String)> = {
let docker = self.docker.0.lock().unwrap();
self.running()
.into_iter()
.filter(|(_, container)| {
self.forced.contains(container)
&& !docker.containers.iter().any(|c| c.id == *container)
})
.collect()
};
for (id, container) in gone {
self.end_job(id, &container, "cancelled");
self.jobs.get_mut(&id).unwrap().phase = Phase::Done { ran: None };
}
}
fn busy_orphan(&mut self) {
let name = format!(
"{}orphan-{}",
self.engine.runner_prefix(),
self.orphans.len()
);
let mut github = self.github.0.lock().unwrap();
github.registrations += 1;
let id = 1000 + github.registrations;
github.runners.push(Runner {
id,
name,
status: "online".to_owned(),
busy: true,
});
drop(github);
self.orphans.push(id);
}
fn oversized_snapshot(&mut self) {
let volume = Names::new("runnerd").base("heavy");
self.docker
.0
.lock()
.unwrap()
.volume_sizes
.insert(volume, u64::MAX / 2);
self.engine.make_trim_due();
}
async fn save_unnamed(&mut self, pick: usize) {
let running = self.running();
let Some((id, container)) = pick_from(&running, pick).cloned() else {
return;
};
let name = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.find(|candidate| candidate.id == container)
.map(|candidate| candidate.name.clone());
let Some(socket) = name.and_then(|name| build_cache::socket_of(&self.docker, &name)) else {
return;
};
if let Some(details) = self.github.0.lock().unwrap().jobs.get_mut(&id) {
details.status = "queued".to_owned();
if let Some(runner) = details.runner_name.take() {
self.unnamed_saves.insert(id, runner);
}
}
build_cache::save(&socket).await;
}
fn runner_named(&mut self, pick: usize) {
let unnamed: Vec<u64> = self.unnamed_saves.keys().copied().collect();
let Some(id) = pick_from(&unnamed, pick).copied() else {
return;
};
let runner = self.unnamed_saves.remove(&id);
let mut github = self.github.0.lock().unwrap();
if let Some(details) = github.jobs.get_mut(&id) {
details.runner_name = runner;
}
github
.triggers
.insert(id, build_cache::trigger("push", "main"));
}
fn job_read_fails(&mut self, pick: usize, how: ReadFailure) {
let candidates: Vec<u64> = self
.jobs
.iter()
.filter(|(_, job)| !matches!(job.phase, Phase::Running { .. }))
.map(|(id, _)| *id)
.collect();
let Some(id) = pick_from(&candidates, pick).copied() else {
return;
};
let mut github = self.github.0.lock().unwrap();
match how {
ReadFailure::Gone => drop(github.gone_jobs.insert(id)),
ReadFailure::Limited => drop(
github
.limited_jobs
.insert(id, SystemTime::now() + Duration::from_secs(3600)),
),
ReadFailure::Other => drop(github.failed_jobs.insert(id)),
}
}
fn lose(&mut self, pick: usize) {
if let Some(container) = self.end_running(pick) {
self.docker
.0
.lock()
.unwrap()
.containers
.retain(|candidate| candidate.id != container);
}
}
fn removing(&mut self, pick: usize) {
if let Some(container) = self.end_running(pick) {
let mut docker = self.docker.0.lock().unwrap();
set_running(&mut docker, &container, false);
docker.being_removed.insert(container);
}
}
fn removals_finish(&mut self) {
let mut docker = self.docker.0.lock().unwrap();
let removed = std::mem::take(&mut docker.being_removed);
docker
.containers
.retain(|candidate| !removed.contains(&candidate.id));
}
fn end_running(&mut self, pick: usize) -> Option<String> {
let running = self.running();
let (id, container) = pick_from(&running, pick).cloned()?;
self.end_job(id, &container, "failure");
if (self.unobserved(&container) || self.capture_unread(id, &container))
&& let Some(job) = self.jobs.get_mut(&id)
{
job.phase = Phase::Done { ran: None };
}
Some(container)
}
fn inspect_fails(&mut self, pick: usize) {
let containers = self.running_containers();
if let Some(container) = pick_from(&containers, pick) {
*self
.docker
.0
.lock()
.unwrap()
.fail_inspects
.entry(container.id.clone())
.or_default() += 1;
}
}
async fn assign(&mut self, pick: usize, listing: Listing) {
let idle = self.idle_containers();
let registered: BTreeSet<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
let candidates: Vec<(u64, Container)> = self
.queued()
.into_iter()
.flat_map(|id| {
let job = &self.jobs[&id];
idle.iter()
.filter(|container| {
let label =
|name: &str| container.labels.get(name).cloned().unwrap_or_default();
let labels: Vec<String> = label("runnerd.labels")
.split(',')
.map(str::to_owned)
.collect();
registered.contains(&container.name)
&& label("runnerd.repository") == job.repository
&& !job.labels.is_empty()
&& job.labels.iter().all(|wanted| {
labels.iter().any(|has| has.eq_ignore_ascii_case(wanted))
})
})
.map(move |container| (id, container.clone()))
.collect::<Vec<_>>()
})
.collect();
let Some((id, container)) = pick_from(&candidates, pick).cloned() else {
return;
};
let profile = container.labels["runnerd.profile"].clone();
let repository = container.labels["runnerd.repository"].clone();
{
let mut github = self.github.0.lock().unwrap();
if listing == Listing::Stale {
self.still_listed.insert(id, None);
} else {
github.queued.retain(|job| job.id != id);
}
let mut runner_id = None;
for runner in &mut github.runners {
if runner.name == container.name {
runner.busy = true;
runner_id = Some(runner.id);
}
}
let mut running = details(id, "in_progress", None);
running.run_id = id;
running.name = self.jobs[&id].name.to_owned();
running.runner_name = Some(container.name.clone());
running.labels = self.jobs[&id].labels.clone();
github.jobs.insert(id, running);
github.gone_jobs.remove(&id);
github.limited_jobs.remove(&id);
github.failed_jobs.remove(&id);
assert!(runner_id.is_some(), "an assigned runner is registered");
}
{
let mut docker = self.docker.0.lock().unwrap();
docker
.container_captures
.insert(container.id.clone(), format!("@@job {id} {id}\n"));
docker
.diag
.insert(container.id.clone(), docker::tests::diag_archive(id));
}
self.took_job.insert(container.id.clone());
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Running {
container: container.id.clone(),
};
job.taken_by = Some((profile, repository));
for _ in 0..100 {
self.handle.store.flush().await;
if self
.handle
.store
.job(id)
.await
.unwrap()
.is_some_and(|job| job.summary.runner.is_some())
{
break;
}
tokio::time::sleep(Duration::from_millis(2)).await;
}
}
fn end_job(&mut self, id: u64, container: &str, conclusion: &str) {
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Done {
ran: job.taken_by.clone(),
};
let by_container = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.find(|candidate| candidate.id == container)
.map(|candidate| candidate.name.clone());
let mut github = self.github.0.lock().unwrap();
let name = github
.jobs
.get(&id)
.and_then(|details| details.runner_name.clone())
.or(by_container);
let mut completed = details(id, "completed", Some(conclusion));
completed.run_id = id;
completed.name = job.name.to_owned();
completed.runner_name = name.clone();
github.jobs.insert(id, completed);
github.queued.retain(|job| job.id != id);
self.still_listed.remove(&id);
github
.runners
.retain(|runner| Some(&runner.name) != name.as_ref());
drop(github);
self.took_job.remove(container);
if let Some((slot, occupant)) = self
.engine
.occupied()
.find(|(_, occupant)| occupant.container == container)
{
let repository = occupant.repository.clone();
self.ended_between
.push((container.to_owned(), repository, slot.profile));
}
}
fn exit_idle(&mut self, pick: usize, exit: Exit) {
let idle = self.idle_containers();
let Some(container) = pick_from(&idle, pick).cloned() else {
return;
};
if exit == Exit::AfterIdleTimeout {
for occupant in self.engine.occupants_mut() {
if occupant.container == container.id {
occupant.started_at -= IDLE_TIMEOUT;
}
}
}
let watch = self
.engine
.occupied()
.find(|(_, occupant)| occupant.container == container.id)
.map(|(slot, occupant)| ExitWatch {
container: container.id.clone(),
repository: occupant.repository.clone(),
profile: slot.profile,
exit: match occupant.started_at.elapsed() {
Ok(waited) if exit == Exit::AtOnce && waited >= IDLE_TIMEOUT => {
Exit::AfterIdleTimeout
}
_ => exit,
},
quiet_before: false,
});
self.exits.extend(watch.map(|watch| ExitWatch {
quiet_before: self.pool_is_quiet(&watch.repository, &watch.profile),
..watch
}));
let log = match exit {
Exit::Connected => "Listening for Jobs\n",
Exit::AtOnce | Exit::AfterIdleTimeout => "",
};
let mut docker = self.docker.0.lock().unwrap();
docker
.container_logs
.insert(container.id.clone(), log.to_owned());
set_running(&mut docker, &container.id, false);
}
fn note_busy(&mut self) {
self.busy_before = self
.engine
.occupied()
.filter(|(_, occupant)| {
occupant.listed_busy()
|| occupant.named_job().is_some()
|| self.took_job.contains(&occupant.container)
})
.map(|(slot, occupant)| {
let repository = occupant.repository.clone();
(occupant.container.clone(), repository, slot.profile)
})
.collect();
self.busy_before.append(&mut self.ended_between);
}
fn pool_is_quiet(&self, repository: &str, profile: &str) -> bool {
self.engine
.pool_is_quiet(repository, profile, SystemTime::now())
}
fn exits_hold_only_what_failed_to_start(&mut self) -> Result<(), Violation> {
let in_slots: BTreeSet<String> = self
.engine
.occupied()
.map(|(_, occupant)| occupant.container.clone())
.collect();
let (waiting, freed): (Vec<ExitWatch>, Vec<ExitWatch>) = std::mem::take(&mut self.exits)
.into_iter()
.partition(|watch| in_slots.contains(&watch.container));
self.exits = waiting;
let now = SystemTime::now();
for watch in &freed {
let Some(hold) = self.engine.pool_hold(&watch.repository, &watch.profile) else {
continue;
};
let standing = hold.held.filter(|held| held.until > now);
let job_ended = self
.busy_before
.iter()
.any(|(container, repository, profile)| {
!in_slots.contains(container)
&& *repository == watch.repository
&& *profile == watch.profile
});
match watch.exit {
Exit::AtOnce if standing.is_none() && !job_ended => {
let detail = format!("{watch:?} failed to start and {hold:?} holds nothing");
return violation(self, "exits_hold_only_what_failed_to_start", detail);
}
Exit::Connected if watch.quiet_before && standing.is_some() => {
let failed = freed.iter().any(|other| {
other.exit == Exit::AtOnce
&& other.repository == watch.repository
&& other.profile == watch.profile
});
let cause = standing.map(|held| held.cause);
if !failed && matches!(cause, Some(HoldCause::NoJob | HoldCause::IdleExits)) {
let detail = format!("{watch:?} connected and {hold:?} holds its pool");
return violation(self, "exits_hold_only_what_failed_to_start", detail);
}
}
_ => {}
}
self.seen.exits_freed += 1;
}
Ok(())
}
fn sample_during_start(&mut self, percent: u64, available: u64, stalled: bool) {
let (store, took_job, mid_batch, clock, launches) = (
self.handle.store.clone(),
self.took_job.clone(),
self.mid_batch.clone(),
self.clock.clone(),
self.engine.launches(),
);
self.docker.0.lock().unwrap().on_start = Some(Box::new(move |docker| {
let now = clock.now_ms();
let reading = sample(docker, &store, &took_job, now, percent, available, stalled);
*mid_batch.lock().unwrap() = Some(MidBatch {
reading: Some(reading),
launched: launches.count(),
});
}));
}
fn unobserved(&self, container: &str) -> bool {
self.engine.occupied().any(|(_, occupant)| {
occupant.container == container
&& !self.engine.watches.is_watched(&occupant.runner_name)
})
}
fn capture_unread(&self, id: u64, container: &str) -> bool {
let faults = self.capture_faults_of(container);
let retried = self.capture_faults.get(container).copied().unwrap_or(0);
!self.jobs[&id].named_by_github && (faults >= 2 || faults > retried)
}
fn capture_faults_of(&self, container: &str) -> u32 {
self.docker
.0
.lock()
.unwrap()
.capture_faults
.get(container)
.copied()
.unwrap_or(0)
}
async fn reload(&mut self, change: Change) -> Result<(), Violation> {
let mut shape = self.shape;
match change {
Change::HeavySlots(slots) => shape.heavy_slots = slots,
Change::LightSlots(slots) => {
if shape.light_slots.is_some() {
shape.light_slots = Some(slots);
}
}
Change::LightPool => shape.light_pool = !shape.light_pool,
Change::SecondRepository => shape.second_repository = !shape.second_repository,
Change::LightProfile => {
shape.light_slots = match shape.light_slots {
Some(_) => None,
None => Some(2),
};
}
Change::Policy => {
shape.admission = match shape.admission {
Admission::Caps => Admission::Measured,
Admission::Measured => Admission::Caps,
};
}
Change::CpuOvercommit => shape.cpu_overcommit = !shape.cpu_overcommit,
Change::PooledCount => shape.pooled_count = !shape.pooled_count,
Change::HeavyMemory => shape.heavy_memory = if shape.heavy_memory == 4 { 2 } else { 4 },
Change::Mirror => shape.mirror = !shape.mirror,
Change::RecordFailures => shape.record_failures = !shape.record_failures,
}
let (reply, answer) = tokio::sync::oneshot::channel();
self.engine
.command(Command::ReplaceConfig {
config: Box::new(Config::parse(&shape.toml()).unwrap()),
memory_max: Some(MEMORY_MAX),
cpus: Some(SLICE_CPUS),
commit: Commit::new(|| Ok(None)),
running: RunningSlots::Drain,
author: Author::Owner,
reply,
})
.await;
let applied = answer.await;
if applied != Ok(Ok(Vec::new())) {
return violation(
self,
"a_reload_applies",
format!("{change:?} answered {applied:?}"),
);
}
self.shape = shape;
Ok(())
}
async fn stop(&mut self, docker_gone: bool) -> Result<(), Violation> {
let running = self.running();
let mode = if docker_gone {
self.docker.set_outage(Outage::Refused);
Shutdown::Graceful
} else {
Shutdown::Detach
};
self.handle.control.shutdown(mode);
let stopped = tokio::time::timeout(Duration::from_secs(300), self.engine.schedule()).await;
self.docker.set_outage(Outage::None);
if stopped.is_err() {
return violation(self, "a_stop_ends", format!("{mode:?} never ended"));
}
let left: BTreeSet<String> = self
.running_containers()
.into_iter()
.map(|container| container.id)
.collect();
let gone: Vec<&(u64, String)> = running
.iter()
.filter(|(_, container)| !left.contains(container))
.collect();
if !gone.is_empty() {
let detail = format!("{mode:?} took the containers of running jobs {gone:?}");
return violation(self, "a_stop_leaves_running_jobs", detail);
}
if mode == Shutdown::Detach {
self.a_detach_deregisters_idle_runners()?;
}
Ok(())
}
fn a_detach_deregisters_idle_runners(&self) -> Result<(), Violation> {
let blocked = self.github.0.lock().unwrap().blocked_until;
if blocked.is_some_and(|until| until > SystemTime::now()) {
return Ok(());
}
let registered: BTreeSet<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
let idle: Vec<String> = self
.idle_containers()
.into_iter()
.map(|container| container.name)
.filter(|name| registered.contains(name))
.collect();
if idle.is_empty() {
return Ok(());
}
violation(
self,
"a_detach_deregisters_idle_runners",
format!("{idle:?} stay registered with no job"),
)
}
async fn restart(&mut self) -> Result<(), Violation> {
self.engine.abort_captures();
self.handle.store.flush().await;
let (engine, handle) = Case::engine(
&self.github,
&self.docker,
&self.dir,
self.shape,
self.clock.clone(),
);
self.engine = engine;
self.engine.named = self.names_from;
self.handle = handle;
self.host = None;
self.exits.clear();
{
let mut docker = self.docker.0.lock().unwrap();
docker.on_start = None;
docker.on_prepare = None;
}
self.github.0.lock().unwrap().on_register = None;
let asked = self.stop_during_start.lock().unwrap().take();
if asked.is_some_and(|asked| asked.docker_gone) {
self.docker.set_outage(Outage::None);
}
self.github.unblock();
self.recover().await
}
fn elapse(&mut self) {
let old = self.clock.now_ms() - 1000 * (3 * api::SAMPLE_INTERVAL + 1);
let docker = self.docker.0.lock().unwrap();
let reading = sample(
&docker,
&self.handle.store,
&self.took_job,
old,
50,
50,
false,
);
drop(docker);
self.host = Some(reading);
for occupant in self.engine.occupants_mut() {
occupant.started_at -= IDLE_TIMEOUT;
}
for watch in &mut self.exits {
if watch.exit == Exit::AtOnce {
watch.exit = Exit::AfterIdleTimeout;
}
}
let past = SystemTime::now() - Duration::from_secs(1);
self.engine.expire_holds(past);
self.engine.expire_github_limit(past);
self.github.unblock();
self.engine.age_waits(PRIORITY_AFTER);
self.engine.expire_cancelling();
self.engine.make_unread_due();
self.engine
.age_ended_uploads(crate::scheduler::engine::build_cache::ENDED_WAIT);
self.engine
.age_followed_jobs(crate::scheduler::engine::history::LOST_AFTER);
}
fn settle(&mut self) {
for occupant in self.engine.occupants_mut() {
occupant.age_job_start(POOL_AFTER);
if let Some(admission) = occupant.admission.as_mut() {
admission.basis = Basis::Job {
runs: 10,
peak: JobPeak::P95,
};
}
}
}
async fn past_runs(&mut self, runs: &[(&str, &str, &str, u64)]) {
let at = time::now_ms() - 3600 * 1000;
let store = &self.handle.store;
for (first, (repository, name, profile, peak)) in (1_u64..).map(|n| n * 1_000_000).zip(runs)
{
for id in first..first + 10 {
let runner = format!("earlier-{id}");
store.queued(
repository,
&QueuedJob {
id,
run_id: id,
name: (*name).to_owned(),
created_at: at,
..QueuedJob::default()
},
at,
);
store.runner_started(RunnerRecord {
purpose: None,
name: runner.clone(),
repository: (*repository).to_owned(),
profile: (*profile).to_owned(),
slot: 1,
container: format!("c-{id}"),
started_at: at,
memory_cap: Some(if *profile == "heavy" { 4 * G } else { G }),
cpu_cap: Some(2000),
admission: None,
});
store.claimed(id, id, &runner, at + 1000);
store.samples(vec![SampleRecord {
runner: runner.clone(),
at: at + 2000,
cpu: 0,
cpu_usage: 0,
memory: *peak,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
written: None,
dirty: None,
events: Events::default(),
}]);
let mut ran = details(id, "completed", Some("success"));
ran.run_id = id;
ran.name = (*name).to_owned();
ran.runner_name = Some(runner.clone());
ran.steps = Vec::new();
store.details(ran);
store.runner_removed(&runner, at + 60_000);
}
}
store.flush().await;
self.engine.forget_predictor();
}
fn unregistered_runners_exit(&mut self) {
let registered: BTreeSet<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
let mut docker = self.docker.0.lock().unwrap();
for container in &mut docker.containers {
if container.running
&& container.labels.contains_key("runnerd.job")
&& !registered.contains(&container.name)
{
container.running = false;
}
}
}
fn failed_late(&mut self) {
let failed = std::mem::take(&mut self.docker.0.lock().unwrap().late_started);
for (container, job) in failed {
let Some(id) = job else { continue };
let left = self
.running_containers()
.into_iter()
.find(|candidate| candidate.id == container);
if let Some(left) = left {
let label = |name: &str| left.labels.get(name).cloned().unwrap_or_default();
self.took_job.insert(container.clone());
let job = self.jobs.get_mut(&id).unwrap();
job.phase = Phase::Running { container };
job.taken_by = Some((label("runnerd.profile"), label("runnerd.repository")));
} else {
self.jobs.get_mut(&id).unwrap().taken_by = None;
self.end_job(id, &container, "failure");
}
}
}
async fn tick(&mut self) -> Result<(), Violation> {
self.tick_as(Comes::Due).await
}
async fn tick_as(&mut self, comes: Comes) -> Result<(), Violation> {
let (by, scope) = match comes {
Comes::Due => (POLL_INTERVAL, Scope::Due),
Comes::Soon(after) => (after, Scope::Due),
Comes::Command => (TICK_EVERY, Scope::Every),
};
self.model += by;
self.between_ticks().await;
self.github_lag_ends();
self.unregistered_runners_exit();
let was_short = self.engine.host_is_short();
let calls_before = self.docker.0.lock().unwrap().calls.len();
let github_before = self.github.calls().len();
self.github.0.lock().unwrap().queued_failures.clear();
let seeds_before: BTreeSet<String> = self.engine.seed_memories().into_keys().collect();
self.capture_faults = self.docker.0.lock().unwrap().capture_faults.clone();
let queued_before: BTreeSet<u64> = self.queued().into_iter().collect();
self.note_busy();
let uploads_waiting = self.engine.ended_uploads_waiting();
self.engine.age_running_reads(by.min(TICK_EVERY));
self.engine.age_listings(by);
self.engine.tick_listing(Shutdown::No, scope).await;
self.note_listings(github_before, &queued_before);
let ended = self.engine.ended_upload_runners();
self.seen.decided_uploads += uploads_waiting
.iter()
.filter(|runner| !ended.contains(*runner))
.count();
self.removals_finish();
self.failed_late();
self.exits_hold_only_what_failed_to_start()?;
let after = self.status();
let running: BTreeSet<String> = self
.running_containers()
.into_iter()
.map(|container| container.name)
.collect();
let starts: Vec<String> = self.docker.0.lock().unwrap().calls[calls_before..]
.iter()
.filter_map(|call| call.strip_prefix("start "))
.filter(|name| running.contains(*name))
.map(str::to_owned)
.collect();
self.note_slow_start(starts.len());
let mut started: BTreeSet<String> = starts.iter().cloned().collect();
self.no_pool_gets_more_runners_than_its_queued_jobs(&after, &started)?;
let seeds = self
.engine
.seed_memories()
.keys()
.filter(|profile| !seeds_before.contains(*profile))
.map(|profile| format!("{profile}{SEED_START}"))
.collect::<Vec<_>>();
self.no_slot_is_left_starting()?;
self.runner_names_are_unique()?;
self.no_build_cache_left_open_without_a_runner()?;
let read_during_start = self.starts_after_a_reading(&starts);
let same_reading = read_during_start.is_none();
let seeded = !seeds.is_empty();
if let Some(after) = read_during_start {
started = after;
}
started.extend(seeds);
self.starts_fit_the_budget(&after, &started)?;
self.docker_runs_no_runner_the_engine_lost(&after)?;
self.no_watch_outlives_its_runner()?;
if !after.shutting_down {
self.only_unserved_jobs_wait_for_a_pool(&after)?;
}
self.one_barrier_and_only_for_a_queued_job(&after)?;
self.no_job_waits_for_a_slot_or_the_host_past_its_runners_start(&after)?;
self.the_count_keeps_its_bounds(&after)?;
self.a_runner_started_finds_the_mirror_up(&starts)?;
if same_reading {
self.the_band_follows_the_waiters(was_short, seeded)?;
}
self.note_states();
self.no_removal_of_a_running_job()
}
fn github_lag_ends(&mut self) {
let model = self.model;
let ended: BTreeSet<u64> = self
.still_listed
.iter()
.filter(|(_, seen)| seen.is_some_and(|seen| model >= seen + POLL_INTERVAL))
.map(|(id, _)| *id)
.collect();
self.still_listed.retain(|id, _| !ended.contains(id));
self.github
.0
.lock()
.unwrap()
.queued
.retain(|job| !ended.contains(&job.id));
}
fn note_listings(&mut self, github_before: usize, queued_before: &BTreeSet<u64>) {
let calls = self.github.calls();
let failed: BTreeSet<String> =
std::mem::take(&mut self.github.0.lock().unwrap().queued_failures)
.into_iter()
.collect();
let listed: BTreeSet<&str> = calls[github_before..]
.iter()
.filter_map(|call| call.strip_prefix("queued "))
.filter(|repository| !failed.contains(*repository))
.collect();
let runners_listed: BTreeSet<&str> = calls[github_before..]
.iter()
.filter_map(|call| call.strip_prefix("runners "))
.collect();
let unlisted_stale = self
.still_listed
.iter()
.any(|(id, seen)| seen.is_some() && !listed.contains(self.jobs[id].repository));
if !listed.is_empty() && unlisted_stale {
self.seen.beside_still_listed += 1;
}
if listed.len() == 1 {
self.seen.one_due += 1;
}
for (id, seen) in &mut self.still_listed {
if runners_listed.contains(self.jobs[id].repository) {
seen.get_or_insert(self.model);
}
}
for (id, job) in &mut self.jobs {
if failed.contains(job.repository) && job.phase == Phase::Queued {
job.shown = false;
}
if !listed.contains(job.repository) {
continue;
}
match job.phase {
Phase::Queued if queued_before.contains(id) => {
job.shown = true;
job.listed |= self.shape.serves(job.repository, &job.labels).is_some();
}
Phase::Running { .. } if job.listed && !self.still_listed.contains_key(id) => {
job.named_by_github = true;
}
_ => {}
}
}
self.cancelled_since_listed
.retain(|repository, _| !listed.contains(repository));
}
fn no_slot_is_left_starting(&self) -> Result<(), Violation> {
if self.engine.any_starting() {
return violation(
self,
"no_slot_is_left_starting",
"a slot is still starting".to_owned(),
);
}
Ok(())
}
fn note_slow_start(&mut self, starts: usize) {
let slow = std::mem::take(&mut self.docker.0.lock().unwrap().slow_started);
if slow > 0 && starts > 1 {
self.seen.slow_starts += 1;
}
}
fn note_states(&mut self) {
let counts = self.engine.state_counts();
let seen = &mut self.seen;
seen.closing += counts.closing;
seen.ended_uploads += counts.ended_uploads;
seen.cancelling += counts.cancelling;
seen.busy_orphans += counts.busy_orphans;
seen.oversized += counts.oversized;
seen.unread += counts.unread;
seen.split_waits += counts.split_waits;
seen.captures_with_recaptures += counts.captures_with_recaptures;
seen.mirrors += usize::from(
self.docker
.0
.lock()
.unwrap()
.containers
.iter()
.any(|container| {
container.running && container.labels.contains_key("runnerd.mirror")
}),
);
}
fn starts_after_a_reading(&mut self, starts: &[String]) -> Option<BTreeSet<String>> {
let mid_batch = self.mid_batch.lock().unwrap().take()?;
self.docker.0.lock().unwrap().hooked_start = None;
match mid_batch.reading {
Some(reading) => self.host = Some(reading),
None => self.seen.stalls_during_start += 1,
}
let launched = self.engine.launches().after(mid_batch.launched);
Some(
launched
.into_iter()
.filter(|name| starts.contains(name))
.collect(),
)
}
async fn pass(&mut self) -> Result<(), Violation> {
self.unregistered_runners_exit();
let calls_before = self.docker.0.lock().unwrap().calls.len();
self.engine.exits.any();
self.note_busy();
self.engine.turnover().await;
self.removals_finish();
self.failed_late();
self.exits_hold_only_what_failed_to_start()?;
let after = self.status();
let running: BTreeSet<String> = self
.running_containers()
.into_iter()
.map(|container| container.name)
.collect();
let order: Vec<String> = self.docker.0.lock().unwrap().calls[calls_before..]
.iter()
.filter_map(|call| call.strip_prefix("start "))
.filter(|name| running.contains(*name))
.map(str::to_owned)
.collect();
self.note_slow_start(order.len());
let starts: BTreeSet<String> = order.iter().cloned().collect();
self.no_pass_starts_a_second_runner_for_a_job(&after, &starts)?;
self.no_slot_is_left_starting()?;
self.runner_names_are_unique()?;
self.no_build_cache_left_open_without_a_runner()?;
let bound = self.starts_after_a_reading(&order);
self.starts_fit_the_budget(&after, bound.as_ref().unwrap_or(&starts))?;
self.docker_runs_no_runner_the_engine_lost(&after)?;
self.no_watch_outlives_its_runner()?;
self.one_barrier_and_only_for_a_queued_job(&after)?;
self.no_job_waits_for_a_slot_or_the_host_past_its_runners_start(&after)?;
self.the_count_keeps_its_bounds(&after)?;
self.note_states();
self.no_removal_of_a_running_job()
}
fn no_job_waits_for_a_slot_or_the_host_past_its_runners_start(
&mut self,
status: &api::Status,
) -> Result<(), Violation> {
let engine = &self.engine;
let still: Vec<u64> = status
.queued
.iter()
.filter(|job| job.waiting == Some(api::Waiting::RunnerStarting))
.map(|job| job.id)
.inspect(|_| self.seen.runners_starting += 1)
.filter(|job| engine.waits_for_a_slot(*job) || engine.waits_for_the_host(*job))
.collect();
if still.is_empty() {
return Ok(());
}
violation(
self,
"no_job_waits_for_a_slot_or_the_host_past_its_runners_start",
format!("jobs {still:?} have a runner starting and still wait for a slot or the host"),
)
}
fn no_pass_starts_a_second_runner_for_a_job(
&self,
status: &api::Status,
started: &BTreeSet<String>,
) -> Result<(), Violation> {
let idle: BTreeSet<String> = self.idle_containers().into_iter().map(|c| c.id).collect();
let pools: BTreeSet<(String, String)> = runners_of(status)
.filter(|(_, runner)| started.contains(&runner.name))
.map(|(profile, runner)| (runner.repository.clone(), profile.to_owned()))
.collect();
for (repository, profile) in pools {
let waiting = runners_of(status)
.filter(|(in_profile, runner)| {
*in_profile == profile
&& runner.repository == repository
&& idle.contains(&runner.container)
})
.count();
let wanted = self
.queued()
.into_iter()
.chain(self.cancelled_since_listed.values().flatten().copied())
.chain(self.listed_after_runner.iter().copied())
.filter(|id| {
let job = &self.jobs[id];
job.repository == repository
&& self.shape.serves(job.repository, &job.labels) == Some(profile.as_str())
})
.count();
if waiting > wanted {
return violation(
self,
"no_pass_starts_a_second_runner_for_a_job",
format!(
"{repository} in {profile}: {waiting} runners wait for {wanted} jobs queued or cancelled"
),
);
}
}
Ok(())
}
fn no_pool_gets_more_runners_than_its_queued_jobs(
&self,
status: &api::Status,
started: &BTreeSet<String>,
) -> Result<(), Violation> {
let mut pools: BTreeSet<(String, String)> = BTreeSet::new();
for (profile, runner) in runners_of(status) {
if started.contains(&runner.name) {
pools.insert((runner.repository.clone(), profile.to_owned()));
}
}
let idle: BTreeSet<String> = self.idle_containers().into_iter().map(|c| c.id).collect();
for (repository, profile) in pools {
let waiting = runners_of(status)
.filter(|(in_profile, runner)| {
*in_profile == profile
&& runner.repository == repository
&& idle.contains(&runner.container)
})
.count();
let queued = self
.queued()
.into_iter()
.chain(self.cancelled_since_listed.values().flatten().copied())
.chain(self.listed_after_runner.iter().copied())
.filter(|id| {
let job = &self.jobs[id];
job.repository == repository
&& self.shape.serves(job.repository, &job.labels) == Some(profile.as_str())
})
.count();
if waiting > queued {
return violation(
self,
"no_pool_gets_more_runners_than_its_queued_jobs",
format!(
"{repository} in {profile}: {waiting} runners wait for {queued} queued jobs"
),
);
}
}
Ok(())
}
fn starts_fit_the_budget(
&self,
status: &api::Status,
started: &BTreeSet<String>,
) -> Result<(), Violation> {
if started.is_empty() {
return Ok(());
}
let budget = &status.budget;
let runner_started = started.iter().any(|name| !name.ends_with(SEED_START));
if self.shape.admission == Admission::Caps {
let running = self.running_containers();
let docker = self.docker.0.lock().unwrap();
let caps: u64 = running
.iter()
.map(|container| docker.memory.get(&container.id).copied().unwrap_or(0))
.sum();
let cpus: u64 = running
.iter()
.map(|container| docker.cpus.get(&container.id).copied().unwrap_or(0))
.sum();
drop(docker);
if runner_started && self.shape.cpu_overcommit && cpus > SLICE_CPUS {
return violation(
self,
"starts_fit_the_budget",
format!("started {started:?} with CPU limits of {cpus} in {SLICE_CPUS}"),
);
}
let seeds: u64 = budget.seed_memory.iter().map(|seed| seed.memory).sum();
if caps.saturating_add(seeds) > MEMORY_MAX {
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} with caps of {caps} and seeds of {seeds} in {MEMORY_MAX}"
),
);
}
}
if let Some(max) = budget.memory_max
&& budget.counted() > max
{
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} with {} of {max} counted, seed containers {:?}",
budget.counted(),
budget.seed_memory
),
);
}
if let Some(cpu) = &budget.cpu
&& runner_started
&& cpu.in_use > cpu.budget
{
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} with {} of {} millicores reserved",
cpu.in_use, cpu.budget
),
);
}
let now = self.clock.now_ms();
if let Some(reading) = self.host.filter(|reading| reading.fresh(now)) {
let limit = reading
.slice
.saturating_add(reading.available)
.saturating_sub(HOST_MARGIN);
let seeds: u64 = self.engine.seed_memories().values().sum();
let credit = host_credit(budget);
let reserved = runners_of(status)
.map(|(_, runner)| runner.reserved)
.fold(seeds, u64::saturating_add)
.saturating_sub(credit);
if reserved > limit {
return violation(
self,
"starts_fit_the_budget",
format!("started {started:?} with {reserved} reserved past the host's {limit}"),
);
}
if reading.stalled && reading.available < 2 * HOST_MARGIN {
return violation(
self,
"starts_fit_the_budget",
format!(
"started {started:?} while the host stalls with {} available",
reading.available
),
);
}
}
Ok(())
}
fn the_count_keeps_its_bounds(&mut self, status: &api::Status) -> Result<(), Violation> {
let budget = &status.budget;
let credit = budget.pooled.map_or(0, |pooled| pooled.credit);
let host_credit = host_credit(budget);
let pools = self.shape.admission == Admission::Measured && self.shape.pooled_count;
if budget.counted.is_some() != pools || ((credit > 0 || host_credit > 0) && !pools) {
let detail = format!("{budget:?} under {:?}", self.shape);
return violation(self, "the_count_keeps_its_bounds", detail);
}
self.plans_read_the_count(budget, host_credit)?;
let committed = self.engine.committed_memory();
if let Some(kept) = budget.counted.as_ref().and_then(|counted| counted.kept) {
return self.the_growth_count_keeps_its_bounds(budget, kept, committed);
}
if credit > pooled::MOST_CREDIT {
let detail = format!("{budget:?} under {:?}", self.shape);
return violation(self, "the_count_keeps_its_bounds", detail);
}
let hosted = budget.memory_in_use - host_credit;
if credit == 0 && host_credit == 0 {
return Ok(());
}
let now = SystemTime::now();
let fresh_after = self
.clock
.now_ms()
.saturating_sub(3000 * api::SAMPLE_INTERVAL);
let mut floor = committed;
let mut headroom = 0;
for (_, occupant) in self.engine.occupied() {
let sample = self.handle.store.latest_sample(&occupant.runner_name);
let settled = occupant
.job_found_at()
.and_then(|started| now.duration_since(started).ok())
.is_some_and(|ran| ran >= POOL_AFTER);
let predicted = occupant.admission.is_some_and(|admission| {
matches!(
admission.basis,
Basis::Job { .. } | Basis::Profile { .. } | Basis::ProfileMax { .. }
)
});
let known = occupant.has_key() && occupant.was_started_here();
let takes = self.engine.takes(occupant);
match sample.filter(|sample| sample.at >= fresh_after) {
Some(sample) if settled && predicted && known => {
floor += sample.memory;
headroom += takes.saturating_sub(sample.memory);
}
_ => floor += takes,
}
}
let slice = self
.handle
.store
.latest_host_sample()
.filter(|sample| sample.at * 1000 >= fresh_after)
.map(|sample| sample.memory);
if host_credit > 0 {
self.seen.host_credits += 1;
let kept = headroom.min(pooled::HOST_RESERVE_FLOOR);
if hosted < floor + kept {
let detail =
format!("{budget:?} counts {hosted} for the host, under {floor} and {kept}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
let read = self
.host
.filter(|reading| reading.fresh(self.clock.now_ms()));
let stalls = budget.host.as_ref().is_none_or(|host| host.stalled);
if read.is_none() || slice.is_none() || stalls {
let detail = format!(
"a host credit of {host_credit} beside the reading {read:?}, the slice's {slice:?} and {:?}",
budget.host
);
return violation(self, "the_count_keeps_its_bounds", detail);
}
}
if credit == 0 {
return Ok(());
}
self.seen.credits += 1;
if budget.counted() < floor {
let detail = format!("{budget:?} counts under {floor}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
let host = budget.host.as_ref();
let keeps = HOST_MARGIN + HOST_TOTAL / 4;
if host.is_none_or(|host| host.stalled || host.memory.available < keeps + credit) {
let detail = format!("{credit} of credit beside the host's {host:?}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
let room = slice.map(|slice| {
(MEMORY_MAX * 90 / 100)
.saturating_sub(slice)
.saturating_sub(pooled::HIGH_WATER_RESERVE)
});
if room.is_none_or(|room| credit > room) {
let detail = format!("{credit} of credit beside the slice's {slice:?}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
Ok(())
}
fn the_growth_count_keeps_its_bounds(
&mut self,
budget: &api::Budget,
kept: u64,
committed: u64,
) -> Result<(), Violation> {
self.seen.growth_counts += 1;
let read = self
.host
.filter(|reading| reading.fresh(self.clock.now_ms()));
let fresh_after = self
.clock
.now_ms()
.saturating_sub(3000 * api::SAMPLE_INTERVAL);
let slice = self
.handle
.store
.latest_host_sample()
.filter(|sample| sample.at * 1000 >= fresh_after)
.map(|sample| sample.memory);
let keeps = HOST_MARGIN + HOST_TOTAL / 4;
let host = budget.host.as_ref();
let (Some(slice), Some(host), Some(_)) = (slice, host, read) else {
let detail = format!(
"{budget:?} counts growth beside the reading {read:?} and the slice's {slice:?}"
);
return violation(self, "the_count_keeps_its_bounds", detail);
};
if host.stalled || host.memory.available < keeps {
let detail = format!("{budget:?} counts growth beside the host's {host:?}");
return violation(self, "the_count_keeps_its_bounds", detail);
}
let mut used = committed;
let mut unsampled = 0;
let mut headroom = 0;
for (_, occupant) in self.engine.occupied() {
let takes = self.engine.takes(occupant);
match self
.handle
.store
.latest_sample(&occupant.runner_name)
.filter(|sample| sample.at >= fresh_after)
{
Some(sample) => {
used += sample.memory.min(takes);
headroom += takes.saturating_sub(sample.memory);
}
None => unsampled += takes,
}
}
let credit = budget.pooled.map_or(0, |pooled| pooled.credit);
let host_credit = host_credit(budget);
let hosted = budget.memory_in_use - host_credit;
let least = (used + unsampled).max(used + headroom.min(pooled::GROWTH_FLOOR));
if kept < headroom.min(pooled::GROWTH_FLOOR) || budget.counted().min(hosted) < least {
let detail = format!(
"{budget:?} counts under {used} used, {unsampled} unsampled and {headroom} of headroom"
);
return violation(self, "the_count_keeps_its_bounds", detail);
}
let host_room = host.memory.available - keeps;
let slice_room = (MEMORY_MAX * 90 / 100)
.saturating_sub(slice)
.saturating_sub(pooled::HIGH_WATER_RESERVE);
if credit.max(host_credit) > host_room.min(pooled::GROWTH_CEILING) || credit > slice_room {
let detail = format!(
"credits of {credit} and {host_credit} beside the host's {host_room}, the slice's {slice_room} and the \
ceiling"
);
return violation(self, "the_count_keeps_its_bounds", detail);
}
Ok(())
}
fn plans_read_the_count(
&mut self,
budget: &api::Budget,
host_credit: u64,
) -> Result<(), Violation> {
let admitting = self.engine.admitting(&mut [], None);
let plan = self.engine.plan_budget(&admitting);
let left = budget
.memory_max
.map(|max| crate::scheduler::left(max, budget.counted()));
if !admitting.held && plan.memory != left {
let detail = format!("a plan reads {:?}, the status {left:?}", plan.memory);
return violation(self, "the_count_keeps_its_bounds", detail);
}
let hosted = budget.memory_in_use - host_credit;
let host = admitting
.host
.as_ref()
.map(|host| host::left(host.limit, hosted, host.stalled));
if !admitting.held && plan.host != host {
let detail = format!(
"a plan's host reads {:?}, the host's count {host:?}",
plan.host
);
return violation(self, "the_count_keeps_its_bounds", detail);
}
let running = self.engine.occupied().next().is_some();
let committed = self.engine.committed_memory();
let drained = admitting.host.as_ref().map(|host| {
let limit = host::limit(&host.memory, running || host.stalled);
crate::scheduler::left(limit, committed)
});
if plan.drained.host != drained {
let detail = format!("a drain leaves {:?}, not {drained:?}", plan.drained.host);
return violation(self, "the_count_keeps_its_bounds", detail);
}
Ok(())
}
fn a_restart_takes_the_host_credit_back(&mut self) -> Result<(), Violation> {
let status = self.status();
if host_credit(&status.budget) > 0 {
let detail = format!("{:?} after a restart", status.budget.host);
return violation(self, "a_restart_takes_the_host_credit_back", detail);
}
Ok(())
}
fn the_band_follows_the_waiters(
&mut self,
was_short: bool,
seeded: bool,
) -> Result<(), Violation> {
let short = self.engine.host_is_short();
let (plan, admitting) = self.engine.plan();
let Some(host) = admitting.host.as_ref() else {
if short {
let detail = "the larger margin with no reading".to_owned();
return violation(self, "the_band_follows_the_waiters", detail);
}
return Ok(());
};
if host.stalled {
if !short {
let detail = format!("a stall without the larger margin: {:?}", host.memory);
return violation(self, "the_band_follows_the_waiters", detail);
}
return Ok(());
}
let in_use = self.engine.count(self.engine.host_remembered()).plain;
let room = host::left(host::limit(&host.memory, false), in_use, false);
let needs: Vec<i64> = plan
.waiting
.values()
.filter_map(|reason| match reason {
api::Waiting::HostMemory { needs, .. } => i64::try_from(*needs).ok(),
_ => None,
})
.collect();
let fits = needs.iter().any(|needs| *needs <= room);
let starts = plan
.decisions
.iter()
.any(|decision| matches!(decision, Decision::Start { .. }));
if short && fits {
let detail = format!("{needs:?} wait, {room} is left by the plain margin");
return violation(self, "the_band_follows_the_waiters", detail);
}
if !short && !fits && !needs.is_empty() && !starts && !seeded {
let detail = format!("{needs:?} wait, {room} is left, and the margin is plain");
return violation(self, "the_band_follows_the_waiters", detail);
}
self.seen.bands += usize::from(short && !needs.is_empty());
self.seen.released += usize::from(was_short && !short && starts);
Ok(())
}
fn one_barrier_and_only_for_a_queued_job(
&mut self,
status: &api::Status,
) -> Result<(), Violation> {
let behind = |reason: &Option<api::Waiting>| match reason {
Some(api::Waiting::BehindMemory { job } | api::Waiting::BehindCpu { job }) => {
Some(*job)
}
_ => None,
};
let owners: BTreeSet<u64> = status
.queued
.iter()
.filter_map(|job| behind(&job.waiting))
.collect();
self.seen.barriers += owners.len();
if owners.len() > 1 {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("jobs wait behind each of {owners:?}"),
);
}
for owner in &owners {
let Some(queued) = status.queued.iter().find(|job| job.id == *owner) else {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("jobs wait behind {owner}, which no longer waits"),
);
};
if behind(&queued.waiting).is_some() {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("{owner} holds others back and says {:?}", queued.waiting),
);
}
if let Some(younger) = status
.queued
.iter()
.find(|job| behind(&job.waiting).is_some() && job.id <= *owner)
{
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!(
"{} is older than {owner}, which it waits behind",
younger.id
),
);
}
}
for listed in self.engine.listed_repositories() {
let (name, queued) = (listed.name, listed.queued);
let waits = listed.waits.iter();
let kept = waits.flat_map(|waits| waits.memory().into_iter().chain(waits.cpu()));
if let Some(wait) = kept
.into_iter()
.find(|wait| !queued.iter().any(|job| job.id == wait.job))
{
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("{name} keeps a wait for job {}, listed no more", wait.job),
);
}
if queued.is_empty() && listed.waits.iter().any(crate::scheduler::Waits::served) {
return violation(
self,
"one_barrier_and_only_for_a_queued_job",
format!("{name} keeps a served mark with nothing queued"),
);
}
}
Ok(())
}
fn a_runner_started_finds_the_mirror_up(&self, starts: &[String]) -> Result<(), Violation> {
let held = self.engine.occupied().next().is_some();
let up = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.any(|container| container.running && container.labels.contains_key("runnerd.mirror"));
if self.shape.mirror && held && !up && !starts.is_empty() {
let detail = format!("{starts:?} started and no mirror runs");
return violation(self, "a_runner_started_finds_the_mirror_up", detail);
}
Ok(())
}
fn no_removal_of_a_running_job(&mut self) -> Result<(), Violation> {
self.forced_jobs_end();
let calls = self.docker.calls();
let running: BTreeMap<String, u64> = self
.running()
.into_iter()
.map(|(id, container)| (container, id))
.collect();
for call in &calls[self.calls_seen..] {
if let Some(container) = call.strip_prefix("remove ")
&& let Some(job) = running.get(container)
&& !self.forced.contains(container)
{
let detail = format!("removed {container}, which runs job {job}");
self.calls_seen = calls.len();
return violation(self, "no_removal_of_a_running_job", detail);
}
}
self.calls_seen = calls.len();
Ok(())
}
fn docker_runs_no_runner_the_engine_lost(&self, status: &api::Status) -> Result<(), Violation> {
let held: BTreeSet<&str> = runners_of(status)
.map(|(_, runner)| runner.container.as_str())
.collect();
let lost: Vec<String> = self
.running_containers()
.into_iter()
.filter(|container| !held.contains(container.id.as_str()))
.map(|container| container.id)
.collect();
if lost.is_empty() {
Ok(())
} else {
violation(
self,
"docker_runs_no_runner_the_engine_lost",
format!("no slot holds {lost:?}"),
)
}
}
fn no_watch_outlives_its_runner(&self) -> Result<(), Violation> {
let held: BTreeSet<&str> = self
.engine
.occupied()
.map(|(_, occupant)| occupant.runner_name.as_str())
.collect();
let outlived: Vec<&str> = self
.engine
.watches
.runners()
.filter(|runner| !held.contains(runner))
.collect();
if outlived.is_empty() {
Ok(())
} else {
violation(
self,
"no_watch_outlives_its_runner",
format!("{outlived:?} are watched with no slot"),
)
}
}
fn only_unserved_jobs_wait_for_a_pool(&self, status: &api::Status) -> Result<(), Violation> {
for listed in &status.queued {
let job = &self.jobs[&listed.id];
let hosted = job.labels == ["ubuntu-latest"];
let unserved = self.shape.serves(job.repository, &job.labels).is_none();
let no_pool = matches!(listed.waiting, Some(api::Waiting::NoPool { .. }));
if hosted || unserved != no_pool {
return violation(
self,
"only_unserved_jobs_wait_for_a_pool",
format!(
"job {} asking for {:?} waits with {:?}",
listed.id, job.labels, listed.waiting
),
);
}
}
if status.github_limited_until.is_some() {
return Ok(());
}
for id in self.queued() {
let job = &self.jobs[&id];
if job.shown
&& self.shape.configures(job.repository)
&& job.labels != ["ubuntu-latest"]
&& !status.queued.iter().any(|listed| listed.id == id)
{
return violation(
self,
"only_unserved_jobs_wait_for_a_pool",
format!("queued job {id} asking for {:?} is not listed", job.labels),
);
}
}
Ok(())
}
async fn drain(&mut self) -> Result<(), Violation> {
self.step += 1;
self.github.0.lock().unwrap().on_register = None;
self.docker.0.lock().unwrap().on_prepare = None;
while !self.queued().is_empty() {
self.apply(Step::Cancel(0)).await?;
}
while !self.running().is_empty() {
self.apply(Step::Finish(0)).await?;
}
while !self.open().is_empty() {
self.open_job_completes(0);
}
let orphans = std::mem::take(&mut self.orphans);
self.github
.0
.lock()
.unwrap()
.runners
.retain(|runner| !orphans.contains(&runner.id));
for _ in 0..6 {
self.elapse();
self.tick().await?;
}
self.no_runner_is_left()?;
self.names_only_rise()?;
self.history_names_the_pool_that_ran_each_job().await?;
self.no_job_stays_running().await
}
fn names_only_rise(&self) -> Result<(), Violation> {
let named: Vec<String> = self
.github
.calls()
.iter()
.filter_map(|call| call.strip_prefix("register "))
.map(str::to_owned)
.collect();
let second = |name: &String| {
name.rsplit_once('-')
.and_then(|(_, secs)| secs.parse::<u64>().ok())
};
match named
.windows(2)
.find(|pair| second(&pair[1]) <= second(&pair[0]))
{
Some(pair) => violation(
self,
"names_only_rise",
format!("{} was named after {}", pair[1], pair[0]),
),
None => Ok(()),
}
}
fn no_runner_is_left(&mut self) -> Result<(), Violation> {
let status = self.status();
let containers: Vec<String> = self
.docker
.0
.lock()
.unwrap()
.containers
.iter()
.filter(|container| container.labels.contains_key("runnerd.job"))
.map(|container| container.id.clone())
.collect();
let slots: Vec<String> = runners_of(&status)
.map(|(_, runner)| runner.name.clone())
.collect();
let registered: Vec<String> = self
.github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.name.clone())
.collect();
if containers.is_empty() && slots.is_empty() && registered.is_empty() {
Ok(())
} else {
violation(
self,
"no_runner_is_left",
format!("containers {containers:?}, slots {slots:?}, registered {registered:?}"),
)
}
}
async fn history_names_the_pool_that_ran_each_job(&mut self) -> Result<(), Violation> {
self.handle.store.flush().await;
let ran: Vec<(u64, (String, String))> = self
.jobs
.iter()
.filter_map(|(id, job)| match &job.phase {
Phase::Done { ran: Some(ran) } => Some((*id, ran.clone())),
_ => None,
})
.collect();
for (id, (profile, repository)) in ran {
let recorded = self.handle.store.job(id).await.unwrap();
let seen = recorded.as_ref().map(|job| {
(
job.summary
.runner
.as_ref()
.map(|runner| runner.profile.clone()),
job.summary.repository.clone(),
)
});
if seen != Some((Some(profile.clone()), repository.clone())) {
return violation(
self,
"history_names_the_pool_that_ran_each_job",
format!("job {id} ran in {repository} {profile}; the history says {seen:?}"),
);
}
}
Ok(())
}
async fn no_job_stays_running(&mut self) -> Result<(), Violation> {
for id in self.jobs.keys().copied().collect::<Vec<_>>() {
let recorded = self.handle.store.job(id).await.unwrap();
if let Some(job) = recorded.filter(|job| job.summary.state == JobState::Running) {
return violation(
self,
"no_job_stays_running",
format!("job {id} is still running on {:?}", job.summary.runner),
);
}
}
Ok(())
}
}
fn sample(
docker: &DockerState,
store: &Store,
took_job: &BTreeSet<String>,
now: u64,
percent: u64,
available: u64,
stalled: bool,
) -> HostReading {
let mut slice = 0;
let mut samples = Vec::new();
for container in docker
.containers
.iter()
.filter(|container| container.running && container.labels.contains_key("runnerd.job"))
{
let cap = docker.memory.get(&container.id).copied().unwrap_or(G);
let memory = if took_job.contains(&container.id) {
cap / 100 * percent
} else {
100 * M
};
slice += memory;
samples.push(SampleRecord {
runner: container.name.clone(),
at: now,
cpu: 0,
cpu_usage: 0,
memory,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
written: None,
dirty: None,
events: Events::default(),
});
}
store.samples(samples);
store.host_sample(api::HostSample {
at: now / 1000,
cpu: 0,
memory: slice,
});
store.host_memory(api::HostMemory {
at: now / 1000,
total: HOST_TOTAL,
available: available * G,
slice,
some: Some(api::Pressure::default()),
full: Some(api::Pressure {
avg10: if stalled { 600 } else { 0 },
avg60: 0,
}),
holders: Vec::new(),
ceiling: None,
swap: None,
});
HostReading {
at: now / 1000,
slice,
available: available * G,
stalled,
}
}
fn pick_from<T>(items: &[T], pick: usize) -> Option<&T> {
(!items.is_empty()).then(|| &items[pick % items.len()])
}
fn set_running(docker: &mut DockerState, container: &str, running: bool) {
for candidate in &mut docker.containers {
if candidate.id == container {
candidate.running = running;
}
}
}
fn host_credit(budget: &api::Budget) -> u64 {
budget
.host
.as_ref()
.and_then(|host| host.credit)
.map_or(0, |credit| credit.bytes)
}
fn runners_of(status: &api::Status) -> impl Iterator<Item = (&str, &api::RunnerStatus)> {
status.profiles.iter().flat_map(|profile| {
profile
.slots
.iter()
.filter_map(|slot| slot.runner.as_ref())
.map(|runner| (profile.name.as_str(), runner))
})
}
fn steady_status(status: &api::Status) -> serde_json::Value {
fn blank(value: &mut serde_json::Value) {
match value {
serde_json::Value::Number(_) => *value = 0.into(),
serde_json::Value::Array(items) => items.iter_mut().for_each(blank),
serde_json::Value::Object(fields) => {
fields.retain(|name, _| !UNSTEADY_STATUS.contains(&name.as_str()));
fields.values_mut().for_each(blank);
}
_ => {}
}
}
let mut value = serde_json::to_value(status).unwrap();
blank(&mut value);
value
}
fn steady(line: &str) -> String {
let mut out = String::with_capacity(line.len());
let bytes = line.as_bytes();
let mut i = 0;
while i < bytes.len() {
let run = bytes[i + 1..]
.iter()
.take_while(|byte| byte.is_ascii_digit())
.count();
if bytes[i] == b'-' && run == 10 {
i += 11;
} else {
out.push(char::from(bytes[i]));
i += 1;
}
}
out
}
async fn run(steps: &[Step]) -> Result<Seen, Violation> {
run_case(steps, Case::new()).await.map(|case| case.seen)
}
async fn run_case(steps: &[Step], mut case: Case) -> Result<Case, Violation> {
case.recover().await?;
case.tick().await?;
case.trace_step("start").await;
for (index, step) in steps.iter().enumerate() {
case.step = index;
case.apply(*step).await?;
case.trace_step(&format!("{index} {step:?}")).await;
}
case.step = steps.len();
case.drain().await?;
case.trace_step("drained").await;
case.note_closes().await;
case.note_start_faults();
Ok(case)
}
const LIGHT: Step = Step::Queue {
second: false,
labels: Labels::Light,
};
const A_BATCH_ABORTED: [Step; 4] = [LIGHT, LIGHT, HEAVY, Step::AbortMidBatch];
const A_STOP_WHILE_A_BATCH_REGISTERS: [Step; 5] = [
LIGHT,
LIGHT,
HEAVY,
Step::StopDuringStart {
docker_gone: false,
after: 1,
at: StopAt::Registration,
},
Step::Tick,
];
const A_SLOW_STALLING_START_IN_A_BATCH: [Step; 8] = [
Step::Reload(Change::Policy),
Step::Sample {
percent: 50,
available: 50,
stalled: false,
},
LIGHT,
LIGHT,
HEAVY,
Step::SlowStart,
Step::SliceSampleStallsDuringStart,
Step::Tick,
];
fn steps_of(seed: u64) -> Vec<Step> {
let mut steps = generate(&mut Rng(seed));
if seed % 2 == 1 {
steps.splice(0..0, TWO_JOBS_SETTLED);
}
match seed % 8 {
0 => drop(steps.splice(0..0, SLOT_FREED_BETWEEN_TICKS)),
2 => drop(steps.splice(0..0, BAND_RELEASED[..BAND_STANDS].iter().copied())),
4 => drop(steps.splice(0..0, THREE_JOBS_SETTLED)),
6 => drop(steps.splice(0..0, BAND_RELEASED)),
_ => {}
}
if seed.is_multiple_of(32) {
steps.splice(0..0, WAITS_ON_DIFFERENT_JOBS);
}
if seed.is_multiple_of(8) && !seed.is_multiple_of(32) {
let batch: &[Step] = match seed % 32 {
8 => &A_BATCH_ABORTED,
16 => &A_STOP_WHILE_A_BATCH_REGISTERS,
_ => &A_SLOW_STALLING_START_IN_A_BATCH,
};
steps.splice(0..0, batch.iter().copied().chain([Step::Tick, Step::Tick]));
}
if seed % 4 == 3 {
steps.splice(
0..0,
A_LISTING_BESIDE_A_NEWLY_BUSY_RUNNER
.iter()
.copied()
.chain([Step::Tick, Step::Tick]),
);
}
if seed % 2 == 1 {
let rarely: &[Step] = match seed % 16 {
1 => &A_JOB_STAYS_OPEN,
3 => &A_JOB_STAYS_LISTED,
5 => &THE_MIRROR_RUNS,
7 => &A_FORCED_PAUSE,
9 => &A_BUSY_ORPHAN,
11 => &AN_OVERSIZED_SNAPSHOT,
13 => &AN_UNREAD_LEFTOVER,
_ if seed % 32 == 31 => &UPLOADS_DECIDED,
_ => &UPLOADS_WAIT_FOR_GITHUB,
};
steps.splice(0..0, rarely.iter().copied().chain([Step::Tick, Step::Tick]));
}
steps
}
async fn attempt(steps: &[Step]) -> Result<Seen, Violation> {
match AssertUnwindSafe(Box::pin(run(steps))).catch_unwind().await {
Ok(outcome) => outcome,
Err(panic) => Err(Violation {
invariant: "no_panic",
step: 0,
detail: panic
.downcast_ref::<&str>()
.map(|message| (*message).to_owned())
.or_else(|| panic.downcast_ref::<String>().cloned())
.unwrap_or_default(),
}),
}
}
async fn shrink(mut steps: Vec<Step>, found: Violation) -> (Vec<Step>, Violation) {
let mut last = found;
let mut chunk = steps.len() / 2;
while chunk >= 1 {
let mut start = 0;
while start < steps.len() {
let mut fewer = steps.clone();
fewer.drain(start..(start + chunk).min(steps.len()));
match attempt(&fewer).await {
Err(found) if found.invariant == last.invariant => {
steps = fewer;
last = found;
}
_ => start += chunk,
}
}
chunk /= 2;
}
(steps, last)
}
const BAND_RELEASED: [Step; 10] = [
Step::Reload(Change::Policy),
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Sample {
percent: 10,
available: 20,
stalled: false,
},
Step::Tick,
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Sample {
percent: 10,
available: 3,
stalled: false,
},
Step::Tick,
Step::Sample {
percent: 10,
available: 9,
stalled: false,
},
Step::Tick,
Step::Tick,
];
const BAND_STANDS: usize = 7;
const SLOT_FREED_BETWEEN_TICKS: [Step; 7] = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Tick,
Step::Assign(0),
Step::Finish(0),
Step::Pass,
];
const TWO_JOBS_SETTLED: [Step; 11] = [
Step::PastRuns,
Step::Reload(Change::Policy),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Assign(0),
Step::Assign(0),
Step::Tick,
Step::Settle,
Step::Sample {
percent: 10,
available: 50,
stalled: false,
},
Step::Tick,
];
const THREE_JOBS_SETTLED: [Step; 17] = [
Step::PastLargeRuns,
Step::Reload(Change::Policy),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Assign(0),
Step::Assign(0),
Step::Tick,
Step::Settle,
Step::Sample {
percent: 10,
available: 50,
stalled: false,
},
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::Settle,
Step::Sample {
percent: 10,
available: 50,
stalled: false,
},
Step::Tick,
];
const HEAVY: Step = Step::Queue {
second: false,
labels: Labels::Heavy,
};
const fn host_has(available: u64) -> Step {
Step::Sample {
percent: 10,
available,
stalled: false,
}
}
const A_JOB_STAYS_OPEN: [Step; 7] = [
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::RunnerGoes(0),
Step::Tick,
Step::OpenJobCompletes(0),
];
const A_JOB_STAYS_LISTED: [Step; 8] = [
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::RunnerGoesStillListed(0),
Step::Tick,
Step::Tick,
Step::OpenJobCompletes(0),
];
const THE_MIRROR_RUNS: [Step; 7] = [
Step::Reload(Change::Mirror),
HEAVY,
Step::Tick,
Step::Tick,
Step::Assign(0),
Step::Finish(0),
Step::Tick,
];
const WAITS_ON_DIFFERENT_JOBS: [Step; 16] = [
Step::Reload(Change::CpuOvercommit),
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
HEAVY,
Step::Queue {
second: false,
labels: Labels::Light,
},
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Cancel(0),
Step::Cancel(0),
Step::Finish(0),
Step::Finish(0),
Step::Reload(Change::CpuOvercommit),
Step::Tick,
];
const A_FORCED_PAUSE: [Step; 9] = [
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::ForcedPause { second: false },
Step::Tick,
Step::Elapse,
Step::Tick,
Step::Resume { second: false },
];
const A_LISTING_BESIDE_A_NEWLY_BUSY_RUNNER: [Step; 13] = [
HEAVY,
Step::Tick,
Step::Assign(0),
Step::TickSoon { after: 5 },
Step::TickSoon { after: 3 },
Step::TickSoon { after: 3 },
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::TickSoon { after: 2 },
Step::TickSoon { after: 2 },
Step::AssignListed(0),
Step::TickSoon { after: 3 },
Step::TickSoon { after: 2 },
Step::TickSoon { after: 3 },
];
const A_BUSY_ORPHAN: [Step; 2] = [Step::BusyOrphan, Step::Tick];
const AN_OVERSIZED_SNAPSHOT: [Step; 5] = [
Step::OversizedSnapshot,
Step::Tick,
host_has(3),
Step::Tick,
host_has(50),
];
const AN_UNREAD_LEFTOVER: [Step; 8] = [
Step::Reload(Change::RecordFailures),
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::Finish(0),
Step::RestartLogsFail,
Step::Tick,
];
const UPLOADS_WAIT_FOR_GITHUB: [Step; 11] = [
Step::UseBuildCache,
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::SaveUnnamed(0),
Step::RunnerGoes(0),
Step::Tick,
Step::Elapse,
Step::OpenJobCompletes(0),
Step::Tick,
];
const UPLOADS_DECIDED: [Step; 10] = [
Step::UseBuildCache,
HEAVY,
Step::Tick,
Step::Assign(0),
Step::Tick,
Step::SaveUnnamed(0),
Step::RunnerGoes(0),
Step::Tick,
Step::RunnerNamed(0),
Step::Tick,
];
fn env_u64(name: &str) -> Option<u64> {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
}
#[tokio::test(start_paused = true)]
#[expect(
clippy::print_stderr,
reason = "names the failing seed before shrinking, which takes a while and may itself fail"
)]
async fn the_scheduler_keeps_its_invariants_over_generated_sequences() {
let seeds: Vec<u64> = match env_u64("RUNNERD_INVARIANT_SEED") {
Some(seed) => vec![seed],
None => (0..env_u64("RUNNERD_INVARIANT_CASES").unwrap_or(CASES)).collect(),
};
let mut seen = Seen::default();
for batch in seeds.chunks(64) {
let runs = batch.iter().map(|&seed| async move {
let steps = steps_of(seed);
(seed, attempt(&steps).await, steps)
});
for (seed, outcome, steps) in futures_util::future::join_all(runs).await {
if let Ok(case) = &outcome {
seen.add(*case);
assert!(
seed % 8 != 2 || case.bands > 0,
"seed {seed} starts from a job no margin fits, so the larger margin stands"
);
assert!(
seed % 8 != 6 || case.released > 0,
"seed {seed} starts from a job between the margins, so the band lets it go"
);
assert!(
seed % 2 == 0 || case.credits > 0,
"seed {seed} starts from two settled jobs, so it reaches a credit"
);
assert!(
seed % 8 != 4 || case.host_credits > 0,
"seed {seed} starts from three settled jobs, so it reaches a host credit"
);
}
if let Err(found) = outcome {
eprintln!(
"seed {seed} breaks {} at step {}: {}; shrinking",
found.invariant, found.step, found.detail
);
let (invariant, detail) = (found.invariant, found.detail.clone());
let (shortest, violation) = Box::pin(shrink(steps, found)).await;
panic!(
"seed {seed} breaks {invariant}: {detail}\nshrunk to {shortest:#?}\nwhich breaks it at step {}: {}",
violation.step, violation.detail
);
}
}
}
assert!(
seen.unreached().is_empty() || seeds.len() == 1,
"the generated runs never reach {:?}, so nothing checks them",
seen.unreached()
);
}
const UNSTEADY_STATUS: &[&str] = &["seed_memory", "reseeding"];
const QUIET_ROUNDS: u32 = 8;
const TRACE_UNSTABLE: &[u64] = &[21, 53];
#[tokio::test]
async fn the_trace_of_each_seed() {
let Some(directory) = std::env::var_os("RUNNERD_INVARIANT_TRACE") else {
return;
};
let directory = std::path::PathBuf::from(directory);
std::fs::create_dir_all(&directory).unwrap();
for seed in (0..env_u64("RUNNERD_INVARIANT_CASES").unwrap_or(CASES))
.filter(|seed| !TRACE_UNSTABLE.contains(seed))
{
let mut case = Case::new();
case.trace = Some(Vec::new());
let case = run_case(&steps_of(seed), case)
.await
.unwrap_or_else(|found| {
panic!("seed {seed} breaks {}: {}", found.invariant, found.detail)
});
let trace = case.finished_trace().await;
std::fs::write(directory.join(format!("{seed}.trace")), trace.join("\n")).unwrap();
}
}
async fn broken(steps: &[Step]) -> Option<(&'static str, String)> {
Box::pin(run(steps))
.await
.err()
.map(|found| (found.invariant, found.detail))
}
#[tokio::test]
async fn a_daemon_started_with_a_job_queued_starts_the_mirror_for_its_runner() {
let steps = [Step::Reload(Change::Mirror), HEAVY, Step::Restart];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_taking_a_waiting_runners_room_still_fits_what_the_host_leaves() {
let steps = [
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Cancel(0),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Sample {
percent: 10,
available: 3,
stalled: false,
},
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_that_ends_before_its_broken_capture_comes_back_keeps_its_runner_in_the_history() {
let broke = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::CaptureFails { mid_stream: true },
Step::RestartInspectFails(0),
Step::InspectFails(1),
Step::InspectFails(6),
];
let tick = [Step::Tick, Step::Tick, Step::Assign(3)];
assert_eq!(broken(&[&broke[..], &tick].concat()).await, None);
let pass = [
Step::StartFails,
Step::Tick,
Step::Tick,
Step::Assign(3),
Step::Finish(0),
Step::Pass,
];
assert_eq!(broken(&[&broke[..], &pass].concat()).await, None);
let restart = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Reload(Change::Policy),
Step::InspectFails(5),
Step::CaptureFails { mid_stream: false },
Step::Reload(Change::SecondRepository),
Step::AssignListed(4),
Step::Finish(4),
Step::RestartInspectFails(5),
];
assert_eq!(broken(&restart).await, None);
}
#[tokio::test]
async fn a_runner_whose_inspect_fails_after_a_reload_lowered_its_memory_counts_at_its_own_cap() {
let heavy = |second| Step::Queue {
second,
labels: Labels::Heavy,
};
let steps = [
heavy(false),
heavy(true),
Step::Tick,
Step::Assign(0),
Step::Assign(0),
Step::Reload(Change::HeavyMemory),
Step::RestartInspectFails(7),
heavy(false),
Step::Reload(Change::HeavySlots(3)),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_stopped_runner_that_recovery_cannot_remove_is_removed_later() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Tick,
Step::Assign(0),
Step::RemoveFails,
Step::Finish(0),
Step::Restart,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_runner_watched_through_a_failed_inspect_still_records_the_job_it_runs() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Tick,
Step::InspectFails(0),
Step::RestartInspectFails(0),
Step::Assign(0),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_taking_a_waiting_runners_room_still_fits_the_cpu_budget() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Reload(Change::CpuOvercommit),
Step::AssignListed(1),
Step::InspectFails(1),
Step::Finish(2),
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_seed_container_starts_past_a_cpu_budget_a_reload_lowered() {
let steps = [
Step::Assign(6),
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Reload(Change::HeavyMemory),
Step::AssignListed(4),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Finish(5),
Step::Reload(Change::LightSlots(1)),
Step::Reload(Change::HeavySlots(3)),
Step::Reload(Change::CpuOvercommit),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_job_taking_a_waiting_runners_room_still_fits_the_slice_under_measured_admission() {
let steps = [
Step::Reload(Change::Policy),
Step::Reload(Change::HeavySlots(3)),
Step::Queue {
second: true,
labels: Labels::Heavy,
},
Step::Tick,
Step::Cancel(0),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Tick,
Step::Assign(0),
Step::Sample {
percent: 120,
available: 50,
stalled: false,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_reseed_starts_only_where_its_seed_container_fits_the_budget() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Light,
},
Step::Tick,
Step::Assign(0),
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Finish(0),
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_runner_first_listed_busy_beside_its_job_still_listed_gets_no_twin() {
let steps = [
Step::StartedInspectFails,
Step::CaptureFails { mid_stream: false },
Step::Queue {
second: false,
labels: Labels::SelfHosted,
},
Step::Restart,
Step::AssignListed(2),
Step::Reload(Change::LightPool),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_pass_starts_no_runner_for_a_job_a_failed_start_may_have_taken() {
let steps = [
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::Queue {
second: false,
labels: Labels::Heavy,
},
Step::StartFailsLate {
takes_job: false,
cleanup_fails: true,
},
Step::Sample {
percent: 120,
available: 8,
stalled: false,
},
Step::StartFailsLate {
takes_job: true,
cleanup_fails: false,
},
Step::Tick,
Step::Restart,
Step::Assign(7),
Step::Lose(5),
Step::Pass,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_runner_started_after_a_restart_never_takes_a_name_nothing_recorded() {
let queue = |second, labels| Step::Queue { second, labels };
let steps = [
queue(false, Labels::SelfHosted),
Step::StartedInspectFails,
queue(false, Labels::Typo),
Step::RestartLogsFail,
Step::CaptureFails { mid_stream: false },
queue(true, Labels::Empty),
queue(true, Labels::Heavy),
Step::RunnerGoes(4),
Step::AssignListed(1),
Step::RunnerGoes(4),
Step::Elapse,
Step::RestartInspectFails(3),
Step::CaptureFails { mid_stream: false },
Step::Reload(Change::SecondRepository),
Step::Cancel(7),
Step::Assign(4),
Step::OpenJobCompletes(2),
Step::Stop { docker_gone: false },
];
let mut case = Case::new();
case.names_from = time::now_ms() / 1000 + 3600;
case.engine.named = case.names_from;
let broken = run_case(&steps, case)
.await
.err()
.map(|found| (found.invariant, found.detail));
assert_eq!(broken, None);
}
#[tokio::test(start_paused = true)]
async fn a_start_fits_a_reading_taken_while_its_launch_waited() {
let queue = |second, labels| Step::Queue { second, labels };
let steps = [
Step::Tick,
Step::PastRuns,
Step::Reload(Change::Policy),
queue(false, Labels::Heavy),
queue(true, Labels::Heavy),
Step::Tick,
Step::Assign(0),
Step::Assign(3),
Step::Finish(1),
Step::SampleDuringStart {
percent: 50,
available: 8,
stalled: false,
},
queue(false, Labels::Heavy),
queue(true, Labels::SelfHosted),
Step::Assign(5),
Step::Lose(4),
Step::Reload(Change::CpuOvercommit),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn an_idle_runner_is_named_for_the_oldest_job_whatever_order_the_listing_gives() {
let heavy = |second| Step::Queue {
second,
labels: Labels::Heavy,
};
let steps = [
heavy(false),
heavy(true),
Step::Tick,
Step::Assign(0),
Step::Assign(0),
heavy(false),
heavy(true),
Step::Reload(Change::HeavyMemory),
Step::Reseed { light: false },
Step::RunnerGoesStillListed(7),
Step::Reload(Change::PooledCount),
Step::Finish(3),
Step::Elapse,
Step::Reload(Change::PooledCount),
Step::RestartInspectFails(2),
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test]
async fn a_start_the_tick_did_not_make_keeps_the_plain_margin() {
let queue = |labels| Step::Queue {
second: false,
labels,
};
let steps = [
queue(Labels::Light),
Step::PrepareFailsMidBatch { after: 2 },
queue(Labels::Heavy),
Step::Stop { docker_gone: false },
queue(Labels::SelfHosted),
Step::Tick,
queue(Labels::Light),
Step::ExitsIdle(7),
Step::Sample {
percent: 10,
available: 6,
stalled: false,
},
Step::Tick,
];
assert_eq!(broken(&steps).await, None);
}
#[tokio::test(start_paused = true)]
async fn a_job_a_failed_listing_forgot_is_shown_again_at_the_next_listing_of_its_repository() {
let heavy = Step::Queue {
second: false,
labels: Labels::Heavy,
};
let steps = [
heavy,
Step::RateLimitedDuringStart { after: 3 },
heavy,
Step::RestartLogsFail,
Step::Stop { docker_gone: false },
Step::Reload(Change::CpuOvercommit),
Step::Elapse,
Step::TickSoon { after: 3 },
Step::TickSoon { after: 3 },
];
assert_eq!(broken(&steps).await, None);
}